From 8b636e2c11526ea842b5f7ac6a0ec42982129b4e Mon Sep 17 00:00:00 2001 From: UniDesk Date: Fri, 15 May 2026 18:16:23 +0000 Subject: [PATCH] refactor(backend-core): split monolithic index.ts into 15 focused modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract config, context, logger, types, db, http, overview, performance, provider-registry, scheduler, task-dispatcher, microservice-proxy, ssh-bridge, and egress-tcp into separate files. index.ts now only handles server setup, HTTP/WebSocket routing, and startup orchestration. No behavioral changes — all existing tests and health checks pass. --- AGENTS.md | 2 +- docs/reference/deployment.md | 2 +- docs/reference/repo-tree.md | 16 +- scripts/src/check.ts | 2 +- src/components/backend-core/src/config.ts | 157 + src/components/backend-core/src/context.ts | 50 + src/components/backend-core/src/db.ts | 582 +++ src/components/backend-core/src/egress-tcp.ts | 95 + src/components/backend-core/src/http.ts | 100 + src/components/backend-core/src/index.ts | 3628 +---------------- src/components/backend-core/src/logger.ts | 29 + .../backend-core/src/microservice-proxy.ts | 795 ++++ src/components/backend-core/src/overview.ts | 316 ++ .../backend-core/src/performance.ts | 211 + .../backend-core/src/provider-registry.ts | 164 + src/components/backend-core/src/scheduler.ts | 752 ++++ src/components/backend-core/src/ssh-bridge.ts | 174 + .../backend-core/src/task-dispatcher.ts | 158 + src/components/backend-core/src/types.ts | 191 + 19 files changed, 3848 insertions(+), 3576 deletions(-) create mode 100644 src/components/backend-core/src/config.ts create mode 100644 src/components/backend-core/src/context.ts create mode 100644 src/components/backend-core/src/db.ts create mode 100644 src/components/backend-core/src/egress-tcp.ts create mode 100644 src/components/backend-core/src/http.ts create mode 100644 src/components/backend-core/src/logger.ts create mode 100644 src/components/backend-core/src/microservice-proxy.ts create mode 100644 src/components/backend-core/src/overview.ts create mode 100644 src/components/backend-core/src/performance.ts create mode 100644 src/components/backend-core/src/provider-registry.ts create mode 100644 src/components/backend-core/src/scheduler.ts create mode 100644 src/components/backend-core/src/ssh-bridge.ts create mode 100644 src/components/backend-core/src/task-dispatcher.ts create mode 100644 src/components/backend-core/src/types.ts diff --git a/AGENTS.md b/AGENTS.md index 881ce77e..192657fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `bun`：TypeScript 运行时固定使用 Bun，组件入口和 CLI 都直接运行 `.ts` 文件，约束见 `docs/reference/config.md`。 - `docker-compose.yml`：主 server 统一编排 core、frontend、database、本机 provider gateway、Todo Note 后端、Baidu Netdisk 后端和 OA Event Flow 后端；Code Queue 由 D601 v3s/k8s 控制面代管，并经 `v3sctl-adapter` 的 Kubernetes API service proxy 单一路径接入，服务拓扑见 `docs/reference/deployment.md`。 - `src/components/frontend`：前端源码固定使用 TypeScript + React，`app.tsx` 只做 shell/router，左侧主模块与顶部子标签统一编译为模块前缀路由：`/ops//`、`/nodes//`、`/tasks//`、`/config//`，只有用户服务使用 `/app//` 深链接，运行总览包含通用性能面板，资源监控含曲线和进程资源排序表，Todo Note、FindJob、Pipeline、MET Nonlinear、Baidu Netdisk、Code Queue、OA Event Flow、V3S Control 等业务页必须拆到独立 TSX 模块，界面规则见 `docs/reference/frontend.md`。 -- `backend-core / frontend performance`：backend-core 暴露 `/api/performance`，frontend 暴露同源 `/api/frontend-performance` 并在 `/ops/performance/` 汇总组件请求、失败请求、内部操作和慢操作，规则见 `docs/reference/observability.md`。 +- `backend-core / frontend performance`：backend-core 暴露 `/api/performance`，frontend 暴露同源 `/api/frontend-performance` 并在 `/ops/performance/` 汇总组件请求、失败请求、内部操作和慢操作，规则见 `docs/reference/observability.md`。backend-core 源码已拆分为 15 个模块，结构见 `docs/reference/repo-tree.md`。 - `Unified OA event flow`：`oa-event-flow` 是独立主 server 用户服务，提供事件表、按 tag 订阅和 Trace/STEP 统计中心，Code Queue 与 Pipeline 都必须接入统一事件流；共享契约见 `docs/reference/oa-event-flow.md`，Pipeline 专有控制流规则见 `docs/reference/pipeline-oa-event-flow.md`。 - `src/components/provider-gateway`：当前主 server `74.48.78.17` 也作为 provider gateway 接入 UniDesk，外部节点通过 `ws://74.48.78.17:18082/ws/provider` 接入，必须以 `restart: always` 部署 always-enabled 远程升级、sleep-and-validate 回滚保护和 Host SSH / WSL SSH 透传并完成自测，部署与 Playwright 公网前端验证方法见 `docs/reference/provider-gateway.md`。 - `microservices`：用户服务配置命名仍保留 `microservices`；用户服务指挂载在 UniDesk 核心服务上的用户业务能力，支持 `unidesk-direct` 与 `v3sctl-managed` 两种部署模式；v3s 代管必须使用标准 k3s/k8s 对象和 Kubernetes API service proxy，禁止业务容器直连、NodePort 和隐藏 fallback；缺少这些服务时核心仍可运行。主 server 本地开发边界固定为只开发 UniDesk frontend；非 UniDesk 核心业务后端、Dockerfile、GPU/训练调试必须在目标计算节点通过 SSH 透传或 v3s 控制面完成，Todo Note 这类明确写入主 server 的例外需单独登记，规则见 `docs/reference/microservices.md`。 diff --git a/docs/reference/deployment.md b/docs/reference/deployment.md index 6a8be6d7..13d921d6 100644 --- a/docs/reference/deployment.md +++ b/docs/reference/deployment.md @@ -5,7 +5,7 @@ ## Services - `database` 使用 `postgres:16-alpine`，数据保存到 named volume `unidesk_pgdata_10gb`，初始化 SQL 位于 `src/components/database/init/`。 -- `backend-core` 是无状态核心服务，提供 Docker 内网 REST API、provider ingress WebSocket、任务调度入口和数据库访问层。 +- `backend-core` 是无状态核心服务，提供 Docker 内网 REST API、provider ingress WebSocket、任务调度入口和数据库访问层。源码拆分为 15 个职责单一的模块（`index.ts` 只做路由和启动），模块结构见 `docs/reference/repo-tree.md`。 - `frontend` 是唯一公开 Web 控制台，提供登录、从 TSX 转译出的 React 应用资产和到 backend-core 的同源代理。 - `provider-gateway` 是当前主 server 的本机计算节点代理，通过 WebSocket 主动连到 provider ingress，挂载 `/var/run/docker.sock` 作为自动任务执行主路径，使用 `pid: "host"` 读取节点级进程资源，并周期性上报系统资源指标、进程占用与 Docker daemon 状态；计算节点 provider-gateway 还必须把 egress proxy 仅发布到宿主 loopback `127.0.0.1:18789` 供本节点执行环境出网，维护用 Host SSH / WSL SSH 私钥目录只读挂载到 `/run/host-ssh`，不得作为自动任务调度主路径。 - `todo-note` 是主 server 承载的 Todo Note 纯后端用户服务，容器名 `todo-note-backend`，只在 Compose 内网暴露 `4211/tcp`，使用主 PostgreSQL 存储迁移后的 Todo Note 数据。 diff --git a/docs/reference/repo-tree.md b/docs/reference/repo-tree.md index f17ef215..b1a0f836 100644 --- a/docs/reference/repo-tree.md +++ b/docs/reference/repo-tree.md @@ -46,7 +46,21 @@ - package.json - tsconfig.json - Dockerfile - - src/index.ts (Internal REST API, public provider ingress WebSocket, scheduler, database access, system/Docker status storage API) + - src/index.ts (Server setup, HTTP/WebSocket routing, startup orchestration) + - src/types.ts (All interfaces and type aliases) + - src/config.ts (Environment variable reading and config validation) + - src/context.ts (Shared mutable state container: db client, config ref, dbReady flag) + - src/logger.ts (Structured logger factory) + - src/http.ts (HTTP/JSON utilities, ISO formatting, nested value extraction) + - src/db.ts (Database init, schema creation, all CRUD queries) + - src/performance.ts (Request/operation performance sampling and database size reporting) + - src/provider-registry.ts (Provider WebSocket lifecycle: register, heartbeat, status, disconnect) + - src/task-dispatcher.ts (Task creation, dispatch to providers, wait-for-result) + - src/ssh-bridge.ts (SSH session bridging over provider WebSocket) + - src/egress-tcp.ts (Egress TCP proxy for provider outbound connections) + - src/scheduler.ts (Scheduled task system: cron-like recurring tasks) + - src/microservice-proxy.ts (Microservice routing, HTTP proxy, health aggregation) + - src/overview.ts (Overview endpoint, node/task/service summary, load test) - frontend/ (Frontend web application container) - package.json - tsconfig.json diff --git a/scripts/src/check.ts b/scripts/src/check.ts index 5e83d89a..436158c0 100644 --- a/scripts/src/check.ts +++ b/scripts/src/check.ts @@ -30,7 +30,7 @@ function commandItem(name: string, command: string[]): CheckItem { function unifiedLogRotationItem(): CheckItem { const serviceFiles = [ - "src/components/backend-core/src/index.ts", + "src/components/backend-core/src/logger.ts", "src/components/frontend/src/index.ts", "src/components/provider-gateway/src/index.ts", "src/components/microservices/code-queue/src/index.ts", diff --git a/src/components/backend-core/src/config.ts b/src/components/backend-core/src/config.ts new file mode 100644 index 00000000..ad55a4a7 --- /dev/null +++ b/src/components/backend-core/src/config.ts @@ -0,0 +1,157 @@ +import type { MicroserviceConfig, RuntimeConfig } from "./types"; + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +function readNumberEnv(name: string): number { + const raw = requiredEnv(name); + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); + } + return parsed; +} + +function readOptionalNumberEnv(name: string, fallback: number): number { + const raw = process.env[name]; + if (raw === undefined || raw.length === 0) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); + } + return parsed; +} + +function asRecord(value: unknown, name: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${name} must be an object`); + return value as Record; +} + +function stringFromRecord(value: Record, key: string, path: string): string { + const field = value[key]; + if (typeof field !== "string" || field.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return field; +} + +function optionalStringFromRecord(value: Record, key: string, path: string): string | undefined { + const field = value[key]; + if (field === undefined) return undefined; + if (typeof field !== "string" || field.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return field; +} + +function numberFromRecord(value: Record, key: string, path: string): number { + const field = value[key]; + if (typeof field !== "number" || !Number.isFinite(field) || field <= 0) throw new Error(`${path}.${key} must be a positive number`); + return field; +} + +function booleanFromRecord(value: Record, key: string, path: string): boolean { + const field = value[key]; + if (typeof field !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return field; +} + +function stringArrayFromRecord(value: Record, key: string, path: string): string[] { + const field = value[key]; + if (!Array.isArray(field) || field.some((item) => typeof item !== "string" || item.length === 0)) { + throw new Error(`${path}.${key} must be an array of non-empty strings`); + } + return field as string[]; +} + +function optionalStringArrayFromRecord(value: Record, key: string, path: string): string[] | undefined { + const field = value[key]; + if (field === undefined) return undefined; + return stringArrayFromRecord(value, key, path); +} + +function parseMicroserviceConfig(value: unknown, index: number): MicroserviceConfig { + const path = `MICROSERVICES_JSON[${index}]`; + const item = asRecord(value, path); + const repository = asRecord(item.repository, `${path}.repository`); + const backend = asRecord(item.backend, `${path}.backend`); + const development = asRecord(item.development, `${path}.development`); + const frontend = asRecord(item.frontend, `${path}.frontend`); + const deployment = item.deployment === undefined ? undefined : asRecord(item.deployment, `${path}.deployment`); + const deploymentMode = deployment === undefined ? "unidesk-direct" : stringFromRecord(deployment, "mode", `${path}.deployment`); + if (deploymentMode !== "unidesk-direct" && deploymentMode !== "v3sctl-managed") { + throw new Error(`${path}.deployment.mode must be unidesk-direct or v3sctl-managed`); + } + return { + id: stringFromRecord(item, "id", path), + name: stringFromRecord(item, "name", path), + providerId: stringFromRecord(item, "providerId", path), + description: stringFromRecord(item, "description", path), + repository: { + url: stringFromRecord(repository, "url", `${path}.repository`), + commitId: stringFromRecord(repository, "commitId", `${path}.repository`), + dockerfile: stringFromRecord(repository, "dockerfile", `${path}.repository`), + composeFile: stringFromRecord(repository, "composeFile", `${path}.repository`), + composeService: stringFromRecord(repository, "composeService", `${path}.repository`), + containerName: stringFromRecord(repository, "containerName", `${path}.repository`), + }, + backend: { + nodeBaseUrl: stringFromRecord(backend, "nodeBaseUrl", `${path}.backend`), + nodeBindHost: stringFromRecord(backend, "nodeBindHost", `${path}.backend`), + nodePort: numberFromRecord(backend, "nodePort", `${path}.backend`), + proxyMode: stringFromRecord(backend, "proxyMode", `${path}.backend`), + frontendOnly: booleanFromRecord(backend, "frontendOnly", `${path}.backend`), + public: booleanFromRecord(backend, "public", `${path}.backend`), + allowedMethods: stringArrayFromRecord(backend, "allowedMethods", `${path}.backend`).map((method) => method.toUpperCase()), + allowedPathPrefixes: stringArrayFromRecord(backend, "allowedPathPrefixes", `${path}.backend`), + healthPath: stringFromRecord(backend, "healthPath", `${path}.backend`), + timeoutMs: numberFromRecord(backend, "timeoutMs", `${path}.backend`), + }, + deployment: deployment === undefined + ? { mode: "unidesk-direct" } + : { + mode: deploymentMode, + adapterServiceId: optionalStringFromRecord(deployment, "adapterServiceId", `${path}.deployment`), + v3sServiceId: optionalStringFromRecord(deployment, "v3sServiceId", `${path}.deployment`), + namespace: optionalStringFromRecord(deployment, "namespace", `${path}.deployment`), + expectedNodeIds: optionalStringArrayFromRecord(deployment, "expectedNodeIds", `${path}.deployment`), + activeNodeId: optionalStringFromRecord(deployment, "activeNodeId", `${path}.deployment`), + }, + development: { + providerId: stringFromRecord(development, "providerId", `${path}.development`), + sshPassthrough: booleanFromRecord(development, "sshPassthrough", `${path}.development`), + worktreePath: stringFromRecord(development, "worktreePath", `${path}.development`), + }, + frontend: { + route: stringFromRecord(frontend, "route", `${path}.frontend`), + integrated: booleanFromRecord(frontend, "integrated", `${path}.frontend`), + }, + }; +} + +function readMicroservicesEnv(): MicroserviceConfig[] { + const raw = process.env.MICROSERVICES_JSON; + if (raw === undefined || raw.length === 0) return []; + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) throw new Error("MICROSERVICES_JSON must be an array"); + return parsed.map(parseMicroserviceConfig); +} + +export function readConfig(): RuntimeConfig { + return { + port: readNumberEnv("PORT"), + providerPort: readNumberEnv("PROVIDER_PORT"), + databaseUrl: requiredEnv("DATABASE_URL"), + providerToken: requiredEnv("PROVIDER_TOKEN"), + heartbeatTimeoutMs: readNumberEnv("HEARTBEAT_TIMEOUT_MS"), + taskPendingTimeoutMs: readOptionalNumberEnv("TASK_PENDING_TIMEOUT_MS", 10 * 60 * 1000), + databaseVolumeName: requiredEnv("DATABASE_VOLUME_NAME"), + databaseVolumeSize: requiredEnv("DATABASE_VOLUME_SIZE"), + pgdataBackupStagingDir: process.env.PGDATA_BACKUP_STAGING_DIR || "/data/baidu-netdisk-staging", + baiduNetdiskInternalUrl: process.env.BAIDU_NETDISK_INTERNAL_URL || "http://baidu-netdisk:4244", + microservices: readMicroservicesEnv(), + logFile: requiredEnv("LOG_FILE"), + databasePoolMax: Math.max(1, Math.min(16, readOptionalNumberEnv("DATABASE_POOL_MAX", 4))), + }; +} diff --git a/src/components/backend-core/src/context.ts b/src/components/backend-core/src/context.ts new file mode 100644 index 00000000..2ff76254 --- /dev/null +++ b/src/components/backend-core/src/context.ts @@ -0,0 +1,50 @@ +import type { JsonValue } from "../../shared/src/index"; +import type { + EgressTcpConnection, + LoggerFn, + MicroserviceAvailabilityEntry, + MicroserviceProxyCacheEntry, + OperationPerformanceSample, + ProviderSocket, + RawTaskRow, + RequestPerformanceSample, + RuntimeConfig, + SqlClient, + TaskTerminalWaiter, +} from "./types"; + +export const ctx = { + sql: null as SqlClient | null, + config: null as RuntimeConfig | null, + logger: null as LoggerFn | null, + dbReady: false, + serviceStartedAt: new Date(), + activeProviders: new Map(), + activeSshClients: new Map(), + activeEgressTcpConnections: new Map(), + taskTerminalWaiters: new Map>(), + microserviceProxyCache: new Map(), + microserviceProxyRefreshes: new Map>(), + microserviceAvailabilityCache: new Map(), + microserviceAvailabilityRefreshes: new Map>>(), + activeScheduledRuns: new Set(), + recentLogs: [] as unknown[], + requestPerformanceSamples: [] as RequestPerformanceSample[], + operationPerformanceSamples: [] as OperationPerformanceSample[], + lastTaskSweepAt: 0, + taskSweepInFlight: null as Promise | null, +}; + +export function sql(): SqlClient { + if (ctx.sql === null) throw new Error("database not initialized"); + return ctx.sql; +} + +export function config(): RuntimeConfig { + if (ctx.config === null) throw new Error("config not initialized"); + return ctx.config; +} + +export function logger(level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue): void { + if (ctx.logger !== null) ctx.logger(level, message, data); +} diff --git a/src/components/backend-core/src/db.ts b/src/components/backend-core/src/db.ts new file mode 100644 index 00000000..3fe3b400 --- /dev/null +++ b/src/components/backend-core/src/db.ts @@ -0,0 +1,582 @@ +import type { JsonValue, ApiEvent, ApiNode, ApiNodeDockerStatus, ApiNodeSystemStatus, ApiTask, ProviderLabels } from "../../shared/src/index"; +import { ctx, sql, config, logger } from "./context"; +import type { RawTaskRow, ScheduledTaskRow, ScheduledTaskRunRow, SqlClient } from "./types"; +import { compactJson, nestedNumber, redactDatabaseUrl, rowIso } from "./http"; + +function errorToJson(error: unknown): JsonValue { + if (error instanceof Error) { + return { name: error.name, message: error.message, stack: error.stack ?? null }; + } + return String(error); +} + +export async function initDatabase(client: SqlClient): Promise { + logger("info", "database_init_start", { databaseUrl: redactDatabaseUrl(config().databaseUrl) }); + await client` + CREATE TABLE IF NOT EXISTS unidesk_nodes ( + provider_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + labels JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL, + connected_at TIMESTAMPTZ, + last_heartbeat TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_events ( + id BIGSERIAL PRIMARY KEY, + type TEXT NOT NULL, + source TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_tasks ( + id TEXT PRIMARY KEY, + provider_id TEXT NOT NULL, + command TEXT NOT NULL, + status TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + result JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_scheduled_tasks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + enabled BOOLEAN NOT NULL DEFAULT true, + schedule_json JSONB NOT NULL DEFAULT '{}'::jsonb, + action_json JSONB NOT NULL DEFAULT '{}'::jsonb, + concurrency_policy TEXT NOT NULL DEFAULT 'skip', + next_run_at TIMESTAMPTZ, + last_run_at TIMESTAMPTZ, + last_run_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_scheduled_task_runs ( + id TEXT PRIMARY KEY, + schedule_id TEXT NOT NULL REFERENCES unidesk_scheduled_tasks(id) ON DELETE CASCADE, + trigger_type TEXT NOT NULL, + status TEXT NOT NULL, + task_id TEXT, + result JSONB, + error TEXT, + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + duration_ms BIGINT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_node_docker_status ( + provider_id TEXT PRIMARY KEY, + status JSONB NOT NULL DEFAULT '{}'::jsonb, + collected_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_node_system_status ( + provider_id TEXT PRIMARY KEY, + status JSONB NOT NULL DEFAULT '{}'::jsonb, + collected_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_node_metric_samples ( + id BIGSERIAL PRIMARY KEY, + provider_id TEXT NOT NULL, + collected_at TIMESTAMPTZ NOT NULL, + cpu_percent DOUBLE PRECISION NOT NULL DEFAULT 0, + memory_percent DOUBLE PRECISION NOT NULL DEFAULT 0, + disk_percent DOUBLE PRECISION NOT NULL DEFAULT 0, + sample JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_tasks_updated_at ON unidesk_tasks(updated_at DESC)`; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_tasks_status_updated_at ON unidesk_tasks(status, updated_at DESC)`; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_scheduled_tasks_next_run ON unidesk_scheduled_tasks(enabled, next_run_at)`; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_scheduled_task_runs_schedule_updated ON unidesk_scheduled_task_runs(schedule_id, updated_at DESC)`; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_scheduled_task_runs_status_updated ON unidesk_scheduled_task_runs(status, updated_at DESC)`; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_node_system_status_updated_at ON unidesk_node_system_status(updated_at DESC)`; + await client`CREATE INDEX IF NOT EXISTS idx_unidesk_node_metric_samples_provider_time ON unidesk_node_metric_samples(provider_id, collected_at DESC)`; + ctx.dbReady = true; + logger("info", "database_init_complete"); +} + +export async function recordEvent(type: string, source: string, payload: Record): Promise { + logger("info", type, { source, payload }); + if (!ctx.dbReady) return; + try { + await sql()` + INSERT INTO unidesk_events (type, source, payload) + VALUES (${type}, ${source}, ${sql().json(payload)}) + `; + } catch (error) { + logger("error", "event_insert_failed", { type, source, error: errorToJson(error) }); + } +} + +export async function upsertProviderNode(providerId: string, name: string, labels: unknown): Promise { + await sql()` + INSERT INTO unidesk_nodes (provider_id, name, labels, status, connected_at, last_heartbeat, updated_at) + VALUES (${providerId}, ${name}, ${sql().json(labels as JsonValue)}, 'online', now(), now(), now()) + ON CONFLICT (provider_id) DO UPDATE SET + name = EXCLUDED.name, + labels = EXCLUDED.labels, + status = 'online', + connected_at = COALESCE(unidesk_nodes.connected_at, EXCLUDED.connected_at), + last_heartbeat = now(), + updated_at = now() + `; +} + +export async function updateProviderHeartbeat(providerId: string, labels: unknown): Promise { + await sql()` + UPDATE unidesk_nodes + SET labels = unidesk_nodes.labels || ${sql().json(labels as JsonValue)}, status = 'online', last_heartbeat = now(), updated_at = now() + WHERE provider_id = ${providerId} + `; +} +export async function upsertDockerStatus(providerId: string, status: unknown, collectedAt: string): Promise { + await sql()` + INSERT INTO unidesk_node_docker_status (provider_id, status, collected_at, updated_at) + VALUES (${providerId}, ${sql().json(status as JsonValue)}, ${collectedAt}, now()) + ON CONFLICT (provider_id) DO UPDATE SET + status = EXCLUDED.status, + collected_at = EXCLUDED.collected_at, + updated_at = now() + `; +} + +export async function upsertSystemStatus(providerId: string, status: unknown, collectedAt: string): Promise { + const cpuPercent = nestedNumber(status, "cpu", "percent"); + const memoryPercent = nestedNumber(status, "memory", "percent"); + const diskPercent = nestedNumber(status, "disk", "percent"); + await sql().begin(async (tx) => { + await tx` + INSERT INTO unidesk_node_system_status (provider_id, status, collected_at, updated_at) + VALUES (${providerId}, ${tx.json(status as JsonValue)}, ${collectedAt}, now()) + ON CONFLICT (provider_id) DO UPDATE SET + status = EXCLUDED.status, + collected_at = EXCLUDED.collected_at, + updated_at = now() + `; + await tx` + INSERT INTO unidesk_node_metric_samples (provider_id, collected_at, cpu_percent, memory_percent, disk_percent, sample) + VALUES (${providerId}, ${collectedAt}, ${cpuPercent}, ${memoryPercent}, ${diskPercent}, ${tx.json(status as JsonValue)}) + `; + await tx` + DELETE FROM unidesk_node_metric_samples + WHERE provider_id = ${providerId} + AND id NOT IN ( + SELECT id FROM unidesk_node_metric_samples + WHERE provider_id = ${providerId} + ORDER BY collected_at DESC + LIMIT 720 + ) + `; + }); +} + +export async function getNodes(): Promise { + const rows = await sql()>>` + SELECT provider_id, name, labels, status, connected_at, last_heartbeat + FROM unidesk_nodes + ORDER BY status DESC, provider_id ASC + `; + return rows.map((row) => ({ + providerId: String(row.provider_id), + name: String(row.name), + status: row.status === "online" ? "online" : "offline", + labels: (row.labels ?? {}) as ProviderLabels, + connectedAt: row.connected_at instanceof Date ? row.connected_at.toISOString() : row.connected_at === null ? null : String(row.connected_at), + lastHeartbeat: row.last_heartbeat instanceof Date ? row.last_heartbeat.toISOString() : row.last_heartbeat === null ? null : String(row.last_heartbeat), + })); +} +export async function getNodeDockerStatuses(): Promise { + const rows = await sql()>>` + SELECT n.provider_id, n.name, n.status AS node_status, d.status AS docker_status, d.updated_at + FROM unidesk_nodes n + LEFT JOIN unidesk_node_docker_status d ON d.provider_id = n.provider_id + ORDER BY n.status DESC, n.provider_id ASC + `; + return rows.map((row) => ({ + providerId: String(row.provider_id), + name: String(row.name), + nodeStatus: row.node_status === "online" ? "online" : "offline", + dockerStatus: row.docker_status === null || row.docker_status === undefined ? null : (row.docker_status as JsonValue), + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at === null || row.updated_at === undefined ? null : String(row.updated_at), + })); +} + +function metricPointFromSample(sample: unknown, collectedAt: string): JsonValue { + return { + at: collectedAt, + cpuPercent: nestedNumber(sample, "cpu", "percent"), + memoryPercent: nestedNumber(sample, "memory", "percent"), + diskPercent: nestedNumber(sample, "disk", "percent"), + memoryUsedBytes: nestedNumber(sample, "memory", "usedBytes"), + memoryTotalBytes: nestedNumber(sample, "memory", "totalBytes"), + diskUsedBytes: nestedNumber(sample, "disk", "usedBytes"), + diskTotalBytes: nestedNumber(sample, "disk", "totalBytes"), + load1: nestedNumber(sample, "cpu", "load1"), + }; +} + +export async function getNodeSystemStatuses(limit: number): Promise { + const currentRows = await sql()>>` + SELECT n.provider_id, n.name, n.status AS node_status, s.status AS system_status, s.updated_at + FROM unidesk_nodes n + LEFT JOIN unidesk_node_system_status s ON s.provider_id = n.provider_id + ORDER BY n.status DESC, n.provider_id ASC + `; + const sampleRows = await sql()>>` + SELECT n.provider_id, recent.collected_at, recent.sample + FROM unidesk_nodes n + LEFT JOIN LATERAL ( + SELECT collected_at, sample + FROM unidesk_node_metric_samples m + WHERE m.provider_id = n.provider_id + ORDER BY collected_at DESC + LIMIT ${limit} + ) recent ON true + WHERE recent.collected_at IS NOT NULL + ORDER BY n.provider_id ASC, recent.collected_at ASC + `; + const historyByProvider = new Map(); + for (const row of sampleRows) { + const providerId = String(row.provider_id); + const collectedAt = row.collected_at instanceof Date ? row.collected_at.toISOString() : String(row.collected_at); + const history = historyByProvider.get(providerId) ?? []; + history.push(metricPointFromSample(row.sample ?? {}, collectedAt)); + historyByProvider.set(providerId, history); + } + return currentRows.map((row) => { + const providerId = String(row.provider_id); + return { + providerId, + name: String(row.name), + nodeStatus: row.node_status === "online" ? "online" : "offline", + current: row.system_status === null || row.system_status === undefined ? null : (row.system_status as JsonValue), + history: historyByProvider.get(providerId) ?? [], + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at === null || row.updated_at === undefined ? null : String(row.updated_at), + }; + }); +} + +export async function getEvents(limit: number): Promise { + const rows = await sql()>>` + SELECT id, type, source, payload, created_at + FROM unidesk_events + ORDER BY id DESC + LIMIT ${limit} + `; + return rows.map((row) => ({ + id: Number(row.id), + type: String(row.type), + source: String(row.source), + payload: compactJson(row.payload ?? {}), + createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), + })); +} + +function rowString(row: Record, key: string): string { + const value = row[key]; + return typeof value === "string" ? value : value === null || value === undefined ? "" : String(value); +} + +function rowNumber(row: Record, key: string): number | null { + const value = row[key]; + const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN; + return Number.isFinite(parsed) ? parsed : null; +} + +function taskJsonSummary(row: Record, prefix: "payload" | "result"): JsonValue { + const type = rowString(row, `${prefix}_type`); + if (type.length === 0) return prefix === "result" ? null : {}; + const summary: Record = { summaryOnly: true, type }; + const fields = prefix === "payload" + ? ["source", "serviceId", "method", "path", "mode", "targetBaseUrl", "timeoutMs", "targetProviderGatewayVersion", "providerGatewayVersion"] + : ["error", "reason", "message", "status", "exitCode", "code", "signal", "timeoutMs", "previousStatus", "mode", "policy", "targetProviderGatewayVersion", "providerGatewayVersion", "updaterContainerId"]; + for (const field of fields) { + const value = row[`${prefix}_${field}`]; + if (value !== null && value !== undefined && String(value).length > 0) { + summary[field] = typeof value === "number" || typeof value === "boolean" ? value : String(value); + } + } + const bodyChars = rowNumber(row, `${prefix}_body_text_chars`); + if (bodyChars !== null) summary.bodyText = ``; + return summary; +} +function taskSummaryFromRow(row: Record): ApiTask { + return { + id: String(row.id), + providerId: String(row.provider_id), + command: String(row.command), + status: String(row.status), + payload: taskJsonSummary(row, "payload"), + result: taskJsonSummary(row, "result"), + createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), + _summaryOnly: true, + } as ApiTask; +} + +export async function getTask(taskId: string): Promise { + const rows = await sql()>>` + SELECT + id, + provider_id, + command, + status, + CASE + WHEN payload ? 'bodyText' THEN jsonb_set(payload - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) + ELSE payload + END AS payload, + CASE + WHEN result IS NOT NULL AND result ? 'bodyText' THEN jsonb_set(result - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) + ELSE result + END AS result, + created_at, + updated_at + FROM unidesk_tasks + WHERE id = ${taskId} + LIMIT 1 + `; + const row = rows[0]; + if (row === undefined) return null; + return { + id: String(row.id), + providerId: String(row.provider_id), + command: String(row.command), + status: String(row.status), + payload: compactJson(row.payload ?? {}), + result: compactJson(row.result ?? null), + createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), + }; +} +let lastTaskSweepAt = 0; +let taskSweepInFlight: Promise | null = null; + +async function markStaleTasksFailed(): Promise { + if (!ctx.dbReady) return; + const timeoutMs = config().taskPendingTimeoutMs; + const rows = await sql()>` + WITH stale AS ( + SELECT id, provider_id, command, status AS previous_status, updated_at + FROM unidesk_tasks + WHERE status IN ('queued', 'dispatched', 'running') + AND updated_at < now() - (${timeoutMs}::bigint * interval '1 millisecond') + FOR UPDATE + ), + updated AS ( + UPDATE unidesk_tasks task + SET + status = 'failed', + result = jsonb_build_object( + 'error', 'task timed out without terminal provider status', + 'timeoutMs', ${timeoutMs}::bigint, + 'previousStatus', stale.previous_status, + 'previousResult', task.result, + 'timedOutAt', now() + ), + updated_at = now() + FROM stale + WHERE task.id = stale.id + RETURNING task.id, task.provider_id, task.command, stale.previous_status, stale.updated_at + ) + SELECT * FROM updated + `; + for (const row of rows) { + await recordEvent("task_timeout", row.provider_id, { + taskId: row.id, + providerId: row.provider_id, + command: row.command, + previousStatus: row.previous_status, + previousUpdatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), + timeoutMs, + }); + } +} + +async function maybeMarkStaleTasksFailed(minIntervalMs = 15_000): Promise { + if (!ctx.dbReady) return; + const now = Date.now(); + if (taskSweepInFlight !== null) return taskSweepInFlight; + if (now - lastTaskSweepAt < minIntervalMs) return; + lastTaskSweepAt = now; + taskSweepInFlight = markStaleTasksFailed() + .catch((error) => { + logger("error", "task_timeout_sweep_failed", { error: errorToJson(error) }); + }) + .finally(() => { + taskSweepInFlight = null; + }); + return taskSweepInFlight; +} +export async function getTasks(limit: number, statusFilter = "all", lite = false, summary = false): Promise { + await maybeMarkStaleTasksFailed(); + if (summary && !lite) { + const rows = statusFilter === "pending" + ? await sql()>>` + SELECT + id, + provider_id, + command, + status, + jsonb_typeof(payload) AS payload_type, + payload->>'source' AS "payload_source", + payload->>'serviceId' AS "payload_serviceId", + payload->>'method' AS "payload_method", + payload->>'path' AS "payload_path", + payload->>'mode' AS "payload_mode", + payload->>'targetBaseUrl' AS "payload_targetBaseUrl", + payload->>'timeoutMs' AS "payload_timeoutMs", + payload->>'targetProviderGatewayVersion' AS "payload_targetProviderGatewayVersion", + payload->>'providerGatewayVersion' AS "payload_providerGatewayVersion", + CASE WHEN payload ? 'bodyText' THEN length(payload->>'bodyText') ELSE NULL END AS payload_body_text_chars, + jsonb_typeof(result) AS result_type, + result->>'error' AS "result_error", + result->>'reason' AS "result_reason", + result->>'message' AS "result_message", + result->>'status' AS "result_status", + result->>'exitCode' AS "result_exitCode", + result->>'code' AS "result_code", + result->>'signal' AS "result_signal", + result->>'timeoutMs' AS "result_timeoutMs", + result->>'previousStatus' AS "result_previousStatus", + result->>'mode' AS "result_mode", + result->>'policy' AS "result_policy", + result->>'targetProviderGatewayVersion' AS "result_targetProviderGatewayVersion", + result->>'providerGatewayVersion' AS "result_providerGatewayVersion", + result->>'updaterContainerId' AS "result_updaterContainerId", + CASE WHEN result IS NOT NULL AND result ? 'bodyText' THEN length(result->>'bodyText') ELSE NULL END AS result_body_text_chars, + created_at, + updated_at + FROM unidesk_tasks + WHERE status IN ('queued', 'dispatched', 'running') + ORDER BY updated_at DESC + LIMIT ${limit} + ` + : await sql()>>` + SELECT + id, + provider_id, + command, + status, + jsonb_typeof(payload) AS payload_type, + payload->>'source' AS "payload_source", + payload->>'serviceId' AS "payload_serviceId", + payload->>'method' AS "payload_method", + payload->>'path' AS "payload_path", + payload->>'mode' AS "payload_mode", + payload->>'targetBaseUrl' AS "payload_targetBaseUrl", + payload->>'timeoutMs' AS "payload_timeoutMs", + payload->>'targetProviderGatewayVersion' AS "payload_targetProviderGatewayVersion", + payload->>'providerGatewayVersion' AS "payload_providerGatewayVersion", + CASE WHEN payload ? 'bodyText' THEN length(payload->>'bodyText') ELSE NULL END AS payload_body_text_chars, + jsonb_typeof(result) AS result_type, + result->>'error' AS "result_error", + result->>'reason' AS "result_reason", + result->>'message' AS "result_message", + result->>'status' AS "result_status", + result->>'exitCode' AS "result_exitCode", + result->>'code' AS "result_code", + result->>'signal' AS "result_signal", + result->>'timeoutMs' AS "result_timeoutMs", + result->>'previousStatus' AS "result_previousStatus", + result->>'mode' AS "result_mode", + COALESCE(result->>'policy', result #>> '{plan,policy}') AS "result_policy", + COALESCE(result->>'targetProviderGatewayVersion', result #>> '{plan,targetProviderGatewayVersion}') AS "result_targetProviderGatewayVersion", + COALESCE(result->>'providerGatewayVersion', result #>> '{plan,providerGatewayVersion}') AS "result_providerGatewayVersion", + result->>'updaterContainerId' AS "result_updaterContainerId", + CASE WHEN result ? 'bodyText' THEN length(result->>'bodyText') ELSE NULL END AS result_body_text_chars, + created_at, + updated_at + FROM unidesk_tasks + ORDER BY updated_at DESC + LIMIT ${limit} + `; + return rows.map(taskSummaryFromRow); + } + const rows = statusFilter === "pending" + ? lite + ? await sql()>>` + SELECT id, provider_id, command, status, created_at, updated_at + FROM unidesk_tasks + WHERE status IN ('queued', 'dispatched', 'running') + ORDER BY updated_at DESC + LIMIT ${limit} + ` + : await sql()>>` + SELECT + id, + provider_id, + command, + status, + CASE + WHEN payload ? 'bodyText' THEN jsonb_set(payload - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) + ELSE payload + END AS payload, + CASE + WHEN result IS NOT NULL AND result ? 'bodyText' THEN jsonb_set(result - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) + ELSE result + END AS result, + created_at, + updated_at + FROM unidesk_tasks + WHERE status IN ('queued', 'dispatched', 'running') + ORDER BY updated_at DESC + LIMIT ${limit} + ` + : lite + ? await sql()>>` + SELECT id, provider_id, command, status, created_at, updated_at + FROM unidesk_tasks + ORDER BY updated_at DESC + LIMIT ${limit} + ` + : await sql()>>` + SELECT + id, + provider_id, + command, + status, + CASE + WHEN payload ? 'bodyText' THEN jsonb_set(payload - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) + ELSE payload + END AS payload, + CASE + WHEN result IS NOT NULL AND result ? 'bodyText' THEN jsonb_set(result - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) + ELSE result + END AS result, + created_at, + updated_at + FROM unidesk_tasks + ORDER BY updated_at DESC + LIMIT ${limit} + `; + return rows.map((row) => ({ + id: String(row.id), + providerId: String(row.provider_id), + command: String(row.command), + status: String(row.status), + payload: lite ? {} : compactJson(row.payload ?? {}), + result: lite ? null : compactJson(row.result ?? null), + createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), + })); +} diff --git a/src/components/backend-core/src/egress-tcp.ts b/src/components/backend-core/src/egress-tcp.ts new file mode 100644 index 00000000..83e49f7a --- /dev/null +++ b/src/components/backend-core/src/egress-tcp.ts @@ -0,0 +1,95 @@ +import { connect as connectTcp } from "node:net"; +import type { + CoreEgressTcpCloseMessage, + CoreEgressTcpDataMessage, + CoreEgressTcpOpenedMessage, + ProviderEgressTcpCloseMessage, + ProviderEgressTcpDataMessage, + ProviderEgressTcpOpenMessage, +} from "../../shared/src/index"; +import { ctx, logger } from "./context"; +import type { EgressTcpConnection, ProviderSocket } from "./types"; +import { wsSendJson } from "./http"; + +function egressTcpKey(providerId: string, connectionId: string): string { + return `${providerId}:${connectionId}`; +} + +function isValidEgressHost(host: string): boolean { + return host.length > 0 && host.length <= 253 && !/[\s/\\:@\0]/u.test(host); +} + +function isValidEgressPort(port: number): boolean { + return Number.isInteger(port) && port > 0 && port <= 65_535; +} + +function sendEgressClose(provider: ProviderSocket, connectionId: string, error?: string): void { + const message: CoreEgressTcpCloseMessage = error === undefined + ? { type: "egress_tcp_close", connectionId } + : { type: "egress_tcp_close", connectionId, error }; + wsSendJson(provider, message); +} + +function closeEgressTcpConnection(providerId: string, connectionId: string, error?: string): void { + const key = egressTcpKey(providerId, connectionId); + const connection = ctx.activeEgressTcpConnections.get(key); + if (connection === undefined) return; + ctx.activeEgressTcpConnections.delete(key); + connection.socket.destroy(); + if (error !== undefined) sendEgressClose(connection.provider, connectionId, error); +} + +export function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressTcpOpenMessage): void { + const host = message.host.trim(); + const port = Number(message.port); + if (!isValidEgressHost(host) || !isValidEgressPort(port)) { + sendEgressClose(ws, message.connectionId, "invalid egress target"); + return; + } + const key = egressTcpKey(message.providerId, message.connectionId); + closeEgressTcpConnection(message.providerId, message.connectionId); + const socket = connectTcp({ host, port }); + const connection: EgressTcpConnection = { providerId: message.providerId, connectionId: message.connectionId, socket, provider: ws }; + ctx.activeEgressTcpConnections.set(key, connection); + socket.on("connect", () => { + const opened: CoreEgressTcpOpenedMessage = { type: "egress_tcp_opened", connectionId: message.connectionId }; + wsSendJson(ws, opened); + }); + socket.on("data", (chunk) => { + const data: CoreEgressTcpDataMessage = { + type: "egress_tcp_data", + connectionId: message.connectionId, + data: chunk.toString("base64"), + encoding: "base64", + }; + wsSendJson(ws, data); + }); + socket.on("close", () => { + if (ctx.activeEgressTcpConnections.get(key) !== connection) return; + ctx.activeEgressTcpConnections.delete(key); + sendEgressClose(ws, message.connectionId); + }); + socket.on("error", (error) => { + if (ctx.activeEgressTcpConnections.get(key) !== connection) return; + ctx.activeEgressTcpConnections.delete(key); + sendEgressClose(ws, message.connectionId, error.message); + }); +} + +export function handleEgressTcpData(message: ProviderEgressTcpDataMessage): void { + const connection = ctx.activeEgressTcpConnections.get(egressTcpKey(message.providerId, message.connectionId)); + if (connection === undefined) return; + connection.socket.write(Buffer.from(message.data, message.encoding === "base64" ? "base64" : "utf8")); +} + +export function handleEgressTcpClose(message: ProviderEgressTcpCloseMessage): void { + closeEgressTcpConnection(message.providerId, message.connectionId); +} + +export function closeEgressTcpConnectionsForProvider(providerId: string): void { + for (const [key, connection] of ctx.activeEgressTcpConnections) { + if (connection.providerId !== providerId) continue; + ctx.activeEgressTcpConnections.delete(key); + connection.socket.destroy(); + } +} diff --git a/src/components/backend-core/src/http.ts b/src/components/backend-core/src/http.ts new file mode 100644 index 00000000..bb3a7e49 --- /dev/null +++ b/src/components/backend-core/src/http.ts @@ -0,0 +1,100 @@ +import type { JsonValue } from "../../shared/src/index"; +import type { ProviderSocket } from "./types"; + +export function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body, null, 2), { + status, + headers: { + "content-type": "application/json; charset=utf-8", + "access-control-allow-origin": "*", + "access-control-allow-methods": "GET,HEAD,POST,PUT,PATCH,DELETE,OPTIONS", + "access-control-allow-headers": "content-type,x-provider-token", + }, + }); +} + +export function textResponse(text: string, status = 200): Response { + return new Response(text, { + status, + headers: { + "content-type": "text/plain; charset=utf-8", + "access-control-allow-origin": "*", + }, + }); +} + +export function readLimit(url: URL, defaultLimit: number): number { + const raw = url.searchParams.get("limit"); + if (raw === null) return defaultLimit; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) return defaultLimit; + return Math.min(parsed, 500); +} + +export function errorToJson(error: unknown): JsonValue { + if (error instanceof Error) { + return { name: error.name, message: error.message, stack: error.stack ?? null }; + } + return String(error); +} + +export function compactJson(value: unknown, depth = 0): JsonValue { + if (value === null || typeof value === "number" || typeof value === "boolean") return value; + if (typeof value === "string") return value.length > 600 ? `${value.slice(0, 600)}...` : value; + if (Array.isArray(value)) { + const items = value.slice(0, 20).map((item) => compactJson(item, depth + 1)); + if (value.length > 20) items.push({ truncatedItems: value.length - 20 }); + return items; + } + if (typeof value === "object" && value !== null) { + if (depth >= 4) return ""; + const entries = Object.entries(value as Record).slice(0, 30); + const result: Record = {}; + for (const [key, item] of entries) result[key] = compactJson(item, depth + 1); + const total = Object.keys(value as Record).length; + if (total > entries.length) result.truncatedKeys = total - entries.length; + return result; + } + return String(value); +} + +export function wsSendJson(ws: ProviderSocket, value: unknown): void { + if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(value)); +} + +export function isJsonRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function nestedNumber(value: unknown, objectKey: string, numberKey: string): number { + const parsed = Number(recordValue(recordValue(value, objectKey), numberKey)); + return Number.isFinite(parsed) ? parsed : 0; +} + +function recordValue(value: unknown, key: string): unknown { + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + return (value as Record)[key]; +} + +export function redactDatabaseUrl(value: string): string { + try { + const url = new URL(value); + if (url.password) url.password = "***"; + return url.toString(); + } catch { + return ""; + } +} + +export function truncateText(value: string, maxBytes: number): string { + return value.length <= maxBytes ? value : value.slice(0, maxBytes); +} + +export function rowIso(value: Date | string | null | undefined): string | null { + if (value === null || value === undefined) return null; + return value instanceof Date ? value.toISOString() : String(value); +} diff --git a/src/components/backend-core/src/index.ts b/src/components/backend-core/src/index.ts index 97653d80..7463aaef 100644 --- a/src/components/backend-core/src/index.ts +++ b/src/components/backend-core/src/index.ts @@ -1,3528 +1,45 @@ -import type { Server, ServerWebSocket } from "bun"; -import { createHash } from "node:crypto"; -import { createReadStream } from "node:fs"; -import { mkdir, rm, stat } from "node:fs/promises"; -import { connect as connectTcp, type Socket } from "node:net"; -import { basename, dirname, relative, resolve, posix as pathPosix } from "node:path"; +import type { Server } from "bun"; import postgres from "postgres"; -import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../shared/src/rotating-jsonl"; -import { - type ApiEvent, - type ApiNode, - type ApiNodeDockerStatus, - type ApiNodeSystemStatus, - type ApiTask, - type CoreHostSshCloseMessage, - type CoreHostSshEofMessage, - type CoreHostSshInputMessage, - type CoreHostSshOpenMessage, - type CoreHostSshResizeMessage, - type CoreDispatchMessage, - type CoreEgressTcpCloseMessage, - type CoreEgressTcpDataMessage, - type CoreEgressTcpOpenedMessage, - type JsonValue, - type ProviderEgressTcpCloseMessage, - type ProviderEgressTcpDataMessage, - type ProviderEgressTcpOpenMessage, - type ProviderHostSshDataMessage, - type ProviderHostSshErrorMessage, - type ProviderHostSshExitMessage, - type ProviderHostSshOpenedMessage, - type ProviderLabels, - isProviderDispatchCommand, - type ProviderToCoreMessage, - isProviderToCoreMessage, -} from "../../shared/src/index"; +import { readConfig } from "./config"; +import { ctx, sql, config, logger } from "./context"; +import { createLogger } from "./logger"; +import { initDatabase } from "./db"; +import { getNodes, getNodeDockerStatuses, getNodeSystemStatuses, getEvents, getTasks, getTask } from "./db"; +import { recordRequestPerformance, withPerformanceOperation, getPerformance } from "./performance"; +import { handleProviderMessage, markProviderOffline, markStaleProvidersOffline } from "./provider-registry"; +import { markStaleTasksFailed, dispatchTask } from "./task-dispatcher"; +import { handleSshClientMessage, sshRoute } from "./ssh-bridge"; +import { closeEgressTcpConnectionsForProvider } from "./egress-tcp"; +import { scheduledTaskRoute, runDueScheduledTasks, recoverScheduledRuns } from "./scheduler"; +import { microserviceRoute, getMicroservices } from "./microservice-proxy"; +import { getOverview, codexQueueLoadTest } from "./overview"; +import { jsonResponse, textResponse, readLimit, errorToJson } from "./http"; +import type { WsData, ProviderSocket } from "./types"; -interface RuntimeConfig { - port: number; - providerPort: number; - databaseUrl: string; - providerToken: string; - heartbeatTimeoutMs: number; - taskPendingTimeoutMs: number; - databaseVolumeName: string; - databaseVolumeSize: string; - pgdataBackupStagingDir: string; - baiduNetdiskInternalUrl: string; - microservices: MicroserviceConfig[]; - logFile: string; - databasePoolMax: number; -} +const runtimeConfig = readConfig(); +const { logger: loggerFn, recentLogs } = createLogger("backend-core", runtimeConfig.logFile); -interface MicroserviceConfig { - id: string; - name: string; - providerId: string; - description: string; - repository: { - url: string; - commitId: string; - dockerfile: string; - composeFile: string; - composeService: string; - containerName: string; - }; - backend: { - nodeBaseUrl: string; - nodeBindHost: string; - nodePort: number; - proxyMode: string; - frontendOnly: boolean; - public: boolean; - allowedMethods: string[]; - allowedPathPrefixes: string[]; - healthPath: string; - timeoutMs: number; - }; - deployment: { - mode: "unidesk-direct" | "v3sctl-managed"; - adapterServiceId?: string; - v3sServiceId?: string; - namespace?: string; - expectedNodeIds?: string[]; - activeNodeId?: string; - }; - development: { - providerId: string; - sshPassthrough: boolean; - worktreePath: string; - }; - frontend: { - route: string; - integrated: boolean; - }; -} - -interface WsData { - providerId?: string; - channel?: "provider" | "ssh"; - sessionId?: string; -} - -type ProviderSocket = ServerWebSocket; - -type SqlClient = ReturnType; - -interface RequestPerformanceSample { - at: string; - component: string; - method: string; - path: string; - status: number; - durationMs: number; - ok: boolean; -} - -interface OperationPerformanceSample { - at: string; - service: string; - operation: string; - durationMs: number; - ok: boolean; - detail: string; -} - -interface RawTaskRow { - id: string; - provider_id: string; - command: string; - status: string; - payload: JsonValue; - result: JsonValue | null; - updated_at: Date | string; -} - -interface ScheduledTaskRow { - id: string; - name: string; - description: string; - enabled: boolean; - schedule_json: JsonValue; - action_json: JsonValue; - concurrency_policy: string; - next_run_at: Date | string | null; - last_run_at: Date | string | null; - last_run_id: string | null; - created_at: Date | string; - updated_at: Date | string; -} - -interface ScheduledTaskRunRow { - id: string; - schedule_id: string; - trigger_type: string; - status: string; - task_id: string | null; - result: JsonValue | null; - error: string | null; - started_at: Date | string | null; - finished_at: Date | string | null; - duration_ms: number | string | null; - created_at: Date | string; - updated_at: Date | string; -} - -interface ScheduleSpec { - type: "daily" | "interval"; - timeOfDay?: string; - timezone?: string; - everySeconds?: number; -} - -interface DispatchScheduleAction { - type: "dispatch"; - providerId: string; - command: CoreDispatchMessage["command"]; - payload: Record; - timeoutMs?: number; -} - -interface PgdataBackupScheduleAction { - type: "pgdata_backup"; - volumeName: string; - remoteBaseDir: string; - stagingSubdir: string; - baiduBaseUrl: string; - timeoutMs: number; - cleanupLocal: boolean; -} - -type ScheduleAction = DispatchScheduleAction | PgdataBackupScheduleAction; - -type TaskTerminalWaiter = (task: RawTaskRow | null) => void; - -interface EgressTcpConnection { - providerId: string; - connectionId: string; - socket: Socket; - provider: ProviderSocket; -} - -interface MicroserviceProxyCacheEntry { - expiresAt: number; - staleExpiresAt: number; - status: number; - contentType: string; - bodyText: string; -} - -interface MicroserviceHealthAssessment { - healthy: boolean; - reason: string; - checks: Record; - body: JsonValue; -} - -interface MicroserviceAvailabilityEntry { - expiresAt: number; - probe: Record; -} - -const recentLogs: unknown[] = []; -const activeProviders = new Map(); -const activeSshClients = new Map(); -const activeEgressTcpConnections = new Map(); -const serviceStartedAt = new Date(); -const config = readConfig(); -const logger = createLogger("backend-core", config.logFile); -const sql = postgres(config.databaseUrl, { - max: config.databasePoolMax, +ctx.config = runtimeConfig; +ctx.logger = loggerFn; +ctx.recentLogs = recentLogs; +ctx.sql = postgres(runtimeConfig.databaseUrl, { + max: runtimeConfig.databasePoolMax, idle_timeout: 20, connect_timeout: 10, connection: { application_name: "unidesk-backend-core" }, }); -let dbReady = false; -const requestPerformanceSamples: RequestPerformanceSample[] = []; -const operationPerformanceSamples: OperationPerformanceSample[] = []; -const maxPerformanceSamples = 3000; -const taskTerminalWaiters = new Map>(); -const microserviceProxyCache = new Map(); -const microserviceProxyRefreshes = new Map>(); -const microserviceAvailabilityCache = new Map(); -const microserviceAvailabilityRefreshes = new Map>>(); -const activeScheduledRuns = new Set(); -let lastTaskSweepAt = 0; -let taskSweepInFlight: Promise | null = null; -const microserviceProxyMaxBodyTextLength = 8 * 1024 * 1024; -const microserviceAvailabilityTtlMs = 30_000; -const microserviceForwardRequestHeaders = [ - "accept", - "content-type", - "range", - "x-auth", - "x-requested-with", - "destination", - "overwrite", - "tus-resumable", - "upload-concat", - "upload-defer-length", - "upload-length", - "upload-metadata", - "upload-offset", -] as const; - -function requiredEnv(name: string): string { - const value = process.env[name]; - if (value === undefined || value.length === 0) { - throw new Error(`Missing required environment variable: ${name}`); - } - return value; -} - -function readNumberEnv(name: string): number { - const raw = requiredEnv(name); - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); - } - return parsed; -} - -function readOptionalNumberEnv(name: string, fallback: number): number { - const raw = process.env[name]; - if (raw === undefined || raw.length === 0) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); - } - return parsed; -} - -function asRecord(value: unknown, name: string): Record { - if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${name} must be an object`); - return value as Record; -} - -function stringFromRecord(value: Record, key: string, path: string): string { - const field = value[key]; - if (typeof field !== "string" || field.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); - return field; -} - -function numberFromRecord(value: Record, key: string, path: string): number { - const field = value[key]; - if (typeof field !== "number" || !Number.isFinite(field) || field <= 0) throw new Error(`${path}.${key} must be a positive number`); - return field; -} - -function booleanFromRecord(value: Record, key: string, path: string): boolean { - const field = value[key]; - if (typeof field !== "boolean") throw new Error(`${path}.${key} must be a boolean`); - return field; -} - -function optionalStringFromRecord(value: Record, key: string, path: string): string | undefined { - const field = value[key]; - if (field === undefined) return undefined; - if (typeof field !== "string" || field.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); - return field; -} - -function stringArrayFromRecord(value: Record, key: string, path: string): string[] { - const field = value[key]; - if (!Array.isArray(field) || field.some((item) => typeof item !== "string" || item.length === 0)) { - throw new Error(`${path}.${key} must be an array of non-empty strings`); - } - return field as string[]; -} - -function optionalStringArrayFromRecord(value: Record, key: string, path: string): string[] | undefined { - const field = value[key]; - if (field === undefined) return undefined; - return stringArrayFromRecord(value, key, path); -} - -function parseMicroserviceConfig(value: unknown, index: number): MicroserviceConfig { - const path = `MICROSERVICES_JSON[${index}]`; - const item = asRecord(value, path); - const repository = asRecord(item.repository, `${path}.repository`); - const backend = asRecord(item.backend, `${path}.backend`); - const development = asRecord(item.development, `${path}.development`); - const frontend = asRecord(item.frontend, `${path}.frontend`); - const deployment = item.deployment === undefined ? undefined : asRecord(item.deployment, `${path}.deployment`); - const deploymentMode = deployment === undefined ? "unidesk-direct" : stringFromRecord(deployment, "mode", `${path}.deployment`); - if (deploymentMode !== "unidesk-direct" && deploymentMode !== "v3sctl-managed") { - throw new Error(`${path}.deployment.mode must be unidesk-direct or v3sctl-managed`); - } - return { - id: stringFromRecord(item, "id", path), - name: stringFromRecord(item, "name", path), - providerId: stringFromRecord(item, "providerId", path), - description: stringFromRecord(item, "description", path), - repository: { - url: stringFromRecord(repository, "url", `${path}.repository`), - commitId: stringFromRecord(repository, "commitId", `${path}.repository`), - dockerfile: stringFromRecord(repository, "dockerfile", `${path}.repository`), - composeFile: stringFromRecord(repository, "composeFile", `${path}.repository`), - composeService: stringFromRecord(repository, "composeService", `${path}.repository`), - containerName: stringFromRecord(repository, "containerName", `${path}.repository`), - }, - backend: { - nodeBaseUrl: stringFromRecord(backend, "nodeBaseUrl", `${path}.backend`), - nodeBindHost: stringFromRecord(backend, "nodeBindHost", `${path}.backend`), - nodePort: numberFromRecord(backend, "nodePort", `${path}.backend`), - proxyMode: stringFromRecord(backend, "proxyMode", `${path}.backend`), - frontendOnly: booleanFromRecord(backend, "frontendOnly", `${path}.backend`), - public: booleanFromRecord(backend, "public", `${path}.backend`), - allowedMethods: stringArrayFromRecord(backend, "allowedMethods", `${path}.backend`).map((method) => method.toUpperCase()), - allowedPathPrefixes: stringArrayFromRecord(backend, "allowedPathPrefixes", `${path}.backend`), - healthPath: stringFromRecord(backend, "healthPath", `${path}.backend`), - timeoutMs: numberFromRecord(backend, "timeoutMs", `${path}.backend`), - }, - deployment: deployment === undefined - ? { mode: "unidesk-direct" } - : { - mode: deploymentMode, - adapterServiceId: optionalStringFromRecord(deployment, "adapterServiceId", `${path}.deployment`), - v3sServiceId: optionalStringFromRecord(deployment, "v3sServiceId", `${path}.deployment`), - namespace: optionalStringFromRecord(deployment, "namespace", `${path}.deployment`), - expectedNodeIds: optionalStringArrayFromRecord(deployment, "expectedNodeIds", `${path}.deployment`), - activeNodeId: optionalStringFromRecord(deployment, "activeNodeId", `${path}.deployment`), - }, - development: { - providerId: stringFromRecord(development, "providerId", `${path}.development`), - sshPassthrough: booleanFromRecord(development, "sshPassthrough", `${path}.development`), - worktreePath: stringFromRecord(development, "worktreePath", `${path}.development`), - }, - frontend: { - route: stringFromRecord(frontend, "route", `${path}.frontend`), - integrated: booleanFromRecord(frontend, "integrated", `${path}.frontend`), - }, - }; -} - -function readMicroservicesEnv(): MicroserviceConfig[] { - const raw = process.env.MICROSERVICES_JSON; - if (raw === undefined || raw.length === 0) return []; - const parsed = JSON.parse(raw) as unknown; - if (!Array.isArray(parsed)) throw new Error("MICROSERVICES_JSON must be an array"); - return parsed.map(parseMicroserviceConfig); -} - -function readConfig(): RuntimeConfig { - return { - port: readNumberEnv("PORT"), - providerPort: readNumberEnv("PROVIDER_PORT"), - databaseUrl: requiredEnv("DATABASE_URL"), - providerToken: requiredEnv("PROVIDER_TOKEN"), - heartbeatTimeoutMs: readNumberEnv("HEARTBEAT_TIMEOUT_MS"), - taskPendingTimeoutMs: readOptionalNumberEnv("TASK_PENDING_TIMEOUT_MS", 10 * 60 * 1000), - databaseVolumeName: requiredEnv("DATABASE_VOLUME_NAME"), - databaseVolumeSize: requiredEnv("DATABASE_VOLUME_SIZE"), - pgdataBackupStagingDir: process.env.PGDATA_BACKUP_STAGING_DIR || "/data/baidu-netdisk-staging", - baiduNetdiskInternalUrl: process.env.BAIDU_NETDISK_INTERNAL_URL || "http://baidu-netdisk:4244", - microservices: readMicroservicesEnv(), - logFile: requiredEnv("LOG_FILE"), - databasePoolMax: Math.max(1, Math.min(16, readOptionalNumberEnv("DATABASE_POOL_MAX", 4))), - }; -} - -function createLogger(service: string, logFile: string) { - const writer = createHourlyJsonlWriter({ - baseLogFile: logFile, - service, - maxBytes: logRetentionBytesForService(service), - }); - writer.prune(); - return (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue): void => { - const entry = data === undefined - ? { ts: new Date().toISOString(), service, level, message } - : { ts: new Date().toISOString(), service, level, message, data }; - recentLogs.push(entry); - while (recentLogs.length > 500) recentLogs.shift(); - const line = `${JSON.stringify(entry)}\n`; - try { - writer.appendLine(line, new Date(entry.ts)); - } catch (error) { - console.error(JSON.stringify({ ts: new Date().toISOString(), service, level: "error", message: "log_write_failed", data: String(error) })); - } - const consoleMethod = level === "error" ? console.error : level === "warn" ? console.warn : console.log; - consoleMethod(line.trimEnd()); - }; -} - -function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body, null, 2), { - status, - headers: { - "content-type": "application/json; charset=utf-8", - "access-control-allow-origin": "*", - "access-control-allow-methods": "GET,HEAD,POST,PUT,PATCH,DELETE,OPTIONS", - "access-control-allow-headers": "content-type,x-provider-token", - }, - }); -} - -function textResponse(text: string, status = 200): Response { - return new Response(text, { - status, - headers: { - "content-type": "text/plain; charset=utf-8", - "access-control-allow-origin": "*", - }, - }); -} - -function trimPerformanceBuffers(): void { - while (requestPerformanceSamples.length > maxPerformanceSamples) requestPerformanceSamples.shift(); - while (operationPerformanceSamples.length > maxPerformanceSamples) operationPerformanceSamples.shift(); -} - -function classifyRequestComponent(pathname: string): string { - if (pathname.startsWith("/api/microservices/") && pathname.includes("/proxy/")) return "microservice_proxy"; - if (pathname.startsWith("/api/microservices/")) return "microservice_registry"; - if (pathname.startsWith("/api/nodes/")) return "node_metrics_api"; - if (pathname === "/api/nodes") return "node_inventory_api"; - if (pathname.startsWith("/api/tasks")) return "scheduler_api"; - if (pathname.startsWith("/api/schedules")) return "scheduler_api"; - if (pathname.startsWith("/api/events")) return "event_api"; - if (pathname.startsWith("/api/performance")) return "performance_api"; - if (pathname.startsWith("/api/")) return "core_api"; - if (pathname.startsWith("/ws/")) return "websocket_api"; - if (pathname === "/logs") return "logs_api"; - return "core_http"; -} - -function recordRequestPerformance(req: Request, pathname: string, response: Response | undefined, durationMs: number): void { - const status = response?.status ?? 101; - requestPerformanceSamples.push({ - at: new Date().toISOString(), - component: classifyRequestComponent(pathname), - method: req.method, - path: pathname, - status, - durationMs, - ok: status < 400, - }); - trimPerformanceBuffers(); -} - -function recordOperationPerformance(service: string, operation: string, durationMs: number, ok: boolean, detail = "-"): void { - operationPerformanceSamples.push({ - at: new Date().toISOString(), - service, - operation, - durationMs, - ok, - detail: detail.length > 260 ? `${detail.slice(0, 257)}...` : detail, - }); - trimPerformanceBuffers(); -} - -async function withPerformanceOperation(service: string, operation: string, detail: string, fn: () => Promise): Promise { - const started = performance.now(); +(async () => { try { - const result = await fn(); - recordOperationPerformance(service, operation, performance.now() - started, true, detail); - return result; + await initDatabase(ctx.sql!); + ctx.dbReady = true; + logger("info", "database_ready", undefined); + await recoverScheduledRuns(); } catch (error) { - recordOperationPerformance(service, operation, performance.now() - started, false, error instanceof Error ? error.message : String(error)); - throw error; + logger("error", "database_init_failed", errorToJson(error)); + setTimeout(() => process.exit(1), 1000); } -} - -function percentile(values: number[], ratio: number): number { - if (values.length === 0) return 0; - const sorted = [...values].sort((left, right) => left - right); - const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1)); - return sorted[index] ?? 0; -} - -function average(values: number[]): number { - if (values.length === 0) return 0; - return values.reduce((sum, value) => sum + value, 0) / values.length; -} - -function roundMs(value: number): number { - return Math.round(value * 10) / 10; -} - -function summarizeRequestPerformance(): JsonValue[] { - const groups = new Map(); - for (const sample of requestPerformanceSamples) { - const rows = groups.get(sample.component) ?? []; - rows.push(sample); - groups.set(sample.component, rows); - } - return Array.from(groups.entries()).map(([component, rows]) => { - const durations = rows.map((row) => row.durationMs); - const failed = rows.filter((row) => !row.ok).length; - return { - component, - requestCount: rows.length, - failureCount: failed, - failureRate: rows.length === 0 ? 0 : failed / rows.length, - averageLatencyMs: roundMs(average(durations)), - p95LatencyMs: roundMs(percentile(durations, 0.95)), - maxLatencyMs: roundMs(Math.max(0, ...durations)), - }; - }).sort((left, right) => Number((right as Record).requestCount ?? 0) - Number((left as Record).requestCount ?? 0)) as JsonValue[]; -} - -function summarizeOperationPerformance(): JsonValue[] { - const groups = new Map(); - for (const sample of operationPerformanceSamples) { - const key = `${sample.service}:${sample.operation}`; - const rows = groups.get(key) ?? []; - rows.push(sample); - groups.set(key, rows); - } - return Array.from(groups.entries()).map(([key, rows]) => { - const [service, ...operationParts] = key.split(":"); - const durations = rows.map((row) => row.durationMs); - const failed = rows.filter((row) => !row.ok).length; - return { - service, - operation: operationParts.join(":"), - count: rows.length, - failureCount: failed, - averageLatencyMs: roundMs(average(durations)), - p95LatencyMs: roundMs(percentile(durations, 0.95)), - maxLatencyMs: roundMs(Math.max(0, ...durations)), - }; - }).sort((left, right) => Number((right as Record).count ?? 0) - Number((left as Record).count ?? 0)) as JsonValue[]; -} - -async function getCodexQueueStoragePerformance(): Promise { - try { - const rows = await sql>` - SELECT status, count(*)::int AS count - FROM unidesk_codex_queue_tasks - GROUP BY status - ORDER BY status ASC - `; - return { - ok: true, - table: "unidesk_codex_queue_tasks", - counts: Object.fromEntries(rows.map((row) => [row.status, Number(row.count)])) as Record, - total: rows.reduce((sum, row) => sum + Number(row.count), 0), - }; - } catch (error) { - return { - ok: false, - table: "unidesk_codex_queue_tasks", - error: error instanceof Error ? error.message : String(error), - }; - } -} - -async function getPerformance(): Promise { - const memory = process.memoryUsage(); - const pgdata = await withPerformanceOperation("database", "pgdata_usage", "pg_database_size", () => getPgdataUsage()); - const codexQueueStorage = await withPerformanceOperation("database", "codex_queue_storage", "unidesk_codex_queue_tasks", () => getCodexQueueStoragePerformance()); - const recentFailures = requestPerformanceSamples - .filter((sample) => !sample.ok) - .slice(-20) - .reverse() - .map((sample) => ({ ...sample }) as Record); - const recentOperationCutoff = Date.now() - 10 * 60 * 1000; - const recentSlowOperations = operationPerformanceSamples - .filter((sample) => { - const at = Date.parse(sample.at); - return Number.isFinite(at) && at >= recentOperationCutoff; - }) - .sort((left, right) => Date.parse(right.at) - Date.parse(left.at)) - .slice(0, 20) - .map((sample) => ({ ...sample }) as Record); - return { - ok: true, - service: "backend-core", - generatedAt: new Date().toISOString(), - startedAt: serviceStartedAt.toISOString(), - uptimeSeconds: Math.floor((Date.now() - serviceStartedAt.getTime()) / 1000), - requests: { - sampleCount: requestPerformanceSamples.length, - componentSummary: summarizeRequestPerformance(), - recentFailures, - }, - operations: { - sampleCount: operationPerformanceSamples.length, - summary: summarizeOperationPerformance(), - recentSlowOperations, - }, - process: { - rssBytes: memory.rss, - heapUsedBytes: memory.heapUsed, - heapTotalBytes: memory.heapTotal, - externalBytes: memory.external, - arrayBuffersBytes: memory.arrayBuffers, - }, - database: { - ready: dbReady, - pgdata, - codexQueueStorage, - }, - }; -} - -async function initDatabase(client: SqlClient): Promise { - logger("info", "database_init_start", { databaseUrl: redactDatabaseUrl(config.databaseUrl) }); - await client` - CREATE TABLE IF NOT EXISTS unidesk_nodes ( - provider_id TEXT PRIMARY KEY, - name TEXT NOT NULL, - labels JSONB NOT NULL DEFAULT '{}'::jsonb, - status TEXT NOT NULL, - connected_at TIMESTAMPTZ, - last_heartbeat TIMESTAMPTZ, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_events ( - id BIGSERIAL PRIMARY KEY, - type TEXT NOT NULL, - source TEXT NOT NULL, - payload JSONB NOT NULL DEFAULT '{}'::jsonb, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_tasks ( - id TEXT PRIMARY KEY, - provider_id TEXT NOT NULL, - command TEXT NOT NULL, - status TEXT NOT NULL, - payload JSONB NOT NULL DEFAULT '{}'::jsonb, - result JSONB, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_scheduled_tasks ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT NOT NULL DEFAULT '', - enabled BOOLEAN NOT NULL DEFAULT true, - schedule_json JSONB NOT NULL DEFAULT '{}'::jsonb, - action_json JSONB NOT NULL DEFAULT '{}'::jsonb, - concurrency_policy TEXT NOT NULL DEFAULT 'skip', - next_run_at TIMESTAMPTZ, - last_run_at TIMESTAMPTZ, - last_run_id TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_scheduled_task_runs ( - id TEXT PRIMARY KEY, - schedule_id TEXT NOT NULL REFERENCES unidesk_scheduled_tasks(id) ON DELETE CASCADE, - trigger_type TEXT NOT NULL, - status TEXT NOT NULL, - task_id TEXT, - result JSONB, - error TEXT, - started_at TIMESTAMPTZ, - finished_at TIMESTAMPTZ, - duration_ms BIGINT, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_node_docker_status ( - provider_id TEXT PRIMARY KEY, - status JSONB NOT NULL DEFAULT '{}'::jsonb, - collected_at TIMESTAMPTZ, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_node_system_status ( - provider_id TEXT PRIMARY KEY, - status JSONB NOT NULL DEFAULT '{}'::jsonb, - collected_at TIMESTAMPTZ, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client` - CREATE TABLE IF NOT EXISTS unidesk_node_metric_samples ( - id BIGSERIAL PRIMARY KEY, - provider_id TEXT NOT NULL, - collected_at TIMESTAMPTZ NOT NULL, - cpu_percent DOUBLE PRECISION NOT NULL DEFAULT 0, - memory_percent DOUBLE PRECISION NOT NULL DEFAULT 0, - disk_percent DOUBLE PRECISION NOT NULL DEFAULT 0, - sample JSONB NOT NULL DEFAULT '{}'::jsonb, - created_at TIMESTAMPTZ NOT NULL DEFAULT now() - ) - `; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_tasks_updated_at ON unidesk_tasks(updated_at DESC)`; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_tasks_status_updated_at ON unidesk_tasks(status, updated_at DESC)`; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_scheduled_tasks_next_run ON unidesk_scheduled_tasks(enabled, next_run_at)`; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_scheduled_task_runs_schedule_updated ON unidesk_scheduled_task_runs(schedule_id, updated_at DESC)`; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_scheduled_task_runs_status_updated ON unidesk_scheduled_task_runs(status, updated_at DESC)`; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_node_system_status_updated_at ON unidesk_node_system_status(updated_at DESC)`; - await client`CREATE INDEX IF NOT EXISTS idx_unidesk_node_metric_samples_provider_time ON unidesk_node_metric_samples(provider_id, collected_at DESC)`; - dbReady = true; - logger("info", "database_init_complete"); -} - -async function initDatabaseWithRetry(): Promise { - const started = Date.now(); - let attempt = 0; - while (!dbReady) { - attempt += 1; - try { - await initDatabase(sql); - return; - } catch (error) { - const elapsedMs = Date.now() - started; - logger("warn", "database_init_retry", { attempt, elapsedMs, error: errorToJson(error) }); - if (elapsedMs > 90_000) { - logger("error", "database_init_failed", { attempt, elapsedMs, error: errorToJson(error) }); - throw error; - } - await Bun.sleep(Math.min(1000 * attempt, 5000)); - } - } -} - -function redactDatabaseUrl(value: string): string { - try { - const url = new URL(value); - if (url.password) url.password = "***"; - return url.toString(); - } catch { - return ""; - } -} - -function errorToJson(error: unknown): JsonValue { - if (error instanceof Error) { - return { name: error.name, message: error.message, stack: error.stack ?? null }; - } - return String(error); -} - -function compactJson(value: unknown, depth = 0): JsonValue { - if (value === null || typeof value === "number" || typeof value === "boolean") return value; - if (typeof value === "string") return value.length > 600 ? `${value.slice(0, 600)}...` : value; - if (Array.isArray(value)) { - const items = value.slice(0, 20).map((item) => compactJson(item, depth + 1)); - if (value.length > 20) items.push({ truncatedItems: value.length - 20 }); - return items; - } - if (typeof value === "object" && value !== null) { - if (depth >= 4) return ""; - const entries = Object.entries(value as Record).slice(0, 30); - const result: Record = {}; - for (const [key, item] of entries) result[key] = compactJson(item, depth + 1); - const total = Object.keys(value as Record).length; - if (total > entries.length) result.truncatedKeys = total - entries.length; - return result; - } - return String(value); -} - -async function recordEvent(type: string, source: string, payload: JsonValue): Promise { - logger("info", type, { source, payload }); - if (!dbReady) return; - try { - await sql` - INSERT INTO unidesk_events (type, source, payload) - VALUES (${type}, ${source}, ${sql.json(payload)}) - `; - } catch (error) { - logger("error", "event_insert_failed", { type, source, error: errorToJson(error) }); - } -} - -async function upsertNodeOnline(providerId: string, name: string, labels: ProviderLabels): Promise { - await sql` - INSERT INTO unidesk_nodes (provider_id, name, labels, status, connected_at, last_heartbeat, updated_at) - VALUES (${providerId}, ${name}, ${sql.json(labels)}, 'online', now(), now(), now()) - ON CONFLICT (provider_id) DO UPDATE SET - name = EXCLUDED.name, - labels = EXCLUDED.labels, - status = 'online', - connected_at = COALESCE(unidesk_nodes.connected_at, EXCLUDED.connected_at), - last_heartbeat = now(), - updated_at = now() - `; -} - -async function touchHeartbeat(providerId: string, labels: ProviderLabels): Promise { - await sql` - UPDATE unidesk_nodes - SET labels = unidesk_nodes.labels || ${sql.json(labels)}, status = 'online', last_heartbeat = now(), updated_at = now() - WHERE provider_id = ${providerId} - `; -} - -async function providerCapabilities(providerId: string): Promise { - if (!dbReady) return []; - const rows = await sql>` - SELECT labels - FROM unidesk_nodes - WHERE provider_id = ${providerId} - LIMIT 1 - `; - const labels = rows[0]?.labels; - if (typeof labels !== "object" || labels === null || Array.isArray(labels)) return []; - const capabilities = (labels as Record).unideskCapabilities; - return Array.isArray(capabilities) ? capabilities.filter((item): item is string => typeof item === "string") : []; -} - -async function providerSupports(providerId: string, capability: string): Promise { - const capabilities = await providerCapabilities(providerId); - return capabilities.includes(capability); -} - -async function upsertDockerStatus(providerId: string, status: JsonValue, collectedAt: string): Promise { - await sql` - INSERT INTO unidesk_node_docker_status (provider_id, status, collected_at, updated_at) - VALUES (${providerId}, ${sql.json(status)}, ${collectedAt}, now()) - ON CONFLICT (provider_id) DO UPDATE SET - status = EXCLUDED.status, - collected_at = EXCLUDED.collected_at, - updated_at = now() - `; -} - -function recordValue(value: unknown, key: string): unknown { - if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; - return (value as Record)[key]; -} - -function nestedNumber(value: unknown, objectKey: string, numberKey: string): number { - const parsed = Number(recordValue(recordValue(value, objectKey), numberKey)); - return Number.isFinite(parsed) ? parsed : 0; -} - -async function upsertSystemStatus(providerId: string, status: JsonValue, collectedAt: string): Promise { - const cpuPercent = nestedNumber(status, "cpu", "percent"); - const memoryPercent = nestedNumber(status, "memory", "percent"); - const diskPercent = nestedNumber(status, "disk", "percent"); - await sql.begin(async (tx) => { - await tx` - INSERT INTO unidesk_node_system_status (provider_id, status, collected_at, updated_at) - VALUES (${providerId}, ${tx.json(status)}, ${collectedAt}, now()) - ON CONFLICT (provider_id) DO UPDATE SET - status = EXCLUDED.status, - collected_at = EXCLUDED.collected_at, - updated_at = now() - `; - await tx` - INSERT INTO unidesk_node_metric_samples (provider_id, collected_at, cpu_percent, memory_percent, disk_percent, sample) - VALUES (${providerId}, ${collectedAt}, ${cpuPercent}, ${memoryPercent}, ${diskPercent}, ${tx.json(status)}) - `; - await tx` - DELETE FROM unidesk_node_metric_samples - WHERE provider_id = ${providerId} - AND id NOT IN ( - SELECT id FROM unidesk_node_metric_samples - WHERE provider_id = ${providerId} - ORDER BY collected_at DESC - LIMIT 720 - ) - `; - }); -} - -async function markProviderOffline(providerId: string): Promise { - activeProviders.delete(providerId); - if (!dbReady) return; - await sql` - UPDATE unidesk_nodes - SET status = 'offline', updated_at = now() - WHERE provider_id = ${providerId} - `; - await recordEvent("provider_offline", providerId, { providerId }); -} - -async function markStaleProvidersOffline(): Promise { - if (!dbReady) return; - const timeoutMs = config.heartbeatTimeoutMs; - const rows = await sql<{ provider_id: string }[]>` - UPDATE unidesk_nodes - SET status = 'offline', updated_at = now() - WHERE status = 'online' - AND last_heartbeat IS NOT NULL - AND last_heartbeat < now() - (${timeoutMs} * interval '1 millisecond') - RETURNING provider_id - `; - for (const row of rows) { - activeProviders.delete(row.provider_id); - await recordEvent("provider_heartbeat_timeout", row.provider_id, { providerId: row.provider_id, timeoutMs }); - } -} - -async function markStaleTasksFailed(): Promise { - if (!dbReady) return; - const timeoutMs = config.taskPendingTimeoutMs; - const rows = await sql>` - WITH stale AS ( - SELECT id, provider_id, command, status AS previous_status, updated_at - FROM unidesk_tasks - WHERE status IN ('queued', 'dispatched', 'running') - AND updated_at < now() - (${timeoutMs}::bigint * interval '1 millisecond') - FOR UPDATE - ), - updated AS ( - UPDATE unidesk_tasks task - SET - status = 'failed', - result = jsonb_build_object( - 'error', 'task timed out without terminal provider status', - 'timeoutMs', ${timeoutMs}::bigint, - 'previousStatus', stale.previous_status, - 'previousResult', task.result, - 'timedOutAt', now() - ), - updated_at = now() - FROM stale - WHERE task.id = stale.id - RETURNING task.id, task.provider_id, task.command, stale.previous_status, stale.updated_at - ) - SELECT * FROM updated - `; - for (const row of rows) { - await recordEvent("task_timeout", row.provider_id, { - taskId: row.id, - providerId: row.provider_id, - command: row.command, - previousStatus: row.previous_status, - previousUpdatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), - timeoutMs, - }); - notifyTaskTerminal(row.id).catch((error) => logger("error", "task_waiter_notify_failed", { taskId: row.id, error: errorToJson(error) })); - } -} - -async function maybeMarkStaleTasksFailed(minIntervalMs = 15_000): Promise { - if (!dbReady) return; - const now = Date.now(); - if (taskSweepInFlight !== null) return taskSweepInFlight; - if (now - lastTaskSweepAt < minIntervalMs) return; - lastTaskSweepAt = now; - taskSweepInFlight = markStaleTasksFailed() - .catch((error) => { - logger("error", "task_timeout_sweep_failed", { error: errorToJson(error) }); - }) - .finally(() => { - taskSweepInFlight = null; - }); - return taskSweepInFlight; -} - -function parseMessage(raw: string | Buffer): ProviderToCoreMessage { - const text = typeof raw === "string" ? raw : raw.toString("utf8"); - const parsed = JSON.parse(text) as unknown; - if (!isProviderToCoreMessage(parsed)) { - throw new Error(`Unsupported provider message: ${text.slice(0, 200)}`); - } - return parsed; -} - -function safeSessionId(): string { - return `ssh_${Date.now()}_${Math.random().toString(16).slice(2)}`; -} - -function wsSendJson(ws: ProviderSocket, value: unknown): void { - if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(value)); -} - -function egressTcpKey(providerId: string, connectionId: string): string { - return `${providerId}:${connectionId}`; -} - -function isValidEgressHost(host: string): boolean { - return host.length > 0 && host.length <= 253 && !/[\s/\\:@\0]/u.test(host); -} - -function isValidEgressPort(port: number): boolean { - return Number.isInteger(port) && port > 0 && port <= 65_535; -} - -function sendEgressClose(provider: ProviderSocket, connectionId: string, error?: string): void { - const message: CoreEgressTcpCloseMessage = error === undefined - ? { type: "egress_tcp_close", connectionId } - : { type: "egress_tcp_close", connectionId, error }; - wsSendJson(provider, message); -} - -function closeEgressTcpConnection(providerId: string, connectionId: string, error?: string): void { - const key = egressTcpKey(providerId, connectionId); - const connection = activeEgressTcpConnections.get(key); - if (connection === undefined) return; - activeEgressTcpConnections.delete(key); - connection.socket.destroy(); - if (error !== undefined) sendEgressClose(connection.provider, connectionId, error); -} - -function handleEgressTcpOpen(ws: ProviderSocket, message: ProviderEgressTcpOpenMessage): void { - const host = message.host.trim(); - const port = Number(message.port); - if (!isValidEgressHost(host) || !isValidEgressPort(port)) { - sendEgressClose(ws, message.connectionId, "invalid egress target"); - return; - } - const key = egressTcpKey(message.providerId, message.connectionId); - closeEgressTcpConnection(message.providerId, message.connectionId); - const socket = connectTcp({ host, port }); - const connection: EgressTcpConnection = { providerId: message.providerId, connectionId: message.connectionId, socket, provider: ws }; - activeEgressTcpConnections.set(key, connection); - socket.on("connect", () => { - const opened: CoreEgressTcpOpenedMessage = { type: "egress_tcp_opened", connectionId: message.connectionId }; - wsSendJson(ws, opened); - }); - socket.on("data", (chunk) => { - const data: CoreEgressTcpDataMessage = { - type: "egress_tcp_data", - connectionId: message.connectionId, - data: chunk.toString("base64"), - encoding: "base64", - }; - wsSendJson(ws, data); - }); - socket.on("close", () => { - if (activeEgressTcpConnections.get(key) !== connection) return; - activeEgressTcpConnections.delete(key); - sendEgressClose(ws, message.connectionId); - }); - socket.on("error", (error) => { - if (activeEgressTcpConnections.get(key) !== connection) return; - activeEgressTcpConnections.delete(key); - sendEgressClose(ws, message.connectionId, error.message); - }); -} - -function handleEgressTcpData(message: ProviderEgressTcpDataMessage): void { - const connection = activeEgressTcpConnections.get(egressTcpKey(message.providerId, message.connectionId)); - if (connection === undefined) return; - connection.socket.write(Buffer.from(message.data, message.encoding === "base64" ? "base64" : "utf8")); -} - -function handleEgressTcpClose(message: ProviderEgressTcpCloseMessage): void { - closeEgressTcpConnection(message.providerId, message.connectionId); -} - -function closeEgressTcpConnectionsForProvider(providerId: string): void { - for (const [key, connection] of activeEgressTcpConnections) { - if (connection.providerId !== providerId) continue; - activeEgressTcpConnections.delete(key); - connection.socket.destroy(); - } -} - -function sshClientFor(sessionId: string): ProviderSocket | null { - return activeSshClients.get(sessionId) ?? null; -} - -function providerForSsh(providerId: string): ProviderSocket | null { - return activeProviders.get(providerId) ?? null; -} - -function closeSshClient(sessionId: string, code = 1000, reason = "ssh session closed"): void { - const client = sshClientFor(sessionId); - activeSshClients.delete(sessionId); - if (client !== null && client.readyState === WebSocket.OPEN) client.close(code, reason); -} - -function forwardSshProviderMessage( - message: ProviderHostSshOpenedMessage | ProviderHostSshDataMessage | ProviderHostSshExitMessage | ProviderHostSshErrorMessage, -): void { - const client = sshClientFor(message.sessionId); - if (client === null) { - logger("warn", "ssh_client_missing", { providerId: message.providerId, sessionId: message.sessionId, type: message.type }); - return; - } - if (message.type === "host_ssh_opened") { - wsSendJson(client, { type: "ssh.opened", providerId: message.providerId, sessionId: message.sessionId }); - return; - } - if (message.type === "host_ssh_data") { - wsSendJson(client, { type: "ssh.data", stream: message.stream, data: message.data, encoding: message.encoding }); - return; - } - if (message.type === "host_ssh_exit") { - wsSendJson(client, { type: "ssh.exit", exitCode: message.exitCode, signal: message.signal }); - setTimeout(() => closeSshClient(message.sessionId), 50); - return; - } - wsSendJson(client, { type: "ssh.error", message: message.message }); - setTimeout(() => closeSshClient(message.sessionId, 1011, "ssh session error"), 50); -} - -async function handleProviderMessage(ws: ProviderSocket, raw: string | Buffer): Promise { - const message = parseMessage(raw); - ws.data.providerId = message.providerId; - activeProviders.set(message.providerId, ws); - - if ( - message.type === "host_ssh_opened" || - message.type === "host_ssh_data" || - message.type === "host_ssh_exit" || - message.type === "host_ssh_error" - ) { - forwardSshProviderMessage(message); - return; - } - - if (message.type === "egress_tcp_open") { - handleEgressTcpOpen(ws, message); - return; - } - if (message.type === "egress_tcp_data") { - handleEgressTcpData(message); - return; - } - if (message.type === "egress_tcp_close") { - handleEgressTcpClose(message); - return; - } - - if (message.type === "register") { - const labels = { ...message.labels, unideskCapabilities: message.capabilities }; - await upsertNodeOnline(message.providerId, message.name, labels); - await recordEvent("provider_registered", message.providerId, { - providerId: message.providerId, - name: message.name, - labels, - capabilities: message.capabilities, - }); - ws.send(JSON.stringify({ type: "ack", requestId: "register", ok: true, message: "registered" })); - return; - } - - if (message.type === "heartbeat") { - await touchHeartbeat(message.providerId, message.labels); - logger("debug", "provider_heartbeat", { providerId: message.providerId, labels: message.labels }); - return; - } - - if (message.type === "system_status") { - await upsertSystemStatus(message.providerId, message.status as unknown as JsonValue, message.status.collectedAt); - logger("debug", "provider_system_status", { - providerId: message.providerId, - cpuPercent: message.status.cpu.percent, - memoryPercent: message.status.memory.percent, - diskPercent: message.status.disk.percent, - ok: message.status.ok, - }); - return; - } - - if (message.type === "docker_status") { - await upsertDockerStatus(message.providerId, message.status as unknown as JsonValue, message.status.collectedAt); - logger("debug", "provider_docker_status", { providerId: message.providerId, counts: message.status.counts, ok: message.status.ok }); - return; - } - - await sql` - WITH incoming AS ( - SELECT ${message.status}::text AS status, ${sql.json(message.result ?? { message: message.message })}::jsonb AS result - ) - UPDATE unidesk_tasks - SET - status = CASE - WHEN unidesk_tasks.status IN ('succeeded', 'failed') AND incoming.status NOT IN ('succeeded', 'failed') THEN unidesk_tasks.status - WHEN unidesk_tasks.status = 'running' AND incoming.status = 'accepted' THEN unidesk_tasks.status - ELSE incoming.status - END, - result = CASE - WHEN unidesk_tasks.status IN ('succeeded', 'failed') AND incoming.status NOT IN ('succeeded', 'failed') THEN unidesk_tasks.result - WHEN unidesk_tasks.status = 'running' AND incoming.status = 'accepted' THEN unidesk_tasks.result - ELSE incoming.result - END, - updated_at = now() - FROM incoming - WHERE id = ${message.taskId} - `; - await recordEvent("task_status", message.providerId, { - providerId: message.providerId, - taskId: message.taskId, - status: message.status, - message: message.message, - result: message.result ?? null, - }); - if (isTerminalTaskStatus(message.status)) { - await notifyTaskTerminal(message.taskId); - } -} - -async function getNodes(): Promise { - const rows = await sql>>` - SELECT provider_id, name, labels, status, connected_at, last_heartbeat - FROM unidesk_nodes - ORDER BY status DESC, provider_id ASC - `; - return rows.map((row) => ({ - providerId: String(row.provider_id), - name: String(row.name), - status: row.status === "online" ? "online" : "offline", - labels: (row.labels ?? {}) as ProviderLabels, - connectedAt: row.connected_at instanceof Date ? row.connected_at.toISOString() : row.connected_at === null ? null : String(row.connected_at), - lastHeartbeat: row.last_heartbeat instanceof Date ? row.last_heartbeat.toISOString() : row.last_heartbeat === null ? null : String(row.last_heartbeat), - })); -} - -async function getNodeDockerStatuses(): Promise { - const rows = await sql>>` - SELECT n.provider_id, n.name, n.status AS node_status, d.status AS docker_status, d.updated_at - FROM unidesk_nodes n - LEFT JOIN unidesk_node_docker_status d ON d.provider_id = n.provider_id - ORDER BY n.status DESC, n.provider_id ASC - `; - return rows.map((row) => ({ - providerId: String(row.provider_id), - name: String(row.name), - nodeStatus: row.node_status === "online" ? "online" : "offline", - dockerStatus: row.docker_status === null || row.docker_status === undefined ? null : (row.docker_status as JsonValue), - updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at === null || row.updated_at === undefined ? null : String(row.updated_at), - })); -} - -function metricPointFromSample(sample: unknown, collectedAt: string): JsonValue { - return { - at: collectedAt, - cpuPercent: nestedNumber(sample, "cpu", "percent"), - memoryPercent: nestedNumber(sample, "memory", "percent"), - diskPercent: nestedNumber(sample, "disk", "percent"), - memoryUsedBytes: nestedNumber(sample, "memory", "usedBytes"), - memoryTotalBytes: nestedNumber(sample, "memory", "totalBytes"), - diskUsedBytes: nestedNumber(sample, "disk", "usedBytes"), - diskTotalBytes: nestedNumber(sample, "disk", "totalBytes"), - load1: nestedNumber(sample, "cpu", "load1"), - }; -} - -async function getNodeSystemStatuses(limit: number): Promise { - const currentRows = await sql>>` - SELECT n.provider_id, n.name, n.status AS node_status, s.status AS system_status, s.updated_at - FROM unidesk_nodes n - LEFT JOIN unidesk_node_system_status s ON s.provider_id = n.provider_id - ORDER BY n.status DESC, n.provider_id ASC - `; - const sampleRows = await sql>>` - SELECT n.provider_id, recent.collected_at, recent.sample - FROM unidesk_nodes n - LEFT JOIN LATERAL ( - SELECT collected_at, sample - FROM unidesk_node_metric_samples m - WHERE m.provider_id = n.provider_id - ORDER BY collected_at DESC - LIMIT ${limit} - ) recent ON true - WHERE recent.collected_at IS NOT NULL - ORDER BY n.provider_id ASC, recent.collected_at ASC - `; - const historyByProvider = new Map(); - for (const row of sampleRows) { - const providerId = String(row.provider_id); - const collectedAt = row.collected_at instanceof Date ? row.collected_at.toISOString() : String(row.collected_at); - const history = historyByProvider.get(providerId) ?? []; - history.push(metricPointFromSample(row.sample ?? {}, collectedAt)); - historyByProvider.set(providerId, history); - } - return currentRows.map((row) => { - const providerId = String(row.provider_id); - return { - providerId, - name: String(row.name), - nodeStatus: row.node_status === "online" ? "online" : "offline", - current: row.system_status === null || row.system_status === undefined ? null : (row.system_status as JsonValue), - history: historyByProvider.get(providerId) ?? [], - updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : row.updated_at === null || row.updated_at === undefined ? null : String(row.updated_at), - }; - }); -} - -async function getEvents(limit: number): Promise { - const rows = await sql>>` - SELECT id, type, source, payload, created_at - FROM unidesk_events - ORDER BY id DESC - LIMIT ${limit} - `; - return rows.map((row) => ({ - id: Number(row.id), - type: String(row.type), - source: String(row.source), - payload: compactJson(row.payload ?? {}), - createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), - })); -} - -function rowString(row: Record, key: string): string { - const value = row[key]; - return typeof value === "string" ? value : value === null || value === undefined ? "" : String(value); -} - -function rowNumber(row: Record, key: string): number | null { - const value = row[key]; - const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN; - return Number.isFinite(parsed) ? parsed : null; -} - -function taskJsonSummary(row: Record, prefix: "payload" | "result"): JsonValue { - const type = rowString(row, `${prefix}_type`); - if (type.length === 0) return prefix === "result" ? null : {}; - const summary: Record = { summaryOnly: true, type }; - const fields = prefix === "payload" - ? ["source", "serviceId", "method", "path", "mode", "targetBaseUrl", "timeoutMs", "targetProviderGatewayVersion", "providerGatewayVersion"] - : ["error", "reason", "message", "status", "exitCode", "code", "signal", "timeoutMs", "previousStatus", "mode", "policy", "targetProviderGatewayVersion", "providerGatewayVersion", "updaterContainerId"]; - for (const field of fields) { - const value = row[`${prefix}_${field}`]; - if (value !== null && value !== undefined && String(value).length > 0) { - summary[field] = typeof value === "number" || typeof value === "boolean" ? value : String(value); - } - } - const bodyChars = rowNumber(row, `${prefix}_body_text_chars`); - if (bodyChars !== null) summary.bodyText = ``; - return summary; -} - -function taskSummaryFromRow(row: Record): ApiTask { - return { - id: String(row.id), - providerId: String(row.provider_id), - command: String(row.command), - status: String(row.status), - payload: taskJsonSummary(row, "payload"), - result: taskJsonSummary(row, "result"), - createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), - updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), - _summaryOnly: true, - } as ApiTask; -} - -async function getTask(taskId: string): Promise { - const rows = await sql>>` - SELECT - id, - provider_id, - command, - status, - CASE - WHEN payload ? 'bodyText' THEN jsonb_set(payload - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) - ELSE payload - END AS payload, - CASE - WHEN result IS NOT NULL AND result ? 'bodyText' THEN jsonb_set(result - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) - ELSE result - END AS result, - created_at, - updated_at - FROM unidesk_tasks - WHERE id = ${taskId} - LIMIT 1 - `; - const row = rows[0]; - if (row === undefined) return null; - return { - id: String(row.id), - providerId: String(row.provider_id), - command: String(row.command), - status: String(row.status), - payload: compactJson(row.payload ?? {}), - result: compactJson(row.result ?? null), - createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), - updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), - }; -} - -async function getTasks(limit: number, statusFilter = "all", lite = false, summary = false): Promise { - await maybeMarkStaleTasksFailed(); - if (summary && !lite) { - const rows = statusFilter === "pending" - ? await sql>>` - SELECT - id, - provider_id, - command, - status, - jsonb_typeof(payload) AS payload_type, - payload->>'source' AS "payload_source", - payload->>'serviceId' AS "payload_serviceId", - payload->>'method' AS "payload_method", - payload->>'path' AS "payload_path", - payload->>'mode' AS "payload_mode", - payload->>'targetBaseUrl' AS "payload_targetBaseUrl", - payload->>'timeoutMs' AS "payload_timeoutMs", - payload->>'targetProviderGatewayVersion' AS "payload_targetProviderGatewayVersion", - payload->>'providerGatewayVersion' AS "payload_providerGatewayVersion", - CASE WHEN payload ? 'bodyText' THEN length(payload->>'bodyText') ELSE NULL END AS payload_body_text_chars, - jsonb_typeof(result) AS result_type, - result->>'error' AS "result_error", - result->>'reason' AS "result_reason", - result->>'message' AS "result_message", - result->>'status' AS "result_status", - result->>'exitCode' AS "result_exitCode", - result->>'code' AS "result_code", - result->>'signal' AS "result_signal", - result->>'timeoutMs' AS "result_timeoutMs", - result->>'previousStatus' AS "result_previousStatus", - result->>'mode' AS "result_mode", - COALESCE(result->>'policy', result #>> '{plan,policy}') AS "result_policy", - COALESCE(result->>'targetProviderGatewayVersion', result #>> '{plan,targetProviderGatewayVersion}') AS "result_targetProviderGatewayVersion", - COALESCE(result->>'providerGatewayVersion', result #>> '{plan,providerGatewayVersion}') AS "result_providerGatewayVersion", - result->>'updaterContainerId' AS "result_updaterContainerId", - CASE WHEN result ? 'bodyText' THEN length(result->>'bodyText') ELSE NULL END AS result_body_text_chars, - created_at, - updated_at - FROM unidesk_tasks - WHERE status IN ('queued', 'dispatched', 'running') - ORDER BY updated_at DESC - LIMIT ${limit} - ` - : await sql>>` - SELECT - id, - provider_id, - command, - status, - jsonb_typeof(payload) AS payload_type, - payload->>'source' AS "payload_source", - payload->>'serviceId' AS "payload_serviceId", - payload->>'method' AS "payload_method", - payload->>'path' AS "payload_path", - payload->>'mode' AS "payload_mode", - payload->>'targetBaseUrl' AS "payload_targetBaseUrl", - payload->>'timeoutMs' AS "payload_timeoutMs", - payload->>'targetProviderGatewayVersion' AS "payload_targetProviderGatewayVersion", - payload->>'providerGatewayVersion' AS "payload_providerGatewayVersion", - CASE WHEN payload ? 'bodyText' THEN length(payload->>'bodyText') ELSE NULL END AS payload_body_text_chars, - jsonb_typeof(result) AS result_type, - result->>'error' AS "result_error", - result->>'reason' AS "result_reason", - result->>'message' AS "result_message", - result->>'status' AS "result_status", - result->>'exitCode' AS "result_exitCode", - result->>'code' AS "result_code", - result->>'signal' AS "result_signal", - result->>'timeoutMs' AS "result_timeoutMs", - result->>'previousStatus' AS "result_previousStatus", - result->>'mode' AS "result_mode", - COALESCE(result->>'policy', result #>> '{plan,policy}') AS "result_policy", - COALESCE(result->>'targetProviderGatewayVersion', result #>> '{plan,targetProviderGatewayVersion}') AS "result_targetProviderGatewayVersion", - COALESCE(result->>'providerGatewayVersion', result #>> '{plan,providerGatewayVersion}') AS "result_providerGatewayVersion", - result->>'updaterContainerId' AS "result_updaterContainerId", - CASE WHEN result ? 'bodyText' THEN length(result->>'bodyText') ELSE NULL END AS result_body_text_chars, - created_at, - updated_at - FROM unidesk_tasks - ORDER BY updated_at DESC - LIMIT ${limit} - `; - return rows.map(taskSummaryFromRow); - } - const rows = statusFilter === "pending" - ? lite - ? await sql>>` - SELECT id, provider_id, command, status, created_at, updated_at - FROM unidesk_tasks - WHERE status IN ('queued', 'dispatched', 'running') - ORDER BY updated_at DESC - LIMIT ${limit} - ` - : await sql>>` - SELECT - id, - provider_id, - command, - status, - CASE - WHEN payload ? 'bodyText' THEN jsonb_set(payload - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) - ELSE payload - END AS payload, - CASE - WHEN result IS NOT NULL AND result ? 'bodyText' THEN jsonb_set(result - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) - ELSE result - END AS result, - created_at, - updated_at - FROM unidesk_tasks - WHERE status IN ('queued', 'dispatched', 'running') - ORDER BY updated_at DESC - LIMIT ${limit} - ` - : lite - ? await sql>>` - SELECT id, provider_id, command, status, created_at, updated_at - FROM unidesk_tasks - ORDER BY updated_at DESC - LIMIT ${limit} - ` - : await sql>>` - SELECT - id, - provider_id, - command, - status, - CASE - WHEN payload ? 'bodyText' THEN jsonb_set(payload - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) - ELSE payload - END AS payload, - CASE - WHEN result IS NOT NULL AND result ? 'bodyText' THEN jsonb_set(result - 'bodyText', '{bodyText}', to_jsonb(('>'bodyText')::text || ' chars>')::text)) - ELSE result - END AS result, - created_at, - updated_at - FROM unidesk_tasks - ORDER BY updated_at DESC - LIMIT ${limit} - `; - return rows.map((row) => ({ - id: String(row.id), - providerId: String(row.provider_id), - command: String(row.command), - status: String(row.status), - payload: lite ? {} : compactJson(row.payload ?? {}), - result: lite ? null : compactJson(row.result ?? null), - createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), - updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), - })); -} - -async function countPendingTasks(): Promise { - await maybeMarkStaleTasksFailed(); - const rows = await sql>` - SELECT count(*)::int AS count - FROM unidesk_tasks - WHERE status IN ('queued', 'dispatched', 'running') - `; - return Number(rows[0]?.count ?? 0); -} - -async function getPgdataUsage(): Promise { - const rows = await sql>` - SELECT - current_database()::text AS database_name, - pg_database_size(current_database())::bigint AS database_bytes, - pg_size_pretty(pg_database_size(current_database()))::text AS database_pretty - `; - const row = rows[0]; - return { - volumeName: config.databaseVolumeName, - volumeSize: config.databaseVolumeSize, - databaseName: row?.database_name ?? "unknown", - databaseBytes: Number(row?.database_bytes ?? 0), - databasePretty: row?.database_pretty ?? "--", - }; -} - -async function getOverview(): Promise { - const [nodeRows, dockerRows, systemRows, pendingTasks, pgdata, microserviceAvailabilitySummary] = await Promise.all([ - sql>` - SELECT - count(*)::int AS node_count, - count(*) FILTER (WHERE status = 'online')::int AS online_node_count - FROM unidesk_nodes - `, - sql>` - SELECT count(*) FILTER (WHERE d.status IS NOT NULL)::int AS docker_status_node_count - FROM unidesk_nodes n - LEFT JOIN unidesk_node_docker_status d ON d.provider_id = n.provider_id - `, - sql>` - SELECT count(*) FILTER (WHERE s.status IS NOT NULL)::int AS system_status_node_count - FROM unidesk_nodes n - LEFT JOIN unidesk_node_system_status s ON s.provider_id = n.provider_id - `, - countPendingTasks(), - getPgdataUsage(), - getMicroserviceAvailabilitySummary(), - ]); - const nodeRow = nodeRows[0]; - const dockerRow = dockerRows[0]; - const systemRow = systemRows[0]; - return { - service: "unidesk-core", - ok: true, - dbReady, - pgdata, - uptimeSeconds: Math.floor((Date.now() - serviceStartedAt.getTime()) / 1000), - nodeCount: Number(nodeRow?.node_count ?? 0), - onlineNodeCount: Number(nodeRow?.online_node_count ?? 0), - dockerStatusNodeCount: Number(dockerRow?.docker_status_node_count ?? 0), - systemStatusNodeCount: Number(systemRow?.system_status_node_count ?? 0), - pendingTaskCount: pendingTasks, - microserviceAvailability: microserviceAvailabilitySummary, - taskPendingTimeoutMs: config.taskPendingTimeoutMs, - activeSocketCount: activeProviders.size, - heartbeatTimeoutMs: config.heartbeatTimeoutMs, - }; -} - -function microserviceById(id: string): MicroserviceConfig | null { - return config.microservices.find((service) => service.id === id) ?? null; -} - -function dockerStatusRecord(status: JsonValue | null): Record { - return typeof status === "object" && status !== null && !Array.isArray(status) ? status as Record : {}; -} - -function findContainer(status: JsonValue | null, containerName: string): JsonValue | null { - const containers = dockerStatusRecord(status).containers; - if (!Array.isArray(containers)) return null; - const container = containers.find((item) => { - if (typeof item !== "object" || item === null || Array.isArray(item)) return false; - return (item as Record).name === containerName; - }); - return container === undefined ? null : container as JsonValue; -} - -function inferredMicroserviceAvailability(service: MicroserviceConfig, providerStatus: string, container: JsonValue | null): JsonValue { - const cached = cachedMicroserviceAvailability(service.id); - if (cached !== null) return cached; - const containerState = isPlainRecord(container) ? String(container.state ?? "").toLowerCase() : ""; - const containerStatus = isPlainRecord(container) ? String(container.status ?? "") : ""; - if (providerStatus !== "online") { - return { - serviceId: service.id, - healthy: false, - status: "unhealthy", - reason: `provider is ${providerStatus}`, - source: "runtime-inference", - }; - } - if (container !== null && containerState !== "running") { - return { - serviceId: service.id, - healthy: false, - status: "unhealthy", - reason: `container is ${containerStatus || containerState || "not running"}`, - source: "runtime-inference", - }; - } - return { - serviceId: service.id, - healthy: null, - status: "unknown", - reason: "strict health has not been probed yet", - source: "runtime-inference", - }; -} - -async function getMicroservices(): Promise { - const nodes = await getNodes(); - const dockerStatuses = await getNodeDockerStatuses(); - return config.microservices.map((service) => { - const node = nodes.find((item) => item.providerId === service.providerId) ?? null; - const docker = dockerStatuses.find((item) => item.providerId === service.providerId) ?? null; - const v3sManaged = isV3sctlManagedMicroservice(service); - const container = v3sManaged ? null : findContainer(docker?.dockerStatus ?? null, service.repository.containerName); - return { - ...service, - runtime: { - orchestrator: v3sManaged ? "v3sctl" : "unidesk-direct", - providerStatus: node?.status ?? "missing", - providerName: node?.name ?? service.providerId, - providerLastHeartbeat: node?.lastHeartbeat ?? null, - container, - availability: inferredMicroserviceAvailability(service, node?.status ?? "missing", container), - backendPortMapping: { - providerId: service.providerId, - node: `${service.backend.nodeBindHost}:${service.backend.nodePort}`, - mainServerAccess: "frontend-only backend proxy", - public: service.backend.public, - }, - }, - } satisfies JsonValue; - }); -} - -async function createAndSendTask( - providerId: string, - command: CoreDispatchMessage["command"], - payload: Record, -): Promise<{ taskId: string; providerOnline: boolean }> { - const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`; - await sql` - INSERT INTO unidesk_tasks (id, provider_id, command, status, payload, result) - VALUES (${taskId}, ${providerId}, ${command}, 'queued', ${sql.json(payload)}, NULL) - `; - const socket = activeProviders.get(providerId); - if (!socket) { - await recordEvent("task_queued_provider_offline", providerId, { taskId, providerId, command }); - return { taskId, providerOnline: false }; - } - const dispatch: CoreDispatchMessage = { type: "dispatch", taskId, command, payload }; - socket.send(JSON.stringify(dispatch)); - await sql` - UPDATE unidesk_tasks - SET status = 'dispatched', updated_at = now() - WHERE id = ${taskId} AND status = 'queued' - `; - await recordEvent("task_dispatched", providerId, { taskId, providerId, command }); - return { taskId, providerOnline: true }; -} - -async function rawTask(taskId: string): Promise { - const rows = await sql` - SELECT id, provider_id, command, status, payload, result, updated_at - FROM unidesk_tasks - WHERE id = ${taskId} - LIMIT 1 - `; - return rows[0] ?? null; -} - -function isTerminalTaskStatus(status: string): boolean { - return status === "succeeded" || status === "failed"; -} - -async function notifyTaskTerminal(taskId: string): Promise { - const waiters = taskTerminalWaiters.get(taskId); - if (waiters === undefined || waiters.size === 0) return; - const task = await rawTask(taskId); - if (task === null || !isTerminalTaskStatus(task.status)) return; - taskTerminalWaiters.delete(taskId); - for (const waiter of waiters) waiter(task); -} - -async function waitForTaskTerminal(taskId: string, timeoutMs: number): Promise { - let latest = await rawTask(taskId); - if (latest !== null && isTerminalTaskStatus(latest.status)) return latest; - return await new Promise((resolve) => { - let settled = false; - let timer: ReturnType; - const settle = (task: RawTaskRow | null): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - const waiters = taskTerminalWaiters.get(taskId); - waiters?.delete(settle); - if (waiters !== undefined && waiters.size === 0) taskTerminalWaiters.delete(taskId); - resolve(task); - }; - const waiters = taskTerminalWaiters.get(taskId) ?? new Set(); - waiters.add(settle); - taskTerminalWaiters.set(taskId, waiters); - timer = setTimeout(() => { - rawTask(taskId) - .then((task) => settle(task ?? latest)) - .catch(() => settle(latest)); - }, Math.max(1, timeoutMs)); - }); -} - -function rowIso(value: Date | string | null | undefined): string | null { - if (value === null || value === undefined) return null; - return value instanceof Date ? value.toISOString() : String(value); -} - -function isJsonRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function scheduleRunView(row: ScheduledTaskRunRow): JsonValue { - return { - id: row.id, - scheduleId: row.schedule_id, - trigger: row.trigger_type, - status: row.status, - taskId: row.task_id, - result: compactJson(row.result ?? null), - error: row.error ?? "", - startedAt: rowIso(row.started_at), - finishedAt: rowIso(row.finished_at), - durationMs: row.duration_ms === null ? null : Number(row.duration_ms), - createdAt: rowIso(row.created_at), - updatedAt: rowIso(row.updated_at), - }; -} - -function scheduledTaskView(row: ScheduledTaskRow, runs: ScheduledTaskRunRow[] = []): JsonValue { - return { - id: row.id, - name: row.name, - description: row.description, - enabled: row.enabled, - schedule: row.schedule_json, - action: row.action_json, - concurrencyPolicy: row.concurrency_policy, - nextRunAt: rowIso(row.next_run_at), - lastRunAt: rowIso(row.last_run_at), - lastRunId: row.last_run_id, - createdAt: rowIso(row.created_at), - updatedAt: rowIso(row.updated_at), - recentRuns: runs.map(scheduleRunView), - }; -} - -function normalizeScheduleId(value: unknown): string { - const raw = typeof value === "string" && value.trim().length > 0 - ? value.trim() - : `schedule_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; - if (!/^[A-Za-z0-9_.:-]{1,120}$/u.test(raw)) { - throw new Error("schedule id must be 1-120 chars using letters, numbers, _, ., :, or -"); - } - return raw; -} - -function normalizeTimeOfDay(value: unknown): string { - const raw = typeof value === "string" && value.trim().length > 0 ? value.trim() : "03:00"; - const match = raw.match(/^([01]\d|2[0-3]):([0-5]\d)$/u); - if (match === null) throw new Error("daily schedule timeOfDay must use HH:MM in UTC"); - return `${match[1]}:${match[2]}`; -} - -function numberInRange(value: unknown, fallback: number, min: number, max: number): number { - const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : fallback; - if (!Number.isFinite(parsed)) return fallback; - return Math.max(min, Math.min(max, Math.floor(parsed))); -} - -function normalizeScheduleSpec(value: unknown): ScheduleSpec { - const record = isJsonRecord(value) ? value : {}; - if (record.type === "interval") { - return { - type: "interval", - everySeconds: numberInRange(record.everySeconds, 3600, 60, 366 * 24 * 3600), - }; - } - return { - type: "daily", - timeOfDay: normalizeTimeOfDay(record.timeOfDay), - timezone: typeof record.timezone === "string" && record.timezone.length > 0 ? record.timezone : "Etc/UTC", - }; -} - -function normalizeScheduleAction(value: unknown): ScheduleAction { - if (!isJsonRecord(value)) throw new Error("scheduled task action must be an object"); - if (value.type === "dispatch") { - const providerId = typeof value.providerId === "string" ? value.providerId.trim() : ""; - if (providerId.length === 0) throw new Error("dispatch action providerId is required"); - if (!isProviderDispatchCommand(value.command)) throw new Error("dispatch action command is invalid"); - const payload = isJsonRecord(value.payload) ? value.payload : {}; - const timeoutMs = value.timeoutMs === undefined ? undefined : numberInRange(value.timeoutMs, config.taskPendingTimeoutMs, 1_000, 24 * 3600_000); - return { type: "dispatch", providerId, command: value.command, payload, ...(timeoutMs === undefined ? {} : { timeoutMs }) }; - } - if (value.type === "pgdata_backup") { - return { - type: "pgdata_backup", - volumeName: typeof value.volumeName === "string" && value.volumeName.length > 0 ? value.volumeName : config.databaseVolumeName, - remoteBaseDir: normalizeRemoteDir(typeof value.remoteBaseDir === "string" ? value.remoteBaseDir : "/SERVER_DATA/UNIDESK_PG_DATA"), - stagingSubdir: normalizeRelativeStagingPath(typeof value.stagingSubdir === "string" ? value.stagingSubdir : "server-data/unidesk-pg-data").relativePath, - baiduBaseUrl: typeof value.baiduBaseUrl === "string" && value.baiduBaseUrl.length > 0 ? value.baiduBaseUrl : config.baiduNetdiskInternalUrl, - timeoutMs: numberInRange(value.timeoutMs, 60 * 60_000, 60_000, 24 * 3600_000), - cleanupLocal: value.cleanupLocal !== false, - }; - } - throw new Error("scheduled task action.type must be dispatch or pgdata_backup"); -} - -function computeNextRunAt(schedule: ScheduleSpec, after = new Date()): Date { - if (schedule.type === "interval") { - return new Date(after.getTime() + Math.max(60, schedule.everySeconds ?? 3600) * 1000); - } - const [hourText = "03", minuteText = "00"] = (schedule.timeOfDay ?? "03:00").split(":"); - const candidate = new Date(Date.UTC(after.getUTCFullYear(), after.getUTCMonth(), after.getUTCDate(), Number(hourText), Number(minuteText), 0, 0)); - if (candidate.getTime() <= after.getTime()) candidate.setUTCDate(candidate.getUTCDate() + 1); - return candidate; -} - -async function getScheduledTasks(limit: number): Promise { - const rows = await sql` - SELECT * - FROM unidesk_scheduled_tasks - ORDER BY enabled DESC, next_run_at ASC NULLS LAST, updated_at DESC - LIMIT ${limit} - `; - const runRows = await sql` - SELECT * - FROM unidesk_scheduled_task_runs - ORDER BY updated_at DESC - LIMIT ${Math.max(20, limit * 5)} - `; - const runsBySchedule = new Map(); - for (const run of runRows) { - const list = runsBySchedule.get(run.schedule_id) ?? []; - if (list.length < 5) list.push(run); - runsBySchedule.set(run.schedule_id, list); - } - return rows.map((row) => scheduledTaskView(row, runsBySchedule.get(row.id) ?? [])); -} - -async function getScheduledTaskRuns(scheduleId: string | null, limit: number): Promise { - const rows = scheduleId === null - ? await sql` - SELECT * - FROM unidesk_scheduled_task_runs - ORDER BY updated_at DESC - LIMIT ${limit} - ` - : await sql` - SELECT * - FROM unidesk_scheduled_task_runs - WHERE schedule_id = ${scheduleId} - ORDER BY updated_at DESC - LIMIT ${limit} - `; - return rows.map(scheduleRunView); -} - -async function getScheduledTaskRow(scheduleId: string): Promise { - const rows = await sql` - SELECT * - FROM unidesk_scheduled_tasks - WHERE id = ${scheduleId} - LIMIT 1 - `; - return rows[0] ?? null; -} - -async function upsertScheduledTask(req: Request, scheduleIdFromPath: string | null): Promise { - const body = await req.json().catch(() => ({})) as Record; - const id = normalizeScheduleId(scheduleIdFromPath ?? body.id); - const schedule = normalizeScheduleSpec(body.schedule); - const action = normalizeScheduleAction(body.action); - const name = typeof body.name === "string" && body.name.trim().length > 0 ? body.name.trim() : id; - const description = typeof body.description === "string" ? body.description : ""; - const enabled = typeof body.enabled === "boolean" ? body.enabled : true; - const concurrencyPolicy = body.concurrencyPolicy === "parallel" ? "parallel" : "skip"; - const existing = await getScheduledTaskRow(id); - const nextRunAt = existing?.next_run_at && JSON.stringify(existing.schedule_json) === JSON.stringify(schedule) - ? rowIso(existing.next_run_at) - : computeNextRunAt(schedule).toISOString(); - const scheduleJson = schedule as unknown as JsonValue; - const actionJson = action as unknown as JsonValue; - const rows = await sql` - INSERT INTO unidesk_scheduled_tasks (id, name, description, enabled, schedule_json, action_json, concurrency_policy, next_run_at, updated_at) - VALUES (${id}, ${name}, ${description}, ${enabled}, ${sql.json(scheduleJson)}, ${sql.json(actionJson)}, ${concurrencyPolicy}, ${nextRunAt}, now()) - ON CONFLICT (id) DO UPDATE SET - name = EXCLUDED.name, - description = EXCLUDED.description, - enabled = EXCLUDED.enabled, - schedule_json = EXCLUDED.schedule_json, - action_json = EXCLUDED.action_json, - concurrency_policy = EXCLUDED.concurrency_policy, - next_run_at = EXCLUDED.next_run_at, - updated_at = now() - RETURNING * - `; - await recordEvent(existing === null ? "scheduled_task_created" : "scheduled_task_updated", "scheduler", { scheduleId: id, name, enabled, schedule: scheduleJson, action: compactJson(action) }); - return jsonResponse({ ok: true, schedule: scheduledTaskView(rows[0]) }, existing === null ? 201 : 200); -} - -async function patchScheduledTask(scheduleId: string, req: Request): Promise { - const current = await getScheduledTaskRow(scheduleId); - if (current === null) return jsonResponse({ ok: false, error: `scheduled task not found: ${scheduleId}` }, 404); - const body = await req.json().catch(() => ({})) as Record; - const schedule = body.schedule === undefined ? normalizeScheduleSpec(current.schedule_json) : normalizeScheduleSpec(body.schedule); - const action = body.action === undefined ? normalizeScheduleAction(current.action_json) : normalizeScheduleAction(body.action); - const name = typeof body.name === "string" && body.name.trim().length > 0 ? body.name.trim() : current.name; - const description = typeof body.description === "string" ? body.description : current.description; - const enabled = typeof body.enabled === "boolean" ? body.enabled : current.enabled; - const concurrencyPolicy = body.concurrencyPolicy === "parallel" || body.concurrencyPolicy === "skip" ? body.concurrencyPolicy : current.concurrency_policy; - const scheduleChanged = body.schedule !== undefined && JSON.stringify(schedule) !== JSON.stringify(current.schedule_json); - const nextRunAt = scheduleChanged || current.next_run_at === null ? computeNextRunAt(schedule).toISOString() : rowIso(current.next_run_at); - const scheduleJson = schedule as unknown as JsonValue; - const actionJson = action as unknown as JsonValue; - const rows = await sql` - UPDATE unidesk_scheduled_tasks - SET name = ${name}, - description = ${description}, - enabled = ${enabled}, - schedule_json = ${sql.json(scheduleJson)}, - action_json = ${sql.json(actionJson)}, - concurrency_policy = ${concurrencyPolicy}, - next_run_at = ${nextRunAt}, - updated_at = now() - WHERE id = ${scheduleId} - RETURNING * - `; - await recordEvent("scheduled_task_updated", "scheduler", { scheduleId, name, enabled, schedule: scheduleJson, action: compactJson(action) }); - return jsonResponse({ ok: true, schedule: scheduledTaskView(rows[0]) }); -} - -async function deleteScheduledTask(scheduleId: string): Promise { - const rows = await sql>` - DELETE FROM unidesk_scheduled_tasks - WHERE id = ${scheduleId} - RETURNING id - `; - if (rows.length === 0) return jsonResponse({ ok: false, error: `scheduled task not found: ${scheduleId}` }, 404); - await recordEvent("scheduled_task_deleted", "scheduler", { scheduleId }); - return jsonResponse({ ok: true, deleted: scheduleId }); -} - -function scheduledRawTaskJson(task: RawTaskRow | null): JsonValue { - if (task === null) return null; - return { - id: task.id, - providerId: task.provider_id, - command: task.command, - status: task.status, - payload: compactJson(task.payload), - result: compactJson(task.result ?? null), - updatedAt: rowIso(task.updated_at), - }; -} - -async function executeDispatchScheduleAction(action: DispatchScheduleAction): Promise<{ ok: boolean; taskId: string; result: JsonValue }> { - const { taskId, providerOnline } = await createAndSendTask(action.providerId, action.command, { - source: "scheduled-task", - ...action.payload, - }); - if (!providerOnline) { - return { ok: false, taskId, result: { providerOnline, taskId, error: `provider is offline: ${action.providerId}` } }; - } - const timeoutMs = action.timeoutMs ?? numberInRange(action.payload.timeoutMs, config.taskPendingTimeoutMs + 5000, 1000, 24 * 3600_000); - const task = await waitForTaskTerminal(taskId, timeoutMs); - const ok = task?.status === "succeeded"; - return { - ok, - taskId, - result: { - providerOnline, - taskId, - timeoutMs, - terminal: task === null ? false : isTerminalTaskStatus(task.status), - task: scheduledRawTaskJson(task), - }, - }; -} - -function normalizeRemoteDir(input: string): string { - const raw = input.trim() || "/"; - if (raw.includes("\0")) throw new Error("remote directory must not contain null bytes"); - const normalized = pathPosix.normalize(raw.startsWith("/") ? raw : `/${raw}`); - return normalized === "/" ? "/" : normalized.replace(/\/+$/u, ""); -} - -function normalizeRelativeStagingPath(input: string): { relativePath: string; absolutePath: string } { - const cleaned = input.replace(/\\/g, "/").replace(/^\/+/u, "").trim(); - const normalized = pathPosix.normalize(cleaned || "."); - if (normalized === "." || normalized.startsWith("../") || normalized === "..") throw new Error("staging path must be a relative child path"); - const absolutePath = resolve(config.pgdataBackupStagingDir, normalized); - const rel = relative(config.pgdataBackupStagingDir, absolutePath); - if (rel.startsWith("..") || rel.includes("\0") || resolve(absolutePath) === resolve(config.pgdataBackupStagingDir)) { - throw new Error("staging path must stay inside PGDATA_BACKUP_STAGING_DIR"); - } - return { relativePath: normalized, absolutePath }; -} - -function remoteFolderChain(remoteDir: string): string[] { - const parts = normalizeRemoteDir(remoteDir).split("/").filter(Boolean); - const folders: string[] = []; - let current = ""; - for (const part of parts) { - current = `${current}/${part}`; - folders.push(current); - } - return folders; -} - -function safeFilePart(value: string): string { - return value.replace(/[^A-Za-z0-9_.-]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 80) || "data"; -} - -function utcBackupParts(date = new Date()): { date: string; time: string; month: string; stamp: string } { - const year = String(date.getUTCFullYear()).padStart(4, "0"); - const monthNumber = String(date.getUTCMonth() + 1).padStart(2, "0"); - const day = String(date.getUTCDate()).padStart(2, "0"); - const hour = String(date.getUTCHours()).padStart(2, "0"); - const minute = String(date.getUTCMinutes()).padStart(2, "0"); - const second = String(date.getUTCSeconds()).padStart(2, "0"); - return { date: `${year}${monthNumber}${day}`, time: `${hour}${minute}${second}`, month: `${year}${monthNumber}`, stamp: `${year}${monthNumber}${day}_${hour}${minute}${second}` }; -} - -function databaseConnectionParts(): { host: string; port: string; user: string; password: string } { - const url = new URL(config.databaseUrl); - return { - host: url.hostname || "database", - port: url.port || "5432", - user: decodeURIComponent(url.username || "postgres"), - password: decodeURIComponent(url.password || ""), - }; -} - -async function runLocalCommand( - command: string, - args: string[], - options: { cwd?: string; env?: Record; timeoutMs: number }, -): Promise<{ ok: boolean; stdout: string; stderr: string; exitCode: number | null; timedOut: boolean }> { - const proc = Bun.spawn([command, ...args], { - cwd: options.cwd, - env: { ...process.env, ...(options.env ?? {}) }, - stdout: "pipe", - stderr: "pipe", - }); - let timedOut = false; - const timer = setTimeout(() => { - timedOut = true; - proc.kill("SIGTERM"); - }, Math.max(1, options.timeoutMs)); - try { - const [stdout, stderr, exitCode] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { ok: exitCode === 0 && !timedOut, stdout: truncateText(stdout, 4000), stderr: truncateText(stderr, 4000), exitCode, timedOut }; - } finally { - clearTimeout(timer); - } -} - -async function sha256File(filePath: string): Promise { - const hash = createHash("sha256"); - await new Promise((resolveHash, rejectHash) => { - const stream = createReadStream(filePath); - stream.on("data", (chunk) => hash.update(chunk)); - stream.on("error", rejectHash); - stream.on("end", resolveHash); - }); - return hash.digest("hex"); -} - -async function fetchJsonWithTimeout(url: string, init: RequestInit, timeoutMs: number): Promise> { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutMs)); - try { - const response = await fetch(url, { ...init, signal: controller.signal }); - const text = await response.text(); - let parsed: unknown = {}; - try { - parsed = text.length > 0 ? JSON.parse(text) as unknown : {}; - } catch { - parsed = { text }; - } - if (!response.ok) throw new Error(`HTTP ${response.status}: ${truncateText(text, 1000)}`); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { value: parsed }; - return parsed as Record; - } finally { - clearTimeout(timer); - } -} - -async function baiduJson(baseUrl: string, path: string, init: RequestInit = {}, timeoutMs = 30_000): Promise> { - const url = new URL(path, baseUrl); - const headers = new Headers(init.headers); - if (init.body !== undefined && !headers.has("content-type")) headers.set("content-type", "application/json"); - const body = await fetchJsonWithTimeout(url.toString(), { ...init, headers }, timeoutMs); - if (body.ok === false) throw new Error(`Baidu Netdisk API failed: ${JSON.stringify(compactJson(body))}`); - return body; -} - -function jobRecord(value: unknown): Record { - return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; -} - -async function waitForBaiduTransfer(baseUrl: string, jobId: string, timeoutMs: number): Promise> { - const deadline = Date.now() + timeoutMs; - let latest: Record = {}; - while (Date.now() < deadline) { - const detail = await baiduJson(baseUrl, `/api/transfers/${encodeURIComponent(jobId)}`, {}, 30_000); - latest = detail; - const job = jobRecord(detail.job); - const status = String(job.status || ""); - if (status === "succeeded") return detail; - if (status === "failed" || status === "canceled") throw new Error(`Baidu transfer ${status}: ${String(job.error || "no error detail")}`); - await Bun.sleep(2000); - } - throw new Error(`Baidu transfer timed out after ${timeoutMs}ms: ${JSON.stringify(compactJson(latest))}`); -} - -async function executePgdataBackupAction(action: PgdataBackupScheduleAction, runId: string): Promise<{ ok: boolean; result: JsonValue }> { - const startedAt = new Date(); - const parts = utcBackupParts(startedAt); - const filename = `${parts.stamp}_${safeFilePart(action.volumeName)}.pg_basebackup.tar.gz`; - const monthRelativeDir = pathPosix.join(action.stagingSubdir, parts.month); - const backupRelativePath = pathPosix.join(monthRelativeDir, filename); - const backupPath = normalizeRelativeStagingPath(backupRelativePath).absolutePath; - const tempRelativeDir = pathPosix.join(action.stagingSubdir, parts.month, `.tmp_${runId}`); - const tempDir = normalizeRelativeStagingPath(tempRelativeDir).absolutePath; - const remoteDir = normalizeRemoteDir(pathPosix.join(action.remoteBaseDir, parts.month)); - const remotePath = normalizeRemoteDir(pathPosix.join(remoteDir, filename)); - const db = databaseConnectionParts(); - await mkdir(dirname(backupPath), { recursive: true }); - await rm(tempDir, { recursive: true, force: true }); - await mkdir(tempDir, { recursive: true }); - let uploadDetail: Record | null = null; - let backupBytes = 0; - let backupSha256 = ""; - try { - const backupTimeoutMs = Math.max(60_000, Math.floor(action.timeoutMs * 0.55)); - const packageTimeoutMs = Math.max(60_000, Math.floor(action.timeoutMs * 0.15)); - const uploadTimeoutMs = Math.max(60_000, action.timeoutMs - backupTimeoutMs - packageTimeoutMs); - const basebackup = await runLocalCommand("pg_basebackup", [ - "-h", db.host, - "-p", db.port, - "-U", db.user, - "-D", tempDir, - "-Ft", - "-X", "stream", - "-z", - "--checkpoint=fast", - "--no-sync", - ], { env: { PGPASSWORD: db.password }, timeoutMs: backupTimeoutMs }); - if (!basebackup.ok) throw new Error(`pg_basebackup failed exit=${basebackup.exitCode} timedOut=${basebackup.timedOut}: ${basebackup.stderr || basebackup.stdout}`); - const packaged = await runLocalCommand("tar", ["-czf", backupPath, "-C", tempDir, "."], { timeoutMs: packageTimeoutMs }); - if (!packaged.ok) throw new Error(`tar packaging failed exit=${packaged.exitCode} timedOut=${packaged.timedOut}: ${packaged.stderr || packaged.stdout}`); - const info = await stat(backupPath); - backupBytes = info.size; - backupSha256 = await sha256File(backupPath); - for (const folder of remoteFolderChain(remoteDir)) { - await baiduJson(action.baiduBaseUrl, "/api/folders", { - method: "POST", - body: JSON.stringify({ path: folder }), - }, 60_000); - } - const created = await baiduJson(action.baiduBaseUrl, "/api/transfers/upload-from-path", { - method: "POST", - body: JSON.stringify({ localPath: backupRelativePath, remotePath }), - }, 60_000); - const jobId = String(jobRecord(created.job).id || ""); - if (jobId.length === 0) throw new Error(`Baidu upload did not return a job id: ${JSON.stringify(compactJson(created))}`); - uploadDetail = await waitForBaiduTransfer(action.baiduBaseUrl, jobId, uploadTimeoutMs); - const uploadedJob = jobRecord(uploadDetail.job); - return { - ok: true, - result: { - backupType: "pg_basebackup", - volumeName: action.volumeName, - backupBytes, - backupSha256, - localRelativePath: backupRelativePath, - remotePath, - remoteDir, - month: parts.month, - timestampPrefix: parts.stamp, - baiduTransferJobId: jobId, - baiduTransferStatus: String(uploadedJob.status || ""), - baiduFsId: String(uploadedJob.fsId || ""), - baiduResult: compactJson(uploadedJob.result ?? null), - }, - }; - } finally { - await rm(tempDir, { recursive: true, force: true }).catch(() => undefined); - if (action.cleanupLocal) await rm(backupPath, { force: true }).catch(() => undefined); - } -} - -async function executeScheduleAction(action: ScheduleAction, runId: string): Promise<{ ok: boolean; taskId: string | null; result: JsonValue }> { - if (action.type === "dispatch") { - const dispatched = await executeDispatchScheduleAction(action); - return { ok: dispatched.ok, taskId: dispatched.taskId, result: dispatched.result }; - } - const backup = await executePgdataBackupAction(action, runId); - return { ok: backup.ok, taskId: null, result: backup.result }; -} - -async function executeScheduledRun(runId: string): Promise { - if (activeScheduledRuns.has(runId)) return; - activeScheduledRuns.add(runId); - const started = Date.now(); - try { - const queuedRuns = await sql` - SELECT * - FROM unidesk_scheduled_task_runs - WHERE id = ${runId} - LIMIT 1 - `; - const queuedRun = queuedRuns[0]; - if (queuedRun === undefined) return; - const scheduleRow = await getScheduledTaskRow(queuedRun.schedule_id); - if (scheduleRow === null) return; - const runRows = await sql` - UPDATE unidesk_scheduled_task_runs - SET status = 'running', started_at = now(), updated_at = now() - WHERE id = ${runId} AND status = 'queued' - RETURNING * - `; - if (runRows.length === 0) return; - try { - const action = normalizeScheduleAction(scheduleRow.action_json); - const outcome = await executeScheduleAction(action, runId); - const durationMs = Date.now() - started; - const status = outcome.ok ? "succeeded" : "failed"; - const error = outcome.ok ? null : "scheduled action failed"; - await sql.begin(async (tx) => { - await tx` - UPDATE unidesk_scheduled_task_runs - SET status = ${status}, - task_id = ${outcome.taskId}, - result = ${tx.json(outcome.result)}, - error = ${error}, - finished_at = now(), - duration_ms = ${durationMs}, - updated_at = now() - WHERE id = ${runId} - `; - await tx` - UPDATE unidesk_scheduled_tasks - SET last_run_at = now(), last_run_id = ${runId}, updated_at = now() - WHERE id = ${scheduleRow.id} - `; - }); - await recordEvent(`scheduled_task_${status}`, "scheduler", { scheduleId: scheduleRow.id, runId, durationMs, result: compactJson(outcome.result) }); - } catch (error) { - const durationMs = Date.now() - started; - await sql.begin(async (tx) => { - await tx` - UPDATE unidesk_scheduled_task_runs - SET status = 'failed', - result = ${tx.json(errorToJson(error))}, - error = ${error instanceof Error ? error.message : String(error)}, - finished_at = now(), - duration_ms = ${durationMs}, - updated_at = now() - WHERE id = ${runId} - `; - await tx` - UPDATE unidesk_scheduled_tasks - SET last_run_at = now(), last_run_id = ${runId}, updated_at = now() - WHERE id = ${scheduleRow.id} - `; - }); - await recordEvent("scheduled_task_failed", "scheduler", { scheduleId: scheduleRow.id, runId, durationMs, error: errorToJson(error) }); - } - } finally { - activeScheduledRuns.delete(runId); - } -} - -async function triggerScheduledTask(scheduleId: string, triggerType: "manual" | "schedule"): Promise { - const runId = `schedrun_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; - let createdRun: ScheduledTaskRunRow | null = null; - await sql.begin(async (tx) => { - const rows = await tx` - SELECT * - FROM unidesk_scheduled_tasks - WHERE id = ${scheduleId} - FOR UPDATE - `; - const scheduleRow = rows[0]; - if (scheduleRow === undefined) throw new Error(`scheduled task not found: ${scheduleId}`); - if (triggerType === "schedule" && !scheduleRow.enabled) return; - const runningRows = await tx>` - SELECT count(*)::int AS count - FROM unidesk_scheduled_task_runs - WHERE schedule_id = ${scheduleId} - AND status IN ('queued', 'running') - `; - const runningCount = Number(runningRows[0]?.count ?? 0); - const schedule = normalizeScheduleSpec(scheduleRow.schedule_json); - const shouldAdvanceNext = triggerType === "schedule"; - if (shouldAdvanceNext) { - await tx` - UPDATE unidesk_scheduled_tasks - SET next_run_at = ${computeNextRunAt(schedule).toISOString()}, updated_at = now() - WHERE id = ${scheduleId} - `; - } - if (scheduleRow.concurrency_policy !== "parallel" && runningCount > 0) { - const skippedRows = await tx` - INSERT INTO unidesk_scheduled_task_runs (id, schedule_id, trigger_type, status, result, error, finished_at, duration_ms) - VALUES (${runId}, ${scheduleId}, ${triggerType}, 'skipped', ${tx.json({ reason: "previous run still active", runningCount })}, 'previous run still active', now(), 0) - RETURNING * - `; - createdRun = skippedRows[0] ?? null; - return; - } - const runRows = await tx` - INSERT INTO unidesk_scheduled_task_runs (id, schedule_id, trigger_type, status) - VALUES (${runId}, ${scheduleId}, ${triggerType}, 'queued') - RETURNING * - `; - createdRun = runRows[0] ?? null; - }); - const run = createdRun as ScheduledTaskRunRow | null; - if (run === null) return null; - if (run.status === "queued") { - setTimeout(() => { - executeScheduledRun(runId).catch((error) => logger("error", "scheduled_run_uncaught", { runId, error: errorToJson(error) })); - }, 0); - } - await recordEvent("scheduled_task_triggered", "scheduler", { scheduleId, runId, triggerType, status: run.status }); - return scheduleRunView(run); -} - -async function scheduledTaskRoute(req: Request, url: URL): Promise { - const prefix = "/api/schedules"; - const rest = url.pathname === prefix ? "" : url.pathname.slice(prefix.length + 1); - const segments = rest.split("/").filter(Boolean).map((part) => decodeURIComponent(part)); - if (url.pathname === prefix && req.method === "GET") { - return jsonResponse({ ok: true, schedules: await getScheduledTasks(readLimit(url, 100)) }); - } - if (url.pathname === prefix && req.method === "POST") return upsertScheduledTask(req, null); - if (segments.length === 1 && segments[0] === "runs" && req.method === "GET") { - return jsonResponse({ ok: true, runs: await getScheduledTaskRuns(null, readLimit(url, 100)) }); - } - if (segments.length === 1 && req.method === "GET") { - const schedule = await getScheduledTaskRow(segments[0]); - if (schedule === null) return jsonResponse({ ok: false, error: `scheduled task not found: ${segments[0]}` }, 404); - const runs = await sql` - SELECT * - FROM unidesk_scheduled_task_runs - WHERE schedule_id = ${segments[0]} - ORDER BY updated_at DESC - LIMIT 20 - `; - return jsonResponse({ ok: true, schedule: scheduledTaskView(schedule, runs) }); - } - if (segments.length === 1 && req.method === "PUT") return upsertScheduledTask(req, segments[0]); - if (segments.length === 1 && req.method === "PATCH") return patchScheduledTask(segments[0], req); - if (segments.length === 1 && req.method === "DELETE") return deleteScheduledTask(segments[0]); - if (segments.length === 2 && segments[1] === "run" && req.method === "POST") { - const run = await triggerScheduledTask(segments[0], "manual"); - if (run === null) return jsonResponse({ ok: false, error: `scheduled task was not triggered: ${segments[0]}` }, 409); - return jsonResponse({ ok: true, run }); - } - if (segments.length === 2 && segments[1] === "runs" && req.method === "GET") { - return jsonResponse({ ok: true, runs: await getScheduledTaskRuns(segments[0], readLimit(url, 100)) }); - } - return jsonResponse({ ok: false, error: "scheduled task route not found", path: url.pathname }, 404); -} - -async function recoverScheduledRuns(): Promise { - const rows = await sql` - UPDATE unidesk_scheduled_task_runs - SET status = 'failed', - error = 'backend-core restarted before scheduled run completed', - result = jsonb_build_object('error', 'backend-core restarted before scheduled run completed'), - finished_at = now(), - updated_at = now() - WHERE status IN ('queued', 'running') - RETURNING * - `; - if (rows.length > 0) await recordEvent("scheduled_runs_recovered", "scheduler", { failedRunCount: rows.length }); - const schedules = await sql` - SELECT * - FROM unidesk_scheduled_tasks - WHERE next_run_at IS NULL - LIMIT 200 - `; - for (const schedule of schedules) { - const nextRunAt = computeNextRunAt(normalizeScheduleSpec(schedule.schedule_json)).toISOString(); - await sql`UPDATE unidesk_scheduled_tasks SET next_run_at = ${nextRunAt}, updated_at = now() WHERE id = ${schedule.id}`; - } -} - -async function runDueScheduledTasks(): Promise { - if (!dbReady) return; - const rows = await sql` - SELECT * - FROM unidesk_scheduled_tasks - WHERE enabled = true - AND (next_run_at IS NULL OR next_run_at <= now()) - ORDER BY next_run_at ASC NULLS FIRST - LIMIT 5 - `; - for (const row of rows) { - try { - await triggerScheduledTask(row.id, "schedule"); - } catch (error) { - logger("error", "scheduled_task_due_trigger_failed", { scheduleId: row.id, error: errorToJson(error) }); - } - } -} - -function isMicroservicePathAllowed(service: MicroserviceConfig, path: string): boolean { - return service.backend.allowedPathPrefixes.some((prefix) => path === prefix || path.startsWith(prefix)); -} - -function isMicroserviceMethodAllowed(service: MicroserviceConfig, method: string): boolean { - return service.backend.allowedMethods.includes(method.toUpperCase()); -} - -function readMicroserviceArrayLimits(url: URL): { query: string; jsonArrayLimits: Record } { - const params = new URLSearchParams(url.searchParams); - const jsonArrayLimits: Record = {}; - const rawLimits = params.getAll("__unideskArrayLimit"); - params.delete("__unideskArrayLimit"); - for (const raw of rawLimits) { - for (const entry of raw.split(",")) { - const [path = "", limitText = ""] = entry.split(":"); - if (!/^[A-Za-z0-9_.-]+$/.test(path)) continue; - const limit = Number(limitText); - if (Number.isInteger(limit) && limit > 0 && limit <= 500) jsonArrayLimits[path] = limit; - } - } - const search = params.toString(); - return { query: search.length > 0 ? `?${search}` : "", jsonArrayLimits }; -} - -function truncateText(value: string, maxBytes: number): string { - return value.length <= maxBytes ? value : value.slice(0, maxBytes); -} - -function contentTypeIsJson(contentType: string): boolean { - return contentType.toLowerCase().includes("json"); -} - -function readMicroserviceRequestHeaders(req: Request): Record { - const requestHeaders: Record = {}; - for (const name of microserviceForwardRequestHeaders) { - const value = req.headers.get(name); - if (value !== null && value.length > 0) requestHeaders[name] = value.slice(0, 4096); - } - return requestHeaders; -} - -function headersFromMicroserviceRequest(requestHeaders: Record): Headers { - const headers = new Headers(); - for (const name of microserviceForwardRequestHeaders) { - const value = requestHeaders[name]; - if (typeof value === "string" && value.length > 0) headers.set(name, value); - } - return headers; -} - -function boundedMicroserviceBodyText( - bodyText: string, - contentType: string, - metadata: { serviceId: string; targetPath: string; status: number; upstreamBodyBytes: number }, -): { bodyText: string; truncated: boolean } { - if (bodyText.length <= microserviceProxyMaxBodyTextLength) { - return { bodyText, truncated: false }; - } - if (contentTypeIsJson(contentType)) { - return { - bodyText: JSON.stringify({ - ok: false, - error: "microservice proxy response body is too large", - serviceId: metadata.serviceId, - targetPath: metadata.targetPath, - upstreamStatus: metadata.status, - upstreamBodyBytes: metadata.upstreamBodyBytes, - transformedBodyBytes: bodyText.length, - responseBodyLimitBytes: microserviceProxyMaxBodyTextLength, - hint: "Use a paged endpoint or tighten __unideskArrayLimit so the response stays below the proxy safety limit.", - }), - truncated: true, - }; - } - return { bodyText: truncateText(bodyText, microserviceProxyMaxBodyTextLength), truncated: true }; -} - -function arrayAtPath(value: unknown, path: string): JsonValue[] | null { - let current: unknown = value; - for (const part of path.split(".")) { - if (typeof current !== "object" || current === null || Array.isArray(current)) return null; - current = (current as Record)[part]; - } - return Array.isArray(current) ? current as JsonValue[] : null; -} - -function applyJsonArrayLimits(bodyText: string, contentType: string, limits: Record): string { - const entries = Object.entries(limits); - if (entries.length === 0 || !contentType.toLowerCase().includes("json")) return bodyText; - try { - const parsed = JSON.parse(bodyText) as unknown; - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return bodyText; - const root = parsed as Record; - const applied: Record = {}; - for (const [path, rawLimit] of entries) { - const limit = Number(rawLimit); - if (!Number.isInteger(limit) || limit <= 0 || limit > 500) continue; - const array = arrayAtPath(root, path); - if (array === null) continue; - const originalLength = array.length; - if (array.length > limit) array.splice(limit); - applied[path] = { limit, originalLength, returnedLength: array.length }; - } - root._unidesk = { arrayLimits: applied }; - return JSON.stringify(parsed); - } catch { - return bodyText; - } -} - -function responseFromMicroserviceResult(task: Awaited>): Response { - if (task === null) return jsonResponse({ ok: false, error: "microservice proxy task missing" }, 502); - if (task.status !== "succeeded") return jsonResponse({ ok: false, error: "microservice proxy task failed", task }, 502); - const result = dockerStatusRecord(task.result); - const status = Number(result.status); - const contentType = typeof result.contentType === "string" ? result.contentType : "application/json; charset=utf-8"; - const bodyText = typeof result.bodyText === "string" ? result.bodyText : ""; - if (!Number.isInteger(status) || status < 100 || status > 599) { - return jsonResponse({ ok: false, error: "microservice proxy returned invalid upstream status", task }, 502); - } - if (result.truncated === true && contentTypeIsJson(contentType)) { - try { - JSON.parse(bodyText); - } catch { - return jsonResponse({ - ok: false, - error: "microservice proxy response was truncated before a JSON boundary", - providerId: task.provider_id, - command: task.command, - upstreamStatus: status, - upstreamBodyBytes: result.upstreamBodyBytes ?? null, - returnedBodyBytes: result.returnedBodyBytes ?? bodyText.length, - responseBodyLimitBytes: result.responseBodyLimitBytes ?? null, - hint: "Upgrade the provider-gateway or request a smaller/paged microservice response.", - }, 502); - } - } - return new Response(bodyText, { - status, - headers: { - "content-type": contentType, - "x-unidesk-response-truncated": result.truncated === true ? "true" : "false", - }, - }); -} - -function isPlainRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function nestedRecord(value: unknown, key: string): Record | null { - if (!isPlainRecord(value)) return null; - const child = value[key]; - return isPlainRecord(child) ? child : null; -} - -function nestedBoolean(value: unknown, key: string): boolean | null { - if (!isPlainRecord(value)) return null; - const child = value[key]; - return typeof child === "boolean" ? child : null; -} - -function nestedString(value: unknown, key: string): string | null { - if (!isPlainRecord(value)) return null; - const child = value[key]; - return typeof child === "string" ? child : null; -} - -function jsonBodyFromText(bodyText: string, contentType: string): JsonValue { - if (!contentTypeIsJson(contentType)) return bodyText.length > 0 ? truncateText(bodyText, 4000) : null; - try { - return compactJson(JSON.parse(bodyText) as unknown); - } catch { - return bodyText.length > 0 ? truncateText(bodyText, 4000) : null; - } -} - -function healthFailure(body: JsonValue, checks: Record, reason: string): MicroserviceHealthAssessment { - return { healthy: false, reason, checks, body }; -} - -function healthSuccess(body: JsonValue, checks: Record, reason = "health endpoint returned a usable service state"): MicroserviceHealthAssessment { - return { healthy: true, reason, checks, body }; -} - -function genericMicroserviceHealthAssessment(upstreamStatus: number, body: JsonValue): MicroserviceHealthAssessment { - const checks: Record = { - upstreamStatus, - upstream2xx: upstreamStatus >= 200 && upstreamStatus < 300, - }; - if (upstreamStatus < 200 || upstreamStatus >= 300) { - return healthFailure(body, checks, `health endpoint returned HTTP ${upstreamStatus}`); - } - if (isPlainRecord(body)) { - const ok = body.ok; - const success = body.success; - const statusText = typeof body.status === "string" ? body.status : ""; - checks.okField = typeof ok === "boolean" ? ok : null; - checks.successField = typeof success === "boolean" ? success : null; - checks.statusField = statusText || null; - if (ok === false) return healthFailure(body, checks, "health body ok=false"); - if (success === false) return healthFailure(body, checks, "health body success=false"); - if (["error", "failed", "fail", "unhealthy", "down", "offline", "not_ready", "not-ready"].includes(statusText.toLowerCase())) { - return healthFailure(body, checks, `health body status=${statusText}`); - } - } - return healthSuccess(body, checks); -} - -function assessMetNonlinearHealth(upstreamStatus: number, body: JsonValue): MicroserviceHealthAssessment { - const base = genericMicroserviceHealthAssessment(upstreamStatus, body); - const checks = { ...base.checks }; - if (!base.healthy) return { ...base, checks }; - if (!isPlainRecord(body)) return healthFailure(body, checks, "MET Nonlinear health body is not JSON"); - checks.service = typeof body.service === "string" ? body.service : null; - checks.queueReady = isPlainRecord(body.queue); - const image = nestedRecord(body, "image"); - checks.mlImage = typeof image?.image === "string" ? image.image : null; - checks.mlImagePresent = image?.present === true; - if (body.service !== "met-nonlinear-unidesk-ts") { - return healthFailure(body, checks, "MET Nonlinear health returned an unexpected service identity"); - } - if (!isPlainRecord(body.queue)) return healthFailure(body, checks, "MET Nonlinear queue state is not available"); - if (image?.present !== true) return healthFailure(body, checks, "MET Nonlinear ML image is not ready"); - return healthSuccess(body, checks, "MET Nonlinear backend, queue, and ML image are ready"); -} - -function assessClaudeQqHealth(upstreamStatus: number, body: JsonValue): MicroserviceHealthAssessment { - const base = genericMicroserviceHealthAssessment(upstreamStatus, body); - const checks = { ...base.checks }; - if (!base.healthy) return { ...base, checks }; - if (!isPlainRecord(body)) return healthFailure(body, checks, "ClaudeQQ health body is not JSON"); - const napcat = nestedRecord(body, "napcat"); - const endpoints = Array.isArray(body.endpoints) ? body.endpoints.map((item) => String(item)) : []; - const httpConnected = nestedBoolean(napcat, "httpConnected") === true; - const wsConnected = nestedBoolean(napcat, "wsConnected") === true; - const napcatConnected = nestedBoolean(napcat, "connected") === true; - const loginState = nestedString(napcat, "loginState") ?? ""; - checks.service = typeof body.service === "string" ? body.service : null; - checks.pushEndpoint = endpoints.includes("/api/push/text"); - checks.napcatHttpLoggedIn = httpConnected; - checks.napcatWebsocketConnected = wsConnected; - checks.napcatConnected = napcatConnected; - checks.loginState = loginState || null; - if (body.service !== "claudeqq") return healthFailure(body, checks, "ClaudeQQ health returned an unexpected service identity"); - if (!endpoints.includes("/api/push/text")) return healthFailure(body, checks, "ClaudeQQ push API endpoint is not advertised"); - if (!httpConnected) return healthFailure(body, checks, "NapCat HTTP is not logged in"); - if (!wsConnected) return healthFailure(body, checks, "NapCat OneBot WebSocket is not connected"); - if (!napcatConnected || loginState !== "logged_in") return healthFailure(body, checks, "NapCat is not in a logged-in send/receive ready state"); - return healthSuccess(body, checks, "ClaudeQQ and NapCat are logged in and send/receive ready"); -} - -function assessMicroserviceHealth(service: MicroserviceConfig, upstreamStatus: number, contentType: string, bodyText: string): MicroserviceHealthAssessment { - const body = jsonBodyFromText(bodyText, contentType); - if (service.id === "met-nonlinear") return assessMetNonlinearHealth(upstreamStatus, body); - if (service.id === "claudeqq") return assessClaudeQqHealth(upstreamStatus, body); - return genericMicroserviceHealthAssessment(upstreamStatus, body); -} - -function healthProbeFromAssessment( - service: MicroserviceConfig, - upstreamStatus: number, - contentType: string, - assessment: MicroserviceHealthAssessment, -): Record { - return { - serviceId: service.id, - name: service.name, - providerId: service.providerId, - healthPath: service.backend.healthPath, - healthy: assessment.healthy, - status: assessment.healthy ? "healthy" : "unhealthy", - reason: assessment.reason, - upstreamStatus, - contentType, - checks: assessment.checks, - checkedAt: new Date().toISOString(), - }; -} - -function rememberMicroserviceAvailability(serviceId: string, probe: Record): void { - microserviceAvailabilityCache.set(serviceId, { expiresAt: Date.now() + microserviceAvailabilityTtlMs, probe }); -} - -function cachedMicroserviceAvailability(serviceId: string): Record | null { - const entry = microserviceAvailabilityCache.get(serviceId); - if (entry === undefined) return null; - if (entry.expiresAt <= Date.now()) { - microserviceAvailabilityCache.delete(serviceId); - return null; - } - return entry.probe; -} - -async function strictMicroserviceHealthProbe(service: MicroserviceConfig, timeoutMs?: number): Promise> { - const probeService = timeoutMs === undefined - ? service - : { ...service, backend: { ...service.backend, timeoutMs } }; - const response = await fetchMicroserviceUpstreamResponse(probeService, "GET", service.backend.healthPath, { query: "", jsonArrayLimits: {} }, { accept: "application/json" }, ""); - const upstreamStatus = response.status; - const contentType = response.headers.get("content-type") ?? "application/octet-stream"; - const bodyText = await response.text(); - const assessment = assessMicroserviceHealth(service, upstreamStatus, contentType, bodyText); - const probe = healthProbeFromAssessment(service, upstreamStatus, contentType, assessment); - rememberMicroserviceAvailability(service.id, probe); - return { ...probe, body: assessment.body }; -} - -async function strictMicroserviceHealthResponse(service: MicroserviceConfig, headOnly: boolean): Promise { - const response = await fetchMicroserviceUpstreamResponse(service, "GET", service.backend.healthPath, { query: "", jsonArrayLimits: {} }, { accept: "application/json" }, ""); - const upstreamStatus = response.status; - const contentType = response.headers.get("content-type") ?? "application/octet-stream"; - const bodyText = await response.text(); - const assessment = assessMicroserviceHealth(service, upstreamStatus, contentType, bodyText); - const probe = healthProbeFromAssessment(service, upstreamStatus, contentType, assessment); - rememberMicroserviceAvailability(service.id, probe); - if (assessment.healthy) { - return new Response(headOnly ? null : bodyText, { - status: upstreamStatus, - headers: { - "content-type": contentType, - "x-unidesk-health": "healthy", - "x-unidesk-health-reason": assessment.reason, - }, - }); - } - const status = upstreamStatus >= 500 ? upstreamStatus : 503; - return jsonResponse({ - ok: false, - status: "unhealthy", - serviceId: service.id, - name: service.name, - providerId: service.providerId, - reason: assessment.reason, - checkedAt: probe.checkedAt, - checks: assessment.checks, - upstream: { - status: upstreamStatus, - contentType, - body: assessment.body, - }, - }, status); -} - -async function microserviceAvailability(service: MicroserviceConfig): Promise> { - const cached = cachedMicroserviceAvailability(service.id); - if (cached !== null) return cached; - const existing = microserviceAvailabilityRefreshes.get(service.id); - if (existing !== undefined) return existing; - const timeoutMs = Math.max(2500, Math.min(service.backend.timeoutMs, 10_000)); - const refresh = strictMicroserviceHealthProbe(service, timeoutMs) - .catch((error) => { - const probe = { - serviceId: service.id, - name: service.name, - providerId: service.providerId, - healthPath: service.backend.healthPath, - healthy: false, - status: "unhealthy", - reason: error instanceof Error ? error.message : String(error), - upstreamStatus: null, - contentType: null, - checks: {}, - checkedAt: new Date().toISOString(), - } satisfies Record; - rememberMicroserviceAvailability(service.id, probe); - return probe; - }) - .finally(() => { - microserviceAvailabilityRefreshes.delete(service.id); - }); - microserviceAvailabilityRefreshes.set(service.id, refresh); - return refresh; -} - -async function getMicroserviceAvailabilitySummary(): Promise> { - const probes = await Promise.all(config.microservices.map((service) => microserviceAvailability(service))); - const healthy = probes.filter((probe) => probe.healthy === true); - const byProvider: Record = {}; - for (const probe of probes) { - const providerId = String(probe.providerId ?? "unknown"); - const current = isPlainRecord(byProvider[providerId]) ? byProvider[providerId] as Record : { total: 0, healthy: 0, unhealthy: 0 }; - current.total = Number(current.total ?? 0) + 1; - if (probe.healthy === true) current.healthy = Number(current.healthy ?? 0) + 1; - else current.unhealthy = Number(current.unhealthy ?? 0) + 1; - byProvider[providerId] = current; - } - return { - totalCount: probes.length, - healthyCount: healthy.length, - unhealthyCount: probes.length - healthy.length, - checkedAt: new Date().toISOString(), - byProvider, - services: probes.map((probe) => ({ - serviceId: probe.serviceId, - name: probe.name, - providerId: probe.providerId, - healthy: probe.healthy, - status: probe.status, - reason: probe.reason, - checkedAt: probe.checkedAt, - })), - }; -} - -function microserviceCacheTtlMs(serviceId: string, targetPath: string): number { - if (targetPath === "/health") return 0; - if (targetPath.endsWith("/stream")) return 0; - if (serviceId === "pipeline" && targetPath === "/api/snapshot") return 6_000; - if (serviceId === "pipeline" && targetPath.startsWith("/api/oa-event-flow/")) return 20_000; - if (serviceId === "pipeline" && targetPath.startsWith("/api/model-quota/")) return 60_000; - if (serviceId === "pipeline" && targetPath.startsWith("/api/node-control/runs/")) return 6_000; - if (serviceId === "pipeline" && targetPath.startsWith("/api/runs/")) return 6_000; - if (serviceId === "findjob" && (targetPath === "/api/summary" || targetPath === "/api/jobs" || targetPath === "/api/drafts")) return 8_000; - if (serviceId === "met-nonlinear" && (targetPath === "/api/images" || targetPath === "/api/projects")) return 15_000; - if (serviceId === "met-nonlinear" && (targetPath === "/api/queue" || targetPath === "/api/summary" || targetPath === "/api/history")) return 5_000; - if (serviceId === "code-queue" && targetPath.includes("/transcript")) return 1_000; - return 750; -} - -function microserviceCacheStaleMs(serviceId: string, targetPath: string): number { - if (serviceId === "pipeline" && targetPath.startsWith("/api/model-quota/")) return 5 * 60_000; - if (serviceId === "pipeline") return 45_000; - if (serviceId === "findjob") return 60_000; - if (serviceId === "met-nonlinear" && (targetPath === "/api/images" || targetPath === "/api/projects")) return 5 * 60_000; - if (serviceId === "met-nonlinear") return 45_000; - return 5_000; -} - -function providerMicroserviceCacheTtlMs(serviceId: string, targetPath: string): number { - if (targetPath === "/health") return 0; - if (serviceId === "met-nonlinear" && (targetPath === "/api/images" || targetPath === "/api/projects")) return 60_000; - if (serviceId === "met-nonlinear" && targetPath === "/api/history") return 10_000; - if (serviceId === "met-nonlinear" && (targetPath === "/api/queue" || targetPath === "/api/summary")) return 3_000; - if (serviceId === "pipeline" && (targetPath === "/api/snapshot" || targetPath.startsWith("/api/oa-event-flow/"))) return 2_000; - if (serviceId === "findjob" && (targetPath === "/api/summary" || targetPath === "/api/jobs")) return 2_000; - return 1_000; -} - -function microserviceCacheKey(service: MicroserviceConfig, method: string, targetPath: string, proxyOptions: { query: string; jsonArrayLimits: Record }): string { - return JSON.stringify([service.id, method, targetPath, proxyOptions.query, proxyOptions.jsonArrayLimits]); -} - -function responseFromMicroserviceCache(entry: MicroserviceProxyCacheEntry, state: "hit" | "stale"): Response { - return new Response(entry.bodyText, { - status: entry.status, - headers: { - "content-type": entry.contentType, - "x-unidesk-cache": state, - }, - }); -} - -function readMicroserviceCache(key: string): Response | null { - const entry = microserviceProxyCache.get(key); - if (entry === undefined) return null; - if (entry.staleExpiresAt <= Date.now()) { - microserviceProxyCache.delete(key); - return null; - } - if (entry.expiresAt <= Date.now()) return null; - return responseFromMicroserviceCache(entry, "hit"); -} - -function readStaleMicroserviceCache(key: string): Response | null { - const entry = microserviceProxyCache.get(key); - if (entry === undefined) return null; - if (entry.staleExpiresAt <= Date.now()) { - microserviceProxyCache.delete(key); - return null; - } - if (entry.expiresAt > Date.now()) return null; - return responseFromMicroserviceCache(entry, "stale"); -} - -async function cacheableResponseSnapshot(response: Response): Promise { - if (response.status < 200 || response.status >= 300) return null; - if (response.headers.get("x-unidesk-response-truncated") === "true") return null; - const bodyText = await response.clone().text(); - if (bodyText.length > 2 * 1024 * 1024) return null; - return { - expiresAt: 0, - staleExpiresAt: 0, - status: response.status, - contentType: response.headers.get("content-type") ?? "application/octet-stream", - bodyText, - }; -} - -function rememberMicroserviceCache(key: string, ttlMs: number, entry: MicroserviceProxyCacheEntry | null): void { - if (entry === null || ttlMs <= 0) return; - const keyParts = JSON.parse(key) as string[]; - entry.expiresAt = Date.now() + ttlMs; - entry.staleExpiresAt = entry.expiresAt + microserviceCacheStaleMs(String(keyParts[0] ?? ""), String(keyParts[2] ?? "")); - microserviceProxyCache.set(key, entry); - if (microserviceProxyCache.size > 300) { - const now = Date.now(); - for (const [cacheKey, cacheEntry] of microserviceProxyCache) { - if (cacheEntry.expiresAt <= now || microserviceProxyCache.size > 240) microserviceProxyCache.delete(cacheKey); - } - } -} - -function invalidateMicroserviceCache(serviceId: string): void { - const prefix = `["${serviceId}",`; - for (const key of microserviceProxyCache.keys()) { - if (key.startsWith(prefix)) microserviceProxyCache.delete(key); - } -} - -function canDirectProxyMicroservice(service: MicroserviceConfig): boolean { - return service.providerId === "main-server"; -} - -function isV3sctlManagedMicroservice(service: MicroserviceConfig): boolean { - return service.deployment.mode === "v3sctl-managed" || service.backend.proxyMode === "v3sctl-adapter-http"; -} - -async function directMicroserviceResponse( - service: MicroserviceConfig, - method: string, - targetPath: string, - proxyOptions: { query: string; jsonArrayLimits: Record }, - requestHeaders: Record, - bodyText: string, - abortSignal?: AbortSignal, -): Promise { - const baseUrl = new URL(service.backend.nodeBaseUrl); - const upstreamUrl = new URL(targetPath, baseUrl); - upstreamUrl.search = proxyOptions.query; - const headers = headersFromMicroserviceRequest(requestHeaders); - const streamExpected = method === "GET" && (targetPath.endsWith("/stream") || String(headers.get("accept") || "").toLowerCase().includes("text/event-stream")); - const controller = new AbortController(); - const timer = streamExpected ? null : setTimeout(() => controller.abort(), Math.max(1000, service.backend.timeoutMs)); - try { - const upstream = await fetch(upstreamUrl, { - method, - headers, - body: method === "GET" || method === "HEAD" ? undefined : bodyText, - signal: streamExpected ? abortSignal : controller.signal, - }); - const upstreamContentType = upstream.headers.get("content-type") ?? "text/plain; charset=utf-8"; - if (upstreamContentType.toLowerCase().includes("text/event-stream")) { - const responseHeaders: Record = { - "content-type": upstreamContentType, - "cache-control": upstream.headers.get("cache-control") || "no-store, no-transform", - "connection": "keep-alive", - "x-unidesk-proxy-mode": "direct", - "x-unidesk-response-truncated": "false", - }; - const buffering = upstream.headers.get("x-accel-buffering"); - if (buffering !== null) responseHeaders["x-accel-buffering"] = buffering; - return new Response(upstream.body, { status: upstream.status, headers: responseHeaders }); - } - const rawBodyText = await upstream.text(); - const limitedBodyText = applyJsonArrayLimits(rawBodyText, upstreamContentType, proxyOptions.jsonArrayLimits); - const bounded = boundedMicroserviceBodyText(limitedBodyText, upstreamContentType, { - serviceId: service.id, - targetPath, - status: upstream.status, - upstreamBodyBytes: rawBodyText.length, - }); - return new Response(bounded.bodyText, { - status: upstream.status, - headers: { - "content-type": upstreamContentType, - "x-unidesk-proxy-mode": "direct", - "x-unidesk-response-truncated": bounded.truncated ? "true" : "false", - }, - }); - } catch (error) { - return jsonResponse({ ok: false, error: "direct microservice proxy failed", serviceId: service.id, detail: errorToJson(error) }, 502); - } finally { - if (timer !== null) clearTimeout(timer); - } -} - -async function v3sctlAdapterMicroserviceResponse( - service: MicroserviceConfig, - method: string, - targetPath: string, - proxyOptions: { query: string; jsonArrayLimits: Record }, - requestHeaders: Record, - bodyText: string, - abortSignal?: AbortSignal, -): Promise { - const adapterServiceId = service.deployment.adapterServiceId ?? "v3sctl-adapter"; - const adapter = microserviceById(adapterServiceId); - if (adapter === null) { - return jsonResponse({ ok: false, error: `v3sctl adapter microservice not found: ${adapterServiceId}`, serviceId: service.id }, 502); - } - if (adapter.id === service.id || isV3sctlManagedMicroservice(adapter)) { - return jsonResponse({ ok: false, error: "v3sctl adapter must be a UniDesk-direct microservice", serviceId: service.id, adapterServiceId }, 500); - } - const v3sServiceId = service.deployment.v3sServiceId ?? service.id; - const adapterTargetPath = `/api/services/${encodeURIComponent(v3sServiceId)}/proxy${targetPath}`; - return fetchMicroserviceUpstreamResponse(adapter, method, adapterTargetPath, proxyOptions, requestHeaders, bodyText, abortSignal); -} - -async function fetchMicroserviceUpstreamResponse( - service: MicroserviceConfig, - method: string, - targetPath: string, - proxyOptions: { query: string; jsonArrayLimits: Record }, - requestHeaders: Record, - bodyText: string, - abortSignal?: AbortSignal, -): Promise { - if (isV3sctlManagedMicroservice(service)) { - return v3sctlAdapterMicroserviceResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, abortSignal); - } - if (canDirectProxyMicroservice(service)) { - return directMicroserviceResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, abortSignal); - } - if (!(await providerSupports(service.providerId, "microservice.http"))) { - return jsonResponse({ ok: false, error: `provider does not declare microservice.http capability: ${service.providerId}` }, 409); - } - const { taskId, providerOnline } = await createAndSendTask(service.providerId, "microservice.http", { - source: "microservice-frontend-proxy", - serviceId: service.id, - method, - targetBaseUrl: service.backend.nodeBaseUrl, - path: targetPath, - query: proxyOptions.query, - requestHeaders, - bodyText, - jsonArrayLimits: proxyOptions.jsonArrayLimits, - timeoutMs: service.backend.timeoutMs, - cacheTtlMs: providerMicroserviceCacheTtlMs(service.id, targetPath), - }); - if (!providerOnline) return jsonResponse({ ok: false, error: `provider is offline: ${service.providerId}`, taskId }, 503); - const task = await waitForTaskTerminal(taskId, service.backend.timeoutMs + 3000); - return responseFromMicroserviceResult(task); -} - -function refreshMicroserviceCacheInBackground( - cacheKey: string, - ttlMs: number, - fetchResponse: () => Promise, -): void { - if (microserviceProxyRefreshes.has(cacheKey)) return; - const refresh = fetchResponse() - .then((response) => cacheableResponseSnapshot(response)) - .then((entry) => rememberMicroserviceCache(cacheKey, ttlMs, entry)) - .catch((error) => { - logger("warn", "microservice_cache_revalidate_failed", { cacheKey, error: errorToJson(error) }); - }) - .finally(() => { - microserviceProxyRefreshes.delete(cacheKey); - }); - microserviceProxyRefreshes.set(cacheKey, refresh); -} - -async function microserviceRoute(req: Request, url: URL): Promise { - const rest = url.pathname.slice("/api/microservices/".length); - const slashIndex = rest.indexOf("/"); - const serviceId = decodeURIComponent(slashIndex === -1 ? rest : rest.slice(0, slashIndex)); - const suffix = slashIndex === -1 ? "" : rest.slice(slashIndex + 1); - const service = microserviceById(serviceId); - if (service === null) return jsonResponse({ ok: false, error: `microservice not found: ${serviceId}` }, 404); - const method = req.method.toUpperCase(); - if ((suffix === "" || suffix === "status") && (method === "GET" || method === "HEAD")) { - return jsonResponse({ ok: true, microservice: (await getMicroservices()).find((item) => recordValue(item, "id") === serviceId) ?? service }); - } - - const proxyPrefix = "proxy"; - const targetPath = suffix === "health" - ? service.backend.healthPath - : suffix === proxyPrefix - ? "/" - : suffix.startsWith(`${proxyPrefix}/`) - ? `/${suffix.slice(proxyPrefix.length + 1)}` - : ""; - if (targetPath.length === 0) return jsonResponse({ ok: false, error: "microservice route must be /status, /health, or /proxy/" }, 404); - if (suffix === "health" && method !== "GET" && method !== "HEAD") { - return jsonResponse({ ok: false, error: "microservice health only supports GET/HEAD" }, 405); - } - if (!isMicroserviceMethodAllowed(service, method)) { - return jsonResponse({ ok: false, error: "microservice method is not allowed", serviceId, method, allowedMethods: service.backend.allowedMethods }, 405); - } - if (!isMicroservicePathAllowed(service, targetPath)) { - return jsonResponse({ ok: false, error: "microservice path is not allowed", serviceId, targetPath }, 403); - } - if (suffix === "health") return strictMicroserviceHealthResponse(service, method === "HEAD"); - const proxyOptions = readMicroserviceArrayLimits(url); - const cacheKey = microserviceCacheKey(service, method, targetPath, proxyOptions); - const cacheTtlMs = microserviceCacheTtlMs(service.id, targetPath); - if (method === "GET" || method === "HEAD") { - const cached = readMicroserviceCache(cacheKey); - if (cached !== null) return cached; - } else { - invalidateMicroserviceCache(service.id); - } - const bodyText = method === "GET" || method === "HEAD" ? "" : await req.text(); - if (bodyText.length > 1024 * 1024) { - return jsonResponse({ ok: false, error: "microservice request body is too large", maxBytes: 1024 * 1024 }, 413); - } - const requestHeaders = readMicroserviceRequestHeaders(req); - if (method === "GET" || method === "HEAD") { - const stale = readStaleMicroserviceCache(cacheKey); - if (stale !== null) { - refreshMicroserviceCacheInBackground(cacheKey, cacheTtlMs, () => fetchMicroserviceUpstreamResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText)); - return stale; - } - } - const response = await fetchMicroserviceUpstreamResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, req.signal); - if ((method === "GET" || method === "HEAD") && cacheTtlMs > 0) rememberMicroserviceCache(cacheKey, cacheTtlMs, await cacheableResponseSnapshot(response)); - return response; -} - -async function dispatchTask(req: Request): Promise { - const body = (await req.json()) as { providerId?: unknown; command?: unknown; payload?: unknown }; - const providerId = typeof body.providerId === "string" ? body.providerId : ""; - const command = isProviderDispatchCommand(body.command) ? body.command : null; - const payload = typeof body.payload === "object" && body.payload !== null ? (body.payload as Record) : {}; - if (!providerId) { - return jsonResponse({ ok: false, error: "providerId is required" }, 400); - } - if (command === null) { - return jsonResponse({ ok: false, error: "command must be one of docker.ps, provider.upgrade, host.ssh, microservice.http, echo" }, 400); - } - if (command === "host.ssh" && !(await providerSupports(providerId, "host.ssh"))) { - return jsonResponse({ ok: false, error: `provider does not declare host.ssh capability: ${providerId}` }, 409); - } - if (command === "microservice.http" && !(await providerSupports(providerId, "microservice.http"))) { - return jsonResponse({ ok: false, error: `provider does not declare microservice.http capability: ${providerId}` }, 409); - } - const { taskId, providerOnline } = await createAndSendTask(providerId, command, payload); - return jsonResponse({ ok: true, taskId, status: providerOnline ? "dispatched" : "queued", providerOnline }); -} - -function shellQuote(value: string): string { - return `'${value.replace(/'/g, `'\\''`)}'`; -} - -function safePerfRunId(): string { - return `codex_queue_perf_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; -} - -function rawTaskJson(task: RawTaskRow | null): JsonValue { - if (task === null) return null; - return { - id: task.id, - providerId: task.provider_id, - command: task.command, - status: task.status, - payload: task.payload, - result: task.result, - updatedAt: task.updated_at instanceof Date ? task.updated_at.toISOString() : String(task.updated_at), - }; -} - -function recordFromJson(value: JsonValue | null): Record { - return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; -} - -async function runHostSshPerfCommand(providerId: string, command: string, timeoutMs = 18_000): Promise<{ ok: boolean; taskId: string; task: RawTaskRow | null; stdout: string; stderr: string; exitCode: number | null; timedOut: boolean }> { - const { taskId, providerOnline } = await createAndSendTask(providerId, "host.ssh", { - source: "code-queue-performance-panel", - mode: "exec", - cwd: "/root/unidesk", - timeoutMs: 15_000, - command, - }); - if (!providerOnline) { - return { ok: false, taskId, task: null, stdout: "", stderr: `provider is offline: ${providerId}`, exitCode: null, timedOut: false }; - } - const task = await waitForTaskTerminal(taskId, timeoutMs); - const result = recordFromJson(task?.result ?? null); - return { - ok: task?.status === "succeeded" && result.ok === true, - taskId, - task, - stdout: typeof result.stdout === "string" ? result.stdout : "", - stderr: typeof result.stderr === "string" ? result.stderr : "", - exitCode: typeof result.exitCode === "number" ? result.exitCode : null, - timedOut: result.timedOut === true, - }; -} - -function parsePerfPoll(stdout: string): { ready: boolean; exitCode: number | null; body: string; stderr: string } { - if (!stdout.includes("__UNIDESK_CODEX_QUEUE_PERF_READY__")) { - return { ready: false, exitCode: null, body: "", stderr: "" }; - } - const exitMatch = stdout.match(/__UNIDESK_CODEX_QUEUE_PERF_READY__\r?\n([0-9]+)/u); - const bodyMatch = stdout.match(/__UNIDESK_CODEX_QUEUE_PERF_STDOUT__\r?\n([\s\S]*?)\r?\n__UNIDESK_CODEX_QUEUE_PERF_STDERR__/u); - const stderrMatch = stdout.match(/__UNIDESK_CODEX_QUEUE_PERF_STDERR__\r?\n([\s\S]*)$/u); - const exitCode = exitMatch === null ? null : Number(exitMatch[1]); - return { - ready: true, - exitCode: Number.isFinite(exitCode) ? exitCode : null, - body: bodyMatch?.[1]?.trim() ?? "", - stderr: stderrMatch?.[1]?.trim() ?? "", - }; -} - -function parsePerfJson(body: string): Record | null { - for (const line of body.split(/\r?\n/u).reverse()) { - const trimmed = line.trim(); - if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) continue; - try { - const parsed = JSON.parse(trimmed) as unknown; - if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) return parsed as Record; - } catch { - // Continue scanning for the compact JSON line emitted by the Playwright script. - } - } - return null; -} - -async function codexQueueLoadTest(req: Request): Promise { - const body = req.method === "POST" ? (await req.json().catch(() => ({}))) as Record : {}; - const codexService = microserviceById("code-queue"); - const providerId = typeof body.providerId === "string" && body.providerId.length > 0 - ? body.providerId - : codexService?.providerId ?? "main-server"; - if (!(await providerSupports(providerId, "host.ssh"))) { - return jsonResponse({ ok: true, measurementOk: false, error: `provider does not declare host.ssh capability: ${providerId}`, providerId }, 200); - } - - const timeoutMs = numberFromUnknown(body.timeoutMs, 90_000, 5_000, 180_000); - const targetMs = numberFromUnknown(body.targetMs, 1_000, 100, 60_000); - const runId = safePerfRunId(); - const dir = ".state/code-queue-perf"; - const browsersPath = ".state/playwright-browsers"; - const outputPath = `${dir}/${runId}.json`; - const stderrPath = `${dir}/${runId}.stderr`; - const exitPath = `${dir}/${runId}.exit`; - const urlArg = typeof body.url === "string" && body.url.length > 0 ? ` --url ${shellQuote(body.url)}` : ""; - const chromePath = `${browsersPath}/chromium-1217/chrome-linux64/chrome`; - const playwrightSetup = [ - `if ! test -x ${shellQuote(chromePath)}; then PLAYWRIGHT_BROWSERS_PATH=${shellQuote(browsersPath)} npx playwright install chromium; fi`, - `if test -x ${shellQuote(chromePath)} && ldd ${shellQuote(chromePath)} 2>&1 | grep -q 'not found'; then npx playwright install-deps chromium; fi`, - ].join(" && "); - const startCommand = [ - `mkdir -p ${shellQuote(dir)}`, - `rm -f ${shellQuote(outputPath)} ${shellQuote(stderrPath)} ${shellQuote(exitPath)}`, - `(${playwrightSetup}; PLAYWRIGHT_BROWSERS_PATH=${shellQuote(browsersPath)} bun scripts/src/code-queue-perf.ts --json --timeout-ms ${timeoutMs} --target-ms ${targetMs}${urlArg} > ${shellQuote(outputPath)}; printf '%s' "$?" > ${shellQuote(exitPath)}) > ${shellQuote(stderrPath)} 2>&1 & printf '%s\\n' ${shellQuote(runId)}`, - ].join("; "); - const startedAt = Date.now(); - const start = await runHostSshPerfCommand(providerId, startCommand); - if (!start.ok) { - return jsonResponse({ ok: true, measurementOk: false, providerId, runId, stage: "start", error: start.stderr || "failed to start Playwright benchmark", taskId: start.taskId, task: rawTaskJson(start.task) }, 200); - } - - const pollCommand = [ - `if test -f ${shellQuote(exitPath)}; then`, - "printf '__UNIDESK_CODEX_QUEUE_PERF_READY__\\n';", - `cat ${shellQuote(exitPath)};`, - "printf '\\n__UNIDESK_CODEX_QUEUE_PERF_STDOUT__\\n';", - `cat ${shellQuote(outputPath)} 2>/dev/null || true;`, - "printf '\\n__UNIDESK_CODEX_QUEUE_PERF_STDERR__\\n';", - `tail -c 3000 ${shellQuote(stderrPath)} 2>/dev/null || true;`, - "else printf '__UNIDESK_CODEX_QUEUE_PERF_PENDING__\\n'; fi", - ].join(" "); - - let latestPoll: Awaited> | null = null; - while (Date.now() - startedAt < timeoutMs + 20_000) { - await Bun.sleep(1_500); - latestPoll = await runHostSshPerfCommand(providerId, pollCommand); - const parsedPoll = parsePerfPoll(latestPoll.stdout); - if (!parsedPoll.ready) continue; - const result = parsePerfJson(parsedPoll.body); - if (result === null) { - return jsonResponse({ - ok: true, - measurementOk: false, - providerId, - runId, - stage: "parse", - exitCode: parsedPoll.exitCode, - error: "Playwright benchmark did not emit JSON", - stderr: parsedPoll.stderr, - taskId: latestPoll.taskId, - }, 200); - } - return jsonResponse({ - ok: true, - measurementOk: result.ok === true, - providerId, - runId, - taskId: latestPoll.taskId, - elapsedMs: Date.now() - startedAt, - exitCode: parsedPoll.exitCode, - result, - stderr: parsedPoll.stderr, - }); - } - - return jsonResponse({ - ok: true, - measurementOk: false, - providerId, - runId, - stage: "timeout", - elapsedMs: Date.now() - startedAt, - error: `Code Queue Playwright benchmark did not finish within ${timeoutMs}ms`, - latestTaskId: latestPoll?.taskId ?? start.taskId, - latestTask: rawTaskJson(latestPoll?.task ?? start.task), - }, 200); -} - -function numberFromUnknown(value: unknown, fallback: number, min: number, max: number): number { - const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : fallback; - if (!Number.isFinite(parsed)) return fallback; - return Math.max(min, Math.min(max, Math.floor(parsed))); -} - -async function handleSshClientMessage(ws: ProviderSocket, raw: string | Buffer): Promise { - const text = typeof raw === "string" ? raw : raw.toString("utf8"); - const message = JSON.parse(text) as { type?: unknown; providerId?: unknown; cwd?: unknown; command?: unknown; tty?: unknown; data?: unknown; encoding?: unknown; cols?: unknown; rows?: unknown }; - if (message.type === "ssh.open") { - const providerId = typeof message.providerId === "string" ? message.providerId : ""; - if (providerId.length === 0) { - wsSendJson(ws, { type: "ssh.error", message: "providerId is required" }); - return; - } - const provider = providerForSsh(providerId); - if (provider === null) { - wsSendJson(ws, { type: "ssh.error", message: `provider is not online: ${providerId}` }); - return; - } - if (!(await providerSupports(providerId, "host.ssh"))) { - wsSendJson(ws, { type: "ssh.error", message: `provider does not declare host.ssh capability: ${providerId}` }); - return; - } - const sessionId = safeSessionId(); - ws.data.channel = "ssh"; - ws.data.providerId = providerId; - ws.data.sessionId = sessionId; - activeSshClients.set(sessionId, ws); - const openMessage: CoreHostSshOpenMessage = { - type: "host_ssh_open", - sessionId, - cols: numberFromUnknown(message.cols, 100, 20, 300), - rows: numberFromUnknown(message.rows, 30, 8, 120), - }; - if (typeof message.cwd === "string" && message.cwd.length > 0) openMessage.cwd = message.cwd; - if (typeof message.command === "string" && message.command.length > 0) openMessage.command = message.command; - if (typeof message.tty === "boolean") openMessage.tty = message.tty; - provider.send(JSON.stringify(openMessage)); - wsSendJson(ws, { type: "ssh.dispatched", providerId, sessionId }); - logger("info", "ssh_session_dispatched", { providerId, sessionId, hasCommand: typeof openMessage.command === "string" }); - return; - } - - const sessionId = ws.data.sessionId; - const providerId = ws.data.providerId; - if (sessionId === undefined || providerId === undefined) { - if (message.type === "ssh.eof" || message.type === "ssh.close") return; - wsSendJson(ws, { type: "ssh.error", message: "ssh session is not open" }); - return; - } - const provider = providerForSsh(providerId); - if (provider === null) { - wsSendJson(ws, { type: "ssh.error", message: `provider went offline: ${providerId}` }); - return; - } - - if (message.type === "ssh.input") { - if (typeof message.data !== "string" || message.encoding !== "base64") { - wsSendJson(ws, { type: "ssh.error", message: "ssh.input requires base64 data" }); - return; - } - const inputMessage: CoreHostSshInputMessage = { type: "host_ssh_input", sessionId, data: message.data, encoding: "base64" }; - provider.send(JSON.stringify(inputMessage)); - return; - } - - if (message.type === "ssh.resize") { - const resizeMessage: CoreHostSshResizeMessage = { - type: "host_ssh_resize", - sessionId, - cols: numberFromUnknown(message.cols, 100, 20, 300), - rows: numberFromUnknown(message.rows, 30, 8, 120), - }; - provider.send(JSON.stringify(resizeMessage)); - return; - } - - if (message.type === "ssh.eof") { - const eofMessage: CoreHostSshEofMessage = { type: "host_ssh_eof", sessionId }; - provider.send(JSON.stringify(eofMessage)); - return; - } - - if (message.type === "ssh.close") { - const closeMessage: CoreHostSshCloseMessage = { type: "host_ssh_close", sessionId }; - provider.send(JSON.stringify(closeMessage)); - closeSshClient(sessionId); - return; - } - - wsSendJson(ws, { type: "ssh.error", message: `unsupported ssh client message: ${String(message.type)}` }); -} - -async function sshRoute(req: Request, server: Server): Promise { - const url = new URL(req.url); - const token = url.searchParams.get("token") ?? req.headers.get("x-provider-token"); - if (token !== config.providerToken) return jsonResponse({ ok: false, error: "invalid ssh bridge token" }, 401); - const upgraded = server.upgrade(req, { data: { channel: "ssh" } satisfies WsData }); - return upgraded ? undefined : jsonResponse({ ok: false, error: "websocket upgrade failed" }, 400); -} +})(); async function routeInner(req: Request, server: Server): Promise { const url = new URL(req.url); @@ -3531,9 +48,9 @@ async function routeInner(req: Request, server: Server): Promise getOverview())); @@ -3561,7 +78,7 @@ async function routeInner(req: Request, server: Server): Promise codexQueueLoadTest(req)); if (url.pathname.startsWith("/api/microservices/")) return withPerformanceOperation("microservices", "route", url.pathname, () => microserviceRoute(req, url)); if (url.pathname === "/api/dispatch" && req.method === "POST") return withPerformanceOperation("scheduler", "dispatch", url.pathname, () => dispatchTask(req)); - if (url.pathname === "/logs") return jsonResponse({ ok: true, logs: recentLogs.slice(-readLimit(url, 100)) }); + if (url.pathname === "/logs") return jsonResponse({ ok: true, logs: ctx.recentLogs.slice(-readLimit(url, 100)) }); if (url.pathname === "/favicon.ico") return textResponse("", 204); return jsonResponse({ ok: false, error: "not found", path: url.pathname }, 404); } catch (error) { @@ -3584,80 +101,47 @@ async function route(req: Request, server: Server): Promise): Promise { const url = new URL(req.url); - if (url.pathname === "/" || url.pathname === "/health") { - return jsonResponse({ ok: true, service: "unidesk-provider-ingress", activeSocketCount: activeProviders.size }); + if (url.pathname === "/ws/provider") { + const token = url.searchParams.get("token") ?? req.headers.get("x-provider-token"); + if (token !== config().providerToken) return jsonResponse({ ok: false, error: "invalid provider token" }, 401); + const upgraded = server.upgrade(req, { data: { channel: "provider" } satisfies WsData }); + return upgraded ? undefined : jsonResponse({ ok: false, error: "websocket upgrade failed" }, 400); } - if (url.pathname !== "/ws/provider") { - return jsonResponse({ ok: false, error: "provider ingress only accepts /ws/provider", path: url.pathname }, 404); - } - const token = url.searchParams.get("token") ?? req.headers.get("x-provider-token"); - if (token !== config.providerToken) { - await recordEvent("provider_auth_failed", "unknown", { remote: req.headers.get("x-forwarded-for") ?? null }); - return jsonResponse({ ok: false, error: "invalid provider token" }, 401); - } - const upgraded = server.upgrade(req, { data: { channel: "provider" } satisfies WsData }); - return upgraded ? undefined : jsonResponse({ ok: false, error: "websocket upgrade failed" }, 400); + return jsonResponse({ ok: false, error: "not found" }, 404); } -function readLimit(url: URL, defaultLimit: number): number { - const raw = url.searchParams.get("limit"); - if (raw === null) return defaultLimit; - const parsed = Number(raw); - if (!Number.isInteger(parsed) || parsed <= 0) return defaultLimit; - return Math.min(parsed, 500); -} - -await initDatabaseWithRetry(); -markStaleTasksFailed().catch((error) => logger("error", "task_timeout_sweep_failed", { error: errorToJson(error) })); -await recoverScheduledRuns(); -runDueScheduledTasks().catch((error) => logger("error", "scheduled_task_sweep_failed", { error: errorToJson(error) })); - const apiServer = Bun.serve({ - port: config.port, - hostname: "0.0.0.0", - idleTimeout: 120, - fetch: route, + port: config().port, + fetch(req, server) { + return route(req, server); + }, websocket: { open(ws) { - if (ws.data.channel === "ssh") logger("info", "ssh_client_open"); + if (ws.data.channel === "ssh") return; + logger("info", "api_socket_open", { remoteAddress: ws.remoteAddress, channel: ws.data.channel ?? null }); }, message(ws, raw) { - if (ws.data.channel !== "ssh") { - ws.close(1008, "unsupported websocket channel"); + if (ws.data.channel === "ssh") { + handleSshClientMessage(ws, raw).catch((error) => { + logger("error", "ssh_client_message_failed", { sessionId: ws.data.sessionId ?? null, error: errorToJson(error) }); + }); return; } - handleSshClientMessage(ws, raw).catch((error) => { - logger("error", "ssh_client_message_failed", { providerId: ws.data.providerId ?? null, sessionId: ws.data.sessionId ?? null, error: errorToJson(error) }); - wsSendJson(ws, { type: "ssh.error", message: error instanceof Error ? error.message : String(error) }); - }); }, close(ws) { - if (ws.data.channel !== "ssh") return; - const { sessionId, providerId } = ws.data; - logger("warn", "ssh_client_close", { providerId: providerId ?? null, sessionId: sessionId ?? null }); - if (sessionId !== undefined) { - activeSshClients.delete(sessionId); - if (providerId !== undefined) { - const provider = providerForSsh(providerId); - if (provider !== null) { - const closeMessage: CoreHostSshCloseMessage = { type: "host_ssh_close", sessionId }; - provider.send(JSON.stringify(closeMessage)); - } - } - } + if (ws.data.channel === "ssh") return; }, }, }); const providerServer = Bun.serve({ - port: config.providerPort, - hostname: "0.0.0.0", + port: config().providerPort, async fetch(req, server) { const started = performance.now(); const url = new URL(req.url); let response: Response | undefined; try { - response = await providerRoute(req, server); + response = await providerRoute(req, server) as Response | undefined; return response; } finally { recordRequestPerformance(req, url.pathname, response, performance.now() - started); @@ -3678,7 +162,7 @@ const providerServer = Bun.serve({ logger("warn", "provider_socket_close", { providerId: providerId ?? null }); if (providerId !== undefined) { closeEgressTcpConnectionsForProvider(providerId); - if (activeProviders.get(providerId) !== ws) { + if (ctx.activeProviders.get(providerId) !== ws) { logger("info", "provider_socket_close_ignored_replaced", { providerId }); return; } @@ -3694,7 +178,7 @@ setInterval(() => { setInterval(() => { markStaleTasksFailed().catch((error) => logger("error", "task_timeout_sweep_failed", { error: errorToJson(error) })); -}, Math.min(config.taskPendingTimeoutMs, 60_000)); +}, Math.min(config().taskPendingTimeoutMs, 60_000)); setInterval(() => { runDueScheduledTasks().catch((error) => logger("error", "scheduled_task_sweep_failed", { error: errorToJson(error) })); @@ -3703,5 +187,5 @@ setInterval(() => { logger("info", "server_listening", { apiUrl: `http://0.0.0.0:${apiServer.port}`, providerIngressUrl: `ws://0.0.0.0:${providerServer.port}/ws/provider`, - logFile: config.logFile, + logFile: config().logFile, }); diff --git a/src/components/backend-core/src/logger.ts b/src/components/backend-core/src/logger.ts new file mode 100644 index 00000000..b475ae41 --- /dev/null +++ b/src/components/backend-core/src/logger.ts @@ -0,0 +1,29 @@ +import type { JsonValue } from "../../shared/src/index"; +import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../shared/src/rotating-jsonl"; +import type { LoggerFn } from "./types"; + +export function createLogger(service: string, logFile: string): { logger: LoggerFn; recentLogs: unknown[] } { + const recentLogs: unknown[] = []; + const writer = createHourlyJsonlWriter({ + baseLogFile: logFile, + service, + maxBytes: logRetentionBytesForService(service), + }); + writer.prune(); + const logger: LoggerFn = (level, message, data?: JsonValue): void => { + const entry = data === undefined + ? { ts: new Date().toISOString(), service, level, message } + : { ts: new Date().toISOString(), service, level, message, data }; + recentLogs.push(entry); + while (recentLogs.length > 500) recentLogs.shift(); + const line = `${JSON.stringify(entry)}\n`; + try { + writer.appendLine(line, new Date(entry.ts)); + } catch (error) { + console.error(JSON.stringify({ ts: new Date().toISOString(), service, level: "error", message: "log_write_failed", data: String(error) })); + } + const consoleMethod = level === "error" ? console.error : level === "warn" ? console.warn : console.log; + consoleMethod(line.trimEnd()); + }; + return { logger, recentLogs }; +} diff --git a/src/components/backend-core/src/microservice-proxy.ts b/src/components/backend-core/src/microservice-proxy.ts new file mode 100644 index 00000000..90c18e6c --- /dev/null +++ b/src/components/backend-core/src/microservice-proxy.ts @@ -0,0 +1,795 @@ +import type { JsonValue } from "../../shared/src/index"; +import { ctx, config, logger } from "./context"; +import type { MicroserviceConfig, MicroserviceProxyCacheEntry, MicroserviceHealthAssessment, MicroserviceAvailabilityEntry, RawTaskRow } from "./types"; +import { jsonResponse, errorToJson, compactJson, isPlainRecord, truncateText } from "./http"; +import { createAndSendTask, waitForTaskTerminal, providerSupports } from "./task-dispatcher"; +import { getNodes, getNodeDockerStatuses } from "./db"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const microserviceProxyMaxBodyTextLength = 8 * 1024 * 1024; +const microserviceAvailabilityTtlMs = 30_000; +const microserviceForwardRequestHeaders = [ + "accept", + "content-type", + "range", + "x-auth", + "x-requested-with", + "destination", + "overwrite", + "tus-resumable", + "upload-concat", + "upload-defer-length", + "upload-length", + "upload-metadata", + "upload-offset", +] as const; + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +function recordValue(value: unknown, key: string): unknown { + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + return (value as Record)[key]; +} + +function dockerStatusRecord(status: JsonValue | null): Record { + return typeof status === "object" && status !== null && !Array.isArray(status) ? status as Record : {}; +} + +function findContainer(status: JsonValue | null, containerName: string): JsonValue | null { + const containers = dockerStatusRecord(status).containers; + if (!Array.isArray(containers)) return null; + const container = containers.find((item) => { + if (typeof item !== "object" || item === null || Array.isArray(item)) return false; + return (item as Record).name === containerName; + }); + return container === undefined ? null : container as JsonValue; +} + +function cachedMicroserviceAvailability(serviceId: string): Record | null { + const entry = ctx.microserviceAvailabilityCache.get(serviceId); + if (entry === undefined) return null; + if (entry.expiresAt <= Date.now()) { + ctx.microserviceAvailabilityCache.delete(serviceId); + return null; + } + return entry.probe; +} + +function inferredMicroserviceAvailability(service: MicroserviceConfig, providerStatus: string, container: JsonValue | null): JsonValue { + const cached = cachedMicroserviceAvailability(service.id); + if (cached !== null) return cached; + const containerState = isPlainRecord(container) ? String(container.state ?? "").toLowerCase() : ""; + const containerStatus = isPlainRecord(container) ? String(container.status ?? "") : ""; + if (providerStatus !== "online") { + return { + serviceId: service.id, + healthy: false, + status: "unhealthy", + reason: `provider is ${providerStatus}`, + source: "runtime-inference", + }; + } + if (container !== null && containerState !== "running") { + return { + serviceId: service.id, + healthy: false, + status: "unhealthy", + reason: `container is ${containerStatus || containerState || "not running"}`, + source: "runtime-inference", + }; + } + return { + serviceId: service.id, + healthy: null, + status: "unknown", + reason: "strict health has not been probed yet", + source: "runtime-inference", + }; +} + +function isMicroservicePathAllowed(service: MicroserviceConfig, path: string): boolean { + return service.backend.allowedPathPrefixes.some((prefix) => path === prefix || path.startsWith(prefix)); +} + +function isMicroserviceMethodAllowed(service: MicroserviceConfig, method: string): boolean { + return service.backend.allowedMethods.includes(method.toUpperCase()); +} + +function readMicroserviceArrayLimits(url: URL): { query: string; jsonArrayLimits: Record } { + const params = new URLSearchParams(url.searchParams); + const jsonArrayLimits: Record = {}; + const rawLimits = params.getAll("__unideskArrayLimit"); + params.delete("__unideskArrayLimit"); + for (const raw of rawLimits) { + for (const entry of raw.split(",")) { + const [path = "", limitText = ""] = entry.split(":"); + if (!/^[A-Za-z0-9_.-]+$/.test(path)) continue; + const limit = Number(limitText); + if (Number.isInteger(limit) && limit > 0 && limit <= 500) jsonArrayLimits[path] = limit; + } + } + const search = params.toString(); + return { query: search.length > 0 ? `?${search}` : "", jsonArrayLimits }; +} +function contentTypeIsJson(contentType: string): boolean { + return contentType.toLowerCase().includes("json"); +} + +function readMicroserviceRequestHeaders(req: Request): Record { + const requestHeaders: Record = {}; + for (const name of microserviceForwardRequestHeaders) { + const value = req.headers.get(name); + if (value !== null && value.length > 0) requestHeaders[name] = value.slice(0, 4096); + } + return requestHeaders; +} + +function headersFromMicroserviceRequest(requestHeaders: Record): Headers { + const headers = new Headers(); + for (const name of microserviceForwardRequestHeaders) { + const value = requestHeaders[name]; + if (typeof value === "string" && value.length > 0) headers.set(name, value); + } + return headers; +} + +function boundedMicroserviceBodyText( + bodyText: string, + contentType: string, + metadata: { serviceId: string; targetPath: string; status: number; upstreamBodyBytes: number }, +): { bodyText: string; truncated: boolean } { + if (bodyText.length <= microserviceProxyMaxBodyTextLength) { + return { bodyText, truncated: false }; + } + if (contentTypeIsJson(contentType)) { + return { + bodyText: JSON.stringify({ + ok: false, + error: "microservice proxy response body is too large", + serviceId: metadata.serviceId, + targetPath: metadata.targetPath, + upstreamStatus: metadata.status, + upstreamBodyBytes: metadata.upstreamBodyBytes, + transformedBodyBytes: bodyText.length, + responseBodyLimitBytes: microserviceProxyMaxBodyTextLength, + hint: "Use a paged endpoint or tighten __unideskArrayLimit so the response stays below the proxy safety limit.", + }), + truncated: true, + }; + } + return { bodyText: truncateText(bodyText, microserviceProxyMaxBodyTextLength), truncated: true }; +} + +function arrayAtPath(value: unknown, path: string): JsonValue[] | null { + let current: unknown = value; + for (const part of path.split(".")) { + if (typeof current !== "object" || current === null || Array.isArray(current)) return null; + current = (current as Record)[part]; + } + return Array.isArray(current) ? current as JsonValue[] : null; +} +function applyJsonArrayLimits(bodyText: string, contentType: string, limits: Record): string { + const entries = Object.entries(limits); + if (entries.length === 0 || !contentType.toLowerCase().includes("json")) return bodyText; + try { + const parsed = JSON.parse(bodyText) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return bodyText; + const root = parsed as Record; + const applied: Record = {}; + for (const [path, rawLimit] of entries) { + const limit = Number(rawLimit); + if (!Number.isInteger(limit) || limit <= 0 || limit > 500) continue; + const array = arrayAtPath(root, path); + if (array === null) continue; + const originalLength = array.length; + if (array.length > limit) array.splice(limit); + applied[path] = { limit, originalLength, returnedLength: array.length }; + } + root._unidesk = { arrayLimits: applied }; + return JSON.stringify(parsed); + } catch { + return bodyText; + } +} + +function responseFromMicroserviceResult(task: RawTaskRow | null): Response { + if (task === null) return jsonResponse({ ok: false, error: "microservice proxy task missing" }, 502); + if (task.status !== "succeeded") return jsonResponse({ ok: false, error: "microservice proxy task failed", task }, 502); + const result = dockerStatusRecord(task.result); + const status = Number(result.status); + const contentType = typeof result.contentType === "string" ? result.contentType : "application/json; charset=utf-8"; + const bodyText = typeof result.bodyText === "string" ? result.bodyText : ""; + if (!Number.isInteger(status) || status < 100 || status > 599) { + return jsonResponse({ ok: false, error: "microservice proxy returned invalid upstream status", task }, 502); + } + if (result.truncated === true && contentTypeIsJson(contentType)) { + try { + JSON.parse(bodyText); + } catch { + return jsonResponse({ + ok: false, + error: "microservice proxy response was truncated before a JSON boundary", + providerId: task.provider_id, + command: task.command, + upstreamStatus: status, + upstreamBodyBytes: result.upstreamBodyBytes ?? null, + returnedBodyBytes: result.returnedBodyBytes ?? bodyText.length, + responseBodyLimitBytes: result.responseBodyLimitBytes ?? null, + hint: "Upgrade the provider-gateway or request a smaller/paged microservice response.", + }, 502); + } + } + return new Response(bodyText, { + status, + headers: { + "content-type": contentType, + "x-unidesk-response-truncated": result.truncated === true ? "true" : "false", + }, + }); +} +function nestedRecord(value: unknown, key: string): Record | null { + if (!isPlainRecord(value)) return null; + const child = value[key]; + return isPlainRecord(child) ? child : null; +} + +function nestedBoolean(value: unknown, key: string): boolean | null { + if (!isPlainRecord(value)) return null; + const child = value[key]; + return typeof child === "boolean" ? child : null; +} + +function nestedString(value: unknown, key: string): string | null { + if (!isPlainRecord(value)) return null; + const child = value[key]; + return typeof child === "string" ? child : null; +} + +function jsonBodyFromText(bodyText: string, contentType: string): JsonValue { + if (!contentTypeIsJson(contentType)) return bodyText.length > 0 ? truncateText(bodyText, 4000) : null; + try { + return compactJson(JSON.parse(bodyText) as unknown); + } catch { + return bodyText.length > 0 ? truncateText(bodyText, 4000) : null; + } +} + +function healthFailure(body: JsonValue, checks: Record, reason: string): MicroserviceHealthAssessment { + return { healthy: false, reason, checks, body }; +} + +function healthSuccess(body: JsonValue, checks: Record, reason = "health endpoint returned a usable service state"): MicroserviceHealthAssessment { + return { healthy: true, reason, checks, body }; +} + +function genericMicroserviceHealthAssessment(upstreamStatus: number, body: JsonValue): MicroserviceHealthAssessment { + const checks: Record = { + upstreamStatus, + upstream2xx: upstreamStatus >= 200 && upstreamStatus < 300, + }; + if (upstreamStatus < 200 || upstreamStatus >= 300) { + return healthFailure(body, checks, `health endpoint returned HTTP ${upstreamStatus}`); + } + if (isPlainRecord(body)) { + const ok = body.ok; + const success = body.success; + const statusText = typeof body.status === "string" ? body.status : ""; + checks.okField = typeof ok === "boolean" ? ok : null; + checks.successField = typeof success === "boolean" ? success : null; + checks.statusField = statusText || null; + if (ok === false) return healthFailure(body, checks, "health body ok=false"); + if (success === false) return healthFailure(body, checks, "health body success=false"); + if (["error", "failed", "fail", "unhealthy", "down", "offline", "not_ready", "not-ready"].includes(statusText.toLowerCase())) { + return healthFailure(body, checks, `health body status=${statusText}`); + } + } + return healthSuccess(body, checks); +} +function assessMetNonlinearHealth(upstreamStatus: number, body: JsonValue): MicroserviceHealthAssessment { + const base = genericMicroserviceHealthAssessment(upstreamStatus, body); + const checks = { ...base.checks }; + if (!base.healthy) return { ...base, checks }; + if (!isPlainRecord(body)) return healthFailure(body, checks, "MET Nonlinear health body is not JSON"); + checks.service = typeof body.service === "string" ? body.service : null; + checks.queueReady = isPlainRecord(body.queue); + const image = nestedRecord(body, "image"); + checks.mlImage = typeof image?.image === "string" ? image.image : null; + checks.mlImagePresent = image?.present === true; + if (body.service !== "met-nonlinear-unidesk-ts") { + return healthFailure(body, checks, "MET Nonlinear health returned an unexpected service identity"); + } + if (!isPlainRecord(body.queue)) return healthFailure(body, checks, "MET Nonlinear queue state is not available"); + if (image?.present !== true) return healthFailure(body, checks, "MET Nonlinear ML image is not ready"); + return healthSuccess(body, checks, "MET Nonlinear backend, queue, and ML image are ready"); +} + +function assessClaudeQqHealth(upstreamStatus: number, body: JsonValue): MicroserviceHealthAssessment { + const base = genericMicroserviceHealthAssessment(upstreamStatus, body); + const checks = { ...base.checks }; + if (!base.healthy) return { ...base, checks }; + if (!isPlainRecord(body)) return healthFailure(body, checks, "ClaudeQQ health body is not JSON"); + const napcat = nestedRecord(body, "napcat"); + const endpoints = Array.isArray(body.endpoints) ? body.endpoints.map((item) => String(item)) : []; + const httpConnected = nestedBoolean(napcat, "httpConnected") === true; + const wsConnected = nestedBoolean(napcat, "wsConnected") === true; + const napcatConnected = nestedBoolean(napcat, "connected") === true; + const loginState = nestedString(napcat, "loginState") ?? ""; + checks.service = typeof body.service === "string" ? body.service : null; + checks.pushEndpoint = endpoints.includes("/api/push/text"); + checks.napcatHttpLoggedIn = httpConnected; + checks.napcatWebsocketConnected = wsConnected; + checks.napcatConnected = napcatConnected; + checks.loginState = loginState || null; + if (body.service !== "claudeqq") return healthFailure(body, checks, "ClaudeQQ health returned an unexpected service identity"); + if (!endpoints.includes("/api/push/text")) return healthFailure(body, checks, "ClaudeQQ push API endpoint is not advertised"); + if (!httpConnected) return healthFailure(body, checks, "NapCat HTTP is not logged in"); + if (!wsConnected) return healthFailure(body, checks, "NapCat OneBot WebSocket is not connected"); + if (!napcatConnected || loginState !== "logged_in") return healthFailure(body, checks, "NapCat is not in a logged-in send/receive ready state"); + return healthSuccess(body, checks, "ClaudeQQ and NapCat are logged in and send/receive ready"); +} + +function assessMicroserviceHealth(service: MicroserviceConfig, upstreamStatus: number, contentType: string, bodyText: string): MicroserviceHealthAssessment { + const body = jsonBodyFromText(bodyText, contentType); + if (service.id === "met-nonlinear") return assessMetNonlinearHealth(upstreamStatus, body); + if (service.id === "claudeqq") return assessClaudeQqHealth(upstreamStatus, body); + return genericMicroserviceHealthAssessment(upstreamStatus, body); +} +function healthProbeFromAssessment( + service: MicroserviceConfig, + upstreamStatus: number, + contentType: string, + assessment: MicroserviceHealthAssessment, +): Record { + return { + serviceId: service.id, + name: service.name, + providerId: service.providerId, + healthPath: service.backend.healthPath, + healthy: assessment.healthy, + status: assessment.healthy ? "healthy" : "unhealthy", + reason: assessment.reason, + upstreamStatus, + contentType, + checks: assessment.checks, + checkedAt: new Date().toISOString(), + }; +} + +function rememberMicroserviceAvailability(serviceId: string, probe: Record): void { + ctx.microserviceAvailabilityCache.set(serviceId, { expiresAt: Date.now() + microserviceAvailabilityTtlMs, probe }); +} + +function canDirectProxyMicroservice(service: MicroserviceConfig): boolean { + return service.providerId === "main-server"; +} + +function isV3sctlManagedMicroservice(service: MicroserviceConfig): boolean { + return service.deployment.mode === "v3sctl-managed" || service.backend.proxyMode === "v3sctl-adapter-http"; +} + +// --------------------------------------------------------------------------- +// Cache helpers +// --------------------------------------------------------------------------- + +function microserviceCacheTtlMs(serviceId: string, targetPath: string): number { + if (targetPath === "/health") return 0; + if (targetPath.endsWith("/stream")) return 0; + if (serviceId === "pipeline" && targetPath === "/api/snapshot") return 6_000; + if (serviceId === "pipeline" && targetPath.startsWith("/api/oa-event-flow/")) return 20_000; + if (serviceId === "pipeline" && targetPath.startsWith("/api/model-quota/")) return 60_000; + if (serviceId === "pipeline" && targetPath.startsWith("/api/node-control/runs/")) return 6_000; + if (serviceId === "pipeline" && targetPath.startsWith("/api/runs/")) return 6_000; + if (serviceId === "findjob" && (targetPath === "/api/summary" || targetPath === "/api/jobs" || targetPath === "/api/drafts")) return 8_000; + if (serviceId === "met-nonlinear" && (targetPath === "/api/images" || targetPath === "/api/projects")) return 15_000; + if (serviceId === "met-nonlinear" && (targetPath === "/api/queue" || targetPath === "/api/summary" || targetPath === "/api/history")) return 5_000; + if (serviceId === "code-queue" && targetPath.includes("/transcript")) return 1_000; + if (serviceId === "code-queue" && targetPath === "/api/tasks/overview") return 3_000; + if (serviceId === "code-queue" && targetPath.startsWith("/api/tasks/")) return 2_000; + return 750; +} + +function microserviceCacheStaleMs(serviceId: string, targetPath: string): number { + if (serviceId === "pipeline" && targetPath.startsWith("/api/model-quota/")) return 5 * 60_000; + if (serviceId === "pipeline") return 45_000; + if (serviceId === "findjob") return 60_000; + if (serviceId === "met-nonlinear" && (targetPath === "/api/images" || targetPath === "/api/projects")) return 5 * 60_000; + if (serviceId === "met-nonlinear") return 45_000; + if (serviceId === "code-queue") return 15_000; + return 5_000; +} +function providerMicroserviceCacheTtlMs(serviceId: string, targetPath: string): number { + if (targetPath === "/health") return 0; + if (serviceId === "met-nonlinear" && (targetPath === "/api/images" || targetPath === "/api/projects")) return 60_000; + if (serviceId === "met-nonlinear" && targetPath === "/api/history") return 10_000; + if (serviceId === "met-nonlinear" && (targetPath === "/api/queue" || targetPath === "/api/summary")) return 3_000; + if (serviceId === "pipeline" && (targetPath === "/api/snapshot" || targetPath.startsWith("/api/oa-event-flow/"))) return 2_000; + if (serviceId === "findjob" && (targetPath === "/api/summary" || targetPath === "/api/jobs")) return 2_000; + return 1_000; +} + +function microserviceCacheKey(service: MicroserviceConfig, method: string, targetPath: string, proxyOptions: { query: string; jsonArrayLimits: Record }): string { + return JSON.stringify([service.id, method, targetPath, proxyOptions.query, proxyOptions.jsonArrayLimits]); +} + +function responseFromMicroserviceCache(entry: MicroserviceProxyCacheEntry, state: "hit" | "stale"): Response { + return new Response(entry.bodyText, { + status: entry.status, + headers: { + "content-type": entry.contentType, + "x-unidesk-cache": state, + }, + }); +} + +function readMicroserviceCache(key: string): Response | null { + const entry = ctx.microserviceProxyCache.get(key); + if (entry === undefined) return null; + if (entry.staleExpiresAt <= Date.now()) { + ctx.microserviceProxyCache.delete(key); + return null; + } + if (entry.expiresAt <= Date.now()) return null; + return responseFromMicroserviceCache(entry, "hit"); +} + +function readStaleMicroserviceCache(key: string): Response | null { + const entry = ctx.microserviceProxyCache.get(key); + if (entry === undefined) return null; + if (entry.staleExpiresAt <= Date.now()) { + ctx.microserviceProxyCache.delete(key); + return null; + } + if (entry.expiresAt > Date.now()) return null; + return responseFromMicroserviceCache(entry, "stale"); +} + +async function cacheableResponseSnapshot(response: Response): Promise { + if (response.status < 200 || response.status >= 300) return null; + if (response.headers.get("x-unidesk-response-truncated") === "true") return null; + const bodyText = await response.clone().text(); + if (bodyText.length > 2 * 1024 * 1024) return null; + return { + expiresAt: 0, + staleExpiresAt: 0, + status: response.status, + contentType: response.headers.get("content-type") ?? "application/octet-stream", + bodyText, + }; +} +function rememberMicroserviceCache(key: string, ttlMs: number, entry: MicroserviceProxyCacheEntry | null): void { + if (entry === null || ttlMs <= 0) return; + const keyParts = JSON.parse(key) as string[]; + entry.expiresAt = Date.now() + ttlMs; + entry.staleExpiresAt = entry.expiresAt + microserviceCacheStaleMs(String(keyParts[0] ?? ""), String(keyParts[2] ?? "")); + ctx.microserviceProxyCache.set(key, entry); + if (ctx.microserviceProxyCache.size > 300) { + const now = Date.now(); + for (const [cacheKey, cacheEntry] of ctx.microserviceProxyCache) { + if (cacheEntry.expiresAt <= now || ctx.microserviceProxyCache.size > 240) ctx.microserviceProxyCache.delete(cacheKey); + } + } +} + +function invalidateMicroserviceCache(serviceId: string): void { + const prefix = `["${serviceId}",`; + for (const key of ctx.microserviceProxyCache.keys()) { + if (key.startsWith(prefix)) ctx.microserviceProxyCache.delete(key); + } +} + +// --------------------------------------------------------------------------- +// Upstream proxy functions +// --------------------------------------------------------------------------- + +async function directMicroserviceResponse( + service: MicroserviceConfig, + method: string, + targetPath: string, + proxyOptions: { query: string; jsonArrayLimits: Record }, + requestHeaders: Record, + bodyText: string, + abortSignal?: AbortSignal, +): Promise { + const baseUrl = new URL(service.backend.nodeBaseUrl); + const upstreamUrl = new URL(targetPath, baseUrl); + upstreamUrl.search = proxyOptions.query; + const headers = headersFromMicroserviceRequest(requestHeaders); + const streamExpected = method === "GET" && (targetPath.endsWith("/stream") || String(headers.get("accept") || "").toLowerCase().includes("text/event-stream")); + const controller = new AbortController(); + const timer = streamExpected ? null : setTimeout(() => controller.abort(), Math.max(1000, service.backend.timeoutMs)); + try { + const upstream = await fetch(upstreamUrl, { + method, + headers, + body: method === "GET" || method === "HEAD" ? undefined : bodyText, + signal: streamExpected ? abortSignal : controller.signal, + }); + const upstreamContentType = upstream.headers.get("content-type") ?? "text/plain; charset=utf-8"; + if (upstreamContentType.toLowerCase().includes("text/event-stream")) { + const responseHeaders: Record = { + "content-type": upstreamContentType, + "cache-control": upstream.headers.get("cache-control") || "no-store, no-transform", + "connection": "keep-alive", + "x-unidesk-proxy-mode": "direct", + "x-unidesk-response-truncated": "false", + }; + const buffering = upstream.headers.get("x-accel-buffering"); + if (buffering !== null) responseHeaders["x-accel-buffering"] = buffering; + return new Response(upstream.body, { status: upstream.status, headers: responseHeaders }); + } + const rawBodyText = await upstream.text(); + const limitedBodyText = applyJsonArrayLimits(rawBodyText, upstreamContentType, proxyOptions.jsonArrayLimits); + const bounded = boundedMicroserviceBodyText(limitedBodyText, upstreamContentType, { + serviceId: service.id, + targetPath, + status: upstream.status, + upstreamBodyBytes: rawBodyText.length, + }); + return new Response(bounded.bodyText, { + status: upstream.status, + headers: { + "content-type": upstreamContentType, + "x-unidesk-proxy-mode": "direct", + "x-unidesk-response-truncated": bounded.truncated ? "true" : "false", + }, + }); + } catch (error) { + return jsonResponse({ ok: false, error: "direct microservice proxy failed", serviceId: service.id, detail: errorToJson(error) }, 502); + } finally { + if (timer !== null) clearTimeout(timer); + } +} + +async function v3sctlAdapterMicroserviceResponse( + service: MicroserviceConfig, + method: string, + targetPath: string, + proxyOptions: { query: string; jsonArrayLimits: Record }, + requestHeaders: Record, + bodyText: string, + abortSignal?: AbortSignal, +): Promise { + const adapterServiceId = service.deployment.adapterServiceId ?? "v3sctl-adapter"; + const adapter = microserviceById(adapterServiceId); + if (adapter === null) { + return jsonResponse({ ok: false, error: `v3sctl adapter microservice not found: ${adapterServiceId}`, serviceId: service.id }, 502); + } + if (adapter.id === service.id || isV3sctlManagedMicroservice(adapter)) { + return jsonResponse({ ok: false, error: "v3sctl adapter must be a UniDesk-direct microservice", serviceId: service.id, adapterServiceId }, 500); + } + const v3sServiceId = service.deployment.v3sServiceId ?? service.id; + const adapterTargetPath = `/api/services/${encodeURIComponent(v3sServiceId)}/proxy${targetPath}`; + return fetchMicroserviceUpstreamResponse(adapter, method, adapterTargetPath, proxyOptions, requestHeaders, bodyText, abortSignal); +} + +async function fetchMicroserviceUpstreamResponse( + service: MicroserviceConfig, + method: string, + targetPath: string, + proxyOptions: { query: string; jsonArrayLimits: Record }, + requestHeaders: Record, + bodyText: string, + abortSignal?: AbortSignal, +): Promise { + if (isV3sctlManagedMicroservice(service)) { + return v3sctlAdapterMicroserviceResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, abortSignal); + } + if (canDirectProxyMicroservice(service)) { + return directMicroserviceResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, abortSignal); + } + if (!(await providerSupports(service.providerId, "microservice.http"))) { + return jsonResponse({ ok: false, error: `provider does not declare microservice.http capability: ${service.providerId}` }, 409); + } + const { taskId, providerOnline } = await createAndSendTask(service.providerId, "microservice.http", { + source: "microservice-frontend-proxy", + serviceId: service.id, + method, + targetBaseUrl: service.backend.nodeBaseUrl, + path: targetPath, + query: proxyOptions.query, + requestHeaders, + bodyText, + jsonArrayLimits: proxyOptions.jsonArrayLimits, + timeoutMs: service.backend.timeoutMs, + cacheTtlMs: providerMicroserviceCacheTtlMs(service.id, targetPath), + }); + if (!providerOnline) return jsonResponse({ ok: false, error: `provider is offline: ${service.providerId}`, taskId }, 503); + const task = await waitForTaskTerminal(taskId, service.backend.timeoutMs + 3000); + return responseFromMicroserviceResult(task); +} + +function refreshMicroserviceCacheInBackground( + cacheKey: string, + ttlMs: number, + fetchResponse: () => Promise, +): void { + if (ctx.microserviceProxyRefreshes.has(cacheKey)) return; + const refresh = fetchResponse() + .then((response) => cacheableResponseSnapshot(response)) + .then((entry) => rememberMicroserviceCache(cacheKey, ttlMs, entry)) + .catch((error) => { + logger("warn", "microservice_cache_revalidate_failed", { cacheKey, error: errorToJson(error) }); + }) + .finally(() => { + ctx.microserviceProxyRefreshes.delete(cacheKey); + }); + ctx.microserviceProxyRefreshes.set(cacheKey, refresh); +} + +// --------------------------------------------------------------------------- +// Exported functions +// --------------------------------------------------------------------------- + +export function microserviceById(serviceId: string): MicroserviceConfig | null { + return config().microservices.find((service) => service.id === serviceId) ?? null; +} + +export async function strictMicroserviceHealthProbe(service: MicroserviceConfig, timeoutMs?: number): Promise> { + const probeService = timeoutMs === undefined + ? service + : { ...service, backend: { ...service.backend, timeoutMs } }; + const response = await fetchMicroserviceUpstreamResponse(probeService, "GET", service.backend.healthPath, { query: "", jsonArrayLimits: {} }, { accept: "application/json" }, ""); + const upstreamStatus = response.status; + const contentType = response.headers.get("content-type") ?? "application/octet-stream"; + const bodyText = await response.text(); + const assessment = assessMicroserviceHealth(service, upstreamStatus, contentType, bodyText); + const probe = healthProbeFromAssessment(service, upstreamStatus, contentType, assessment); + rememberMicroserviceAvailability(service.id, probe); + return { ...probe, body: assessment.body }; +} + +export async function strictMicroserviceHealthResponse(service: MicroserviceConfig, headOnly: boolean): Promise { + const response = await fetchMicroserviceUpstreamResponse(service, "GET", service.backend.healthPath, { query: "", jsonArrayLimits: {} }, { accept: "application/json" }, ""); + const upstreamStatus = response.status; + const contentType = response.headers.get("content-type") ?? "application/octet-stream"; + const bodyText = await response.text(); + const assessment = assessMicroserviceHealth(service, upstreamStatus, contentType, bodyText); + const probe = healthProbeFromAssessment(service, upstreamStatus, contentType, assessment); + rememberMicroserviceAvailability(service.id, probe); + if (assessment.healthy) { + return new Response(headOnly ? null : bodyText, { + status: upstreamStatus, + headers: { + "content-type": contentType, + "x-unidesk-health": "healthy", + "x-unidesk-health-reason": assessment.reason, + }, + }); + } + const status = upstreamStatus >= 500 ? upstreamStatus : 503; + return jsonResponse({ + ok: false, + status: "unhealthy", + serviceId: service.id, + name: service.name, + providerId: service.providerId, + reason: assessment.reason, + checkedAt: probe.checkedAt, + checks: assessment.checks, + upstream: { + status: upstreamStatus, + contentType, + body: assessment.body, + }, + }, status); +} + +export async function microserviceAvailability(service: MicroserviceConfig): Promise> { + const cached = cachedMicroserviceAvailability(service.id); + if (cached !== null) return cached; + const existing = ctx.microserviceAvailabilityRefreshes.get(service.id); + if (existing !== undefined) return existing; + const timeoutMs = Math.max(2500, Math.min(service.backend.timeoutMs, 10_000)); + const refresh = strictMicroserviceHealthProbe(service, timeoutMs) + .catch((error) => { + const probe = { + serviceId: service.id, + name: service.name, + providerId: service.providerId, + healthPath: service.backend.healthPath, + healthy: false, + status: "unhealthy", + reason: error instanceof Error ? error.message : String(error), + upstreamStatus: null, + contentType: null, + checks: {}, + checkedAt: new Date().toISOString(), + } satisfies Record; + rememberMicroserviceAvailability(service.id, probe); + return probe; + }) + .finally(() => { + ctx.microserviceAvailabilityRefreshes.delete(service.id); + }); + ctx.microserviceAvailabilityRefreshes.set(service.id, refresh); + return refresh; +} + +export async function getMicroservices(): Promise { + const nodes = await getNodes(); + const dockerStatuses = await getNodeDockerStatuses(); + return config().microservices.map((service) => { + const node = nodes.find((item) => item.providerId === service.providerId) ?? null; + const docker = dockerStatuses.find((item) => item.providerId === service.providerId) ?? null; + const v3sManaged = isV3sctlManagedMicroservice(service); + const container = v3sManaged ? null : findContainer(docker?.dockerStatus ?? null, service.repository.containerName); + return { + ...service, + runtime: { + orchestrator: v3sManaged ? "v3sctl" : "unidesk-direct", + providerStatus: node?.status ?? "missing", + providerName: node?.name ?? service.providerId, + providerLastHeartbeat: node?.lastHeartbeat ?? null, + container, + availability: inferredMicroserviceAvailability(service, node?.status ?? "missing", container), + backendPortMapping: { + providerId: service.providerId, + node: `${service.backend.nodeBindHost}:${service.backend.nodePort}`, + mainServerAccess: "frontend-only backend proxy", + public: service.backend.public, + }, + }, + } satisfies JsonValue; + }); +} +export async function microserviceRoute(req: Request, url: URL): Promise { + const rest = url.pathname.slice("/api/microservices/".length); + const slashIndex = rest.indexOf("/"); + const serviceId = decodeURIComponent(slashIndex === -1 ? rest : rest.slice(0, slashIndex)); + const suffix = slashIndex === -1 ? "" : rest.slice(slashIndex + 1); + const service = microserviceById(serviceId); + if (service === null) return jsonResponse({ ok: false, error: `microservice not found: ${serviceId}` }, 404); + const method = req.method.toUpperCase(); + if ((suffix === "" || suffix === "status") && (method === "GET" || method === "HEAD")) { + return jsonResponse({ ok: true, microservice: (await getMicroservices()).find((item) => recordValue(item, "id") === serviceId) ?? service }); + } + + const proxyPrefix = "proxy"; + const targetPath = suffix === "health" + ? service.backend.healthPath + : suffix === proxyPrefix + ? "/" + : suffix.startsWith(`${proxyPrefix}/`) + ? `/${suffix.slice(proxyPrefix.length + 1)}` + : ""; + if (targetPath.length === 0) return jsonResponse({ ok: false, error: "microservice route must be /status, /health, or /proxy/" }, 404); + if (suffix === "health" && method !== "GET" && method !== "HEAD") { + return jsonResponse({ ok: false, error: "microservice health only supports GET/HEAD" }, 405); + } + if (!isMicroserviceMethodAllowed(service, method)) { + return jsonResponse({ ok: false, error: "microservice method is not allowed", serviceId, method, allowedMethods: service.backend.allowedMethods }, 405); + } + if (!isMicroservicePathAllowed(service, targetPath)) { + return jsonResponse({ ok: false, error: "microservice path is not allowed", serviceId, targetPath }, 403); + } + if (suffix === "health") return strictMicroserviceHealthResponse(service, method === "HEAD"); + const proxyOptions = readMicroserviceArrayLimits(url); + const cacheKey = microserviceCacheKey(service, method, targetPath, proxyOptions); + const cacheTtlMs = microserviceCacheTtlMs(service.id, targetPath); + if (method === "GET" || method === "HEAD") { + const cached = readMicroserviceCache(cacheKey); + if (cached !== null) return cached; + } else { + invalidateMicroserviceCache(service.id); + } + const bodyText = method === "GET" || method === "HEAD" ? "" : await req.text(); + if (bodyText.length > 1024 * 1024) { + return jsonResponse({ ok: false, error: "microservice request body is too large", maxBytes: 1024 * 1024 }, 413); + } + const requestHeaders = readMicroserviceRequestHeaders(req); + if (method === "GET" || method === "HEAD") { + const stale = readStaleMicroserviceCache(cacheKey); + if (stale !== null) { + refreshMicroserviceCacheInBackground(cacheKey, cacheTtlMs, () => fetchMicroserviceUpstreamResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText)); + return stale; + } + } + const response = await fetchMicroserviceUpstreamResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, req.signal); + if ((method === "GET" || method === "HEAD") && cacheTtlMs > 0) rememberMicroserviceCache(cacheKey, cacheTtlMs, await cacheableResponseSnapshot(response)); + return response; +} diff --git a/src/components/backend-core/src/overview.ts b/src/components/backend-core/src/overview.ts new file mode 100644 index 00000000..6afd14e6 --- /dev/null +++ b/src/components/backend-core/src/overview.ts @@ -0,0 +1,316 @@ +import type { JsonValue } from "../../shared/src/index"; +import { ctx, sql, config, logger } from "./context"; +import type { RawTaskRow, MicroserviceConfig } from "./types"; +import { jsonResponse, compactJson, errorToJson } from "./http"; +import { createAndSendTask, waitForTaskTerminal } from "./task-dispatcher"; +import { microserviceById, getMicroservices, microserviceAvailability } from "./microservice-proxy"; +import { getNodes, getNodeDockerStatuses } from "./db"; + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +function safePerfRunId(): string { + return `codex_queue_perf_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; +} + +function numberFromUnknown(value: unknown, fallback: number, min: number, max: number): number { + const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : fallback; + if (!Number.isFinite(parsed)) return fallback; + return Math.max(min, Math.min(max, Math.floor(parsed))); +} + +function recordFromJson(value: JsonValue | null): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function parsePerfPoll(stdout: string): { ready: boolean; exitCode: number | null; body: string; stderr: string } { + if (!stdout.includes("__UNIDESK_CODEX_QUEUE_PERF_READY__")) { + return { ready: false, exitCode: null, body: "", stderr: "" }; + } + const exitMatch = stdout.match(/__UNIDESK_CODEX_QUEUE_PERF_READY__\r?\n([0-9]+)/u); + const bodyMatch = stdout.match(/__UNIDESK_CODEX_QUEUE_PERF_STDOUT__\r?\n([\s\S]*?)\r?\n__UNIDESK_CODEX_QUEUE_PERF_STDERR__/u); + const stderrMatch = stdout.match(/__UNIDESK_CODEX_QUEUE_PERF_STDERR__\r?\n([\s\S]*)$/u); + const exitCode = exitMatch === null ? null : Number(exitMatch[1]); + return { + ready: true, + exitCode: Number.isFinite(exitCode) ? exitCode : null, + body: bodyMatch?.[1]?.trim() ?? "", + stderr: stderrMatch?.[1]?.trim() ?? "", + }; +} + +function parsePerfJson(body: string): Record | null { + for (const line of body.split(/\r?\n/u).reverse()) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) continue; + try { + const parsed = JSON.parse(trimmed) as unknown; + if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) return parsed as Record; + } catch { + // Continue scanning for the compact JSON line emitted by the Playwright script. + } + } + return null; +} +// --------------------------------------------------------------------------- +// Exported helpers +// --------------------------------------------------------------------------- + +export function rawTaskJson(task: RawTaskRow | null): JsonValue { + if (task === null) return null; + return { + id: task.id, + providerId: task.provider_id, + command: task.command, + status: task.status, + payload: task.payload, + result: task.result, + updatedAt: task.updated_at instanceof Date ? task.updated_at.toISOString() : String(task.updated_at), + }; +} + +export async function providerSupports(providerId: string, capability: string): Promise { + if (!ctx.dbReady) return false; + const rows = await sql()>` + SELECT labels + FROM unidesk_nodes + WHERE provider_id = ${providerId} + LIMIT 1 + `; + const labels = rows[0]?.labels; + if (typeof labels !== "object" || labels === null || Array.isArray(labels)) return false; + const capabilities = (labels as Record).unideskCapabilities; + return Array.isArray(capabilities) ? capabilities.filter((item): item is string => typeof item === "string").includes(capability) : false; +} + +export async function runHostSshPerfCommand(providerId: string, command: string, timeoutMs = 18_000): Promise<{ ok: boolean; taskId: string; task: RawTaskRow | null; stdout: string; stderr: string; exitCode: number | null; timedOut: boolean }> { + const { taskId, providerOnline } = await createAndSendTask(providerId, "host.ssh", { + source: "code-queue-performance-panel", + mode: "exec", + cwd: "/root/unidesk", + timeoutMs: 15_000, + command, + }); + if (!providerOnline) { + return { ok: false, taskId, task: null, stdout: "", stderr: `provider is offline: ${providerId}`, exitCode: null, timedOut: false }; + } + const task = await waitForTaskTerminal(taskId, timeoutMs); + const result = recordFromJson(task?.result ?? null); + return { + ok: task?.status === "succeeded" && result.ok === true, + taskId, + task, + stdout: typeof result.stdout === "string" ? result.stdout : "", + stderr: typeof result.stderr === "string" ? result.stderr : "", + exitCode: typeof result.exitCode === "number" ? result.exitCode : null, + timedOut: result.timedOut === true, + }; +} +// --------------------------------------------------------------------------- +// countPendingTasks / getPgdataUsage / getMicroserviceAvailabilitySummary +// --------------------------------------------------------------------------- + +async function countPendingTasks(): Promise { + const rows = await sql()>` + SELECT count(*)::int AS count + FROM unidesk_tasks + WHERE status IN ('queued', 'dispatched', 'running') + `; + return Number(rows[0]?.count ?? 0); +} + +async function getPgdataUsage(): Promise { + const rows = await sql()>` + SELECT + current_database()::text AS database_name, + pg_database_size(current_database())::bigint AS database_bytes, + pg_size_pretty(pg_database_size(current_database()))::text AS database_pretty + `; + const row = rows[0]; + return { + volumeName: config().databaseVolumeName, + volumeSize: config().databaseVolumeSize, + databaseName: row?.database_name ?? "unknown", + databaseBytes: Number(row?.database_bytes ?? 0), + databasePretty: row?.database_pretty ?? "--", + }; +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function getMicroserviceAvailabilitySummary(): Promise> { + const probes = await Promise.all(config().microservices.map((service) => microserviceAvailability(service))); + const healthy = probes.filter((probe) => probe.healthy === true); + const byProvider: Record = {}; + for (const probe of probes) { + const providerId = String(probe.providerId ?? "unknown"); + const current = isPlainRecord(byProvider[providerId]) ? byProvider[providerId] as Record : { total: 0, healthy: 0, unhealthy: 0 }; + current.total = Number(current.total ?? 0) + 1; + if (probe.healthy === true) current.healthy = Number(current.healthy ?? 0) + 1; + else current.unhealthy = Number(current.unhealthy ?? 0) + 1; + byProvider[providerId] = current; + } + return { + totalCount: probes.length, + healthyCount: healthy.length, + unhealthyCount: probes.length - healthy.length, + checkedAt: new Date().toISOString(), + byProvider, + services: probes.map((probe) => ({ + serviceId: probe.serviceId, + name: probe.name, + providerId: probe.providerId, + healthy: probe.healthy, + status: probe.status, + reason: probe.reason, + checkedAt: probe.checkedAt, + })), + }; +} +// --------------------------------------------------------------------------- +// getOverview +// --------------------------------------------------------------------------- + +export async function getOverview(): Promise { + const [nodeRows, dockerRows, systemRows, pendingTasks, pgdata, microserviceAvailabilitySummary] = await Promise.all([ + sql()>` + SELECT + count(*)::int AS node_count, + count(*) FILTER (WHERE status = 'online')::int AS online_node_count + FROM unidesk_nodes + `, + sql()>` + SELECT count(*) FILTER (WHERE d.status IS NOT NULL)::int AS docker_status_node_count + FROM unidesk_nodes n + LEFT JOIN unidesk_node_docker_status d ON d.provider_id = n.provider_id + `, + sql()>` + SELECT count(*) FILTER (WHERE s.status IS NOT NULL)::int AS system_status_node_count + FROM unidesk_nodes n + LEFT JOIN unidesk_node_system_status s ON s.provider_id = n.provider_id + `, + countPendingTasks(), + getPgdataUsage(), + getMicroserviceAvailabilitySummary(), + ]); + const nodeRow = nodeRows[0]; + const dockerRow = dockerRows[0]; + const systemRow = systemRows[0]; + return { + service: "unidesk-core", + ok: true, + dbReady: ctx.dbReady, + pgdata, + uptimeSeconds: Math.floor((Date.now() - ctx.serviceStartedAt.getTime()) / 1000), + nodeCount: Number(nodeRow?.node_count ?? 0), + onlineNodeCount: Number(nodeRow?.online_node_count ?? 0), + dockerStatusNodeCount: Number(dockerRow?.docker_status_node_count ?? 0), + systemStatusNodeCount: Number(systemRow?.system_status_node_count ?? 0), + pendingTaskCount: pendingTasks, + microserviceAvailability: microserviceAvailabilitySummary, + taskPendingTimeoutMs: config().taskPendingTimeoutMs, + activeSocketCount: ctx.activeProviders.size, + heartbeatTimeoutMs: config().heartbeatTimeoutMs, + }; +} +// --------------------------------------------------------------------------- +// codexQueueLoadTest +// --------------------------------------------------------------------------- + +export async function codexQueueLoadTest(req: Request): Promise { + const body = req.method === "POST" ? (await req.json().catch(() => ({}))) as Record : {}; + const codexService = microserviceById("code-queue"); + const providerId = typeof body.providerId === "string" && body.providerId.length > 0 + ? body.providerId + : codexService?.providerId ?? "main-server"; + if (!(await providerSupports(providerId, "host.ssh"))) { + return jsonResponse({ ok: true, measurementOk: false, error: `provider does not declare host.ssh capability: ${providerId}`, providerId }, 200); + } + + const timeoutMs = numberFromUnknown(body.timeoutMs, 90_000, 5_000, 180_000); + const targetMs = numberFromUnknown(body.targetMs, 1_000, 100, 60_000); + const runId = safePerfRunId(); + const dir = ".state/code-queue-perf"; + const browsersPath = ".state/playwright-browsers"; + const outputPath = `${dir}/${runId}.json`; + const stderrPath = `${dir}/${runId}.stderr`; + const exitPath = `${dir}/${runId}.exit`; + const urlArg = typeof body.url === "string" && body.url.length > 0 ? ` --url ${shellQuote(body.url)}` : ""; + const chromePath = `${browsersPath}/chromium-1217/chrome-linux64/chrome`; + const playwrightSetup = [ + `if ! test -x ${shellQuote(chromePath)}; then PLAYWRIGHT_BROWSERS_PATH=${shellQuote(browsersPath)} npx playwright install chromium; fi`, + `if test -x ${shellQuote(chromePath)} && ldd ${shellQuote(chromePath)} 2>&1 | grep -q 'not found'; then npx playwright install-deps chromium; fi`, + ].join(" && "); + const startCommand = [ + `mkdir -p ${shellQuote(dir)}`, + `rm -f ${shellQuote(outputPath)} ${shellQuote(stderrPath)} ${shellQuote(exitPath)}`, + `(${playwrightSetup}; PLAYWRIGHT_BROWSERS_PATH=${shellQuote(browsersPath)} bun scripts/src/code-queue-perf.ts --json --timeout-ms ${timeoutMs} --target-ms ${targetMs}${urlArg} > ${shellQuote(outputPath)}; printf '%s' "$?" > ${shellQuote(exitPath)}) > ${shellQuote(stderrPath)} 2>&1 & printf '%s\\n' ${shellQuote(runId)}`, + ].join("; "); + const startedAt = Date.now(); + const start = await runHostSshPerfCommand(providerId, startCommand); + if (!start.ok) { + return jsonResponse({ ok: true, measurementOk: false, providerId, runId, stage: "start", error: start.stderr || "failed to start Playwright benchmark", taskId: start.taskId, task: rawTaskJson(start.task) }, 200); + } + + const pollCommand = [ + `if test -f ${shellQuote(exitPath)}; then`, + "printf '__UNIDESK_CODEX_QUEUE_PERF_READY__\\n';", + `cat ${shellQuote(exitPath)};`, + "printf '\\n__UNIDESK_CODEX_QUEUE_PERF_STDOUT__\\n';", + `cat ${shellQuote(outputPath)} 2>/dev/null || true;`, + "printf '\\n__UNIDESK_CODEX_QUEUE_PERF_STDERR__\\n';", + `tail -c 3000 ${shellQuote(stderrPath)} 2>/dev/null || true;`, + "else printf '__UNIDESK_CODEX_QUEUE_PERF_PENDING__\\n'; fi", + ].join(" "); + let latestPoll: Awaited> | null = null; + while (Date.now() - startedAt < timeoutMs + 20_000) { + await Bun.sleep(1_500); + latestPoll = await runHostSshPerfCommand(providerId, pollCommand); + const parsedPoll = parsePerfPoll(latestPoll.stdout); + if (!parsedPoll.ready) continue; + const result = parsePerfJson(parsedPoll.body); + if (result === null) { + return jsonResponse({ + ok: true, + measurementOk: false, + providerId, + runId, + stage: "parse", + exitCode: parsedPoll.exitCode, + error: "Playwright benchmark did not emit JSON", + stderr: parsedPoll.stderr, + taskId: latestPoll.taskId, + }, 200); + } + return jsonResponse({ + ok: true, + measurementOk: result.ok === true, + providerId, + runId, + taskId: latestPoll.taskId, + elapsedMs: Date.now() - startedAt, + exitCode: parsedPoll.exitCode, + result, + stderr: parsedPoll.stderr, + }); + } + + return jsonResponse({ + ok: true, + measurementOk: false, + providerId, + runId, + stage: "timeout", + elapsedMs: Date.now() - startedAt, + error: `Code Queue Playwright benchmark did not finish within ${timeoutMs}ms`, + latestTaskId: latestPoll?.taskId ?? start.taskId, + latestTask: rawTaskJson(latestPoll?.task ?? start.task), + }, 200); +} diff --git a/src/components/backend-core/src/performance.ts b/src/components/backend-core/src/performance.ts new file mode 100644 index 00000000..ed691cb4 --- /dev/null +++ b/src/components/backend-core/src/performance.ts @@ -0,0 +1,211 @@ +import type { JsonValue } from "../../shared/src/index"; +import { ctx, sql, config, logger } from "./context"; +import type { RequestPerformanceSample, OperationPerformanceSample } from "./types"; + +const maxPerformanceSamples = 3000; + +function trimPerformanceBuffers(): void { + while (ctx.requestPerformanceSamples.length > maxPerformanceSamples) ctx.requestPerformanceSamples.shift(); + while (ctx.operationPerformanceSamples.length > maxPerformanceSamples) ctx.operationPerformanceSamples.shift(); +} + +function classifyRequestComponent(pathname: string): string { + if (pathname.startsWith("/api/microservices/") && pathname.includes("/proxy/")) return "microservice_proxy"; + if (pathname.startsWith("/api/microservices/")) return "microservice_registry"; + if (pathname.startsWith("/api/nodes/")) return "node_metrics_api"; + if (pathname === "/api/nodes") return "node_inventory_api"; + if (pathname.startsWith("/api/tasks")) return "scheduler_api"; + if (pathname.startsWith("/api/schedules")) return "scheduler_api"; + if (pathname.startsWith("/api/events")) return "event_api"; + if (pathname.startsWith("/api/performance")) return "performance_api"; + if (pathname.startsWith("/api/")) return "core_api"; + if (pathname.startsWith("/ws/")) return "websocket_api"; + if (pathname === "/logs") return "logs_api"; + return "core_http"; +} + +export function recordRequestPerformance(req: Request, pathname: string, response: Response | undefined, durationMs: number): void { + const status = response?.status ?? 101; + ctx.requestPerformanceSamples.push({ + at: new Date().toISOString(), + component: classifyRequestComponent(pathname), + method: req.method, + path: pathname, + status, + durationMs, + ok: status < 400, + }); + trimPerformanceBuffers(); +} + +export function recordOperationPerformance(service: string, operation: string, detail: string, durationMs: number, ok: boolean): void { + ctx.operationPerformanceSamples.push({ + at: new Date().toISOString(), + service, + operation, + durationMs, + ok, + detail: detail.length > 260 ? `${detail.slice(0, 257)}...` : detail, + }); + trimPerformanceBuffers(); +} +export async function withPerformanceOperation(service: string, operation: string, detail: string, fn: () => Promise): Promise { + const started = performance.now(); + try { + const result = await fn(); + recordOperationPerformance(service, operation, detail, performance.now() - started, true); + return result; + } catch (error) { + recordOperationPerformance(service, operation, error instanceof Error ? error.message : String(error), performance.now() - started, false); + throw error; + } +} + +function percentile(values: number[], ratio: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1)); + return sorted[index] ?? 0; +} + +function average(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function roundMs(value: number): number { + return Math.round(value * 10) / 10; +} + +function summarizeRequestPerformance(): JsonValue[] { + const groups = new Map(); + for (const sample of ctx.requestPerformanceSamples) { + const rows = groups.get(sample.component) ?? []; + rows.push(sample); + groups.set(sample.component, rows); + } + return Array.from(groups.entries()).map(([component, rows]) => { + const durations = rows.map((row) => row.durationMs); + const failed = rows.filter((row) => !row.ok).length; + return { + component, + requestCount: rows.length, + failureCount: failed, + failureRate: rows.length === 0 ? 0 : failed / rows.length, + averageLatencyMs: roundMs(average(durations)), + p95LatencyMs: roundMs(percentile(durations, 0.95)), + maxLatencyMs: roundMs(Math.max(0, ...durations)), + }; + }).sort((left, right) => Number((right as Record).requestCount ?? 0) - Number((left as Record).requestCount ?? 0)) as JsonValue[]; +} +function summarizeOperationPerformance(): JsonValue[] { + const groups = new Map(); + for (const sample of ctx.operationPerformanceSamples) { + const key = `${sample.service}:${sample.operation}`; + const rows = groups.get(key) ?? []; + rows.push(sample); + groups.set(key, rows); + } + return Array.from(groups.entries()).map(([key, rows]) => { + const [service, ...operationParts] = key.split(":"); + const durations = rows.map((row) => row.durationMs); + const failed = rows.filter((row) => !row.ok).length; + return { + service, + operation: operationParts.join(":"), + count: rows.length, + failureCount: failed, + averageLatencyMs: roundMs(average(durations)), + p95LatencyMs: roundMs(percentile(durations, 0.95)), + maxLatencyMs: roundMs(Math.max(0, ...durations)), + }; + }).sort((left, right) => Number((right as Record).count ?? 0) - Number((left as Record).count ?? 0)) as JsonValue[]; +} + +async function getCodexQueueStoragePerformance(): Promise { + try { + const rows = await sql()>` + SELECT status, count(*)::int AS count + FROM unidesk_codex_queue_tasks + GROUP BY status + ORDER BY status ASC + `; + return { + ok: true, + table: "unidesk_codex_queue_tasks", + counts: Object.fromEntries(rows.map((row) => [row.status, Number(row.count)])) as Record, + total: rows.reduce((sum, row) => sum + Number(row.count), 0), + }; + } catch (error) { + return { + ok: false, + table: "unidesk_codex_queue_tasks", + error: error instanceof Error ? error.message : String(error), + }; + } +} + +async function getPgdataUsage(): Promise { + const rows = await sql()>` + SELECT + current_database()::text AS database_name, + pg_database_size(current_database())::bigint AS database_bytes, + pg_size_pretty(pg_database_size(current_database()))::text AS database_pretty + `; + const row = rows[0]; + return { + volumeName: config().databaseVolumeName, + volumeSize: config().databaseVolumeSize, + databaseName: row?.database_name ?? "unknown", + databaseBytes: Number(row?.database_bytes ?? 0), + databasePretty: row?.database_pretty ?? "--", + }; +} +export async function getPerformance(): Promise { + const memory = process.memoryUsage(); + const pgdata = await withPerformanceOperation("database", "pgdata_usage", "pg_database_size", () => getPgdataUsage()); + const codexQueueStorage = await withPerformanceOperation("database", "codex_queue_storage", "unidesk_codex_queue_tasks", () => getCodexQueueStoragePerformance()); + const recentFailures = ctx.requestPerformanceSamples + .filter((sample) => !sample.ok) + .slice(-20) + .reverse() + .map((sample) => ({ ...sample }) as Record); + const recentOperationCutoff = Date.now() - 10 * 60 * 1000; + const recentSlowOperations = ctx.operationPerformanceSamples + .filter((sample) => { + const at = Date.parse(sample.at); + return Number.isFinite(at) && at >= recentOperationCutoff; + }) + .sort((left, right) => Date.parse(right.at) - Date.parse(left.at)) + .slice(0, 20) + .map((sample) => ({ ...sample }) as Record); + return { + ok: true, + service: "backend-core", + generatedAt: new Date().toISOString(), + startedAt: ctx.serviceStartedAt.toISOString(), + uptimeSeconds: Math.floor((Date.now() - ctx.serviceStartedAt.getTime()) / 1000), + requests: { + sampleCount: ctx.requestPerformanceSamples.length, + componentSummary: summarizeRequestPerformance(), + recentFailures, + }, + operations: { + sampleCount: ctx.operationPerformanceSamples.length, + summary: summarizeOperationPerformance(), + recentSlowOperations, + }, + process: { + rssBytes: memory.rss, + heapUsedBytes: memory.heapUsed, + heapTotalBytes: memory.heapTotal, + externalBytes: memory.external, + arrayBuffersBytes: memory.arrayBuffers, + }, + database: { + ready: ctx.dbReady, + pgdata, + codexQueueStorage, + }, + }; +} diff --git a/src/components/backend-core/src/provider-registry.ts b/src/components/backend-core/src/provider-registry.ts new file mode 100644 index 00000000..35818509 --- /dev/null +++ b/src/components/backend-core/src/provider-registry.ts @@ -0,0 +1,164 @@ +import type { + JsonValue, + ProviderToCoreMessage, +} from "../../shared/src/index"; +import { isProviderToCoreMessage as checkProviderMessage } from "../../shared/src/index"; +import { ctx, sql, logger, config } from "./context"; +import type { ProviderSocket, WsData } from "./types"; +import { errorToJson, wsSendJson } from "./http"; +import { recordEvent, upsertProviderNode, updateProviderHeartbeat, upsertDockerStatus, upsertSystemStatus } from "./db"; +import { notifyTaskTerminal } from "./task-dispatcher"; +import { forwardSshProviderMessage } from "./ssh-bridge"; +import { handleEgressTcpOpen, handleEgressTcpData, handleEgressTcpClose } from "./egress-tcp"; + +function isTerminalTaskStatus(status: string): boolean { + return status === "succeeded" || status === "failed"; +} + +export function parseMessage(raw: string | Buffer): ProviderToCoreMessage { + const text = typeof raw === "string" ? raw : raw.toString("utf8"); + const parsed = JSON.parse(text) as unknown; + if (!checkProviderMessage(parsed)) { + throw new Error(`Unsupported provider message: ${text.slice(0, 200)}`); + } + return parsed; +} + +export async function markProviderOffline(providerId: string): Promise { + ctx.activeProviders.delete(providerId); + if (!ctx.dbReady) return; + await sql()` + UPDATE unidesk_nodes + SET status = 'offline', updated_at = now() + WHERE provider_id = ${providerId} + `; + await recordEvent("provider_offline", providerId, { providerId }); +} + +export async function markStaleProvidersOffline(): Promise { + if (!ctx.dbReady) return; + const timeoutMs = config().heartbeatTimeoutMs; + const rows = await sql()<{ provider_id: string }[]>` + UPDATE unidesk_nodes + SET status = 'offline', updated_at = now() + WHERE status = 'online' + AND last_heartbeat IS NOT NULL + AND last_heartbeat < now() - (${timeoutMs} * interval '1 millisecond') + RETURNING provider_id + `; + for (const row of rows) { + ctx.activeProviders.delete(row.provider_id); + await recordEvent("provider_heartbeat_timeout", row.provider_id, { providerId: row.provider_id, timeoutMs }); + } +} + +export async function providerSupports(providerId: string, capability: string): Promise { + if (!ctx.dbReady) return false; + const rows = await sql()>` + SELECT labels + FROM unidesk_nodes + WHERE provider_id = ${providerId} + LIMIT 1 + `; + const labels = rows[0]?.labels; + if (typeof labels !== "object" || labels === null || Array.isArray(labels)) return false; + const capabilities = (labels as Record).unideskCapabilities; + return Array.isArray(capabilities) ? capabilities.filter((item): item is string => typeof item === "string").includes(capability) : false; +} + +export async function handleProviderMessage(ws: ProviderSocket, raw: string | Buffer): Promise { + const message = parseMessage(raw); + ws.data.providerId = message.providerId; + ctx.activeProviders.set(message.providerId, ws); + + if ( + message.type === "host_ssh_opened" || + message.type === "host_ssh_data" || + message.type === "host_ssh_exit" || + message.type === "host_ssh_error" + ) { + forwardSshProviderMessage(message); + return; + } + + if (message.type === "egress_tcp_open") { + handleEgressTcpOpen(ws, message); + return; + } + if (message.type === "egress_tcp_data") { + handleEgressTcpData(message); + return; + } + if (message.type === "egress_tcp_close") { + handleEgressTcpClose(message); + return; + } + + if (message.type === "register") { + const labels = { ...message.labels, unideskCapabilities: message.capabilities }; + await upsertProviderNode(message.providerId, message.name, labels); + await recordEvent("provider_registered", message.providerId, { + providerId: message.providerId, + name: message.name, + labels, + capabilities: message.capabilities, + }); + ws.send(JSON.stringify({ type: "ack", requestId: "register", ok: true, message: "registered" })); + return; + } + + if (message.type === "heartbeat") { + await updateProviderHeartbeat(message.providerId, message.labels); + logger("debug", "provider_heartbeat", { providerId: message.providerId, labels: message.labels }); + return; + } + + if (message.type === "system_status") { + await upsertSystemStatus(message.providerId, message.status as unknown as JsonValue, message.status.collectedAt); + logger("debug", "provider_system_status", { + providerId: message.providerId, + cpuPercent: message.status.cpu.percent, + memoryPercent: message.status.memory.percent, + diskPercent: message.status.disk.percent, + ok: message.status.ok, + }); + return; + } + + if (message.type === "docker_status") { + await upsertDockerStatus(message.providerId, message.status as unknown as JsonValue, message.status.collectedAt); + logger("debug", "provider_docker_status", { providerId: message.providerId, counts: message.status.counts, ok: message.status.ok }); + return; + } + + await sql()` + WITH incoming AS ( + SELECT ${message.status}::text AS status, ${sql().json(message.result ?? { message: message.message })}::jsonb AS result + ) + UPDATE unidesk_tasks + SET + status = CASE + WHEN unidesk_tasks.status IN ('succeeded', 'failed') AND incoming.status NOT IN ('succeeded', 'failed') THEN unidesk_tasks.status + WHEN unidesk_tasks.status = 'running' AND incoming.status = 'accepted' THEN unidesk_tasks.status + ELSE incoming.status + END, + result = CASE + WHEN unidesk_tasks.status IN ('succeeded', 'failed') AND incoming.status NOT IN ('succeeded', 'failed') THEN unidesk_tasks.result + WHEN unidesk_tasks.status = 'running' AND incoming.status = 'accepted' THEN unidesk_tasks.result + ELSE incoming.result + END, + updated_at = now() + FROM incoming + WHERE id = ${message.taskId} + `; + await recordEvent("task_status", message.providerId, { + providerId: message.providerId, + taskId: message.taskId, + status: message.status, + message: message.message, + result: message.result ?? null, + }); + if (isTerminalTaskStatus(message.status)) { + await notifyTaskTerminal(message.taskId); + } +} diff --git a/src/components/backend-core/src/scheduler.ts b/src/components/backend-core/src/scheduler.ts new file mode 100644 index 00000000..664c835d --- /dev/null +++ b/src/components/backend-core/src/scheduler.ts @@ -0,0 +1,752 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { mkdir, rm, stat } from "node:fs/promises"; +import { dirname, relative, resolve, posix as pathPosix } from "node:path"; +import type { JsonValue, CoreDispatchMessage } from "../../shared/src/index"; +import { isProviderDispatchCommand } from "../../shared/src/index"; +import { ctx, sql, config, logger } from "./context"; +import type { + ScheduledTaskRow, + ScheduledTaskRunRow, + ScheduleSpec, + ScheduleAction, + DispatchScheduleAction, + PgdataBackupScheduleAction, + RawTaskRow, +} from "./types"; +import { jsonResponse, compactJson, errorToJson, isJsonRecord, rowIso, truncateText } from "./http"; +import { createAndSendTask, waitForTaskTerminal, rawTask } from "./task-dispatcher"; +import { recordEvent } from "./db"; + +/* ─── helpers ─────────────────────────────────────────────────────────── */ + +function readLimit(url: URL, defaultLimit: number): number { + const raw = url.searchParams.get("limit"); + if (raw === null) return defaultLimit; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) return defaultLimit; + return Math.min(parsed, 500); +} + +function isTerminalTaskStatus(status: string): boolean { + return status === "succeeded" || status === "failed"; +} + +/* ─── views ───────────────────────────────────────────────────────────── */ + +function scheduleRunView(row: ScheduledTaskRunRow): JsonValue { + return { + id: row.id, + scheduleId: row.schedule_id, + triggerType: row.trigger_type, + status: row.status, + taskId: row.task_id, + result: row.result, + error: row.error, + startedAt: rowIso(row.started_at), + finishedAt: rowIso(row.finished_at), + durationMs: row.duration_ms === null ? null : Number(row.duration_ms), + createdAt: rowIso(row.created_at), + updatedAt: rowIso(row.updated_at), + }; +} + +function scheduledTaskView(row: ScheduledTaskRow, runs: ScheduledTaskRunRow[] = []): JsonValue { + return { + id: row.id, + name: row.name, + description: row.description, + enabled: row.enabled, + schedule: row.schedule_json, + action: row.action_json, + concurrencyPolicy: row.concurrency_policy, + nextRunAt: rowIso(row.next_run_at), + lastRunAt: rowIso(row.last_run_at), + lastRunId: row.last_run_id, + createdAt: rowIso(row.created_at), + updatedAt: rowIso(row.updated_at), + recentRuns: runs.map(scheduleRunView), + }; +} + +function scheduledRawTaskJson(task: RawTaskRow | null): JsonValue { + if (task === null) return null; + return { + id: task.id, + providerId: task.provider_id, + command: task.command, + status: task.status, + payload: compactJson(task.payload), + result: compactJson(task.result ?? null), + updatedAt: rowIso(task.updated_at), + }; +} + +/* ─── normalization ───────────────────────────────────────────────────── */ + +function normalizeScheduleId(value: unknown): string { + const raw = typeof value === "string" && value.trim().length > 0 + ? value.trim() + : `schedule_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; + if (!/^[A-Za-z0-9_.:-]{1,120}$/u.test(raw)) { + throw new Error("schedule id must be 1-120 chars using letters, numbers, _, ., :, or -"); + } + return raw; +} + +function normalizeTimeOfDay(value: unknown): string { + const raw = typeof value === "string" && value.trim().length > 0 ? value.trim() : "03:00"; + const match = raw.match(/^([01]\d|2[0-3]):([0-5]\d)$/u); + if (match === null) throw new Error("daily schedule timeOfDay must use HH:MM in UTC"); + return `${match[1]}:${match[2]}`; +} + +function numberInRange(value: unknown, fallback: number, min: number, max: number): number { + const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : fallback; + if (!Number.isFinite(parsed)) return fallback; + return Math.max(min, Math.min(max, Math.floor(parsed))); +} + +function normalizeScheduleSpec(value: unknown): ScheduleSpec { + const record = isJsonRecord(value) ? value : {}; + if (record.type === "interval") { + return { + type: "interval", + everySeconds: numberInRange(record.everySeconds, 3600, 60, 366 * 24 * 3600), + }; + } + return { + type: "daily", + timeOfDay: normalizeTimeOfDay(record.timeOfDay), + timezone: typeof record.timezone === "string" && record.timezone.length > 0 ? record.timezone : "Etc/UTC", + }; +} + +function normalizeScheduleAction(value: unknown): ScheduleAction { + if (!isJsonRecord(value)) throw new Error("scheduled task action must be an object"); + if (value.type === "dispatch") { + const providerId = typeof value.providerId === "string" ? value.providerId.trim() : ""; + if (providerId.length === 0) throw new Error("dispatch action providerId is required"); + if (!isProviderDispatchCommand(value.command)) throw new Error("dispatch action command is invalid"); + const payload = isJsonRecord(value.payload) ? value.payload : {}; + const timeoutMs = value.timeoutMs === undefined ? undefined : numberInRange(value.timeoutMs, config().taskPendingTimeoutMs, 1_000, 24 * 3600_000); + return { type: "dispatch", providerId, command: value.command, payload, ...(timeoutMs === undefined ? {} : { timeoutMs }) }; + } + if (value.type === "pgdata_backup") { + return { + type: "pgdata_backup", + volumeName: typeof value.volumeName === "string" && value.volumeName.length > 0 ? value.volumeName : config().databaseVolumeName, + remoteBaseDir: normalizeRemoteDir(typeof value.remoteBaseDir === "string" ? value.remoteBaseDir : "/SERVER_DATA/UNIDESK_PG_DATA"), + stagingSubdir: normalizeRelativeStagingPath(typeof value.stagingSubdir === "string" ? value.stagingSubdir : "server-data/unidesk-pg-data").relativePath, + baiduBaseUrl: typeof value.baiduBaseUrl === "string" && value.baiduBaseUrl.length > 0 ? value.baiduBaseUrl : config().baiduNetdiskInternalUrl, + timeoutMs: numberInRange(value.timeoutMs, 60 * 60_000, 60_000, 24 * 3600_000), + cleanupLocal: value.cleanupLocal !== false, + }; + } + throw new Error("scheduled task action.type must be dispatch or pgdata_backup"); +} + +function computeNextRunAt(schedule: ScheduleSpec, after = new Date()): Date { + if (schedule.type === "interval") { + return new Date(after.getTime() + Math.max(60, schedule.everySeconds ?? 3600) * 1000); + } + const [hourText = "03", minuteText = "00"] = (schedule.timeOfDay ?? "03:00").split(":"); + const candidate = new Date(Date.UTC(after.getUTCFullYear(), after.getUTCMonth(), after.getUTCDate(), Number(hourText), Number(minuteText), 0, 0)); + if (candidate.getTime() <= after.getTime()) candidate.setUTCDate(candidate.getUTCDate() + 1); + return candidate; +} + +/* ─── path helpers ────────────────────────────────────────────────────── */ + +function normalizeRemoteDir(input: string): string { + const raw = input.trim() || "/"; + if (raw.includes("\0")) throw new Error("remote directory must not contain null bytes"); + const normalized = pathPosix.normalize(raw.startsWith("/") ? raw : `/${raw}`); + return normalized === "/" ? "/" : normalized.replace(/\/+$/u, ""); +} + +function normalizeRelativeStagingPath(input: string): { relativePath: string; absolutePath: string } { + const cleaned = input.replace(/\\/g, "/").replace(/^\/+/u, "").trim(); + const normalized = pathPosix.normalize(cleaned || "."); + if (normalized === "." || normalized.startsWith("../") || normalized === "..") throw new Error("staging path must be a relative child path"); + const absolutePath = resolve(config().pgdataBackupStagingDir, normalized); + const rel = relative(config().pgdataBackupStagingDir, absolutePath); + if (rel.startsWith("..") || rel.includes("\0") || resolve(absolutePath) === resolve(config().pgdataBackupStagingDir)) { + throw new Error("staging path must stay inside PGDATA_BACKUP_STAGING_DIR"); + } + return { relativePath: normalized, absolutePath }; +} + +function remoteFolderChain(remoteDir: string): string[] { + const parts = normalizeRemoteDir(remoteDir).split("/").filter(Boolean); + const folders: string[] = []; + let current = ""; + for (const part of parts) { + current = `${current}/${part}`; + folders.push(current); + } + return folders; +} + +function safeFilePart(value: string): string { + return value.replace(/[^A-Za-z0-9_.-]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 80) || "data"; +} + +function utcBackupParts(date = new Date()): { date: string; time: string; month: string; stamp: string } { + const year = String(date.getUTCFullYear()).padStart(4, "0"); + const monthNumber = String(date.getUTCMonth() + 1).padStart(2, "0"); + const day = String(date.getUTCDate()).padStart(2, "0"); + const hour = String(date.getUTCHours()).padStart(2, "0"); + const minute = String(date.getUTCMinutes()).padStart(2, "0"); + const second = String(date.getUTCSeconds()).padStart(2, "0"); + return { date: `${year}${monthNumber}${day}`, time: `${hour}${minute}${second}`, month: `${year}${monthNumber}`, stamp: `${year}${monthNumber}${day}_${hour}${minute}${second}` }; +} + +function databaseConnectionParts(): { host: string; port: string; user: string; password: string } { + const url = new URL(config().databaseUrl); + return { + host: url.hostname || "database", + port: url.port || "5432", + user: decodeURIComponent(url.username || "postgres"), + password: decodeURIComponent(url.password || ""), + }; +} + +/* ─── local command / file utilities ──────────────────────────────────── */ + +async function runLocalCommand( + command: string, + args: string[], + options: { cwd?: string; env?: Record; timeoutMs: number }, +): Promise<{ ok: boolean; stdout: string; stderr: string; exitCode: number | null; timedOut: boolean }> { + const proc = Bun.spawn([command, ...args], { + cwd: options.cwd, + env: { ...process.env, ...(options.env ?? {}) }, + stdout: "pipe", + stderr: "pipe", + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill("SIGTERM"); + }, Math.max(1, options.timeoutMs)); + try { + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { ok: exitCode === 0 && !timedOut, stdout: truncateText(stdout, 4000), stderr: truncateText(stderr, 4000), exitCode, timedOut }; + } finally { + clearTimeout(timer); + } +} + +async function sha256File(filePath: string): Promise { + const hash = createHash("sha256"); + await new Promise((resolveHash, rejectHash) => { + const stream = createReadStream(filePath); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("error", rejectHash); + stream.on("end", resolveHash); + }); + return hash.digest("hex"); +} + +async function fetchJsonWithTimeout(url: string, init: RequestInit, timeoutMs: number): Promise> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutMs)); + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + const text = await response.text(); + let parsed: unknown = {}; + try { + parsed = text.length > 0 ? JSON.parse(text) as unknown : {}; + } catch { + parsed = { text }; + } + if (!response.ok) throw new Error(`HTTP ${response.status}: ${truncateText(text, 1000)}`); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { value: parsed }; + return parsed as Record; + } finally { + clearTimeout(timer); + } +} + +async function baiduJson(baseUrl: string, path: string, init: RequestInit = {}, timeoutMs = 30_000): Promise> { + const url = new URL(path, baseUrl); + const headers = new Headers(init.headers); + if (init.body !== undefined && !headers.has("content-type")) headers.set("content-type", "application/json"); + const body = await fetchJsonWithTimeout(url.toString(), { ...init, headers }, timeoutMs); + if (body.ok === false) throw new Error(`Baidu Netdisk API failed: ${JSON.stringify(compactJson(body))}`); + return body; +} + +function jobRecord(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +async function waitForBaiduTransfer(baseUrl: string, jobId: string, timeoutMs: number): Promise> { + const deadline = Date.now() + timeoutMs; + let latest: Record = {}; + while (Date.now() < deadline) { + const detail = await baiduJson(baseUrl, `/api/transfers/${encodeURIComponent(jobId)}`, {}, 30_000); + latest = detail; + const job = jobRecord(detail.job); + const status = String(job.status || ""); + if (status === "succeeded") return detail; + if (status === "failed" || status === "canceled") throw new Error(`Baidu transfer ${status}: ${String(job.error || "no error detail")}`); + await Bun.sleep(2000); + } + throw new Error(`Baidu transfer timed out after ${timeoutMs}ms: ${JSON.stringify(compactJson(latest))}`); +} + +/* ─── action execution ────────────────────────────────────────────────── */ + +async function executeDispatchScheduleAction(action: DispatchScheduleAction): Promise<{ ok: boolean; taskId: string; result: JsonValue }> { + const { taskId, providerOnline } = await createAndSendTask(action.providerId, action.command, { + source: "scheduled-task", + ...action.payload, + }); + if (!providerOnline) { + return { ok: false, taskId, result: { providerOnline, taskId, error: `provider is offline: ${action.providerId}` } }; + } + const timeoutMs = action.timeoutMs ?? numberInRange(action.payload.timeoutMs, config().taskPendingTimeoutMs + 5000, 1000, 24 * 3600_000); + const task = await waitForTaskTerminal(taskId, timeoutMs); + const ok = task?.status === "succeeded"; + return { + ok, + taskId, + result: { + providerOnline, + taskId, + timeoutMs, + terminal: task === null ? false : isTerminalTaskStatus(task.status), + task: scheduledRawTaskJson(task), + }, + }; +} + +async function executePgdataBackupAction(action: PgdataBackupScheduleAction, runId: string): Promise<{ ok: boolean; result: JsonValue }> { + const startedAt = new Date(); + const parts = utcBackupParts(startedAt); + const filename = `${parts.stamp}_${safeFilePart(action.volumeName)}.pg_basebackup.tar.gz`; + const monthRelativeDir = pathPosix.join(action.stagingSubdir, parts.month); + const backupRelativePath = pathPosix.join(monthRelativeDir, filename); + const backupPath = normalizeRelativeStagingPath(backupRelativePath).absolutePath; + const tempRelativeDir = pathPosix.join(action.stagingSubdir, parts.month, `.tmp_${runId}`); + const tempDir = normalizeRelativeStagingPath(tempRelativeDir).absolutePath; + const remoteDir = normalizeRemoteDir(pathPosix.join(action.remoteBaseDir, parts.month)); + const remotePath = normalizeRemoteDir(pathPosix.join(remoteDir, filename)); + const db = databaseConnectionParts(); + await mkdir(dirname(backupPath), { recursive: true }); + await rm(tempDir, { recursive: true, force: true }); + await mkdir(tempDir, { recursive: true }); + let uploadDetail: Record | null = null; + let backupBytes = 0; + let backupSha256 = ""; + try { + const backupTimeoutMs = Math.max(60_000, Math.floor(action.timeoutMs * 0.55)); + const packageTimeoutMs = Math.max(60_000, Math.floor(action.timeoutMs * 0.15)); + const uploadTimeoutMs = Math.max(60_000, action.timeoutMs - backupTimeoutMs - packageTimeoutMs); + const basebackup = await runLocalCommand("pg_basebackup", [ + "-h", db.host, + "-p", db.port, + "-U", db.user, + "-D", tempDir, + "-Ft", + "-X", "stream", + "-z", + "--checkpoint=fast", + "--no-sync", + ], { env: { PGPASSWORD: db.password }, timeoutMs: backupTimeoutMs }); + if (!basebackup.ok) throw new Error(`pg_basebackup failed exit=${basebackup.exitCode} timedOut=${basebackup.timedOut}: ${basebackup.stderr || basebackup.stdout}`); + const packaged = await runLocalCommand("tar", ["-czf", backupPath, "-C", tempDir, "."], { timeoutMs: packageTimeoutMs }); + if (!packaged.ok) throw new Error(`tar packaging failed exit=${packaged.exitCode} timedOut=${packaged.timedOut}: ${packaged.stderr || packaged.stdout}`); + const info = await stat(backupPath); + backupBytes = info.size; + backupSha256 = await sha256File(backupPath); + for (const folder of remoteFolderChain(remoteDir)) { + await baiduJson(action.baiduBaseUrl, "/api/folders", { + method: "POST", + body: JSON.stringify({ path: folder }), + }, 60_000); + } + const created = await baiduJson(action.baiduBaseUrl, "/api/transfers/upload-from-path", { + method: "POST", + body: JSON.stringify({ localPath: backupRelativePath, remotePath }), + }, 60_000); + const jobId = String(jobRecord(created.job).id || ""); + if (jobId.length === 0) throw new Error(`Baidu upload did not return a job id: ${JSON.stringify(compactJson(created))}`); + uploadDetail = await waitForBaiduTransfer(action.baiduBaseUrl, jobId, uploadTimeoutMs); + const uploadedJob = jobRecord(uploadDetail.job); + return { + ok: true, + result: { + backupType: "pg_basebackup", + volumeName: action.volumeName, + backupBytes, + backupSha256, + localRelativePath: backupRelativePath, + remotePath, + remoteDir, + month: parts.month, + timestampPrefix: parts.stamp, + baiduTransferJobId: jobId, + baiduTransferStatus: String(uploadedJob.status || ""), + baiduFsId: String(uploadedJob.fsId || ""), + baiduResult: compactJson(uploadedJob.result ?? null), + }, + }; + } finally { + await rm(tempDir, { recursive: true, force: true }).catch(() => undefined); + if (action.cleanupLocal) await rm(backupPath, { force: true }).catch(() => undefined); + } +} + +async function executeScheduleAction(action: ScheduleAction, runId: string): Promise<{ ok: boolean; taskId: string | null; result: JsonValue }> { + if (action.type === "dispatch") { + const dispatched = await executeDispatchScheduleAction(action); + return { ok: dispatched.ok, taskId: dispatched.taskId, result: dispatched.result }; + } + const backup = await executePgdataBackupAction(action, runId); + return { ok: backup.ok, taskId: null, result: backup.result }; +} + +/* ─── scheduled run execution ─────────────────────────────────────────── */ + +async function executeScheduledRun(runId: string): Promise { + if (ctx.activeScheduledRuns.has(runId)) return; + ctx.activeScheduledRuns.add(runId); + const started = Date.now(); + try { + const queuedRuns = await sql()` + SELECT * + FROM unidesk_scheduled_task_runs + WHERE id = ${runId} + LIMIT 1 + `; + const queuedRun = queuedRuns[0]; + if (queuedRun === undefined) return; + const scheduleRow = await getScheduledTaskRow(queuedRun.schedule_id); + if (scheduleRow === null) return; + const runRows = await sql()` + UPDATE unidesk_scheduled_task_runs + SET status = 'running', started_at = now(), updated_at = now() + WHERE id = ${runId} AND status = 'queued' + RETURNING * + `; + if (runRows.length === 0) return; + try { + const action = normalizeScheduleAction(scheduleRow.action_json); + const outcome = await executeScheduleAction(action, runId); + const durationMs = Date.now() - started; + const status = outcome.ok ? "succeeded" : "failed"; + const error = outcome.ok ? null : "scheduled action failed"; + await sql().begin(async (tx) => { + await tx` + UPDATE unidesk_scheduled_task_runs + SET status = ${status}, + task_id = ${outcome.taskId}, + result = ${tx.json(outcome.result)}, + error = ${error}, + finished_at = now(), + duration_ms = ${durationMs}, + updated_at = now() + WHERE id = ${runId} + `; + await tx` + UPDATE unidesk_scheduled_tasks + SET last_run_at = now(), last_run_id = ${runId}, updated_at = now() + WHERE id = ${scheduleRow.id} + `; + }); + await recordEvent(`scheduled_task_${status}`, "scheduler", { scheduleId: scheduleRow.id, runId, durationMs, result: compactJson(outcome.result) }); + } catch (error) { + const durationMs = Date.now() - started; + await sql().begin(async (tx) => { + await tx` + UPDATE unidesk_scheduled_task_runs + SET status = 'failed', + result = ${tx.json(errorToJson(error))}, + error = ${error instanceof Error ? error.message : String(error)}, + finished_at = now(), + duration_ms = ${durationMs}, + updated_at = now() + WHERE id = ${runId} + `; + await tx` + UPDATE unidesk_scheduled_tasks + SET last_run_at = now(), last_run_id = ${runId}, updated_at = now() + WHERE id = ${scheduleRow.id} + `; + }); + await recordEvent("scheduled_task_failed", "scheduler", { scheduleId: scheduleRow.id, runId, durationMs, error: errorToJson(error) }); + } + } finally { + ctx.activeScheduledRuns.delete(runId); + } +} + +/* ─── CRUD ────────────────────────────────────────────────────────────── */ + +async function getScheduledTasks(limit: number): Promise { + const rows = await sql()` + SELECT * + FROM unidesk_scheduled_tasks + ORDER BY enabled DESC, next_run_at ASC NULLS LAST, updated_at DESC + LIMIT ${limit} + `; + const runRows = await sql()` + SELECT * + FROM unidesk_scheduled_task_runs + ORDER BY updated_at DESC + LIMIT ${Math.max(20, limit * 5)} + `; + const runsBySchedule = new Map(); + for (const run of runRows) { + const list = runsBySchedule.get(run.schedule_id) ?? []; + if (list.length < 5) list.push(run); + runsBySchedule.set(run.schedule_id, list); + } + return rows.map((row) => scheduledTaskView(row, runsBySchedule.get(row.id) ?? [])); +} + +async function getScheduledTaskRuns(scheduleId: string | null, limit: number): Promise { + const rows = scheduleId === null + ? await sql()` + SELECT * + FROM unidesk_scheduled_task_runs + ORDER BY updated_at DESC + LIMIT ${limit} + ` + : await sql()` + SELECT * + FROM unidesk_scheduled_task_runs + WHERE schedule_id = ${scheduleId} + ORDER BY updated_at DESC + LIMIT ${limit} + `; + return rows.map(scheduleRunView); +} + +async function getScheduledTaskRow(scheduleId: string): Promise { + const rows = await sql()` + SELECT * + FROM unidesk_scheduled_tasks + WHERE id = ${scheduleId} + LIMIT 1 + `; + return rows[0] ?? null; +} + +async function upsertScheduledTask(req: Request, scheduleIdFromPath: string | null): Promise { + const body = await req.json().catch(() => ({})) as Record; + const id = normalizeScheduleId(scheduleIdFromPath ?? body.id); + const schedule = normalizeScheduleSpec(body.schedule); + const action = normalizeScheduleAction(body.action); + const name = typeof body.name === "string" && body.name.trim().length > 0 ? body.name.trim() : id; + const description = typeof body.description === "string" ? body.description : ""; + const enabled = typeof body.enabled === "boolean" ? body.enabled : true; + const concurrencyPolicy = body.concurrencyPolicy === "parallel" ? "parallel" : "skip"; + const existing = await getScheduledTaskRow(id); + const nextRunAt = existing?.next_run_at && JSON.stringify(existing.schedule_json) === JSON.stringify(schedule) + ? rowIso(existing.next_run_at) + : computeNextRunAt(schedule).toISOString(); + const scheduleJson = schedule as unknown as JsonValue; + const actionJson = action as unknown as JsonValue; + const rows = await sql()` + INSERT INTO unidesk_scheduled_tasks (id, name, description, enabled, schedule_json, action_json, concurrency_policy, next_run_at, updated_at) + VALUES (${id}, ${name}, ${description}, ${enabled}, ${sql().json(scheduleJson)}, ${sql().json(actionJson)}, ${concurrencyPolicy}, ${nextRunAt}, now()) + ON CONFLICT (id) DO UPDATE SET + name = EXCLUDED.name, + description = EXCLUDED.description, + enabled = EXCLUDED.enabled, + schedule_json = EXCLUDED.schedule_json, + action_json = EXCLUDED.action_json, + concurrency_policy = EXCLUDED.concurrency_policy, + next_run_at = EXCLUDED.next_run_at, + updated_at = now() + RETURNING * + `; + await recordEvent(existing === null ? "scheduled_task_created" : "scheduled_task_updated", "scheduler", { scheduleId: id, name, enabled, schedule: scheduleJson, action: compactJson(action) }); + return jsonResponse({ ok: true, schedule: scheduledTaskView(rows[0]) }, existing === null ? 201 : 200); +} + +async function patchScheduledTask(scheduleId: string, req: Request): Promise { + const current = await getScheduledTaskRow(scheduleId); + if (current === null) return jsonResponse({ ok: false, error: `scheduled task not found: ${scheduleId}` }, 404); + const body = await req.json().catch(() => ({})) as Record; + const schedule = body.schedule === undefined ? normalizeScheduleSpec(current.schedule_json) : normalizeScheduleSpec(body.schedule); + const action = body.action === undefined ? normalizeScheduleAction(current.action_json) : normalizeScheduleAction(body.action); + const name = typeof body.name === "string" && body.name.trim().length > 0 ? body.name.trim() : current.name; + const description = typeof body.description === "string" ? body.description : current.description; + const enabled = typeof body.enabled === "boolean" ? body.enabled : current.enabled; + const concurrencyPolicy = body.concurrencyPolicy === "parallel" || body.concurrencyPolicy === "skip" ? body.concurrencyPolicy : current.concurrency_policy; + const scheduleChanged = body.schedule !== undefined && JSON.stringify(schedule) !== JSON.stringify(current.schedule_json); + const nextRunAt = scheduleChanged || current.next_run_at === null ? computeNextRunAt(schedule).toISOString() : rowIso(current.next_run_at); + const scheduleJson = schedule as unknown as JsonValue; + const actionJson = action as unknown as JsonValue; + const rows = await sql()` + UPDATE unidesk_scheduled_tasks + SET name = ${name}, + description = ${description}, + enabled = ${enabled}, + schedule_json = ${sql().json(scheduleJson)}, + action_json = ${sql().json(actionJson)}, + concurrency_policy = ${concurrencyPolicy}, + next_run_at = ${nextRunAt}, + updated_at = now() + WHERE id = ${scheduleId} + RETURNING * + `; + await recordEvent("scheduled_task_updated", "scheduler", { scheduleId, name, enabled, schedule: scheduleJson, action: compactJson(action) }); + return jsonResponse({ ok: true, schedule: scheduledTaskView(rows[0]) }); +} + +async function deleteScheduledTask(scheduleId: string): Promise { + const rows = await sql()>` + DELETE FROM unidesk_scheduled_tasks + WHERE id = ${scheduleId} + RETURNING id + `; + if (rows.length === 0) return jsonResponse({ ok: false, error: `scheduled task not found: ${scheduleId}` }, 404); + await recordEvent("scheduled_task_deleted", "scheduler", { scheduleId }); + return jsonResponse({ ok: true, deleted: scheduleId }); +} + +async function triggerScheduledTask(scheduleId: string, triggerType: "manual" | "schedule"): Promise { + const runId = `schedrun_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; + let createdRun: ScheduledTaskRunRow | null = null; + await sql().begin(async (tx) => { + const rows = await tx` + SELECT * + FROM unidesk_scheduled_tasks + WHERE id = ${scheduleId} + FOR UPDATE + `; + const scheduleRow = rows[0]; + if (scheduleRow === undefined) throw new Error(`scheduled task not found: ${scheduleId}`); + if (triggerType === "schedule" && !scheduleRow.enabled) return; + const runningRows = await tx>` + SELECT count(*)::int AS count + FROM unidesk_scheduled_task_runs + WHERE schedule_id = ${scheduleId} + AND status IN ('queued', 'running') + `; + const runningCount = Number(runningRows[0]?.count ?? 0); + const schedule = normalizeScheduleSpec(scheduleRow.schedule_json); + const shouldAdvanceNext = triggerType === "schedule"; + if (shouldAdvanceNext) { + await tx` + UPDATE unidesk_scheduled_tasks + SET next_run_at = ${computeNextRunAt(schedule).toISOString()}, updated_at = now() + WHERE id = ${scheduleId} + `; + } + if (scheduleRow.concurrency_policy !== "parallel" && runningCount > 0) { + const skippedRows = await tx` + INSERT INTO unidesk_scheduled_task_runs (id, schedule_id, trigger_type, status, result, error, finished_at, duration_ms) + VALUES (${runId}, ${scheduleId}, ${triggerType}, 'skipped', ${tx.json({ reason: "previous run still active", runningCount })}, 'previous run still active', now(), 0) + RETURNING * + `; + createdRun = skippedRows[0] ?? null; + return; + } + const runRows = await tx` + INSERT INTO unidesk_scheduled_task_runs (id, schedule_id, trigger_type, status) + VALUES (${runId}, ${scheduleId}, ${triggerType}, 'queued') + RETURNING * + `; + createdRun = runRows[0] ?? null; + }); + const run = createdRun as ScheduledTaskRunRow | null; + if (run === null) return null; + if (run.status === "queued") { + setTimeout(() => { + executeScheduledRun(runId).catch((error) => logger("error", "scheduled_run_uncaught", { runId, error: errorToJson(error) })); + }, 0); + } + await recordEvent("scheduled_task_triggered", "scheduler", { scheduleId, runId, triggerType, status: run.status }); + return scheduleRunView(run); +} + +export async function scheduledTaskRoute(req: Request, url: URL): Promise { + const prefix = "/api/schedules"; + const rest = url.pathname === prefix ? "" : url.pathname.slice(prefix.length + 1); + const segments = rest.split("/").filter(Boolean).map((part) => decodeURIComponent(part)); + if (url.pathname === prefix && req.method === "GET") { + return jsonResponse({ ok: true, schedules: await getScheduledTasks(readLimit(url, 100)) }); + } + if (url.pathname === prefix && req.method === "POST") return upsertScheduledTask(req, null); + if (segments.length === 1 && segments[0] === "runs" && req.method === "GET") { + return jsonResponse({ ok: true, runs: await getScheduledTaskRuns(null, readLimit(url, 100)) }); + } + if (segments.length === 1 && req.method === "GET") { + const schedule = await getScheduledTaskRow(segments[0]); + if (schedule === null) return jsonResponse({ ok: false, error: `scheduled task not found: ${segments[0]}` }, 404); + const runs = await sql()` + SELECT * + FROM unidesk_scheduled_task_runs + WHERE schedule_id = ${segments[0]} + ORDER BY updated_at DESC + LIMIT 20 + `; + return jsonResponse({ ok: true, schedule: scheduledTaskView(schedule, runs) }); + } + if (segments.length === 1 && req.method === "PUT") return upsertScheduledTask(req, segments[0]); + if (segments.length === 1 && req.method === "PATCH") return patchScheduledTask(segments[0], req); + if (segments.length === 1 && req.method === "DELETE") return deleteScheduledTask(segments[0]); + if (segments.length === 2 && segments[1] === "run" && req.method === "POST") { + const run = await triggerScheduledTask(segments[0], "manual"); + if (run === null) return jsonResponse({ ok: false, error: `scheduled task was not triggered: ${segments[0]}` }, 409); + return jsonResponse({ ok: true, run }); + } + if (segments.length === 2 && segments[1] === "runs" && req.method === "GET") { + return jsonResponse({ ok: true, runs: await getScheduledTaskRuns(segments[0], readLimit(url, 100)) }); + } + return jsonResponse({ ok: false, error: "scheduled task route not found", path: url.pathname }, 404); +} + +export async function recoverScheduledRuns(): Promise { + const rows = await sql()` + UPDATE unidesk_scheduled_task_runs + SET status = 'failed', + error = 'backend-core restarted before scheduled run completed', + result = jsonb_build_object('error', 'backend-core restarted before scheduled run completed'), + finished_at = now(), + updated_at = now() + WHERE status IN ('queued', 'running') + RETURNING * + `; + if (rows.length > 0) await recordEvent("scheduled_runs_recovered", "scheduler", { failedRunCount: rows.length }); + const schedules = await sql()` + SELECT * + FROM unidesk_scheduled_tasks + WHERE next_run_at IS NULL + LIMIT 200 + `; + for (const schedule of schedules) { + const nextRunAt = computeNextRunAt(normalizeScheduleSpec(schedule.schedule_json)).toISOString(); + await sql()`UPDATE unidesk_scheduled_tasks SET next_run_at = ${nextRunAt}, updated_at = now() WHERE id = ${schedule.id}`; + } +} + +export async function runDueScheduledTasks(): Promise { + if (!ctx.dbReady) return; + const rows = await sql()` + SELECT * + FROM unidesk_scheduled_tasks + WHERE enabled = true + AND (next_run_at IS NULL OR next_run_at <= now()) + ORDER BY next_run_at ASC NULLS FIRST + LIMIT 5 + `; + for (const row of rows) { + try { + await triggerScheduledTask(row.id, "schedule"); + } catch (error) { + logger("error", "scheduled_task_due_trigger_failed", { scheduleId: row.id, error: errorToJson(error) }); + } + } +} diff --git a/src/components/backend-core/src/ssh-bridge.ts b/src/components/backend-core/src/ssh-bridge.ts new file mode 100644 index 00000000..db84ec21 --- /dev/null +++ b/src/components/backend-core/src/ssh-bridge.ts @@ -0,0 +1,174 @@ +import type { Server } from "bun"; +import type { + CoreHostSshCloseMessage, + CoreHostSshEofMessage, + CoreHostSshInputMessage, + CoreHostSshOpenMessage, + CoreHostSshResizeMessage, + ProviderHostSshDataMessage, + ProviderHostSshErrorMessage, + ProviderHostSshExitMessage, + ProviderHostSshOpenedMessage, +} from "../../shared/src/index"; +import { ctx, sql, config, logger } from "./context"; +import type { ProviderSocket, WsData } from "./types"; +import { jsonResponse, wsSendJson } from "./http"; + +function safeSessionId(): string { + return `ssh_${Date.now()}_${Math.random().toString(16).slice(2)}`; +} + +function sshClientFor(sessionId: string): ProviderSocket | null { + return ctx.activeSshClients.get(sessionId) ?? null; +} + +function providerForSsh(providerId: string): ProviderSocket | null { + return ctx.activeProviders.get(providerId) ?? null; +} + +export function closeSshClient(sessionId: string, code = 1000, reason = "ssh session closed"): void { + const client = sshClientFor(sessionId); + ctx.activeSshClients.delete(sessionId); + if (client !== null && client.readyState === WebSocket.OPEN) client.close(code, reason); +} + +export function forwardSshProviderMessage( + message: ProviderHostSshOpenedMessage | ProviderHostSshDataMessage | ProviderHostSshExitMessage | ProviderHostSshErrorMessage, +): void { + const client = sshClientFor(message.sessionId); + if (client === null) { + logger("warn", "ssh_client_missing", { providerId: message.providerId, sessionId: message.sessionId, type: message.type }); + return; + } + if (message.type === "host_ssh_opened") { + wsSendJson(client, { type: "ssh.opened", providerId: message.providerId, sessionId: message.sessionId }); + return; + } + if (message.type === "host_ssh_data") { + wsSendJson(client, { type: "ssh.data", stream: message.stream, data: message.data, encoding: message.encoding }); + return; + } + if (message.type === "host_ssh_exit") { + wsSendJson(client, { type: "ssh.exit", exitCode: message.exitCode, signal: message.signal }); + setTimeout(() => closeSshClient(message.sessionId), 50); + return; + } + wsSendJson(client, { type: "ssh.error", message: message.message }); + setTimeout(() => closeSshClient(message.sessionId, 1011, "ssh session error"), 50); +} + +function numberFromUnknown(value: unknown, fallback: number, min: number, max: number): number { + const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : fallback; + if (!Number.isFinite(parsed)) return fallback; + return Math.max(min, Math.min(max, Math.floor(parsed))); +} + +async function providerSupports(providerId: string, capability: string): Promise { + if (!ctx.dbReady) return false; + const rows = await sql()>` + SELECT labels + FROM unidesk_nodes + WHERE provider_id = ${providerId} + LIMIT 1 + `; + const labels = rows[0]?.labels; + if (typeof labels !== "object" || labels === null || Array.isArray(labels)) return false; + const capabilities = (labels as Record).unideskCapabilities; + return Array.isArray(capabilities) ? capabilities.filter((item): item is string => typeof item === "string").includes(capability) : false; +} + +export async function handleSshClientMessage(ws: ProviderSocket, raw: string | Buffer): Promise { + const text = typeof raw === "string" ? raw : raw.toString("utf8"); + const message = JSON.parse(text) as { type?: unknown; providerId?: unknown; cwd?: unknown; command?: unknown; tty?: unknown; data?: unknown; encoding?: unknown; cols?: unknown; rows?: unknown }; + if (message.type === "ssh.open") { + const providerId = typeof message.providerId === "string" ? message.providerId : ""; + if (providerId.length === 0) { + wsSendJson(ws, { type: "ssh.error", message: "providerId is required" }); + return; + } + const provider = providerForSsh(providerId); + if (provider === null) { + wsSendJson(ws, { type: "ssh.error", message: `provider is not online: ${providerId}` }); + return; + } + if (!(await providerSupports(providerId, "host.ssh"))) { + wsSendJson(ws, { type: "ssh.error", message: `provider does not declare host.ssh capability: ${providerId}` }); + return; + } + const sessionId = safeSessionId(); + ws.data.channel = "ssh"; + ws.data.providerId = providerId; + ws.data.sessionId = sessionId; + ctx.activeSshClients.set(sessionId, ws); + const openMessage: CoreHostSshOpenMessage = { + type: "host_ssh_open", + sessionId, + cols: numberFromUnknown(message.cols, 100, 20, 300), + rows: numberFromUnknown(message.rows, 30, 8, 120), + }; + if (typeof message.cwd === "string" && message.cwd.length > 0) openMessage.cwd = message.cwd; + if (typeof message.command === "string" && message.command.length > 0) openMessage.command = message.command; + if (typeof message.tty === "boolean") openMessage.tty = message.tty; + provider.send(JSON.stringify(openMessage)); + wsSendJson(ws, { type: "ssh.dispatched", providerId, sessionId }); + logger("info", "ssh_session_dispatched", { providerId, sessionId, hasCommand: typeof openMessage.command === "string" }); + return; + } + + const sessionId = ws.data.sessionId; + const providerId = ws.data.providerId; + if (sessionId === undefined || providerId === undefined) { + if (message.type === "ssh.eof" || message.type === "ssh.close") return; + wsSendJson(ws, { type: "ssh.error", message: "ssh session is not open" }); + return; + } + const provider = providerForSsh(providerId); + if (provider === null) { + wsSendJson(ws, { type: "ssh.error", message: `provider went offline: ${providerId}` }); + return; + } + + if (message.type === "ssh.input") { + if (typeof message.data !== "string" || message.encoding !== "base64") { + wsSendJson(ws, { type: "ssh.error", message: "ssh.input requires base64 data" }); + return; + } + const inputMessage: CoreHostSshInputMessage = { type: "host_ssh_input", sessionId, data: message.data, encoding: "base64" }; + provider.send(JSON.stringify(inputMessage)); + return; + } + + if (message.type === "ssh.resize") { + const resizeMessage: CoreHostSshResizeMessage = { + type: "host_ssh_resize", + sessionId, + cols: numberFromUnknown(message.cols, 100, 20, 300), + rows: numberFromUnknown(message.rows, 30, 8, 120), + }; + provider.send(JSON.stringify(resizeMessage)); + return; + } + + if (message.type === "ssh.eof") { + const eofMessage: CoreHostSshEofMessage = { type: "host_ssh_eof", sessionId }; + provider.send(JSON.stringify(eofMessage)); + return; + } + + if (message.type === "ssh.close") { + const closeMessage: CoreHostSshCloseMessage = { type: "host_ssh_close", sessionId }; + provider.send(JSON.stringify(closeMessage)); + closeSshClient(sessionId); + return; + } + + wsSendJson(ws, { type: "ssh.error", message: `unsupported ssh client message: ${String(message.type)}` }); +} + +export async function sshRoute(req: Request, server: Server): Promise { + const url = new URL(req.url); + const token = url.searchParams.get("token") ?? req.headers.get("x-provider-token"); + if (token !== config().providerToken) return jsonResponse({ ok: false, error: "invalid ssh bridge token" }, 401); + const upgraded = server.upgrade(req, { data: { channel: "ssh" } satisfies WsData }); + return upgraded ? undefined : jsonResponse({ ok: false, error: "websocket upgrade failed" }, 400); +} diff --git a/src/components/backend-core/src/task-dispatcher.ts b/src/components/backend-core/src/task-dispatcher.ts new file mode 100644 index 00000000..b4ba524f --- /dev/null +++ b/src/components/backend-core/src/task-dispatcher.ts @@ -0,0 +1,158 @@ +import type { CoreDispatchMessage, JsonValue } from "../../shared/src/index"; +import { isProviderDispatchCommand } from "../../shared/src/index"; +import { ctx, sql, logger, config } from "./context"; +import type { RawTaskRow, ProviderSocket } from "./types"; +import { jsonResponse, errorToJson } from "./http"; +import { recordEvent } from "./db"; + +function isTerminalTaskStatus(status: string): boolean { + return status === "succeeded" || status === "failed"; +} + +export async function rawTask(taskId: string): Promise { + const rows = await sql()` + SELECT id, provider_id, command, status, payload, result, updated_at + FROM unidesk_tasks + WHERE id = ${taskId} + LIMIT 1 + `; + return rows[0] ?? null; +} + +export async function notifyTaskTerminal(taskId: string): Promise { + const waiters = ctx.taskTerminalWaiters.get(taskId); + if (waiters === undefined || waiters.size === 0) return; + const task = await rawTask(taskId); + if (task === null || !isTerminalTaskStatus(task.status)) return; + ctx.taskTerminalWaiters.delete(taskId); + for (const waiter of waiters) waiter(task); +} + +export async function waitForTaskTerminal(taskId: string, timeoutMs: number): Promise { + let latest = await rawTask(taskId); + if (latest !== null && isTerminalTaskStatus(latest.status)) return latest; + return await new Promise((resolve) => { + let settled = false; + let timer: ReturnType; + const settle = (task: RawTaskRow | null): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + const waiters = ctx.taskTerminalWaiters.get(taskId); + waiters?.delete(settle); + if (waiters !== undefined && waiters.size === 0) ctx.taskTerminalWaiters.delete(taskId); + resolve(task); + }; + const waiters = ctx.taskTerminalWaiters.get(taskId) ?? new Set<(task: RawTaskRow | null) => void>(); + waiters.add(settle); + ctx.taskTerminalWaiters.set(taskId, waiters); + timer = setTimeout(() => { + rawTask(taskId) + .then((task) => settle(task ?? latest)) + .catch(() => settle(latest)); + }, Math.max(1, timeoutMs)); + }); +} + +export async function createAndSendTask( + providerId: string, + command: CoreDispatchMessage["command"], + payload: Record, +): Promise<{ taskId: string; providerOnline: boolean }> { + const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`; + await sql()` + INSERT INTO unidesk_tasks (id, provider_id, command, status, payload, result) + VALUES (${taskId}, ${providerId}, ${command}, 'queued', ${sql().json(payload)}, NULL) + `; + const socket = ctx.activeProviders.get(providerId); + if (!socket) { + await recordEvent("task_queued_provider_offline", providerId, { taskId, providerId, command }); + return { taskId, providerOnline: false }; + } + const dispatch: CoreDispatchMessage = { type: "dispatch", taskId, command, payload }; + socket.send(JSON.stringify(dispatch)); + await sql()` + UPDATE unidesk_tasks + SET status = 'dispatched', updated_at = now() + WHERE id = ${taskId} AND status = 'queued' + `; + await recordEvent("task_dispatched", providerId, { taskId, providerId, command }); + return { taskId, providerOnline: true }; +} + +export async function markStaleTasksFailed(): Promise { + if (!ctx.dbReady) return; + const timeoutMs = config().taskPendingTimeoutMs; + const rows = await sql()>` + WITH stale AS ( + SELECT id, provider_id, command, status AS previous_status, updated_at + FROM unidesk_tasks + WHERE status IN ('queued', 'dispatched', 'running') + AND updated_at < now() - (${timeoutMs}::bigint * interval '1 millisecond') + FOR UPDATE + ), + updated AS ( + UPDATE unidesk_tasks task + SET + status = 'failed', + result = jsonb_build_object( + 'error', 'task timed out without terminal provider status', + 'timeoutMs', ${timeoutMs}::bigint, + 'previousStatus', stale.previous_status, + 'previousResult', task.result, + 'timedOutAt', now() + ), + updated_at = now() + FROM stale + WHERE task.id = stale.id + RETURNING task.id, task.provider_id, task.command, stale.previous_status, stale.updated_at + ) + SELECT * FROM updated + `; + for (const row of rows) { + await recordEvent("task_timeout", row.provider_id, { + taskId: row.id, + providerId: row.provider_id, + command: row.command, + previousStatus: row.previous_status, + previousUpdatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), + timeoutMs, + }); + notifyTaskTerminal(row.id).catch((error) => logger("error", "task_waiter_notify_failed", { taskId: row.id, error: errorToJson(error) })); + } +} + +export async function providerSupports(providerId: string, capability: string): Promise { + if (!ctx.dbReady) return false; + const rows = await sql()>` + SELECT labels + FROM unidesk_nodes + WHERE provider_id = ${providerId} + LIMIT 1 + `; + const labels = rows[0]?.labels; + if (typeof labels !== "object" || labels === null || Array.isArray(labels)) return false; + const capabilities = (labels as Record).unideskCapabilities; + return Array.isArray(capabilities) ? capabilities.filter((item): item is string => typeof item === "string").includes(capability) : false; +} + +export async function dispatchTask(req: Request): Promise { + const body = (await req.json()) as { providerId?: unknown; command?: unknown; payload?: unknown }; + const providerId = typeof body.providerId === "string" ? body.providerId : ""; + const command = isProviderDispatchCommand(body.command) ? body.command : null; + const payload = typeof body.payload === "object" && body.payload !== null ? (body.payload as Record) : {}; + if (!providerId) { + return jsonResponse({ ok: false, error: "providerId is required" }, 400); + } + if (command === null) { + return jsonResponse({ ok: false, error: "command must be one of docker.ps, provider.upgrade, host.ssh, microservice.http, echo" }, 400); + } + if (command === "host.ssh" && !(await providerSupports(providerId, "host.ssh"))) { + return jsonResponse({ ok: false, error: `provider does not declare host.ssh capability: ${providerId}` }, 409); + } + if (command === "microservice.http" && !(await providerSupports(providerId, "microservice.http"))) { + return jsonResponse({ ok: false, error: `provider does not declare microservice.http capability: ${providerId}` }, 409); + } + const { taskId, providerOnline } = await createAndSendTask(providerId, command, payload); + return jsonResponse({ ok: true, taskId, status: providerOnline ? "dispatched" : "queued", providerOnline }); +} diff --git a/src/components/backend-core/src/types.ts b/src/components/backend-core/src/types.ts new file mode 100644 index 00000000..4fcee9bc --- /dev/null +++ b/src/components/backend-core/src/types.ts @@ -0,0 +1,191 @@ +import type { ServerWebSocket } from "bun"; +import type { CoreDispatchMessage, JsonValue } from "../../shared/src/index"; +import type { Socket } from "node:net"; +import type postgres from "postgres"; + +export interface RuntimeConfig { + port: number; + providerPort: number; + databaseUrl: string; + providerToken: string; + heartbeatTimeoutMs: number; + taskPendingTimeoutMs: number; + databaseVolumeName: string; + databaseVolumeSize: string; + pgdataBackupStagingDir: string; + baiduNetdiskInternalUrl: string; + microservices: MicroserviceConfig[]; + logFile: string; + databasePoolMax: number; +} + +export interface MicroserviceConfig { + id: string; + name: string; + providerId: string; + description: string; + repository: { + url: string; + commitId: string; + dockerfile: string; + composeFile: string; + composeService: string; + containerName: string; + }; + backend: { + nodeBaseUrl: string; + nodeBindHost: string; + nodePort: number; + proxyMode: string; + frontendOnly: boolean; + public: boolean; + allowedMethods: string[]; + allowedPathPrefixes: string[]; + healthPath: string; + timeoutMs: number; + }; + deployment: { + mode: "unidesk-direct" | "v3sctl-managed"; + adapterServiceId?: string; + v3sServiceId?: string; + namespace?: string; + expectedNodeIds?: string[]; + activeNodeId?: string; + }; + development: { + providerId: string; + sshPassthrough: boolean; + worktreePath: string; + }; + frontend: { + route: string; + integrated: boolean; + }; +} + +export interface WsData { + providerId?: string; + channel?: "provider" | "ssh"; + sessionId?: string; +} + +export type ProviderSocket = ServerWebSocket; + +export type SqlClient = ReturnType; + +export interface RequestPerformanceSample { + at: string; + component: string; + method: string; + path: string; + status: number; + durationMs: number; + ok: boolean; +} + +export interface OperationPerformanceSample { + at: string; + service: string; + operation: string; + durationMs: number; + ok: boolean; + detail: string; +} + +export interface RawTaskRow { + id: string; + provider_id: string; + command: string; + status: string; + payload: JsonValue; + result: JsonValue | null; + updated_at: Date | string; +} + +export interface ScheduledTaskRow { + id: string; + name: string; + description: string; + enabled: boolean; + schedule_json: JsonValue; + action_json: JsonValue; + concurrency_policy: string; + next_run_at: Date | string | null; + last_run_at: Date | string | null; + last_run_id: string | null; + created_at: Date | string; + updated_at: Date | string; +} + +export interface ScheduledTaskRunRow { + id: string; + schedule_id: string; + trigger_type: string; + status: string; + task_id: string | null; + result: JsonValue | null; + error: string | null; + started_at: Date | string | null; + finished_at: Date | string | null; + duration_ms: number | string | null; + created_at: Date | string; + updated_at: Date | string; +} + +export interface ScheduleSpec { + type: "daily" | "interval"; + timeOfDay?: string; + timezone?: string; + everySeconds?: number; +} + +export interface DispatchScheduleAction { + type: "dispatch"; + providerId: string; + command: CoreDispatchMessage["command"]; + payload: Record; + timeoutMs?: number; +} + +export interface PgdataBackupScheduleAction { + type: "pgdata_backup"; + volumeName: string; + remoteBaseDir: string; + stagingSubdir: string; + baiduBaseUrl: string; + timeoutMs: number; + cleanupLocal: boolean; +} + +export type ScheduleAction = DispatchScheduleAction | PgdataBackupScheduleAction; + +export type TaskTerminalWaiter = (task: RawTaskRow | null) => void; + +export interface EgressTcpConnection { + providerId: string; + connectionId: string; + socket: Socket; + provider: ProviderSocket; +} + +export interface MicroserviceProxyCacheEntry { + expiresAt: number; + staleExpiresAt: number; + status: number; + contentType: string; + bodyText: string; +} + +export interface MicroserviceHealthAssessment { + healthy: boolean; + reason: string; + checks: Record; + body: JsonValue; +} + +export interface MicroserviceAvailabilityEntry { + expiresAt: number; + probe: Record; +} + +export type LoggerFn = (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue) => void;