From caa80ee5e7856f86927baf71c78c78f537759fed Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 4 May 2026 11:09:35 +0000 Subject: [PATCH] feat: initialize unidesk platform --- .gitignore | 11 + AGENTS.md | 30 ++ TEST.md | 41 ++ bun.lock | 32 ++ config.json | 62 +++ docker-compose.yml | 118 +++++ docs/reference/arch.md | 73 +++ docs/reference/cli.md | 28 + docs/reference/config.md | 19 + docs/reference/deployment.md | 22 + docs/reference/e2e.md | 36 ++ docs/reference/frontend.md | 15 + docs/reference/observability.md | 15 + docs/reference/provider-gateway.md | 15 + docs/reference/repo-tree.md | 66 +++ package.json | 17 + reference | 1 + scripts/cli.ts | 133 +++++ scripts/src/check.ts | 59 +++ scripts/src/command.ts | 55 ++ scripts/src/config.ts | 117 +++++ scripts/src/debug.ts | 30 ++ scripts/src/docker.ts | 213 ++++++++ scripts/src/e2e.ts | 154 ++++++ scripts/src/jobs.ts | 95 ++++ scripts/src/output.ts | 31 ++ scripts/tsconfig.json | 13 + src/bun.lock | 50 ++ src/components/backend-core/Dockerfile | 7 + src/components/backend-core/package.json | 12 + src/components/backend-core/src/index.ts | 484 ++++++++++++++++++ src/components/backend-core/tsconfig.json | 18 + .../database/config/postgresql.conf | 6 + .../database/init/001_unidesk_init.sql | 32 ++ src/components/frontend/Dockerfile | 7 + src/components/frontend/package.json | 9 + src/components/frontend/public/app.js | 153 ++++++ src/components/frontend/public/index.html | 68 +++ src/components/frontend/public/style.css | 329 ++++++++++++ src/components/frontend/src/index.ts | 98 ++++ src/components/frontend/tsconfig.json | 17 + .../microservices/example-service/Dockerfile | 3 + .../example-service/package.json | 5 + .../example-service/src/index.ts | 1 + .../example-service/tsconfig.json | 10 + src/components/provider-gateway/Dockerfile | 10 + src/components/provider-gateway/package.json | 9 + .../scripts/host-ssh-shell.sh | 29 ++ src/components/provider-gateway/src/index.ts | 231 +++++++++ src/components/provider-gateway/tsconfig.json | 18 + src/components/shared/package.json | 7 + src/components/shared/src/index.ts | 104 ++++ src/components/shared/tsconfig.json | 15 + src/package.json | 16 + src/tsconfig.base.json | 9 + src/tsconfig.check.json | 15 + 56 files changed, 3273 insertions(+) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 TEST.md create mode 100644 bun.lock create mode 100644 config.json create mode 100644 docker-compose.yml create mode 100644 docs/reference/arch.md create mode 100644 docs/reference/cli.md create mode 100644 docs/reference/config.md create mode 100644 docs/reference/deployment.md create mode 100644 docs/reference/e2e.md create mode 100644 docs/reference/frontend.md create mode 100644 docs/reference/observability.md create mode 100644 docs/reference/provider-gateway.md create mode 100644 docs/reference/repo-tree.md create mode 100644 package.json create mode 120000 reference create mode 100644 scripts/cli.ts create mode 100644 scripts/src/check.ts create mode 100644 scripts/src/command.ts create mode 100644 scripts/src/config.ts create mode 100644 scripts/src/debug.ts create mode 100644 scripts/src/docker.ts create mode 100644 scripts/src/e2e.ts create mode 100644 scripts/src/jobs.ts create mode 100644 scripts/src/output.ts create mode 100644 scripts/tsconfig.json create mode 100644 src/bun.lock create mode 100644 src/components/backend-core/Dockerfile create mode 100644 src/components/backend-core/package.json create mode 100644 src/components/backend-core/src/index.ts create mode 100644 src/components/backend-core/tsconfig.json create mode 100644 src/components/database/config/postgresql.conf create mode 100644 src/components/database/init/001_unidesk_init.sql create mode 100644 src/components/frontend/Dockerfile create mode 100644 src/components/frontend/package.json create mode 100644 src/components/frontend/public/app.js create mode 100644 src/components/frontend/public/index.html create mode 100644 src/components/frontend/public/style.css create mode 100644 src/components/frontend/src/index.ts create mode 100644 src/components/frontend/tsconfig.json create mode 100644 src/components/microservices/example-service/Dockerfile create mode 100644 src/components/microservices/example-service/package.json create mode 100644 src/components/microservices/example-service/src/index.ts create mode 100644 src/components/microservices/example-service/tsconfig.json create mode 100644 src/components/provider-gateway/Dockerfile create mode 100644 src/components/provider-gateway/package.json create mode 100755 src/components/provider-gateway/scripts/host-ssh-shell.sh create mode 100644 src/components/provider-gateway/src/index.ts create mode 100644 src/components/provider-gateway/tsconfig.json create mode 100644 src/components/shared/package.json create mode 100644 src/components/shared/src/index.ts create mode 100644 src/components/shared/tsconfig.json create mode 100644 src/package.json create mode 100644 src/tsconfig.base.json create mode 100644 src/tsconfig.check.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..1c1c7d2a --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.state/ +logs/ +node_modules/ +package-lock.json +npm-debug.log* +.DS_Store +.env +.env.* +!.env.example +dist/ +coverage/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..40dd7669 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,30 @@ +# UniDesk Agent Index + +UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文件是项目顶级索引，也承担 `scripts/cli.ts` 的 CLI 使用说明入口。 + +## CLI + +- `bun scripts/cli.ts help`：输出所有可用命令的 JSON 索引，详细规范见 `docs/reference/cli.md`。 +- `bun scripts/cli.ts config show`：校验并展示根目录 `config.json`，配置来源规则见 `docs/reference/config.md`。 +- `bun scripts/cli.ts check`：运行配置、TypeScript、文件存在性和 Docker Compose 配置检查，测试入口见 `TEST.md`。 +- `bun scripts/cli.ts server start`：以异步 job 启动 database、backend-core、frontend、provider-gateway，部署规则见 `docs/reference/deployment.md`。 +- `bun scripts/cli.ts server status`：查询固定端口、容器状态、健康检查和访问 URL，判定标准见 `docs/reference/deployment.md`。 +- `bun scripts/cli.ts server logs`：分页返回文件日志与 Docker 日志尾部，日志规则见 `docs/reference/observability.md`。 +- `bun scripts/cli.ts server stop`：以异步 job 停止固定 Compose 项目中的全部 UniDesk 服务，停止后用 `server status` 复核。 +- `bun scripts/cli.ts job list` / `bun scripts/cli.ts job status latest`：查询 `.state/jobs/` 中的异步任务状态，job 机制见 `docs/reference/cli.md`。 +- `bun scripts/cli.ts debug health` / `bun scripts/cli.ts debug dispatch`：通过真实 HTTP 和 WebSocket 流程调试健康检查与任务下发，调试规则见 `docs/reference/cli.md`。 +- `bun scripts/cli.ts e2e run`：用公开 URL 验证 core、database、frontend、provider-gateway 和 Playwright 页面可见性，验收规则见 `docs/reference/e2e.md`。 + +## Runtime + +- `bun`：TypeScript 运行时固定使用 Bun，组件入口和 CLI 都直接运行 `.ts` 文件，约束见 `docs/reference/config.md`。 +- `docker-compose.yml`：主 server 统一编排 core、frontend、database 和本机 provider gateway，服务拓扑见 `docs/reference/deployment.md`。 +- `src/components/frontend`：前端采用高信息密度、左侧主模块和顶部子模块标签的工业化控制台设计，界面规则见 `docs/reference/frontend.md`。 +- `src/components/provider-gateway`：当前主 server 也作为 provider gateway 接入 UniDesk，节点接入规则见 `docs/reference/provider-gateway.md`。 +- `docs/reference/e2e.md`：交付前必须执行的自测门禁、Playwright 前端断言和数据库命名卷持久化要求。 + +## Architecture Docs + +- `docs/reference/arch.md`：UniDesk 分布式工作平台的长期架构约束。 +- `docs/reference/repo-tree.md`：仓库结构目标与组件边界。 +- `reference`：兼容旧路径的符号链接，指向 `docs/reference/`。 diff --git a/TEST.md b/TEST.md new file mode 100644 index 00000000..aa51dd7f --- /dev/null +++ b/TEST.md @@ -0,0 +1,41 @@ +# UniDesk Manual Test Plan + +## T1 CLI 可观测性与配置校验 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：运行 `bun scripts/cli.ts help`、`bun scripts/cli.ts config show`、`bun scripts/cli.ts check`，确认每条命令都有 JSON 输出、失败时包含错误对象、`config.json` 是唯一配置来源，且 TypeScript 检查覆盖 `scripts/` 与 `src/components/`。 + +## T2 Docker 栈异步启动 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：运行 `bun scripts/cli.ts server start`，确认命令立即返回 job id；随后运行 `bun scripts/cli.ts job status latest` 观察构建和启动进度，直到 job 状态为 `succeeded`，且输出包含 `.state/jobs/` 日志路径和 `logs/{YYYYMMDD}/` 服务日志路径。 + +## T3 主 server 自接入 Provider Gateway + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：运行 `bun scripts/cli.ts server status` 和 `bun scripts/cli.ts debug health`，确认 backend-core、frontend、database 端口均监听，`/api/nodes` 中存在 `main-server` provider，状态为 `online`，且 provider 标签中能看到 Docker socket 可用性。 + +## T4 前端控制台连通 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：先用 `bun scripts/cli.ts server status` 获取 frontend URL，再用浏览器访问该 URL，确认左侧主模块、顶部子标签、核心指标、Provider 表格和事件流可见；页面布局应紧凑、信息密度高、字体不过大，且移动端宽度下左侧栏转为横向模块条。 + +## T5 真实任务下发链路 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：运行 `bun scripts/cli.ts debug dispatch main-server docker.ps`，随后运行 `bun scripts/cli.ts debug health` 或在前端事件流中确认任务状态经历 `accepted`、`running`、`succeeded`，并能看到 provider 通过 Docker socket 返回容器列表摘要。 + +## T6 日志第一现场验证 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：运行 `bun scripts/cli.ts server logs --tail-bytes 20000`，实际读取输出中列出的 `logs/{YYYYMMDD}/` 文件，确认 backend-core、frontend、provider-gateway、database 都有实时日志；日志不得只有启动行，错误日志必须包含可定位的错误消息或 stack。 + +## T7 停止与端口释放 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：运行 `bun scripts/cli.ts server stop`，确认立即返回 job id；等待 `bun scripts/cli.ts job status latest` 成功后运行 `bun scripts/cli.ts server status`，确认 backend-core、frontend、database 固定端口不再监听，容器状态不再运行。 + +## Issue 记录 + +测试发现的问题写入 `docs/issue/`，文件命名为 `{NN}_{issue_title}_{YYYYMMDD}.md`，其中 `NN` 从 `01` 递增；问题文档必须包含复现命令、实际输出摘要、期望行为和修复判定标准。 + +## T8 Playwright 公网前端 E2E + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：确认 `config.json` 的 `network.publicHost` 是主 server 公网地址，运行 `bun scripts/cli.ts e2e run`，要求 JSON 中 `core:public-overview`、`core:public-nodes`、`database:named-volume-write`、`database:public-port`、`frontend:public-page-provider-visible` 全部 passed；打开输出的 screenshotPath，确认页面上能看到 `main-server`、`Main Server Provider`、Online Nodes 指标和 Provider 表格。 + +## T9 Database 命名卷持久化 + +阅读 `AGENTS.md`（本项目 `AGENTS.md` 同时承担 `SKILL.md` 对 `scripts/cli.ts` 的解释职责），然后用 cli 手动测试以下内容：向 `unidesk_e2e_markers` 插入一个唯一 marker，运行 `bun scripts/cli.ts server start` 并等待 `job status latest` 为 `succeeded`，再用 `docker exec unidesk-database psql -U unidesk -d unidesk` 查询该 marker 仍存在；同时审查 `docker-compose.yml` 和 `scripts/src/docker.ts`，确认 CLI server 控制没有使用 `down -v` 或 volume 删除命令。 diff --git a/bun.lock b/bun.lock new file mode 100644 index 00000000..f119ac77 --- /dev/null +++ b/bun.lock @@ -0,0 +1,32 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "unidesk-root", + "devDependencies": { + "@types/bun": "latest", + "@types/node": "latest", + "playwright": "^1.59.1", + "typescript": "latest", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + + "@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], + + "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + + "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + + "playwright": ["playwright@1.59.1", "", { "dependencies": { "playwright-core": "1.59.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw=="], + + "playwright-core": ["playwright-core@1.59.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg=="], + + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + + "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], + } +} diff --git a/config.json b/config.json new file mode 100644 index 00000000..c25dd5b2 --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "project": { + "name": "unidesk", + "timezone": "Etc/UTC" + }, + "runtime": { + "typescript": "bun", + "bunVersion": "1.3.13" + }, + "network": { + "host": "0.0.0.0", + "publicHost": "74.48.78.17", + "core": { + "port": 18080, + "containerPort": 8080 + }, + "frontend": { + "port": 18081, + "containerPort": 8080 + }, + "database": { + "port": 15432, + "containerPort": 5432 + } + }, + "database": { + "user": "unidesk", + "password": "unidesk_dev_password", + "name": "unidesk", + "volume": "unidesk_pgdata_10gb", + "volumeSize": "10GB" + }, + "providerGateway": { + "id": "main-server", + "name": "Main Server Provider", + "token": "unidesk-dev-token-change-me", + "labels": { + "host": "main-server", + "role": "self-provider", + "docker": true + }, + "heartbeatIntervalMs": 15000, + "reconnectBaseMs": 1000, + "reconnectMaxMs": 30000 + }, + "docker": { + "composeFile": "docker-compose.yml", + "projectName": "unidesk" + }, + "paths": { + "stateDir": ".state", + "logsDir": "logs", + "docsReferenceDir": "docs/reference" + }, + "sshForwarding": { + "mode": "optional-maintenance-only", + "keyDir": "/root/.unidesk/host-ssh", + "host": "host.docker.internal", + "port": 22, + "user": "root" + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..923e787d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,118 @@ +version: "3.8" + +services: + database: + image: postgres:16-alpine + container_name: unidesk-database + restart: unless-stopped + ports: + - "${UNIDESK_DATABASE_PORT}:5432" + environment: + POSTGRES_USER: "${UNIDESK_DATABASE_USER}" + POSTGRES_PASSWORD: "${UNIDESK_DATABASE_PASSWORD}" + POSTGRES_DB: "${UNIDESK_DATABASE_NAME}" + command: + - "postgres" + - "-c" + - "config_file=/etc/postgresql/postgresql.conf" + - "-c" + - "logging_collector=on" + - "-c" + - "log_directory=/var/log/unidesk" + - "-c" + - "log_filename=${UNIDESK_LOG_PREFIX}_database.log" + - "-c" + - "log_connections=on" + - "-c" + - "log_disconnections=on" + volumes: + - unidesk_pgdata_10gb:/var/lib/postgresql/data + - ./src/components/database/init:/docker-entrypoint-initdb.d:ro + - ./src/components/database/config/postgresql.conf:/etc/postgresql/postgresql.conf:ro + - ${UNIDESK_LOG_DIR}:/var/log/unidesk + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${UNIDESK_DATABASE_USER} -d ${UNIDESK_DATABASE_NAME}"] + interval: 5s + timeout: 3s + retries: 20 + + backend-core: + build: + context: . + dockerfile: src/components/backend-core/Dockerfile + container_name: unidesk-backend-core + restart: unless-stopped + depends_on: + - database + ports: + - "${UNIDESK_CORE_PORT}:8080" + environment: + PORT: "8080" + DATABASE_URL: "postgres://${UNIDESK_DATABASE_USER}:${UNIDESK_DATABASE_PASSWORD}@database:5432/${UNIDESK_DATABASE_NAME}" + PROVIDER_TOKEN: "${UNIDESK_PROVIDER_TOKEN}" + HEARTBEAT_TIMEOUT_MS: "${UNIDESK_HEARTBEAT_TIMEOUT_MS}" + LOG_FILE: "/var/log/unidesk/${UNIDESK_LOG_PREFIX}_backend-core.jsonl" + volumes: + - ${UNIDESK_LOG_DIR}:/var/log/unidesk + healthcheck: + test: ["CMD", "bun", "-e", "fetch('http://127.0.0.1:8080/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 5s + timeout: 3s + retries: 20 + + frontend: + build: + context: . + dockerfile: src/components/frontend/Dockerfile + container_name: unidesk-frontend + restart: unless-stopped + depends_on: + - backend-core + ports: + - "${UNIDESK_FRONTEND_PORT}:8080" + environment: + PORT: "8080" + CORE_PUBLIC_URL: "http://${UNIDESK_PUBLIC_HOST}:${UNIDESK_CORE_PORT}" + LOG_FILE: "/var/log/unidesk/${UNIDESK_LOG_PREFIX}_frontend.jsonl" + volumes: + - ${UNIDESK_LOG_DIR}:/var/log/unidesk + healthcheck: + test: ["CMD", "bun", "-e", "fetch('http://127.0.0.1:8080/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 5s + timeout: 3s + retries: 20 + + provider-gateway: + build: + context: . + dockerfile: src/components/provider-gateway/Dockerfile + container_name: unidesk-provider-gateway-main + restart: unless-stopped + depends_on: + - backend-core + environment: + PROVIDER_SERVER_URL: "ws://backend-core:8080/ws/provider" + PROVIDER_TOKEN: "${UNIDESK_PROVIDER_TOKEN}" + PROVIDER_ID: "${UNIDESK_PROVIDER_ID}" + PROVIDER_NAME: "${UNIDESK_PROVIDER_NAME}" + PROVIDER_LABELS_JSON: "${UNIDESK_PROVIDER_LABELS_JSON}" + HEARTBEAT_INTERVAL_MS: "${UNIDESK_HEARTBEAT_INTERVAL_MS}" + RECONNECT_BASE_MS: "${UNIDESK_RECONNECT_BASE_MS}" + RECONNECT_MAX_MS: "${UNIDESK_RECONNECT_MAX_MS}" + DOCKER_SOCKET_PATH: "/var/run/docker.sock" + LOG_FILE: "/var/log/unidesk/${UNIDESK_LOG_PREFIX}_provider-gateway.jsonl" + HOST_SSH_HOST: "${UNIDESK_HOST_SSH_HOST}" + HOST_SSH_PORT: "${UNIDESK_HOST_SSH_PORT}" + HOST_SSH_USER: "${UNIDESK_HOST_SSH_USER}" + HOST_SSH_KEY: "/run/host-ssh/id_ed25519" + HOST_REMOTE_CWD: "/root" + HOST_LOGIN_SHELL: "/bin/bash" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${UNIDESK_LOG_DIR}:/var/log/unidesk + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + unidesk_pgdata_10gb: + name: unidesk_pgdata_10gb diff --git a/docs/reference/arch.md b/docs/reference/arch.md new file mode 100644 index 00000000..6d6a9c5a --- /dev/null +++ b/docs/reference/arch.md @@ -0,0 +1,73 @@ +- Requirements + - Build a distributed work platform covering research, project development, and project management + - Deploy the main entry point on a server with a public IP, providing a unified interface + - Multiple computing resource machines join the platform to execute computing tasks + - The platform must support task scheduling, state monitoring, versioned code distribution, and large file storage + - Design goals are high availability, high concurrency, centralized state management, and stateless compute nodes +- Key Assumptions + - The main server has a public IP and can be accessed from the internet + - Computing resource machines have no public IP, possibly behind NAT or firewalls + - Computing resource machines have stable outbound network connectivity (within intranet or internet) + - Computing resource machines can run Docker and support WSL (some nodes are Windows workstations) + - Users interact with the platform only through the main server entry point, never directly with compute nodes + - The main server's availability is higher than that of computing resource machines; compute nodes may go offline frequently due to hardware, network, or human factors + - Tasks prone to single points of failure are deployed on the main server first, leveraging its high-availability environment to protect the critical path +- UniDesk Distributed Work Platform Architecture + - Overview + - The main server hosts all stateless business logic as the unified entry point + - Computing resource nodes actively connect via lightweight Provider Gateway containers + - All state is stored centrally in PostgreSQL, never scattered across nodes + - Code and environments are distributed via GitHub versions; large file storage solution is to be determined + - The main server also connects itself to the platform as a compute node, using the exact same method as ordinary compute nodes + - This design allows verification of the full distributed dispatching flow on a single main server + - Main Server Components + - UniDesk Stateless Services + - Run all business microservices as Docker containers + - Includes API gateway, task scheduler, project management, and other stateless modules + - Instances can scale horizontally; failure recovery requires no state synchronization + - PostgreSQL Database + - Deployed as a Docker container with a 10 GB named volume + - Stores all task metadata, node heartbeats, resource labels, and business state + - Backed up periodically via `pg_dump`, keeping the last 7 daily snapshots + - The named volume ensures data survives container recreation or upgrades + - Code and Environment Distribution + - Code repositories and execution environment definitions may reside in multiple GitHub repositories + - When dispatching a task, five metadata items must be specified: `code_repo_url`, `code_commit_id`, `env_repo_url`, `env_commit_id`, and `dockerfile_path` + - A single env repo can contain multiple Dockerfiles defining different execution environments, distinguished by `dockerfile_path` + - Compute nodes maintain a local Git cache and only incrementally fetch the specified version each time + - Docker layer caching accelerates environment builds, making subsequent builds nearly instantaneous after the first + - Compute Node Connection Scheme + - Provider Gateway Docker + - Each computing resource machine runs a Provider Gateway container + - Acts as the node-side gateway, bridging the main server and the local execution environment + - The container houses the agent logic, implementing a WebSocket client and local scheduling + - WebSocket Persistent Connection + - Provider Gateway actively initiates a WebSocket connection to the main server + - Commands, heartbeats, and task statuses are exchanged bidirectionally over this persistent connection + - The main server never initiates connections to nodes, perfectly adapting to environments without public IP and behind NAT + - Interaction with Local Execution Environment + - The primary path for automated task dispatching and execution is via the local Docker socket + - Access to the local environment via WSL SSH is reserved solely as an auxiliary path for emergency maintenance and troubleshooting + - Automating task deployment or dispatching through the WSL SSH channel is forbidden + - Connection Management + - When registering, a node carries an authentication token to verify its identity and declares resources such as GPU/CPU + - The authentication token is pre-issued by the main server and configured at Provider Gateway startup + - Heartbeats are sent every 15 seconds; if no heartbeat arrives for 90 seconds, the node is marked offline + - Automatic reconnection on disconnect with exponential backoff to avoid a thundering herd on the main server + - Data Flow and State Management + - Task commands are delivered over WebSocket and never contain large file content + - All state changes are reported to the main server in real time by Provider Gateway + - The main server writes state updates to PostgreSQL, completing the unified closed loop + - Critical Task Deployment Principles + - Single-point components such as the database, core scheduler logic, and API gateway are deployed on the main server + - The high-availability environment of the main server ensures the critical scheduling path never breaks + - Compute nodes are only responsible for task execution; their offline status does not affect overall platform availability + - Large File Storage Solution + - The concrete implementation is to be determined, and must meet the following requirements + - Support automated pull and upload by compute nodes without human intervention + - Provide a programmable interface for the scheduler to generate temporary access credentials + - Have sufficient bandwidth so that concurrent reads/writes never become the bottleneck for training tasks + - Deployment Notes + - Use `docker-compose` on the main server to orchestrate all services uniformly + - PostgreSQL uses a named volume to guarantee data persistence + - The Provider Gateway image is built uniformly and distributed to all compute nodes in a versioned manner diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 00000000..eb35e7b4 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,28 @@ +# UniDesk CLI Reference + +UniDesk 的统一 CLI 入口是根目录 `scripts/cli.ts`，运行方式固定为 `bun scripts/cli.ts `。CLI 默认输出 JSON，所有成功和失败路径都必须向 stdout 写出结构化对象，避免无输出造成状态不可观测。 + +## Command Model + +- `help` 输出命令索引，适合作为交互式入口。 +- `config show` 读取并校验根目录 `config.json`，不从环境变量、默认值或隐藏文件静默补配置。 +- `check` 执行配置校验、文件存在性检查、`scripts/` TypeScript 检查、`src/components/` TypeScript 检查和 Docker Compose 配置检查。 +- `server start` 创建异步 job，在后台执行 Docker 构建和启动；命令本身只负责返回 job id、日志路径和启动命令。 +- `server stop` 创建异步 job，在后台停止固定 Compose project 中的全部 UniDesk 服务。 +- `server status` 查询固定端口、Compose 容器、core/frontend 健康检查和访问 URL。 +- `server logs` 返回 `logs/` 文件日志和 Docker 容器日志的尾部，默认限制输出大小，避免日志爆炸。 +- `job list` 与 `job status` 查询 `.state/jobs/` 文件系统状态，是异步命令的可观测入口。 +- `debug health` 与 `debug dispatch` 走真实 HTTP、WebSocket、数据库和 provider 流程，只用于开发调试，不写入 `TEST.md` 的正式验收步骤。 +- `e2e run` 使用 publicHost 派生的公开 URL 验证 core API、PostgreSQL、provider self-connection 和 Playwright 前端页面，是交付前的自动化 E2E 门禁。 + +## Async Job State + +长时操作采用 Fire-and-Forget 模式：CLI 创建 `.state/jobs/{jobId}.json`，后台进程执行真实命令，并将 stdout、stderr 分别写入 `.state/jobs/{jobId}.stdout.log` 与 `.state/jobs/{jobId}.stderr.log`。调用者通过 `bun scripts/cli.ts job status ` 查询进度和尾部输出。 + +## Output Contract + +每条命令的最外层 JSON 包含 `ok`、`command` 和 `data` 或 `error`。失败时 CLI 设置非零退出码，但仍然输出 JSON 错误对象；错误对象应包含 `name`、`message` 和可用的 `stack`。 + +## Debug Contract + +`debug` 子命令必须复用真实模块与真实端点，禁止维护平行实现。`debug dispatch` 会调用 core 的 `/api/dispatch`，core 再通过 WebSocket 将任务下发给 provider gateway，因此它可以验证核心调度闭环。 diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 00000000..977061b6 --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,19 @@ +# UniDesk Configuration Reference + +根目录 `config.json` 是 UniDesk CLI 的唯一配置来源。CLI 启动时必须完整校验配置结构，读取失败或字段不合法时直接返回 JSON 错误，不允许静默 fallback。 + +## Runtime + +TypeScript 运行时固定为 Bun。根目录 CLI、backend-core、frontend 和 provider-gateway 都直接运行 `.ts` 入口；Docker 镜像使用 `oven/bun` 基础镜像，本机命令使用 `bun scripts/cli.ts`。 + +## Fixed Ports + +`config.json` 中固定三个对外端口：backend-core、frontend、database。`network.publicHost` 必须是浏览器和外部客户端可访问的主 server 地址；公网 E2E 不允许把它保留为 `127.0.0.1`。`server start` 会在启动前检查这些端口，避免因端口冲突产生多个版本混乱的服务实例。 + +## Compose Env Generation + +Docker Compose 本身不读取 JSON，因此 CLI 会从 `config.json` 生成 `.state/docker-compose.env`。该文件是派生状态，不应手写；如需改端口、token、provider 标签或主机名，应修改 `config.json` 后重新运行 CLI。 + +## Secrets + +当前配置面向主 server 开发部署，包含开发用数据库密码和 provider token。公网暴露前必须在 `config.json` 中修改这些值，并重新启动栈以刷新派生环境文件。 diff --git a/docs/reference/deployment.md b/docs/reference/deployment.md new file mode 100644 index 00000000..c62bf27b --- /dev/null +++ b/docs/reference/deployment.md @@ -0,0 +1,22 @@ +# UniDesk Deployment Reference + +主 server 使用根目录 `docker-compose.yml` 统一编排 database、backend-core、frontend 和 provider-gateway。当前环境本身就是主 server，因此 provider-gateway 也在同一台机器上启动，用与普通计算节点相同的 WebSocket 方式接入 core。 + +## Services + +- `database` 使用 `postgres:16-alpine`，数据保存到 named volume `unidesk_pgdata_10gb`，初始化 SQL 位于 `src/components/database/init/`。 +- `backend-core` 是无状态核心服务，提供 REST API、provider WebSocket、任务调度入口和数据库访问层。 +- `frontend` 是独立 Web 容器，通过浏览器访问 core 的公开 API URL。 +- `provider-gateway` 是当前主 server 的本机计算节点代理，通过 WebSocket 主动连到 backend-core，并挂载 `/var/run/docker.sock` 作为自动任务执行主路径。 + +## Start And Stop + +`bun scripts/cli.ts server start` 与 `bun scripts/cli.ts server stop` 都是异步 job。启动 job 会先清理固定 Compose project 的旧容器，再重新构建并启动，避免主 server 上残留旧容器或旧镜像配置。启动后用 `job status latest` 观察后台命令，用 `server status` 验证端口、容器和健康检查。 + +## Health Criteria + +服务跑通的最低标准是：backend-core `/health` 返回 ok，frontend `/health` 返回 ok，database 端口监听，`/api/nodes` 中出现 `main-server` provider 且状态为 `online`，`debug dispatch main-server docker.ps` 能完成真实任务下发。交付前还必须运行 `bun scripts/cli.ts e2e run`，并以 `docs/reference/e2e.md` 的门禁作为最终判定。 + +## Database Volume + +架构要求数据库使用 10 GB named volume；当前实现将 volume 命名为 `unidesk_pgdata_10gb` 以固定生命周期。Docker named volume 默认不强制容量上限；如需硬配额，应在主机存储层或 Docker volume driver 层配置。CLI server 控制只能使用不删除 volume 的 `down` / `up` 流程，禁止使用 `down -v` 或删除 `unidesk_pgdata_10gb`。 diff --git a/docs/reference/e2e.md b/docs/reference/e2e.md new file mode 100644 index 00000000..d99782d2 --- /dev/null +++ b/docs/reference/e2e.md @@ -0,0 +1,36 @@ +# UniDesk E2E Reference + +UniDesk delivery is not complete until the public frontend, public core API, PostgreSQL database, and local provider-gateway self-connection pass one end-to-end check. The canonical automated command is `bun scripts/cli.ts e2e run`. + +## Required Preconditions + +- `config.json` `network.publicHost` must be the externally reachable host name or IP of the main server, not `127.0.0.1`, when validating browser access from outside the server. +- `bunx playwright install chromium` and `bunx playwright install-deps chromium` must have been run on hosts that execute browser E2E tests. +- The Docker stack must be running through `bun scripts/cli.ts server start`, and `bun scripts/cli.ts server status` must report healthy core, frontend, database, and provider-gateway containers. + +## Automated E2E Scope + +`bun scripts/cli.ts e2e run` validates the following through the public URLs derived from `config.json`: + +- Core API: `GET /api/overview` reports `dbReady: true` and at least one online node. +- Provider self-connection: `GET /api/nodes` contains `main-server` with `status: online`. +- Database: the command writes an `unidesk_e2e_markers` row through `docker exec unidesk-database psql`, confirms provider state is stored in PostgreSQL, and probes the public PostgreSQL port with `pg_isready`. +- Frontend: Playwright opens the public frontend URL, waits for `核心在线`, asserts that `main-server` and `Main Server Provider` are visible, checks the metrics panel, and captures a screenshot under `.state/e2e/`. + +## Public Frontend Rule + +The frontend must not inject `127.0.0.1` as the browser-facing core API URL for public deployments. If a loopback URL is accidentally injected and the page itself is opened from a non-loopback host, `public/app.js` rewrites the API host to `window.location.hostname` as a safety net; however the correct fix is still to set `network.publicHost` correctly in `config.json` and restart the stack. + +## Database Persistence Rule + +The PostgreSQL data volume is the named Docker volume `unidesk_pgdata_10gb`. CLI server control commands must never use `docker compose down -v`, `docker volume rm`, or any equivalent data-volume removal. To validate persistence, insert a marker row into `unidesk_e2e_markers`, run `bun scripts/cli.ts server start` or a full stop/start cycle, and verify the marker row still exists. + +## Delivery Gate + +Before claiming delivery, run these checks and keep their JSON output or screenshot path available for review: + +1. `bun scripts/cli.ts check` +2. `bun scripts/cli.ts server start`, then `bun scripts/cli.ts job status latest` until `succeeded` +3. `bun scripts/cli.ts server status` +4. `bun scripts/cli.ts e2e run` +5. a database persistence marker check across at least one CLI-controlled restart diff --git a/docs/reference/frontend.md b/docs/reference/frontend.md new file mode 100644 index 00000000..3f00f91d --- /dev/null +++ b/docs/reference/frontend.md @@ -0,0 +1,15 @@ +# UniDesk Frontend Reference + +UniDesk 前端是工业化控制台，不追求展示型大屏效果。设计目标是高信息密度、低装饰、低字号、低间距，并让调度、节点、事件和配置入口在单屏内快速切换。 + +## Layout + +左侧边栏切换主模块：运行总览、资源节点、任务调度、系统配置。顶部标签切换子模块：Overview、Live Nodes、Event Log、Dispatch。桌面端采用双列内容网格，移动端将左侧栏压缩为横向模块条。 + +## Visual Language + +界面使用深钢蓝、炭黑、琥珀和冷青作为工业控制台色板；字体选择窄体和等宽组合，以减少横向浪费。字号、表格行高和面板间距保持克制，避免大标题和松散卡片造成信息密度下降。 + +## Data Flow + +frontend 容器只服务静态资产和轻量 HTML 注入；浏览器根据 `CORE_PUBLIC_URL` 调用 backend-core 的 REST API。调度表单调用 `/api/dispatch`，事件表和节点表通过轮询刷新。 diff --git a/docs/reference/observability.md b/docs/reference/observability.md new file mode 100644 index 00000000..c397b3bd --- /dev/null +++ b/docs/reference/observability.md @@ -0,0 +1,15 @@ +# UniDesk Observability Reference + +UniDesk 的可观测性优先级高于静默成功。CLI、服务日志、Docker 日志和数据库状态都必须能通过短命令查询。 + +## CLI Logs + +异步 job 的 stdout 和 stderr 位于 `.state/jobs/`。`job status` 会返回有限尾部，避免输出爆炸，同时保留完整日志文件路径便于继续排查。 + +## Service Logs + +服务日志位于 `logs/{YYYYMMDD}/`，每次 `server start` 都生成新的本地时间戳前缀。backend-core、frontend 和 provider-gateway 输出 JSONL 文件；database 通过 PostgreSQL logging collector 写入同一目录。 + +## Log Access + +`bun scripts/cli.ts server logs` 同时读取文件日志和 Docker logs 尾部。文件日志是服务崩溃时的第一现场，Docker logs 是容器启动失败和 stdout/stderr 的辅助来源。 diff --git a/docs/reference/provider-gateway.md b/docs/reference/provider-gateway.md new file mode 100644 index 00000000..73cc6a1c --- /dev/null +++ b/docs/reference/provider-gateway.md @@ -0,0 +1,15 @@ +# Provider Gateway Reference + +Provider Gateway 是计算节点侧容器。它只主动连出到 backend-core 的 WebSocket，不要求计算节点有公网 IP，适合 NAT、内网和防火墙后的机器。 + +## Main Server Self Provider + +当前主 server 也运行一个 provider-gateway，`providerId` 固定来自 `config.json` 的 `providerGateway.id`。这让单机环境也能验证完整的分布式调度闭环：frontend 发起任务，core 写数据库并通过 WebSocket 下发，provider gateway 执行后回传状态。 + +## Docker Socket Path + +自动任务执行只允许走本地 Docker socket。Compose 将 `/var/run/docker.sock` 挂入 provider-gateway，provider 标签会报告 `dockerSocketPresent`，`docker.ps` 调试任务会通过该 socket 查询宿主 Docker 容器。 + +## Host SSH Maintenance Bridge + +宿主 SSH 转发只作为应急维护辅助路径，不用于自动任务调度。实现参考 `../web-terminal` 的经验：容器内使用只读挂载的私钥，通过 `ssh -tt` 主动连接宿主 sshd，并设置 `StrictHostKeyChecking=accept-new`、`ServerAliveInterval` 和 `ServerAliveCountMax`。本仓库保留 `src/components/provider-gateway/scripts/host-ssh-shell.sh` 作为维护桥接脚本，默认 Compose 不挂载私钥，避免把 SSH 路径误用为调度通道。 diff --git a/docs/reference/repo-tree.md b/docs/reference/repo-tree.md new file mode 100644 index 00000000..aae6c440 --- /dev/null +++ b/docs/reference/repo-tree.md @@ -0,0 +1,66 @@ +- unidesk/ (Repository root: configuration, orchestration, CLI, and documentation) + - AGENTS.md (Top-level agent index and `scripts/cli.ts` usage guide) + - TEST.md (Manual CLI test plan following cli-spec expectations) + - config.json (Single source of truth for ports, tokens, runtime, paths, and provider identity) + - docker-compose.yml (Main server orchestration for database, backend-core, frontend, provider-gateway) + - package.json / bun.lock (Root Bun tooling for CLI checks) + - .gitignore + - reference -> docs/reference (Compatibility symlink for older references) + - scripts/ (Unified CLI and implementation modules) + - cli.ts (Single Bun CLI entry) + - tsconfig.json (TypeScript check scope for CLI) + - src/ (CLI business logic modules; `cli.ts` remains a thin router) + - config.ts (Root config loading and validation) + - docker.ts (Docker Compose env generation, start/stop/status/logs) + - jobs.ts (Fire-and-Forget job state under `.state/jobs/`) + - check.ts (Formal checks) + - debug.ts (Real-flow debug helpers) + - command.ts (Bounded command execution helpers) + - output.ts (JSON output helpers) + - e2e.ts (Public API, database, provider, and Playwright frontend E2E checks) + - logs/ (Generated service logs; ignored by git) + - .state/ (Generated job state and compose env; ignored by git) + - docs/ + - issue/ (Manual test issue records) + - reference/ (Long-term reference documents) + - arch.md (Distributed work platform architecture) + - repo-tree.md (This repository structure reference) + - cli.md (CLI command model and async job contract) + - config.md (Config and runtime rules) + - deployment.md (Docker stack deployment and health criteria) + - frontend.md (Frontend layout and design rules) + - provider-gateway.md (Provider connection and host SSH maintenance bridge) + - observability.md (Logs and status visibility) + - e2e.md (Delivery gate, Playwright frontend E2E, and database persistence checks) + - src/ (TypeScript component monorepo) + - package.json (Component workspace metadata) + - bun.lock (Component dependency lockfile) + - tsconfig.base.json (Project references for component checks) + - tsconfig.check.json (No-emit TypeScript check scope for all components) + - components/ + - shared/ (Shared message types and utilities) + - package.json + - tsconfig.json + - src/index.ts + - backend-core/ (UniDesk stateless core service container) + - package.json + - tsconfig.json + - Dockerfile + - src/index.ts (REST API, WebSocket provider server, scheduler, database access) + - frontend/ (Frontend web application container) + - package.json + - tsconfig.json + - Dockerfile + - src/index.ts (Bun static server and runtime config injection) + - public/ (HTML/CSS/JS assets for the compact industrial console) + - provider-gateway/ (Compute node Provider Gateway container) + - package.json + - tsconfig.json + - Dockerfile + - src/index.ts (WebSocket client, heartbeat, Docker adapter) + - scripts/host-ssh-shell.sh (Optional maintenance-only SSH bridge) + - database/ (PostgreSQL initialization and configuration) + - config/postgresql.conf + - init/001_unidesk_init.sql + - microservices/ (Reserved for future stateless microservices) + - example-service/ diff --git a/package.json b/package.json new file mode 100644 index 00000000..ce121d6f --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "unidesk-root", + "private": true, + "type": "module", + "packageManager": "bun@1.3.13", + "scripts": { + "cli": "bun scripts/cli.ts", + "check": "bun scripts/cli.ts check", + "e2e": "bun scripts/cli.ts e2e run" + }, + "devDependencies": { + "@types/bun": "latest", + "@types/node": "latest", + "playwright": "^1.59.1", + "typescript": "latest" + } +} diff --git a/reference b/reference new file mode 120000 index 00000000..2c76037f --- /dev/null +++ b/reference @@ -0,0 +1 @@ +docs/reference \ No newline at end of file diff --git a/scripts/cli.ts b/scripts/cli.ts new file mode 100644 index 00000000..4618eec6 --- /dev/null +++ b/scripts/cli.ts @@ -0,0 +1,133 @@ +import { readConfig } from "./src/config"; +import { debugDispatch, debugHealth } from "./src/debug"; +import { stackLogs, stackStatus, startStack, stopStack } from "./src/docker"; +import { runE2E } from "./src/e2e"; +import { emitError, emitJson } from "./src/output"; +import { jobWithTail, listJobs, readJob, runJob } from "./src/jobs"; +import { runChecks } from "./src/check"; + +const args = process.argv.slice(2); +const commandName = args.join(" ") || "help"; + +function help(): unknown { + return { + entry: "bun scripts/cli.ts", + output: "json", + commands: [ + { command: "help", description: "List supported commands." }, + { command: "config show", description: "Validate and print config.json as the single source of truth." }, + { command: "check", description: "Run config, TypeScript, file presence, and docker-compose config checks." }, + { command: "server start", description: "Fire-and-forget build/start for database, backend-core, frontend, and provider gateway." }, + { command: "server stop", description: "Fire-and-forget docker-compose down for the fixed UniDesk stack." }, + { command: "server status", description: "Show fixed ports, containers, service health, and public URLs." }, + { command: "server logs [--tail-bytes N]", description: "Return bounded tails from file logs and docker logs." }, + { command: "job list", description: "List async jobs from .state/jobs." }, + { command: "job status [--tail-bytes N]", description: "Show job state with bounded stdout/stderr tails." }, + { command: "debug health", description: "Probe core, overview, nodes, and frontend using real HTTP endpoints." }, + { command: "debug dispatch [providerId] [docker.ps|echo]", description: "Submit a real dispatch request to core for CLI debugging." }, + { command: "e2e run", description: "Run public API, database, provider, and Playwright frontend E2E checks." }, + ], + }; +} + +function numberOption(name: string, defaultValue: number): number { + const index = args.indexOf(name); + if (index === -1) return defaultValue; + const raw = args[index + 1]; + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); + return value; +} + +function latestJobId(): string { + const jobs = listJobs(); + if (jobs.length === 0) throw new Error("No jobs found"); + return jobs[0].id; +} + +async function main(): Promise { + const [top, sub, third, fourth] = args; + if (top === undefined || top === "help" || top === "--help" || top === "-h") { + emitJson(commandName, help()); + return; + } + + if (top === "internal" && sub === "run-job") { + if (!third) throw new Error("internal run-job requires job id"); + emitJson(commandName, await runJob(third)); + return; + } + + const config = readConfig(); + + if (top === "config" && sub === "show") { + emitJson(commandName, { config }); + return; + } + + if (top === "check") { + const result = runChecks(config); + emitJson(commandName, result, result.ok); + if (!result.ok) process.exitCode = 1; + return; + } + + if (top === "server") { + if (sub === "start") { + emitJson(commandName, startStack(config)); + return; + } + if (sub === "stop") { + emitJson(commandName, stopStack(config)); + return; + } + if (sub === "status") { + emitJson(commandName, await stackStatus(config)); + return; + } + if (sub === "logs") { + emitJson(commandName, stackLogs(config, numberOption("--tail-bytes", 3000))); + return; + } + } + + if (top === "job") { + if (sub === "list") { + emitJson(commandName, { jobs: listJobs() }); + return; + } + if (sub === "status") { + const id = third === "latest" || third === undefined ? latestJobId() : third; + emitJson(commandName, { job: jobWithTail(readJob(id), numberOption("--tail-bytes", 12000)) }); + return; + } + } + + if (top === "debug") { + if (sub === "health") { + emitJson(commandName, await debugHealth(config)); + return; + } + if (sub === "dispatch") { + const providerId = third ?? config.providerGateway.id; + const dispatchCommand = fourth === "docker.ps" || fourth === "echo" ? fourth : "docker.ps"; + emitJson(commandName, await debugDispatch(config, providerId, dispatchCommand)); + return; + } + } + + if (top === "e2e" && sub === "run") { + const result = await runE2E(config); + const ok = (result as { ok?: unknown }).ok === true; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } + + throw new Error(`Unknown command: ${commandName}`); +} + +main().catch((error) => { + emitError(commandName, error); + process.exitCode = 1; +}); diff --git a/scripts/src/check.ts b/scripts/src/check.ts new file mode 100644 index 00000000..068b9273 --- /dev/null +++ b/scripts/src/check.ts @@ -0,0 +1,59 @@ +import { existsSync } from "node:fs"; +import { runCommand } from "./command"; +import { type UniDeskConfig, repoRoot, rootPath } from "./config"; +import { composeConfig } from "./docker"; + +interface CheckItem { + name: string; + ok: boolean; + detail: unknown; +} + +function fileItem(path: string): CheckItem { + const absolute = rootPath(path); + return { name: `file:${path}`, ok: existsSync(absolute), detail: absolute }; +} + +function commandItem(name: string, command: string[]): CheckItem { + const result = runCommand(command, repoRoot); + return { + name, + ok: result.exitCode === 0, + detail: { + command, + exitCode: result.exitCode, + stdoutTail: result.stdout.slice(-4000), + stderrTail: result.stderr.slice(-4000), + }, + }; +} + +export function runChecks(config: UniDeskConfig): { ok: boolean; items: CheckItem[] } { + const items: CheckItem[] = [ + { name: "config:validated", ok: true, detail: { project: config.project.name, runtime: config.runtime } }, + fileItem("scripts/cli.ts"), + fileItem("AGENTS.md"), + fileItem("TEST.md"), + fileItem("docker-compose.yml"), + fileItem("src/components/backend-core/src/index.ts"), + fileItem("src/components/frontend/src/index.ts"), + fileItem("src/components/provider-gateway/src/index.ts"), + fileItem("scripts/src/e2e.ts"), + commandItem("bun:version", ["bun", "--version"]), + commandItem("typescript:scripts", ["bunx", "tsc", "-p", "scripts/tsconfig.json", "--noEmit", "--pretty", "false"]), + commandItem("typescript:components", ["bunx", "tsc", "-p", "src/tsconfig.check.json", "--pretty", "false"]), + ]; + const compose = composeConfig(config); + items.push({ + name: "docker-compose:config", + ok: compose.result.exitCode === 0, + detail: { + command: compose.command, + exitCode: compose.result.exitCode, + stdoutTail: compose.result.stdout.slice(-4000), + stderrTail: compose.result.stderr.slice(-4000), + runtimeEnv: compose.runtimeEnv, + }, + }); + return { ok: items.every((item) => item.ok), items }; +} diff --git a/scripts/src/command.ts b/scripts/src/command.ts new file mode 100644 index 00000000..468992dd --- /dev/null +++ b/scripts/src/command.ts @@ -0,0 +1,55 @@ +import { spawn, spawnSync } from "node:child_process"; +import { createWriteStream, existsSync, readFileSync } from "node:fs"; + +export interface CommandResult { + command: string[]; + cwd: string; + exitCode: number | null; + stdout: string; + stderr: string; +} + +export function runCommand(command: string[], cwd: string): CommandResult { + const result = spawnSync(command[0], command.slice(1), { + cwd, + encoding: "utf8", + maxBuffer: 1024 * 1024 * 8, + }); + return { + command, + cwd, + exitCode: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? result.error?.message ?? "", + }; +} + +export function commandOk(command: string[], cwd: string): boolean { + return runCommand(command, cwd).exitCode === 0; +} + +export async function runCommandToFiles(command: string[], cwd: string, stdoutFile: string, stderrFile: string): Promise { + const stdout = createWriteStream(stdoutFile, { flags: "a" }); + const stderr = createWriteStream(stderrFile, { flags: "a" }); + stdout.write(`$ ${command.map((part) => JSON.stringify(part)).join(" ")}\n`); + const child = spawn(command[0], command.slice(1), { cwd, env: process.env }); + child.stdout.pipe(stdout, { end: false }); + child.stderr.pipe(stderr, { end: false }); + const exitCode = await new Promise((resolve) => { + child.on("close", (code) => resolve(code)); + child.on("error", (error) => { + stderr.write(`${error.stack ?? error.message}\n`); + resolve(127); + }); + }); + stdout.write(`\n[exit_code=${exitCode}]\n`); + stdout.end(); + stderr.end(); + return exitCode; +} + +export function tailFile(path: string, maxBytes = 8192): string { + if (!existsSync(path)) return ""; + const content = readFileSync(path); + return content.subarray(Math.max(0, content.length - maxBytes)).toString("utf8"); +} diff --git a/scripts/src/config.ts b/scripts/src/config.ts new file mode 100644 index 00000000..f1a23a40 --- /dev/null +++ b/scripts/src/config.ts @@ -0,0 +1,117 @@ +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export interface UniDeskConfig { + project: { name: string; timezone: string }; + runtime: { typescript: "bun"; bunVersion: string }; + network: { + host: string; + publicHost: string; + core: { port: number; containerPort: number }; + frontend: { port: number; containerPort: number }; + database: { port: number; containerPort: number }; + }; + database: { user: string; password: string; name: string; volume: string; volumeSize: string }; + providerGateway: { + id: string; + name: string; + token: string; + labels: Record; + heartbeatIntervalMs: number; + reconnectBaseMs: number; + reconnectMaxMs: number; + }; + docker: { composeFile: string; projectName: string }; + paths: { stateDir: string; logsDir: string; docsReferenceDir: string }; + sshForwarding: { mode: string; keyDir: string; host: string; port: number; user: string }; +} + +const moduleDir = dirname(fileURLToPath(import.meta.url)); +export const repoRoot = join(moduleDir, "..", ".."); + +export function rootPath(...parts: string[]): string { + return join(repoRoot, ...parts); +} + +function asRecord(value: unknown, name: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${name} must be an object`); + } + return value as Record; +} + +function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value; +} + +function numberField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) throw new Error(`${path}.${key} must be a positive number`); + return value; +} + +function portPair(obj: Record, key: string): { port: number; containerPort: number } { + const value = asRecord(obj[key], `network.${key}`); + return { port: numberField(value, "port", `network.${key}`), containerPort: numberField(value, "containerPort", `network.${key}`) }; +} + +export function readConfig(): UniDeskConfig { + const configPath = rootPath("config.json"); + if (!existsSync(configPath)) throw new Error(`config.json not found at ${configPath}`); + const raw = readFileSync(configPath, "utf8"); + const parsed = asRecord(JSON.parse(raw) as unknown, "config.json"); + const project = asRecord(parsed.project, "project"); + const runtime = asRecord(parsed.runtime, "runtime"); + const network = asRecord(parsed.network, "network"); + const database = asRecord(parsed.database, "database"); + const providerGateway = asRecord(parsed.providerGateway, "providerGateway"); + const docker = asRecord(parsed.docker, "docker"); + const paths = asRecord(parsed.paths, "paths"); + const sshForwarding = asRecord(parsed.sshForwarding, "sshForwarding"); + const labels = asRecord(providerGateway.labels, "providerGateway.labels"); + const typescript = stringField(runtime, "typescript", "runtime"); + if (typescript !== "bun") throw new Error("runtime.typescript must be bun"); + return { + project: { name: stringField(project, "name", "project"), timezone: stringField(project, "timezone", "project") }, + runtime: { typescript, bunVersion: stringField(runtime, "bunVersion", "runtime") }, + network: { + host: stringField(network, "host", "network"), + publicHost: stringField(network, "publicHost", "network"), + core: portPair(network, "core"), + frontend: portPair(network, "frontend"), + database: portPair(network, "database"), + }, + database: { + user: stringField(database, "user", "database"), + password: stringField(database, "password", "database"), + name: stringField(database, "name", "database"), + volume: stringField(database, "volume", "database"), + volumeSize: stringField(database, "volumeSize", "database"), + }, + providerGateway: { + id: stringField(providerGateway, "id", "providerGateway"), + name: stringField(providerGateway, "name", "providerGateway"), + token: stringField(providerGateway, "token", "providerGateway"), + labels, + heartbeatIntervalMs: numberField(providerGateway, "heartbeatIntervalMs", "providerGateway"), + reconnectBaseMs: numberField(providerGateway, "reconnectBaseMs", "providerGateway"), + reconnectMaxMs: numberField(providerGateway, "reconnectMaxMs", "providerGateway"), + }, + docker: { composeFile: stringField(docker, "composeFile", "docker"), projectName: stringField(docker, "projectName", "docker") }, + paths: { + stateDir: stringField(paths, "stateDir", "paths"), + logsDir: stringField(paths, "logsDir", "paths"), + docsReferenceDir: stringField(paths, "docsReferenceDir", "paths"), + }, + sshForwarding: { + mode: stringField(sshForwarding, "mode", "sshForwarding"), + keyDir: stringField(sshForwarding, "keyDir", "sshForwarding"), + host: stringField(sshForwarding, "host", "sshForwarding"), + port: numberField(sshForwarding, "port", "sshForwarding"), + user: stringField(sshForwarding, "user", "sshForwarding"), + }, + }; +} diff --git a/scripts/src/debug.ts b/scripts/src/debug.ts new file mode 100644 index 00000000..7ecb1172 --- /dev/null +++ b/scripts/src/debug.ts @@ -0,0 +1,30 @@ +import { type UniDeskConfig } from "./config"; + +async function readJson(url: string, init?: RequestInit): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 5000); + try { + const res = await fetch(url, { ...init, signal: controller.signal }); + const text = await res.text(); + return { ok: res.ok, status: res.status, body: text.length > 0 ? JSON.parse(text) : null }; + } finally { + clearTimeout(timer); + } +} + +export async function debugHealth(config: UniDeskConfig): Promise { + return { + core: await readJson(`http://127.0.0.1:${config.network.core.port}/health`), + overview: await readJson(`http://127.0.0.1:${config.network.core.port}/api/overview`), + nodes: await readJson(`http://127.0.0.1:${config.network.core.port}/api/nodes`), + frontend: await readJson(`http://127.0.0.1:${config.network.frontend.port}/health`), + }; +} + +export async function debugDispatch(config: UniDeskConfig, providerId: string, command: "docker.ps" | "echo"): Promise { + return readJson(`http://127.0.0.1:${config.network.core.port}/api/dispatch`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ providerId, command, payload: { source: "cli-debug" } }), + }); +} diff --git a/scripts/src/docker.ts b/scripts/src/docker.ts new file mode 100644 index 00000000..c679c53b --- /dev/null +++ b/scripts/src/docker.ts @@ -0,0 +1,213 @@ +import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; +import { commandOk, runCommand, tailFile } from "./command"; +import { type UniDeskConfig, repoRoot, rootPath } from "./config"; +import { startJob } from "./jobs"; + +export interface ComposeRuntimeEnv { + envFile: string; + logDir: string; + logPrefix: string; +} + +export interface ContainerStatus { + id: string; + name: string; + image: string; + status: string; + ports: string; +} + +export function resolveComposeCommand(config: UniDeskConfig, envFile: string): string[] { + const composeFile = rootPath(config.docker.composeFile); + if (commandOk(["docker", "compose", "version"], repoRoot)) { + return ["docker", "compose", "--env-file", envFile, "-f", composeFile, "-p", config.docker.projectName]; + } + if (commandOk(["docker-compose", "--version"], repoRoot)) { + return ["docker-compose", "--env-file", envFile, "-f", composeFile, "-p", config.docker.projectName]; + } + throw new Error("Neither docker compose plugin nor docker-compose binary is available"); +} + +function pad(value: number): string { + return String(value).padStart(2, "0"); +} + +function localDateParts(date: Date): { day: string; stamp: string } { + const day = `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}`; + const stamp = `${day}_${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; + return { day, stamp }; +} + +function envValue(value: string): string { + if (/^[A-Za-z0-9_./:@-]+$/.test(value)) return value; + return JSON.stringify(value); +} + +export function writeComposeEnv(config: UniDeskConfig, freshLogPrefix: boolean): ComposeRuntimeEnv { + const stateDir = rootPath(config.paths.stateDir); + mkdirSync(stateDir, { recursive: true }); + const envFile = join(stateDir, "docker-compose.env"); + if (!freshLogPrefix && existsSync(envFile)) { + const raw = readFileSync(envFile, "utf8"); + const logDir = raw.match(/^UNIDESK_LOG_DIR=(.*)$/m)?.[1]?.replace(/^"|"$/g, "") ?? rootPath(config.paths.logsDir); + const logPrefix = raw.match(/^UNIDESK_LOG_PREFIX=(.*)$/m)?.[1]?.replace(/^"|"$/g, "") ?? localDateParts(new Date()).stamp; + return { envFile, logDir, logPrefix }; + } + const parts = localDateParts(new Date()); + const logDir = resolve(rootPath(config.paths.logsDir, parts.day)); + mkdirSync(logDir, { recursive: true }); + chmodSync(logDir, 0o777); + const labels = JSON.stringify(config.providerGateway.labels); + const lines = { + UNIDESK_PUBLIC_HOST: config.network.publicHost, + UNIDESK_CORE_PORT: String(config.network.core.port), + UNIDESK_FRONTEND_PORT: String(config.network.frontend.port), + UNIDESK_DATABASE_PORT: String(config.network.database.port), + UNIDESK_DATABASE_USER: config.database.user, + UNIDESK_DATABASE_PASSWORD: config.database.password, + UNIDESK_DATABASE_NAME: config.database.name, + UNIDESK_PROVIDER_TOKEN: config.providerGateway.token, + UNIDESK_PROVIDER_ID: config.providerGateway.id, + UNIDESK_PROVIDER_NAME: config.providerGateway.name, + UNIDESK_PROVIDER_LABELS_JSON: labels, + UNIDESK_HEARTBEAT_INTERVAL_MS: String(config.providerGateway.heartbeatIntervalMs), + UNIDESK_HEARTBEAT_TIMEOUT_MS: "90000", + UNIDESK_RECONNECT_BASE_MS: String(config.providerGateway.reconnectBaseMs), + UNIDESK_RECONNECT_MAX_MS: String(config.providerGateway.reconnectMaxMs), + UNIDESK_LOG_DIR: logDir, + UNIDESK_LOG_PREFIX: parts.stamp, + UNIDESK_HOST_SSH_HOST: config.sshForwarding.host, + UNIDESK_HOST_SSH_PORT: String(config.sshForwarding.port), + UNIDESK_HOST_SSH_USER: config.sshForwarding.user, + }; + writeFileSync(envFile, Object.entries(lines).map(([key, value]) => `${key}=${envValue(value)}`).join("\n") + "\n", "utf8"); + return { envFile, logDir, logPrefix: parts.stamp }; +} + +export function composeConfig(config: UniDeskConfig): { runtimeEnv: ComposeRuntimeEnv; command: string[]; result: ReturnType } { + const runtimeEnv = writeComposeEnv(config, false); + const compose = resolveComposeCommand(config, runtimeEnv.envFile); + const command = [...compose, "config", "-q"]; + return { runtimeEnv, command, result: runCommand(command, repoRoot) }; +} + +export function startStack(config: UniDeskConfig): unknown { + const runtimeEnv = writeComposeEnv(config, true); + const compose = resolveComposeCommand(config, runtimeEnv.envFile); + const containers = dockerContainers(config); + const occupiedPorts = fixedPorts(config).filter((item) => item.listening); + if (occupiedPorts.length > 0 && containers.length === 0) { + throw new Error(`Fixed UniDesk port is occupied before start: ${occupiedPorts.map((p) => `${p.name}:${p.port}`).join(", ")}`); + } + const downCommand = [...compose, "down", "--remove-orphans"]; + const upCommand = [...compose, "up", "-d", "--build"]; + const command = ["bash", "-lc", `set -euo pipefail; ${shellJoin(downCommand)}; ${shellJoin(upCommand)}`]; + const job = startJob("server_start", command, "Build and start UniDesk database, core, frontend, and provider gateway containers"); + return { job, runtimeEnv, command, ports: fixedPorts(config) }; +} + +export function stopStack(config: UniDeskConfig): unknown { + const runtimeEnv = writeComposeEnv(config, false); + const compose = resolveComposeCommand(config, runtimeEnv.envFile); + const command = [...compose, "down", "--remove-orphans"]; + const job = startJob("server_stop", command, "Stop all UniDesk Docker services managed by the fixed compose project"); + return { job, runtimeEnv, command, portsBeforeStop: fixedPorts(config) }; +} + +function fixedPorts(config: UniDeskConfig): Array<{ name: string; port: number; listening: boolean }> { + return [ + { name: "backend-core", port: config.network.core.port, listening: isPortListening(config.network.core.port) }, + { name: "frontend", port: config.network.frontend.port, listening: isPortListening(config.network.frontend.port) }, + { name: "database", port: config.network.database.port, listening: isPortListening(config.network.database.port) }, + ]; +} + +function shellJoin(args: string[]): string { + return args.map((arg) => `'${arg.replace(/'/g, `'\\''`)}'`).join(" "); +} + +function isPortListening(port: number): boolean { + const result = runCommand(["ss", "-ltn"], repoRoot); + if (result.exitCode !== 0) return false; + return result.stdout.split("\n").some((line) => line.includes(`:${port} `) || line.includes(`:${port}\t`)); +} + +export function dockerContainers(config: UniDeskConfig): ContainerStatus[] { + const result = runCommand([ + "docker", + "ps", + "-a", + "--filter", + `label=com.docker.compose.project=${config.docker.projectName}`, + "--format", + "{{json .}}", + ], repoRoot); + if (result.exitCode !== 0 || result.stdout.trim().length === 0) return []; + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as Record) + .map((row) => ({ id: row.ID ?? "", name: row.Names ?? "", image: row.Image ?? "", status: row.Status ?? "", ports: row.Ports ?? "" })); +} + +async function probe(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 2500); + try { + const res = await fetch(url, { signal: controller.signal }); + const body = await res.text(); + return { ok: res.ok, status: res.status, body: body.slice(0, 1200) }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + clearTimeout(timer); + } +} + +export async function stackStatus(config: UniDeskConfig): Promise { + const runtimeEnv = writeComposeEnv(config, false); + return { + runtimeEnv, + ports: fixedPorts(config), + containers: dockerContainers(config), + health: { + core: await probe(`http://127.0.0.1:${config.network.core.port}/health`), + frontend: await probe(`http://127.0.0.1:${config.network.frontend.port}/health`), + overview: await probe(`http://127.0.0.1:${config.network.core.port}/api/overview`), + }, + urls: { + core: `http://${config.network.publicHost}:${config.network.core.port}`, + frontend: `http://${config.network.publicHost}:${config.network.frontend.port}`, + database: `postgres://${config.database.user}:***@${config.network.publicHost}:${config.network.database.port}/${config.database.name}`, + }, + }; +} + +function listLogFiles(root: string): string[] { + if (!existsSync(root)) return []; + const entries = readdirSync(root, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...listLogFiles(path)); + if (entry.isFile()) files.push(path); + } + return files.sort(); +} + +export function stackLogs(config: UniDeskConfig, tailBytes: number): unknown { + const logRoot = rootPath(config.paths.logsDir); + const runtimeEnv = writeComposeEnv(config, false); + const allFiles = listLogFiles(logRoot); + const currentFiles = allFiles.filter((path) => basename(path).startsWith(runtimeEnv.logPrefix)); + const selectedFiles = (currentFiles.length > 0 ? currentFiles : allFiles.slice(-12)).slice(-12); + const files = selectedFiles.map((path) => ({ path, name: basename(path), tail: tailFile(path, tailBytes) })); + const containerNames = ["unidesk-database", "unidesk-backend-core", "unidesk-frontend", "unidesk-provider-gateway-main"]; + const docker = containerNames.map((name) => { + const result = runCommand(["docker", "logs", "--tail", "40", name], repoRoot); + return { name, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-tailBytes), stderrTail: result.stderr.slice(-tailBytes) }; + }); + return { logRoot, runtimeEnv, files, docker }; +} diff --git a/scripts/src/e2e.ts b/scripts/src/e2e.ts new file mode 100644 index 00000000..82c52c75 --- /dev/null +++ b/scripts/src/e2e.ts @@ -0,0 +1,154 @@ +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { chromium } from "playwright"; +import { runCommand } from "./command"; +import { type UniDeskConfig, repoRoot, rootPath } from "./config"; + +type CheckStatus = "passed" | "failed"; + +interface E2ECheck { + name: string; + status: CheckStatus; + detail: unknown; +} + +interface PublicUrls { + frontendUrl: string; + coreUrl: string; + databaseHost: string; + databasePort: number; +} + +function publicUrls(config: UniDeskConfig): PublicUrls { + return { + frontendUrl: `http://${config.network.publicHost}:${config.network.frontend.port}`, + coreUrl: `http://${config.network.publicHost}:${config.network.core.port}`, + databaseHost: config.network.publicHost, + databasePort: config.network.database.port, + }; +} + +async function fetchJson(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 8000); + try { + const response = await fetch(url, { signal: controller.signal }); + const text = await response.text(); + return { ok: response.ok, status: response.status, body: text.length > 0 ? JSON.parse(text) : null }; + } finally { + clearTimeout(timer); + } +} + +function addCheck(checks: E2ECheck[], name: string, passed: boolean, detail: unknown): void { + checks.push({ name, status: passed ? "passed" : "failed", detail }); +} + +function runPsql(config: UniDeskConfig, sql: string): { ok: boolean; stdout: string; stderr: string; exitCode: number | null } { + const result = runCommand([ + "docker", + "exec", + "unidesk-database", + "psql", + "-U", + config.database.user, + "-d", + config.database.name, + "-v", + "ON_ERROR_STOP=1", + "-tA", + "-c", + sql, + ], repoRoot); + return { ok: result.exitCode === 0, stdout: result.stdout.trim(), stderr: result.stderr.trim(), exitCode: result.exitCode }; +} + +async function apiChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2ECheck[]): Promise { + const overview = await fetchJson(`${urls.coreUrl}/api/overview`); + const nodes = await fetchJson(`${urls.coreUrl}/api/nodes`); + addCheck(checks, "core:public-overview", (overview as { ok?: boolean; body?: { ok?: boolean; onlineNodeCount?: number } }).ok === true && (overview as { body?: { ok?: boolean; onlineNodeCount?: number } }).body?.ok === true && ((overview as { body?: { onlineNodeCount?: number } }).body?.onlineNodeCount ?? 0) >= 1, overview); + const nodeList = (nodes as { body?: { nodes?: Array<{ providerId?: string; status?: string }> } }).body?.nodes ?? []; + addCheck(checks, "core:public-nodes", nodeList.some((node) => node.providerId === config.providerGateway.id && node.status === "online"), nodes); +} + +function databaseChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2ECheck[]): string { + const markerId = `e2e_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; + const markerSql = ` + CREATE TABLE IF NOT EXISTS unidesk_e2e_markers ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); + INSERT INTO unidesk_e2e_markers (id, source) VALUES ('${markerId}', 'cli-e2e'); + SELECT 'marker=' || id FROM unidesk_e2e_markers WHERE id = '${markerId}'; + SELECT 'marker_count=' || count(*) FROM unidesk_e2e_markers; + SELECT 'online_main_server=' || count(*) FROM unidesk_nodes WHERE provider_id = '${config.providerGateway.id}' AND status = 'online'; + `; + const marker = runPsql(config, markerSql); + addCheck(checks, "database:named-volume-write", marker.ok && marker.stdout.includes(`marker=${markerId}`), marker); + addCheck(checks, "database:provider-state", marker.ok && marker.stdout.includes("online_main_server=1"), marker); + + const publicProbe = runCommand([ + "docker", + "run", + "--rm", + "postgres:16-alpine", + "pg_isready", + "-h", + urls.databaseHost, + "-p", + String(urls.databasePort), + "-U", + config.database.user, + ], repoRoot); + addCheck(checks, "database:public-port", publicProbe.exitCode === 0, { + exitCode: publicProbe.exitCode, + stdout: publicProbe.stdout.trim(), + stderr: publicProbe.stderr.trim(), + }); + return markerId; +} + +async function frontendCheck(config: UniDeskConfig, urls: PublicUrls, checks: E2ECheck[]): Promise<{ screenshotPath: string; bodyText: string; consoleErrors: string[] }> { + const e2eDir = rootPath(".state", "e2e"); + mkdirSync(e2eDir, { recursive: true }); + const screenshotPath = join(e2eDir, `${new Date().toISOString().replace(/[-:.TZ]/g, "")}_frontend.png`); + const consoleErrors: string[] = []; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage({ viewport: { width: 1440, height: 920 } }); + page.on("console", (message) => { + if (message.type() === "error") consoleErrors.push(message.text()); + }); + page.on("pageerror", (error) => consoleErrors.push(error.message)); + await page.goto(urls.frontendUrl, { waitUntil: "domcontentloaded", timeout: 15000 }); + await page.waitForFunction(() => document.querySelector("#conn-text")?.textContent?.includes("核心在线"), undefined, { timeout: 15000 }); + await page.waitForSelector(`text=${config.providerGateway.id}`, { timeout: 10000 }); + await page.waitForSelector("text=Online Nodes", { timeout: 5000 }); + const bodyText = await page.locator("body").innerText({ timeout: 5000 }); + const nodeCount = await page.locator("#node-count").innerText({ timeout: 5000 }); + const metricText = await page.locator("#metric-grid").innerText({ timeout: 5000 }); + await page.screenshot({ path: screenshotPath, fullPage: true }); + addCheck(checks, "frontend:public-page-provider-visible", bodyText.includes(config.providerGateway.id) && bodyText.includes(config.providerGateway.name), { nodeCount, metricText, screenshotPath }); + addCheck(checks, "frontend:no-console-errors", consoleErrors.length === 0, { consoleErrors }); + return { screenshotPath, bodyText, consoleErrors }; + } finally { + await browser.close(); + } +} + +export async function runE2E(config: UniDeskConfig): Promise { + const checks: E2ECheck[] = []; + const urls = publicUrls(config); + await apiChecks(config, urls, checks); + const markerId = databaseChecks(config, urls, checks); + const frontend = await frontendCheck(config, urls, checks); + const ok = checks.every((check) => check.status === "passed"); + return { + ok, + urls, + markerId, + screenshotPath: frontend.screenshotPath, + checks, + }; +} diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts new file mode 100644 index 00000000..8e8e13e3 --- /dev/null +++ b/scripts/src/jobs.ts @@ -0,0 +1,95 @@ +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { repoRoot, rootPath } from "./config"; +import { runCommandToFiles, tailFile } from "./command"; + +export type JobStatus = "queued" | "running" | "succeeded" | "failed"; + +export interface JobRecord { + id: string; + name: string; + status: JobStatus; + command: string[]; + cwd: string; + createdAt: string; + startedAt: string | null; + finishedAt: string | null; + exitCode: number | null; + stdoutFile: string; + stderrFile: string; + note: string; +} + +function jobsDir(): string { + const dir = rootPath(".state", "jobs"); + mkdirSync(dir, { recursive: true }); + return dir; +} + +function jobPath(id: string): string { + return join(jobsDir(), `${id}.json`); +} + +function writeJob(job: JobRecord): void { + writeFileSync(jobPath(job.id), `${JSON.stringify(job, null, 2)}\n`, "utf8"); +} + +export function readJob(id: string): JobRecord { + const path = jobPath(id); + if (!existsSync(path)) throw new Error(`job not found: ${id}`); + return JSON.parse(readFileSync(path, "utf8")) as JobRecord; +} + +export function listJobs(): JobRecord[] { + return readdirSync(jobsDir()) + .filter((name) => name.endsWith(".json")) + .map((name) => JSON.parse(readFileSync(join(jobsDir(), name), "utf8")) as JobRecord) + .sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +} + +export function startJob(name: string, command: string[], note: string): JobRecord { + const id = `${name}_${new Date().toISOString().replace(/[-:.TZ]/g, "")}_${Math.random().toString(16).slice(2, 8)}`; + const stdoutFile = rootPath(".state", "jobs", `${id}.stdout.log`); + const stderrFile = rootPath(".state", "jobs", `${id}.stderr.log`); + const job: JobRecord = { + id, + name, + status: "queued", + command, + cwd: repoRoot, + createdAt: new Date().toISOString(), + startedAt: null, + finishedAt: null, + exitCode: null, + stdoutFile, + stderrFile, + note, + }; + writeJob(job); + const child = spawn(process.execPath, [rootPath("scripts", "cli.ts"), "internal", "run-job", id], { + cwd: repoRoot, + detached: true, + stdio: "ignore", + env: process.env, + }); + child.unref(); + return job; +} + +export async function runJob(id: string): Promise { + const job = readJob(id); + job.status = "running"; + job.startedAt = new Date().toISOString(); + writeJob(job); + const exitCode = await runCommandToFiles(job.command, job.cwd, job.stdoutFile, job.stderrFile); + job.exitCode = exitCode; + job.status = exitCode === 0 ? "succeeded" : "failed"; + job.finishedAt = new Date().toISOString(); + writeJob(job); + return job; +} + +export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { stdoutTail: string; stderrTail: string } { + return { ...job, stdoutTail: tailFile(job.stdoutFile, maxBytes), stderrTail: tailFile(job.stderrFile, maxBytes) }; +} diff --git a/scripts/src/output.ts b/scripts/src/output.ts new file mode 100644 index 00000000..addfde5c --- /dev/null +++ b/scripts/src/output.ts @@ -0,0 +1,31 @@ +export interface JsonEnvelope { + ok: boolean; + command: string; + data?: T; + error?: unknown; +} + +export function emitJson(command: string, data: T, ok = true): void { + const envelope: JsonEnvelope = { ok, command, data }; + safeStdoutWrite(`${JSON.stringify(envelope, null, 2)}\n`); +} + +export function emitError(command: string, error: unknown): void { + const payload = error instanceof Error + ? { name: error.name, message: error.message, stack: error.stack ?? null } + : { message: String(error) }; + const envelope: JsonEnvelope = { ok: false, command, error: payload }; + safeStdoutWrite(`${JSON.stringify(envelope, null, 2)}\n`); +} + +function safeStdoutWrite(text: string): void { + try { + process.stdout.write(text); + } catch (error) { + if (typeof error === "object" && error !== null && "code" in error && (error as { code?: unknown }).code === "EPIPE") { + process.exitCode = 0; + return; + } + throw error; + } +} diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json new file mode 100644 index 00000000..37402e91 --- /dev/null +++ b/scripts/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["bun", "node"], + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true + }, + "include": ["cli.ts", "src/**/*.ts"] +} diff --git a/src/bun.lock b/src/bun.lock new file mode 100644 index 00000000..7e9b84ce --- /dev/null +++ b/src/bun.lock @@ -0,0 +1,50 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@unidesk/components", + "devDependencies": { + "@types/bun": "latest", + "@types/node": "latest", + "typescript": "latest", + }, + }, + "components/backend-core": { + "name": "@unidesk/backend-core", + "dependencies": { + "postgres": "latest", + }, + }, + "components/frontend": { + "name": "@unidesk/frontend", + }, + "components/provider-gateway": { + "name": "@unidesk/provider-gateway", + }, + "components/shared": { + "name": "@unidesk/shared", + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + + "@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], + + "@unidesk/backend-core": ["@unidesk/backend-core@workspace:components/backend-core"], + + "@unidesk/frontend": ["@unidesk/frontend@workspace:components/frontend"], + + "@unidesk/provider-gateway": ["@unidesk/provider-gateway@workspace:components/provider-gateway"], + + "@unidesk/shared": ["@unidesk/shared@workspace:components/shared"], + + "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + + "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + + "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], + } +} diff --git a/src/components/backend-core/Dockerfile b/src/components/backend-core/Dockerfile new file mode 100644 index 00000000..4d1cd683 --- /dev/null +++ b/src/components/backend-core/Dockerfile @@ -0,0 +1,7 @@ +FROM oven/bun:1-alpine +WORKDIR /app/src/components/backend-core +COPY src/components/backend-core/package.json ./package.json +RUN bun install --production +COPY src/components/shared /app/src/components/shared +COPY src/components/backend-core/src ./src +CMD ["bun", "run", "src/index.ts"] diff --git a/src/components/backend-core/package.json b/src/components/backend-core/package.json new file mode 100644 index 00000000..9b835e24 --- /dev/null +++ b/src/components/backend-core/package.json @@ -0,0 +1,12 @@ +{ + "name": "@unidesk/backend-core", + "private": true, + "type": "module", + "scripts": { + "start": "bun run src/index.ts", + "check": "tsc -p tsconfig.json --noEmit" + }, + "dependencies": { + "postgres": "latest" + } +} diff --git a/src/components/backend-core/src/index.ts b/src/components/backend-core/src/index.ts new file mode 100644 index 00000000..da810b6c --- /dev/null +++ b/src/components/backend-core/src/index.ts @@ -0,0 +1,484 @@ +import { appendFileSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import type { Server, ServerWebSocket } from "bun"; +import postgres from "postgres"; +import { + type ApiEvent, + type ApiNode, + type ApiTask, + type CoreDispatchMessage, + type JsonValue, + type ProviderLabels, + type ProviderToCoreMessage, + isProviderToCoreMessage, +} from "../../shared/src/index"; + +interface RuntimeConfig { + port: number; + databaseUrl: string; + providerToken: string; + heartbeatTimeoutMs: number; + logFile: string; +} + +interface WsData { + providerId?: string; +} + +type ProviderSocket = ServerWebSocket; + +type SqlClient = ReturnType; + +const recentLogs: unknown[] = []; +const activeProviders = new Map(); +const serviceStartedAt = new Date(); +const config = readConfig(); +const logger = createLogger("backend-core", config.logFile); +const sql = postgres(config.databaseUrl, { + max: 8, + idle_timeout: 20, + connect_timeout: 10, +}); + +let dbReady = false; + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +function readNumberEnv(name: string): number { + const raw = requiredEnv(name); + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); + } + return parsed; +} + +function readConfig(): RuntimeConfig { + return { + port: readNumberEnv("PORT"), + databaseUrl: requiredEnv("DATABASE_URL"), + providerToken: requiredEnv("PROVIDER_TOKEN"), + heartbeatTimeoutMs: readNumberEnv("HEARTBEAT_TIMEOUT_MS"), + logFile: requiredEnv("LOG_FILE"), + }; +} + +function createLogger(service: string, logFile: string) { + mkdirSync(dirname(logFile), { recursive: true }); + return (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue): void => { + const entry = data === undefined + ? { ts: new Date().toISOString(), service, level, message } + : { ts: new Date().toISOString(), service, level, message, data }; + recentLogs.push(entry); + while (recentLogs.length > 500) recentLogs.shift(); + const line = `${JSON.stringify(entry)}\n`; + try { + appendFileSync(logFile, line, "utf8"); + } catch (error) { + console.error(JSON.stringify({ ts: new Date().toISOString(), service, level: "error", message: "log_write_failed", data: String(error) })); + } + const consoleMethod = level === "error" ? console.error : level === "warn" ? console.warn : console.log; + consoleMethod(line.trimEnd()); + }; +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body, null, 2), { + status, + headers: { + "content-type": "application/json; charset=utf-8", + "access-control-allow-origin": "*", + "access-control-allow-methods": "GET,POST,OPTIONS", + "access-control-allow-headers": "content-type,x-provider-token", + }, + }); +} + +function textResponse(text: string, status = 200): Response { + return new Response(text, { + status, + headers: { + "content-type": "text/plain; charset=utf-8", + "access-control-allow-origin": "*", + }, + }); +} + +async function initDatabase(client: SqlClient): Promise { + logger("info", "database_init_start", { databaseUrl: redactDatabaseUrl(config.databaseUrl) }); + await client` + CREATE TABLE IF NOT EXISTS unidesk_nodes ( + provider_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + labels JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL, + connected_at TIMESTAMPTZ, + last_heartbeat TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_events ( + id BIGSERIAL PRIMARY KEY, + type TEXT NOT NULL, + source TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + await client` + CREATE TABLE IF NOT EXISTS unidesk_tasks ( + id TEXT PRIMARY KEY, + provider_id TEXT NOT NULL, + command TEXT NOT NULL, + status TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + result JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + `; + dbReady = true; + logger("info", "database_init_complete"); +} + +async function initDatabaseWithRetry(): Promise { + const started = Date.now(); + let attempt = 0; + while (!dbReady) { + attempt += 1; + try { + await initDatabase(sql); + return; + } catch (error) { + const elapsedMs = Date.now() - started; + logger("warn", "database_init_retry", { attempt, elapsedMs, error: errorToJson(error) }); + if (elapsedMs > 90_000) { + logger("error", "database_init_failed", { attempt, elapsedMs, error: errorToJson(error) }); + throw error; + } + await Bun.sleep(Math.min(1000 * attempt, 5000)); + } + } +} + +function redactDatabaseUrl(value: string): string { + try { + const url = new URL(value); + if (url.password) url.password = "***"; + return url.toString(); + } catch { + return ""; + } +} + +function errorToJson(error: unknown): JsonValue { + if (error instanceof Error) { + return { name: error.name, message: error.message, stack: error.stack ?? null }; + } + return String(error); +} + +function compactJson(value: unknown, depth = 0): JsonValue { + if (value === null || typeof value === "number" || typeof value === "boolean") return value; + if (typeof value === "string") return value.length > 600 ? `${value.slice(0, 600)}...` : value; + if (Array.isArray(value)) { + const items = value.slice(0, 20).map((item) => compactJson(item, depth + 1)); + if (value.length > 20) items.push({ truncatedItems: value.length - 20 }); + return items; + } + if (typeof value === "object" && value !== null) { + if (depth >= 4) return ""; + const entries = Object.entries(value as Record).slice(0, 30); + const result: Record = {}; + for (const [key, item] of entries) result[key] = compactJson(item, depth + 1); + const total = Object.keys(value as Record).length; + if (total > entries.length) result.truncatedKeys = total - entries.length; + return result; + } + return String(value); +} + +async function recordEvent(type: string, source: string, payload: JsonValue): Promise { + logger("info", type, { source, payload }); + if (!dbReady) return; + try { + await sql` + INSERT INTO unidesk_events (type, source, payload) + VALUES (${type}, ${source}, ${sql.json(payload)}) + `; + } catch (error) { + logger("error", "event_insert_failed", { type, source, error: errorToJson(error) }); + } +} + +async function upsertNodeOnline(providerId: string, name: string, labels: ProviderLabels): Promise { + await sql` + INSERT INTO unidesk_nodes (provider_id, name, labels, status, connected_at, last_heartbeat, updated_at) + VALUES (${providerId}, ${name}, ${sql.json(labels)}, 'online', now(), now(), now()) + ON CONFLICT (provider_id) DO UPDATE SET + name = EXCLUDED.name, + labels = EXCLUDED.labels, + status = 'online', + connected_at = COALESCE(unidesk_nodes.connected_at, EXCLUDED.connected_at), + last_heartbeat = now(), + updated_at = now() + `; +} + +async function touchHeartbeat(providerId: string, labels: ProviderLabels): Promise { + await sql` + UPDATE unidesk_nodes + SET labels = ${sql.json(labels)}, status = 'online', last_heartbeat = now(), updated_at = now() + WHERE provider_id = ${providerId} + `; +} + +async function markProviderOffline(providerId: string): Promise { + activeProviders.delete(providerId); + if (!dbReady) return; + await sql` + UPDATE unidesk_nodes + SET status = 'offline', updated_at = now() + WHERE provider_id = ${providerId} + `; + await recordEvent("provider_offline", providerId, { providerId }); +} + +async function markStaleProvidersOffline(): Promise { + if (!dbReady) return; + const timeoutMs = config.heartbeatTimeoutMs; + const rows = await sql<{ provider_id: string }[]>` + UPDATE unidesk_nodes + SET status = 'offline', updated_at = now() + WHERE status = 'online' + AND last_heartbeat IS NOT NULL + AND last_heartbeat < now() - (${timeoutMs} * interval '1 millisecond') + RETURNING provider_id + `; + for (const row of rows) { + activeProviders.delete(row.provider_id); + await recordEvent("provider_heartbeat_timeout", row.provider_id, { providerId: row.provider_id, timeoutMs }); + } +} + +function parseMessage(raw: string | Buffer): ProviderToCoreMessage { + const text = typeof raw === "string" ? raw : raw.toString("utf8"); + const parsed = JSON.parse(text) as unknown; + if (!isProviderToCoreMessage(parsed)) { + throw new Error(`Unsupported provider message: ${text.slice(0, 200)}`); + } + return parsed; +} + +async function handleProviderMessage(ws: ProviderSocket, raw: string | Buffer): Promise { + const message = parseMessage(raw); + ws.data.providerId = message.providerId; + activeProviders.set(message.providerId, ws); + + if (message.type === "register") { + await upsertNodeOnline(message.providerId, message.name, message.labels); + await recordEvent("provider_registered", message.providerId, { + providerId: message.providerId, + name: message.name, + labels: message.labels, + capabilities: message.capabilities, + }); + ws.send(JSON.stringify({ type: "ack", requestId: "register", ok: true, message: "registered" })); + return; + } + + if (message.type === "heartbeat") { + await touchHeartbeat(message.providerId, message.labels); + logger("debug", "provider_heartbeat", { providerId: message.providerId, labels: message.labels }); + return; + } + + await sql` + UPDATE unidesk_tasks + SET status = ${message.status}, result = ${sql.json(message.result ?? { message: message.message })}, updated_at = now() + WHERE id = ${message.taskId} + `; + await recordEvent("task_status", message.providerId, { + providerId: message.providerId, + taskId: message.taskId, + status: message.status, + message: message.message, + result: message.result ?? null, + }); +} + +async function getNodes(): Promise { + const rows = await sql>>` + SELECT provider_id, name, labels, status, connected_at, last_heartbeat + FROM unidesk_nodes + ORDER BY status DESC, provider_id ASC + `; + return rows.map((row) => ({ + providerId: String(row.provider_id), + name: String(row.name), + status: row.status === "online" ? "online" : "offline", + labels: (row.labels ?? {}) as ProviderLabels, + connectedAt: row.connected_at instanceof Date ? row.connected_at.toISOString() : row.connected_at === null ? null : String(row.connected_at), + lastHeartbeat: row.last_heartbeat instanceof Date ? row.last_heartbeat.toISOString() : row.last_heartbeat === null ? null : String(row.last_heartbeat), + })); +} + +async function getEvents(limit: number): Promise { + const rows = await sql>>` + SELECT id, type, source, payload, created_at + FROM unidesk_events + ORDER BY id DESC + LIMIT ${limit} + `; + return rows.map((row) => ({ + id: Number(row.id), + type: String(row.type), + source: String(row.source), + payload: compactJson(row.payload ?? {}), + createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), + })); +} + +async function getTasks(limit: number): Promise { + const rows = await sql>>` + SELECT id, provider_id, command, status, payload, result, created_at, updated_at + FROM unidesk_tasks + ORDER BY updated_at DESC + LIMIT ${limit} + `; + return rows.map((row) => ({ + id: String(row.id), + providerId: String(row.provider_id), + command: String(row.command), + status: String(row.status), + payload: compactJson(row.payload ?? {}), + result: compactJson(row.result ?? null), + createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at), + updatedAt: row.updated_at instanceof Date ? row.updated_at.toISOString() : String(row.updated_at), + })); +} + +async function getOverview(): Promise { + const nodes = await getNodes(); + const tasks = await getTasks(50); + const online = nodes.filter((node) => node.status === "online").length; + const pendingTasks = tasks.filter((task) => task.status === "queued" || task.status === "dispatched" || task.status === "running").length; + return { + service: "unidesk-core", + ok: true, + dbReady, + uptimeSeconds: Math.floor((Date.now() - serviceStartedAt.getTime()) / 1000), + nodeCount: nodes.length, + onlineNodeCount: online, + pendingTaskCount: pendingTasks, + activeSocketCount: activeProviders.size, + heartbeatTimeoutMs: config.heartbeatTimeoutMs, + }; +} + +async function dispatchTask(req: Request): Promise { + const body = (await req.json()) as { providerId?: unknown; command?: unknown; payload?: unknown }; + const providerId = typeof body.providerId === "string" ? body.providerId : ""; + const command = body.command === "docker.ps" || body.command === "echo" ? body.command : "echo"; + const payload = typeof body.payload === "object" && body.payload !== null ? (body.payload as Record) : {}; + if (!providerId) { + return jsonResponse({ ok: false, error: "providerId is required" }, 400); + } + const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`; + await sql` + INSERT INTO unidesk_tasks (id, provider_id, command, status, payload, result) + VALUES (${taskId}, ${providerId}, ${command}, 'queued', ${sql.json(payload)}, NULL) + `; + const socket = activeProviders.get(providerId); + if (!socket) { + await recordEvent("task_queued_provider_offline", providerId, { taskId, providerId, command }); + return jsonResponse({ ok: true, taskId, status: "queued", providerOnline: false }); + } + const dispatch: CoreDispatchMessage = { type: "dispatch", taskId, command, payload }; + socket.send(JSON.stringify(dispatch)); + await sql`UPDATE unidesk_tasks SET status = 'dispatched', updated_at = now() WHERE id = ${taskId}`; + await recordEvent("task_dispatched", providerId, { taskId, providerId, command }); + return jsonResponse({ ok: true, taskId, status: "dispatched", providerOnline: true }); +} + +async function route(req: Request, server: Server): Promise { + const url = new URL(req.url); + if (req.method === "OPTIONS") return jsonResponse({ ok: true }); + + if (url.pathname === "/ws/provider") { + const token = url.searchParams.get("token") ?? req.headers.get("x-provider-token"); + if (token !== config.providerToken) { + await recordEvent("provider_auth_failed", "unknown", { remote: req.headers.get("x-forwarded-for") ?? null }); + return jsonResponse({ ok: false, error: "invalid provider token" }, 401); + } + const upgraded = server.upgrade(req, { data: {} satisfies WsData }); + return upgraded ? undefined : jsonResponse({ ok: false, error: "websocket upgrade failed" }, 400); + } + + try { + if (url.pathname === "/" || url.pathname === "/health") { + return jsonResponse({ ok: true, service: "unidesk-core", dbReady, startedAt: serviceStartedAt.toISOString() }); + } + if (!dbReady && url.pathname.startsWith("/api/")) { + return jsonResponse({ ok: false, error: "database not ready" }, 503); + } + if (url.pathname === "/api/overview") return jsonResponse(await getOverview()); + if (url.pathname === "/api/nodes") return jsonResponse({ ok: true, nodes: await getNodes() }); + if (url.pathname === "/api/events") return jsonResponse({ ok: true, events: await getEvents(readLimit(url, 100)) }); + if (url.pathname === "/api/tasks") return jsonResponse({ ok: true, tasks: await getTasks(readLimit(url, 100)) }); + if (url.pathname === "/api/dispatch" && req.method === "POST") return dispatchTask(req); + if (url.pathname === "/logs") return jsonResponse({ ok: true, logs: recentLogs.slice(-readLimit(url, 100)) }); + if (url.pathname === "/favicon.ico") return textResponse("", 204); + return jsonResponse({ ok: false, error: "not found", path: url.pathname }, 404); + } catch (error) { + logger("error", "request_failed", { path: url.pathname, error: errorToJson(error) }); + return jsonResponse({ ok: false, error: errorToJson(error) }, 500); + } +} + +function readLimit(url: URL, defaultLimit: number): number { + const raw = url.searchParams.get("limit"); + if (raw === null) return defaultLimit; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) return defaultLimit; + return Math.min(parsed, 500); +} + +await initDatabaseWithRetry(); + +const server = Bun.serve({ + port: config.port, + hostname: "0.0.0.0", + fetch: route, + websocket: { + open(ws) { + logger("info", "provider_socket_open", { remoteAddress: ws.remoteAddress }); + }, + message(ws, raw) { + handleProviderMessage(ws, raw).catch((error) => { + logger("error", "provider_message_failed", { providerId: ws.data.providerId ?? null, error: errorToJson(error) }); + ws.send(JSON.stringify({ type: "ack", requestId: "message", ok: false, message: String(error) })); + }); + }, + close(ws) { + const providerId = ws.data.providerId; + logger("warn", "provider_socket_close", { providerId: providerId ?? null }); + if (providerId !== undefined) { + markProviderOffline(providerId).catch((error) => logger("error", "provider_offline_mark_failed", { providerId, error: errorToJson(error) })); + } + }, + }, +}); + +setInterval(() => { + markStaleProvidersOffline().catch((error) => logger("error", "heartbeat_sweep_failed", { error: errorToJson(error) })); +}, 10_000); + +logger("info", "server_listening", { url: `http://0.0.0.0:${server.port}`, logFile: config.logFile }); diff --git a/src/components/backend-core/tsconfig.json b/src/components/backend-core/tsconfig.json new file mode 100644 index 00000000..df85698f --- /dev/null +++ b/src/components/backend-core/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "composite": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["bun", "node"], + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "skipLibCheck": true + }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../shared" }] +} diff --git a/src/components/database/config/postgresql.conf b/src/components/database/config/postgresql.conf new file mode 100644 index 00000000..cd3a7693 --- /dev/null +++ b/src/components/database/config/postgresql.conf @@ -0,0 +1,6 @@ +# UniDesk keeps PostgreSQL mostly stock; runtime logging options are injected by docker-compose.yml. +shared_buffers = '256MB' +work_mem = '16MB' +maintenance_work_mem = '64MB' +max_connections = 100 +listen_addresses = '*' diff --git a/src/components/database/init/001_unidesk_init.sql b/src/components/database/init/001_unidesk_init.sql new file mode 100644 index 00000000..fb32ff36 --- /dev/null +++ b/src/components/database/init/001_unidesk_init.sql @@ -0,0 +1,32 @@ +CREATE TABLE IF NOT EXISTS unidesk_nodes ( + provider_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + labels JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL, + connected_at TIMESTAMPTZ, + last_heartbeat TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS unidesk_events ( + id BIGSERIAL PRIMARY KEY, + type TEXT NOT NULL, + source TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS unidesk_tasks ( + id TEXT PRIMARY KEY, + provider_id TEXT NOT NULL, + command TEXT NOT NULL, + status TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + result JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_unidesk_nodes_status ON unidesk_nodes(status); +CREATE INDEX IF NOT EXISTS idx_unidesk_events_created_at ON unidesk_events(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_unidesk_tasks_provider_status ON unidesk_tasks(provider_id, status); diff --git a/src/components/frontend/Dockerfile b/src/components/frontend/Dockerfile new file mode 100644 index 00000000..88823d18 --- /dev/null +++ b/src/components/frontend/Dockerfile @@ -0,0 +1,7 @@ +FROM oven/bun:1-alpine +WORKDIR /app/src/components/frontend +COPY src/components/frontend/package.json ./package.json +RUN bun install --production +COPY src/components/frontend/src ./src +COPY src/components/frontend/public ./public +CMD ["bun", "run", "src/index.ts"] diff --git a/src/components/frontend/package.json b/src/components/frontend/package.json new file mode 100644 index 00000000..711d1eab --- /dev/null +++ b/src/components/frontend/package.json @@ -0,0 +1,9 @@ +{ + "name": "@unidesk/frontend", + "private": true, + "type": "module", + "scripts": { + "start": "bun run src/index.ts", + "check": "tsc -p tsconfig.json --noEmit" + } +} diff --git a/src/components/frontend/public/app.js b/src/components/frontend/public/app.js new file mode 100644 index 00000000..f2cf1dc5 --- /dev/null +++ b/src/components/frontend/public/app.js @@ -0,0 +1,153 @@ +const rawCfg = window.UNIDESK_CONFIG || { coreApiUrl: "http://127.0.0.1:18080", corePort: "18080" }; +const cfg = { ...rawCfg, coreApiUrl: resolveCoreApiUrl(rawCfg) }; +const state = { overview: null, nodes: [], events: [], tasks: [], activeModule: "ops", activeTab: "overview", lastRefresh: null }; + +const $ = (id) => document.getElementById(id); +const fmtTime = (value) => value ? new Date(value).toLocaleTimeString() : "--"; +const compactJson = (value) => JSON.stringify(value ?? {}, null, 0); + +function resolveCoreApiUrl(config) { + const api = new URL(config.coreApiUrl, window.location.href); + const pageHost = window.location.hostname; + const apiIsLoopback = api.hostname === "127.0.0.1" || api.hostname === "localhost"; + const pageIsLoopback = pageHost === "127.0.0.1" || pageHost === "localhost"; + if (apiIsLoopback && !pageIsLoopback) { + api.hostname = pageHost; + api.port = String(config.corePort || api.port || "18080"); + } + return api.origin; +} + +function setConnection(ok, text) { + const dot = $("conn-dot"); + dot.className = `dot ${ok ? "ok" : "fail"}`; + $("conn-text").textContent = text; +} + +async function api(path, options) { + const res = await fetch(`${cfg.coreApiUrl}${path}`, options); + const json = await res.json(); + if (!res.ok || json.ok === false) throw new Error(json.error?.message || json.error || `HTTP ${res.status}`); + return json; +} + +function metric(label, value, hint) { + return `
${label}
${value}
${hint}
`; +} + +function renderMetrics() { + const o = state.overview || {}; + $("metric-grid").innerHTML = [ + metric("DB Ready", o.dbReady ? "YES" : "NO", "PostgreSQL central state"), + metric("Online Nodes", o.onlineNodeCount ?? 0, `${o.nodeCount ?? 0} registered`), + metric("Active Sockets", o.activeSocketCount ?? 0, "Provider WebSocket"), + metric("Pending Tasks", o.pendingTaskCount ?? 0, "queued / running"), + ].join(""); + $("refresh-age").textContent = state.lastRefresh ? `刷新 ${fmtTime(state.lastRefresh)}` : "--"; +} + +function renderNodes() { + $("node-count").textContent = `${state.nodes.length} nodes`; + $("nodes-body").innerHTML = state.nodes.map((node) => ` + + ${node.status} +
${node.name}
${node.providerId}
+ ${compactJson(node.labels)} + ${fmtTime(node.lastHeartbeat)} + + `).join("") || `暂无 Provider 节点`; + if (state.nodes[0] && !$("dispatch-provider").value) $("dispatch-provider").value = state.nodes[0].providerId; +} + +function renderEvents() { + $("events-body").innerHTML = state.events.map((event) => ` + + ${event.id} + ${event.type} + ${event.source} + ${compactJson(event.payload)} + ${fmtTime(event.createdAt)} + + `).join("") || `暂无事件`; +} + +function render() { + renderMetrics(); + renderNodes(); + renderEvents(); + document.querySelectorAll("[data-panel]").forEach((panel) => { + const key = panel.getAttribute("data-panel"); + panel.style.display = state.activeTab === "overview" ? "" : key === state.activeTab ? "" : key === "overview" && state.activeModule === "ops" ? "" : "none"; + }); +} + +async function refresh() { + try { + const [overview, nodes, events] = await Promise.all([ + api("/api/overview"), + api("/api/nodes"), + api("/api/events?limit=100"), + ]); + state.overview = overview; + state.nodes = nodes.nodes || []; + state.events = events.events || []; + state.lastRefresh = new Date(); + setConnection(true, "核心在线"); + render(); + } catch (error) { + setConnection(false, error.message); + } +} + +function bindNav() { + document.querySelectorAll(".module").forEach((btn) => { + btn.addEventListener("click", () => { + document.querySelectorAll(".module").forEach((b) => b.classList.remove("active")); + btn.classList.add("active"); + state.activeModule = btn.dataset.module; + if (state.activeModule === "nodes") setTab("nodes"); + if (state.activeModule === "tasks") setTab("dispatch"); + if (state.activeModule === "config") setTab("events"); + if (state.activeModule === "ops") setTab("overview"); + }); + }); + document.querySelectorAll(".tab").forEach((btn) => btn.addEventListener("click", () => setTab(btn.dataset.tab))); +} + +function setTab(tab) { + state.activeTab = tab; + document.querySelectorAll(".tab").forEach((b) => b.classList.toggle("active", b.dataset.tab === tab)); + render(); +} + +function bindDispatch() { + $("dispatch-form").addEventListener("submit", async (event) => { + event.preventDefault(); + const providerId = $("dispatch-provider").value.trim(); + const command = $("dispatch-command").value; + const payloadText = $("dispatch-payload").value.trim(); + try { + const payload = payloadText ? JSON.parse(payloadText) : {}; + const result = await api("/api/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ providerId, command, payload }), + }); + $("dispatch-result").textContent = JSON.stringify(result, null, 2); + await refresh(); + } catch (error) { + $("dispatch-result").textContent = `ERROR: ${error.message}`; + } + }); +} + +function tick() { + $("clock").textContent = new Date().toLocaleTimeString(); +} + +bindNav(); +bindDispatch(); +tick(); +setInterval(tick, 1000); +refresh(); +setInterval(refresh, 5000); diff --git a/src/components/frontend/public/index.html b/src/components/frontend/public/index.html new file mode 100644 index 00000000..e038a403 --- /dev/null +++ b/src/components/frontend/public/index.html @@ -0,0 +1,68 @@ + + + + + + UniDesk Control Plane + + + + +
+ +
+
+
+

Distributed Work Platform

+

控制平面

+
+
+ + 连接中 + --:--:-- +
+
+ +
+
+

核心指标

--
+
+
+
+

Provider 节点

0
+
状态Provider标签最后心跳
+
+
+

事件流

最近 100 条
+
ID类型来源载荷时间
+
+
+

调度试运行

真实 WebSocket 下发
+
+ + + + +
+ 
等待操作
+
+
+
+
+ + + diff --git a/src/components/frontend/public/style.css b/src/components/frontend/public/style.css new file mode 100644 index 00000000..714909f6 --- /dev/null +++ b/src/components/frontend/public/style.css @@ -0,0 +1,329 @@ +:root { + --bg: #111820; + --panel: #17212b; + --panel-2: #1d2a35; + --line: #30404d; + --line-soft: #24323e; + --text: #dce7ec; + --muted: #8496a3; + --accent: #e2a329; + --accent-2: #4bb7aa; + --danger: #d86b55; + --ok: #6fbe73; + --rail: #0c1218; + --shadow: 0 16px 40px rgba(0, 0, 0, 0.28); + font-family: "DIN Condensed", "Aptos Narrow", "Liberation Sans Narrow", "Noto Sans", sans-serif; +} + +* { box-sizing: border-box; } +html, body { min-height: 100%; } +body { + margin: 0; + color: var(--text); + background: + linear-gradient(135deg, rgba(226, 163, 41, 0.08), transparent 28%), + linear-gradient(315deg, rgba(75, 183, 170, 0.08), transparent 30%), + repeating-linear-gradient(90deg, rgba(255,255,255,0.025) 0, rgba(255,255,255,0.025) 1px, transparent 1px, transparent 36px), + var(--bg); + font-size: 13px; + letter-spacing: 0.01em; +} + +button, input, select, textarea { + font: inherit; +} + +.shell { + display: grid; + grid-template-columns: 176px minmax(0, 1fr); + min-height: 100vh; +} + +.rail { + position: sticky; + top: 0; + height: 100vh; + padding: 12px 10px; + border-right: 1px solid var(--line); + background: linear-gradient(180deg, #0a1015, var(--rail)); +} + +.brand { + display: flex; + align-items: center; + gap: 9px; + height: 40px; + padding: 0 5px 12px; + border-bottom: 1px solid var(--line-soft); + margin-bottom: 10px; +} + +.brand-mark { + display: grid; + place-items: center; + width: 32px; + height: 26px; + border: 1px solid var(--accent); + color: var(--accent); + font-weight: 800; + letter-spacing: 0.08em; +} + +.brand-text { + font-size: 15px; + text-transform: uppercase; + letter-spacing: 0.14em; +} + +.module, .tab, .dispatch-form button { + border: 1px solid transparent; + color: var(--muted); + background: transparent; + cursor: pointer; +} + +.module { + display: block; + width: 100%; + padding: 8px 10px; + margin: 4px 0; + text-align: left; + border-left: 3px solid transparent; +} + +.module:hover, .module.active { + color: var(--text); + background: rgba(255,255,255,0.045); + border-left-color: var(--accent); +} + +.workspace { + min-width: 0; + padding: 12px 14px 16px; +} + +.topbar { + display: flex; + justify-content: space-between; + align-items: center; + gap: 14px; + min-height: 54px; + padding: 0 0 10px; + border-bottom: 1px solid var(--line); +} + +.eyebrow { + margin: 0 0 2px; + color: var(--accent); + font-size: 10px; + letter-spacing: 0.22em; + text-transform: uppercase; +} + +h1, h2 { margin: 0; font-weight: 650; } +h1 { font-size: 22px; letter-spacing: 0.08em; } +h2 { font-size: 14px; text-transform: uppercase; letter-spacing: 0.09em; } + +.status-strip { + display: flex; + align-items: center; + gap: 8px; + padding: 7px 9px; + border: 1px solid var(--line); + background: rgba(0,0,0,0.14); + color: var(--muted); + white-space: nowrap; +} + +.dot { + width: 8px; + height: 8px; + border-radius: 999px; + background: var(--muted); + box-shadow: 0 0 0 2px rgba(255,255,255,0.06); +} +.dot.ok { background: var(--ok); } +.dot.warn { background: var(--accent); } +.dot.fail { background: var(--danger); } + +.tabs { + display: flex; + gap: 6px; + padding: 10px 0; + overflow-x: auto; +} + +.tab { + padding: 7px 12px; + border-color: var(--line); + background: rgba(12, 18, 24, 0.58); + color: var(--muted); + min-width: 108px; +} + +.tab.active, .tab:hover { + color: var(--text); + border-color: var(--accent-2); + background: rgba(75, 183, 170, 0.12); +} + +.content-grid { + display: grid; + grid-template-columns: minmax(320px, 0.9fr) minmax(520px, 1.6fr); + gap: 10px; + align-items: stretch; +} + +.panel { + border: 1px solid var(--line); + background: linear-gradient(180deg, rgba(255,255,255,0.035), rgba(255,255,255,0.015)), var(--panel); + box-shadow: var(--shadow); + min-width: 0; +} + +.panel-head { + display: flex; + justify-content: space-between; + align-items: center; + gap: 10px; + height: 38px; + padding: 0 10px; + border-bottom: 1px solid var(--line); + color: var(--muted); +} + +.metrics-panel { grid-column: 1 / 2; } +.table-panel[data-panel="nodes"] { grid-column: 2 / 3; } +.table-panel[data-panel="events"], .dispatch-panel { grid-column: 1 / -1; } + +.metric-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; + padding: 10px; +} + +.metric { + padding: 10px; + min-height: 74px; + border: 1px solid var(--line-soft); + background: var(--panel-2); +} + +.metric .label { + color: var(--muted); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.12em; +} +.metric .value { + margin-top: 8px; + color: var(--text); + font-size: 24px; + font-weight: 720; +} +.metric .hint { + margin-top: 3px; + color: var(--muted); + font-size: 11px; +} + +.table-wrap { overflow: auto; max-height: 46vh; } +table { width: 100%; border-collapse: collapse; min-width: 680px; } +th, td { + padding: 7px 9px; + border-bottom: 1px solid var(--line-soft); + text-align: left; + vertical-align: top; +} +th { + position: sticky; + top: 0; + background: #121b24; + color: var(--accent); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.11em; + z-index: 1; +} +td { color: var(--text); } +.code, code { + font-family: "Cascadia Mono", "IBM Plex Mono", "Liberation Mono", monospace; + font-size: 12px; + color: #bfd7dc; +} +.badge { + display: inline-flex; + align-items: center; + gap: 5px; + padding: 2px 7px; + border: 1px solid var(--line); + color: var(--muted); + background: rgba(0,0,0,0.18); +} +.badge.online { color: var(--ok); border-color: rgba(111, 190, 115, 0.45); } +.badge.offline { color: var(--danger); border-color: rgba(216, 107, 85, 0.45); } + +.dispatch-panel { min-height: 230px; } +.dispatch-form { + display: grid; + grid-template-columns: 1fr 180px auto; + gap: 8px; + padding: 10px; + align-items: end; +} +.dispatch-form label { + display: grid; + gap: 4px; + color: var(--muted); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.12em; +} +.dispatch-form .wide { grid-column: 1 / -1; } +input, select, textarea { + width: 100%; + border: 1px solid var(--line); + color: var(--text); + background: #0d151c; + padding: 7px 8px; + outline: none; +} +textarea { min-height: 76px; resize: vertical; font-family: "Cascadia Mono", "IBM Plex Mono", monospace; } +.dispatch-form button { + height: 33px; + padding: 0 14px; + border-color: var(--accent); + color: #130f08; + background: var(--accent); + font-weight: 700; +} +.result-block { + margin: 0 10px 10px; + padding: 8px; + max-height: 170px; + overflow: auto; + border: 1px solid var(--line-soft); + background: #0d151c; + color: #bfd7dc; +} + +@media (max-width: 980px) { + .shell { grid-template-columns: 1fr; } + .rail { + position: static; + height: auto; + display: flex; + align-items: center; + gap: 6px; + overflow-x: auto; + border-right: 0; + border-bottom: 1px solid var(--line); + } + .brand { border-bottom: 0; margin-bottom: 0; padding-bottom: 0; flex: 0 0 auto; } + .module { width: auto; white-space: nowrap; border-left: 0; border-bottom: 2px solid transparent; } + .module.active, .module:hover { border-bottom-color: var(--accent); } + .content-grid { grid-template-columns: 1fr; } + .metrics-panel, .table-panel[data-panel="nodes"], .table-panel[data-panel="events"], .dispatch-panel { grid-column: 1; } + .dispatch-form { grid-template-columns: 1fr; } +} diff --git a/src/components/frontend/src/index.ts b/src/components/frontend/src/index.ts new file mode 100644 index 00000000..51609546 --- /dev/null +++ b/src/components/frontend/src/index.ts @@ -0,0 +1,98 @@ +import { appendFileSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +interface RuntimeConfig { + port: number; + corePublicUrl: string; + logFile: string; +} + +type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; + +const config = readConfig(); +const logger = createLogger("frontend", config.logFile); +const publicDir = join(import.meta.dir, "..", "public"); +const indexHtml = readFileSync(join(publicDir, "index.html"), "utf8").replace( + "__UNIDESK_CONFIG__", + JSON.stringify({ coreApiUrl: config.corePublicUrl, corePort: new URL(config.corePublicUrl).port }), +); + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +function readNumberEnv(name: string): number { + const raw = requiredEnv(name); + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); + } + return parsed; +} + +function readConfig(): RuntimeConfig { + return { + port: readNumberEnv("PORT"), + corePublicUrl: requiredEnv("CORE_PUBLIC_URL"), + logFile: requiredEnv("LOG_FILE"), + }; +} + +function createLogger(service: string, logFile: string) { + mkdirSync(dirname(logFile), { recursive: true }); + return (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue): void => { + const entry = { ts: new Date().toISOString(), service, level, message, data }; + const line = `${JSON.stringify(entry)}\n`; + try { + appendFileSync(logFile, line, "utf8"); + } catch (error) { + console.error(JSON.stringify({ ts: new Date().toISOString(), service, level: "error", message: "log_write_failed", data: String(error) })); + } + const consoleMethod = level === "error" ? console.error : level === "warn" ? console.warn : console.log; + consoleMethod(line.trimEnd()); + }; +} + +function contentType(pathname: string): string { + if (pathname.endsWith(".css")) return "text/css; charset=utf-8"; + if (pathname.endsWith(".js")) return "text/javascript; charset=utf-8"; + if (pathname.endsWith(".svg")) return "image/svg+xml"; + return "text/plain; charset=utf-8"; +} + +function jsonResponse(body: JsonValue, status = 200): Response { + return new Response(JSON.stringify(body, null, 2), { + status, + headers: { "content-type": "application/json; charset=utf-8" }, + }); +} + +const server = Bun.serve({ + port: config.port, + hostname: "0.0.0.0", + async fetch(req) { + const url = new URL(req.url); + logger("debug", "request", { path: url.pathname }); + if (url.pathname === "/health") { + return jsonResponse({ ok: true, service: "unidesk-frontend", coreApiUrl: config.corePublicUrl }); + } + if (url.pathname === "/" || url.pathname === "/index.html") { + return new Response(indexHtml, { headers: { "content-type": "text/html; charset=utf-8" } }); + } + const safePath = url.pathname.replace(/^\/+/, ""); + if (safePath.includes("..")) { + return jsonResponse({ ok: false, error: "invalid path" }, 400); + } + const file = Bun.file(join(publicDir, safePath)); + if (!(await file.exists())) { + return jsonResponse({ ok: false, error: "not found", path: url.pathname }, 404); + } + return new Response(file, { headers: { "content-type": contentType(url.pathname) } }); + }, +}); + +logger("info", "server_listening", { url: `http://0.0.0.0:${server.port}`, coreApiUrl: config.corePublicUrl, logFile: config.logFile }); diff --git a/src/components/frontend/tsconfig.json b/src/components/frontend/tsconfig.json new file mode 100644 index 00000000..f62969e7 --- /dev/null +++ b/src/components/frontend/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "composite": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["bun", "node"], + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} diff --git a/src/components/microservices/example-service/Dockerfile b/src/components/microservices/example-service/Dockerfile new file mode 100644 index 00000000..a73f8a24 --- /dev/null +++ b/src/components/microservices/example-service/Dockerfile @@ -0,0 +1,3 @@ +FROM oven/bun:1-alpine +WORKDIR /app +CMD ["bun", "--version"] diff --git a/src/components/microservices/example-service/package.json b/src/components/microservices/example-service/package.json new file mode 100644 index 00000000..2aed62d4 --- /dev/null +++ b/src/components/microservices/example-service/package.json @@ -0,0 +1,5 @@ +{ + "name": "@unidesk/example-service", + "private": true, + "type": "module" +} diff --git a/src/components/microservices/example-service/src/index.ts b/src/components/microservices/example-service/src/index.ts new file mode 100644 index 00000000..13f65dd5 --- /dev/null +++ b/src/components/microservices/example-service/src/index.ts @@ -0,0 +1 @@ +export const reserved = true; diff --git a/src/components/microservices/example-service/tsconfig.json b/src/components/microservices/example-service/tsconfig.json new file mode 100644 index 00000000..9573fe03 --- /dev/null +++ b/src/components/microservices/example-service/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} diff --git a/src/components/provider-gateway/Dockerfile b/src/components/provider-gateway/Dockerfile new file mode 100644 index 00000000..c9a78de0 --- /dev/null +++ b/src/components/provider-gateway/Dockerfile @@ -0,0 +1,10 @@ +FROM oven/bun:1-alpine +RUN apk add --no-cache bash docker-cli openssh-client +WORKDIR /app/src/components/provider-gateway +COPY src/components/provider-gateway/package.json ./package.json +RUN bun install --production +COPY src/components/shared /app/src/components/shared +COPY src/components/provider-gateway/src ./src +COPY src/components/provider-gateway/scripts/host-ssh-shell.sh /usr/local/bin/unidesk-host-ssh-shell +RUN chmod +x /usr/local/bin/unidesk-host-ssh-shell +CMD ["bun", "run", "src/index.ts"] diff --git a/src/components/provider-gateway/package.json b/src/components/provider-gateway/package.json new file mode 100644 index 00000000..d6e7ded0 --- /dev/null +++ b/src/components/provider-gateway/package.json @@ -0,0 +1,9 @@ +{ + "name": "@unidesk/provider-gateway", + "private": true, + "type": "module", + "scripts": { + "start": "bun run src/index.ts", + "check": "tsc -p tsconfig.json --noEmit" + } +} diff --git a/src/components/provider-gateway/scripts/host-ssh-shell.sh b/src/components/provider-gateway/scripts/host-ssh-shell.sh new file mode 100755 index 00000000..e7508229 --- /dev/null +++ b/src/components/provider-gateway/scripts/host-ssh-shell.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +host="${HOST_SSH_HOST}" +port="${HOST_SSH_PORT}" +user="${HOST_SSH_USER}" +key="${HOST_SSH_KEY}" +fallback_cwd="${HOST_REMOTE_CWD}" +requested_cwd="${UNIDESK_REQUESTED_CWD:-$fallback_cwd}" +login_shell="${HOST_LOGIN_SHELL}" + +quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\''/g")" +} + +q_requested=$(quote "$requested_cwd") +q_fallback=$(quote "$fallback_cwd") +q_shell=$(quote "$login_shell") +remote_cmd="cd $q_requested 2>/dev/null || cd $q_fallback 2>/dev/null || cd; export UNIDESK_BRIDGE=host-ssh; exec $q_shell -l" + +exec ssh -tt \ + -i "$key" \ + -p "$port" \ + -o BatchMode=yes \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile=/tmp/unidesk-host-known-hosts \ + -o ServerAliveInterval=20 \ + -o ServerAliveCountMax=3 \ + "$user@$host" "$remote_cmd" diff --git a/src/components/provider-gateway/src/index.ts b/src/components/provider-gateway/src/index.ts new file mode 100644 index 00000000..a0c944f4 --- /dev/null +++ b/src/components/provider-gateway/src/index.ts @@ -0,0 +1,231 @@ +import { appendFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { + type CoreDispatchMessage, + type JsonValue, + type ProviderLabels, + type ProviderTaskStatusMessage, + parseJsonObject, +} from "../../shared/src/index"; + +interface RuntimeConfig { + serverUrl: string; + token: string; + providerId: string; + providerName: string; + labels: ProviderLabels; + heartbeatIntervalMs: number; + reconnectBaseMs: number; + reconnectMaxMs: number; + dockerSocketPath: string; + logFile: string; +} + +const startedAt = new Date(); +const config = readConfig(); +const logger = createLogger("provider-gateway", config.logFile); +let socket: WebSocket | null = null; +let heartbeatTimer: ReturnType | null = null; +let reconnectAttempt = 0; +let stopping = false; + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +function readNumberEnv(name: string): number { + const raw = requiredEnv(name); + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Environment variable ${name} must be a positive number, got ${raw}`); + } + return parsed; +} + +function readConfig(): RuntimeConfig { + return { + serverUrl: requiredEnv("PROVIDER_SERVER_URL"), + token: requiredEnv("PROVIDER_TOKEN"), + providerId: requiredEnv("PROVIDER_ID"), + providerName: requiredEnv("PROVIDER_NAME"), + labels: parseJsonObject(requiredEnv("PROVIDER_LABELS_JSON"), "PROVIDER_LABELS_JSON"), + heartbeatIntervalMs: readNumberEnv("HEARTBEAT_INTERVAL_MS"), + reconnectBaseMs: readNumberEnv("RECONNECT_BASE_MS"), + reconnectMaxMs: readNumberEnv("RECONNECT_MAX_MS"), + dockerSocketPath: requiredEnv("DOCKER_SOCKET_PATH"), + logFile: requiredEnv("LOG_FILE"), + }; +} + +function createLogger(service: string, logFile: string) { + mkdirSync(dirname(logFile), { recursive: true }); + return (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue): void => { + const entry = data === undefined + ? { ts: new Date().toISOString(), service, level, message } + : { ts: new Date().toISOString(), service, level, message, data }; + const line = `${JSON.stringify(entry)}\n`; + try { + appendFileSync(logFile, line, "utf8"); + } catch (error) { + console.error(JSON.stringify({ ts: new Date().toISOString(), service, level: "error", message: "log_write_failed", data: String(error) })); + } + const consoleMethod = level === "error" ? console.error : level === "warn" ? console.warn : console.log; + consoleMethod(line.trimEnd()); + }; +} + +function withToken(rawUrl: string, token: string): string { + const url = new URL(rawUrl); + url.searchParams.set("token", token); + return url.toString(); +} + +function currentLabels(): ProviderLabels { + return { + ...config.labels, + dockerSocketPresent: existsSync(config.dockerSocketPath), + runtime: "bun", + gatewayUptimeSeconds: Math.floor((Date.now() - startedAt.getTime()) / 1000), + }; +} + +function sendJson(value: unknown): void { + if (!socket || socket.readyState !== WebSocket.OPEN) return; + socket.send(JSON.stringify(value)); +} + +function sendRegister(): void { + sendJson({ + type: "register", + providerId: config.providerId, + name: config.providerName, + labels: currentLabels(), + startedAt: startedAt.toISOString(), + capabilities: ["heartbeat", "docker.ps", "echo"], + }); +} + +function sendHeartbeat(): void { + sendJson({ + type: "heartbeat", + providerId: config.providerId, + labels: currentLabels(), + at: new Date().toISOString(), + }); +} + +async function sendTaskStatus(taskId: string, status: ProviderTaskStatusMessage["status"], message: string, result?: JsonValue): Promise { + sendJson({ + type: "task_status", + providerId: config.providerId, + taskId, + status, + message, + at: new Date().toISOString(), + result, + }); +} + +async function runDockerPs(): Promise { + const proc = Bun.spawn(["docker", "ps", "--format", "{{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"], { + stdout: "pipe", + stderr: "pipe", + }); + const timeout = setTimeout(() => proc.kill("SIGKILL"), 5000); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + clearTimeout(timeout); + if (exitCode !== 0) { + throw new Error(`docker ps failed with exit ${exitCode}: ${stderr}`); + } + const containers = stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .slice(0, 50) + .map((line) => { + const [id, name, image, status, ports] = line.split("\t"); + return { id, name, image, status, ports }; + }); + return { containers, count: containers.length, stderr }; +} + +async function handleDispatch(message: CoreDispatchMessage): Promise { + logger("info", "dispatch_received", { taskId: message.taskId, command: message.command, payload: message.payload }); + await sendTaskStatus(message.taskId, "accepted", "provider accepted task"); + try { + await sendTaskStatus(message.taskId, "running", "provider running task"); + if (message.command === "docker.ps") { + const result = await runDockerPs(); + await sendTaskStatus(message.taskId, "succeeded", "docker ps completed", result); + return; + } + await sendTaskStatus(message.taskId, "succeeded", "echo completed", { echo: message.payload }); + } catch (error) { + const text = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + logger("error", "dispatch_failed", { taskId: message.taskId, error: text }); + await sendTaskStatus(message.taskId, "failed", text, { error: text }); + } +} + +function handleMessage(raw: MessageEvent): void { + try { + const parsed = JSON.parse(raw.data) as { type?: unknown }; + if (parsed.type === "dispatch") { + handleDispatch(parsed as CoreDispatchMessage).catch((error) => logger("error", "dispatch_handler_failed", { error: String(error) })); + return; + } + logger("debug", "core_message", parsed as JsonValue); + } catch (error) { + logger("error", "core_message_parse_failed", { error: String(error) }); + } +} + +function scheduleReconnect(): void { + if (stopping) return; + reconnectAttempt += 1; + const delayMs = Math.min(config.reconnectMaxMs, config.reconnectBaseMs * 2 ** Math.min(reconnectAttempt, 8)); + logger("warn", "reconnect_scheduled", { reconnectAttempt, delayMs }); + setTimeout(connect, delayMs); +} + +function connect(): void { + const url = withToken(config.serverUrl, config.token); + logger("info", "connect_start", { serverUrl: config.serverUrl, providerId: config.providerId }); + socket = new WebSocket(url); + socket.addEventListener("open", () => { + reconnectAttempt = 0; + logger("info", "connect_open", { providerId: config.providerId }); + sendRegister(); + sendHeartbeat(); + if (heartbeatTimer !== null) clearInterval(heartbeatTimer); + heartbeatTimer = setInterval(sendHeartbeat, config.heartbeatIntervalMs); + }); + socket.addEventListener("message", (event) => handleMessage(event as MessageEvent)); + socket.addEventListener("close", (event) => { + logger("warn", "connect_close", { code: event.code, reason: event.reason }); + if (heartbeatTimer !== null) clearInterval(heartbeatTimer); + heartbeatTimer = null; + scheduleReconnect(); + }); + socket.addEventListener("error", () => { + logger("error", "connect_error", { providerId: config.providerId }); + }); +} + +process.on("SIGTERM", () => { + stopping = true; + logger("warn", "sigterm_received"); + if (heartbeatTimer !== null) clearInterval(heartbeatTimer); + socket?.close(1000, "provider shutdown"); + process.exit(0); +}); + +connect(); diff --git a/src/components/provider-gateway/tsconfig.json b/src/components/provider-gateway/tsconfig.json new file mode 100644 index 00000000..df85698f --- /dev/null +++ b/src/components/provider-gateway/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "composite": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["bun", "node"], + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "skipLibCheck": true + }, + "include": ["src/**/*.ts"], + "references": [{ "path": "../shared" }] +} diff --git a/src/components/shared/package.json b/src/components/shared/package.json new file mode 100644 index 00000000..3807a2ac --- /dev/null +++ b/src/components/shared/package.json @@ -0,0 +1,7 @@ +{ + "name": "@unidesk/shared", + "private": true, + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts" +} diff --git a/src/components/shared/src/index.ts b/src/components/shared/src/index.ts new file mode 100644 index 00000000..345bd5eb --- /dev/null +++ b/src/components/shared/src/index.ts @@ -0,0 +1,104 @@ +export type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; + +export type ProviderLabels = Record; + +export interface ProviderRegisterMessage { + type: "register"; + providerId: string; + name: string; + labels: ProviderLabels; + startedAt: string; + capabilities: string[]; +} + +export interface ProviderHeartbeatMessage { + type: "heartbeat"; + providerId: string; + labels: ProviderLabels; + at: string; +} + +export interface ProviderTaskStatusMessage { + type: "task_status"; + providerId: string; + taskId: string; + status: "accepted" | "running" | "succeeded" | "failed"; + message: string; + at: string; + result?: JsonValue; +} + +export interface CoreDispatchMessage { + type: "dispatch"; + taskId: string; + command: "docker.ps" | "echo"; + payload: Record; +} + +export interface CoreAcknowledgeMessage { + type: "ack"; + requestId: string; + ok: boolean; + message: string; +} + +export type ProviderToCoreMessage = + | ProviderRegisterMessage + | ProviderHeartbeatMessage + | ProviderTaskStatusMessage; + +export type CoreToProviderMessage = CoreDispatchMessage | CoreAcknowledgeMessage; + +export interface ApiNode { + providerId: string; + name: string; + status: "online" | "offline"; + labels: ProviderLabels; + connectedAt: string | null; + lastHeartbeat: string | null; +} + +export interface ApiTask { + id: string; + providerId: string; + command: string; + status: string; + payload: JsonValue; + result: JsonValue; + createdAt: string; + updatedAt: string; +} + +export interface ApiEvent { + id: number; + type: string; + source: string; + payload: JsonValue; + createdAt: string; +} + +export interface ServiceLogEntry { + ts: string; + service: string; + level: "debug" | "info" | "warn" | "error"; + message: string; + data?: JsonValue; +} + +export function parseJsonObject(value: string, name: string): Record { + const parsed = JSON.parse(value) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new Error(`${name} must be a JSON object`); + } + return parsed as Record; +} + +export function isProviderToCoreMessage(value: unknown): value is ProviderToCoreMessage { + if (typeof value !== "object" || value === null || !("type" in value)) return false; + const msg = value as { type?: unknown; providerId?: unknown }; + return ( + (msg.type === "register" || msg.type === "heartbeat" || msg.type === "task_status") && + typeof msg.providerId === "string" && + msg.providerId.length > 0 + ); +} diff --git a/src/components/shared/tsconfig.json b/src/components/shared/tsconfig.json new file mode 100644 index 00000000..aabbf6ea --- /dev/null +++ b/src/components/shared/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "composite": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["bun", "node"], + "strict": true, + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} diff --git a/src/package.json b/src/package.json new file mode 100644 index 00000000..50fd74f7 --- /dev/null +++ b/src/package.json @@ -0,0 +1,16 @@ +{ + "name": "@unidesk/components", + "private": true, + "type": "module", + "workspaces": [ + "components/*" + ], + "scripts": { + "check": "tsc -b tsconfig.base.json" + }, + "devDependencies": { + "@types/bun": "latest", + "@types/node": "latest", + "typescript": "latest" + } +} diff --git a/src/tsconfig.base.json b/src/tsconfig.base.json new file mode 100644 index 00000000..76588187 --- /dev/null +++ b/src/tsconfig.base.json @@ -0,0 +1,9 @@ +{ + "files": [], + "references": [ + { "path": "components/shared" }, + { "path": "components/backend-core" }, + { "path": "components/provider-gateway" }, + { "path": "components/frontend" } + ] +} diff --git a/src/tsconfig.check.json b/src/tsconfig.check.json new file mode 100644 index 00000000..6c03efc6 --- /dev/null +++ b/src/tsconfig.check.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["bun", "node"], + "strict": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["components/**/*.ts"], + "exclude": ["components/**/dist/**"] +}