fix: improve egress and job diagnostics (#969)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
@@ -53,8 +53,8 @@ PipelineRun 失败或长时间未完成时,先按定点 `control-plane status
|
||||
- `--main-server-ip <ip> <command>` 默认通过公网 frontend 登录态调用主 server 的同源 API 代理,不要求计算节点持有主 server SSH key;显式提供 `--main-server-key` 或 `--main-server-transport ssh` 时才使用旧 SSH 传输。远程 frontend 传输下的 `ssh <route> ...` 必须复用同一套结构化 route parser,支持 `D601`、`G14`、host workspace、`D601:win`、`D601:win/c/test`、`D601:k3s` 和 `D601:k3s:<namespace>:<workload>` 这类定位路径;它不向调用容器下发 provider token,也不要求调用容器能解析 backend-core 内网 DNS。
|
||||
- `config show` 读取并校验根目录 `config.json`,不从环境变量、默认值或隐藏文件静默补配置。
|
||||
- `check` 默认只执行轻量配置校验、Bun 版本检查和 Bun Transpiler 语法解析(覆盖 CLI 入口、主要 `scripts/` 模块和核心组件入口,不做类型推导)。除非用户明确要求,CLI 改动不运行单元测试、合同测试或新增测试脚本;默认最多做语法检查和必要的帮助/命令形态人工确认。关键文件存在性、`scripts/` TypeScript 类型检查、`src/components/` TypeScript 类型检查、Docker Compose config、日志轮转策略扫描和 D601 recovery guardrails 默认不启用,分别通过 `--files`、`--scripts-typecheck`、`--components`、`--compose`、`--logs`、`--recovery-guardrails` 开启,或用 `--full` 一次性开启。`--scripts-typecheck` 只跑 scripts TypeScript 类型检查,不触发测试脚本或 GitHub issue/PR live API check。长命令项必须在 stderr 输出 `unidesk.check.progress` JSON lines,stdout 保持最终 JSON 结果,避免 post-task 或人工运行时长时间无可见进度。`typescript:scripts` 固定通过 `bun --bun tsc -p scripts/tsconfig.json --noEmit --pretty false` 执行,默认 `--scripts-typecheck-timeout-ms 120000`,可按目标运行面显式调小或调大但 CLI 会封顶;`--check-heartbeat-ms` 控制运行中心跳间隔,默认 `15000`。所有命令项的最终 item detail 必须包含 `durationMs`、`timeoutMs`、`heartbeatMs`、`exitCode`、`signal`、`timedOut`、stdout/stderr byte count、truncation flag 和有界 tail;超时必须返回 `timedOut=true`,不得只留下被外层命令杀死的空输出。不要把 `bun --check scripts/cli.ts` 当作低噪声 CLI 自检入口;它可能执行根 CLI help 并触发长 help dump。CLI 入口级自检使用 `bun scripts/cli.ts check`,单文件语法确认只针对具体模块文件运行。`check recovery-guardrails` 是同一诊断的低噪声直接入口,报告 malformed `/proc/mounts`、kubelet validation risk、stale CRI sandbox count、Code Queue worktree/symlink、Code Queue/MDTODO hostPath 和 `ContainerCreating` 分类;它不得重启 k3s、删除 CRI sandbox、修改 hostPath、deploy/rollout 或 prune/reset。`--rust` 只允许在 D601 CI/dev execution 中配合 `UNIDESK_D601_RUST_CHECK=1` 使用,长期规则见 `docs/reference/dev-environment.md` 和 `docs/reference/devops-hygiene.md`。
|
||||
- `server start` 创建异步 job,在后台执行 Docker 构建和启动;命令本身只负责返回 job id、日志路径和启动命令。
|
||||
- `server stop` 创建异步 job,在后台停止固定 Compose project 中的全部 UniDesk 服务。
|
||||
- `server start` 创建异步 job,在后台执行 Docker 构建和启动;命令默认只返回低噪声 async job 摘要、stdout/stderr 路径和 `job status` 后续命令,完整 JSON 只能通过 `--full`/`--raw` 显式展开。
|
||||
- `server stop` 创建异步 job,在后台停止固定 Compose project 中的全部 UniDesk 服务;默认输出同样是 async job 摘要。
|
||||
- `server status` 查询公开端口、受限宿主端口、内部端口、主机 swap 摘要、Compose 容器、core/frontend/dev-frontend/provider/database 健康检查和访问 URL;D601 Code Queue 使用的 PostgreSQL/OA Event Flow host mapping 必须出现在受限宿主端口而不是无条件公开入口中。低内存主 server 上 `swap.warning` 非空时,先执行 `server swap status` 或 `server swap ensure`。
|
||||
- `server swap status|ensure [--path /swapfile] [--size 2GiB] [--dry-run]` 是主 server swap 管理入口。`status` 仅读 `/proc/meminfo`、`/proc/swaps` 和 `/etc/fstab` 并返回 JSON;`ensure` 在已有任何 active swap 时只报告 no-op,在无 active swap 时创建固定 swapfile、`chmod 600`、`mkswap`、`swapon` 并尽量写入 `/etc/fstab`。输出必须包含 `before`、`after`、total memory、active swap、持久化状态、关键动作和错误详情;若 swap 已启用但 fstab 写入失败,状态为 `degraded`,调用者需按返回的 detail 修复持久化。
|
||||
- `server logs` 返回 `logs/` 文件日志和 Docker 容器日志的尾部,默认限制输出大小,避免日志爆炸。实现必须只读取文件末尾字节,不得为了 tail 先把巨大日志完整读入 CLI 内存。
|
||||
@@ -143,7 +143,7 @@ PipelineRun 失败或长时间未完成时,先按定点 `control-plane status
|
||||
- `codex interrupt|cancel <taskId>` 通过 Code Queue 私有代理请求中断;running/judging 任务会请求 D601 当前 agent run 停止,queued/retry_wait 任务的取消也必须保持与 WebUI 相同代理路径,返回有界 task 摘要和后续查询命令。任何需要接触 active run 的动作仍属于 D601 执行面。
|
||||
- 旧 Code Queue 多队列 lane 现在是归档视图:`codex queues [--full|--all] [--limit N] [--page N|--offset N]` 只读展示历史 queue 摘要、activity、commanderConcurrency、counts 和 execution diagnostics。`queue create`、`queue merge`、`move` 等旧队列写入口冻结并返回 `legacy-code-queue-frozen`;AgentRun 新任务的排队、派发和取消必须使用 `agentrun create|apply|get|cancel`。
|
||||
- 所有旧 `codex` 历史查询、已读和残留 interrupt/cancel 命令必须走与 WebUI 相同的 backend-core 私有代理路径 `/api/microservices/code-queue/proxy/...`。旧 submit/steer/resume/queue mutation/move/workdir mutation 不得绕过冻结;若需要新任务或新 session 控制,使用 AgentRun 资源原语。
|
||||
- `job list [--limit N] [--include-command]` 与 `job status <jobId|latest> [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态,是异步命令的可观测入口。`job list` 默认只返回最新 50 条摘要,并为已知异步工作流返回轻量 `progress.summary` 与后续查询命令;`job status` 默认返回结构化 `progress`、stdout/stderr 末尾 12000 字节、`tailPolicy` 与完整日志路径。已知工作流应从有界日志尾部抽取阶段、关键对象名和下一步命令,避免为了判断当前阶段而手工打开完整 stdout/stderr。`hwlab_g14_v02_trigger_current` 的 progress 必须暴露 trigger 阶段、source commit 和 PipelineRun;`hwlab_g14_v02_pr_monitor` 的 progress 必须暴露 preflight、merge、source-head、cd-trigger、cd-status、git-mirror-flush 和 pr-comment 阶段,以及 PR、source commit、PipelineRun、targetValidation/pendingFlush 摘要;`agentrun_vNN_trigger_current` 的 progress 必须识别 YAML lane 的 source-bootstrap、image-build、gitops-publish、git-mirror 阶段,暴露 source commit、PipelineRun、stage diagnostics、timing 和 `agentrun control-plane status --node <node> --lane <lane> --pipeline-run <name>`;`hwlab_g14_git_mirror_sync|flush` 与 `agentrun_v01_git_mirror_sync|flush` 的 progress 必须暴露 sync/flush 状态、Job 名、pendingFlush 与 fetch/push/total/SSH timing,并给出对应 repo 的 mirror status 命令。
|
||||
- `job|jobs list [--limit N] [--include-command]` 与 `job status|get|read <jobId|latest> [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态,是异步命令的可观测入口。`jobs get/read` 是 `job status` 的兼容别名,不应引导调用者直接读取 `.state/jobs` 文件。`job list` 默认只返回最新 50 条摘要,并为已知异步工作流返回轻量 `progress.summary` 与后续查询命令;`job status` 默认返回结构化 `progress`、stdout/stderr 末尾 12000 字节、`tailPolicy` 与完整日志路径。已知工作流应从有界日志尾部抽取阶段、关键对象名和下一步命令,避免为了判断当前阶段而手工打开完整 stdout/stderr。`hwlab_g14_v02_trigger_current` 的 progress 必须暴露 trigger 阶段、source commit 和 PipelineRun;`hwlab_g14_v02_pr_monitor` 的 progress 必须暴露 preflight、merge、source-head、cd-trigger、cd-status、git-mirror-flush 和 pr-comment 阶段,以及 PR、source commit、PipelineRun、targetValidation/pendingFlush 摘要;`agentrun_vNN_trigger_current` 的 progress 必须识别 YAML lane 的 source-bootstrap、image-build、gitops-publish、git-mirror 阶段,暴露 source commit、PipelineRun、stage diagnostics、timing 和 `agentrun control-plane status --node <node> --lane <lane> --pipeline-run <name>`;`hwlab_g14_git_mirror_sync|flush` 与 `agentrun_v01_git_mirror_sync|flush` 的 progress 必须暴露 sync/flush 状态、Job 名、pendingFlush 与 fetch/push/total/SSH timing,并给出对应 repo 的 mirror status 命令。
|
||||
- `debug health`、`debug ssh-pool <providerId>`、`debug dispatch` 与 `debug task` 走真实内部 core、WebSocket、数据库、provider、系统指标、Docker 状态和 Host SSH 维护桥流程,只用于开发调试,不写入 `TEST.md` 的正式验收步骤;`debug ssh-pool` 只裁剪单个 provider 的 `providerGatewaySshData*` labels,用于低噪声判断 tcp-pool 是否 ready、claimed、exhausted 或有 lastError。
|
||||
- `e2e run [--only pattern[,pattern...]] [--skip pattern[,pattern...]]` 使用 publicHost 派生的公开 production frontend/dev frontend/provider ingress URL,并通过 Docker 内网验证 core API、PostgreSQL、provider self-connection、系统指标曲线、Docker 状态快照、provider.upgrade 预检和 Playwright 前端页面,是交付前的自动化 E2E 门禁;CLI 默认输出 check 状态摘要,完整诊断写入 `resultPath`,日常迭代应优先用 `--only` / `--skip` 跑最小必要集合。
|
||||
|
||||
@@ -153,7 +153,7 @@ UniDesk/HWLAB Web 开发、Playwright wrapper、`trans <route> playwright`、HWL
|
||||
|
||||
## Async Job State
|
||||
|
||||
长时操作采用 Fire-and-Forget 模式:CLI 创建 `.state/jobs/{jobId}.json`,后台进程执行真实命令,并将 stdout、stderr 分别写入 `.state/jobs/{jobId}.stdout.log` 与 `.state/jobs/{jobId}.stderr.log`。调用者通过 `bun scripts/cli.ts job status <jobId>` 查询进度和尾部输出。
|
||||
长时操作采用 Fire-and-Forget 模式:CLI 创建 `.state/jobs/{jobId}.json`,后台进程执行真实命令,并将 stdout、stderr 分别写入 `.state/jobs/{jobId}.stdout.log` 与 `.state/jobs/{jobId}.stderr.log`。调用者通过 `bun scripts/cli.ts job status <jobId>` 或兼容别名 `bun scripts/cli.ts jobs get <jobId>` 查询进度和尾部输出。
|
||||
|
||||
异步 job 的返回值只表示控制动作已经排入后台执行,不表示目标运行面对象已经创建或收敛。所有带 `statusCommand` 的返回都必须先用 `job status <jobId>` 查看 `progress.stage`、`progress.stageStatus`、关键对象名和 `nextCommand`;只有 progress 已进入对应创建/完成阶段后,才进入更重的运行面 status。对于 `hwlab g14 control-plane trigger-current --lane v02 --confirm`,`progress.pipelineRun` 在 refresh 或 mirror pre-sync 阶段可能只是预期 PipelineRun 名称;在 `progress.stage=create-pipelinerun` 且 `progress.pipelineCreated=true` 前,`control-plane status --pipeline-run <name>` 返回 not found 只能说明 PipelineRun 尚未创建,不能当作 CI/CD 失败。对于 `git-mirror sync|flush --confirm`,先看 job progress 和 timing 摘要,再用对应 `git-mirror status` 确认 `pendingFlush`、local/github refs 和 `githubInSync`;node-scoped `flush` 的 progress 若出现 `partialSuccess=push-succeeded-fetch-failed`,先看是否同时有 `partialSuccessRecovered=true` 和 `postPushRecovery`,未恢复时再按 `nextCommand` 执行同 node/lane 的 `sync --confirm --wait` 刷新 mirror-stage cache。
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ Web/Workbench trace、Web 哨兵和 `web-probe observe` 的人工判定入口以
|
||||
|
||||
## CLI Logs
|
||||
|
||||
异步 job 的 stdout 和 stderr 位于 `.state/jobs/`。`job list` 默认只返回最新 50 条摘要,并为已知异步工作流返回轻量 `progress.summary`;`job status` 会返回结构化 `progress` 与有限尾部,避免输出爆炸,同时保留完整日志文件路径便于继续排查。实现必须只读取日志尾部字节,不得先把完整 job 日志读入 CLI 内存;长时命令的阶段、关键对象名和下一步查询命令应优先沉淀到 `progress`,不能要求调用者先阅读完整日志才能知道是否卡在提交、构建、发布或观测阶段。
|
||||
异步 job 的 stdout 和 stderr 位于 `.state/jobs/`。`job|jobs list` 默认只返回最新 50 条摘要,并为已知异步工作流返回轻量 `progress.summary`;`job status <id>` 与兼容别名 `jobs get/read <id>` 会返回结构化 `progress` 与有限尾部,避免输出爆炸,同时保留完整日志文件路径便于继续排查。实现必须只读取日志尾部字节,不得先把完整 job 日志读入 CLI 内存;长时命令的阶段、关键对象名和下一步查询命令应优先沉淀到 `progress`,不能要求调用者先阅读完整日志才能知道是否卡在提交、构建、发布或观测阶段。
|
||||
|
||||
## Service Logs
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ provider-gateway 可以提供 egress HTTP CONNECT 代理,用于让 Code Queue
|
||||
|
||||
egress proxy 的长期边界是“统一 provider 通道,不引入第二控制面”。backend-core 只接受在线 provider socket 上的 `egress_tcp_*` 消息,并在该 socket 关闭时销毁全部对应 TCP relay;provider-gateway 只维护本地 HTTP proxy 与 WebSocket 消息映射,不保存业务状态,不参与任务调度、统计或节点注册以外的控制面。执行容器、用户服务、Pipeline runner 和 provider-side deploy build 不允许直接连接 backend-core provider ingress,也不允许携带 provider token 自行注册;需要出网时只能连接同节点 provider-gateway 的私有 proxy endpoint。当前 k3s/k8s Code Queue 通过 `d601-provider-egress-proxy` 或 `g14-provider-egress-proxy` Kubernetes Service 连接同节点 provider-gateway egress endpoint,这是 Pod 内的出网入口,不是业务 HTTP 代理入口,也不能替代 Kubernetes API service proxy。部署构建同样不得新建 SSH SOCKS、公网 master proxy 或未登记的宿主全局代理;构建脚本默认只能把 provider-gateway WS egress 作为短生命周期环境变量和 Docker build-arg 注入,并配合目标节点本地 BuildKit/image cache 避免重复下载大依赖层。G14 的节点本地 VPN proxy 是已登记的基础设施 bootstrap 例外,只允许按 `docs/reference/g14.md` 用于 G14 host-side 镜像构建、cache prewarm 或恢复下载;k3s runtime Pod 仍必须使用 `g14-provider-egress-proxy` 和 `g14-tcp-egress-gateway`。
|
||||
|
||||
egress tunnel 必须有生命周期边界:provider-gateway 发出 `egress_tcp_open` 后如果主 server 未在 `openTimeoutMs` 内返回 `egress_tcp_opened` 或 close,必须主动关闭本地 client 并向 core 发送 `egress_tcp_close`;provider-gateway 与 backend-core 都必须对长时间无数据的 relay 执行 idle 清理,避免 provider WebSocket 抖动、TCP connect 卡住或上游未关闭时留下 stale tunnel。排障时如果 `activeTunnels` 持续增长、`pendingTunnels` 非零或 `oldestTunnelAgeMs` 明显超过业务请求耗时,应先看 provider-gateway 与 backend-core egress 清理日志,再判断 Code Queue、PostgreSQL 或 OA Event Flow 本身是否慢。
|
||||
egress tunnel 必须有生命周期边界:provider-gateway 发出 `egress_tcp_open` 后如果主 server 未在 `openTimeoutMs` 内返回 `egress_tcp_opened` 或 close,必须主动关闭本地 client 并向 core 发送 `egress_tcp_close`;provider-gateway 与 backend-core 都必须对长时间无数据的 relay 执行 idle 清理,避免 provider WebSocket 抖动、TCP connect 卡住或上游未关闭时留下 stale tunnel。生命周期日志必须覆盖 provider proxy open request/opened/remote close/local close、backend-core open requested/connect start/connected/connect failed/connect timeout/read-write failed,并包含 `providerId`、`connectionId`、目标 host/port、method、ageMs、pendingBytes 和低基数 `failureKind`;不得记录 payload、DSN 密码、Authorization、Secret 或 token。排障时如果 `activeTunnels` 持续增长、`pendingTunnels` 非零或 `oldestTunnelAgeMs` 明显超过业务请求耗时,应先看 provider-gateway 与 backend-core egress 清理日志,再判断 Code Queue、PostgreSQL 或 OA Event Flow 本身是否慢。
|
||||
|
||||
故障语义必须显式,不允许静默 fallback。provider-gateway 到 backend-core 的 WebSocket 未连接时,本地 proxy 必须返回 503;执行容器不能自动绕过到 D601 本地直连公网、外部公共代理或主 server 公网 HTTP 端口。`NO_PROXY` 只用于 PostgreSQL、OA Event Flow、ClaudeQQ、frontend/backend-core 内网代理、provider-gateway health 等明确内网链路,不能把 GitHub、npm registry 等外部目标加入绕过列表。`hyueapi.com` 与 MiniMax judge 上游 `api.minimaxi.com` 是明确的模型 API 例外:前者会拒绝 provider-gateway egress proxy 出口,后者在 judge 高频短请求上容易受 provider egress 抖动影响导致任务误重试;Code Queue 必须用 `CODE_QUEUE_EGRESS_PROXY_NO_PROXY` / `NO_PROXY` 将 `hyueapi.com,.hyueapi.com,api.minimaxi.com,.minimaxi.com` 配成直连,其它模型 API 仍不得默认绕过 proxy。验收必须同时证明 provider-gateway labels、业务服务 `/health` 和执行容器内 `curl -I https://...` 都走同一 proxy path,hyueapi/MiniMax 例外则以 Code Queue `/health.egressProxy.noProxy`、MiniMax judge 探测和目标任务成功完成作为证据。
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
| 短名 | 运维监控 |
|
||||
| 层级 | L2 课题 |
|
||||
| 状态 | 已生效 |
|
||||
| 实现引用版本 | draft-2026-06-26-p8-egress-job-friction |
|
||||
| 需求规格模板 | [ISO/IEC/IEEE 29148 需求规格模板](../../templates/iso-iec-ieee-29148-requirements-spec-template.md) |
|
||||
| 上级规格 | [PJ2026-0106 平台运维](PJ2026-0106-platform-ops.md) |
|
||||
| 规格治理索引 | [规格治理](spec-governance.md) |
|
||||
@@ -42,6 +43,7 @@
|
||||
- Web/API/AgentRun/HWPOD/Harness/用户管理等服务的运行面健康、资源状态、公开入口健康和用户可感知性能观测。
|
||||
- 发布后 runtime readiness、resource usage、error rate、queue depth、target availability 和 alert 状态摘要。
|
||||
- AgentRun rolling recovery 与 cancel lifecycle 的 active runner、stale lease、cancel request、cancel delivery、terminal report retry、reconciler backlog、projection lag、late-write fencing 和 Job TTL cleanup 观测。
|
||||
- provider-gateway egress TCP tunnel、backend-core egress bridge、受控 relay、异步 job 和 server lifecycle 命令的阶段化可见性。
|
||||
- 监控和 trace 数据的受控查询、低噪声摘要、失败归因和敏感输出约束。
|
||||
|
||||
### 2.3 范围外
|
||||
@@ -62,6 +64,8 @@
|
||||
| Trace Explorer URL 模板 | 由 YAML 声明的 trace backend 打开链接模板,例如包含 `{trace_id}` 占位符的 Tempo、Jaeger 或 Grafana Explore URL;前端只替换合法 OTel trace id,不拼接其他用户输入。 |
|
||||
| span | 一次请求在 admission、billing、AgentRun dispatch、projection、read/replay 等阶段中的一个可观察工作单元。 |
|
||||
| trace context | W3C `traceparent`/`baggage` 等跨进程传播上下文;业务 `traceId/sessionId/runId/turnId` 只能作为 span attribute,不能替代标准 trace context。 |
|
||||
| provider egress TCP tunnel | target node 上 provider-gateway egress proxy 通过 backend-core WebSocket 控制通道请求 master 侧打开 TCP 连接,并在两端转发字节流的出站链路。 |
|
||||
| async job 摘要 | `server start|stop|rebuild|restart` 等长操作创建 `.state/jobs` 后默认返回的低噪声身份、状态、日志路径和后续查询命令,不默认展开完整命令或 stdout/stderr。 |
|
||||
| 用户可感知性能 | 用户在 Web、CLI 或 API 入口中直接感受到的等待时间、加载时间、首个可读内容出现时间和完整可用状态。 |
|
||||
| YAML-first 运维监控 | Prometheus scrape、rule、summary 和 target 归属先进入 UniDesk YAML,再通过受控 CLI 渲染和验证的运维形态。 |
|
||||
| rolling recovery signal | 用于判断 manager/cloud-api rolling 后同一 run/command/runnerJob 是否恢复控制权的低基数 metrics、span attribute 和 CLI 摘要字段。 |
|
||||
@@ -94,6 +98,7 @@
|
||||
| PJ2026-01060506 | Metrics接入 | 本规格 6.6 | metrics endpoint、scrape target 和 label 口径 | 各 L1 服务健康指标 | Prometheus 查询 |
|
||||
| PJ2026-01060507 | Rolling恢复观测 | 本规格 6.7 | active runner、reconciler backlog、terminal retry、projection lag 和不可恢复 blocker 的查询口径 | AgentRun核心、HWLAB接入、Workbench唯一投影、发布Lane | rolling 发布判定、故障收口 |
|
||||
| PJ2026-01060508 | Web哨兵 | [PJ2026-01060508 Web哨兵](PJ2026-01060508-web-probe-sentinel.md) | `web-probe observe` 的 YAML-first 生产哨兵、持续 canary、报告视图、dashboard 分析工作台和发布恢复验证 | Web工作台、Workbench唯一投影、YAML运维、公开入口、发布流水 | 平台值守、CI/CD targetValidation、用户入口恢复判定 |
|
||||
| PJ2026-01060509 | 出站诊断 | 本规格 6.9 | provider egress TCP tunnel、受控 relay、async job/server lifecycle 的阶段化日志、低噪声摘要和诊断入口 | provider-gateway、backend-core、YAML运维、发布流水 | D601/G14 runtime 出站排障、OpenFGA/Postgres/OA Event Flow 链路验收 |
|
||||
|
||||
## 6. 原子需求
|
||||
|
||||
@@ -274,6 +279,77 @@ Web 哨兵的首条生产 canary 应覆盖 `workbench-dsflash-go-tool-call-10x`
|
||||
|
||||
发布流水可调用 Web 哨兵 maintenance start/stop。maintenance start 暂停告警但继续采样;maintenance stop 触发同一 observe CLI quick verify 和 analyze。CI/CD targetValidation 对 HWLAB Web 恢复的判断必须包含 quick verify、analysis report SHA、finding 摘要、public origin、scenario id 和 observer/run id;Argo green 但哨兵 red/timeout 时不得判定发布恢复成功。
|
||||
|
||||
### 6.9 OPS-MON-REQ-009 Provider 出站与 Job 可见性
|
||||
|
||||
| 编号 | 短名 | 主责模块 | 关联模块 |
|
||||
| --- | --- | --- | --- |
|
||||
| OPS-MON-REQ-009 | 出站诊断 | PJ2026-01060509 出站诊断 | [YAML运维](PJ2026-010603-yaml-first-ops.md)、[公开入口](PJ2026-010604-public-entry.md)、[发布流水](PJ2026-010601-controlled-release.md) |
|
||||
|
||||
运维监控应为 provider-gateway egress TCP tunnel、backend-core egress bridge、受控 relay 和 CLI async job 提供阶段化可见性,使 D601/G14 等节点的 PostgreSQL、OpenFGA、OA Event Flow、Git mirror 或外部下载问题可以被定位到 provider 本地 proxy、backend-core open、master 侧 TCP connect、relay ACL、目标服务或调用方读写阶段。
|
||||
|
||||
provider egress tunnel 生命周期至少应记录:provider proxy 收到 client 请求、发出 `egress_tcp_open`、core 确认 opened、core connect failed/timeout、remote close、本地 open timeout、client close、pending buffer 超限和 idle close。日志和受控摘要必须包含 `providerId`、`connectionId`、目标 `host`/`port`、`method`、`ageMs`、`pendingBytes` 和低基数 failureKind;不得记录 TCP payload、PostgreSQL DSN 密码、HTTP Authorization、Secret、token 或完整业务正文。
|
||||
|
||||
`server start|stop|rebuild|restart` 等创建 `.state/jobs` 的命令默认必须返回 async job 摘要,而不是完整 JSON payload、完整 command、完整 runtime env 或 stdout/stderr。摘要至少包含 job id、目标 service、当前状态、stdout/stderr 路径、`job status` 查询命令和必要的 `server logs`/`server status` 后续命令。完整结构只能通过 `--full`/`--raw` 或 `job status <id>` 渐进展开。`job` 与常见自然输入 `jobs`、`status|get|read` 应指向同一受控状态入口,避免调用者因为别名缺失退回手工文件读取。
|
||||
|
||||
#### 6.9.1 目标架构图
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Pod[目标 runtime Pod] --> Proxy[provider-gateway egress proxy]
|
||||
Proxy -- egress_tcp_open/data/close --> Core[backend-core provider WS]
|
||||
Core --> Relay[受控 relay 或目标 TCP 服务]
|
||||
Relay --> Target[Postgres / OA Event Flow / external endpoint]
|
||||
Core --> Logs[backend-core logs]
|
||||
Proxy --> ProviderLogs[provider-gateway logs]
|
||||
CLI[UniDesk CLI job/status/triage] --> Logs
|
||||
CLI --> ProviderLogs
|
||||
```
|
||||
|
||||
#### 6.9.2 数据流图
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Request[CONNECT 或 HTTP proxy 请求] --> LocalEvent[provider proxy stage event]
|
||||
LocalEvent --> OpenMsg[egress_tcp_open]
|
||||
OpenMsg --> CoreEvent[backend-core open/connect stage event]
|
||||
CoreEvent --> Outcome{connect outcome}
|
||||
Outcome -->|opened| DataFlow[egress_tcp_data 双向转发]
|
||||
Outcome -->|failed/timeout| CloseMsg[egress_tcp_close with failureKind]
|
||||
DataFlow --> CloseMsg
|
||||
CloseMsg --> Summary[provider triage / job status / logs 摘要]
|
||||
```
|
||||
|
||||
#### 6.9.3 关键时序图
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant C as Runtime client
|
||||
participant P as provider egress proxy
|
||||
participant B as backend-core
|
||||
participant R as Relay/Target
|
||||
participant CLI as UniDesk CLI
|
||||
|
||||
C->>P: CONNECT host:port
|
||||
P->>P: log egress_proxy_tunnel_open_request
|
||||
P->>B: egress_tcp_open(providerId, connectionId, host, port)
|
||||
B->>B: log egress_tcp_open_requested/connect_start
|
||||
B->>R: TCP connect
|
||||
alt connect ok
|
||||
B->>P: egress_tcp_opened
|
||||
P->>P: log egress_proxy_tunnel_opened
|
||||
P-->>C: 200 Connection Established
|
||||
else connect failed or timeout
|
||||
B->>P: egress_tcp_close(error, failureKind)
|
||||
P->>P: log remote close
|
||||
P-->>C: connection close
|
||||
end
|
||||
CLI->>CLI: job status / logs / triage show bounded stage summary
|
||||
```
|
||||
|
||||
#### 6.9.4 代码引用规则
|
||||
|
||||
本需求范围内新增或修改的源码文件头部必须标注 `SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction`。纯配置、生成 manifest 或无法承载注释头的文件必须能从 owning CLI、渲染器或运行日志追溯到本 SPEC。
|
||||
|
||||
## 7. 过程控制
|
||||
|
||||
本规格不单独索引过程 issue;跨 L1 的内测、灰度和阶段活动索引统一保留在 [PJ2026-01 HWLAB 总规格](PJ2026-01-HWLAB.md) 的 `7. 过程控制`。
|
||||
|
||||
+42
-11
@@ -1,8 +1,10 @@
|
||||
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
||||
// UniDesk CLI dispatcher with bounded server lifecycle and job drill-down output.
|
||||
import { readConfig } from "./src/config";
|
||||
import { debugDispatch, debugHealth, debugSshPool, debugTask, isDebugDispatchCommand, type DebugDispatchCommand } from "./src/debug";
|
||||
import { isRebuildableService, isRestartableService, rebuildService, restartService, stackLogs, stackStatus, startStack, stopStack, unsupportedRebuildService, unsupportedRestartService } from "./src/docker";
|
||||
import { emitError, emitJson, emitText, isRenderedCliResult } from "./src/output";
|
||||
import { cancelJob, jobWithTail, listJobs, listJobsSummary, readJob, renderJobStatusSummary, runJob } from "./src/jobs";
|
||||
import { cancelJob, jobWithTail, listJobs, listJobsSummary, readJob, renderJobLaunchSummary, renderJobStatusSummary, runJob } from "./src/jobs";
|
||||
import { checkHelp, parseCheckOptions, runChecks, runRecoveryGuardrailsCheck } from "./src/check";
|
||||
import { runSsh } from "./src/ssh";
|
||||
import { autoRemoteCiPublishUserServiceDryRunPlan, extractRemoteCliOptions, runRemoteCli } from "./src/remote";
|
||||
@@ -193,6 +195,23 @@ function latestJobId(): string {
|
||||
return jobs[0].id;
|
||||
}
|
||||
|
||||
function wantsFullDisclosure(): boolean {
|
||||
return args.includes("--full") || args.includes("--raw");
|
||||
}
|
||||
|
||||
function emitServerLifecycleResult(result: unknown, ok = true): void {
|
||||
if (!wantsFullDisclosure()) {
|
||||
const rendered = renderJobLaunchSummary(commandName, result);
|
||||
if (rendered !== null) {
|
||||
emitText(rendered.renderedText, rendered.command || commandName);
|
||||
if (!rendered.ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (remoteOptions.host !== null) {
|
||||
process.exitCode = await runRemoteCli(remoteOptions, readConfig());
|
||||
@@ -444,12 +463,11 @@ async function main(): Promise<void> {
|
||||
if (sub === "start") {
|
||||
const result = startStack(config);
|
||||
const ok = (result as { ok?: unknown }).ok !== false;
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
emitServerLifecycleResult(result, ok);
|
||||
return;
|
||||
}
|
||||
if (sub === "stop") {
|
||||
emitJson(commandName, stopStack(config));
|
||||
emitServerLifecycleResult(stopStack(config));
|
||||
return;
|
||||
}
|
||||
if (sub === "status") {
|
||||
@@ -484,7 +502,7 @@ async function main(): Promise<void> {
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
emitJson(commandName, rebuildService(config, third));
|
||||
emitServerLifecycleResult(rebuildService(config, third));
|
||||
return;
|
||||
}
|
||||
if (sub === "restart") {
|
||||
@@ -494,7 +512,7 @@ async function main(): Promise<void> {
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
emitJson(commandName, restartService(config, third));
|
||||
emitServerLifecycleResult(restartService(config, third));
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -564,22 +582,35 @@ async function main(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (top === "job") {
|
||||
if (sub === "list") {
|
||||
if (top === "job" || top === "jobs") {
|
||||
const jobSub = sub === "get" || sub === "read" ? "status" : sub;
|
||||
if (jobSub === "list" || jobSub === undefined || isHelpToken(jobSub)) {
|
||||
if (jobSub === undefined || isHelpToken(jobSub)) {
|
||||
emitJson(commandName, {
|
||||
command: "job|jobs list|status|get|read|cancel",
|
||||
aliases: ["jobs list", "jobs get <jobId|latest>", "jobs read <jobId|latest>"],
|
||||
usage: [
|
||||
"bun scripts/cli.ts job list [--limit N] [--include-command]",
|
||||
"bun scripts/cli.ts job status <jobId|latest> [--tail-bytes N] [--full|--raw]",
|
||||
"bun scripts/cli.ts jobs get <jobId|latest>",
|
||||
],
|
||||
});
|
||||
return;
|
||||
}
|
||||
emitJson(commandName, listJobsSummary({ limit: boundedNumberOption("--limit", 50, 500), includeCommand: args.includes("--include-command") }));
|
||||
return;
|
||||
}
|
||||
if (sub === "status") {
|
||||
if (jobSub === "status") {
|
||||
const id = third === "latest" || third === undefined ? latestJobId() : third;
|
||||
const job = jobWithTail(readJob(id), boundedNumberOption("--tail-bytes", 12000, 500_000));
|
||||
if (args.includes("--full") || args.includes("--raw")) {
|
||||
if (wantsFullDisclosure()) {
|
||||
emitJson(commandName, { job });
|
||||
return;
|
||||
}
|
||||
emitText(renderJobStatusSummary(job).renderedText, commandName);
|
||||
return;
|
||||
}
|
||||
if (sub === "cancel") {
|
||||
if (jobSub === "cancel") {
|
||||
if (!third) throw new Error("job cancel requires job id");
|
||||
emitJson(commandName, cancelJob(third));
|
||||
return;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
||||
// Main-server Docker lifecycle helpers with bounded status probes and job summaries.
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { commandOk, runCommand, tailFile } from "./command";
|
||||
@@ -608,7 +610,7 @@ export function dockerContainers(config: UniDeskConfig): ContainerStatus[] {
|
||||
"label=com.docker.compose.oneoff=False",
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
], repoRoot);
|
||||
], repoRoot, { timeoutMs: 5000 });
|
||||
if (result.exitCode !== 0 || result.stdout.trim().length === 0) return [];
|
||||
return result.stdout
|
||||
.split("\n")
|
||||
@@ -643,9 +645,9 @@ function dockerExecJson(container: string, path: string): unknown {
|
||||
"export url",
|
||||
"bun -e 'const url=process.env.url; fetch(url).then(async r=>{const text=await r.text(); console.log(JSON.stringify({ok:r.ok,status:r.status,body:text?JSON.parse(text):null})); process.exit(r.ok?0:1);}).catch(e=>{console.error(e); process.exit(1)})'",
|
||||
].join("\n");
|
||||
const result = runCommand(["docker", "exec", container, "sh", "-lc", script], repoRoot);
|
||||
const result = runCommand(["docker", "exec", container, "sh", "-lc", script], repoRoot, { timeoutMs: 5000 });
|
||||
if (result.exitCode !== 0) {
|
||||
return { ok: false, exitCode: result.exitCode, stdout: result.stdout.slice(-1200), stderr: result.stderr.slice(-1200) };
|
||||
return { ok: false, exitCode: result.exitCode, timedOut: result.timedOut, signal: result.signal, stdout: result.stdout.slice(-1200), stderr: result.stderr.slice(-1200) };
|
||||
}
|
||||
try {
|
||||
return JSON.parse(result.stdout.trim()) as unknown;
|
||||
@@ -655,8 +657,8 @@ function dockerExecJson(container: string, path: string): unknown {
|
||||
}
|
||||
|
||||
function dockerExec(config: UniDeskConfig, container: string, command: string[]): unknown {
|
||||
const result = runCommand(["docker", "exec", container, ...command], repoRoot);
|
||||
return { ok: result.exitCode === 0, exitCode: result.exitCode, stdout: result.stdout.slice(-1200), stderr: result.stderr.slice(-1200) };
|
||||
const result = runCommand(["docker", "exec", container, ...command], repoRoot, { timeoutMs: 5000 });
|
||||
return { ok: result.exitCode === 0, exitCode: result.exitCode, timedOut: result.timedOut, signal: result.signal, stdout: result.stdout.slice(-1200), stderr: result.stderr.slice(-1200) };
|
||||
}
|
||||
|
||||
export async function stackStatus(config: UniDeskConfig): Promise<unknown> {
|
||||
|
||||
+7
-4
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
||||
// Static CLI help for job aliases and server lifecycle progressive disclosure.
|
||||
import { ghHelp, ghScopedHelp } from "./gh";
|
||||
import { authBrokerHelp } from "./auth-broker";
|
||||
import { platformDbHelp } from "./platform-db";
|
||||
@@ -86,8 +88,8 @@ export function rootHelp(): unknown {
|
||||
{ command: "codex steer-confirm <taskId> --steer-id <id> [--raw]", description: "Read-only lookup for a steerId in task trace so deliveryUnconfirmed can be resolved without resending the corrective prompt." },
|
||||
{ command: "codex interrupt|cancel <taskId>", description: "Request interrupt for a running Code Queue task, or cancel a queued/retry_wait task, through the same private proxy." },
|
||||
{ command: "codex queues [--full|--all] [--limit N] [--page N|--offset N]", description: "Read legacy Code Queue archive summaries. Legacy queue create/merge and move are frozen; use agentrun create/apply/get/cancel for new work." },
|
||||
{ command: "job list [--limit N] [--include-command]", description: "List async jobs from .state/jobs with a bounded default page and progress summaries." },
|
||||
{ command: "job status <jobId|latest> [--tail-bytes N]", description: "Show job state with a structured progress summary and bounded stdout/stderr tails." },
|
||||
{ command: "job|jobs list [--limit N] [--include-command]", description: "List async jobs from .state/jobs with a bounded default page and progress summaries." },
|
||||
{ command: "job status|get|read <jobId|latest> [--tail-bytes N]", description: "Show job state with a structured progress summary and bounded stdout/stderr tails." },
|
||||
{ command: "job cancel <jobId>", description: "Cancel a queued/running async job through the .state/jobs control entry and keep a terminal canceled record." },
|
||||
{ command: "debug health", description: "Probe internal core, nodes, system/Docker status, frontend, provider ingress, and public boundary." },
|
||||
{ command: "debug ssh-pool <providerId>", description: "Show bounded host.ssh.tcp-pool labels for one provider, including ready/claimed/desired/lastError." },
|
||||
@@ -503,14 +505,15 @@ function codexHelp(): unknown {
|
||||
|
||||
function jobHelp(): unknown {
|
||||
return {
|
||||
command: "job list|status|cancel",
|
||||
command: "job|jobs list|status|get|read|cancel",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts job list [--limit N] [--include-command]",
|
||||
"bun scripts/cli.ts job status <jobId|latest> [--tail-bytes N]",
|
||||
"bun scripts/cli.ts jobs get <jobId|latest> [--tail-bytes N]",
|
||||
"bun scripts/cli.ts job cancel <jobId>",
|
||||
],
|
||||
description: "Inspect or cancel fire-and-forget job state from .state/jobs with structured progress summaries and bounded log tails.",
|
||||
description: "Inspect or cancel fire-and-forget job state from .state/jobs with structured progress summaries and bounded log tails. `jobs get/read` are compatibility aliases for `job status`; server lifecycle commands return this drill-down by default and reserve full JSON for --full/--raw.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
||||
// Async job state, summaries, and lifecycle drill-down rendering for UniDesk CLI.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
@@ -270,6 +272,50 @@ export function renderJobStatusSummary(job: ReturnType<typeof jobWithTail>): Ren
|
||||
};
|
||||
}
|
||||
|
||||
export function renderJobLaunchSummary(command: string, result: unknown): RenderedCliResult | null {
|
||||
if (typeof result !== "object" || result === null || !("job" in result)) return null;
|
||||
const record = result as Record<string, unknown>;
|
||||
const job = record.job as Partial<JobRecord> | undefined;
|
||||
if (typeof job !== "object" || job === null || typeof job.id !== "string") return null;
|
||||
const service = typeof record.service === "string" ? record.service : "-";
|
||||
const runtimeEnv = typeof record.runtimeEnv === "object" && record.runtimeEnv !== null ? record.runtimeEnv as Record<string, unknown> : {};
|
||||
const stdoutFile = typeof job.stdoutFile === "string" ? job.stdoutFile : "-";
|
||||
const stderrFile = typeof job.stderrFile === "string" ? job.stderrFile : "-";
|
||||
const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`;
|
||||
const fullCommand = command.includes("--full") || command.includes("--raw") ? null : `bun scripts/cli.ts ${command} --full`;
|
||||
const lines = [
|
||||
"ASYNC_JOB",
|
||||
jobStatusTable(
|
||||
["JOB", "NAME", "STATUS", "SERVICE", "RUNNER", "CREATED"],
|
||||
[[job.id, job.name ?? "-", job.status ?? "-", service, job.runner ?? "-", job.createdAt ?? "-"]],
|
||||
),
|
||||
"",
|
||||
jobStatusTable(
|
||||
["STREAM", "PATH"],
|
||||
[
|
||||
["stdout", stdoutFile],
|
||||
["stderr", stderrFile],
|
||||
["runtimeEnv", typeof runtimeEnv.envFile === "string" ? runtimeEnv.envFile : "-"],
|
||||
["logDir", typeof runtimeEnv.logDir === "string" ? runtimeEnv.logDir : "-"],
|
||||
],
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` status: ${statusCommand}`,
|
||||
" latest: bun scripts/cli.ts job status latest --tail-bytes 12000",
|
||||
" list: bun scripts/cli.ts job list --limit 20",
|
||||
" logs: bun scripts/cli.ts server logs --tail-bytes 3000",
|
||||
" server: bun scripts/cli.ts server status",
|
||||
fullCommand === null ? null : ` full: ${fullCommand}`,
|
||||
].filter((line): line is string => line !== null);
|
||||
return {
|
||||
ok: job.status !== "failed",
|
||||
command,
|
||||
contentType: "text/plain",
|
||||
renderedText: `${lines.join("\n")}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
|
||||
const nowMs = Date.now();
|
||||
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
||||
// Backend-core side of provider egress TCP bridge with stage logs.
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -43,6 +45,23 @@ fn send_egress_close(provider: &Arc<ProviderConnection>, connection_id: &str, er
|
||||
let _ = provider.sender.send(Message::Text(message.to_string()));
|
||||
}
|
||||
|
||||
fn egress_failure_kind(error: &str) -> &'static str {
|
||||
let normalized = error.to_ascii_lowercase();
|
||||
if normalized.contains("connect timeout") {
|
||||
"connect-timeout"
|
||||
} else if normalized.contains("timed out") || normalized.contains("timeout") {
|
||||
"timeout"
|
||||
} else if normalized.contains("invalid") {
|
||||
"invalid-target"
|
||||
} else if normalized.contains("refused") {
|
||||
"connect-refused"
|
||||
} else if normalized.contains("unreachable") {
|
||||
"connect-unreachable"
|
||||
} else {
|
||||
"connect-failed"
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_egress_tcp_open(
|
||||
state: &Arc<AppState>,
|
||||
provider: Arc<ProviderConnection>,
|
||||
@@ -70,10 +89,42 @@ pub async fn handle_egress_tcp_open(
|
||||
|| provider_id.is_empty()
|
||||
|| connection_id.is_empty()
|
||||
{
|
||||
state.log(
|
||||
"warn",
|
||||
"egress_tcp_open_rejected",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"targetHost": host.as_str(),
|
||||
"targetPort": port,
|
||||
"failureKind": "invalid-target"
|
||||
})),
|
||||
);
|
||||
send_egress_close(&provider, &connection_id, Some("invalid egress target"));
|
||||
return Ok(());
|
||||
}
|
||||
state.log(
|
||||
"info",
|
||||
"egress_tcp_open_requested",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"targetHost": host.as_str(),
|
||||
"targetPort": port
|
||||
})),
|
||||
);
|
||||
close_egress_tcp_connection(state, &provider_id, &connection_id, None).await;
|
||||
state.log(
|
||||
"info",
|
||||
"egress_tcp_connect_start",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"targetHost": host.as_str(),
|
||||
"targetPort": port,
|
||||
"timeoutMs": 15_000
|
||||
})),
|
||||
);
|
||||
let stream = match tokio::time::timeout(
|
||||
Duration::from_secs(15),
|
||||
TcpStream::connect((host.as_str(), port as u16)),
|
||||
@@ -82,10 +133,35 @@ pub async fn handle_egress_tcp_open(
|
||||
{
|
||||
Ok(Ok(stream)) => stream,
|
||||
Ok(Err(error)) => {
|
||||
send_egress_close(&provider, &connection_id, Some(&error.to_string()));
|
||||
let error_text = error.to_string();
|
||||
state.log(
|
||||
"warn",
|
||||
"egress_tcp_connect_failed",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"targetHost": host.as_str(),
|
||||
"targetPort": port,
|
||||
"failureKind": egress_failure_kind(&error_text),
|
||||
"error": error_text.as_str()
|
||||
})),
|
||||
);
|
||||
send_egress_close(&provider, &connection_id, Some(&error_text));
|
||||
return Ok(());
|
||||
}
|
||||
Err(_) => {
|
||||
state.log(
|
||||
"warn",
|
||||
"egress_tcp_connect_timeout",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"targetHost": host.as_str(),
|
||||
"targetPort": port,
|
||||
"failureKind": "connect-timeout",
|
||||
"timeoutMs": 15_000
|
||||
})),
|
||||
);
|
||||
send_egress_close(
|
||||
&provider,
|
||||
&connection_id,
|
||||
@@ -108,6 +184,16 @@ pub async fn handle_egress_tcp_open(
|
||||
let _ = provider.sender.send(Message::Text(
|
||||
json!({ "type": "egress_tcp_opened", "connectionId": connection_id }).to_string(),
|
||||
));
|
||||
state.log(
|
||||
"info",
|
||||
"egress_tcp_connected",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"targetHost": host.as_str(),
|
||||
"targetPort": port
|
||||
})),
|
||||
);
|
||||
|
||||
tokio::spawn({
|
||||
let state = state.clone();
|
||||
@@ -125,7 +211,18 @@ pub async fn handle_egress_tcp_open(
|
||||
let _ = provider.sender.send(Message::Text(json!({ "type": "egress_tcp_data", "connectionId": connection_id, "data": data, "encoding": "base64" }).to_string()));
|
||||
}
|
||||
Err(error) => {
|
||||
send_egress_close(&provider, &connection_id, Some(&error.to_string()));
|
||||
let error_text = error.to_string();
|
||||
state.log(
|
||||
"warn",
|
||||
"egress_tcp_read_failed",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"failureKind": "read-failed",
|
||||
"error": error_text.as_str()
|
||||
})),
|
||||
);
|
||||
send_egress_close(&provider, &connection_id, Some(&error_text));
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -144,7 +241,17 @@ pub async fn handle_egress_tcp_open(
|
||||
let connection_id = connection_id.clone();
|
||||
async move {
|
||||
while let Some(data) = rx.recv().await {
|
||||
if writer.write_all(&data).await.is_err() {
|
||||
if let Err(error) = writer.write_all(&data).await {
|
||||
state.log(
|
||||
"warn",
|
||||
"egress_tcp_write_failed",
|
||||
Some(json!({
|
||||
"providerId": provider_id.as_str(),
|
||||
"connectionId": connection_id.as_str(),
|
||||
"failureKind": "write-failed",
|
||||
"error": error.to_string()
|
||||
})),
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -204,13 +311,27 @@ pub async fn close_egress_tcp_connection(
|
||||
state: &Arc<AppState>,
|
||||
provider_id: &str,
|
||||
connection_id: &str,
|
||||
_error: Option<&str>,
|
||||
error: Option<&str>,
|
||||
) {
|
||||
state
|
||||
let removed = state
|
||||
.active_egress_tcp
|
||||
.lock()
|
||||
.await
|
||||
.remove(&egress_tcp_key(provider_id, connection_id));
|
||||
.remove(&egress_tcp_key(provider_id, connection_id))
|
||||
.is_some();
|
||||
if removed || error.is_some() {
|
||||
state.log(
|
||||
if error.is_some() { "warn" } else { "info" },
|
||||
"egress_tcp_connection_closed",
|
||||
Some(json!({
|
||||
"providerId": provider_id,
|
||||
"connectionId": connection_id,
|
||||
"removed": removed,
|
||||
"failureKind": error.map(egress_failure_kind).unwrap_or("normal-close"),
|
||||
"error": error.unwrap_or("")
|
||||
})),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn close_egress_tcp_connections_for_provider(state: &Arc<AppState>, provider_id: &str) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// SPEC: PJ2026-01060509 出站诊断 draft-2026-06-26-p8-egress-job-friction.
|
||||
// Provider-side egress TCP proxy with lifecycle logs for tunnel diagnosis.
|
||||
import { createServer, type Server, type Socket } from "node:net";
|
||||
import type {
|
||||
CoreEgressTcpCloseMessage,
|
||||
@@ -33,6 +35,8 @@ interface Tunnel {
|
||||
closed: boolean;
|
||||
createdAt: number;
|
||||
lastActivityAt: number;
|
||||
targetHost: string;
|
||||
targetPort: number;
|
||||
openTimer: ReturnType<typeof setTimeout> | null;
|
||||
idleTimer: ReturnType<typeof setTimeout> | null;
|
||||
}
|
||||
@@ -116,6 +120,18 @@ function proxyUrlFor(host: string, port: number): string {
|
||||
return `http://${host}:${port}`;
|
||||
}
|
||||
|
||||
function egressFailureKind(error: string | undefined): string {
|
||||
if (error === undefined || error.length === 0) return "normal-close";
|
||||
const normalized = error.toLowerCase();
|
||||
if (normalized.includes("open timeout")) return "open-timeout";
|
||||
if (normalized.includes("idle timeout")) return "idle-timeout";
|
||||
if (normalized.includes("pending buffer")) return "pending-buffer-exceeded";
|
||||
if (normalized.includes("not connected")) return "core-channel-not-connected";
|
||||
if (normalized.includes("timeout")) return "timeout";
|
||||
if (normalized.includes("closed")) return "closed";
|
||||
return "remote-or-socket-error";
|
||||
}
|
||||
|
||||
const tunnelOpenTimeoutMs = 15_000;
|
||||
const tunnelIdleTimeoutMs = 600_000;
|
||||
const maxPendingBytes = 4 * 1024 * 1024;
|
||||
@@ -163,14 +179,19 @@ export function startProviderEgressProxy(options: ProviderEgressProxyOptions): P
|
||||
tunnel.pendingBytes = 0;
|
||||
if (!tunnel.client.destroyed) tunnel.client.destroy();
|
||||
if (notifyCore) send({ type: "egress_tcp_close", providerId: options.providerId, connectionId: tunnel.id, at: nowIso() });
|
||||
if (error !== undefined) {
|
||||
options.logger("warn", "egress_proxy_tunnel_closed", {
|
||||
connectionId: tunnel.id,
|
||||
opened: tunnel.opened,
|
||||
ageMs: Date.now() - tunnel.createdAt,
|
||||
error,
|
||||
});
|
||||
}
|
||||
const event = {
|
||||
providerId: options.providerId,
|
||||
connectionId: tunnel.id,
|
||||
method: tunnel.method,
|
||||
targetHost: tunnel.targetHost,
|
||||
targetPort: tunnel.targetPort,
|
||||
opened: tunnel.opened,
|
||||
ageMs: Date.now() - tunnel.createdAt,
|
||||
pendingBytes: tunnel.pendingBytes,
|
||||
failureKind: egressFailureKind(error),
|
||||
...(error === undefined ? {} : { error }),
|
||||
};
|
||||
options.logger(error === undefined ? "info" : "warn", "egress_proxy_tunnel_closed", event);
|
||||
};
|
||||
|
||||
const closeTunnel = (id: string, error?: string): void => {
|
||||
@@ -206,6 +227,15 @@ export function startProviderEgressProxy(options: ProviderEgressProxyOptions): P
|
||||
tunnel.openTimer = null;
|
||||
}
|
||||
refreshTunnelIdle(tunnel);
|
||||
options.logger("info", "egress_proxy_tunnel_opened", {
|
||||
providerId: options.providerId,
|
||||
connectionId: tunnel.id,
|
||||
method: tunnel.method,
|
||||
targetHost: tunnel.targetHost,
|
||||
targetPort: tunnel.targetPort,
|
||||
ageMs: Date.now() - tunnel.createdAt,
|
||||
pendingBytes: tunnel.pendingBytes,
|
||||
});
|
||||
if (tunnel.method === "CONNECT") {
|
||||
tunnel.client.write("HTTP/1.1 200 Connection Established\r\nProxy-Agent: UniDesk-ProviderGateway\r\n\r\n");
|
||||
}
|
||||
@@ -226,7 +256,14 @@ export function startProviderEgressProxy(options: ProviderEgressProxyOptions): P
|
||||
destroyTunnel(tunnel, false, message.error);
|
||||
}
|
||||
if (message.error !== undefined && message.error.length > 0) {
|
||||
options.logger("warn", "egress_proxy_remote_close", { connectionId: message.connectionId, error: message.error });
|
||||
options.logger("warn", "egress_proxy_remote_close", {
|
||||
providerId: options.providerId,
|
||||
connectionId: message.connectionId,
|
||||
targetHost: tunnel?.targetHost ?? null,
|
||||
targetPort: tunnel?.targetPort ?? null,
|
||||
failureKind: egressFailureKind(message.error),
|
||||
error: message.error,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -295,10 +332,20 @@ export function startProviderEgressProxy(options: ProviderEgressProxyOptions): P
|
||||
closed: false,
|
||||
createdAt,
|
||||
lastActivityAt: createdAt,
|
||||
targetHost: target.host,
|
||||
targetPort: target.port,
|
||||
openTimer: null,
|
||||
idleTimer: null,
|
||||
};
|
||||
tunnels.set(id, tunnel);
|
||||
options.logger("info", "egress_proxy_tunnel_open_request", {
|
||||
providerId: options.providerId,
|
||||
connectionId: id,
|
||||
method: parsed.method,
|
||||
targetHost: target.host,
|
||||
targetPort: target.port,
|
||||
firstPayloadBytes: firstPayload === null ? 0 : firstPayload.byteLength,
|
||||
});
|
||||
client.on("data", (nextChunk) => {
|
||||
const nextBuffer = Buffer.isBuffer(nextChunk) ? nextChunk : Buffer.from(nextChunk);
|
||||
if (!tunnel.opened) {
|
||||
@@ -315,6 +362,14 @@ export function startProviderEgressProxy(options: ProviderEgressProxyOptions): P
|
||||
tunnels.delete(id);
|
||||
tunnel.closed = true;
|
||||
clearTunnelTimers(tunnel);
|
||||
options.logger("warn", "egress_proxy_tunnel_open_not_sent", {
|
||||
providerId: options.providerId,
|
||||
connectionId: id,
|
||||
method: parsed.method,
|
||||
targetHost: target.host,
|
||||
targetPort: target.port,
|
||||
failureKind: "core-channel-not-connected",
|
||||
});
|
||||
fail("503 Service Unavailable", "provider-gateway core channel is not connected\n");
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user