feat(deploy): distribute code-queue github ssh identity

2026-05-16 11:42:29 +00:00
parent 772b48b29e
commit 83ed99f659
3 changed files with 368 additions and 8 deletions
@@ -19,14 +19,15 @@ bun scripts/cli.ts job status <jobId> --tail-bytes 30000

 部署 job 的步骤固定为：

-1. 在 D601 的 deploy cache 中通过本机 provider-gateway WS egress proxy 执行 `git fetch` remote，并用 `git archive <commitId>` 导出 tracked files 到一次性 export 目录；不得让 D601 直连 GitHub，也不得临时创建 SSH SOCKS、公网 master proxy 或 backend-core/provider-ingress fallback。
-2. 用 `rsync --delete` 同步导出的 repo 到 `/home/ubuntu/cq-deploy`，保留 `.state/`、`logs/`、`.git/`、`node_modules/` 和 `dist/`。
-3. 在 D601 用目标 Docker daemon 的本地 BuildKit builder 构建 `unidesk-code-queue:d601`，复用 D601 上已有基础镜像、inline cache 和 Code Queue build-base；provider-gateway WS egress 是唯一允许的构建代理通道，只作为本次 build 的环境变量与 build-arg 注入，并配合本次 build 的 `--network host` 让 RUN 阶段访问 D601 宿主 loopback proxy，不能污染 D601 宿主 Docker/HTTP proxy 配置，不能新建 SSH SOCKS、公网 master proxy 或直连 fallback。
-4. `docker save` 镜像并导入 k3s containerd：`docker exec -i unidesk-v8s-server ctr -n k8s.io images import -`。
-5. `kubectl apply -f src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml`，其中包含 Code Queue 和 `d601-tcp-egress-gateway`。
-6. 将解析后的 40 位 remote commit 写入 `deployment/code-queue` 的 `CODE_QUEUE_DEPLOY_COMMIT` / `CODE_QUEUE_DEPLOY_REQUESTED_COMMIT`，并记录到 Deployment annotation。
-7. `kubectl -n unidesk rollout restart deployment/d601-tcp-egress-gateway deployment/code-queue` 并等待 rollout 完成。
-8. 通过 backend-core 的真实微服务代理读取 Code Queue `/health`，强制校验 `deploy.commit` 等于本次解析出的 remote commit；如果健康的是旧服务或旧 Pod，job 必须失败。
+1. 对 Code Queue 部署先确保 PostgreSQL 中存在 `unidesk_deploy_ssh_identities(id='github.com')`，该记录保存 GitHub deploy SSH identity 的 private key、public key fingerprint 和 github.com `known_hosts` 行。`codex deploy` 会用主 server 当前 `/root/.ssh/id_ed25519` 种子化这条记录，然后通过 backend-core `/ws/ssh` 交互通道把 identity 流式分发到 D601 WSL `/home/ubuntu/.ssh/id_ed25519`、`id_ed25519.pub` 和 `known_hosts`，并在 D601 侧执行 `ssh -T git@github.com` 验证；secret 不得写入 `host.ssh` task payload、deploy 日志、Docker image 或 Kubernetes Secret。
+2. 在 D601 的 deploy cache 中通过本机 provider-gateway WS egress proxy 执行 `git fetch` remote，并用 `git archive <commitId>` 导出 tracked files 到一次性 export 目录；不得让 D601 直连 GitHub，也不得临时创建 SSH SOCKS、公网 master proxy 或 backend-core/provider-ingress fallback。
+3. 用 `rsync --delete` 同步导出的 repo 到 `/home/ubuntu/cq-deploy`，保留 `.state/`、`logs/`、`.git/`、`node_modules/` 和 `dist/`。
+4. 在 D601 用目标 Docker daemon 的本地 BuildKit builder 构建 `unidesk-code-queue:d601`，复用 D601 上已有基础镜像、inline cache 和 Code Queue build-base；provider-gateway WS egress 是唯一允许的构建代理通道，只作为本次 build 的环境变量与 build-arg 注入，并配合本次 build 的 `--network host` 让 RUN 阶段访问 D601 宿主 loopback proxy，不能污染 D601 宿主 Docker/HTTP proxy 配置，不能新建 SSH SOCKS、公网 master proxy 或直连 fallback。
+5. `docker save` 镜像并导入 k3s containerd：`docker exec -i unidesk-v8s-server ctr -n k8s.io images import -`。
+6. `kubectl apply -f src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml`，其中包含 Code Queue 和 `d601-tcp-egress-gateway`。
+7. 将解析后的 40 位 remote commit 写入 `deployment/code-queue` 的 `CODE_QUEUE_DEPLOY_COMMIT` / `CODE_QUEUE_DEPLOY_REQUESTED_COMMIT`，并记录到 Deployment annotation。
+8. `kubectl -n unidesk rollout restart deployment/d601-tcp-egress-gateway deployment/code-queue` 并等待 rollout 完成。
+9. 通过 backend-core 的真实微服务代理读取 Code Queue `/health`，强制校验 `deploy.commit` 等于本次解析出的 remote commit；如果健康的是旧服务或旧 Pod，job 必须失败。

 ## Observability

@@ -0,0 +1,320 @@
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { type UniDeskConfig, repoRoot, rootPath } from "./config";
+
+export interface GithubSshIdentityDistribution {
+  ok: boolean;
+  detail: string;
+  fingerprint: string | null;
+  seededFromLocal: boolean;
+  raw?: unknown;
+}
+
+interface GithubSshIdentity {
+  privateKey: string;
+  publicKey: string;
+  knownHosts: string;
+  updatedAt: string | null;
+}
+
+const identityId = "github.com";
+const defaultPrivateKeyPath = "/root/.ssh/id_ed25519";
+const defaultKnownHostsPath = "/root/.ssh/known_hosts";
+
+function pgLiteral(value: string): string {
+  return `'${value.replace(/'/gu, "''")}'`;
+}
+
+function commandOutput(command: string[], input?: string, timeoutMs = 30_000): { ok: boolean; stdout: string; stderr: string; exitCode: number | null } {
+  const result = spawnSync(command[0], command.slice(1), {
+    cwd: repoRoot,
+    input,
+    encoding: "utf8",
+    maxBuffer: 1024 * 1024 * 8,
+    timeout: timeoutMs,
+  });
+  return {
+    ok: result.status === 0,
+    stdout: result.stdout ?? "",
+    stderr: result.stderr ?? result.error?.message ?? "",
+    exitCode: result.status,
+  };
+}
+
+function runPsql(config: UniDeskConfig, sql: string): { ok: boolean; stdout: string; stderr: string; exitCode: number | null } {
+  return commandOutput([
+    "docker",
+    "exec",
+    "-i",
+    "unidesk-database",
+    "psql",
+    "-v",
+    "ON_ERROR_STOP=1",
+    "-U",
+    config.database.user,
+    "-d",
+    config.database.name,
+    "-X",
+    "-q",
+    "-t",
+    "-A",
+  ], sql);
+}
+
+function ensureIdentityTableSql(): string {
+  return `
+CREATE TABLE IF NOT EXISTS unidesk_deploy_ssh_identities (
+  id TEXT PRIMARY KEY,
+  host TEXT NOT NULL,
+  private_key TEXT NOT NULL,
+  public_key TEXT NOT NULL,
+  public_key_fingerprint TEXT NOT NULL DEFAULT '',
+  known_hosts TEXT NOT NULL DEFAULT '',
+  source TEXT NOT NULL DEFAULT 'operator-local',
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+`;
+}
+
+function publicKeyFingerprint(publicKey: string): string {
+  const parts = publicKey.trim().split(/\s+/u);
+  const encoded = parts[1] ?? "";
+  if (encoded.length === 0) return "";
+  const digest = createHash("sha256").update(Buffer.from(encoded, "base64")).digest("base64").replace(/=+$/u, "");
+  return `SHA256:${digest}`;
+}
+
+function githubKnownHostLinesFromText(text: string): string[] {
+  const rows: string[] = [];
+  const seen = new Set<string>();
+  for (const rawLine of text.split(/\r?\n/u)) {
+    const line = rawLine.trim();
+    if (line.length === 0 || line.startsWith("#")) continue;
+    const parts = line.split(/\s+/u);
+    if (parts.length < 3) continue;
+    const hostList = parts[0]?.split(",") ?? [];
+    if (!hostList.some((host) => host === "github.com" || host === "[github.com]:22")) continue;
+    if (seen.has(line)) continue;
+    seen.add(line);
+    rows.push(line);
+  }
+  return rows;
+}
+
+function readGithubKnownHosts(): string {
+  const knownHostsPath = process.env.UNIDESK_GITHUB_KNOWN_HOSTS_PATH || defaultKnownHostsPath;
+  const localLines = existsSync(knownHostsPath) ? githubKnownHostLinesFromText(readFileSync(knownHostsPath, "utf8")) : [];
+  if (localLines.length > 0) return `${localLines.join("\n")}\n`;
+  const scan = commandOutput(["ssh-keyscan", "-t", "rsa,ecdsa,ed25519", "github.com"], undefined, 20_000);
+  const scannedLines = githubKnownHostLinesFromText(scan.stdout);
+  if (scannedLines.length === 0) throw new Error(scan.stderr || "failed to collect github.com SSH host keys");
+  return `${scannedLines.join("\n")}\n`;
+}
+
+function readLocalGithubIdentity(): GithubSshIdentity | null {
+  const privateKeyPath = process.env.UNIDESK_GITHUB_SSH_KEY_PATH || defaultPrivateKeyPath;
+  if (!existsSync(privateKeyPath)) return null;
+  const privateKey = readFileSync(privateKeyPath, "utf8");
+  if (!/-----BEGIN [A-Z ]*PRIVATE KEY-----/u.test(privateKey)) throw new Error(`invalid private key format: ${privateKeyPath}`);
+  const publicKeyPath = `${privateKeyPath}.pub`;
+  const publicKey = existsSync(publicKeyPath)
+    ? readFileSync(publicKeyPath, "utf8").trim()
+    : commandOutput(["ssh-keygen", "-y", "-f", privateKeyPath]).stdout.trim();
+  if (!/^(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp256)\s+\S+/u.test(publicKey)) throw new Error(`invalid public key for ${privateKeyPath}`);
+  return {
+    privateKey: privateKey.endsWith("\n") ? privateKey : `${privateKey}\n`,
+    publicKey,
+    knownHosts: readGithubKnownHosts(),
+    updatedAt: null,
+  };
+}
+
+function upsertGithubIdentity(config: UniDeskConfig, identity: GithubSshIdentity): void {
+  const fingerprint = publicKeyFingerprint(identity.publicKey);
+  const sql = `
+${ensureIdentityTableSql()}
+INSERT INTO unidesk_deploy_ssh_identities (id, host, private_key, public_key, public_key_fingerprint, known_hosts, source, updated_at)
+VALUES (
+  ${pgLiteral(identityId)},
+  ${pgLiteral(identityId)},
+  ${pgLiteral(identity.privateKey)},
+  ${pgLiteral(identity.publicKey)},
+  ${pgLiteral(fingerprint)},
+  ${pgLiteral(identity.knownHosts)},
+  'operator-local',
+  now()
+)
+ON CONFLICT (id) DO UPDATE SET
+  host = EXCLUDED.host,
+  private_key = EXCLUDED.private_key,
+  public_key = EXCLUDED.public_key,
+  public_key_fingerprint = EXCLUDED.public_key_fingerprint,
+  known_hosts = EXCLUDED.known_hosts,
+  source = EXCLUDED.source,
+  updated_at = now();
+`;
+  const result = runPsql(config, sql);
+  if (!result.ok) throw new Error(`failed to upsert GitHub SSH identity in PostgreSQL: ${result.stderr || `exit=${result.exitCode}`}`);
+}
+
+function readGithubIdentityFromDatabase(config: UniDeskConfig): GithubSshIdentity {
+  const sql = `
+${ensureIdentityTableSql()}
+SELECT json_build_object(
+  'privateKey', private_key,
+  'publicKey', public_key,
+  'knownHosts', known_hosts,
+  'updatedAt', updated_at
+)::text
+FROM unidesk_deploy_ssh_identities
+WHERE id = ${pgLiteral(identityId)}
+LIMIT 1;
+`;
+  const result = runPsql(config, sql);
+  if (!result.ok) throw new Error(`failed to read GitHub SSH identity from PostgreSQL: ${result.stderr || `exit=${result.exitCode}`}`);
+  const line = result.stdout.trim().split(/\r?\n/u).find((item) => item.trim().startsWith("{"));
+  if (line === undefined) throw new Error("GitHub SSH identity is missing in PostgreSQL");
+  const parsed = JSON.parse(line) as Partial<GithubSshIdentity>;
+  const privateKey = typeof parsed.privateKey === "string" ? parsed.privateKey : "";
+  const publicKey = typeof parsed.publicKey === "string" ? parsed.publicKey : "";
+  const knownHosts = typeof parsed.knownHosts === "string" ? parsed.knownHosts : "";
+  if (!/-----BEGIN [A-Z ]*PRIVATE KEY-----/u.test(privateKey)) throw new Error("PostgreSQL GitHub SSH identity has invalid private key material");
+  if (!/^(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp256)\s+\S+/u.test(publicKey)) throw new Error("PostgreSQL GitHub SSH identity has invalid public key material");
+  if (githubKnownHostLinesFromText(knownHosts).length === 0) throw new Error("PostgreSQL GitHub SSH identity has no github.com known_hosts rows");
+  return {
+    privateKey: privateKey.endsWith("\n") ? privateKey : `${privateKey}\n`,
+    publicKey,
+    knownHosts: knownHosts.endsWith("\n") ? knownHosts : `${knownHosts}\n`,
+    updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : null,
+  };
+}
+
+function remoteInstallPythonSource(): string {
+  return String.raw`
+import json
+import os
+import pathlib
+import stat
+import subprocess
+import sys
+
+data = json.load(sys.stdin)
+ssh_dir = pathlib.Path("/home/ubuntu/.ssh")
+private_key = str(data.get("privateKey") or "")
+public_key = str(data.get("publicKey") or "").strip()
+known_hosts = str(data.get("knownHosts") or "")
+
+if "PRIVATE KEY-----" not in private_key:
+    raise SystemExit("invalid private key payload")
+if not public_key.startswith(("ssh-ed25519 ", "ssh-rsa ", "ecdsa-sha2-nistp256 ")):
+    raise SystemExit("invalid public key payload")
+
+ssh_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
+os.chmod(ssh_dir, 0o700)
+private_path = ssh_dir / "id_ed25519"
+public_path = ssh_dir / "id_ed25519.pub"
+known_hosts_path = ssh_dir / "known_hosts"
+
+private_path.write_text(private_key if private_key.endswith("\n") else private_key + "\n")
+public_path.write_text(public_key + "\n")
+os.chmod(private_path, stat.S_IRUSR | stat.S_IWUSR)
+os.chmod(public_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+
+derived = subprocess.run(["ssh-keygen", "-y", "-f", str(private_path)], text=True, capture_output=True)
+if derived.returncode != 0:
+    raise SystemExit("installed private key failed ssh-keygen verification")
+if derived.stdout.strip() != public_key:
+    raise SystemExit("installed private key does not match public key")
+
+known_hosts_path.touch(mode=0o600, exist_ok=True)
+subprocess.run(["ssh-keygen", "-R", "github.com", "-f", str(known_hosts_path)], text=True, capture_output=True)
+subprocess.run(["ssh-keygen", "-R", "[github.com]:22", "-f", str(known_hosts_path)], text=True, capture_output=True)
+existing = known_hosts_path.read_text().splitlines()
+rows = []
+seen = set()
+for line in existing + known_hosts.splitlines():
+    line = line.strip()
+    if not line or line.startswith("#"):
+        continue
+    parts = line.split()
+    if len(parts) < 3:
+        continue
+    if line in seen:
+        continue
+    seen.add(line)
+    rows.append(line)
+known_hosts_path.write_text("\n".join(rows) + "\n")
+os.chmod(known_hosts_path, stat.S_IRUSR | stat.S_IWUSR)
+print(f"github_ssh_identity_written path={private_path} known_hosts_rows={len(rows)}")
+`;
+}
+
+function shellQuote(value: string): string {
+  return `'${value.replace(/'/gu, `'\\''`)}'`;
+}
+
+function distributeGithubIdentity(providerId: string, identity: GithubSshIdentity): { ok: boolean; detail: string; raw: unknown } {
+  const payload = JSON.stringify({
+    privateKey: identity.privateKey,
+    publicKey: identity.publicKey,
+    knownHosts: identity.knownHosts,
+  });
+  const remotePython = remoteInstallPythonSource();
+  const remoteCommand = [
+    `python3 -c ${shellQuote(remotePython)}`,
+    "auth_output=$(ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -T git@github.com 2>&1 || true)",
+    "printf '%s\\n' \"$auth_output\"",
+    "printf '%s\\n' \"$auth_output\" | grep -q 'successfully authenticated'",
+  ].join(" && ");
+  const result = spawnSync(process.execPath, [
+    rootPath("scripts", "cli.ts"),
+    "ssh",
+    providerId,
+    "--",
+    remoteCommand,
+  ], {
+    cwd: repoRoot,
+    input: payload,
+    encoding: "utf8",
+    timeout: 90_000,
+    maxBuffer: 1024 * 1024 * 4,
+    env: { ...process.env, UNIDESK_SSH_OPEN_TIMEOUT_MS: process.env.UNIDESK_SSH_OPEN_TIMEOUT_MS || "60000" },
+  });
+  const stdout = result.stdout ?? "";
+  const stderr = result.stderr ?? result.error?.message ?? "";
+  return {
+    ok: result.status === 0,
+    detail: [stdout, stderr].filter(Boolean).join("\n").slice(-2000),
+    raw: { exitCode: result.status, stdoutTail: stdout.slice(-1200), stderrTail: stderr.slice(-1200) },
+  };
+}
+
+export async function ensureGithubSshIdentityForProvider(config: UniDeskConfig, providerId: string): Promise<GithubSshIdentityDistribution> {
+  let seededFromLocal = false;
+  const localIdentity = readLocalGithubIdentity();
+  if (localIdentity !== null) {
+    upsertGithubIdentity(config, localIdentity);
+    seededFromLocal = true;
+  }
+  const identity = readGithubIdentityFromDatabase(config);
+  const fingerprint = publicKeyFingerprint(identity.publicKey);
+  const distribution = distributeGithubIdentity(providerId, identity);
+  if (!distribution.ok) {
+    return {
+      ok: false,
+      detail: `failed to distribute GitHub SSH identity from PostgreSQL to ${providerId}: ${distribution.detail || "remote ssh failed"}`,
+      fingerprint,
+      seededFromLocal,
+      raw: distribution.raw,
+    };
+  }
+  return {
+    ok: true,
+    detail: `GitHub SSH identity distributed from PostgreSQL to ${providerId}; fingerprint=${fingerprint}; seededFromLocal=${seededFromLocal}`,
+    fingerprint,
+    seededFromLocal,
+    raw: distribution.raw,
+  };
+}
@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { runCommand } from "./command";
 import { type UniDeskConfig, type UniDeskMicroserviceConfig, repoRoot, rootPath } from "./config";
+import { ensureGithubSshIdentityForProvider } from "./deploy-ssh-identity";
 import { startJob } from "./jobs";
 import { coreInternalFetch } from "./microservices";

@@ -893,6 +894,40 @@ function pushStep(steps: StepResult[], result: StepResult): boolean {
  return result.ok;
 }

+async function ensureGithubSshIdentityStep(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<StepResult> {
+  const startedAt = nowIso();
+  const startedMs = Date.now();
+  progressLine("github-ssh-identity", "start", { serviceId: service.id, providerId: service.providerId });
+  try {
+    const result = await ensureGithubSshIdentityForProvider(config, service.providerId);
+    progressLine("github-ssh-identity", result.ok ? "succeeded" : "failed", {
+      serviceId: service.id,
+      providerId: service.providerId,
+      elapsedMs: elapsedMs(startedMs),
+      fingerprint: result.fingerprint,
+      seededFromLocal: result.seededFromLocal,
+      detail: result.ok ? result.detail : compactTail(result.detail, 1200),
+    });
+    return {
+      step: "github-ssh-identity",
+      ok: result.ok,
+      detail: result.detail,
+      startedAt,
+      finishedAt: nowIso(),
+      raw: result.raw,
+    };
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    progressLine("github-ssh-identity", "failed", {
+      serviceId: service.id,
+      providerId: service.providerId,
+      elapsedMs: elapsedMs(startedMs),
+      detail: compactTail(detail, 1200),
+    });
+    return { step: "github-ssh-identity", ok: false, detail, startedAt, finishedAt: nowIso(), raw: null };
+  }
+}
+
 async function applyOneService(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService, options: DeployOptions): Promise<Record<string, unknown>> {
  const steps: StepResult[] = [];
  const startedAt = nowIso();
@@ -904,6 +939,10 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi

  const runId = `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
  const exportDir = targetExportDir(service, runId);
+  if (service.id === "code-queue" && !targetIsMain(service)) {
+    const identity = await ensureGithubSshIdentityStep(config, service);
+    if (!pushStep(steps, identity)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), before, steps };
+  }
  const prepare = await step(config, service, "prepare-source", prepareSourceScript(service, desired, exportDir), targetIsMain(service) ? repoRoot : "/home/ubuntu", Math.min(options.timeoutMs, 180_000), !targetIsMain(service));
  if (!pushStep(steps, prepare)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), before, steps };
  const resolvedCommit = parseFullCommit(prepare.detail) || parseFullCommit(JSON.stringify(prepare.raw));