import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { runCommand } from "./command"; import { type UniDeskConfig, type UniDeskMicroserviceConfig, repoRoot, rootPath } from "./config"; import { ensureGithubSshIdentityForProvider } from "./deploy-ssh-identity"; import { startJob } from "./jobs"; import { coreInternalFetch } from "./microservices"; type DeployAction = "check" | "plan" | "apply"; interface DeployManifestService { id: string; repo: string; commitId: string; } interface DeployManifest { schemaVersion: 1; services: DeployManifestService[]; } interface DeployOptions { file: string; serviceId: string | null; runNow: boolean; dryRun: boolean; force: boolean; timeoutMs: number; } interface StepResult { step: string; ok: boolean; detail: string; startedAt: string; finishedAt: string; raw?: unknown; } interface DispatchResult { ok: boolean; taskId: string | null; status: string | null; stdout: string; stderr: string; exitCode: number | null; raw: unknown; } interface BackgroundPoll { done: boolean; exitCode: number | null; logTail: string; raw: unknown; } interface RemoteScriptUpload { ok: boolean; path: string; error: string; raw: unknown[]; } interface ServiceRuntimeState { serviceId: string; ok: boolean; supported: boolean; reason: string | null; desiredRepo: string; desiredCommit: string; providerId: string; deploymentMode: string; currentCommit: string | null; healthCommit: string | null; imageCommit: string | null; orchestratorCommit: string | null; healthOk: boolean; upToDate: boolean; raw?: unknown; } const defaultDeployFile = "deploy.json"; const defaultTimeoutMs = 900_000; const resolveFetchTimeout = "120s"; const shortDispatchWaitMs = 25_000; const shortRemoteTimeoutMs = 20_000; const pollIntervalMs = 5_000; const remoteDeployRoot = "/home/ubuntu/.unidesk/deploy"; const k8sNamespace = "unidesk"; const k8sKubeconfig = "/etc/rancher/k3s/k3s.yaml"; const k3sDeployDir = "/home/ubuntu/cq-deploy"; const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789"; const nativeK3sInstallVersion = "v1.34.1+k3s1"; const nativeK3sImage = "rancher/k3s:v1.34.1-k3s1"; const nativeK3sCtrAddress = "/run/k3s/containerd/containerd.sock"; function isHelpArg(value: string | undefined): boolean { return value === "help" || value === "--help" || value === "-h"; } function deployHelp(action: string | undefined = undefined): Record { const command = action === undefined || isHelpArg(action) ? "deploy check|plan|apply" : `deploy ${action}`; return { ok: true, command, usage: { check: "bun scripts/cli.ts deploy check [--file deploy.json] [--service id]", plan: "bun scripts/cli.ts deploy plan [--file deploy.json] [--service id]", apply: "bun scripts/cli.ts deploy apply [--file deploy.json] [--service id] [--dry-run] [--force] [--timeout-ms N] [--run-now]", }, actions: { check: "Validate desired repo+commit state against live service health and commit markers.", plan: "Show desired/live drift without requiring live health to be healthy.", apply: "Start an async target-side reconcile job unless --run-now is explicitly present.", }, options: [ { name: "--file ", default: defaultDeployFile, description: "Desired-state manifest path relative to the repo root." }, { name: "--service ", description: "Limit reconcile to one service from the manifest." }, { name: "--dry-run", description: "Prepare and validate without mutating the target service." }, { name: "--force", description: "Redeploy even when the live commit appears up to date." }, { name: "--timeout-ms ", default: defaultTimeoutMs, description: "Per-step timeout budget where supported." }, { name: "--run-now", description: "Run apply in the foreground worker process; omit it for fire-and-forget async job mode." }, ], }; } function nowIso(): string { return new Date().toISOString(); } function elapsedMs(startedAt: number): number { return Math.max(0, Date.now() - startedAt); } function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'\\''`)}'`; } function compactTail(text: string, maxChars = 1600): string { return text.length > maxChars ? text.slice(text.length - maxChars) : text; } function progressLine(step: string, message: string, detail?: unknown): void { const payload = detail === undefined ? { at: nowIso(), step, message } : { at: nowIso(), step, message, detail }; process.stderr.write(`${JSON.stringify(payload)}\n`); } function asRecord(value: unknown): Record | null { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; } function asString(value: unknown): string { return typeof value === "string" ? value : ""; } function parseFullCommit(value: string): string { const match = value.match(/\b[0-9a-f]{40}\b/iu); return match?.[0]?.toLowerCase() ?? ""; } function safeId(value: string): string { const sanitized = value.replace(/[^A-Za-z0-9_.-]/gu, "-"); if (sanitized.length === 0) throw new Error(`invalid service id: ${value}`); return sanitized; } function repoResolveCacheDir(repo: string): string { return rootPath(".state", "deploy", "resolve", createHash("sha256").update(repo).digest("hex").slice(0, 16)); } function repoSlug(repo: string): string | null { const trimmed = repo.trim().replace(/\.git$/u, ""); const https = trimmed.match(/^https:\/\/([^/]+)\/(.+)$/u); if (https !== null) return `${https[1]?.toLowerCase()}/${https[2]}`; const ssh = trimmed.match(/^git@([^:]+):(.+)$/u); if (ssh !== null) return `${ssh[1]?.toLowerCase()}/${ssh[2]}`; return null; } function sshUrlForSlug(slug: string | null): string | null { if (slug === null) return null; const [host = "", ...pathParts] = slug.split("/"); const path = pathParts.join("/"); if (host.length === 0 || path.length === 0) return null; if (host !== "github.com" && host !== "gitee.com") return null; return `git@${host}:${path}.git`; } function candidateRepoUrls(repo: string): string[] { const desiredSlug = repoSlug(repo); const urls = [repo]; const localOrigin = runCommand(["git", "remote", "get-url", "origin"], repoRoot); const localOriginUrl = localOrigin.exitCode === 0 ? localOrigin.stdout.trim() : ""; if (localOriginUrl.length > 0 && repoSlug(localOriginUrl) === desiredSlug) urls.push(localOriginUrl); const sshUrl = sshUrlForSlug(desiredSlug); if (sshUrl !== null) urls.push(sshUrl); return [...new Set(urls)]; } function providerSourceRepoUrl(repo: string): string { const slug = repoSlug(repo); if (slug === null || !slug.startsWith("github.com/")) return repo; return sshUrlForSlug(slug) ?? repo; } function localResolvedCommitForDesired(desired: DeployManifestService): string | null { const desiredSlug = repoSlug(desired.repo); if (desiredSlug === null || desired.commitId.length !== 40) return null; const localOrigin = runCommand(["git", "remote", "get-url", "origin"], repoRoot); const localOriginUrl = localOrigin.exitCode === 0 ? localOrigin.stdout.trim() : ""; if (localOriginUrl.length === 0 || repoSlug(localOriginUrl) !== desiredSlug) return null; const resolved = runCommand(["git", "rev-parse", "--verify", `${desired.commitId}^{commit}`], repoRoot); const fullCommit = parseFullCommit(resolved.stdout); return resolved.exitCode === 0 && fullCommit === desired.commitId.toLowerCase() ? fullCommit : null; } function commandFailure(result: ReturnType, maxChars = 1200): string { const detail = [result.stderr, result.stdout].filter(Boolean).join("\n"); return compactTail(detail, maxChars) || `exit ${result.exitCode}`; } function runGitOrThrow(args: string[], cwd: string, message: string): ReturnType { const result = runCommand(["git", ...args], cwd); if (result.exitCode !== 0) throw new Error(`${message}: ${commandFailure(result)}`); return result; } function runResolveGit(args: string[], cwd: string): ReturnType { return runCommand(["timeout", resolveFetchTimeout, "git", ...args], cwd); } function resolveDesiredCommit(desired: DeployManifestService): DeployManifestService { const localCommit = localResolvedCommitForDesired(desired); if (localCommit !== null) return { ...desired, commitId: localCommit }; const cacheDir = repoResolveCacheDir(desired.repo); mkdirSync(cacheDir, { recursive: true }); if (!existsSync(join(cacheDir, ".git", "config"))) { const init = runCommand(["git", "init", cacheDir], repoRoot); if (init.exitCode !== 0 && !existsSync(join(cacheDir, ".git", "config"))) { throw new Error(`failed to initialize deploy commit resolver for ${desired.repo}: ${commandFailure(init)}`); } } const fetchErrors: string[] = []; for (const repoUrl of candidateRepoUrls(desired.repo)) { runCommand(["git", "-C", cacheDir, "remote", "remove", "origin"], repoRoot); const addRemote = runCommand(["git", "-C", cacheDir, "remote", "add", "origin", repoUrl], repoRoot); if (addRemote.exitCode !== 0) { fetchErrors.push(`${repoUrl}: ${commandFailure(addRemote)}`); continue; } const fetches = desired.commitId.length === 40 ? [ runResolveGit(["-C", cacheDir, "fetch", "--no-tags", "--depth=1", "origin", desired.commitId], repoRoot), runResolveGit([ "-C", cacheDir, "fetch", "--no-tags", "--prune", "origin", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*", ], repoRoot), ] : [ runResolveGit([ "-C", cacheDir, "fetch", "--no-tags", "--prune", "origin", "+refs/heads/*:refs/remotes/origin/*", "+refs/tags/*:refs/tags/*", ], repoRoot), ]; const fetch = fetches.find((result) => result.exitCode === 0) ?? fetches[fetches.length - 1]; if (fetch.exitCode === 0) { fetchErrors.length = 0; break; } fetchErrors.push(`${repoUrl}: ${commandFailure(fetch)}`); } if (fetchErrors.length > 0) { throw new Error(`deploy manifest service ${desired.id} cannot fetch ${desired.repo} before deploy: ${compactTail(fetchErrors.join("\n"), 1600)}`); } const resolved = runCommand(["git", "-C", cacheDir, "rev-parse", "--verify", `${desired.commitId}^{commit}`], repoRoot); const fullCommit = parseFullCommit(resolved.stdout); if (resolved.exitCode !== 0 || fullCommit.length !== 40) { throw new Error(`deploy manifest service ${desired.id} commitId ${desired.commitId} cannot be resolved in ${desired.repo}; use a pushed unique 7-40 char SHA. ${commandFailure(resolved)}`); } return { ...desired, commitId: fullCommit }; } function resolveManifestCommits(manifest: DeployManifest, serviceId: string | null): DeployManifest { return { schemaVersion: manifest.schemaVersion, services: manifest.services.map((service) => (serviceId === null || service.id === serviceId ? resolveDesiredCommit(service) : service)), }; } function optionValue(args: string[], names: string[]): string | undefined { for (const name of names) { const index = args.indexOf(name); if (index === -1) continue; const raw = args[index + 1]; if (raw === undefined || raw.length === 0 || raw.startsWith("--")) throw new Error(`${name} requires a non-empty value`); return raw; } return undefined; } function positiveIntegerOption(args: string[], names: string[], defaultValue: number): number { const raw = optionValue(args, names); if (raw === undefined) return defaultValue; const parsed = Number(raw); if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${names[0]} must be a positive integer`); return parsed; } function parseOptions(args: string[]): DeployOptions { return { file: optionValue(args, ["--file"]) ?? defaultDeployFile, serviceId: optionValue(args, ["--service", "--service-id"]) ?? null, runNow: args.includes("--run-now"), dryRun: args.includes("--dry-run"), force: args.includes("--force"), timeoutMs: positiveIntegerOption(args, ["--timeout-ms"], defaultTimeoutMs), }; } function positionalArgs(args: string[]): string[] { const result: string[] = []; for (let index = 0; index < args.length; index += 1) { const value = args[index] ?? ""; if (value.startsWith("--")) { if (!["--run-now", "--force"].includes(value)) index += 1; continue; } result.push(value); } return result; } function readDeployManifest(file: string): DeployManifest { const path = resolve(repoRoot, file); if (!existsSync(path)) throw new Error(`deploy manifest not found: ${path}`); const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown; const record = asRecord(parsed); if (record === null) throw new Error("deploy manifest must be an object"); if (record.schemaVersion !== 1) throw new Error("deploy manifest schemaVersion must be 1"); if (!Array.isArray(record.services)) throw new Error("deploy manifest services must be an array"); const seen = new Set(); const services = record.services.map((item, index) => { const service = asRecord(item); if (service === null) throw new Error(`deploy manifest services[${index}] must be an object`); const id = asString(service.id); const repo = asString(service.repo); const commitId = asString(service.commitId).toLowerCase(); if (id.length === 0) throw new Error(`deploy manifest services[${index}].id must be a non-empty string`); if (repo.length === 0) throw new Error(`deploy manifest services[${index}].repo must be a non-empty string`); if (!/^[0-9a-f]{7,40}$/iu.test(commitId)) throw new Error(`deploy manifest services[${index}].commitId must be a 7-40 char git SHA`); if (seen.has(id)) throw new Error(`duplicate deploy manifest service id: ${id}`); seen.add(id); return { id, repo, commitId }; }); return { schemaVersion: 1, services }; } function selectServices(config: UniDeskConfig, manifest: DeployManifest, serviceId: string | null): Array<{ desired: DeployManifestService; config: UniDeskMicroserviceConfig; }> { const configById = new Map(config.microservices.map((service) => [service.id, service])); const selected = serviceId === null ? manifest.services : manifest.services.filter((service) => service.id === serviceId); if (serviceId !== null && selected.length === 0) throw new Error(`deploy manifest does not contain service: ${serviceId}`); return selected.map((desired) => { const service = configById.get(desired.id); if (service === undefined) throw new Error(`deploy manifest service ${desired.id} is not present in config.json microservices`); return { desired, config: service }; }); } function unsupportedReason(service: UniDeskMicroserviceConfig): string | null { if (service.repository.dockerfile.startsWith("docker.io/")) return "image-only service has no Dockerfile source artifact"; if (service.repository.composeFile.startsWith("docker run")) return "docker-run image-only service has no compose/k8s build path"; if (service.repository.commitId === "local") return null; return null; } function targetIsMain(service: UniDeskMicroserviceConfig): boolean { return service.providerId === "main-server"; } function targetDeployRoot(service: UniDeskMicroserviceConfig): string { return targetIsMain(service) ? rootPath(".state", "deploy") : remoteDeployRoot; } function targetRepoDir(service: UniDeskMicroserviceConfig): string { return `${targetDeployRoot(service)}/repos/${safeId(service.id)}`; } function targetExportDir(service: UniDeskMicroserviceConfig, runId: string): string { return `${targetDeployRoot(service)}/exports/${safeId(service.id)}-${runId}`; } function targetWorkDir(service: UniDeskMicroserviceConfig): string { if (service.deployment.mode === "k3sctl-managed") return k3sDeployDir; if (targetIsMain(service) && service.repository.url === "https://github.com/pikasTech/unidesk") { return rootPath(".state", "deploy", "work", safeId(service.id)); } return service.development.worktreePath; } function buildImageTag(service: UniDeskMicroserviceConfig): string { if (service.deployment.mode === "k3sctl-managed") return `unidesk-${service.id}:d601`; if (targetIsMain(service)) { if (["project-manager", "baidu-netdisk", "oa-event-flow"].includes(service.repository.composeService)) return service.repository.composeService; return `unidesk-${service.repository.composeService}`; } return `unidesk-${service.id}:${service.providerId.toLowerCase()}`; } function directComposeFile(service: UniDeskMicroserviceConfig): string { return targetIsMain(service) ? rootPath("docker-compose.yml") : `${targetWorkDir(service)}/${service.repository.composeFile}`; } function directComposeEnvFile(service: UniDeskMicroserviceConfig): string { return targetIsMain(service) ? writeComposeEnvFallbackPath() : ""; } function directBuildContextOverride(service: UniDeskMicroserviceConfig): string { if (targetIsMain(service) && service.repository.url === "https://github.com/pikasTech/unidesk") return targetWorkDir(service); return ""; } function directDockerfileOverride(service: UniDeskMicroserviceConfig): string { if (targetIsMain(service) && service.repository.url === "https://github.com/pikasTech/unidesk") return service.repository.dockerfile; return ""; } function k8sManifestPath(service: UniDeskMicroserviceConfig): string { const composeFile = service.repository.composeFile; if (!composeFile.endsWith(".k3s.json")) throw new Error(`${service.id} k3s service composeFile must point to *.k3s.json`); return composeFile.replace(/\.k3s\.json$/u, ".k8s.yaml"); } function sourceProxyPrelude(service: UniDeskMicroserviceConfig): string { if (targetIsMain(service)) return ""; return [ `build_proxy=${shellQuote(providerGatewayWsEgressProxyUrl)}`, "export HTTP_PROXY=\"$build_proxy\" HTTPS_PROXY=\"$build_proxy\" ALL_PROXY=\"$build_proxy\"", "export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal\"", "git_ssh_proxy=/tmp/unidesk-git-ssh-http-connect.py", "cat > \"$git_ssh_proxy\" <<'UNIDESK_GIT_SSH_PROXY'", String.raw`#!/usr/bin/env python3 import os import select import socket import sys from urllib.parse import urlparse if len(sys.argv) != 3: raise SystemExit("usage: unidesk-git-ssh-http-connect.py host port") target_host = sys.argv[1] target_port = int(sys.argv[2]) proxy = urlparse(os.environ.get("UNIDESK_GIT_SSH_HTTP_PROXY", "http://127.0.0.1:18789")) proxy_host = proxy.hostname or "127.0.0.1" proxy_port = proxy.port or 80 sock = socket.create_connection((proxy_host, proxy_port), timeout=20) sock.sendall(f"CONNECT {target_host}:{target_port} HTTP/1.1\r\nHost: {target_host}:{target_port}\r\n\r\n".encode("ascii")) header = b"" while b"\r\n\r\n" not in header: chunk = sock.recv(4096) if not chunk: raise SystemExit("proxy closed before CONNECT response") header += chunk head, rest = header.split(b"\r\n\r\n", 1) if not (head.startswith(b"HTTP/1.1 200") or head.startswith(b"HTTP/1.0 200")): sys.stderr.write(head.decode("latin1", "replace") + "\n") raise SystemExit(1) if rest: os.write(1, rest) stdin_open = True sock.setblocking(False) while True: readers = [sock] if stdin_open: readers.append(sys.stdin.buffer) ready, _, _ = select.select(readers, [], []) if sock in ready: try: data = sock.recv(65536) except BlockingIOError: data = b"" if not data: break os.write(1, data) if stdin_open and sys.stdin.buffer in ready: data = os.read(0, 65536) if data: sock.sendall(data) else: stdin_open = False try: sock.shutdown(socket.SHUT_WR) except OSError: pass `, "UNIDESK_GIT_SSH_PROXY", "chmod 700 \"$git_ssh_proxy\"", "export UNIDESK_GIT_SSH_HTTP_PROXY=\"$build_proxy\"", "export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=$git_ssh_proxy %h %p'\"", "curl -fsSI --max-time 20 -x \"$build_proxy\" https://github.com >/dev/null", "echo target_source_proxy=provider-gateway-ws-egress:$build_proxy", "echo target_build_proxy=provider-gateway-ws-egress:$build_proxy", "echo target_build_proxy_probe=ok", ].join("\n"); } function buildCachePrelude(dockerfileVariable: string): string[] { return [ "cache_args=(--cache-to type=inline --build-arg BUILDKIT_INLINE_CACHE=1)", "if docker image inspect \"$image\" >/dev/null 2>&1; then cache_args+=(--cache-from \"$image\"); echo target_build_cache_from_image=$image; else echo target_build_cache_from_image=missing:$image; fi", "echo target_build_cache_to=inline", "base_args=()", "build_base_image=\"${image}-build-base\"", `if grep -Eq '^ARG[[:space:]]+CODE_QUEUE_BASE_IMAGE([=[:space:]]|$)' "${dockerfileVariable}" 2>/dev/null; then if docker image inspect "$build_base_image" >/dev/null 2>&1; then base_args=(--build-arg "CODE_QUEUE_BASE_IMAGE=$build_base_image"); echo target_build_base_image=$build_base_image; else echo target_build_base_image=default; fi; else echo target_build_base_image=unsupported; fi`, ]; } function prepareSourceScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, exportDir: string): string { if (targetIsMain(service) && desired.repo === "https://github.com/pikasTech/unidesk") { return [ "set -euo pipefail", `repo=${shellQuote(repoRoot)}`, `commit=${shellQuote(desired.commitId)}`, `export_dir=${shellQuote(exportDir)}`, "mkdir -p \"$(dirname \"$export_dir\")\"", "git -C \"$repo\" fetch --no-tags origin \"$commit\" || git -C \"$repo\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git -C \"$repo\" rev-parse --verify \"$commit^{commit}\")", "rm -rf \"$export_dir\"", "mkdir -p \"$export_dir\"", "git -C \"$repo\" archive --format=tar \"$resolved\" | tar -xf - -C \"$export_dir\"", "printf 'resolved_commit=%s\\nexport_dir=%s\\nsource_repo=%s\\n' \"$resolved\" \"$export_dir\" \"$repo\"", ].join("\n"); } return [ "set -euo pipefail", sourceProxyPrelude(service), `repo=${shellQuote(targetRepoDir(service))}`, `repo_url=${shellQuote(providerSourceRepoUrl(desired.repo))}`, `commit=${shellQuote(desired.commitId)}`, `export_dir=${shellQuote(exportDir)}`, "mkdir -p \"$(dirname \"$repo\")\" \"$(dirname \"$export_dir\")\"", "if [ ! -d \"$repo/.git\" ]; then rm -rf \"$repo\"; git clone --no-checkout \"$repo_url\" \"$repo\"; fi", "cd \"$repo\"", "git remote set-url origin \"$repo_url\"", "git fetch --no-tags origin \"$commit\" || git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git rev-parse --verify \"$commit^{commit}\")", "rm -rf \"$export_dir\"", "mkdir -p \"$export_dir\"", "git archive --format=tar \"$resolved\" | tar -xf - -C \"$export_dir\"", "printf 'resolved_commit=%s\\nexport_dir=%s\\n' \"$resolved\" \"$export_dir\"", ].filter((line) => line.length > 0).join("\n"); } function syncSourceScript(service: UniDeskMicroserviceConfig, exportDir: string): string { const workDir = targetWorkDir(service); return [ "set -euo pipefail", `export_dir=${shellQuote(exportDir)}`, `work_dir=${shellQuote(workDir)}`, "mkdir -p \"$work_dir\"", [ "rsync -a --delete", "--exclude '.git/'", "--exclude '.state/'", "--exclude 'logs/'", "--exclude '**/node_modules/'", "--exclude '**/dist/'", "\"$export_dir/\"", "\"$work_dir/\"", ].join(" "), `test -f "$work_dir/${service.repository.dockerfile}"`, "printf 'synced deploy worktree to %s\\n' \"$work_dir\"", ].join("\n"); } function buildImageScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string): string { const image = buildImageTag(service); const workDir = targetWorkDir(service); const dockerfile = `${workDir}/${service.repository.dockerfile}`; const labelArgs = [ "--label", `unidesk.ai/service-id=${service.id}`, "--label", `unidesk.ai/source-repo=${desired.repo}`, "--label", `unidesk.ai/source-commit=${resolvedCommit}`, "--label", `unidesk.ai/dockerfile=${service.repository.dockerfile}`, ]; const commonArgs = [ "--progress=plain", ...labelArgs, "-t", image, "-f", dockerfile, workDir, ]; const proxyBuildArgs = targetIsMain(service) ? [] : [ "--network", "host", "--build-arg", `HTTP_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `HTTPS_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `ALL_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", "NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal", ]; return [ "set -euo pipefail", sourceProxyPrelude(service), `image=${shellQuote(image)}`, `dockerfile=${shellQuote(dockerfile)}`, "docker buildx version >/dev/null", "builder_args=()", "if docker buildx inspect --builder default >/dev/null 2>&1; then builder_args=(--builder default); echo target_build_builder=default; else echo target_build_builder=implicit; fi", "docker buildx inspect \"${builder_args[@]}\" --bootstrap || true", "echo target_build_builder_cleanup=not-required", ...buildCachePrelude("$dockerfile"), `docker buildx build "\${builder_args[@]}" --load "\${cache_args[@]}" "\${base_args[@]}" ${[...proxyBuildArgs, ...commonArgs].map(shellQuote).join(" ")}`, "docker image inspect \"$image\" --format 'image_id={{.Id}} labels={{json .Config.Labels}}'", ].filter((line) => line.length > 0).join("\n"); } function directComposeResolveScript(service: UniDeskMicroserviceConfig): string { const projectHint = targetIsMain(service) ? "unidesk" : ""; return [ `work_dir=${shellQuote(targetWorkDir(service))}`, `compose_file=${shellQuote(directComposeFile(service))}`, `compose_env_file=${shellQuote(directComposeEnvFile(service))}`, `compose_service=${shellQuote(service.repository.composeService)}`, `container=${shellQuote(service.repository.containerName)}`, `project_hint=${shellQuote(projectHint)}`, `build_context_override=${shellQuote(directBuildContextOverride(service))}`, `build_dockerfile_override=${shellQuote(directDockerfileOverride(service))}`, "compose_env_args=()", "if [ -n \"$compose_env_file\" ]; then compose_env_args=(--env-file \"$compose_env_file\"); fi", "running_project=$(docker inspect -f '{{ index .Config.Labels \"com.docker.compose.project\" }}' \"$container\" 2>/dev/null || true)", "running_service=$(docker inspect -f '{{ index .Config.Labels \"com.docker.compose.service\" }}' \"$container\" 2>/dev/null || true)", "running_image=$(docker inspect -f '{{.Config.Image}}' \"$container\" 2>/dev/null || true)", "if [ \"$running_project\" = '' ]; then running_project=''; fi", "if [ \"$running_service\" = '' ]; then running_service=''; fi", "project=\"$running_project\"", "if [ -z \"$project\" ]; then project=\"$project_hint\"; fi", "if [ -z \"$project\" ]; then project=$(basename \"$work_dir\"); fi", "config_json=$(docker compose \"${compose_env_args[@]}\" -f \"$compose_file\" -p \"$project\" config --format json)", "compose_image=$(printf '%s' \"$config_json\" | python3 -c 'import json,sys; svc=sys.argv[1]; d=json.load(sys.stdin); print(((d.get(\"services\") or {}).get(svc) or {}).get(\"image\") or \"\")' \"$compose_service\")", "image=\"$compose_image\"", "if [ -z \"$image\" ]; then image=\"$running_image\"; fi", "if [ -z \"$image\" ]; then image=\"${project}-${compose_service}\"; fi", "build_context=$(printf '%s' \"$config_json\" | python3 -c 'import json,sys; svc=sys.argv[1]; b=(((json.load(sys.stdin).get(\"services\") or {}).get(svc) or {}).get(\"build\") or {}); print((b if isinstance(b,str) else b.get(\"context\")) or \".\")' \"$compose_service\")", "build_dockerfile=$(printf '%s' \"$config_json\" | python3 -c 'import json,sys; svc=sys.argv[1]; b=(((json.load(sys.stdin).get(\"services\") or {}).get(svc) or {}).get(\"build\") or {}); print(\"Dockerfile\" if isinstance(b,str) else (b.get(\"dockerfile\") or \"Dockerfile\"))' \"$compose_service\")", "build_target=$(printf '%s' \"$config_json\" | python3 -c 'import json,sys; svc=sys.argv[1]; b=(((json.load(sys.stdin).get(\"services\") or {}).get(svc) or {}).get(\"build\") or {}); print(\"\" if isinstance(b,str) else (b.get(\"target\") or \"\"))' \"$compose_service\")", "build_args_file=$(mktemp /tmp/unidesk-compose-build-args.XXXXXX)", "printf '%s' \"$config_json\" | python3 -c 'import json,sys; svc=sys.argv[1]; b=(((json.load(sys.stdin).get(\"services\") or {}).get(svc) or {}).get(\"build\") or {}); args={} if isinstance(b,str) else (b.get(\"args\") or {}); items=args.items() if isinstance(args,dict) else [(str(x).split(\"=\",1)[0], str(x).split(\"=\",1)[1] if \"=\" in str(x) else \"\") for x in args]; [print(str(k)+\"=\"+(\"\" if v is None else str(v))) for k,v in items]' \"$compose_service\" > \"$build_args_file\"", "if [ -n \"$build_context_override\" ]; then build_context=\"$build_context_override\"; fi", "if [ -n \"$build_dockerfile_override\" ]; then build_dockerfile=\"$build_dockerfile_override\"; fi", "case \"$build_context\" in /*) ;; *) build_context=$(cd \"$(dirname \"$compose_file\")\" && cd \"$build_context\" && pwd) ;; esac", "case \"$build_dockerfile\" in /*) dockerfile_abs=\"$build_dockerfile\" ;; *) dockerfile_abs=\"$build_context/$build_dockerfile\" ;; esac", "test -f \"$dockerfile_abs\"", "printf 'compose_project=%s\\ncompose_service=%s\\ncompose_image=%s\\nbuild_context=%s\\nbuild_dockerfile=%s\\nbuild_target=%s\\n' \"$project\" \"$compose_service\" \"$image\" \"$build_context\" \"$dockerfile_abs\" \"$build_target\"", ].join("\n"); } function buildDirectImageScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string): string { const proxyBuildArgs = targetIsMain(service) ? [] : [ "--network", "host", "--build-arg", `HTTP_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `HTTPS_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `ALL_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", "NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal", ]; const labelArgs = [ "--label", `unidesk.ai/service-id=${service.id}`, "--label", `unidesk.ai/source-repo=${desired.repo}`, "--label", `unidesk.ai/source-commit=${resolvedCommit}`, "--label", `unidesk.ai/dockerfile=${service.repository.dockerfile}`, ]; return [ "set -euo pipefail", sourceProxyPrelude(service), directComposeResolveScript(service), "builder_args=()", "if docker buildx inspect --builder default >/dev/null 2>&1; then builder_args=(--builder default); echo target_build_builder=default; else echo target_build_builder=implicit; fi", "docker buildx inspect \"${builder_args[@]}\" --bootstrap || true", ...buildCachePrelude("$dockerfile_abs"), "compose_build_args=()", "while IFS= read -r item; do [ -n \"$item\" ] && compose_build_args+=(--build-arg \"$item\"); done < \"$build_args_file\"", "target_args=()", "if [ -n \"$build_target\" ]; then target_args=(--target \"$build_target\"); fi", `docker buildx build "\${builder_args[@]}" --load "\${cache_args[@]}" "\${base_args[@]}" ${[...proxyBuildArgs, ...labelArgs].map(shellQuote).join(" ")} "\${compose_build_args[@]}" "\${target_args[@]}" --progress=plain -t "$image" -f "$dockerfile_abs" "$build_context"`, "docker image inspect \"$image\" --format 'image_id={{.Id}} labels={{json .Config.Labels}}'", ].filter((line) => line.length > 0).join("\n"); } function imageLabelVerifyScript(service: UniDeskMicroserviceConfig, expectedCommit: string): string { return [ "set -euo pipefail", `container=${shellQuote(service.repository.containerName)}`, `expected=${shellQuote(expectedCommit)}`, "cid=$(docker ps -q -f name=\"^/${container}$\" | head -1)", "test -n \"$cid\"", "image_id=$(docker inspect -f '{{.Image}}' \"$cid\")", "actual=$(docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\")", "test \"$actual\" = \"$expected\"", "printf 'container=%s image_id=%s deploy_commit=%s\\n' \"$container\" \"$image_id\" \"$actual\"", ].join("\n"); } function composeDeployScript(service: UniDeskMicroserviceConfig): string { return [ "set -euo pipefail", directComposeResolveScript(service), "docker compose \"${compose_env_args[@]}\" -f \"$compose_file\" -p \"$project\" up -d --no-build --no-deps --force-recreate \"$compose_service\"", "ready=0", "for attempt in $(seq 1 90); do", " cid=$(docker ps -q -f name=\"^/${container}$\" | head -1)", " if [ -n \"$cid\" ]; then", " health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' \"$cid\" 2>/dev/null || true)", " echo compose_container_probe container=$container attempt=$attempt health=$health", " if [ \"$health\" = \"healthy\" ] || [ \"$health\" = \"running\" ]; then ready=1; break; fi", " else", " echo compose_container_probe container=$container attempt=$attempt cid=missing", " fi", " sleep 1", "done", "test \"$ready\" = \"1\"", ].join("\n"); } function writeComposeEnvFallbackPath(): string { return rootPath(".state", "docker-compose.env"); } function rootAccessPrelude(): string[] { return [ "root_exec() {", " if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return; fi", " if sudo -n true >/dev/null 2>&1; then sudo -n \"$@\"; return; fi", " if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return; fi", " echo 'native_k3s_root_access=missing' >&2", " return 1", "}", "root_shell() {", " script=\"$1\"", " if [ \"$(id -u)\" = \"0\" ]; then bash -lc \"$script\"; return; fi", " if sudo -n true >/dev/null 2>&1; then sudo -n bash -lc \"$script\"; return; fi", " if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- bash -lc \"$script\"; return; fi", " echo 'native_k3s_root_access=missing' >&2", " return 1", "}", ]; } function ensureNativeK3sScript(): string { const installCommand = [ "INSTALL_K3S_SKIP_DOWNLOAD=true", `INSTALL_K3S_VERSION=${nativeK3sInstallVersion}`, "INSTALL_K3S_EXEC=\"server --disable traefik --disable servicelb --disable metrics-server --node-name D601 --node-label unidesk.ai/node-id=D601 --node-label unidesk.ai/provider-id=D601 --tls-san 127.0.0.1 --tls-san host.docker.internal --write-kubeconfig-mode 644\"", "sh /tmp/unidesk-install-k3s.sh", ].join(" "); return [ "set -euo pipefail", ...rootAccessPrelude(), `native_k3s_image=${shellQuote(nativeK3sImage)}`, `native_ctr_address=${shellQuote(nativeK3sCtrAddress)}`, "install_native_k3s_binaries() {", " missing=0", " for binary in k3s ctr containerd containerd-shim-runc-v2 crictl runc cni flannel bridge host-local loopback portmap bandwidth firewall; do", " command -v \"$binary\" >/dev/null 2>&1 || missing=1", " done", " if [ \"$missing\" = \"0\" ]; then return; fi", " docker image inspect \"$native_k3s_image\" >/dev/null 2>&1 || docker pull \"$native_k3s_image\"", " tmp_dir=$(mktemp -d)", " tmp_container=$(docker create \"$native_k3s_image\" sh -lc true)", " docker cp \"$tmp_container:/bin/.\" \"$tmp_dir/bin\"", " docker rm \"$tmp_container\" >/dev/null", " for binary in k3s ctr containerd containerd-shim-runc-v2 crictl runc cni flannel bridge host-local loopback portmap bandwidth firewall; do", " if [ -e \"$tmp_dir/bin/$binary\" ]; then", " root_exec install -m 755 \"$tmp_dir/bin/$binary\" \"/usr/local/bin/$binary\"", " echo native_k3s_binary_installed=$binary", " fi", " done", " rm -rf \"$tmp_dir\"", "}", "unmount_invalid_wsl_kubelet_mounts() {", " if awk 'NF != 6 && $0 ~ /\\/Docker\\/host/ {found=1} END {exit found ? 0 : 1}' /proc/mounts; then", " root_exec umount /Docker/host >/dev/null 2>&1 || root_exec umount -l /Docker/host >/dev/null 2>&1 || true", " echo native_k3s_unmounted_invalid_mount=/Docker/host", " fi", " awk 'NF != 6 {print \"native_k3s_invalid_mount_line=\" NR \":\" $0; bad=1} END {exit bad}' /proc/mounts", "}", "install_system_images_from_legacy_k3s() {", " legacy_container=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1; exit}')", " if [ -n \"$legacy_container\" ]; then", " if docker exec \"$legacy_container\" ctr -n k8s.io images export /tmp/unidesk-k3s-system-images.tar docker.io/rancher/local-path-provisioner:v0.0.32 docker.io/rancher/mirrored-coredns-coredns:1.12.3 >/dev/null 2>&1; then", " docker cp \"$legacy_container:/tmp/unidesk-k3s-system-images.tar\" /tmp/unidesk-k3s-system-images.tar >/dev/null", " root_exec ctr --address \"$native_ctr_address\" -n k8s.io images import /tmp/unidesk-k3s-system-images.tar >/dev/null", " echo native_k3s_imported_legacy_system_images=$legacy_container", " fi", " fi", "}", "install_pause_image() {", " if root_exec ctr --address \"$native_ctr_address\" -n k8s.io images ls | grep -q 'docker.io/rancher/mirrored-pause:3.6'; then return; fi", " docker image inspect rancher/mirrored-pause:3.6 >/dev/null 2>&1 || {", " docker image inspect \"$native_k3s_image\" >/dev/null 2>&1 || docker pull \"$native_k3s_image\"", " tmp_dir=$(mktemp -d)", " printf '%s\\n' \"FROM $native_k3s_image\" 'ENTRYPOINT [\"/bin/sleep\", \"365d\"]' > \"$tmp_dir/Dockerfile\"", " docker build -q -t rancher/mirrored-pause:3.6 \"$tmp_dir\" >/dev/null", " rm -rf \"$tmp_dir\"", " }", " docker save rancher/mirrored-pause:3.6 -o /tmp/unidesk-k3s-pause.tar", " root_exec ctr --address \"$native_ctr_address\" -n k8s.io images import /tmp/unidesk-k3s-pause.tar >/dev/null", " echo native_k3s_imported_pause_image=rancher/mirrored-pause:3.6", "}", "install_native_k3s_binaries", "unmount_invalid_wsl_kubelet_mounts", "if ! systemctl cat k3s.service >/dev/null 2>&1; then", " curl -fsSL --max-time 60 https://get.k3s.io -o /tmp/unidesk-install-k3s.sh", ` root_shell ${shellQuote(installCommand)}`, "fi", "if ! systemctl is-active --quiet k3s; then", " root_exec systemctl enable --now k3s", "fi", "for attempt in $(seq 1 60); do", ` if KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes >/dev/null 2>&1; then break; fi`, " sleep 2", "done", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -l unidesk.ai/node-id=D601 --no-headers | grep -q .`, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl wait --for=condition=Ready node -l unidesk.ai/node-id=D601 --timeout=180s`, "install_system_images_from_legacy_k3s", "install_pause_image", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n kube-system delete pod -l k8s-app=kube-dns --ignore-not-found >/dev/null 2>&1 || true`, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n kube-system delete pod -l app=local-path-provisioner --ignore-not-found >/dev/null 2>&1 || true`, "legacy_k3s_containers=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1}')", "for container in $legacy_k3s_containers; do", " docker update --restart=no \"$container\" >/dev/null 2>&1 || true", " docker stop \"$container\" >/dev/null", " echo stopped_containerized_k3s=$container", "done", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -o wide`, "printf 'native_k3s=ready kubeconfig=%s\\n' /etc/rancher/k3s/k3s.yaml", ].join("\n"); } function importK3sImageScript(service: UniDeskMicroserviceConfig): string { const image = buildImageTag(service); return [ "set -euo pipefail", ...rootAccessPrelude(), `image=${shellQuote(image)}`, "docker image inspect \"$image\" >/dev/null", `docker save "$image" | root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images import -`, `root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images ls | grep -F "$image" || true`, ].join("\n"); } function k8sDeploymentsForService(service: UniDeskMicroserviceConfig): string[] { if (service.id === "code-queue") return ["d601-provider-egress-proxy", "d601-tcp-egress-gateway", "code-queue"]; return [service.repository.composeService]; } function applyK8sScript(service: UniDeskMicroserviceConfig): string { const manifest = `${targetWorkDir(service)}/${k8sManifestPath(service)}`; const cleanup = service.id === "code-queue" ? `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} delete endpointslice d601-provider-egress-proxy --ignore-not-found` : ""; return [ cleanup, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl apply -f ${shellQuote(manifest)}`, ].filter(Boolean).join("\n"); } function stampK8sScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string): string { const deployments = k8sDeploymentsForService(service).map((name) => `deployment/${name}`); const envPairs = [ `UNIDESK_DEPLOY_SERVICE_ID=${service.id}`, `UNIDESK_DEPLOY_REPO=${desired.repo}`, `UNIDESK_DEPLOY_COMMIT=${resolvedCommit}`, `UNIDESK_DEPLOY_REQUESTED_COMMIT=${desired.commitId}`, ...(service.id === "code-queue" ? [ `CODE_QUEUE_DEPLOY_COMMIT=${resolvedCommit}`, `CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=${desired.commitId}`, ] : []), ]; const annotatePairs = [ `unidesk.ai/deploy-service-id=${service.id}`, `unidesk.ai/deploy-repo=${desired.repo}`, `unidesk.ai/deploy-commit=${resolvedCommit}`, `unidesk.ai/deploy-requested-commit=${desired.commitId}`, ]; return [ "set -euo pipefail", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} set env ${deployments.map(shellQuote).join(" ")} ${envPairs.map(shellQuote).join(" ")}`, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} annotate ${deployments.map(shellQuote).join(" ")} ${annotatePairs.map(shellQuote).join(" ")} --overwrite`, `actual=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} get deployment ${shellQuote(service.repository.composeService)} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}')`, `test "$actual" = ${shellQuote(resolvedCommit)}`, "printf 'k8s_deploy_commit=%s\\n' \"$actual\"", ].join("\n"); } function rolloutK8sScript(service: UniDeskMicroserviceConfig): string { const deployments = k8sDeploymentsForService(service); return [ "set -euo pipefail", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} rollout restart ${deployments.map((name) => shellQuote(`deployment/${name}`)).join(" ")}`, ...deployments.map((name) => `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} rollout status ${shellQuote(`deployment/${name}`)} --timeout=180s`), `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} get deploy ${deployments.map(shellQuote).join(" ")} -o wide`, ].join("\n"); } function k8sCommitProbeScript(service: UniDeskMicroserviceConfig): string { return [ "set -euo pipefail", `commit=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(k8sNamespace)} get deployment ${shellQuote(service.repository.composeService)} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}' 2>/dev/null || true)`, "printf '%s\\n' \"$commit\"", ].join("\n"); } function dockerCommitProbeScript(service: UniDeskMicroserviceConfig): string { return [ "set -euo pipefail", `container=${shellQuote(service.repository.containerName)}`, "cid=$(docker ps -q -f name=\"^/${container}$\" | head -1)", "if [ -z \"$cid\" ]; then exit 0; fi", "image_id=$(docker inspect -f '{{.Image}}' \"$cid\")", "docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\" 2>/dev/null || true", ].join("\n"); } function healthDeployCommit(body: Record | null): string | null { const deploy = asRecord(body?.deploy); const commit = asString(deploy?.commit).toLowerCase(); return commit.length > 0 ? commit : null; } function healthSummary(response: unknown): Record { const record = asRecord(response); const body = asRecord(record?.body); return { ok: record?.ok ?? false, status: record?.status ?? null, body: body === null ? null : { ok: body.ok ?? null, service: body.service ?? null, instanceId: body.instanceId ?? null, deploy: body.deploy ?? null, status: body.status ?? null, startedAt: body.startedAt ?? null, }, }; } function commitMatches(actual: string | null, desired: string): boolean { if (actual === null || actual.length === 0) return false; const normalized = actual.toLowerCase(); return normalized === desired.toLowerCase() || (desired.length < 40 && normalized.startsWith(desired.toLowerCase())); } function runtimeCommitVerified( service: UniDeskMicroserviceConfig, healthCommit: string | null, imageCommit: string | null, orchestratorCommit: string | null, desired: string, ): boolean { if (healthCommit !== null && healthCommit.length > 0 && !commitMatches(healthCommit, desired)) return false; if (service.deployment.mode === "k3sctl-managed") return commitMatches(orchestratorCommit, desired); return commitMatches(imageCommit, desired); } function coreBody(response: unknown): Record | null { const record = asRecord(response); return asRecord(record?.body); } async function dispatchSsh( config: UniDeskConfig, providerId: string, command: string, cwd: string | null, waitMs = shortDispatchWaitMs, remoteTimeoutMs = shortRemoteTimeoutMs, ): Promise { const dispatchResponse = coreInternalFetch("/api/dispatch", { method: "POST", body: { providerId, command: "host.ssh", payload: { source: "deploy-reconciler", mode: "exec", command, timeoutMs: remoteTimeoutMs, ...(cwd === null ? {} : { cwd }), }, }, }); const dispatchBody = coreBody(dispatchResponse); const taskId = asString(dispatchBody?.taskId); if (dispatchBody?.ok !== true || taskId.length === 0) { return { ok: false, taskId: taskId || null, status: null, stdout: "", stderr: asString(dispatchBody?.error) || "dispatch did not return a task id", exitCode: null, raw: dispatchResponse, }; } const deadline = Date.now() + waitMs; let latest: unknown = null; while (Date.now() < deadline) { latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`); const task = asRecord(coreBody(latest)?.task); const status = asString(task?.status); if (status === "succeeded" || status === "failed") { const result = asRecord(task?.result); const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null; const stdout = asString(result?.stdout); const stderr = asString(result?.stderr); return { ok: status === "succeeded" && (exitCode === null || exitCode === 0), taskId, status, stdout, stderr, exitCode, raw: task, }; } await Bun.sleep(500); } return { ok: false, taskId, status: "timeout", stdout: "", stderr: `host.ssh task ${taskId} did not finish within ${waitMs}ms`, exitCode: null, raw: latest, }; } async function runTargetCommand(config: UniDeskConfig, service: UniDeskMicroserviceConfig, command: string, cwd: string | null, waitMs = 60_000, remoteTimeoutMs = 45_000): Promise { if (targetIsMain(service)) { const result = runCommand(["bash", "-lc", command], cwd ?? repoRoot); return { ok: result.exitCode === 0, taskId: null, status: result.exitCode === 0 ? "succeeded" : "failed", stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, raw: result, }; } return await dispatchSsh(config, service.providerId, command, cwd, waitMs, remoteTimeoutMs); } function splitFixed(value: string, size: number): string[] { const chunks: string[] = []; for (let index = 0; index < value.length; index += size) chunks.push(value.slice(index, index + size)); return chunks; } async function uploadRemoteShellScript( config: UniDeskConfig, service: UniDeskMicroserviceConfig, script: string, cwd: string, name: string, ): Promise { const remotePath = `/tmp/unidesk-deploy-${safeId(service.id)}-${safeId(name)}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}.sh`; const b64Path = `${remotePath}.b64`; const raw: unknown[] = []; const init = await runTargetCommand(config, service, `umask 077; rm -f ${shellQuote(remotePath)} ${shellQuote(b64Path)}; : > ${shellQuote(b64Path)}`, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs); raw.push(init.raw); if (!init.ok) return { ok: false, path: remotePath, error: init.stderr || init.stdout || "failed to initialize remote script upload", raw }; const encoded = Buffer.from(script, "utf8").toString("base64"); for (const chunk of splitFixed(encoded, 2400)) { const append = await runTargetCommand(config, service, `printf %s ${shellQuote(chunk)} >> ${shellQuote(b64Path)}`, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs); raw.push(append.raw); if (!append.ok) return { ok: false, path: remotePath, error: append.stderr || append.stdout || "failed to append remote script chunk", raw }; } const finalize = await runTargetCommand(config, service, `base64 -d ${shellQuote(b64Path)} > ${shellQuote(remotePath)}; chmod 700 ${shellQuote(remotePath)}; rm -f ${shellQuote(b64Path)}; wc -c ${shellQuote(remotePath)}`, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs); raw.push(finalize.raw); if (!finalize.ok) return { ok: false, path: remotePath, error: finalize.stderr || finalize.stdout || "failed to finalize remote script upload", raw }; return { ok: true, path: remotePath, error: "", raw }; } async function launchRemoteBackground( config: UniDeskConfig, service: UniDeskMicroserviceConfig, shellScript: string, cwd: string, logFile: string, sentinelFile: string, ): Promise<{ ok: boolean; pid: string; raw: unknown; error: string }> { const wrapped = [ `bash -lc ${shellQuote(shellScript)}`, "code=$?", `printf '%s\\n' "$code" > ${shellQuote(sentinelFile)}`, "exit \"$code\"", ].join("; "); const command = [ `rm -f ${shellQuote(sentinelFile)} ${shellQuote(logFile)}`, `nohup bash -lc ${shellQuote(wrapped)} > ${shellQuote(logFile)} 2>&1 < /dev/null & echo $!`, ].join("; "); const result = await runTargetCommand(config, service, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs); const pid = result.stdout.trim().split("\n").pop()?.trim() ?? ""; if (!result.ok || !/^\d+$/u.test(pid)) { return { ok: false, pid: "", raw: result.raw, error: result.stderr || result.stdout || "failed to launch background command" }; } return { ok: true, pid, raw: result.raw, error: "" }; } async function pollRemoteBackground(config: UniDeskConfig, service: UniDeskMicroserviceConfig, cwd: string, logFile: string, sentinelFile: string): Promise { const command = [ `if [ -f ${shellQuote(sentinelFile)} ]; then printf 'SENTINEL:%s\\n' "$(cat ${shellQuote(sentinelFile)} 2>/dev/null || true)"; else echo RUNNING; fi`, `tail -n 100 ${shellQuote(logFile)} 2>/dev/null | tr -d '\\000' | LC_ALL=C sed 's/[^[:print:]\t]//g' || true`, ].join("; "); const result = await runTargetCommand(config, service, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs); const stdout = result.stdout.trimEnd(); const [head = "", ...rest] = stdout.split("\n"); if (head.startsWith("SENTINEL:")) { const rawExitCode = head.slice("SENTINEL:".length).trim(); const exitCode = /^\d+$/u.test(rawExitCode) ? Number(rawExitCode) : null; return { done: true, exitCode, logTail: rest.join("\n").trim(), raw: result.raw }; } return { done: false, exitCode: null, logTail: rest.join("\n").trim(), raw: result.raw }; } async function step( config: UniDeskConfig, service: UniDeskMicroserviceConfig, name: string, command: string, cwd: string | null, timeoutMs: number, background = false, ): Promise { const startedAt = nowIso(); const startedMs = Date.now(); progressLine(name, "start", { serviceId: service.id, providerId: service.providerId, cwd }); if (!background || targetIsMain(service)) { const result = await runTargetCommand(config, service, command, cwd, timeoutMs, timeoutMs); const detail = compactTail([result.stdout, result.stderr].filter(Boolean).join("\n"), 2000); const ok = result.ok; progressLine(name, ok ? "succeeded" : "failed", { serviceId: service.id, elapsedMs: elapsedMs(startedMs), detail }); return { step: name, ok, detail: ok ? detail || `completed in ${elapsedMs(startedMs)}ms` : detail || "command failed", startedAt, finishedAt: nowIso(), raw: result.raw }; } const runId = `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`; const logFile = `/tmp/unidesk-deploy-${safeId(service.id)}-${name}-${runId}.log`; const sentinelFile = `/tmp/unidesk-deploy-${safeId(service.id)}-${name}-${runId}.done`; let launchCommand = command; if (launchCommand.length > 1800) { const upload = await uploadRemoteShellScript(config, service, command, cwd ?? "/home/ubuntu", name); if (!upload.ok) return { step: name, ok: false, detail: upload.error, startedAt, finishedAt: nowIso(), raw: upload.raw }; launchCommand = `bash ${shellQuote(upload.path)}`; progressLine(name, "remote script uploaded", { serviceId: service.id, path: upload.path, bytes: command.length }); } const launch = await launchRemoteBackground(config, service, launchCommand, cwd ?? "/home/ubuntu", logFile, sentinelFile); if (!launch.ok) return { step: name, ok: false, detail: launch.error, startedAt, finishedAt: nowIso(), raw: launch.raw }; progressLine(name, "remote background started", { serviceId: service.id, pid: launch.pid, logFile }); const deadline = Date.now() + timeoutMs; let lastTail = ""; while (Date.now() < deadline) { await Bun.sleep(pollIntervalMs); const poll = await pollRemoteBackground(config, service, cwd ?? "/home/ubuntu", logFile, sentinelFile); const tail = compactTail(poll.logTail, 1800); if (tail.length > 0 && tail !== lastTail) { lastTail = tail; progressLine(name, "remote log tail", { serviceId: service.id, elapsedMs: elapsedMs(startedMs), tail }); } if (poll.done) { const ok = poll.exitCode === 0; const detail = ok ? `completed in ${elapsedMs(startedMs)}ms; log=${logFile}` : `failed with exit ${poll.exitCode}; log=${logFile}; tail=${compactTail(poll.logTail)}`; progressLine(name, ok ? "succeeded" : "failed", { serviceId: service.id, detail }); return { step: name, ok, detail, startedAt, finishedAt: nowIso(), raw: poll.raw }; } } return { step: name, ok: false, detail: `timed out after ${timeoutMs}ms`, startedAt, finishedAt: nowIso(), raw: null }; } async function readDockerImageCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise { const result = await runTargetCommand(config, service, dockerCommitProbeScript(service), targetIsMain(service) ? repoRoot : "/home/ubuntu", 30_000, 20_000); const commit = parseFullCommit(result.stdout); return commit.length > 0 ? commit : null; } async function readK8sCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise { if (service.deployment.mode !== "k3sctl-managed") return null; const result = await runTargetCommand(config, service, k8sCommitProbeScript(service), "/home/ubuntu", 30_000, 20_000); const commit = parseFullCommit(result.stdout); return commit.length > 0 ? commit : null; } async function readRuntimeState(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService): Promise { const reason = unsupportedReason(service); const health = coreInternalFetch(`/api/microservices/${encodeURIComponent(service.id)}/health`); const healthBody = coreBody(health); const healthCommit = healthDeployCommit(healthBody); const healthRecord = asRecord(health); const healthOk = healthRecord?.ok === true && healthBody?.ok !== false; const [imageCommit, orchestratorCommit] = await Promise.all([ readDockerImageCommit(config, service).catch(() => null), readK8sCommit(config, service).catch(() => null), ]); const currentCommit = healthCommit ?? orchestratorCommit ?? imageCommit; return { serviceId: service.id, ok: reason === null, supported: reason === null, reason, desiredRepo: desired.repo, desiredCommit: desired.commitId, providerId: service.providerId, deploymentMode: service.deployment.mode, currentCommit, healthCommit, imageCommit, orchestratorCommit, healthOk, upToDate: reason === null && healthOk && runtimeCommitVerified(service, healthCommit, imageCommit, orchestratorCommit, desired.commitId), raw: { health: healthSummary(health) }, }; } async function healthVerify(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string, timeoutMs: number): Promise { const startedAt = nowIso(); const startedMs = Date.now(); const deadline = Date.now() + timeoutMs; let latest: ServiceRuntimeState | null = null; while (Date.now() < deadline) { latest = await readRuntimeState(config, service, { ...desired, commitId: resolvedCommit }); const commitOk = runtimeCommitVerified(service, latest.healthCommit, latest.imageCommit, latest.orchestratorCommit, resolvedCommit); const ok = latest.healthOk && commitOk; progressLine("live-health", "probe", { serviceId: service.id, ok, healthOk: latest.healthOk, expectedCommit: resolvedCommit, healthCommit: latest.healthCommit, imageCommit: latest.imageCommit, orchestratorCommit: latest.orchestratorCommit }); if (ok) { return { step: "live-health", ok: true, detail: `service ${service.id} health passed with deployed commit ${resolvedCommit} in ${elapsedMs(startedMs)}ms`, startedAt, finishedAt: nowIso(), raw: latest, }; } await Bun.sleep(3_000); } return { step: "live-health", ok: false, detail: `service ${service.id} did not report expected commit ${resolvedCommit} within ${timeoutMs}ms`, startedAt, finishedAt: nowIso(), raw: latest, }; } function pushStep(steps: StepResult[], result: StepResult): boolean { steps.push(result); return result.ok; } async function ensureGithubSshIdentityStep(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise { const startedAt = nowIso(); const startedMs = Date.now(); progressLine("github-ssh-identity", "start", { serviceId: service.id, providerId: service.providerId }); try { const result = await ensureGithubSshIdentityForProvider(config, service.providerId); progressLine("github-ssh-identity", result.ok ? "succeeded" : "failed", { serviceId: service.id, providerId: service.providerId, elapsedMs: elapsedMs(startedMs), fingerprint: result.fingerprint, seededFromLocal: result.seededFromLocal, detail: result.ok ? result.detail : compactTail(result.detail, 1200), }); return { step: "github-ssh-identity", ok: result.ok, detail: result.detail, startedAt, finishedAt: nowIso(), raw: result.raw, }; } catch (error) { const detail = error instanceof Error ? error.message : String(error); progressLine("github-ssh-identity", "failed", { serviceId: service.id, providerId: service.providerId, elapsedMs: elapsedMs(startedMs), detail: compactTail(detail, 1200), }); return { step: "github-ssh-identity", ok: false, detail, startedAt, finishedAt: nowIso(), raw: null }; } } async function applyOneService(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService, options: DeployOptions): Promise> { const steps: StepResult[] = []; const startedAt = nowIso(); const reason = unsupportedReason(service); if (reason !== null) return { ok: false, serviceId: service.id, skipped: true, reason, steps }; const before = await readRuntimeState(config, service, desired); if (!options.force && before.upToDate) return { ok: true, serviceId: service.id, action: "noop", before, steps }; if (options.dryRun) return { ok: true, serviceId: service.id, action: "would-deploy", before, steps }; const runId = `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`; const exportDir = targetExportDir(service, runId); if (service.id === "code-queue" && !targetIsMain(service)) { const identity = await ensureGithubSshIdentityStep(config, service); if (!pushStep(steps, identity)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), before, steps }; } const prepare = await step(config, service, "prepare-source", prepareSourceScript(service, desired, exportDir), targetIsMain(service) ? repoRoot : "/home/ubuntu", Math.min(options.timeoutMs, 180_000), !targetIsMain(service)); if (!pushStep(steps, prepare)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), before, steps }; const resolvedCommit = parseFullCommit(prepare.detail) || parseFullCommit(JSON.stringify(prepare.raw)); if (resolvedCommit.length !== 40) { const failure = { step: "resolve-commit", ok: false, detail: "prepare-source did not expose a full 40-char commit", startedAt: nowIso(), finishedAt: nowIso(), raw: prepare }; steps.push(failure); return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), before, steps }; } const sync = await step(config, service, "sync-source", syncSourceScript(service, exportDir), targetIsMain(service) ? repoRoot : "/home/ubuntu", 90_000, false); if (!pushStep(steps, sync)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; const buildScript = service.deployment.mode === "unidesk-direct" ? buildDirectImageScript(service, desired, resolvedCommit) : buildImageScript(service, desired, resolvedCommit); const build = await step(config, service, "docker-build", buildScript, targetWorkDir(service), Math.min(options.timeoutMs, 540_000), !targetIsMain(service)); if (!pushStep(steps, build)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; if (service.deployment.mode === "k3sctl-managed") { const nativeK3s = await step(config, service, "ensure-native-k3s", ensureNativeK3sScript(), "/home/ubuntu", 600_000, true); if (!pushStep(steps, nativeK3s)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; const imageImport = await step(config, service, "import-k3s-image", importK3sImageScript(service), targetWorkDir(service), 180_000, true); if (!pushStep(steps, imageImport)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; const apply = await step(config, service, "kubectl-apply", applyK8sScript(service), targetWorkDir(service), 60_000, false); if (!pushStep(steps, apply)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; const stamp = await step(config, service, "stamp-deploy-commit", stampK8sScript(service, desired, resolvedCommit), targetWorkDir(service), 60_000, false); if (!pushStep(steps, stamp)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; const rollout = await step(config, service, "rollout", rolloutK8sScript(service), targetWorkDir(service), 240_000, true); if (!pushStep(steps, rollout)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; } else { const deploy = await step(config, service, "compose-up", composeDeployScript(service), targetIsMain(service) ? repoRoot : targetWorkDir(service), 180_000, !targetIsMain(service)); if (!pushStep(steps, deploy)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; const imageVerify = await step(config, service, "image-label-verify", imageLabelVerifyScript(service, resolvedCommit), targetIsMain(service) ? repoRoot : targetWorkDir(service), 60_000, false); if (!pushStep(steps, imageVerify)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps }; } const health = await healthVerify(config, service, desired, resolvedCommit, 90_000); steps.push(health); return { ok: health.ok, serviceId: service.id, action: "deployed", startedAt, finishedAt: nowIso(), resolvedCommit, before, after: health.raw, steps, }; } async function checkOrPlan(config: UniDeskConfig, manifest: DeployManifest, options: DeployOptions, action: "check" | "plan"): Promise> { const services = selectServices(config, manifest, options.serviceId); const items = []; for (const item of services) { const state = await readRuntimeState(config, item.config, item.desired); items.push({ ...state, action: action === "plan" ? state.supported ? state.upToDate ? "noop" : "deploy" : "unsupported" : undefined, }); } return { ok: items.every((item) => item.supported && (action === "plan" || item.healthOk)), action, file: options.file, services: items, }; } async function runApplyNow(config: UniDeskConfig, manifest: DeployManifest, options: DeployOptions): Promise> { const selected = selectServices(config, manifest, options.serviceId); const startedAt = nowIso(); const results = []; for (const item of selected) { progressLine("service", "reconcile", { serviceId: item.config.id, providerId: item.config.providerId, mode: item.config.deployment.mode }); results.push(await applyOneService(config, item.config, item.desired, options)); const latest = results.at(-1) as Record; if (latest.ok !== true) break; } return { ok: results.every((result) => result.ok === true), action: "apply", file: options.file, dryRun: options.dryRun, startedAt, finishedAt: nowIso(), results, }; } function applyJob(config: UniDeskConfig, args: string[], options: DeployOptions): Record { const runArgs = args.includes("--run-now") ? args : [...args, "--run-now"]; const command = [process.execPath, rootPath("scripts", "cli.ts"), "deploy", ...runArgs]; const job = startJob("deploy_apply", command, `Reconcile services from ${options.file}${options.serviceId === null ? "" : ` service=${options.serviceId}`}`); return { ok: true, mode: "async-job", job, statusCommand: `bun scripts/cli.ts job status ${job.id}`, tailCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`, note: "Deployment continues in the background: target-side fetch, one-shot proxied build, deploy, stamp, and live health verification.", configProject: config.project.name, }; } export async function runDeployCommand(config: UniDeskConfig, args: string[]): Promise { const [actionRaw = "check"] = args; if (isHelpArg(actionRaw) || args.slice(1).some(isHelpArg)) return deployHelp(isHelpArg(actionRaw) ? undefined : actionRaw); if (!["check", "plan", "apply"].includes(actionRaw)) throw new Error("deploy command must be one of: check, plan, apply"); const action = actionRaw as DeployAction; const options = parseOptions(args.slice(1)); const manifest = resolveManifestCommits(readDeployManifest(options.file), options.serviceId); if (action === "check" || action === "plan") return await checkOrPlan(config, manifest, options, action); if (!options.runNow) return applyJob(config, args, options); return await runApplyNow(config, manifest, options); } export async function runCodeQueueDeployCompatCommand(config: UniDeskConfig, args: string[]): Promise { if (args.includes("--skip-build")) throw new Error("codex deploy no longer supports --skip-build; target-side Docker build is mandatory"); const providerId = optionValue(args, ["--provider-id", "--provider"]) ?? "D601"; if (providerId !== "D601") throw new Error(`codex deploy compatibility path only supports D601; got ${providerId}`); const commitId = optionValue(args, ["--commit", "--commit-id"]) ?? positionalArgs(args)[0] ?? ""; if (!/^[0-9a-f]{7,40}$/iu.test(commitId)) throw new Error("codex deploy requires a 7-40 char commit id"); const service = config.microservices.find((item) => item.id === "code-queue"); if (service === undefined) throw new Error("config.json does not contain microservice id=code-queue"); const manifestRelDir = join(".state", "deploy", "manifests"); mkdirSync(rootPath(manifestRelDir), { recursive: true }); const manifestRelPath = join(manifestRelDir, `code-queue-${commitId.toLowerCase()}.json`); writeFileSync(rootPath(manifestRelPath), `${JSON.stringify({ schemaVersion: 1, services: [{ id: "code-queue", repo: service.repository.url, commitId: commitId.toLowerCase() }], }, null, 2)}\n`, "utf8"); const deployArgs = [ "apply", "--file", manifestRelPath, "--service", "code-queue", ...(args.includes("--run-now") ? ["--run-now"] : []), ...(args.includes("--force") ? ["--force"] : []), ...(optionValue(args, ["--timeout-ms"]) === undefined ? [] : ["--timeout-ms", optionValue(args, ["--timeout-ms"]) as string]), ]; return await runDeployCommand(config, deployArgs); }