// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote module for scripts/src/ci.ts. // Moved mechanically from scripts/src/ci.ts:888-1205 for #903. import { randomUUID } from "node:crypto"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, posix as posixPath } from "node:path"; import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "../ci-catalog"; import { runCommand } from "../command"; import { type UniDeskConfig, repoRoot, rootPath } from "../config"; import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "../deploy-ssh-identity"; import { jobWithTail, listJobs, readJob, startJob } from "../jobs"; import { coreInternalFetch } from "../microservices"; import { artifactRegistryReadonlyResultFromCommand, buildArtifactRegistryReadonlyProbe, parseArtifactRegistryOptions, type ArtifactRegistryReadonlyProbe, } from "../artifact-registry"; import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard"; import { runSshCommandCapture } from "../ssh"; import type { DispatchResult, PublishPreflightTransport } from "./types"; import { status } from "./cleanup"; import { boundedHintLine, ciTargetGuardShellLines, extractCiLogFailureHints, shellQuote, tailTextLines } from "./options"; import { asRecord, asString, chunks, coreBody, emitCiInstallProgress, hostSshBase64UploadChunkChars } from "./preflight"; import { artifactRegistryProbeCommand, dispatchReadonlySsh } from "./runner-preflight"; import { ciRuntimeImages, ciTarget, providerDispatchCompletionLagMs, providerGatewayWsEgressProxyUrl } from "./types"; export const localPublishPreflightTransport: PublishPreflightTransport = { kind: "local-docker", coreFetch: (path, init) => coreInternalFetch(path, init), dispatchHostSsh: dispatchReadonlySsh, commandCwd: repoRoot, artifactRegistryCommand: artifactRegistryProbeCommand, }; export function positiveManifestNumber(value: unknown, fallback: number, path: string): number { if (value === undefined || value === null) return fallback; if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`); return value; } export function requireManifestString(value: unknown, path: string): string { if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`); return value; } export function requireCiScriptPath(value: unknown): string { const scriptPath = requireManifestString(value, "environments.dev.ci.scriptPath"); if (!scriptPath.startsWith("scripts/ci/") || scriptPath.includes("..") || scriptPath.startsWith("/") || !scriptPath.endsWith(".sh")) { throw new Error("environments.dev.ci.scriptPath must be a repo-relative scripts/ci/*.sh path"); } return scriptPath; } export async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number, pollCompletion = true, target = ciTarget(null)): Promise { const dispatchResponse = coreInternalFetch("/api/dispatch", { method: "POST", timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000), body: { providerId: target.providerId, command: "host.ssh", payload: { source: "ci-cli", mode: "exec", command, timeoutMs: remoteTimeoutMs, cwd: target.hostCwd, }, }, }); const dispatchBody = coreBody(dispatchResponse); const taskId = asString(dispatchBody?.taskId); if (dispatchBody?.ok !== true || taskId.length === 0) { const failureKind = asRecord(dispatchResponse)?.timedOut === true ? "backend-core-dispatch-timeout" : "backend-core-dispatch-submit-failed"; return { ok: false, taskId: taskId || null, status: null, stdout: "", stderr: asString(dispatchBody?.error) || `${failureKind}: dispatch did not return a task id`, exitCode: null, raw: { failureKind, dispatchResponse, timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000), }, }; } if (!pollCompletion) { return { ok: true, taskId, status: "submitted", stdout: "", stderr: "", exitCode: null, raw: dispatchBody, }; } const effectiveWaitMs = Math.max(waitMs, Math.min(remoteTimeoutMs + providerDispatchCompletionLagMs, 120_000)); const deadline = Date.now() + Math.max(effectiveWaitMs, 1_000); let latest: unknown = null; while (Date.now() < deadline) { latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000, timeoutMs: 15_000 }); const task = asRecord(coreBody(latest)?.task); const status = asString(task?.status); if (status === "succeeded" || status === "failed") { const result = asRecord(task?.result); const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null; const stdout = asString(result?.stdout); const stderr = asString(result?.stderr); return { ok: status === "succeeded" && (exitCode === null || exitCode === 0), taskId, status, stdout, stderr, exitCode, raw: task, }; } await Bun.sleep(500); } return { ok: false, taskId, status: "timeout", stdout: "", stderr: `host.ssh task ${taskId} did not finish within ${Math.max(effectiveWaitMs, 1_000)}ms`, exitCode: null, raw: latest, }; } export async function runRemoteKubectl(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise { const result = await runRemoteKubectlRaw(script, waitMs, remoteTimeoutMs, target); if (!result.ok) { throw new Error(`${target.providerId} kubectl command failed: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); } return result; } export async function runRemoteKubectlRaw(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise { const command = [ "set -eu", ...ciTargetGuardShellLines(target, { passOutput: "stderr" }), script, ].join("\n"); return dispatchSsh(command, waitMs, remoteTimeoutMs, true, target); } export async function uploadRemoteBase64(path: string, encoded: string, target = ciTarget(null)): Promise { const init = await dispatchSsh([ "set -euo pipefail", `target=${shellQuote(path)}`, "rm -f \"$target\"", ": > \"$target\"", "chmod 600 \"$target\"", ].join("\n"), 20_000, 10_000, true, target); if (!init.ok) return init; // D601 provider-gateway rejects long host.ssh commands; keep each append // envelope comfortably below the transport limit. for (const chunk of chunks(encoded, hostSshBase64UploadChunkChars)) { const append = await dispatchSsh([ "set -euo pipefail", `target=${shellQuote(path)}`, `printf %s ${shellQuote(chunk)} >> "$target"`, ].join("\n"), 20_000, 10_000, true, target); if (!append.ok) return append; } return dispatchSsh([ "set -euo pipefail", `target=${shellQuote(path)}`, `expected=${shellQuote(String(Buffer.byteLength(encoded)))}`, "actual=$(wc -c < \"$target\" | tr -d ' ')", "printf 'uploaded_bytes=%s expected_bytes=%s path=%s\\n' \"$actual\" \"$expected\" \"$target\"", "test \"$actual\" = \"$expected\"", ].join("\n"), 20_000, 10_000, true, target); } export async function runRemoteBackground(label: string, script: string, timeoutMs: number, target = ciTarget(null)): Promise { const token = randomUUID().replace(/-/gu, "").slice(0, 12); const safeLabel = label.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 48); const base = `/tmp/unidesk-ci-${safeLabel}-${token}`; const scriptPath = `${base}.sh`; const logPath = `${base}.log`; const donePath = `${base}.done`; const encoded = Buffer.from(script, "utf8").toString("base64"); const upload = await uploadRemoteBase64(`${scriptPath}.b64`, encoded, target); if (!upload.ok) return upload; const start = await dispatchSsh([ "set -euo pipefail", `script_path=${shellQuote(scriptPath)}`, `log_path=${shellQuote(logPath)}`, `done_path=${shellQuote(donePath)}`, "rm -f \"$script_path\" \"$log_path\" \"$done_path\"", "base64 -d \"$script_path.b64\" > \"$script_path\"", "rm -f \"$script_path.b64\"", "chmod 700 \"$script_path\"", "nohup bash -lc \"bash '$script_path' >'$log_path' 2>&1; code=\\$?; printf '%s\\n' \\\"\\$code\\\" >'$done_path'\" >/tmp/unidesk-ci-nohup.log 2>&1 &", "printf 'remote_job_pid=%s\\nlog=%s\\ndone=%s\\n' \"$!\" \"$log_path\" \"$done_path\"", ].join("\n"), 20_000, 10_000, true, target); if (!start.ok) return start; const deadline = Date.now() + timeoutMs; let latest: DispatchResult = start; while (Date.now() < deadline) { await Bun.sleep(8_000); latest = await dispatchSsh([ "set -euo pipefail", `log_path=${shellQuote(logPath)}`, `done_path=${shellQuote(donePath)}`, "if [ -f \"$done_path\" ]; then", " code=\"$(cat \"$done_path\" 2>/dev/null || printf 1)\"", " printf 'REMOTE_DONE:%s\\n' \"$code\"", "else", " printf 'REMOTE_RUNNING\\n'", "fi", "tail -n 160 \"$log_path\" 2>/dev/null || true", ].join("\n"), 75_000, 12_000, true, target); if (!latest.ok) { if (latest.status === "timeout" || latest.stderr.includes("did not finish within")) { continue; } return latest; } const firstLine = latest.stdout.split(/\r?\n/u)[0] ?? ""; if (firstLine.startsWith("REMOTE_DONE:")) { const code = Number(firstLine.slice("REMOTE_DONE:".length).trim()); return { ...latest, ok: code === 0, exitCode: Number.isInteger(code) ? code : 1, status: code === 0 ? "succeeded" : "failed", }; } } return { ...latest, ok: false, status: "timeout", exitCode: 124, stderr: `remote background job ${label} did not finish within ${timeoutMs}ms`, }; } export async function remoteApplyManifest(config: UniDeskConfig, path: string, target = ciTarget(null)): Promise { const absolute = rootPath(path); if (!existsSync(absolute)) throw new Error(`manifest not found: ${path}`); const manifest = readFileSync(absolute, "utf8"); const encoded = Buffer.from(manifest, "utf8").toString("base64"); emitCiInstallProgress("upload-manifest", "started", { providerId: target.providerId, manifest: path, bytes: Buffer.byteLength(manifest) }); const script = [ "set -eu", ...ciTargetGuardShellLines(target), "tmp=$(mktemp /tmp/unidesk-ci-apply.XXXXXX.yaml)", "trap 'rm -f \"$tmp\"' EXIT", "base64 -d > \"$tmp\" <<'UNIDESK_CI_MANIFEST_B64'", encoded, "UNIDESK_CI_MANIFEST_B64", "test -s \"$tmp\"", "printf 'manifest_bytes=%s\\n' \"$(wc -c < \"$tmp\" | tr -d ' ')\"", "kubectl apply -f \"$tmp\"", ].join("\n"); emitCiInstallProgress("kubectl-apply", "started", { providerId: target.providerId, manifest: path }); const result = await runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], script); if (result.exitCode !== 0) throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`); emitCiInstallProgress("upload-manifest", "succeeded", { providerId: target.providerId, manifest: path, upload: result.stdout.split(/\r?\n/u).find((line) => line.startsWith("manifest_bytes=")) ?? "" }); emitCiInstallProgress("kubectl-apply", "succeeded", { providerId: target.providerId, manifest: path }); } export async function prewarmCiRuntimeImages(target = ciTarget(null)): Promise { const images = ciRuntimeImages.map(shellQuote).join(" "); const script = [ "set -euo pipefail", ...ciTargetGuardShellLines(target), "export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config", "mkdir -p \"$DOCKER_CONFIG\"", "printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"", `images=(${images})`, "for image in \"${images[@]}\"; do", " if ! docker image inspect \"$image\" >/dev/null 2>&1; then", " echo ci_runtime_image_pull=$image", ` HTTP_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} HTTPS_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} ALL_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal docker pull --platform linux/amd64 "$image"`, " else", " echo ci_runtime_image_cached=$image", " fi", "done", "pause_entrypoint=$(docker image inspect rancher/mirrored-pause:3.6 --format '{{json .Config.Entrypoint}}' 2>/dev/null || true)", "if ! printf '%s' \"$pause_entrypoint\" | grep -q '\"/pause\"'; then echo native_k3s_pause_image_invalid_entrypoint=$pause_entrypoint >&2; exit 1; fi", "root_exec() {", " if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return $?; fi", " if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then sudo \"$@\"; return $?; fi", " if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return $?; fi", " echo ci_runtime_image_containerd_root_required=true >&2", " \"$@\"", "}", "containerd_images=$(root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls 2>/tmp/unidesk-ci-containerd-images.err || true)", "containerd_ready=1", "for image in \"${images[@]}\"; do", " case \"$image\" in", " rancher/*|oven/*|alpine/*) ref=\"docker.io/$image\" ;;", " unidesk-*) ref=\"docker.io/library/$image\" ;;", " *) ref=\"$image\" ;;", " esac", " if ! printf '%s\\n' \"$containerd_images\" | grep -F \"$ref\" >/dev/null; then", " containerd_ready=0", " echo ci_runtime_image_containerd_missing=$ref", " fi", "done", "if [ \"$containerd_ready\" = \"1\" ]; then", " echo ci_runtime_images_containerd_cached=all", " exit 0", "fi", "rm -f /tmp/unidesk-ci-runtime-images.tar", "docker save \"${images[@]}\" -o /tmp/unidesk-ci-runtime-images.tar", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import --digests --all-platforms /tmp/unidesk-ci-runtime-images.tar >/tmp/unidesk-ci-runtime-images-import.log", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/rancher/mirrored-pause:3.6' >/dev/null", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/oven/bun:1-debian' >/dev/null", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/alpine/git:2.45.2' >/dev/null", `root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F ${shellQuote(`docker.io/library/${target.codeQueueImage}`)} >/dev/null`, ].join("\n"); const result = await runRemoteBackground("prewarm-runtime-images", script, 900_000, target); if (!result.ok) { const combined = `${result.stdout}\n${result.stderr}`.trim(); const failureLines = extractCiLogFailureHints(combined).concat( combined.split(/\r?\n/u).filter((line) => /ci_runtime_image_|containerd_|root_required|sudo/iu.test(line)).map(boundedHintLine).slice(-40), ); throw new Error(`CI runtime image prewarm failed: ${JSON.stringify({ providerId: target.providerId, exitCode: result.exitCode, failureLines: Array.from(new Set(failureLines)).slice(-50), stdoutTail: tailTextLines(result.stdout, 80), stderrTail: tailTextLines(result.stderr, 80), recovery: [ "If Tekton is already installed and only CI manifests need refreshing, rerun with: bun scripts/cli.ts ci install --skip-prewarm", "If runtime helper images are missing from containerd, restore passwordless/root containerd import on the provider and rerun without --skip-prewarm.", ], })}`); } }