Codex
345 lines
16 KiB
TypeScript
345 lines
16 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote module for scripts/src/ci.ts.
|
|
|
|
// Moved mechanically from scripts/src/ci.ts:888-1205 for #903.
|
|
|
|
import { randomUUID } from "node:crypto";
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, posix as posixPath } from "node:path";
|
|
import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "../ci-catalog";
|
|
import { runCommand } from "../command";
|
|
import { type UniDeskConfig, repoRoot, rootPath } from "../config";
|
|
import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "../deploy-ssh-identity";
|
|
import { jobWithTail, listJobs, readJob, startJob } from "../jobs";
|
|
import { coreInternalFetch } from "../microservices";
|
|
import {
|
|
artifactRegistryReadonlyResultFromCommand,
|
|
buildArtifactRegistryReadonlyProbe,
|
|
parseArtifactRegistryOptions,
|
|
type ArtifactRegistryReadonlyProbe,
|
|
} from "../artifact-registry";
|
|
import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard";
|
|
import { runSshCommandCapture } from "../ssh";
|
|
|
|
import type { DispatchResult, PublishPreflightTransport } from "./types";
|
|
import { status } from "./cleanup";
|
|
import { boundedHintLine, ciTargetGuardShellLines, extractCiLogFailureHints, shellQuote, tailTextLines } from "./options";
|
|
import { asRecord, asString, chunks, coreBody, emitCiInstallProgress, hostSshBase64UploadChunkChars } from "./preflight";
|
|
import { artifactRegistryProbeCommand, dispatchReadonlySsh } from "./runner-preflight";
|
|
import { ciRuntimeImages, ciTarget, providerDispatchCompletionLagMs, providerGatewayWsEgressProxyUrl } from "./types";
|
|
|
|
export const localPublishPreflightTransport: PublishPreflightTransport = {
|
|
kind: "local-docker",
|
|
coreFetch: (path, init) => coreInternalFetch(path, init),
|
|
dispatchHostSsh: dispatchReadonlySsh,
|
|
commandCwd: repoRoot,
|
|
artifactRegistryCommand: artifactRegistryProbeCommand,
|
|
};
|
|
|
|
export function positiveManifestNumber(value: unknown, fallback: number, path: string): number {
|
|
if (value === undefined || value === null) return fallback;
|
|
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
export function requireManifestString(value: unknown, path: string): string {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
export function requireCiScriptPath(value: unknown): string {
|
|
const scriptPath = requireManifestString(value, "environments.dev.ci.scriptPath");
|
|
if (!scriptPath.startsWith("scripts/ci/") || scriptPath.includes("..") || scriptPath.startsWith("/") || !scriptPath.endsWith(".sh")) {
|
|
throw new Error("environments.dev.ci.scriptPath must be a repo-relative scripts/ci/*.sh path");
|
|
}
|
|
return scriptPath;
|
|
}
|
|
|
|
export async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number, pollCompletion = true, target = ciTarget(null)): Promise<DispatchResult> {
|
|
const dispatchResponse = coreInternalFetch("/api/dispatch", {
|
|
method: "POST",
|
|
timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000),
|
|
body: {
|
|
providerId: target.providerId,
|
|
command: "host.ssh",
|
|
payload: {
|
|
source: "ci-cli",
|
|
mode: "exec",
|
|
command,
|
|
timeoutMs: remoteTimeoutMs,
|
|
cwd: target.hostCwd,
|
|
},
|
|
},
|
|
});
|
|
const dispatchBody = coreBody(dispatchResponse);
|
|
const taskId = asString(dispatchBody?.taskId);
|
|
if (dispatchBody?.ok !== true || taskId.length === 0) {
|
|
const failureKind = asRecord(dispatchResponse)?.timedOut === true ? "backend-core-dispatch-timeout" : "backend-core-dispatch-submit-failed";
|
|
return {
|
|
ok: false,
|
|
taskId: taskId || null,
|
|
status: null,
|
|
stdout: "",
|
|
stderr: asString(dispatchBody?.error) || `${failureKind}: dispatch did not return a task id`,
|
|
exitCode: null,
|
|
raw: {
|
|
failureKind,
|
|
dispatchResponse,
|
|
timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000),
|
|
},
|
|
};
|
|
}
|
|
if (!pollCompletion) {
|
|
return {
|
|
ok: true,
|
|
taskId,
|
|
status: "submitted",
|
|
stdout: "",
|
|
stderr: "",
|
|
exitCode: null,
|
|
raw: dispatchBody,
|
|
};
|
|
}
|
|
const effectiveWaitMs = Math.max(waitMs, Math.min(remoteTimeoutMs + providerDispatchCompletionLagMs, 120_000));
|
|
const deadline = Date.now() + Math.max(effectiveWaitMs, 1_000);
|
|
let latest: unknown = null;
|
|
while (Date.now() < deadline) {
|
|
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000, timeoutMs: 15_000 });
|
|
const task = asRecord(coreBody(latest)?.task);
|
|
const status = asString(task?.status);
|
|
if (status === "succeeded" || status === "failed") {
|
|
const result = asRecord(task?.result);
|
|
const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null;
|
|
const stdout = asString(result?.stdout);
|
|
const stderr = asString(result?.stderr);
|
|
return {
|
|
ok: status === "succeeded" && (exitCode === null || exitCode === 0),
|
|
taskId,
|
|
status,
|
|
stdout,
|
|
stderr,
|
|
exitCode,
|
|
raw: task,
|
|
};
|
|
}
|
|
await Bun.sleep(500);
|
|
}
|
|
return {
|
|
ok: false,
|
|
taskId,
|
|
status: "timeout",
|
|
stdout: "",
|
|
stderr: `host.ssh task ${taskId} did not finish within ${Math.max(effectiveWaitMs, 1_000)}ms`,
|
|
exitCode: null,
|
|
raw: latest,
|
|
};
|
|
}
|
|
|
|
export async function runRemoteKubectl(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise<DispatchResult> {
|
|
const result = await runRemoteKubectlRaw(script, waitMs, remoteTimeoutMs, target);
|
|
if (!result.ok) {
|
|
throw new Error(`${target.providerId} kubectl command failed: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
export async function runRemoteKubectlRaw(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise<DispatchResult> {
|
|
const command = [
|
|
"set -eu",
|
|
...ciTargetGuardShellLines(target, { passOutput: "stderr" }),
|
|
script,
|
|
].join("\n");
|
|
return dispatchSsh(command, waitMs, remoteTimeoutMs, true, target);
|
|
}
|
|
|
|
export async function uploadRemoteBase64(path: string, encoded: string, target = ciTarget(null)): Promise<DispatchResult> {
|
|
const init = await dispatchSsh([
|
|
"set -euo pipefail",
|
|
`target=${shellQuote(path)}`,
|
|
"rm -f \"$target\"",
|
|
": > \"$target\"",
|
|
"chmod 600 \"$target\"",
|
|
].join("\n"), 20_000, 10_000, true, target);
|
|
if (!init.ok) return init;
|
|
// D601 provider-gateway rejects long host.ssh commands; keep each append
|
|
// envelope comfortably below the transport limit.
|
|
for (const chunk of chunks(encoded, hostSshBase64UploadChunkChars)) {
|
|
const append = await dispatchSsh([
|
|
"set -euo pipefail",
|
|
`target=${shellQuote(path)}`,
|
|
`printf %s ${shellQuote(chunk)} >> "$target"`,
|
|
].join("\n"), 20_000, 10_000, true, target);
|
|
if (!append.ok) return append;
|
|
}
|
|
return dispatchSsh([
|
|
"set -euo pipefail",
|
|
`target=${shellQuote(path)}`,
|
|
`expected=${shellQuote(String(Buffer.byteLength(encoded)))}`,
|
|
"actual=$(wc -c < \"$target\" | tr -d ' ')",
|
|
"printf 'uploaded_bytes=%s expected_bytes=%s path=%s\\n' \"$actual\" \"$expected\" \"$target\"",
|
|
"test \"$actual\" = \"$expected\"",
|
|
].join("\n"), 20_000, 10_000, true, target);
|
|
}
|
|
|
|
export async function runRemoteBackground(label: string, script: string, timeoutMs: number, target = ciTarget(null)): Promise<DispatchResult> {
|
|
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
|
|
const safeLabel = label.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 48);
|
|
const base = `/tmp/unidesk-ci-${safeLabel}-${token}`;
|
|
const scriptPath = `${base}.sh`;
|
|
const logPath = `${base}.log`;
|
|
const donePath = `${base}.done`;
|
|
const encoded = Buffer.from(script, "utf8").toString("base64");
|
|
const upload = await uploadRemoteBase64(`${scriptPath}.b64`, encoded, target);
|
|
if (!upload.ok) return upload;
|
|
const start = await dispatchSsh([
|
|
"set -euo pipefail",
|
|
`script_path=${shellQuote(scriptPath)}`,
|
|
`log_path=${shellQuote(logPath)}`,
|
|
`done_path=${shellQuote(donePath)}`,
|
|
"rm -f \"$script_path\" \"$log_path\" \"$done_path\"",
|
|
"base64 -d \"$script_path.b64\" > \"$script_path\"",
|
|
"rm -f \"$script_path.b64\"",
|
|
"chmod 700 \"$script_path\"",
|
|
"nohup bash -lc \"bash '$script_path' >'$log_path' 2>&1; code=\\$?; printf '%s\\n' \\\"\\$code\\\" >'$done_path'\" >/tmp/unidesk-ci-nohup.log 2>&1 &",
|
|
"printf 'remote_job_pid=%s\\nlog=%s\\ndone=%s\\n' \"$!\" \"$log_path\" \"$done_path\"",
|
|
].join("\n"), 20_000, 10_000, true, target);
|
|
if (!start.ok) return start;
|
|
|
|
const deadline = Date.now() + timeoutMs;
|
|
let latest: DispatchResult = start;
|
|
while (Date.now() < deadline) {
|
|
await Bun.sleep(8_000);
|
|
latest = await dispatchSsh([
|
|
"set -euo pipefail",
|
|
`log_path=${shellQuote(logPath)}`,
|
|
`done_path=${shellQuote(donePath)}`,
|
|
"if [ -f \"$done_path\" ]; then",
|
|
" code=\"$(cat \"$done_path\" 2>/dev/null || printf 1)\"",
|
|
" printf 'REMOTE_DONE:%s\\n' \"$code\"",
|
|
"else",
|
|
" printf 'REMOTE_RUNNING\\n'",
|
|
"fi",
|
|
"tail -n 160 \"$log_path\" 2>/dev/null || true",
|
|
].join("\n"), 75_000, 12_000, true, target);
|
|
if (!latest.ok) {
|
|
if (latest.status === "timeout" || latest.stderr.includes("did not finish within")) {
|
|
continue;
|
|
}
|
|
return latest;
|
|
}
|
|
const firstLine = latest.stdout.split(/\r?\n/u)[0] ?? "";
|
|
if (firstLine.startsWith("REMOTE_DONE:")) {
|
|
const code = Number(firstLine.slice("REMOTE_DONE:".length).trim());
|
|
return {
|
|
...latest,
|
|
ok: code === 0,
|
|
exitCode: Number.isInteger(code) ? code : 1,
|
|
status: code === 0 ? "succeeded" : "failed",
|
|
};
|
|
}
|
|
}
|
|
return {
|
|
...latest,
|
|
ok: false,
|
|
status: "timeout",
|
|
exitCode: 124,
|
|
stderr: `remote background job ${label} did not finish within ${timeoutMs}ms`,
|
|
};
|
|
}
|
|
|
|
export async function remoteApplyManifest(config: UniDeskConfig, path: string, target = ciTarget(null)): Promise<void> {
|
|
const absolute = rootPath(path);
|
|
if (!existsSync(absolute)) throw new Error(`manifest not found: ${path}`);
|
|
const manifest = readFileSync(absolute, "utf8");
|
|
const encoded = Buffer.from(manifest, "utf8").toString("base64");
|
|
emitCiInstallProgress("upload-manifest", "started", { providerId: target.providerId, manifest: path, bytes: Buffer.byteLength(manifest) });
|
|
const script = [
|
|
"set -eu",
|
|
...ciTargetGuardShellLines(target),
|
|
"tmp=$(mktemp /tmp/unidesk-ci-apply.XXXXXX.yaml)",
|
|
"trap 'rm -f \"$tmp\"' EXIT",
|
|
"base64 -d > \"$tmp\" <<'UNIDESK_CI_MANIFEST_B64'",
|
|
encoded,
|
|
"UNIDESK_CI_MANIFEST_B64",
|
|
"test -s \"$tmp\"",
|
|
"printf 'manifest_bytes=%s\\n' \"$(wc -c < \"$tmp\" | tr -d ' ')\"",
|
|
"kubectl apply -f \"$tmp\"",
|
|
].join("\n");
|
|
emitCiInstallProgress("kubectl-apply", "started", { providerId: target.providerId, manifest: path });
|
|
const result = await runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], script);
|
|
if (result.exitCode !== 0) throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`);
|
|
emitCiInstallProgress("upload-manifest", "succeeded", { providerId: target.providerId, manifest: path, upload: result.stdout.split(/\r?\n/u).find((line) => line.startsWith("manifest_bytes=")) ?? "" });
|
|
emitCiInstallProgress("kubectl-apply", "succeeded", { providerId: target.providerId, manifest: path });
|
|
}
|
|
|
|
export async function prewarmCiRuntimeImages(target = ciTarget(null)): Promise<void> {
|
|
const images = ciRuntimeImages.map(shellQuote).join(" ");
|
|
const script = [
|
|
"set -euo pipefail",
|
|
...ciTargetGuardShellLines(target),
|
|
"export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config",
|
|
"mkdir -p \"$DOCKER_CONFIG\"",
|
|
"printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"",
|
|
`images=(${images})`,
|
|
"for image in \"${images[@]}\"; do",
|
|
" if ! docker image inspect \"$image\" >/dev/null 2>&1; then",
|
|
" echo ci_runtime_image_pull=$image",
|
|
` HTTP_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} HTTPS_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} ALL_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal docker pull --platform linux/amd64 "$image"`,
|
|
" else",
|
|
" echo ci_runtime_image_cached=$image",
|
|
" fi",
|
|
"done",
|
|
"pause_entrypoint=$(docker image inspect rancher/mirrored-pause:3.6 --format '{{json .Config.Entrypoint}}' 2>/dev/null || true)",
|
|
"if ! printf '%s' \"$pause_entrypoint\" | grep -q '\"/pause\"'; then echo native_k3s_pause_image_invalid_entrypoint=$pause_entrypoint >&2; exit 1; fi",
|
|
"root_exec() {",
|
|
" if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return $?; fi",
|
|
" if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then sudo \"$@\"; return $?; fi",
|
|
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return $?; fi",
|
|
" echo ci_runtime_image_containerd_root_required=true >&2",
|
|
" \"$@\"",
|
|
"}",
|
|
"containerd_images=$(root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls 2>/tmp/unidesk-ci-containerd-images.err || true)",
|
|
"containerd_ready=1",
|
|
"for image in \"${images[@]}\"; do",
|
|
" case \"$image\" in",
|
|
" rancher/*|oven/*|alpine/*) ref=\"docker.io/$image\" ;;",
|
|
" unidesk-*) ref=\"docker.io/library/$image\" ;;",
|
|
" *) ref=\"$image\" ;;",
|
|
" esac",
|
|
" if ! printf '%s\\n' \"$containerd_images\" | grep -F \"$ref\" >/dev/null; then",
|
|
" containerd_ready=0",
|
|
" echo ci_runtime_image_containerd_missing=$ref",
|
|
" fi",
|
|
"done",
|
|
"if [ \"$containerd_ready\" = \"1\" ]; then",
|
|
" echo ci_runtime_images_containerd_cached=all",
|
|
" exit 0",
|
|
"fi",
|
|
"rm -f /tmp/unidesk-ci-runtime-images.tar",
|
|
"docker save \"${images[@]}\" -o /tmp/unidesk-ci-runtime-images.tar",
|
|
"root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import --digests --all-platforms /tmp/unidesk-ci-runtime-images.tar >/tmp/unidesk-ci-runtime-images-import.log",
|
|
"root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/rancher/mirrored-pause:3.6' >/dev/null",
|
|
"root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/oven/bun:1-debian' >/dev/null",
|
|
"root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/alpine/git:2.45.2' >/dev/null",
|
|
`root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F ${shellQuote(`docker.io/library/${target.codeQueueImage}`)} >/dev/null`,
|
|
].join("\n");
|
|
const result = await runRemoteBackground("prewarm-runtime-images", script, 900_000, target);
|
|
if (!result.ok) {
|
|
const combined = `${result.stdout}\n${result.stderr}`.trim();
|
|
const failureLines = extractCiLogFailureHints(combined).concat(
|
|
combined.split(/\r?\n/u).filter((line) => /ci_runtime_image_|containerd_|root_required|sudo/iu.test(line)).map(boundedHintLine).slice(-40),
|
|
);
|
|
throw new Error(`CI runtime image prewarm failed: ${JSON.stringify({
|
|
providerId: target.providerId,
|
|
exitCode: result.exitCode,
|
|
failureLines: Array.from(new Set(failureLines)).slice(-50),
|
|
stdoutTail: tailTextLines(result.stdout, 80),
|
|
stderrTail: tailTextLines(result.stderr, 80),
|
|
recovery: [
|
|
"If Tekton is already installed and only CI manifests need refreshing, rerun with: bun scripts/cli.ts ci install --skip-prewarm",
|
|
"If runtime helper images are missing from containerd, restore passwordless/root containerd import on the provider and rerun without --skip-prewarm.",
|
|
],
|
|
})}`);
|
|
}
|
|
}
|