feat: add devops-controlled dev ci flow
This commit is contained in:
Codex
+492
-117
@@ -1,12 +1,11 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { runCommand } from "./command";
|
||||
import { type UniDeskConfig, repoRoot, rootPath } from "./config";
|
||||
import { startJob } from "./jobs";
|
||||
import { coreInternalFetch } from "./microservices";
|
||||
|
||||
const k3sctlContainerName = "k3sctl-adapter";
|
||||
const k3sctlSshKey = "/run/host-ssh/id_ed25519";
|
||||
const d601SshTarget = "ubuntu@host.docker.internal";
|
||||
const d601ProviderId = "D601";
|
||||
const d601Kubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
const tektonPipelineVersion = "v1.12.0";
|
||||
const tektonTriggersVersion = "v0.34.0";
|
||||
@@ -14,6 +13,7 @@ const tektonPipelineReleaseUrl = `https://infra.tekton.dev/tekton-releases/pipel
|
||||
const tektonTriggersReleaseUrl = `https://infra.tekton.dev/tekton-releases/triggers/previous/${tektonTriggersVersion}/release.yaml`;
|
||||
const tektonTriggersInterceptorsUrl = `https://infra.tekton.dev/tekton-releases/triggers/previous/${tektonTriggersVersion}/interceptors.yaml`;
|
||||
const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789";
|
||||
const ciCodeQueueImage = "unidesk-code-queue:dev";
|
||||
const ciRuntimeImages = [
|
||||
"rancher/mirrored-pause:3.6",
|
||||
"rancher/mirrored-library-busybox:1.36.1",
|
||||
@@ -25,7 +25,7 @@ const ciRuntimeImages = [
|
||||
"ghcr.io/tektoncd/triggers/eventlistenersink-7ad1faa98cddbcb0c24990303b220bb8:v0.34.0",
|
||||
"oven/bun:1-debian",
|
||||
"alpine/git:2.45.2",
|
||||
"unidesk-code-queue:d601",
|
||||
ciCodeQueueImage,
|
||||
];
|
||||
|
||||
interface CiOptions {
|
||||
@@ -34,6 +34,35 @@ interface CiOptions {
|
||||
waitMs: number;
|
||||
}
|
||||
|
||||
interface CiDevE2EOptions {
|
||||
repoUrl: string;
|
||||
desiredRef: string;
|
||||
deployCommit: string;
|
||||
environment: "dev";
|
||||
services: Array<{ id: string; commitId: string; repo: string }>;
|
||||
runId: string;
|
||||
keepNamespace: boolean;
|
||||
waitMs: number;
|
||||
direct: boolean;
|
||||
}
|
||||
|
||||
interface DispatchResult {
|
||||
ok: boolean;
|
||||
taskId: string | null;
|
||||
status: string | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
exitCode: number | null;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
interface DeployDevManifestSummary {
|
||||
deployCommit: string;
|
||||
desiredRef: string;
|
||||
environment: "dev";
|
||||
services: Array<{ id: string; commitId: string; repo: string }>;
|
||||
}
|
||||
|
||||
function stringOption(args: string[], name: string): string | null {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return null;
|
||||
@@ -56,88 +85,243 @@ function requireRevision(value: string | null): string {
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireDesiredRef(value: string | null): string {
|
||||
const ref = value ?? "master";
|
||||
if (!/^[A-Za-z0-9._/-]{1,160}$/u.test(ref) || ref.startsWith("-") || ref.includes("..")) {
|
||||
throw new Error("ci run-dev-e2e --desired-ref contains unsupported characters");
|
||||
}
|
||||
return ref;
|
||||
}
|
||||
|
||||
function boolFlag(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/gu, "'\\''")}'`;
|
||||
}
|
||||
|
||||
function dockerExecK3sctl(args: string[]) {
|
||||
return runCommand(["docker", "exec", k3sctlContainerName, ...args], repoRoot);
|
||||
}
|
||||
|
||||
function dockerExecK3sctlWithInput(args: string[], input: string) {
|
||||
const command = ["docker", "exec", "-i", k3sctlContainerName, ...args];
|
||||
const result = spawnSync(command[0], command.slice(1), {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
input,
|
||||
maxBuffer: 1024 * 1024 * 8,
|
||||
});
|
||||
return {
|
||||
command,
|
||||
cwd: repoRoot,
|
||||
exitCode: result.status,
|
||||
stdout: result.stdout ?? "",
|
||||
stderr: result.stderr ?? result.error?.message ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
function remoteKubectlCommand(script: string): string[] {
|
||||
return [
|
||||
"sh",
|
||||
"-lc",
|
||||
[
|
||||
"ssh",
|
||||
"-i",
|
||||
shellQuote(k3sctlSshKey),
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/tmp/unidesk-ci-known-hosts",
|
||||
"-o",
|
||||
"ConnectTimeout=10",
|
||||
shellQuote(d601SshTarget),
|
||||
shellQuote(`KUBECONFIG=${d601Kubeconfig} bash -lc ${shellQuote(script)}`),
|
||||
].join(" "),
|
||||
];
|
||||
}
|
||||
|
||||
function runRemoteKubectl(script: string) {
|
||||
const result = dockerExecK3sctl(remoteKubectlCommand(script));
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(`D601 kubectl command failed: ${result.stderr || result.stdout}`);
|
||||
function chunks(value: string, size: number): string[] {
|
||||
const result: string[] = [];
|
||||
for (let index = 0; index < value.length; index += size) {
|
||||
result.push(value.slice(index, index + size));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function remoteApplyManifest(path: string): void {
|
||||
const absolute = rootPath(path);
|
||||
if (!existsSync(absolute)) throw new Error(`manifest not found: ${path}`);
|
||||
const result = dockerExecK3sctlWithInput([
|
||||
"sh",
|
||||
"-lc",
|
||||
[
|
||||
"ssh",
|
||||
"-i",
|
||||
shellQuote(k3sctlSshKey),
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/tmp/unidesk-ci-known-hosts",
|
||||
"-o",
|
||||
"ConnectTimeout=10",
|
||||
shellQuote(d601SshTarget),
|
||||
shellQuote(`KUBECONFIG=${d601Kubeconfig} kubectl apply -f -`),
|
||||
].join(" "),
|
||||
], readFileSync(absolute, "utf8"));
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function prewarmCiRuntimeImages(): void {
|
||||
const images = ciRuntimeImages.map(shellQuote).join(" ");
|
||||
runRemoteKubectl([
|
||||
function asString(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function coreBody(response: unknown): Record<string, unknown> | null {
|
||||
return asRecord(asRecord(response)?.body);
|
||||
}
|
||||
|
||||
function proxyBody(response: unknown): Record<string, unknown> | null {
|
||||
const body = coreBody(response);
|
||||
const nested = asRecord(body?.body);
|
||||
return nested ?? body;
|
||||
}
|
||||
|
||||
async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number): Promise<DispatchResult> {
|
||||
const dispatchResponse = coreInternalFetch("/api/dispatch", {
|
||||
method: "POST",
|
||||
body: {
|
||||
providerId: d601ProviderId,
|
||||
command: "host.ssh",
|
||||
payload: {
|
||||
source: "ci-cli",
|
||||
mode: "exec",
|
||||
command,
|
||||
timeoutMs: remoteTimeoutMs,
|
||||
cwd: "/home/ubuntu",
|
||||
},
|
||||
},
|
||||
});
|
||||
const dispatchBody = coreBody(dispatchResponse);
|
||||
const taskId = asString(dispatchBody?.taskId);
|
||||
if (dispatchBody?.ok !== true || taskId.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
taskId: taskId || null,
|
||||
status: null,
|
||||
stdout: "",
|
||||
stderr: asString(dispatchBody?.error) || "dispatch did not return a task id",
|
||||
exitCode: null,
|
||||
raw: dispatchResponse,
|
||||
};
|
||||
}
|
||||
const deadline = Date.now() + Math.max(waitMs, remoteTimeoutMs + 10_000);
|
||||
let latest: unknown = null;
|
||||
while (Date.now() < deadline) {
|
||||
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000 });
|
||||
const task = asRecord(coreBody(latest)?.task);
|
||||
const status = asString(task?.status);
|
||||
if (status === "succeeded" || status === "failed") {
|
||||
const result = asRecord(task?.result);
|
||||
const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null;
|
||||
const stdout = asString(result?.stdout);
|
||||
const stderr = asString(result?.stderr);
|
||||
return {
|
||||
ok: status === "succeeded" && (exitCode === null || exitCode === 0),
|
||||
taskId,
|
||||
status,
|
||||
stdout,
|
||||
stderr,
|
||||
exitCode,
|
||||
raw: task,
|
||||
};
|
||||
}
|
||||
await Bun.sleep(500);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
taskId,
|
||||
status: "timeout",
|
||||
stdout: "",
|
||||
stderr: `host.ssh task ${taskId} did not finish within ${Math.max(waitMs, remoteTimeoutMs + 10_000)}ms`,
|
||||
exitCode: null,
|
||||
raw: latest,
|
||||
};
|
||||
}
|
||||
|
||||
async function runRemoteKubectl(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000): Promise<DispatchResult> {
|
||||
const result = await runRemoteKubectlRaw(script, waitMs, remoteTimeoutMs);
|
||||
if (!result.ok) {
|
||||
throw new Error(`D601 kubectl command failed: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runRemoteKubectlRaw(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000): Promise<DispatchResult> {
|
||||
const command = [
|
||||
"set -euo pipefail",
|
||||
`export KUBECONFIG=${shellQuote(d601Kubeconfig)}`,
|
||||
script,
|
||||
].join("\n");
|
||||
return dispatchSsh(command, waitMs, remoteTimeoutMs);
|
||||
}
|
||||
|
||||
async function uploadRemoteBase64(path: string, encoded: string): Promise<DispatchResult> {
|
||||
const init = await dispatchSsh([
|
||||
"set -euo pipefail",
|
||||
`target=${shellQuote(path)}`,
|
||||
"rm -f \"$target\"",
|
||||
": > \"$target\"",
|
||||
"chmod 600 \"$target\"",
|
||||
].join("\n"), 20_000, 10_000);
|
||||
if (!init.ok) return init;
|
||||
for (const chunk of chunks(encoded, 950)) {
|
||||
const append = await dispatchSsh([
|
||||
"set -euo pipefail",
|
||||
`target=${shellQuote(path)}`,
|
||||
`printf %s ${shellQuote(chunk)} >> "$target"`,
|
||||
].join("\n"), 20_000, 10_000);
|
||||
if (!append.ok) return append;
|
||||
}
|
||||
return dispatchSsh([
|
||||
"set -euo pipefail",
|
||||
`target=${shellQuote(path)}`,
|
||||
"wc -c \"$target\"",
|
||||
].join("\n"), 20_000, 10_000);
|
||||
}
|
||||
|
||||
async function runRemoteBackground(label: string, script: string, timeoutMs: number): Promise<DispatchResult> {
|
||||
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
|
||||
const safeLabel = label.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 48);
|
||||
const base = `/tmp/unidesk-ci-${safeLabel}-${token}`;
|
||||
const scriptPath = `${base}.sh`;
|
||||
const logPath = `${base}.log`;
|
||||
const donePath = `${base}.done`;
|
||||
const encoded = Buffer.from(script, "utf8").toString("base64");
|
||||
const upload = await uploadRemoteBase64(`${scriptPath}.b64`, encoded);
|
||||
if (!upload.ok) return upload;
|
||||
const start = await dispatchSsh([
|
||||
"set -euo pipefail",
|
||||
`script_path=${shellQuote(scriptPath)}`,
|
||||
`log_path=${shellQuote(logPath)}`,
|
||||
`done_path=${shellQuote(donePath)}`,
|
||||
"rm -f \"$script_path\" \"$log_path\" \"$done_path\"",
|
||||
"base64 -d \"$script_path.b64\" > \"$script_path\"",
|
||||
"rm -f \"$script_path.b64\"",
|
||||
"chmod 700 \"$script_path\"",
|
||||
"nohup bash -lc \"bash '$script_path' >'$log_path' 2>&1; code=\\$?; printf '%s\\n' \\\"\\$code\\\" >'$done_path'\" >/tmp/unidesk-ci-nohup.log 2>&1 &",
|
||||
"printf 'remote_job_pid=%s\\nlog=%s\\ndone=%s\\n' \"$!\" \"$log_path\" \"$done_path\"",
|
||||
].join("\n"), 20_000, 10_000);
|
||||
if (!start.ok) return start;
|
||||
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let latest: DispatchResult = start;
|
||||
while (Date.now() < deadline) {
|
||||
await Bun.sleep(8_000);
|
||||
latest = await dispatchSsh([
|
||||
"set -euo pipefail",
|
||||
`log_path=${shellQuote(logPath)}`,
|
||||
`done_path=${shellQuote(donePath)}`,
|
||||
"if [ -f \"$done_path\" ]; then",
|
||||
" code=\"$(cat \"$done_path\" 2>/dev/null || printf 1)\"",
|
||||
" printf 'REMOTE_DONE:%s\\n' \"$code\"",
|
||||
"else",
|
||||
" printf 'REMOTE_RUNNING\\n'",
|
||||
"fi",
|
||||
"tail -n 160 \"$log_path\" 2>/dev/null || true",
|
||||
].join("\n"), 75_000, 12_000);
|
||||
if (!latest.ok) {
|
||||
if (latest.status === "timeout" || latest.stderr.includes("did not finish within")) {
|
||||
continue;
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
const firstLine = latest.stdout.split(/\r?\n/u)[0] ?? "";
|
||||
if (firstLine.startsWith("REMOTE_DONE:")) {
|
||||
const code = Number(firstLine.slice("REMOTE_DONE:".length).trim());
|
||||
return {
|
||||
...latest,
|
||||
ok: code === 0,
|
||||
exitCode: Number.isInteger(code) ? code : 1,
|
||||
status: code === 0 ? "succeeded" : "failed",
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
...latest,
|
||||
ok: false,
|
||||
status: "timeout",
|
||||
exitCode: 124,
|
||||
stderr: `remote background job ${label} did not finish within ${timeoutMs}ms`,
|
||||
};
|
||||
}
|
||||
|
||||
async function remoteApplyManifest(path: string): Promise<void> {
|
||||
const absolute = rootPath(path);
|
||||
if (!existsSync(absolute)) throw new Error(`manifest not found: ${path}`);
|
||||
const encoded = Buffer.from(readFileSync(absolute, "utf8"), "utf8").toString("base64");
|
||||
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
|
||||
const b64Path = `/tmp/unidesk-ci-apply-${token}.b64`;
|
||||
const upload = await uploadRemoteBase64(b64Path, encoded);
|
||||
if (!upload.ok) throw new Error(`failed to upload manifest ${path}: ${upload.stderr || upload.stdout}`);
|
||||
const script = [
|
||||
"set -euo pipefail",
|
||||
`export KUBECONFIG=${shellQuote(d601Kubeconfig)}`,
|
||||
"tmp=$(mktemp /tmp/unidesk-ci-apply.XXXXXX.yaml)",
|
||||
`b64_path=${shellQuote(b64Path)}`,
|
||||
"trap 'rm -f \"$tmp\" \"$b64_path\"' EXIT",
|
||||
"base64 -d \"$b64_path\" > \"$tmp\"",
|
||||
"kubectl apply -f \"$tmp\"",
|
||||
].join("\n");
|
||||
const result = await runRemoteBackground(`apply-${path.split("/").pop() ?? "manifest"}`, script, 180_000);
|
||||
if (!result.ok) throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
|
||||
async function prewarmCiRuntimeImages(): Promise<void> {
|
||||
const images = ciRuntimeImages.map(shellQuote).join(" ");
|
||||
const script = [
|
||||
"set -euo pipefail",
|
||||
`export KUBECONFIG=${shellQuote(d601Kubeconfig)}`,
|
||||
"export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config",
|
||||
"mkdir -p \"$DOCKER_CONFIG\"",
|
||||
"printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"",
|
||||
@@ -152,18 +336,37 @@ function prewarmCiRuntimeImages(): void {
|
||||
"done",
|
||||
"pause_entrypoint=$(docker image inspect rancher/mirrored-pause:3.6 --format '{{json .Config.Entrypoint}}' 2>/dev/null || true)",
|
||||
"if ! printf '%s' \"$pause_entrypoint\" | grep -q '\"/pause\"'; then echo native_k3s_pause_image_invalid_entrypoint=$pause_entrypoint >&2; exit 1; fi",
|
||||
"containerd_images=$(/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls 2>/tmp/unidesk-ci-containerd-images.err || true)",
|
||||
"containerd_ready=1",
|
||||
"for image in \"${images[@]}\"; do",
|
||||
" case \"$image\" in",
|
||||
" rancher/*|oven/*|alpine/*) ref=\"docker.io/$image\" ;;",
|
||||
" unidesk-*) ref=\"docker.io/library/$image\" ;;",
|
||||
" *) ref=\"$image\" ;;",
|
||||
" esac",
|
||||
" if ! printf '%s\\n' \"$containerd_images\" | grep -F \"$ref\" >/dev/null; then",
|
||||
" containerd_ready=0",
|
||||
" echo ci_runtime_image_containerd_missing=$ref",
|
||||
" fi",
|
||||
"done",
|
||||
"if [ \"$containerd_ready\" = \"1\" ]; then",
|
||||
" echo ci_runtime_images_containerd_cached=all",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"rm -f /tmp/unidesk-ci-runtime-images.tar",
|
||||
"docker save \"${images[@]}\" -o /tmp/unidesk-ci-runtime-images.tar",
|
||||
"/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import --digests --all-platforms /tmp/unidesk-ci-runtime-images.tar >/tmp/unidesk-ci-runtime-images-import.log",
|
||||
"/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/rancher/mirrored-pause:3.6' >/dev/null",
|
||||
"/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/oven/bun:1-debian' >/dev/null",
|
||||
"/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/alpine/git:2.45.2' >/dev/null",
|
||||
"/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/library/unidesk-code-queue:d601' >/dev/null",
|
||||
].join("\n"));
|
||||
`/mnt/c/Windows/System32/wsl.exe -u root -- ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F ${shellQuote(`docker.io/library/${ciCodeQueueImage}`)} >/dev/null`,
|
||||
].join("\n");
|
||||
const result = await runRemoteBackground("prewarm-runtime-images", script, 900_000);
|
||||
if (!result.ok) throw new Error(`CI runtime image prewarm failed: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
|
||||
function status(): Record<string, unknown> {
|
||||
const summary = runRemoteKubectl([
|
||||
async function status(): Promise<Record<string, unknown>> {
|
||||
const summary = await runRemoteKubectl([
|
||||
"set -euo pipefail",
|
||||
"printf 'tekton_pipelines='",
|
||||
"kubectl get deploy -n tekton-pipelines -o name 2>/dev/null | tr '\\n' ' ' || true",
|
||||
@@ -175,7 +378,7 @@ function status(): Record<string, unknown> {
|
||||
].join("\n"));
|
||||
return {
|
||||
ok: true,
|
||||
providerId: "D601",
|
||||
providerId: d601ProviderId,
|
||||
orchestrator: "native-k3s",
|
||||
tekton: {
|
||||
pipelineVersion: tektonPipelineVersion,
|
||||
@@ -185,23 +388,26 @@ function status(): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function install(): Record<string, unknown> {
|
||||
async function install(): Promise<Record<string, unknown>> {
|
||||
if (!existsSync(rootPath("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml"))) {
|
||||
throw new Error("CI manifests are missing");
|
||||
}
|
||||
prewarmCiRuntimeImages();
|
||||
runRemoteKubectl([
|
||||
await prewarmCiRuntimeImages();
|
||||
const installTektonScript = [
|
||||
"set -euo pipefail",
|
||||
`export KUBECONFIG=${shellQuote(d601Kubeconfig)}`,
|
||||
`kubectl apply -f ${shellQuote(tektonPipelineReleaseUrl)}`,
|
||||
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s",
|
||||
`kubectl apply -f ${shellQuote(tektonTriggersReleaseUrl)}`,
|
||||
`kubectl apply -f ${shellQuote(tektonTriggersInterceptorsUrl)}`,
|
||||
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s",
|
||||
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines-resolvers --timeout=900s",
|
||||
].join("\n"));
|
||||
remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/tekton-install.yaml");
|
||||
remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml");
|
||||
remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.triggers.yaml");
|
||||
].join("\n");
|
||||
const installTekton = await runRemoteBackground("install-tekton", installTektonScript, 1_200_000);
|
||||
if (!installTekton.ok) throw new Error(`Tekton install failed: ${installTekton.stderr || installTekton.stdout}`);
|
||||
await remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/tekton-install.yaml");
|
||||
await remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml");
|
||||
await remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.triggers.yaml");
|
||||
return status();
|
||||
}
|
||||
|
||||
@@ -233,31 +439,66 @@ spec:
|
||||
`;
|
||||
}
|
||||
|
||||
function remoteCreatePipelineRun(manifest: string): string {
|
||||
const result = dockerExecK3sctlWithInput([
|
||||
"sh",
|
||||
"-lc",
|
||||
[
|
||||
"ssh",
|
||||
"-i",
|
||||
shellQuote(k3sctlSshKey),
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/tmp/unidesk-ci-known-hosts",
|
||||
shellQuote(d601SshTarget),
|
||||
shellQuote(`KUBECONFIG=${d601Kubeconfig} kubectl create -f - -o jsonpath='{.metadata.name}'`),
|
||||
].join(" "),
|
||||
], manifest);
|
||||
if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout);
|
||||
function devE2EPipelineRunManifest(options: CiDevE2EOptions): string {
|
||||
const deployRevisionLabel = options.deployCommit.slice(0, 40);
|
||||
return `apiVersion: tekton.dev/v1
|
||||
kind: PipelineRun
|
||||
metadata:
|
||||
generateName: unidesk-dev-e2e-${options.runId}-
|
||||
namespace: unidesk-ci
|
||||
labels:
|
||||
app.kubernetes.io/name: unidesk-dev-namespace-e2e
|
||||
app.kubernetes.io/part-of: unidesk
|
||||
unidesk.ai/ci-kind: dev-namespace-e2e
|
||||
unidesk.ai/deploy-ref: master-deploy-json-dev
|
||||
unidesk.ai/deploy-commit: ${JSON.stringify(deployRevisionLabel)}
|
||||
spec:
|
||||
pipelineRef:
|
||||
name: unidesk-dev-namespace-e2e
|
||||
taskRunTemplate:
|
||||
serviceAccountName: unidesk-ci-runner
|
||||
params:
|
||||
- name: repo-url
|
||||
value: ${JSON.stringify(options.repoUrl)}
|
||||
- name: desired-ref
|
||||
value: ${JSON.stringify(options.desiredRef)}
|
||||
- name: deploy-commit
|
||||
value: ${JSON.stringify(options.deployCommit)}
|
||||
- name: environment
|
||||
value: ${JSON.stringify(options.environment)}
|
||||
- name: run-id
|
||||
value: ${JSON.stringify(options.runId)}
|
||||
- name: keep-namespace
|
||||
value: ${JSON.stringify(options.keepNamespace ? "true" : "false")}
|
||||
workspaces:
|
||||
- name: shared-workspace
|
||||
persistentVolumeClaim:
|
||||
claimName: unidesk-ci-cache
|
||||
`;
|
||||
}
|
||||
|
||||
async function remoteCreatePipelineRun(manifest: string): Promise<string> {
|
||||
const encoded = Buffer.from(manifest, "utf8").toString("base64");
|
||||
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
|
||||
const b64Path = `/tmp/unidesk-ci-pipelinerun-${token}.b64`;
|
||||
const upload = await uploadRemoteBase64(b64Path, encoded);
|
||||
if (!upload.ok) throw new Error(`failed to upload PipelineRun manifest: ${upload.stderr || upload.stdout}`);
|
||||
const result = await runRemoteKubectl([
|
||||
"tmp=$(mktemp /tmp/unidesk-ci-run.XXXXXX.yaml)",
|
||||
`b64_path=${shellQuote(b64Path)}`,
|
||||
"trap 'rm -f \"$tmp\" \"$b64_path\"' EXIT",
|
||||
"base64 -d \"$b64_path\" > \"$tmp\"",
|
||||
"kubectl create -f \"$tmp\" -o jsonpath='{.metadata.name}'",
|
||||
].join("\n"), 60_000, 45_000);
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
function run(options: CiOptions): Record<string, unknown> {
|
||||
const name = remoteCreatePipelineRun(pipelineRunManifest(options));
|
||||
const wait = options.waitMs > 0 ? dockerExecK3sctl(remoteKubectlCommand([
|
||||
async function waitForPipelineRun(name: string, waitMs: number): Promise<DispatchResult | null> {
|
||||
if (waitMs <= 0) return null;
|
||||
const command = [
|
||||
"set -euo pipefail",
|
||||
`deadline=$((SECONDS + ${Math.ceil(options.waitMs / 1000)}))`,
|
||||
`export KUBECONFIG=${shellQuote(d601Kubeconfig)}`,
|
||||
`deadline=$((SECONDS + ${Math.ceil(waitMs / 1000)}))`,
|
||||
"while [ \"$SECONDS\" -lt \"$deadline\" ]; do",
|
||||
` condition="$(kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o jsonpath='{range .status.conditions[?(@.type==\"Succeeded\")]}{.status}{\"\\t\"}{.reason}{\"\\t\"}{.message}{end}' 2>/dev/null || true)"`,
|
||||
" case \"$condition\" in",
|
||||
@@ -277,7 +518,13 @@ function run(options: CiOptions): Record<string, unknown> {
|
||||
`echo "Timed out waiting for pipelinerun/${name}" >&2`,
|
||||
`kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o json`,
|
||||
"exit 124",
|
||||
].join("\n"))) : null;
|
||||
].join("\n");
|
||||
return dispatchSsh(command, waitMs + 30_000, waitMs + 20_000);
|
||||
}
|
||||
|
||||
async function run(options: CiOptions): Promise<Record<string, unknown>> {
|
||||
const name = await remoteCreatePipelineRun(pipelineRunManifest(options));
|
||||
const wait = await waitForPipelineRun(name, options.waitMs);
|
||||
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
|
||||
return {
|
||||
ok: waitSucceeded,
|
||||
@@ -296,14 +543,111 @@ function run(options: CiOptions): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function logs(name: string): Record<string, unknown> {
|
||||
function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary {
|
||||
const remoteRef = `refs/remotes/origin/${desiredRef}`;
|
||||
const fetch = runCommand(["git", "fetch", "--quiet", "origin", `+refs/heads/${desiredRef}:${remoteRef}`], repoRoot);
|
||||
if (fetch.exitCode !== 0) throw new Error(`failed to fetch origin/${desiredRef}: ${fetch.stderr || fetch.stdout}`);
|
||||
const deployCommitResult = runCommand(["git", "rev-parse", `origin/${desiredRef}`], repoRoot);
|
||||
if (deployCommitResult.exitCode !== 0) throw new Error(`failed to resolve origin/${desiredRef}: ${deployCommitResult.stderr || deployCommitResult.stdout}`);
|
||||
const show = runCommand(["git", "show", `origin/${desiredRef}:deploy.json`], repoRoot);
|
||||
if (show.exitCode !== 0) throw new Error(`failed to read deploy.json from origin/${desiredRef}: ${show.stderr || show.stdout}`);
|
||||
const parsed = JSON.parse(show.stdout) as unknown;
|
||||
const record = asRecord(parsed);
|
||||
if (record?.schemaVersion !== 2) throw new Error(`origin/${desiredRef}:deploy.json must use schemaVersion=2`);
|
||||
const environments = asRecord(record.environments);
|
||||
const dev = asRecord(environments?.dev);
|
||||
const rawServices = Array.isArray(dev?.services) ? dev.services : [];
|
||||
const services = rawServices.map((item) => {
|
||||
const service = asRecord(item);
|
||||
return {
|
||||
id: asString(service?.id),
|
||||
commitId: asString(service?.commitId).toLowerCase(),
|
||||
repo: asString(service?.repo),
|
||||
};
|
||||
}).filter((service) => service.id.length > 0 && service.commitId.length > 0);
|
||||
if (services.length === 0) throw new Error(`origin/${desiredRef}:deploy.json has no environments.dev services with commitId`);
|
||||
return {
|
||||
deployCommit: deployCommitResult.stdout.trim(),
|
||||
desiredRef,
|
||||
environment: "dev",
|
||||
services,
|
||||
};
|
||||
}
|
||||
|
||||
function makeRunId(deployCommit: string): string {
|
||||
const stamp = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase();
|
||||
return `${stamp}-${deployCommit.slice(0, 8).toLowerCase()}`.replace(/[^a-z0-9-]/gu, "-").slice(0, 48);
|
||||
}
|
||||
|
||||
async function runDevE2E(options: CiDevE2EOptions): Promise<Record<string, unknown>> {
|
||||
if (!options.direct) {
|
||||
const devopsResponse = coreInternalFetch("/api/microservices/devops/proxy/api/ci/dev-e2e/run", {
|
||||
method: "POST",
|
||||
body: {
|
||||
repoUrl: options.repoUrl,
|
||||
desiredRef: options.desiredRef,
|
||||
environment: options.environment,
|
||||
runId: options.runId,
|
||||
keepNamespace: options.keepNamespace,
|
||||
},
|
||||
maxResponseBytes: 2_000_000,
|
||||
});
|
||||
const devopsBody = proxyBody(devopsResponse);
|
||||
if (devopsBody?.ok === true) {
|
||||
const pipelineRun = asString(devopsBody.pipelineRun);
|
||||
const wait = pipelineRun.length > 0 ? await waitForPipelineRun(pipelineRun, options.waitMs) : null;
|
||||
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
|
||||
return {
|
||||
...devopsBody,
|
||||
ok: waitSucceeded,
|
||||
triggerMode: "devops-service",
|
||||
wait: wait === null ? null : {
|
||||
stdoutTail: wait.stdout.slice(-6000),
|
||||
stderrTail: wait.stderr.slice(-6000),
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
triggerMode: "devops-service",
|
||||
error: "DevOps service trigger failed or did not return ok=true; use --direct only for CI bootstrap/recovery, never as a service deployment path.",
|
||||
devopsResponse,
|
||||
};
|
||||
}
|
||||
const name = await remoteCreatePipelineRun(devE2EPipelineRunManifest(options));
|
||||
const wait = await waitForPipelineRun(name, options.waitMs);
|
||||
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
|
||||
return {
|
||||
ok: waitSucceeded,
|
||||
pipelineRun: name,
|
||||
namespace: "unidesk-ci",
|
||||
temporaryNamespace: `unidesk-ci-e2e-${options.runId}`,
|
||||
repoUrl: options.repoUrl,
|
||||
desiredRef: options.desiredRef,
|
||||
deployCommit: options.deployCommit,
|
||||
environment: options.environment,
|
||||
services: options.services,
|
||||
keepNamespace: options.keepNamespace,
|
||||
triggerMode: "direct-maintenance",
|
||||
wait: wait === null ? null : {
|
||||
stdoutTail: wait.stdout.slice(-6000),
|
||||
stderrTail: wait.stderr.slice(-6000),
|
||||
},
|
||||
next: [
|
||||
`bun scripts/cli.ts ci logs ${name}`,
|
||||
"bun scripts/cli.ts ci status",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function logs(name: string): Promise<Record<string, unknown>> {
|
||||
if (name.length === 0) throw new Error("ci logs requires PipelineRun name");
|
||||
const result = runRemoteKubectl([
|
||||
const result = await runRemoteKubectl([
|
||||
"set -euo pipefail",
|
||||
`kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`,
|
||||
`kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o wide`,
|
||||
`for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail=160; done`,
|
||||
].join("\n"));
|
||||
`for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail=160 || true; done`,
|
||||
].join("\n"), 60_000, 45_000);
|
||||
return {
|
||||
ok: true,
|
||||
pipelineRun: name,
|
||||
@@ -314,11 +658,12 @@ function logs(name: string): Record<string, unknown> {
|
||||
|
||||
function help(): Record<string, unknown> {
|
||||
return {
|
||||
command: "ci install|status|run|logs",
|
||||
command: "ci install|status|run|run-dev-e2e|logs",
|
||||
description: "Manage the D601 k3s Tekton CI gate. This intentionally does not deploy CD.",
|
||||
examples: [
|
||||
"bun scripts/cli.ts ci install",
|
||||
"bun scripts/cli.ts ci run --revision <commit>",
|
||||
"bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000",
|
||||
"bun scripts/cli.ts ci logs <pipelineRun>",
|
||||
],
|
||||
tekton: {
|
||||
@@ -330,10 +675,22 @@ function help(): Record<string, unknown> {
|
||||
interceptors: tektonTriggersInterceptorsUrl,
|
||||
},
|
||||
},
|
||||
runDevE2E: {
|
||||
defaultTriggerMode: "devops-service",
|
||||
directMaintenanceFlag: "--direct",
|
||||
desiredState: "origin/master:deploy.json#environments.dev",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runCiCommand(_config: UniDeskConfig, args: string[]): Record<string, unknown> {
|
||||
function requireRunId(value: string): string {
|
||||
if (!/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(value)) {
|
||||
throw new Error("ci run-dev-e2e run id must be DNS-safe lowercase alnum/dash, max 48 chars");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function runCiCommand(_config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
||||
const [action = "status", nameArg] = args;
|
||||
if (action === "help" || action === "--help" || action === "-h") return help();
|
||||
if (action === "install") return install();
|
||||
@@ -344,8 +701,26 @@ export function runCiCommand(_config: UniDeskConfig, args: string[]): Record<str
|
||||
const waitMs = numberOption(args, "--wait-ms", 0);
|
||||
return run({ repoUrl, revision, waitMs });
|
||||
}
|
||||
if (action === "run-dev-e2e") {
|
||||
const repoUrl = stringOption(args, "--repo") ?? stringOption(args, "--repo-url") ?? "https://github.com/pikasTech/unidesk";
|
||||
const desiredRef = requireDesiredRef(stringOption(args, "--desired-ref") ?? stringOption(args, "--deploy-branch"));
|
||||
const manifest = resolveDeployDevManifest(desiredRef);
|
||||
const waitMs = numberOption(args, "--wait-ms", 0);
|
||||
const runId = requireRunId(stringOption(args, "--run-id") ?? makeRunId(manifest.deployCommit));
|
||||
return runDevE2E({
|
||||
repoUrl,
|
||||
desiredRef,
|
||||
deployCommit: manifest.deployCommit,
|
||||
environment: manifest.environment,
|
||||
services: manifest.services,
|
||||
runId,
|
||||
keepNamespace: boolFlag(args, "--keep-namespace"),
|
||||
waitMs,
|
||||
direct: boolFlag(args, "--direct"),
|
||||
});
|
||||
}
|
||||
if (action === "logs") return logs(nameArg ?? "");
|
||||
throw new Error("ci command must be one of: install, status, run, logs");
|
||||
throw new Error("ci command must be one of: install, status, run, run-dev-e2e, logs");
|
||||
}
|
||||
|
||||
export function startCiInstallJob(): Record<string, unknown> {
|
||||
|
||||
Reference in New Issue
Block a user