mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make CI install observable

Browse Source
This commit is contained in:
Codex
2026-06-11 10:34:35 +00:00
parent a10fa26c17
commit 63030fa255
5 changed files with 369 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/ci.ts
+222 -42
View File
@@ -6,7 +6,7 @@ import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadC
import { runCommand } from "./command";
import { type UniDeskConfig, repoRoot, rootPath } from "./config";
import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "./deploy-ssh-identity";
import { startJob } from "./jobs";
import { jobWithTail, listJobs, readJob, startJob } from "./jobs";
import { coreInternalFetch } from "./microservices";
import {
artifactRegistryReadonlyResultFromCommand,
@@ -83,6 +83,13 @@ interface CiOptions {
target: CiTarget;
}
interface CiInstallOptions {
target: CiTarget;
skipPrewarm: boolean;
skipTektonInstall: boolean;
wait: boolean;
}
interface CiTarget {
providerId: string;
kubeconfig: string;
@@ -553,6 +560,16 @@ function coreBody(response: unknown): Record<string, unknown> | null {
return asRecord(asRecord(response)?.body);
}
function emitCiInstallProgress(stage: string, status: "started" | "skipped" | "succeeded" | "failed", detail: Record<string, unknown> = {}): void {
console.error(JSON.stringify({
event: "ci.install.progress",
at: new Date().toISOString(),
stage,
status,
...detail,
}));
}
function responseOk(response: unknown): boolean {
if (typeof response !== "object" || response === null) return false;
const record = response as Record<string, unknown>;
@@ -850,6 +867,7 @@ function requireCiScriptPath(value: unknown): string {
async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number, pollCompletion = true, target = ciTarget(null)): Promise<DispatchResult> {
const dispatchResponse = coreInternalFetch("/api/dispatch", {
method: "POST",
timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000),
body: {
providerId: target.providerId,
command: "host.ssh",
@@ -865,14 +883,19 @@ async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: num
const dispatchBody = coreBody(dispatchResponse);
const taskId = asString(dispatchBody?.taskId);
if (dispatchBody?.ok !== true || taskId.length === 0) {
const failureKind = asRecord(dispatchResponse)?.timedOut === true ? "backend-core-dispatch-timeout" : "backend-core-dispatch-submit-failed";
return {
ok: false,
taskId: taskId || null,
status: null,
stdout: "",
stderr: asString(dispatchBody?.error) || "dispatch did not return a task id",
stderr: asString(dispatchBody?.error) || `${failureKind}: dispatch did not return a task id`,
exitCode: null,
raw: dispatchResponse,
raw: {
failureKind,
dispatchResponse,
timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000),
},
};
}
if (!pollCompletion) {
@@ -890,7 +913,7 @@ async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: num
const deadline = Date.now() + Math.max(effectiveWaitMs, 1_000);
let latest: unknown = null;
while (Date.now() < deadline) {
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000 });
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000, timeoutMs: 15_000 });
const task = asRecord(coreBody(latest)?.task);
const status = asString(task?.status);
if (status === "succeeded" || status === "failed") {
@@ -947,7 +970,7 @@ async function uploadRemoteBase64(path: string, encoded: string, target = ciTarg
"chmod 600 \"$target\"",
].join("\n"), 20_000, 10_000, true, target);
if (!init.ok) return init;
for (const chunk of chunks(encoded, 950)) {
for (const chunk of chunks(encoded, 6_000)) {
const append = await dispatchSsh([
"set -euo pipefail",
`target=${shellQuote(path)}`,
@@ -958,7 +981,10 @@ async function uploadRemoteBase64(path: string, encoded: string, target = ciTarg
return dispatchSsh([
"set -euo pipefail",
`target=${shellQuote(path)}`,
"wc -c \"$target\"",
`expected=${shellQuote(String(Buffer.byteLength(encoded)))}`,
"actual=$(wc -c < \"$target\" | tr -d ' ')",
"printf 'uploaded_bytes=%s expected_bytes=%s path=%s\\n' \"$actual\" \"$expected\" \"$target\"",
"test \"$actual\" = \"$expected\"",
].join("\n"), 20_000, 10_000, true, target);
}
@@ -1028,25 +1054,29 @@ async function runRemoteBackground(label: string, script: string, timeoutMs: num
};
}
async function remoteApplyManifest(path: string, target = ciTarget(null)): Promise<void> {
async function remoteApplyManifest(config: UniDeskConfig, path: string, target = ciTarget(null)): Promise<void> {
const absolute = rootPath(path);
if (!existsSync(absolute)) throw new Error(`manifest not found: ${path}`);
const encoded = Buffer.from(readFileSync(absolute, "utf8"), "utf8").toString("base64");
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
const b64Path = `/tmp/unidesk-ci-apply-${token}.b64`;
const upload = await uploadRemoteBase64(b64Path, encoded, target);
if (!upload.ok) throw new Error(`failed to upload manifest ${path}: ${upload.stderr || upload.stdout}`);
const manifest = readFileSync(absolute, "utf8");
const encoded = Buffer.from(manifest, "utf8").toString("base64");
emitCiInstallProgress("upload-manifest", "started", { providerId: target.providerId, manifest: path, bytes: Buffer.byteLength(manifest) });
const script = [
"set -euo pipefail",
"set -eu",
...ciTargetGuardShellLines(target),
"tmp=$(mktemp /tmp/unidesk-ci-apply.XXXXXX.yaml)",
`b64_path=${shellQuote(b64Path)}`,
"trap 'rm -f \"$tmp\" \"$b64_path\"' EXIT",
"base64 -d \"$b64_path\" > \"$tmp\"",
"trap 'rm -f \"$tmp\"' EXIT",
"base64 -d > \"$tmp\" <<'UNIDESK_CI_MANIFEST_B64'",
encoded,
"UNIDESK_CI_MANIFEST_B64",
"test -s \"$tmp\"",
"printf 'manifest_bytes=%s\\n' \"$(wc -c < \"$tmp\" | tr -d ' ')\"",
"kubectl apply -f \"$tmp\"",
].join("\n");
const result = await runRemoteBackground(`apply-${path.split("/").pop() ?? "manifest"}`, script, 180_000, target);
if (!result.ok) throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`);
emitCiInstallProgress("kubectl-apply", "started", { providerId: target.providerId, manifest: path });
const result = await runSshCommandCapture(config, `${target.providerId}:k3s`, ["script"], script);
if (result.exitCode !== 0) throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`);
emitCiInstallProgress("upload-manifest", "succeeded", { providerId: target.providerId, manifest: path, upload: result.stdout.split(/\r?\n/u).find((line) => line.startsWith("manifest_bytes=")) ?? "" });
emitCiInstallProgress("kubectl-apply", "succeeded", { providerId: target.providerId, manifest: path });
}
async function prewarmCiRuntimeImages(target = ciTarget(null)): Promise<void> {
@@ -1070,8 +1100,9 @@ async function prewarmCiRuntimeImages(target = ciTarget(null)): Promise<void> {
"if ! printf '%s' \"$pause_entrypoint\" | grep -q '\"/pause\"'; then echo native_k3s_pause_image_invalid_entrypoint=$pause_entrypoint >&2; exit 1; fi",
"root_exec() {",
" if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return $?; fi",
" if command -v sudo >/dev/null 2>&1; then sudo \"$@\"; return $?; fi",
" if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then sudo \"$@\"; return $?; fi",
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return $?; fi",
" echo ci_runtime_image_containerd_root_required=true >&2",
" \"$@\"",
"}",
"containerd_images=$(root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls 2>/tmp/unidesk-ci-containerd-images.err || true)",
@@ -1100,7 +1131,23 @@ async function prewarmCiRuntimeImages(target = ciTarget(null)): Promise<void> {
`root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F ${shellQuote(`docker.io/library/${target.codeQueueImage}`)} >/dev/null`,
].join("\n");
const result = await runRemoteBackground("prewarm-runtime-images", script, 900_000, target);
if (!result.ok) throw new Error(`CI runtime image prewarm failed: ${result.stderr || result.stdout}`);
if (!result.ok) {
const combined = `${result.stdout}\n${result.stderr}`.trim();
const failureLines = extractCiLogFailureHints(combined).concat(
combined.split(/\r?\n/u).filter((line) => /ci_runtime_image_|containerd_|root_required|sudo/iu.test(line)).map(boundedHintLine).slice(-40),
);
throw new Error(`CI runtime image prewarm failed: ${JSON.stringify({
providerId: target.providerId,
exitCode: result.exitCode,
failureLines: Array.from(new Set(failureLines)).slice(-50),
stdoutTail: tailTextLines(result.stdout, 80),
stderrTail: tailTextLines(result.stderr, 80),
recovery: [
"If Tekton is already installed and only CI manifests need refreshing, rerun with: bun scripts/cli.ts ci install --skip-prewarm",
"If runtime helper images are missing from containerd, restore passwordless/root containerd import on the provider and rerun without --skip-prewarm.",
],
})}`);
}
}
async function status(target = ciTarget(null)): Promise<Record<string, unknown>> {
@@ -1126,27 +1173,135 @@ async function status(target = ciTarget(null)): Promise<Record<string, unknown>>
};
}
async function install(target = ciTarget(null)): Promise<Record<string, unknown>> {
if (!existsSync(rootPath(target.pipelineManifest))) {
async function install(config: UniDeskConfig, options: CiInstallOptions): Promise<Record<string, unknown>> {
if (!existsSync(rootPath(options.target.pipelineManifest))) {
throw new Error("CI manifests are missing");
}
await prewarmCiRuntimeImages(target);
const installTektonScript = [
"set -euo pipefail",
...ciTargetGuardShellLines(target),
`kubectl apply -f ${shellQuote(tektonPipelineReleaseUrl)}`,
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s",
`kubectl apply -f ${shellQuote(tektonTriggersReleaseUrl)}`,
`kubectl apply -f ${shellQuote(tektonTriggersInterceptorsUrl)}`,
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s",
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines-resolvers --timeout=900s",
].join("\n");
const installTekton = await runRemoteBackground("install-tekton", installTektonScript, 1_200_000, target);
if (!installTekton.ok) throw new Error(`Tekton install failed: ${installTekton.stderr || installTekton.stdout}`);
await remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/tekton-install.yaml", target);
await remoteApplyManifest(target.pipelineManifest, target);
await remoteApplyManifest("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.triggers.yaml", target);
return status(target);
emitCiInstallProgress("install", "started", {
providerId: options.target.providerId,
skipPrewarm: options.skipPrewarm,
skipTektonInstall: options.skipTektonInstall,
});
try {
if (options.skipPrewarm) {
emitCiInstallProgress("prewarm", "skipped", { providerId: options.target.providerId });
} else {
emitCiInstallProgress("prewarm", "started", { providerId: options.target.providerId });
await prewarmCiRuntimeImages(options.target);
emitCiInstallProgress("prewarm", "succeeded", { providerId: options.target.providerId });
}
if (options.skipTektonInstall) {
emitCiInstallProgress("install-tekton", "skipped", { providerId: options.target.providerId });
} else {
emitCiInstallProgress("install-tekton", "started", { providerId: options.target.providerId });
const installTektonScript = [
"set -euo pipefail",
...ciTargetGuardShellLines(options.target),
`kubectl apply -f ${shellQuote(tektonPipelineReleaseUrl)}`,
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s",
`kubectl apply -f ${shellQuote(tektonTriggersReleaseUrl)}`,
`kubectl apply -f ${shellQuote(tektonTriggersInterceptorsUrl)}`,
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s",
"kubectl wait --for=condition=Available deployment --all -n tekton-pipelines-resolvers --timeout=900s",
].join("\n");
const installTekton = await runRemoteBackground("install-tekton", installTektonScript, 1_200_000, options.target);
if (!installTekton.ok) throw new Error(`Tekton install failed: ${installTekton.stderr || installTekton.stdout}`);
emitCiInstallProgress("install-tekton", "succeeded", { providerId: options.target.providerId });
}
for (const manifest of [
"src/components/microservices/k3sctl-adapter/k3s/ci/tekton-install.yaml",
options.target.pipelineManifest,
"src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.triggers.yaml",
]) {
emitCiInstallProgress("apply-manifest", "started", { providerId: options.target.providerId, manifest });
await remoteApplyManifest(config, manifest, options.target);
emitCiInstallProgress("apply-manifest", "succeeded", { providerId: options.target.providerId, manifest });
}
emitCiInstallProgress("status", "started", { providerId: options.target.providerId });
const summary = await status(options.target);
emitCiInstallProgress("status", "succeeded", { providerId: options.target.providerId });
emitCiInstallProgress("install", "succeeded", { providerId: options.target.providerId });
return {
...summary,
install: {
providerId: options.target.providerId,
prewarm: options.skipPrewarm ? "skipped" : "completed",
tektonInstall: options.skipTektonInstall ? "skipped" : "completed",
},
};
} catch (error) {
emitCiInstallProgress("install", "failed", {
providerId: options.target.providerId,
errorTail: (error instanceof Error ? error.message : String(error)).slice(-1200),
});
throw error;
}
}
function ciInstallCommand(options: CiInstallOptions): string[] {
return [
"bun",
"scripts/cli.ts",
"ci",
"install",
"--provider-id",
options.target.providerId,
"--wait",
...(options.skipPrewarm ? ["--skip-prewarm"] : []),
...(options.skipTektonInstall ? ["--skip-tekton-install"] : []),
];
}
function installAsync(options: CiInstallOptions): Record<string, unknown> {
const command = ciInstallCommand(options);
const job = startJob(
"ci_install",
command,
`Install/refresh Tekton CI on ${options.target.providerId} native k3s`,
);
return {
ok: true,
mode: "async",
providerId: options.target.providerId,
skipped: {
prewarm: options.skipPrewarm,
tektonInstall: options.skipTektonInstall,
},
job: {
id: job.id,
status: job.status,
command: job.command,
stdoutFile: job.stdoutFile,
stderrFile: job.stderrFile,
note: job.note,
},
next: [
`bun scripts/cli.ts ci install-status ${job.id}`,
`bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
],
boundary: "ci install is fire-and-forget by default; use --wait only for explicit synchronous debugging.",
};
}
function latestCiInstallJobId(): string {
const job = listJobs().find((item) => item.name === "ci_install");
if (job === undefined) throw new Error("no ci_install job found");
return job.id;
}
function installStatus(id: string): Record<string, unknown> {
const jobId = id === "latest" || id.length === 0 ? latestCiInstallJobId() : id;
const job = readJob(jobId);
if (job.name !== "ci_install") {
throw new Error(`job ${jobId} is ${job.name}, not ci_install`);
}
return {
ok: job.status !== "failed" && job.status !== "canceled",
job: jobWithTail(job, 12_000),
next: job.status === "running" || job.status === "queued"
? [`bun scripts/cli.ts ci install-status ${job.id}`]
: ["bun scripts/cli.ts ci status"],
};
}
function pipelineRunManifest(options: CiOptions): string {
@@ -2893,6 +3048,10 @@ export function ciHelp(): Record<string, unknown> {
description: "Manage native k3s Tekton CI on D601 or G14. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.",
examples: [
"bun scripts/cli.ts ci install",
"bun scripts/cli.ts ci install --skip-prewarm",
"bun scripts/cli.ts ci install --skip-prewarm --skip-tekton-install",
"bun scripts/cli.ts ci install-status latest",
"bun scripts/cli.ts ci install --wait --skip-prewarm --skip-tekton-install",
"bun scripts/cli.ts ci install --provider-id G14",
"bun scripts/cli.ts ci run --revision <commit>",
"bun scripts/cli.ts ci run --provider-id G14 --revision <commit>",
@@ -2923,6 +3082,14 @@ export function ciHelp(): Record<string, unknown> {
},
},
},
install: {
defaultMode: "async-job",
waitFlag: "Use --wait only for explicit synchronous debugging; the default returns a .state/jobs job immediately with install-status follow-up.",
statusCommand: "bun scripts/cli.ts ci install-status <jobId|latest>",
prewarmDefault: true,
skipPrewarm: "Use --skip-prewarm only to refresh Tekton/CI manifests when runtime images are already present or prewarm is blocked by provider root/containerd permissions.",
skipTektonInstall: "Use --skip-tekton-install with --skip-prewarm only when Tekton is already installed and only UniDesk CI manifests/triggers need refreshing.",
},
backendCoreArtifact: {
producer: "D601 CI",
registry: "127.0.0.1:5000/unidesk/backend-core:<commit>",
@@ -2973,7 +3140,16 @@ function requireRunId(value: string): string {
export async function runCiCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
const [action = "status", nameArg] = args;
if (isHelpArg(action) || args.slice(1).some(isHelpArg)) return ciHelp();
if (action === "install") return install(ciTarget(providerIdOption(args)));
if (action === "install") {
const options = {
target: ciTarget(providerIdOption(args)),
skipPrewarm: boolFlag(args, "--skip-prewarm"),
skipTektonInstall: boolFlag(args, "--skip-tekton-install"),
wait: boolFlag(args, "--wait"),
};
return options.wait ? install(config, options) : installAsync(options);
}
if (action === "install-status") return installStatus(nameArg ?? "latest");
if (action === "status") return status(ciTarget(providerIdOption(args)));
if (action === "run") {
const target = ciTarget(providerIdOption(args));
@@ -3062,6 +3238,10 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi
}
export function startCiInstallJob(providerId = d601ProviderId): Record<string, unknown> {
const job = startJob("ci_install", ["bun", "scripts/cli.ts", "ci", "install", "--provider-id", providerId], `Install/refresh Tekton CI on ${providerId} native k3s`);
return { ok: true, job };
return installAsync({
target: ciTarget(providerId),
skipPrewarm: false,
skipTektonInstall: false,
wait: false,
});
}
scripts/src/jobs.ts
+54 -1
View File
@@ -223,12 +223,14 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
}
function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
const nowMs = Date.now();
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
const ciInstallWorkflow = job.name === "ci_install";
if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs);
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
const nowMs = Date.now();
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
@@ -357,6 +359,57 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
};
}
function summarizeCiInstallJobProgress(job: JobRecord, stderrTailOverride?: string, nowMs = Date.now()): JobProgressSummary {
const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);
const events = parseJsonLineEvents(stderrTail, "ci.install.progress")
.sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
const lastEvent = events.at(-1) ?? {};
const stage = stringField(lastEvent.stage);
const stageStatus = stringField(lastEvent.status);
const lastEventAt = stringField(lastEvent.at);
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
const warnings = jobProgressWarnings({
job,
eventsObserved: events.length,
elapsedSeconds,
stage,
stageStatus,
stageElapsedSeconds,
lastEventAgeSeconds,
});
return {
kind: "generic",
stage,
stageStatus,
sourceCommit: null,
pipelineRun: null,
pipelineCreated: null,
elapsedSeconds,
stageElapsedSeconds,
lastEventAt,
lastEventAgeSeconds,
eventsObserved: events.length,
slow: warnings.length > 0,
warnings,
timings: {},
summary: [
job.status,
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
typeof lastEvent.providerId === "string" ? `provider=${lastEvent.providerId}` : null,
typeof lastEvent.manifest === "string" ? `manifest=${lastEvent.manifest}` : null,
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
warnings.length > 0 ? "visibility-warning" : null,
].filter(Boolean).join(" "),
nextCommand: job.status === "running"
? `bun scripts/cli.ts ci install-status ${job.id}`
: "bun scripts/cli.ts ci status",
};
}
function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
const events = parseJsonLineEvents(stderrTail, "hwlab.runtime-lane.trigger.progress")
.sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));