import { randomUUID } from "node:crypto"; import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, posix as posixPath } from "node:path"; import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "./ci-catalog"; import { runCommand } from "./command"; import { type UniDeskConfig, repoRoot, rootPath } from "./config"; import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "./deploy-ssh-identity"; import { jobWithTail, listJobs, readJob, startJob } from "./jobs"; import { coreInternalFetch } from "./microservices"; import { artifactRegistryReadonlyResultFromCommand, buildArtifactRegistryReadonlyProbe, parseArtifactRegistryOptions, type ArtifactRegistryReadonlyProbe, } from "./artifact-registry"; import { d601K3sGuardShellLines, d601NativeKubeconfig } from "./d601-k3s-guard"; import { runSshCommandCapture } from "./ssh"; const d601ProviderId = "D601"; const d601Kubeconfig = d601NativeKubeconfig; const tektonPipelineVersion = "v1.12.0"; const tektonTriggersVersion = "v0.34.0"; const tektonPipelineReleaseUrl = `https://infra.tekton.dev/tekton-releases/pipeline/previous/${tektonPipelineVersion}/release.yaml`; const tektonTriggersReleaseUrl = `https://infra.tekton.dev/tekton-releases/triggers/previous/${tektonTriggersVersion}/release.yaml`; const tektonTriggersInterceptorsUrl = `https://infra.tekton.dev/tekton-releases/triggers/previous/${tektonTriggersVersion}/interceptors.yaml`; const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789"; const ciCodeQueueImage = "unidesk-code-queue:dev"; const codeQueueDirectDockerBaseImage = "unidesk-code-queue:d601"; const providerDispatchCompletionLagMs = 45_000; const defaultCiPipelineManifest = "src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml"; const g14CiPipelineManifest = "src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.g14.yaml"; const ciRuntimeImages = [ "rancher/mirrored-pause:3.6", "rancher/mirrored-library-busybox:1.36.1", "cgr.dev/chainguard/busybox@sha256:19f02276bf8dbdd62f069b922f10c65262cc34b710eea26ff928129a736be791", "ghcr.io/tektoncd/pipeline/entrypoint-bff0a22da108bc2f16c818c97641a296:v1.12.0", "ghcr.io/tektoncd/pipeline/workingdirinit-0c558922ec6a1b739e550e349f2d5fc1:v1.12.0", "ghcr.io/tektoncd/pipeline/nop-8eac7c133edad5df719dc37b36b62482:v1.12.0", "ghcr.io/tektoncd/pipeline/events-a9042f7efb0cbade2a868a1ee5ddd52c:v1.12.0", "ghcr.io/tektoncd/triggers/eventlistenersink-7ad1faa98cddbcb0c24990303b220bb8:v0.34.0", "oven/bun:1-debian", "alpine/git:2.45.2", ciCodeQueueImage, ]; function ciTarget(providerId: string | null): CiTarget { const normalized = providerId ?? d601ProviderId; if (normalized === d601ProviderId) { return { providerId: d601ProviderId, kubeconfig: d601Kubeconfig, hostCwd: "/home/ubuntu", homeDir: "/home/ubuntu", pipelineManifest: defaultCiPipelineManifest, codeQueueImage: ciCodeQueueImage, guardName: "d601_native_k3s_guard", requiredNodeName: "d601", }; } if (normalized === "G14") { return { providerId: "G14", kubeconfig: d601Kubeconfig, hostCwd: "/root", homeDir: "/root", pipelineManifest: g14CiPipelineManifest, codeQueueImage: ciCodeQueueImage, guardName: "g14_native_k3s_guard", requiredNodeLabel: { key: "unidesk.ai/node-id", value: "G14" }, }; } throw new Error(`ci --provider-id currently supports D601 or G14, got ${normalized}`); } function providerIdOption(args: string[]): string | null { return stringOption(args, "--provider-id") ?? stringOption(args, "--provider"); } interface CiOptions { repoUrl: string; revision: string; waitMs: number; target: CiTarget; } interface CiInstallOptions { target: CiTarget; skipPrewarm: boolean; skipTektonInstall: boolean; wait: boolean; } interface CiTarget { providerId: string; kubeconfig: string; hostCwd: string; homeDir: string; pipelineManifest: string; codeQueueImage: string; guardName: string; requiredNodeName?: string; requiredNodeLabel?: { key: string; value: string }; } interface CiPublishBackendCoreOptions { repoUrl: string; commit: string; waitMs: number; sourceHostPath: string; dockerfile: string; imageRepository: string; dryRun: boolean; } interface CiPublishUserServiceArtifactOptions { repoUrl: string; commit: string; waitMs: number; sourceHostPath: string; serviceId: string; dockerfile: string; imageRepository: string; dryRun: boolean; transport: CiPublishTransport; } type CiPublishTransport = "auto" | "tekton" | "direct-docker"; interface CiDevE2EOptions { repoUrl: string; desiredRef: string; deployCommit: string; environment: "dev"; scriptRepo: string; scriptPath: string; scriptTimeoutMs: number; services: Array<{ id: string; commitId: string; repo: string }>; runId: string; keepNamespace: boolean; waitMs: number; } interface DispatchResult { ok: boolean; taskId: string | null; status: string | null; stdout: string; stderr: string; exitCode: number | null; raw: unknown; } interface CiLogsOptions { tailLines: number; capture: "dispatch-task" | "ssh-stream"; } interface CiCleanupRunsOptions { target: CiTarget; minAgeMinutes: number; limit: number; confirm: boolean; } interface CiCleanupFailedPodsOptions { target: CiTarget; namespace: string; minAgeMinutes: number; limit: number; confirm: boolean; } type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible" | "ci-runner-not-ready"; type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry"; type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry"; interface PublishPreflightChannelProbe { channel: PublishPreflightDetailedChannel; controlChannel: PublishPreflightControlChannel; ok: boolean; requiredFor: string; detail: unknown; } interface PublishPreflightControlChannelProbe { channel: PublishPreflightControlChannel; ok: boolean; requiredFor: string[]; probes: PublishPreflightDetailedChannel[]; } interface PublishPreflight { ok: boolean; runnerDisposition: "ready" | "infra-blocked"; failureClassification: PublishPreflightFailureClassification | null; serviceId: string; commit: string; providerId: string; supportedArtifactPublish: boolean; missingChannels: string[]; missingControlChannels: PublishPreflightControlChannel[]; controlChannels: PublishPreflightControlChannelProbe[]; channels: PublishPreflightChannelProbe[]; registry: unknown; controlPlane: Record; recommendedAction: string; remoteCommandShape: string; next: string[]; boundary: string; } export interface PublishPreflightTransport { kind?: "local-docker" | "remote-frontend" | "provider-tunnel"; remoteHost?: string | null; coreFetch: (path: string, init?: { method?: string; body?: unknown; maxResponseBytes?: number }) => unknown | Promise; dispatchHostSsh: (command: string, waitMs: number, remoteTimeoutMs: number) => Promise; commandCwd: string; artifactRegistryCommand: (probe: ArtifactRegistryReadonlyProbe) => string[]; } interface PipelineRunCondition { ok: boolean | null; status: string; reason: string; message: string; query: { ok: boolean; status: string | null; exitCode: number | null; stdoutTail: string; stderrTail: string; }; } interface DeployDevManifestSummary { deployCommit: string; desiredRef: string; environment: "dev"; ci: { repo: string; scriptPath: string; timeoutMs: number; }; services: Array<{ id: string; commitId: string; repo: string }>; } interface ArtifactSummary { serviceId: string; sourceCommit: string; sourceRepo: string; dockerfile: string; registry: string; repository: string; tag: string; imageRef: string; digest: string | null; digestRef: string | null; } interface ArtifactSummaryContext { serviceId: string; commit: string; repoUrl: string; dockerfile: string; imageRepository: string; } function stringOption(args: string[], name: string): string | null { const index = args.indexOf(name); if (index === -1) return null; const value = args[index + 1]; if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } function numberOption(args: string[], name: string, fallback: number): number { const raw = stringOption(args, name); if (raw === null) return fallback; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return value; } function ciLogsOptions(args: string[]): CiLogsOptions { const rawTail = stringOption(args, "--tail-lines") ?? stringOption(args, "--tail"); const tailLines = rawTail === null ? 160 : Number(rawTail); if (!Number.isInteger(tailLines) || tailLines <= 0 || tailLines > 2000) { throw new Error("ci logs --tail-lines must be an integer between 1 and 2000"); } return { tailLines, capture: args.includes("--dispatch-task") ? "dispatch-task" : "ssh-stream" }; } function ciCleanupRunsOptions(args: string[]): CiCleanupRunsOptions { const limit = numberOption(args, "--limit", 50); if (limit <= 0 || limit > 500) throw new Error("ci cleanup-runs --limit must be an integer between 1 and 500"); const confirm = boolFlag(args, "--confirm"); if (confirm && boolFlag(args, "--dry-run")) throw new Error("ci cleanup-runs accepts only one of --confirm or --dry-run"); return { target: ciTarget(providerIdOption(args)), minAgeMinutes: numberOption(args, "--min-age-minutes", 60), limit, confirm, }; } function ciCleanupFailedPodsOptions(args: string[]): CiCleanupFailedPodsOptions { const limit = numberOption(args, "--limit", 50); if (limit <= 0 || limit > 500) throw new Error("ci cleanup-failed-pods --limit must be an integer between 1 and 500"); const confirm = boolFlag(args, "--confirm"); if (confirm && boolFlag(args, "--dry-run")) throw new Error("ci cleanup-failed-pods accepts only one of --confirm or --dry-run"); return { target: ciTarget(providerIdOption(args)), namespace: requireK8sName(stringOption(args, "--namespace") ?? "unidesk-ci", "--namespace"), minAgeMinutes: numberOption(args, "--min-age-minutes", 60), limit, confirm, }; } function requireK8sName(value: string, option: string): string { if (!/^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$/u.test(value)) { throw new Error(`${option} must be a Kubernetes DNS label`); } return value; } function countTextLines(value: string): number { if (value.length === 0) return 0; const breaks = value.match(/\r\n|\r|\n/gu)?.length ?? 0; return breaks + (/(\r\n|\r|\n)$/u.test(value) ? 0 : 1); } function tailTextLines(value: string, limit: number): string { if (value.length === 0) return ""; return value.split(/\r?\n/u).slice(-limit).join("\n"); } function boundedHintLine(line: string): string { const normalized = line.trim(); if (normalized.length <= 1000) return normalized; return `${normalized.slice(0, 1000)}`; } function boundedHeadTail(value: string, headChars: number, tailChars: number): string { if (value.length <= headChars + tailChars + 80) return value; return `${value.slice(0, headChars)}\n...[omitted ${value.length - headChars - tailChars} chars]...\n${value.slice(-tailChars)}`; } function extractCiLogConditionHints(value: string): string[] { return value .split(/\r?\n/u) .filter((line) => /^(condition=|taskrun=)/u.test(line.trim())) .map(boundedHintLine) .filter((line) => line.length > 0) .slice(-30); } function extractCiLogFailureHints(value: string): string[] { const failurePattern = /\b(error|failed|failure|stepfailed|exit code|exited|denied|forbidden|unauthorized|timeout|timed out|panic|exception|could not|cannot|no such|killed|oom|back-off|imagepull|errimagepull|invalid|not found)\b/iu; return value .split(/\r?\n/u) .filter((line) => failurePattern.test(line)) .map(boundedHintLine) .filter((line) => line.length > 0) .slice(-40); } function requireRevision(value: string | null): string { if (value === null || value.length === 0) throw new Error("ci run requires --revision "); if (!/^[A-Za-z0-9._/@:-]{1,160}$/u.test(value) || value.startsWith("-") || value.includes("..")) throw new Error("ci --revision contains unsupported characters"); return value; } function requireFullCommit(value: string | null, option = "--commit"): string { const commit = value?.toLowerCase() ?? ""; if (!/^[0-9a-f]{40}$/u.test(commit)) throw new Error(`${option} requires a full 40-character pushed Git commit SHA`); return commit; } function requireServiceId(value: string | null): string { const serviceId = value ?? ""; if (!/^[a-z0-9]([-a-z0-9]{0,62}[a-z0-9])?$/u.test(serviceId)) { throw new Error("ci publish-user-service requires --service "); } return serviceId; } function requireDesiredRef(value: string | null): string { const ref = value ?? "master"; if (!/^[A-Za-z0-9._/-]{1,160}$/u.test(ref) || ref.startsWith("-") || ref.includes("..")) { throw new Error("ci run-dev-e2e --desired-ref contains unsupported characters"); } return ref; } function boolFlag(args: string[], name: string): boolean { return args.includes(name); } function publishTransportOption(value: string | null): CiPublishTransport { if (value === null || value === "auto") return "auto"; if (value === "tekton" || value === "direct-docker") return value; throw new Error("ci publish-user-service --transport must be one of: auto, tekton, direct-docker"); } function isHelpArg(value: string | undefined): boolean { return value === "help" || value === "--help" || value === "-h"; } function shellQuote(value: string): string { return `'${value.replace(/'/gu, "'\\''")}'`; } function ciTargetGuardShellLines(target: CiTarget, options: { passOutput?: "stdout" | "stderr" | "quiet" } = {}): string[] { if (target.providerId === d601ProviderId) { return d601K3sGuardShellLines(target.kubeconfig, options); } const passOutput = options.passOutput ?? "stdout"; const labelSelector = target.requiredNodeLabel === undefined ? "" : `${target.requiredNodeLabel.key}=${target.requiredNodeLabel.value}`; const passLine = passOutput === "quiet" ? ":" : passOutput === "stderr" ? `printf '${target.guardName}=pass kubeconfig=%s context=%s server=%s nodes=%s\\n' "$required_kubeconfig" "$context" "$server" "$(printf '%s' "$nodes" | tr '\\n' ',')" >&2` : `printf '${target.guardName}=pass kubeconfig=%s context=%s server=%s nodes=%s\\n' "$required_kubeconfig" "$context" "$server" "$(printf '%s' "$nodes" | tr '\\n' ',')"`; return [ `export KUBECONFIG=${shellQuote(target.kubeconfig)}`, "unidesk_ci_k3s_guard() {", ` required_kubeconfig=${shellQuote(target.kubeconfig)}`, ` required_node=${shellQuote(target.requiredNodeName ?? "")}`, ` required_selector=${shellQuote(labelSelector)}`, ` guard_name=${shellQuote(target.guardName)}`, " if [ \"${KUBECONFIG:-}\" != \"$required_kubeconfig\" ]; then", " printf '%s=blocked reason=wrong-kubeconfig expected=%s actual=%s\\n' \"$guard_name\" \"$required_kubeconfig\" \"${KUBECONFIG:-}\" >&2", " return 1", " fi", " if ! command -v kubectl >/dev/null 2>&1; then", " printf '%s=blocked reason=kubectl-missing\\n' \"$guard_name\" >&2", " return 1", " fi", " if ! context=$(kubectl config current-context 2>&1); then", " printf '%s=blocked reason=context-read-failed detail=%s\\n' \"$guard_name\" \"$context\" >&2", " return 1", " fi", " if ! server=$(kubectl config view --minify -o 'jsonpath={.clusters[0].cluster.server}' 2>&1); then", " printf '%s=blocked reason=server-read-failed detail=%s\\n' \"$guard_name\" \"$server\" >&2", " return 1", " fi", " if ! nodes=$(kubectl get nodes -o 'jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}' 2>&1); then", " printf '%s=blocked reason=nodes-read-failed detail=%s\\n' \"$guard_name\" \"$nodes\" >&2", " return 1", " fi", " combined=$(printf '%s\\n%s\\n%s\\n' \"$context\" \"$server\" \"$nodes\")", " if printf '%s\\n' \"$combined\" | grep -Eiq 'docker-desktop|desktop-control-plane|127\\.0\\.0\\.1:11700'; then", " printf '%s=refused reason=forbidden-control-plane context=%s server=%s nodes=%s\\n' \"$guard_name\" \"$context\" \"$server\" \"$(printf '%s' \"$nodes\" | tr '\\n' ',')\" >&2", " return 1", " fi", " if [ -n \"$required_node\" ] && ! printf '%s\\n' \"$nodes\" | grep -Fx \"$required_node\" >/dev/null; then", " printf '%s=blocked reason=missing-required-node expected=%s nodes=%s\\n' \"$guard_name\" \"$required_node\" \"$(printf '%s' \"$nodes\" | tr '\\n' ',')\" >&2", " return 1", " fi", " if [ -n \"$required_selector\" ]; then", " if ! labeled_nodes=$(kubectl get nodes -l \"$required_selector\" -o 'jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}' 2>&1); then", " printf '%s=blocked reason=label-query-failed selector=%s detail=%s\\n' \"$guard_name\" \"$required_selector\" \"$labeled_nodes\" >&2", " return 1", " fi", " if [ -z \"$(printf '%s' \"$labeled_nodes\" | tr -d '\\n')\" ]; then", " printf '%s=blocked reason=missing-required-label selector=%s nodes=%s\\n' \"$guard_name\" \"$required_selector\" \"$(printf '%s' \"$nodes\" | tr '\\n' ',')\" >&2", " return 1", " fi", " nodes=\"$labeled_nodes\"", " fi", ` ${passLine}`, "}", "unidesk_ci_k3s_guard", ]; } function safePathToken(value: string): string { return value.replace(/[^a-z0-9._-]/giu, "-").toLowerCase().replace(/^-+|-+$/gu, "").slice(0, 80) || "artifact"; } function repoSshUrl(repoUrl: string): string { if (repoUrl.startsWith("git@")) return repoUrl; if (repoUrl.startsWith("https://github.com/")) { const repoPath = repoUrl.slice("https://github.com/".length).replace(/\.git$/u, ""); return `git@github.com:${repoPath}.git`; } return repoUrl; } function repoNeedsGithubSshIdentity(repoFetchUrl: string): boolean { return repoFetchUrl.startsWith("git@github.com:"); } function repoConnectivityProbeUrl(repoFetchUrl: string): string { const sshMatch = /^git@([^:]+):/u.exec(repoFetchUrl); if (sshMatch !== null) return `https://${sshMatch[1]}`; try { const parsed = new URL(repoFetchUrl); return `${parsed.protocol}//${parsed.host}`; } catch { return repoFetchUrl.startsWith("https://gitee.com/") ? "https://gitee.com" : "https://github.com"; } } function requireRepoRelativePath(path: string, label: string): string { if (path.length === 0 || path.startsWith("/") || path.includes("\0") || path.split("/").includes("..")) { throw new Error(`${label} must be a repo-relative path`); } const normalized = posixPath.normalize(path); if (normalized === "." || normalized.startsWith("../")) throw new Error(`${label} must be a repo-relative path`); return normalized; } function resolveCatalogArtifact(serviceId: string): CiCatalogArtifact { const artifact = findCiCatalogArtifact(serviceId); if (artifact === null) { const known = loadCiCatalog().artifacts.map((item) => item.serviceId).sort().join(", "); throw new Error(`unknown CI artifact service: ${serviceId}. Known services: ${known}`); } return artifact; } function blockedArtifactResult(artifact: CiUpstreamImageCatalogArtifact | CiSourceBuildCatalogArtifact, commit: string, note: string): Record { const base = { ok: false, status: "blocked", error: "blocked", serviceId: artifact.serviceId, commit, reason: note, catalogArtifact: artifact, boundary: "CI catalog marks this service as blocked; it must not be treated as a source-build artifact producer", }; return artifact.kind === "upstream-image" ? { ...base, upstream: artifact.upstream, next: [ `document the upstream image contract in CI.json for ${artifact.serviceId}`, ], } : { ...base, next: [ `unblock ${artifact.serviceId} in CI.json before attempting source-build publication`, ], }; } function blockedReason(artifact: CiSourceBuildCatalogArtifact): string { if (artifact.blockedReason === undefined) throw new Error(`${artifact.serviceId} is blocked in CI.json but has no blockedReason`); return artifact.blockedReason; } function userServicePublishBoundaryBlock( config: UniDeskConfig, serviceId: string, commit: string, artifact: CiSourceBuildCatalogArtifact, ): Record | null { const configService = config.microservices.find((item) => item.id === serviceId); if (configService === undefined) return null; const isD601K3sService = configService.providerId === d601ProviderId && configService.development.providerId === d601ProviderId && configService.deployment.mode === "k3sctl-managed"; const isD601DirectService = configService.providerId === d601ProviderId && configService.development.providerId === d601ProviderId && configService.deployment.mode === "unidesk-direct"; const isMainServerDirectService = configService.providerId === "main-server" && configService.development.providerId === "main-server" && configService.deployment.mode === "unidesk-direct"; const isMainServerInternalSidecar = configService.providerId === "main-server" && configService.development.providerId === "main-server" && configService.deployment.mode === "internal-sidecar"; if (isD601K3sService || isD601DirectService || isMainServerDirectService || isMainServerInternalSidecar) return null; return { ok: false, status: "blocked", error: "blocked", serviceId, commit, reason: `config.json marks ${serviceId} as ${configService.providerId}/${configService.deployment.mode}, which is outside the reviewed CI artifact producer boundary`, catalogArtifact: artifact, configService: { providerId: configService.providerId, deploymentMode: configService.deployment.mode, }, }; } function chunks(value: string, size: number): string[] { const result: string[] = []; for (let index = 0; index < value.length; index += size) { result.push(value.slice(index, index + size)); } return result; } const hostSshBase64UploadChunkChars = 3000; function asRecord(value: unknown): Record | null { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; } function asString(value: unknown): string { return typeof value === "string" ? value : ""; } function coreBody(response: unknown): Record | null { return asRecord(asRecord(response)?.body); } function emitCiInstallProgress(stage: string, status: "started" | "skipped" | "succeeded" | "failed", detail: Record = {}): void { console.error(JSON.stringify({ event: "ci.install.progress", at: new Date().toISOString(), stage, status, ...detail, })); } function responseOk(response: unknown): boolean { if (typeof response !== "object" || response === null) return false; const record = response as Record; if ("ok" in record && record.ok === false) return false; const body = asRecord(record.body); if (body !== null && "ok" in body && body.ok === false) return false; return true; } function channelProbe( channel: PublishPreflightDetailedChannel, controlChannel: PublishPreflightControlChannel, ok: boolean, requiredFor: string, detail: unknown, ): PublishPreflightChannelProbe { return { channel, controlChannel, ok, requiredFor, detail }; } const publishPreflightControlChannelOrder: PublishPreflightControlChannel[] = ["backend-core", "database", "provider", "registry"]; function summarizePublishControlChannels(channels: PublishPreflightChannelProbe[]): PublishPreflightControlChannelProbe[] { return publishPreflightControlChannelOrder.map((channel) => { const probes = channels.filter((item) => item.controlChannel === channel); return { channel, ok: probes.length > 0 && probes.every((item) => item.ok), requiredFor: Array.from(new Set(probes.map((item) => item.requiredFor))), probes: probes.map((item) => item.channel), }; }); } function backendCoreUnavailable(value: unknown): boolean { const record = asRecord(value); const body = coreBody(value); if (record?.runnerDisposition === "infra-blocked") return true; if (record?.failureKind === "target-stack-not-running") return true; if (body?.runnerDisposition === "infra-blocked") return true; if (body?.failureKind === "target-stack-not-running") return true; if (body?.degradedReason === "backend-core-container-missing") return true; const text = JSON.stringify(value) ?? ""; return text.includes("No such container: unidesk-backend-core") || text.includes("No such container: unidesk-database"); } function transportKind(transport: PublishPreflightTransport): "local-docker" | "remote-frontend" | "provider-tunnel" { return transport.kind ?? "local-docker"; } function classifyPublishPreflightFailure( transport: PublishPreflightTransport, overview: unknown, sshProbe: DispatchResult, registry: unknown, ): PublishPreflightFailureClassification | null { const kind = transportKind(transport); const overviewRecord = asRecord(overview); if (overviewRecord?.failureClassification === "auth-missing") return "auth-missing"; if (overviewRecord?.failureClassification === "remote-proxy-missing") return "remote-proxy-missing"; if (overviewRecord?.failureClassification === "provider-unreachable") return "provider-unreachable"; if (kind === "local-docker" && backendCoreUnavailable(overview)) return "local-docker-required"; if (kind !== "local-docker" && !responseOk(overview)) return "remote-proxy-missing"; if (!sshProbe.ok) return "provider-unreachable"; const registryRecord = asRecord(registry); if (registryRecord?.failureClassification === "local-docker-required") return "local-docker-required"; if (registryRecord?.failureClassification === "provider-ssh-command-missing") return "provider-unreachable"; if (registryRecord?.failureClassification === "ssh-helper-command-shape-incompatible") return "ssh-helper-command-shape-incompatible"; if (registryRecord?.failureClassification === "remote-command-timeout") return "remote-command-timeout"; if (registryRecord?.failureClassification === "registry-not-installed") return "registry-not-installed"; if (registryRecord?.failureClassification === "registry-unhealthy") return "registry-unhealthy"; if (registryRecord?.ok === false) return kind === "local-docker" ? "provider-unreachable" : "registry-unhealthy"; return null; } function recommendedPublishPreflightAction( failureClassification: PublishPreflightFailureClassification | null, registry: unknown, missingControlChannels: PublishPreflightControlChannel[], ): string { if (failureClassification === null && missingControlChannels.length === 0) return "none"; const registryRecommendedAction = asString(asRecord(registry)?.recommendedAction); if (registryRecommendedAction.length > 0 && registryRecommendedAction !== "none") return registryRecommendedAction; if (failureClassification === "local-docker-required") return "rerun this read-only preflight through --main-server-ip or from a main-server CLI with backend-core available"; if (failureClassification === "auth-missing") return "restore frontend control-plane authentication before rerunning the dry-run preflight"; if (failureClassification === "remote-proxy-missing") return "restore frontend to backend-core proxy reachability before rerunning the dry-run preflight"; if (failureClassification === "provider-unreachable") return "restore D601 provider-gateway host.ssh reachability before rerunning the dry-run preflight"; if (failureClassification === "ssh-helper-command-shape-incompatible") return "upgrade or shorten the host.ssh readonly command shape before rerunning the dry-run preflight"; if (failureClassification === "remote-command-timeout") return "rerun artifact-registry health and inspect D601 host.ssh latency if the timeout repeats"; if (failureClassification === "registry-not-installed") return "install the D601 artifact registry through the controlled artifact-registry install path, then rerun health"; if (failureClassification === "registry-unhealthy") return "restore the D601 artifact registry service/container/API until artifact-registry health passes"; if (failureClassification === "ci-runner-not-ready") return "restore D601 unidesk-ci Tekton runner prerequisites before authorizing backend-core artifact publication"; if (missingControlChannels.includes("registry")) return "restore or install the D601 artifact registry until artifact-registry health passes"; return `restore missing control channel(s): ${missingControlChannels.join(", ") || "unknown"}`; } function publishPreflightControlPlane( transport: PublishPreflightTransport, failureClassification: PublishPreflightFailureClassification | null, remoteCommandShape = "bun scripts/cli.ts --main-server-ip ci publish-user-service --service --commit --dry-run", ): Record { const kind = transportKind(transport); const remoteCapable = kind !== "local-docker"; return { transport: kind, remoteCapable, remoteHost: transport.remoteHost ?? null, localDockerRequired: kind === "local-docker", failureClassification, preferredRunnerPath: "frontend-private-proxy", fallbackRunnerPath: "main-server-local-docker", remoteCommandShape, providerTunnel: { providerId: d601ProviderId, command: "host.ssh", requiredFor: ["provider-host-ssh", "artifact-registry"], }, }; } function publishPreflightFailedScopes(preflight: PublishPreflight): string[] { if (preflight.ok) return []; const registryScopes = asRecord(preflight.registry)?.failedScopes; if (Array.isArray(registryScopes)) return registryScopes.map(String); return preflight.missingChannels; } function ciRunnerPreflightScript(sourceHostPath: string): string { return [ "set -eu", ...d601K3sGuardShellLines(d601Kubeconfig), "printf 'provider_host_ssh=ok\\n'", "printf 'kubectl='", "command -v kubectl >/dev/null && printf 'ok\\n' || { printf 'missing\\n'; exit 127; }", "printf 'docker='", "command -v docker >/dev/null && printf 'ok\\n' || { printf 'missing\\n'; exit 127; }", "printf 'docker_socket='", "if [ -S /var/run/docker.sock ] || [ -S /run/docker.sock ]; then printf 'true\\n'; else printf 'false\\n'; fi", "printf 'namespace='", "kubectl get namespace unidesk-ci >/dev/null 2>&1 && printf 'true\\n' || printf 'false\\n'", "printf 'tekton_pipeline='", "kubectl get pipeline/unidesk-backend-core-artifact-publish -n unidesk-ci >/dev/null 2>&1 && printf 'true\\n' || printf 'false\\n'", "printf 'tekton_task='", "kubectl get task/unidesk-backend-core-artifact-publish -n unidesk-ci >/dev/null 2>&1 && printf 'true\\n' || printf 'false\\n'", "printf 'service_account='", "kubectl get serviceaccount/unidesk-ci-runner -n unidesk-ci >/dev/null 2>&1 && printf 'true\\n' || printf 'false\\n'", "printf 'pvc='", "kubectl get pvc/unidesk-ci-cache -n unidesk-ci >/dev/null 2>&1 && printf 'true\\n' || printf 'false\\n'", "printf 'source_parent_directory='", `test -d ${shellQuote(posixPath.dirname(sourceHostPath))} && printf 'true\\n' || printf 'false\\n'`, ].join("\n"); } function keyValueBool(stdout: string, key: string): boolean { const match = new RegExp(`^${key}=(.*)$`, "mu").exec(stdout); if (match === null) return false; const value = match[1]?.trim().toLowerCase() ?? ""; return value === "true" || value === "ok" || value === "pass" || value.startsWith("pass "); } function backendCoreCiRunnerReady(result: DispatchResult): boolean { return result.ok && keyValueBool(result.stdout, "d601_native_k3s_guard") && keyValueBool(result.stdout, "kubectl") && keyValueBool(result.stdout, "docker") && keyValueBool(result.stdout, "namespace") && keyValueBool(result.stdout, "tekton_pipeline") && keyValueBool(result.stdout, "tekton_task") && keyValueBool(result.stdout, "service_account") && keyValueBool(result.stdout, "pvc") && keyValueBool(result.stdout, "source_parent_directory"); } function backendCoreArtifactRequirements(options: CiPublishBackendCoreOptions, plannedArtifact: ArtifactSummary): Record { return { requiredLabels: { "unidesk.ai/service-id": "backend-core", "unidesk.ai/source-commit": options.commit, "unidesk.ai/source-repo": options.repoUrl, "unidesk.ai/dockerfile": options.dockerfile, }, digest: { requiredAfterPublish: true, dryRunValue: plannedArtifact.digest, fields: ["artifactSummary.digest", "artifactSummary.digestRef"], source: "D601 registry manifest HEAD after ci publish-backend-core succeeds", }, imageRef: plannedArtifact.imageRef, }; } function backendCoreDevApplyPath(options: CiPublishBackendCoreOptions, plannedArtifact: ArtifactSummary): Record { return { environment: "dev", pullOnly: true, apply: `bun scripts/cli.ts deploy apply --env dev --service backend-core --commit ${options.commit}`, dryRun: `bun scripts/cli.ts deploy apply --env dev --service backend-core --commit ${options.commit} --dry-run`, sourceImage: plannedArtifact.imageRef, forbidden: ["--env prod", "cargo build", "docker build", "docker compose build", "server rebuild backend-core"], note: "PROD is intentionally not part of backend-core publish preflight; dev apply still needs explicit authorization after artifact publication.", }; } function backendCoreRemoteCommandShape(commit: string): string { return `bun scripts/cli.ts --main-server-ip ci publish-backend-core --commit ${commit} --dry-run`; } function dispatchPreflightFailure(command: string, result: DispatchResult): DispatchResult { return { ok: false, taskId: result.taskId, status: result.status, stdout: result.stdout.slice(-4000), stderr: result.stderr.slice(-4000), exitCode: result.exitCode, raw: { command, taskId: result.taskId, status: result.status, exitCode: result.exitCode, stderrTail: result.stderr.slice(-1200), stdoutTail: result.stdout.slice(-1200), raw: result.raw, }, }; } function commandResultFromDispatch(command: string[], cwd: string, result: DispatchResult) { return { command, cwd, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr, signal: null, timedOut: result.status === "timeout", }; } async function dispatchReadonlySsh(command: string, waitMs: number, remoteTimeoutMs: number): Promise { try { const result = await dispatchSsh(command, waitMs, remoteTimeoutMs); if (!result.ok && backendCoreUnavailable(result.raw)) { return { ...result, status: "infra-blocked", stderr: "backend-core bridge unavailable while dispatching readonly SSH task", }; } return result; } catch (error) { return { ok: false, taskId: null, status: null, stdout: "", stderr: error instanceof Error ? error.message : String(error), exitCode: null, raw: error instanceof Error ? { name: error.name, message: error.message, stack: error.stack ?? null } : String(error), }; } } function artifactRegistryProbeCommand(probe: ArtifactRegistryReadonlyProbe): string[] { return [process.execPath, "scripts/cli.ts", "ssh", probe.providerId, "argv", "bash", "-lc", probe.script]; } const localPublishPreflightTransport: PublishPreflightTransport = { kind: "local-docker", coreFetch: (path, init) => coreInternalFetch(path, init), dispatchHostSsh: dispatchReadonlySsh, commandCwd: repoRoot, artifactRegistryCommand: artifactRegistryProbeCommand, }; function positiveManifestNumber(value: unknown, fallback: number, path: string): number { if (value === undefined || value === null) return fallback; if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`); return value; } function requireManifestString(value: unknown, path: string): string { if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`); return value; } function requireCiScriptPath(value: unknown): string { const scriptPath = requireManifestString(value, "environments.dev.ci.scriptPath"); if (!scriptPath.startsWith("scripts/ci/") || scriptPath.includes("..") || scriptPath.startsWith("/") || !scriptPath.endsWith(".sh")) { throw new Error("environments.dev.ci.scriptPath must be a repo-relative scripts/ci/*.sh path"); } return scriptPath; } async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number, pollCompletion = true, target = ciTarget(null)): Promise { const dispatchResponse = coreInternalFetch("/api/dispatch", { method: "POST", timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000), body: { providerId: target.providerId, command: "host.ssh", payload: { source: "ci-cli", mode: "exec", command, timeoutMs: remoteTimeoutMs, cwd: target.hostCwd, }, }, }); const dispatchBody = coreBody(dispatchResponse); const taskId = asString(dispatchBody?.taskId); if (dispatchBody?.ok !== true || taskId.length === 0) { const failureKind = asRecord(dispatchResponse)?.timedOut === true ? "backend-core-dispatch-timeout" : "backend-core-dispatch-submit-failed"; return { ok: false, taskId: taskId || null, status: null, stdout: "", stderr: asString(dispatchBody?.error) || `${failureKind}: dispatch did not return a task id`, exitCode: null, raw: { failureKind, dispatchResponse, timeoutMs: Math.min(Math.max(waitMs, 15_000), 45_000), }, }; } if (!pollCompletion) { return { ok: true, taskId, status: "submitted", stdout: "", stderr: "", exitCode: null, raw: dispatchBody, }; } const effectiveWaitMs = Math.max(waitMs, Math.min(remoteTimeoutMs + providerDispatchCompletionLagMs, 120_000)); const deadline = Date.now() + Math.max(effectiveWaitMs, 1_000); let latest: unknown = null; while (Date.now() < deadline) { latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000, timeoutMs: 15_000 }); const task = asRecord(coreBody(latest)?.task); const status = asString(task?.status); if (status === "succeeded" || status === "failed") { const result = asRecord(task?.result); const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null; const stdout = asString(result?.stdout); const stderr = asString(result?.stderr); return { ok: status === "succeeded" && (exitCode === null || exitCode === 0), taskId, status, stdout, stderr, exitCode, raw: task, }; } await Bun.sleep(500); } return { ok: false, taskId, status: "timeout", stdout: "", stderr: `host.ssh task ${taskId} did not finish within ${Math.max(effectiveWaitMs, 1_000)}ms`, exitCode: null, raw: latest, }; } async function runRemoteKubectl(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise { const result = await runRemoteKubectlRaw(script, waitMs, remoteTimeoutMs, target); if (!result.ok) { throw new Error(`${target.providerId} kubectl command failed: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); } return result; } async function runRemoteKubectlRaw(script: string, waitMs = 60_000, remoteTimeoutMs = 45_000, target = ciTarget(null)): Promise { const command = [ "set -eu", ...ciTargetGuardShellLines(target, { passOutput: "stderr" }), script, ].join("\n"); return dispatchSsh(command, waitMs, remoteTimeoutMs, true, target); } async function uploadRemoteBase64(path: string, encoded: string, target = ciTarget(null)): Promise { const init = await dispatchSsh([ "set -euo pipefail", `target=${shellQuote(path)}`, "rm -f \"$target\"", ": > \"$target\"", "chmod 600 \"$target\"", ].join("\n"), 20_000, 10_000, true, target); if (!init.ok) return init; // D601 provider-gateway rejects long host.ssh commands; keep each append // envelope comfortably below the transport limit. for (const chunk of chunks(encoded, hostSshBase64UploadChunkChars)) { const append = await dispatchSsh([ "set -euo pipefail", `target=${shellQuote(path)}`, `printf %s ${shellQuote(chunk)} >> "$target"`, ].join("\n"), 20_000, 10_000, true, target); if (!append.ok) return append; } return dispatchSsh([ "set -euo pipefail", `target=${shellQuote(path)}`, `expected=${shellQuote(String(Buffer.byteLength(encoded)))}`, "actual=$(wc -c < \"$target\" | tr -d ' ')", "printf 'uploaded_bytes=%s expected_bytes=%s path=%s\\n' \"$actual\" \"$expected\" \"$target\"", "test \"$actual\" = \"$expected\"", ].join("\n"), 20_000, 10_000, true, target); } async function runRemoteBackground(label: string, script: string, timeoutMs: number, target = ciTarget(null)): Promise { const token = randomUUID().replace(/-/gu, "").slice(0, 12); const safeLabel = label.replace(/[^a-z0-9-]/giu, "-").toLowerCase().slice(0, 48); const base = `/tmp/unidesk-ci-${safeLabel}-${token}`; const scriptPath = `${base}.sh`; const logPath = `${base}.log`; const donePath = `${base}.done`; const encoded = Buffer.from(script, "utf8").toString("base64"); const upload = await uploadRemoteBase64(`${scriptPath}.b64`, encoded, target); if (!upload.ok) return upload; const start = await dispatchSsh([ "set -euo pipefail", `script_path=${shellQuote(scriptPath)}`, `log_path=${shellQuote(logPath)}`, `done_path=${shellQuote(donePath)}`, "rm -f \"$script_path\" \"$log_path\" \"$done_path\"", "base64 -d \"$script_path.b64\" > \"$script_path\"", "rm -f \"$script_path.b64\"", "chmod 700 \"$script_path\"", "nohup bash -lc \"bash '$script_path' >'$log_path' 2>&1; code=\\$?; printf '%s\\n' \\\"\\$code\\\" >'$done_path'\" >/tmp/unidesk-ci-nohup.log 2>&1 &", "printf 'remote_job_pid=%s\\nlog=%s\\ndone=%s\\n' \"$!\" \"$log_path\" \"$done_path\"", ].join("\n"), 20_000, 10_000, true, target); if (!start.ok) return start; const deadline = Date.now() + timeoutMs; let latest: DispatchResult = start; while (Date.now() < deadline) { await Bun.sleep(8_000); latest = await dispatchSsh([ "set -euo pipefail", `log_path=${shellQuote(logPath)}`, `done_path=${shellQuote(donePath)}`, "if [ -f \"$done_path\" ]; then", " code=\"$(cat \"$done_path\" 2>/dev/null || printf 1)\"", " printf 'REMOTE_DONE:%s\\n' \"$code\"", "else", " printf 'REMOTE_RUNNING\\n'", "fi", "tail -n 160 \"$log_path\" 2>/dev/null || true", ].join("\n"), 75_000, 12_000, true, target); if (!latest.ok) { if (latest.status === "timeout" || latest.stderr.includes("did not finish within")) { continue; } return latest; } const firstLine = latest.stdout.split(/\r?\n/u)[0] ?? ""; if (firstLine.startsWith("REMOTE_DONE:")) { const code = Number(firstLine.slice("REMOTE_DONE:".length).trim()); return { ...latest, ok: code === 0, exitCode: Number.isInteger(code) ? code : 1, status: code === 0 ? "succeeded" : "failed", }; } } return { ...latest, ok: false, status: "timeout", exitCode: 124, stderr: `remote background job ${label} did not finish within ${timeoutMs}ms`, }; } async function remoteApplyManifest(config: UniDeskConfig, path: string, target = ciTarget(null)): Promise { const absolute = rootPath(path); if (!existsSync(absolute)) throw new Error(`manifest not found: ${path}`); const manifest = readFileSync(absolute, "utf8"); const encoded = Buffer.from(manifest, "utf8").toString("base64"); emitCiInstallProgress("upload-manifest", "started", { providerId: target.providerId, manifest: path, bytes: Buffer.byteLength(manifest) }); const script = [ "set -eu", ...ciTargetGuardShellLines(target), "tmp=$(mktemp /tmp/unidesk-ci-apply.XXXXXX.yaml)", "trap 'rm -f \"$tmp\"' EXIT", "base64 -d > \"$tmp\" <<'UNIDESK_CI_MANIFEST_B64'", encoded, "UNIDESK_CI_MANIFEST_B64", "test -s \"$tmp\"", "printf 'manifest_bytes=%s\\n' \"$(wc -c < \"$tmp\" | tr -d ' ')\"", "kubectl apply -f \"$tmp\"", ].join("\n"); emitCiInstallProgress("kubectl-apply", "started", { providerId: target.providerId, manifest: path }); const result = await runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], script); if (result.exitCode !== 0) throw new Error(`kubectl apply failed for ${path}: ${result.stderr || result.stdout}`); emitCiInstallProgress("upload-manifest", "succeeded", { providerId: target.providerId, manifest: path, upload: result.stdout.split(/\r?\n/u).find((line) => line.startsWith("manifest_bytes=")) ?? "" }); emitCiInstallProgress("kubectl-apply", "succeeded", { providerId: target.providerId, manifest: path }); } async function prewarmCiRuntimeImages(target = ciTarget(null)): Promise { const images = ciRuntimeImages.map(shellQuote).join(" "); const script = [ "set -euo pipefail", ...ciTargetGuardShellLines(target), "export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config", "mkdir -p \"$DOCKER_CONFIG\"", "printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"", `images=(${images})`, "for image in \"${images[@]}\"; do", " if ! docker image inspect \"$image\" >/dev/null 2>&1; then", " echo ci_runtime_image_pull=$image", ` HTTP_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} HTTPS_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} ALL_PROXY=${shellQuote(providerGatewayWsEgressProxyUrl)} NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal docker pull --platform linux/amd64 "$image"`, " else", " echo ci_runtime_image_cached=$image", " fi", "done", "pause_entrypoint=$(docker image inspect rancher/mirrored-pause:3.6 --format '{{json .Config.Entrypoint}}' 2>/dev/null || true)", "if ! printf '%s' \"$pause_entrypoint\" | grep -q '\"/pause\"'; then echo native_k3s_pause_image_invalid_entrypoint=$pause_entrypoint >&2; exit 1; fi", "root_exec() {", " if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return $?; fi", " if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then sudo \"$@\"; return $?; fi", " if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return $?; fi", " echo ci_runtime_image_containerd_root_required=true >&2", " \"$@\"", "}", "containerd_images=$(root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls 2>/tmp/unidesk-ci-containerd-images.err || true)", "containerd_ready=1", "for image in \"${images[@]}\"; do", " case \"$image\" in", " rancher/*|oven/*|alpine/*) ref=\"docker.io/$image\" ;;", " unidesk-*) ref=\"docker.io/library/$image\" ;;", " *) ref=\"$image\" ;;", " esac", " if ! printf '%s\\n' \"$containerd_images\" | grep -F \"$ref\" >/dev/null; then", " containerd_ready=0", " echo ci_runtime_image_containerd_missing=$ref", " fi", "done", "if [ \"$containerd_ready\" = \"1\" ]; then", " echo ci_runtime_images_containerd_cached=all", " exit 0", "fi", "rm -f /tmp/unidesk-ci-runtime-images.tar", "docker save \"${images[@]}\" -o /tmp/unidesk-ci-runtime-images.tar", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import --digests --all-platforms /tmp/unidesk-ci-runtime-images.tar >/tmp/unidesk-ci-runtime-images-import.log", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/rancher/mirrored-pause:3.6' >/dev/null", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/oven/bun:1-debian' >/dev/null", "root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F 'docker.io/alpine/git:2.45.2' >/dev/null", `root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls | grep -F ${shellQuote(`docker.io/library/${target.codeQueueImage}`)} >/dev/null`, ].join("\n"); const result = await runRemoteBackground("prewarm-runtime-images", script, 900_000, target); if (!result.ok) { const combined = `${result.stdout}\n${result.stderr}`.trim(); const failureLines = extractCiLogFailureHints(combined).concat( combined.split(/\r?\n/u).filter((line) => /ci_runtime_image_|containerd_|root_required|sudo/iu.test(line)).map(boundedHintLine).slice(-40), ); throw new Error(`CI runtime image prewarm failed: ${JSON.stringify({ providerId: target.providerId, exitCode: result.exitCode, failureLines: Array.from(new Set(failureLines)).slice(-50), stdoutTail: tailTextLines(result.stdout, 80), stderrTail: tailTextLines(result.stderr, 80), recovery: [ "If Tekton is already installed and only CI manifests need refreshing, rerun with: bun scripts/cli.ts ci install --skip-prewarm", "If runtime helper images are missing from containerd, restore passwordless/root containerd import on the provider and rerun without --skip-prewarm.", ], })}`); } } async function status(target = ciTarget(null)): Promise> { const summary = await runRemoteKubectl([ "set -euo pipefail", "printf 'tekton_pipelines='", "kubectl get deploy -n tekton-pipelines -o name 2>/dev/null | tr '\\n' ' ' || true", "printf '\\ntekton_triggers='", "kubectl get deploy -n tekton-pipelines-resolvers -o name 2>/dev/null | tr '\\n' ' ' || true", "printf '\\nunidesk_ci='", "kubectl get pipeline,task,pipelinerun,eventlistener,svc -n unidesk-ci -o name 2>/dev/null | tr '\\n' ' ' || true", "printf '\\n'", ].join("\n"), 60_000, 45_000, target); return { ok: true, providerId: target.providerId, orchestrator: "native-k3s", tekton: { pipelineVersion: tektonPipelineVersion, triggersVersion: tektonTriggersVersion, }, summary: summary.stdout.trim(), }; } interface CiCleanupPodCandidate { name: string; phase: "Succeeded" | "Failed"; createdAt: string; ageMinutes: number; selected: boolean; selectedReason: string; } function cleanupRunsQueryScript(): string { const jsonPath = "{range .items[*]}{.metadata.name}{\"\\t\"}{.status.phase}{\"\\t\"}{.metadata.creationTimestamp}{\"\\n\"}{end}"; return [ "set -eu", `kubectl get pods -n unidesk-ci -o jsonpath=${shellQuote(jsonPath)} || true`, ].join("\n"); } function parseCleanupPodCandidates(stdout: string, generatedAtMs: number, minAgeMinutes: number, limit: number): CiCleanupPodCandidate[] { const byName = new Map>(); for (const line of stdout.split(/\r?\n/u)) { const [name, phaseRaw, createdAt] = line.trim().split("\t"); if (!name || !createdAt) continue; if (phaseRaw !== "Succeeded" && phaseRaw !== "Failed") continue; if (!/^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$/u.test(name)) continue; const createdAtMs = Date.parse(createdAt); if (!Number.isFinite(createdAtMs)) continue; const ageMinutes = Math.max(0, Math.floor((generatedAtMs - createdAtMs) / 60_000)); if (ageMinutes < minAgeMinutes) continue; byName.set(name, { name, phase: phaseRaw, createdAt, ageMinutes }); } const sorted = Array.from(byName.values()).sort((left, right) => { const byAge = right.ageMinutes - left.ageMinutes; return byAge !== 0 ? byAge : left.name.localeCompare(right.name); }); const selectedNames = new Set(sorted.slice(0, limit).map((item) => item.name)); return sorted.map((item) => ({ ...item, selected: selectedNames.has(item.name), selectedReason: selectedNames.has(item.name) ? "terminal-pod-age-within-limit" : "over-limit", })); } function cleanupRunsDeleteScript(names: string[]): string { const lines = ["set -eu"]; for (let index = 0; index < names.length; index += 50) { const chunk = names.slice(index, index + 50).map(shellQuote).join(" "); lines.push(`kubectl -n unidesk-ci delete pod ${chunk} --ignore-not-found=true --wait=false`); } return lines.join("\n"); } async function runCleanupCapture(config: UniDeskConfig, target: CiTarget, script: string) { const command = [ "set -eu", ...ciTargetGuardShellLines(target, { passOutput: "stderr" }), script, ].join("\n"); return runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], command); } async function cleanupRuns(config: UniDeskConfig, options: CiCleanupRunsOptions): Promise> { const generatedAt = new Date(); const query = await runCleanupCapture(config, options.target, cleanupRunsQueryScript()); const queryOk = "ok" in query ? query.ok : query.exitCode === 0; if (!queryOk) { return { ok: false, command: "ci cleanup-runs", providerId: options.target.providerId, namespace: "unidesk-ci", mutation: false, failureKind: "ci-cleanup-query-failed", query: { exitCode: query.exitCode, stdoutTail: tailTextLines(query.stdout, 80), stderrTail: tailTextLines(query.stderr, 80), }, }; } const candidates = parseCleanupPodCandidates(query.stdout, generatedAt.getTime(), options.minAgeMinutes, options.limit); const selected = candidates.filter((item) => item.selected); const deletion = options.confirm && selected.length > 0 ? await runCleanupCapture(config, options.target, cleanupRunsDeleteScript(selected.map((item) => item.name))) : null; const deletionOk = deletion === null ? true : ("ok" in deletion ? deletion.ok : deletion.exitCode === 0); const ok = deletionOk; return { ok, command: "ci cleanup-runs", providerId: options.target.providerId, namespace: "unidesk-ci", generatedAt: generatedAt.toISOString(), mode: options.confirm ? "confirmed-cleanup" : "dry-run", minAgeMinutes: options.minAgeMinutes, limit: options.limit, mutation: options.confirm, candidateCount: candidates.length, selectedPodCount: selected.length, candidates: candidates.slice(0, Math.min(candidates.length, 120)), truncated: candidates.length > 120, deletedPodCount: deletionOk && deletion !== null ? selected.length : 0, deletion: deletion === null ? null : { exitCode: deletion.exitCode, stdoutTail: tailTextLines(deletion.stdout, 120), stderrTail: tailTextLines(deletion.stderr, 80), }, next: options.confirm ? { status: `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}` } : { confirm: `bun scripts/cli.ts ci cleanup-runs --provider-id ${options.target.providerId} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` }, }; } async function cleanupFailedPods(config: UniDeskConfig, options: CiCleanupFailedPodsOptions): Promise> { const script = [ "set -eu", ...ciTargetGuardShellLines(options.target, { passOutput: "stderr" }), `namespace=${shellQuote(options.namespace)}`, `min_age_minutes=${String(options.minAgeMinutes)}`, `limit=${String(options.limit)}`, `confirm=${options.confirm ? "true" : "false"}`, "tmp=$(mktemp /tmp/unidesk-ci-failed-pods.XXXXXX.tsv)", "selected=$(mktemp /tmp/unidesk-ci-failed-pods-selected.XXXXXX.txt)", "trap 'rm -f \"$tmp\" \"$selected\"' EXIT", "kubectl -n \"$namespace\" get pods --field-selector=status.phase=Failed --sort-by=.metadata.creationTimestamp -o 'custom-columns=NAME:.metadata.name,CREATED:.metadata.creationTimestamp,PHASE:.status.phase,REASON:.status.reason' --no-headers > \"$tmp\" || true", "now_epoch=$(date -u +%s)", "candidate_count=0", "while IFS= read -r line; do", " set -- $line", " name=${1:-}", " created=${2:-}", " phase=${3:-}", " reason=${4:-}", " [ -n \"$name\" ] || continue", " [ \"$phase\" = \"Failed\" ] || continue", " created_epoch=$(date -u -d \"$created\" +%s 2>/dev/null || printf '0')", " [ \"$created_epoch\" -gt 0 ] || continue", " age_minutes=$(( (now_epoch - created_epoch) / 60 ))", " [ \"$age_minutes\" -ge \"$min_age_minutes\" ] || continue", " [ \"$candidate_count\" -lt \"$limit\" ] || continue", " candidate_count=$((candidate_count + 1))", " printf 'candidate\\t%s\\t%s\\t%s\\t%s\\t%s\\n' \"$name\" \"$created\" \"$phase\" \"$age_minutes\" \"$reason\"", " printf '%s\\n' \"$name\" >> \"$selected\"", "done < \"$tmp\"", "delete_exit=0", "delete_stdout=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.out)", "delete_stderr=$(mktemp /tmp/unidesk-ci-failed-pods-delete.XXXXXX.err)", "trap 'rm -f \"$tmp\" \"$selected\" \"$delete_stdout\" \"$delete_stderr\"' EXIT", "if [ \"$confirm\" = \"true\" ] && [ -s \"$selected\" ]; then", " xargs -r kubectl -n \"$namespace\" delete pod --ignore-not-found=true --wait=false < \"$selected\" > \"$delete_stdout\" 2> \"$delete_stderr\" || delete_exit=$?", "fi", "printf 'candidateCount\\t%s\\n' \"$candidate_count\"", "if [ \"$confirm\" = \"true\" ]; then printf 'deletedPodCount\\t%s\\n' \"$(wc -l < \"$selected\" | tr -d ' ')\"; else printf 'deletedPodCount\\t0\\n'; fi", "printf 'deleteExit\\t%s\\n' \"$delete_exit\"", "printf 'deleteStdoutTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stdout\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"", "printf 'deleteStderrTail\\t%s\\n' \"$(tail -c 2000 \"$delete_stderr\" | tr '\\n' ' ' | sed 's/[[:space:]]\\+/ /g')\"", "exit \"$delete_exit\"", ].join("\n"); const result = await runSshCommandCapture(config, `${options.target.providerId}:k3s`, ["sh"], script); const fields = new Map(); const candidates: Record[] = []; for (const line of result.stdout.split(/\r?\n/u)) { if (!line.trim()) continue; const [kind = "", ...rest] = line.split("\t"); if (kind === "candidate") { const [name = "", createdAt = "", phase = "", ageMinutes = "", reason = ""] = rest; candidates.push({ name, namespace: options.namespace, phase, createdAt, ageMinutes: Number(ageMinutes), reason }); continue; } fields.set(kind, rest.join("\t")); } const deleteExit = Number(fields.get("deleteExit") ?? result.exitCode); const ok = result.exitCode === 0 && deleteExit === 0; return { ok, command: "ci cleanup-failed-pods", mode: options.confirm ? "confirmed-cleanup" : "dry-run", mutation: options.confirm, providerId: options.target.providerId, namespace: options.namespace, minAgeMinutes: options.minAgeMinutes, limit: options.limit, candidateCount: Number(fields.get("candidateCount") ?? candidates.length), candidates, deletedPodCount: Number(fields.get("deletedPodCount") ?? 0), deletion: { exitCode: deleteExit, stdoutTail: fields.get("deleteStdoutTail") ?? "", stderrTail: fields.get("deleteStderrTail") ?? "", }, probe: { exitCode: result.exitCode, stdoutTail: tailTextLines(result.stdout, 40), stderrTail: tailTextLines(result.stderr, 20), }, next: options.confirm ? { status: `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}` } : { confirm: `bun scripts/cli.ts ci cleanup-failed-pods --provider-id ${options.target.providerId} --namespace ${options.namespace} --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm` }, }; } async function install(config: UniDeskConfig, options: CiInstallOptions): Promise> { if (!existsSync(rootPath(options.target.pipelineManifest))) { throw new Error("CI manifests are missing"); } emitCiInstallProgress("install", "started", { providerId: options.target.providerId, skipPrewarm: options.skipPrewarm, skipTektonInstall: options.skipTektonInstall, }); try { if (options.skipPrewarm) { emitCiInstallProgress("prewarm", "skipped", { providerId: options.target.providerId }); } else { emitCiInstallProgress("prewarm", "started", { providerId: options.target.providerId }); await prewarmCiRuntimeImages(options.target); emitCiInstallProgress("prewarm", "succeeded", { providerId: options.target.providerId }); } if (options.skipTektonInstall) { emitCiInstallProgress("install-tekton", "skipped", { providerId: options.target.providerId }); } else { emitCiInstallProgress("install-tekton", "started", { providerId: options.target.providerId }); const installTektonScript = [ "set -euo pipefail", ...ciTargetGuardShellLines(options.target), `kubectl apply -f ${shellQuote(tektonPipelineReleaseUrl)}`, "kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s", `kubectl apply -f ${shellQuote(tektonTriggersReleaseUrl)}`, `kubectl apply -f ${shellQuote(tektonTriggersInterceptorsUrl)}`, "kubectl wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=900s", "kubectl wait --for=condition=Available deployment --all -n tekton-pipelines-resolvers --timeout=900s", ].join("\n"); const installTekton = await runRemoteBackground("install-tekton", installTektonScript, 1_200_000, options.target); if (!installTekton.ok) throw new Error(`Tekton install failed: ${installTekton.stderr || installTekton.stdout}`); emitCiInstallProgress("install-tekton", "succeeded", { providerId: options.target.providerId }); } for (const manifest of [ "src/components/microservices/k3sctl-adapter/k3s/ci/tekton-install.yaml", options.target.pipelineManifest, "src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.triggers.yaml", ]) { emitCiInstallProgress("apply-manifest", "started", { providerId: options.target.providerId, manifest }); await remoteApplyManifest(config, manifest, options.target); emitCiInstallProgress("apply-manifest", "succeeded", { providerId: options.target.providerId, manifest }); } emitCiInstallProgress("status", "started", { providerId: options.target.providerId }); const summary = await status(options.target); emitCiInstallProgress("status", "succeeded", { providerId: options.target.providerId }); emitCiInstallProgress("install", "succeeded", { providerId: options.target.providerId }); return { ...summary, install: { providerId: options.target.providerId, prewarm: options.skipPrewarm ? "skipped" : "completed", tektonInstall: options.skipTektonInstall ? "skipped" : "completed", }, }; } catch (error) { emitCiInstallProgress("install", "failed", { providerId: options.target.providerId, errorTail: (error instanceof Error ? error.message : String(error)).slice(-1200), }); throw error; } } function ciInstallCommand(options: CiInstallOptions): string[] { return [ "bun", "scripts/cli.ts", "ci", "install", "--provider-id", options.target.providerId, "--wait", ...(options.skipPrewarm ? ["--skip-prewarm"] : []), ...(options.skipTektonInstall ? ["--skip-tekton-install"] : []), ]; } function installAsync(options: CiInstallOptions): Record { const command = ciInstallCommand(options); const job = startJob( "ci_install", command, `Install/refresh Tekton CI on ${options.target.providerId} native k3s`, ); return { ok: true, mode: "async", providerId: options.target.providerId, skipped: { prewarm: options.skipPrewarm, tektonInstall: options.skipTektonInstall, }, job: { id: job.id, status: job.status, command: job.command, stdoutFile: job.stdoutFile, stderrFile: job.stderrFile, note: job.note, }, next: [ `bun scripts/cli.ts ci install-status ${job.id}`, `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, ], boundary: "ci install is fire-and-forget by default; use --wait only for explicit synchronous debugging.", }; } function latestCiInstallJobId(): string { const job = listJobs().find((item) => item.name === "ci_install"); if (job === undefined) throw new Error("no ci_install job found"); return job.id; } function installStatus(id: string): Record { const jobId = id === "latest" || id.length === 0 ? latestCiInstallJobId() : id; const job = readJob(jobId); if (job.name !== "ci_install") { throw new Error(`job ${jobId} is ${job.name}, not ci_install`); } return { ok: job.status !== "failed" && job.status !== "canceled", job: jobWithTail(job, 12_000), next: job.status === "running" || job.status === "queued" ? [`bun scripts/cli.ts ci install-status ${job.id}`] : ["bun scripts/cli.ts ci status"], }; } function pipelineRunManifest(options: CiOptions): string { const safeSuffix = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase(); return `apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: unidesk-ci-${safeSuffix}- namespace: unidesk-ci labels: app.kubernetes.io/name: unidesk-ci app.kubernetes.io/part-of: unidesk unidesk.ai/revision: ${JSON.stringify(options.revision)} spec: pipelineRef: name: unidesk-ci taskRunTemplate: serviceAccountName: unidesk-ci-runner params: - name: repo-url value: ${JSON.stringify(options.repoUrl)} - name: revision value: ${JSON.stringify(options.revision)} workspaces: - name: shared-workspace persistentVolumeClaim: claimName: unidesk-ci-cache `; } function backendCoreArtifactPipelineRunManifest(options: CiPublishBackendCoreOptions): string { const safeSuffix = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase(); return `apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: backend-core-artifact-${safeSuffix}- namespace: unidesk-ci labels: app.kubernetes.io/name: unidesk-backend-core-artifact-publish app.kubernetes.io/part-of: unidesk unidesk.ai/service-id: backend-core unidesk.ai/revision: ${JSON.stringify(options.commit)} spec: pipelineRef: name: unidesk-backend-core-artifact-publish taskRunTemplate: serviceAccountName: unidesk-ci-runner params: - name: repo-url value: ${JSON.stringify(options.repoUrl)} - name: revision value: ${JSON.stringify(options.commit)} - name: dockerfile value: ${JSON.stringify(options.dockerfile)} - name: image-repository value: ${JSON.stringify(options.imageRepository)} - name: source-host-path value: ${JSON.stringify(options.sourceHostPath)} workspaces: - name: shared-workspace persistentVolumeClaim: claimName: unidesk-ci-cache `; } function userServiceArtifactPipelineRunManifest(options: CiPublishUserServiceArtifactOptions): string { const safeSuffix = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase(); return `apiVersion: tekton.dev/v1 kind: PipelineRun metadata: generateName: user-service-artifact-${options.serviceId}-${safeSuffix}- namespace: unidesk-ci labels: app.kubernetes.io/name: unidesk-user-service-artifact-publish app.kubernetes.io/part-of: unidesk unidesk.ai/service-id: ${JSON.stringify(options.serviceId)} unidesk.ai/revision: ${JSON.stringify(options.commit)} spec: pipelineRef: name: unidesk-user-service-artifact-publish taskRunTemplate: serviceAccountName: unidesk-ci-runner params: - name: repo-url value: ${JSON.stringify(options.repoUrl)} - name: revision value: ${JSON.stringify(options.commit)} - name: service-id value: ${JSON.stringify(options.serviceId)} - name: dockerfile value: ${JSON.stringify(options.dockerfile)} - name: image-repository value: ${JSON.stringify(options.imageRepository)} - name: source-host-path value: ${JSON.stringify(options.sourceHostPath)} workspaces: - name: shared-workspace persistentVolumeClaim: claimName: unidesk-ci-cache `; } function backendCoreArtifactSourceHostPath(commit: string): string { return `/home/ubuntu/.unidesk/ci/backend-core-artifacts/${commit}`; } function userServiceArtifactSourceHostPath(serviceId: string, commit: string): string { return `/home/ubuntu/.unidesk/ci/user-service-artifacts/${serviceId}/${commit}`; } async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: CiPublishBackendCoreOptions): Promise> { const sourceRoot = "/home/ubuntu/.unidesk/ci/backend-core-artifacts"; const sourceHostPath = options.sourceHostPath; const repoCache = "/home/ubuntu/.unidesk/ci/git/unidesk.git"; const repoFetchUrl = repoSshUrl(options.repoUrl); const sshIdentity = repoNeedsGithubSshIdentity(repoFetchUrl) ? await ensureGithubSshIdentityForProvider(config, d601ProviderId) : null; if (sshIdentity !== null && !sshIdentity.ok) throw new Error(sshIdentity.detail); const proxyPython = gitSshHttpConnectProxySource(); const dockerfile = requireRepoRelativePath(options.dockerfile, "CI.json.artifacts.backend-core.source.dockerfile"); const script = [ "set -euo pipefail", `commit=${shellQuote(options.commit)}`, `repo_url=${shellQuote(options.repoUrl)}`, `repo_fetch_url=${shellQuote(repoFetchUrl)}`, `dockerfile=${shellQuote(dockerfile)}`, `source_root=${shellQuote(sourceRoot)}`, `source_dir=${shellQuote(sourceHostPath)}`, `repo_cache=${shellQuote(repoCache)}`, `proxy_url=${shellQuote(providerGatewayWsEgressProxyUrl)}`, "mkdir -p \"$(dirname \"$repo_cache\")\" \"$source_root\"", "export HTTP_PROXY=\"$proxy_url\" HTTPS_PROXY=\"$proxy_url\" ALL_PROXY=\"$proxy_url\"", "export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"", "curl -fsSI --max-time 20 -x \"$proxy_url\" https://github.com >/dev/null", "git_ssh_proxy=/tmp/unidesk-git-ssh-http-connect.py", "cat > \"$git_ssh_proxy\" <<'UNIDESK_GIT_SSH_PROXY'", proxyPython, "UNIDESK_GIT_SSH_PROXY", "chmod 700 \"$git_ssh_proxy\"", "export UNIDESK_GIT_SSH_HTTP_PROXY=\"$proxy_url\"", "export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=$git_ssh_proxy %h %p'\"", "echo backend_core_artifact_source_proxy=provider-gateway-ws-egress:$proxy_url", "echo backend_core_artifact_repo_fetch_url=$repo_fetch_url", "if [ ! -d \"$repo_cache\" ]; then git clone --mirror \"$repo_fetch_url\" \"$repo_cache\"; fi", "git -C \"$repo_cache\" remote set-url origin \"$repo_fetch_url\"", "git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")", "test \"$resolved\" = \"$commit\" || { echo \"backend_core_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }", "git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile\"", "git -C \"$repo_cache\" cat-file -e \"$commit:src/components/backend-core/src\"", "tmp_dir=\"$source_root/.tmp-$commit-$$\"", "rm -rf \"$tmp_dir\"", "mkdir -p \"$tmp_dir\"", "git -C \"$repo_cache\" archive \"$commit\" | tar -x -C \"$tmp_dir\"", "printf '%s\\n' \"$commit\" > \"$tmp_dir/.unidesk-source-commit\"", "printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"", "rm -rf \"$source_dir\"", "mv \"$tmp_dir\" \"$source_dir\"", "test -f \"$source_dir/$dockerfile\"", "test -d \"$source_dir/src/components/backend-core/src\"", "echo backend_core_artifact_source_host_path=$source_dir", ].join("\n"); const result = await runRemoteBackground("prepare-backend-core-source", script, 300_000); if (!result.ok) throw new Error(`failed to prepare backend-core source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); return { ok: true, mode: repoNeedsGithubSshIdentity(repoFetchUrl) ? "d601-host-git-ssh-export" : "d601-host-git-https-export", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl, commit: options.commit, sourceHostPath, dockerfile, identity: sshIdentity === null ? null : { fingerprint: sshIdentity.fingerprint, seededFromLocal: sshIdentity.seededFromLocal, }, stdoutTail: result.stdout.slice(-4000), }; } async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions): Promise> { const sourceRoot = `/home/ubuntu/.unidesk/ci/user-service-artifacts/${options.serviceId}`; const sourceHostPath = options.sourceHostPath; const repoCache = `/home/ubuntu/.unidesk/ci/git/${safePathToken(options.serviceId)}.git`; const repoFetchUrl = repoSshUrl(options.repoUrl); const repoProbeUrl = repoConnectivityProbeUrl(repoFetchUrl); const sshIdentity = repoNeedsGithubSshIdentity(repoFetchUrl) ? await ensureGithubSshIdentityForProvider(config, d601ProviderId) : null; if (sshIdentity !== null && !sshIdentity.ok) throw new Error(sshIdentity.detail); const proxyPython = gitSshHttpConnectProxySource(); const script = [ "set -euo pipefail", `service_id=${shellQuote(options.serviceId)}`, `commit=${shellQuote(options.commit)}`, `repo_url=${shellQuote(options.repoUrl)}`, `repo_fetch_url=${shellQuote(repoFetchUrl)}`, `repo_probe_url=${shellQuote(repoProbeUrl)}`, `dockerfile=${shellQuote(options.dockerfile)}`, `source_root=${shellQuote(sourceRoot)}`, `source_dir=${shellQuote(sourceHostPath)}`, `repo_cache=${shellQuote(repoCache)}`, `proxy_url=${shellQuote(providerGatewayWsEgressProxyUrl)}`, "mkdir -p \"$(dirname \"$repo_cache\")\" \"$source_root\"", "export HTTP_PROXY=\"$proxy_url\" HTTPS_PROXY=\"$proxy_url\" ALL_PROXY=\"$proxy_url\"", "export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"", "curl -fsSI --max-time 20 -x \"$proxy_url\" \"$repo_probe_url\" >/dev/null", "git_ssh_proxy=/tmp/unidesk-git-ssh-http-connect.py", "cat > \"$git_ssh_proxy\" <<'UNIDESK_GIT_SSH_PROXY'", proxyPython, "UNIDESK_GIT_SSH_PROXY", "chmod 700 \"$git_ssh_proxy\"", "export UNIDESK_GIT_SSH_HTTP_PROXY=\"$proxy_url\"", "export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=$git_ssh_proxy %h %p'\"", "echo user_service_artifact_source_proxy=provider-gateway-ws-egress:$proxy_url", "echo user_service_artifact_repo_fetch_url=$repo_fetch_url", "echo user_service_artifact_repo_probe_url=$repo_probe_url", "echo user_service_artifact_service_id=$service_id", "if [ ! -d \"$repo_cache\" ]; then git clone --mirror \"$repo_fetch_url\" \"$repo_cache\"; fi", "git -C \"$repo_cache\" remote set-url origin \"$repo_fetch_url\"", "git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")", "test \"$resolved\" = \"$commit\" || { echo \"user_service_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }", "git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile\"", "tmp_dir=\"$source_root/.tmp-$commit-$$\"", "rm -rf \"$tmp_dir\"", "mkdir -p \"$tmp_dir\"", "git -C \"$repo_cache\" archive \"$commit\" | tar -x -C \"$tmp_dir\"", "printf '%s\\n' \"$commit\" > \"$tmp_dir/.unidesk-source-commit\"", "printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"", "printf '%s\\n' \"$service_id\" > \"$tmp_dir/.unidesk-service-id\"", "printf '%s\\n' \"$dockerfile\" > \"$tmp_dir/.unidesk-dockerfile\"", "rm -rf \"$source_dir\"", "mv \"$tmp_dir\" \"$source_dir\"", "test -f \"$source_dir/$dockerfile\"", "echo user_service_artifact_source_host_path=$source_dir", ].join("\n"); const result = await runRemoteBackground(`prepare-${options.serviceId}-source`, script, 300_000); if (!result.ok) throw new Error(`failed to prepare ${options.serviceId} source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); return { ok: true, mode: repoNeedsGithubSshIdentity(repoFetchUrl) ? "d601-host-git-ssh-export" : "d601-host-git-https-export", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl, repoProbeUrl, commit: options.commit, serviceId: options.serviceId, dockerfile: options.dockerfile, sourceHostPath, identity: sshIdentity === null ? null : { fingerprint: sshIdentity.fingerprint, seededFromLocal: sshIdentity.seededFromLocal, }, stdoutTail: result.stdout.slice(-4000), }; } async function prepareClaudeqqArtifactSource(config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions): Promise> { const sourceRoot = `/home/ubuntu/.unidesk/ci/user-service-artifacts/${options.serviceId}`; const sourceHostPath = options.sourceHostPath; const repoCache = "/home/ubuntu/.unidesk/ci/git/claudeqq-agent-skills.git"; const repoFetchUrl = options.repoUrl; const assets = [ { relativePath: "claudeqq/Dockerfile", sourcePath: rootPath("src/components/microservices/claudeqq/Dockerfile"), label: "Dockerfile", }, { relativePath: "claudeqq/unidesk-adapter.cjs", sourcePath: rootPath("src/components/microservices/claudeqq/adapter.js"), label: "unidesk-adapter.cjs", }, ]; for (const asset of assets) { if (!existsSync(asset.sourcePath)) throw new Error(`claudeqq artifact asset missing: ${asset.sourcePath}`); } const overlayCommands = assets.flatMap((asset) => { const encoded = Buffer.from(readFileSync(asset.sourcePath, "utf8"), "utf8").toString("base64"); return [ `mkdir -p "$tmp_dir/$(dirname ${shellQuote(asset.relativePath)})"`, `printf %s ${shellQuote(encoded)} | base64 -d > "$tmp_dir/${asset.relativePath}"`, `printf 'user_service_artifact_overlay=%s\\n' ${shellQuote(asset.label)}`, ]; }); const script = [ "set -euo pipefail", `service_id=${shellQuote(options.serviceId)}`, `commit=${shellQuote(options.commit)}`, `repo_url=${shellQuote(options.repoUrl)}`, `repo_fetch_url=${shellQuote(repoFetchUrl)}`, `dockerfile=${shellQuote(options.dockerfile)}`, `source_root=${shellQuote(sourceRoot)}`, `source_dir=${shellQuote(sourceHostPath)}`, `repo_cache=${shellQuote(repoCache)}`, `proxy_url=${shellQuote(providerGatewayWsEgressProxyUrl)}`, "mkdir -p \"$(dirname \"$repo_cache\")\" \"$source_root\"", "export HTTP_PROXY=\"$proxy_url\" HTTPS_PROXY=\"$proxy_url\" ALL_PROXY=\"$proxy_url\"", "export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"", "curl -fsSI --max-time 20 -x \"$proxy_url\" https://gitee.com >/dev/null", "echo user_service_artifact_source_proxy=provider-gateway-ws-egress:$proxy_url", "echo user_service_artifact_repo_fetch_url=$repo_fetch_url", "echo user_service_artifact_service_id=$service_id", "if [ ! -d \"$repo_cache\" ]; then git clone --mirror \"$repo_fetch_url\" \"$repo_cache\"; fi", "git -C \"$repo_cache\" remote set-url origin \"$repo_fetch_url\"", "git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")", "test \"$resolved\" = \"$commit\" || { echo \"user_service_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }", "git -C \"$repo_cache\" cat-file -e \"$commit:claudeqq/scripts/src/server_ts/package.json\"", "git -C \"$repo_cache\" cat-file -e \"$commit:claudeqq/scripts/src/server_ts/src\"", "tmp_dir=\"$source_root/.tmp-$commit-$$\"", "rm -rf \"$tmp_dir\"", "mkdir -p \"$tmp_dir\"", "git -C \"$repo_cache\" archive \"$commit\" claudeqq | tar -x -C \"$tmp_dir\"", ...overlayCommands, "printf '%s\\n' \"$commit\" > \"$tmp_dir/.unidesk-source-commit\"", "printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"", "printf '%s\\n' \"$service_id\" > \"$tmp_dir/.unidesk-service-id\"", "printf '%s\\n' \"$dockerfile\" > \"$tmp_dir/.unidesk-dockerfile\"", "rm -rf \"$source_dir\"", "mv \"$tmp_dir\" \"$source_dir\"", "test -f \"$source_dir/$dockerfile\"", "test -f \"$source_dir/claudeqq/unidesk-adapter.cjs\"", "test -d \"$source_dir/claudeqq/scripts/src/server_ts/src\"", "echo user_service_artifact_source_host_path=$source_dir", ].join("\n"); const result = await runRemoteBackground(`prepare-${options.serviceId}-source`, script, 300_000); if (!result.ok) throw new Error(`failed to prepare ${options.serviceId} source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); return { ok: true, mode: "d601-host-gitee-https-export-with-unidesk-overlay", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl, commit: options.commit, serviceId: options.serviceId, dockerfile: options.dockerfile, sourceHostPath, stdoutTail: result.stdout.slice(-4000), }; } async function remoteCreatePipelineRun(manifest: string, target = ciTarget(null)): Promise { const encoded = Buffer.from(manifest, "utf8").toString("base64"); const token = randomUUID().replace(/-/gu, "").slice(0, 12); const b64Path = `/tmp/unidesk-ci-pipelinerun-${token}.b64`; const upload = await uploadRemoteBase64(b64Path, encoded, target); if (!upload.ok) throw new Error(`failed to upload PipelineRun manifest: ${upload.stderr || upload.stdout}`); const result = await runRemoteKubectl([ "tmp=$(mktemp /tmp/unidesk-ci-run.XXXXXX.yaml)", `b64_path=${shellQuote(b64Path)}`, "trap 'rm -f \"$tmp\" \"$b64_path\"' EXIT", "base64 -d \"$b64_path\" > \"$tmp\"", "kubectl create -f \"$tmp\" -o jsonpath='{.metadata.name}'", ].join("\n"), 60_000, 45_000, target); return result.stdout.trim(); } async function waitForPipelineRun(name: string, waitMs: number, target = ciTarget(null)): Promise { if (waitMs <= 0) return null; const command = [ "set -euo pipefail", ...ciTargetGuardShellLines(target), `printf 'waiting_pipelinerun=%s\\n' ${shellQuote(name)}`, `deadline=$((SECONDS + ${Math.ceil(waitMs / 1000)}))`, "while [ \"$SECONDS\" -lt \"$deadline\" ]; do", ` condition="$(kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o jsonpath='{range .status.conditions[?(@.type==\"Succeeded\")]}{.status}{\"\\t\"}{.reason}{\"\\t\"}{.message}{end}' 2>/dev/null || true)"`, " case \"$condition\" in", " True*)", " printf 'condition=%s\\n' \"$condition\"", ` kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} --no-headers 2>/dev/null || true`, " exit 0", " ;;", " False*)", " printf 'condition=%s\\n' \"$condition\"", ` kubectl get taskrun,pod -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} 2>/dev/null || true`, " exit 1", " ;;", " esac", " sleep 2", "done", `echo "Timed out waiting for pipelinerun/${name}" >&2`, `kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o json`, "exit 124", ].join("\n"); return dispatchSsh(command, waitMs + 30_000, waitMs + 20_000, true, target); } async function readPipelineRunCondition(name: string, target = ciTarget(null)): Promise { const result = await runRemoteKubectlRaw([ "set -euo pipefail", `condition="$(kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o jsonpath='{range .status.conditions[?(@.type==\"Succeeded\")]}{.status}{\"\\t\"}{.reason}{\"\\t\"}{.message}{end}' 2>/dev/null || true)"`, "printf '%s\\n' \"$condition\"", ].join("\n"), 30_000, 15_000, target); const [status = "", reason = "", ...messageParts] = result.stdout.trim().split("\t"); const message = messageParts.join("\t"); return { ok: status === "True" ? true : status === "False" ? false : null, status, reason, message, query: { ok: result.ok, status: result.status, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200), }, }; } function pipelineRunWaitSucceeded(wait: DispatchResult | null, condition: PipelineRunCondition | null): boolean { if (wait === null) return true; if (condition?.ok === true) return true; if (condition?.ok === false) return false; return wait.ok || wait.exitCode === 0 || wait.stdout.includes("condition=True\tSucceeded\t"); } function artifactSummaryDefaults(context: ArtifactSummaryContext): ArtifactSummary { const registry = "127.0.0.1:5000"; const repository = `${registry}/${context.imageRepository}`; return { serviceId: context.serviceId, sourceCommit: context.commit, sourceRepo: context.repoUrl, dockerfile: context.dockerfile, registry, repository, tag: context.commit, imageRef: `${repository}:${context.commit}`, digest: null, digestRef: null, }; } function artifactSummaryField(fields: Map, suffix: string): string | null { const value = fields.get(`user_service_artifact_${suffix}`) ?? fields.get(`backend_core_artifact_${suffix}`) ?? null; return value === null || value.length === 0 ? null : value; } function parseArtifactSummaryFromFields(fields: Map, context: ArtifactSummaryContext): ArtifactSummary { const planned = artifactSummaryDefaults(context); const registry = artifactSummaryField(fields, "registry") ?? planned.registry; const repository = artifactSummaryField(fields, "repository") ?? planned.repository; const tag = artifactSummaryField(fields, "tag") ?? planned.tag; const imageRef = artifactSummaryField(fields, "image") ?? (repository.length > 0 && tag.length > 0 ? `${repository}:${tag}` : planned.imageRef); const digest = artifactSummaryField(fields, "digest"); const digestRef = artifactSummaryField(fields, "digest_ref") ?? (digest === null || digest.length === 0 || repository.length === 0 ? null : `${repository}@${digest}`); return { serviceId: artifactSummaryField(fields, "service_id") ?? planned.serviceId, sourceCommit: artifactSummaryField(fields, "source_commit") ?? planned.sourceCommit, sourceRepo: artifactSummaryField(fields, "source_repo") ?? planned.sourceRepo, dockerfile: artifactSummaryField(fields, "dockerfile") ?? planned.dockerfile, registry, repository, tag, imageRef, digest, digestRef, }; } function parseArtifactSummaryFromOutput(output: string, context: ArtifactSummaryContext): ArtifactSummary { const fields = new Map(); for (const line of output.split(/\r?\n/u)) { const match = /^(user_service_artifact_[a-z_]+|backend_core_artifact_[a-z_]+)=(.*)$/u.exec(line.trim()); if (match !== null) fields.set(match[1], match[2]); } return parseArtifactSummaryFromFields(fields, context); } async function completeArtifactSummaryFromRegistry(artifact: ArtifactSummary, context: ArtifactSummaryContext): Promise { if (artifact.digest !== null && artifact.digest.length > 0 && artifact.digestRef !== null && artifact.digestRef.length > 0) return artifact; const planned = artifactSummaryDefaults(context); const registry = artifact.registry.length > 0 ? artifact.registry : planned.registry; const repository = artifact.repository.length > 0 ? artifact.repository : planned.repository; const tag = artifact.tag.length > 0 ? artifact.tag : planned.tag; const imageRef = artifact.imageRef.length > 0 ? artifact.imageRef : `${repository}:${tag}`; if (registry !== "127.0.0.1:5000" || !repository.startsWith(`${registry}/`)) return artifact; const repositoryPath = repository.slice(`${registry}/`.length); if (repositoryPath.length === 0 || repositoryPath.includes("..") || tag.length === 0) return artifact; const result = await dispatchSsh([ "set -euo pipefail", `manifest_url=${shellQuote(`http://127.0.0.1:5000/v2/${repositoryPath}/manifests/${tag}`)}`, "headers=$(mktemp /tmp/unidesk-artifact-summary.XXXXXX.headers)", "trap 'rm -f \"$headers\"' EXIT", "curl -fsSI -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' -D \"$headers\" -o /dev/null \"$manifest_url\"", "manifest_digest=$(awk 'BEGIN{IGNORECASE=1} /^Docker-Content-Digest:/ {gsub(/\\r/, \"\", $2); print $2; exit}' \"$headers\")", "test -n \"$manifest_digest\"", "printf 'artifact_registry_manifest_digest=%s\\n' \"$manifest_digest\"", ].join("\n"), 60_000, 45_000); const digest = /^artifact_registry_manifest_digest=(sha256:[0-9a-f]{64})$/mu.exec(result.stdout)?.[1] ?? null; if (!result.ok || digest === null) return { ...artifact, registry, repository, tag, imageRef }; return { ...artifact, registry, repository, tag, imageRef, digest, digestRef: `${repository}@${digest}`, }; } function missingArtifactSummaryFields(artifact: ArtifactSummary): string[] { const missing: string[] = []; if (artifact.serviceId.length === 0) missing.push("serviceId"); if (!/^[0-9a-f]{40}$/u.test(artifact.sourceCommit)) missing.push("sourceCommit"); if (artifact.sourceRepo.length === 0) missing.push("sourceRepo"); if (artifact.dockerfile.length === 0) missing.push("dockerfile"); if (artifact.imageRef.length === 0) missing.push("imageRef"); if (artifact.tag.length === 0) missing.push("tag"); if (artifact.digest === null || artifact.digest.length === 0) missing.push("digest"); if (artifact.digestRef === null || artifact.digestRef.length === 0) missing.push("digestRef"); return missing; } function dockerArtifactDigest(repository: string, imageRef: string): string | null { const inspect = runCommand(["docker", "image", "inspect", imageRef, "--format", "{{range .RepoDigests}}{{println .}}{{end}}"], repoRoot, { timeoutMs: 30_000 }); if (inspect.exitCode !== 0) return null; for (const line of inspect.stdout.split(/\r?\n/u).map((item) => item.trim()).filter(Boolean)) { const [repo, digest] = line.split("@"); if (repo === repository && /^sha256:[0-9a-f]{64}$/u.test(digest ?? "")) return digest ?? null; } return null; } function directDockerRegistryCurl(args: string[], timeoutMs = 30_000): ReturnType { const local = runCommand(["curl", ...args], repoRoot, { timeoutMs }); if (local.exitCode === 0 && !local.timedOut) return local; const basePresent = runCommand(["docker", "image", "inspect", codeQueueDirectDockerBaseImage], repoRoot, { timeoutMs: 30_000 }); if (basePresent.exitCode !== 0) return local; return runCommand([ "docker", "run", "--rm", "--network", "host", "--pull", "never", "--entrypoint", "curl", codeQueueDirectDockerBaseImage, ...args, ], repoRoot, { timeoutMs }); } function registryManifestDigest(repository: string, tag: string): string | null { const registry = "127.0.0.1:5000"; if (!repository.startsWith(`${registry}/`)) return null; const repositoryPath = repository.slice(`${registry}/`.length); if (repositoryPath.length === 0 || repositoryPath.includes("..") || tag.length === 0) return null; const result = directDockerRegistryCurl([ "-fsSI", "-H", "Accept: application/vnd.docker.distribution.manifest.v2+json", `http://${registry}/v2/${repositoryPath}/manifests/${tag}`, ], 30_000); if (result.exitCode !== 0) return null; const match = /^Docker-Content-Digest:\s*(sha256:[0-9a-f]{64})\s*$/imu.exec(result.stdout); return match?.[1] ?? null; } function assertCommandOk(result: ReturnType, label: string): void { if (result.exitCode === 0 && !result.timedOut) return; throw new Error(`${label} failed: ${result.stderr.slice(-2000) || result.stdout.slice(-2000) || `exitCode=${result.exitCode}`}`); } function buildContextForService(serviceId: string, dockerfile: string): string { return serviceId === "claudeqq" ? posixPath.dirname(dockerfile) : "."; } function directDockerBuildNetworkArgs(serviceId: string): string[] { if (serviceId !== "code-queue") return []; const noProxy = [ "localhost", "127.0.0.1", "::1", "host.docker.internal", "d601-tcp-egress-gateway", "d601-tcp-egress-gateway.unidesk", "d601-tcp-egress-gateway.unidesk.svc", "d601-tcp-egress-gateway.unidesk.svc.cluster.local", "backend-core", "oa-event-flow", "database", ].join(","); return [ "--network", "host", "--build-arg", `HTTP_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `HTTPS_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `http_proxy=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `https_proxy=${providerGatewayWsEgressProxyUrl}`, "--build-arg", `NO_PROXY=${noProxy}`, "--build-arg", `no_proxy=${noProxy}`, ]; } function directDockerSourceGitMode(repoUrl: string): "local-unidesk-worktree" | "git-archive" { return repoUrl === "https://github.com/pikasTech/unidesk" ? "local-unidesk-worktree" : "git-archive"; } async function prepareDirectDockerUserServiceSource(options: CiPublishUserServiceArtifactOptions): Promise<{ path: string; cleanup: () => void; summary: Record }> { const tempRoot = mkdtempSync(join(tmpdir(), `unidesk-ci-${options.serviceId}-${options.commit.slice(0, 8)}-`)); const sourcePath = join(tempRoot, "source"); const gitMode = directDockerSourceGitMode(options.repoUrl); if (gitMode !== "local-unidesk-worktree") { rmSync(tempRoot, { recursive: true, force: true }); throw new Error("direct-docker publish currently supports only UniDesk repo-owned source-build artifacts; use --transport tekton for external repositories"); } const resolved = runCommand(["git", "rev-parse", "--verify", `${options.commit}^{commit}`], repoRoot, { timeoutMs: 30_000 }); assertCommandOk(resolved, "resolve source commit"); if (resolved.stdout.trim() !== options.commit) { rmSync(tempRoot, { recursive: true, force: true }); throw new Error(`direct-docker source commit mismatch: resolved ${resolved.stdout.trim()} expected ${options.commit}`); } const dockerfileExists = runCommand(["git", "cat-file", "-e", `${options.commit}:${options.dockerfile}`], repoRoot, { timeoutMs: 30_000 }); assertCommandOk(dockerfileExists, "verify source dockerfile"); const worktree = runCommand(["git", "worktree", "add", "--detach", sourcePath, options.commit], repoRoot, { timeoutMs: 120_000 }); if (worktree.exitCode !== 0) { rmSync(tempRoot, { recursive: true, force: true }); throw new Error(`prepare source worktree failed: ${worktree.stderr || worktree.stdout}`); } return { path: sourcePath, cleanup: () => { const removed = runCommand(["git", "worktree", "remove", "--force", sourcePath], repoRoot, { timeoutMs: 60_000 }); if (removed.exitCode !== 0) rmSync(tempRoot, { recursive: true, force: true }); else rmSync(tempRoot, { recursive: true, force: true }); }, summary: { ok: true, mode: gitMode, providerId: "local", repoUrl: options.repoUrl, commit: options.commit, serviceId: options.serviceId, dockerfile: options.dockerfile, sourceHostPath: sourcePath, valuesPrinted: false, }, }; } async function publishUserServiceArtifactDirectDocker(options: CiPublishUserServiceArtifactOptions, context: ArtifactSummaryContext): Promise> { const source = await prepareDirectDockerUserServiceSource(options); const planned = artifactSummaryDefaults(context); const localImage = `${options.imageRepository}:${options.commit}`; const buildContext = buildContextForService(options.serviceId, options.dockerfile); const networkArgs = directDockerBuildNetworkArgs(options.serviceId); const baseArgs = options.serviceId === "code-queue" && runCommand(["docker", "image", "inspect", codeQueueDirectDockerBaseImage], repoRoot, { timeoutMs: 30_000 }).exitCode === 0 ? ["--build-arg", `CODE_QUEUE_BASE_IMAGE=${codeQueueDirectDockerBaseImage}`] : []; try { if (options.serviceId === "code-queue" && baseArgs.length === 0) { return { ok: false, runnerDisposition: "infra-blocked", failureClassification: "ci-runner-not-ready", failureKind: "code-queue-base-image-missing", serviceId: options.serviceId, commit: options.commit, artifactSummary: planned, source: source.summary, artifact: planned.imageRef, transport: "direct-docker", controlledPublish: { environment: "DEV-local-artifact", command: `bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --transport direct-docker`, noRollout: true, noRuntimeMutation: true, }, boundary: "direct-docker CI artifact publish builds and pushes a commit-pinned image only; no deploy apply, no rollout, no Code Queue restart, no active task mutation", next: [ "Restore or warm local unidesk-code-queue:d601 base image, then rerun the same direct-docker publish command.", ], }; } const build = runCommand([ "docker", "build", "--label", `unidesk.ai/service-id=${options.serviceId}`, "--label", `unidesk.ai/source-repo=${options.repoUrl}`, "--label", `unidesk.ai/source-commit=${options.commit}`, "--label", `unidesk.ai/dockerfile=${options.dockerfile}`, ...networkArgs, ...baseArgs, "-t", localImage, "-t", planned.imageRef, "-f", options.dockerfile, buildContext, ], source.path, { timeoutMs: Math.max(options.waitMs, 20 * 60_000) }); assertCommandOk(build, "direct docker build"); const inspectLabels = runCommand(["docker", "image", "inspect", planned.imageRef, "--format", "{{ index .Config.Labels \"unidesk.ai/service-id\" }} {{ index .Config.Labels \"unidesk.ai/source-commit\" }}"], repoRoot, { timeoutMs: 30_000 }); assertCommandOk(inspectLabels, "inspect built image labels"); if (!inspectLabels.stdout.includes(`${options.serviceId} ${options.commit}`)) throw new Error(`direct docker image labels did not match ${options.serviceId}/${options.commit}`); const registryCheck = directDockerRegistryCurl(["-fsS", "http://127.0.0.1:5000/v2/"], 15_000); assertCommandOk(registryCheck, "D601 loopback artifact registry health"); const push = runCommand(["docker", "push", planned.imageRef], repoRoot, { timeoutMs: Math.max(options.waitMs, 10 * 60_000) }); assertCommandOk(push, "direct docker push"); const pull = runCommand(["docker", "pull", planned.imageRef], repoRoot, { timeoutMs: 120_000 }); assertCommandOk(pull, "direct docker pull verification"); const digest = dockerArtifactDigest(planned.repository, planned.imageRef) ?? registryManifestDigest(planned.repository, planned.tag); const artifact: ArtifactSummary = { ...planned, digest, digestRef: digest === null ? null : `${planned.repository}@${digest}`, }; assertArtifactSummaryComplete(artifact, "direct-docker"); return { ok: true, transport: "direct-docker", pipelineRun: null, namespace: null, repoUrl: options.repoUrl, commit: options.commit, serviceId: options.serviceId, source: source.summary, artifact: artifact.imageRef, artifactSummary: artifact, controlledPublish: { environment: "DEV-local-artifact", command: `bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --transport direct-docker`, noRollout: true, noRuntimeMutation: true, dependsOnLocalUnideskDatabase: false, }, boundary: "direct-docker CI artifact publish builds and pushes a commit-pinned image only; no deploy apply, no rollout, no Code Queue restart, no active task mutation", wait: { ok: true, dispatchOk: null, dispatchStatus: null, dispatchExitCode: null, stdoutTail: push.stdout.slice(-6000), stderrTail: push.stderr.slice(-6000), }, condition: null, next: [ "use artifactSummary.imageRef or artifactSummary.digestRef as later dev artifact consumer input", ], }; } finally { source.cleanup(); } } function assertArtifactSummaryComplete(artifact: ArtifactSummary, pipelineRun: string): void { const missing = missingArtifactSummaryFields(artifact); if (missing.length > 0) { throw new Error(`artifact summary for ${pipelineRun} is missing required field(s): ${missing.join(", ")}`); } } async function publishUserServicePreflight( _config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions, plannedArtifact: ArtifactSummary, transport: PublishPreflightTransport, ): Promise { const providerId = d601ProviderId; const channels: PublishPreflightChannelProbe[] = []; const overview = await transport.coreFetch("/api/overview", { maxResponseBytes: 500_000 }); const overviewBody = coreBody(overview); const backendCoreOk = responseOk(overview) && overviewBody?.dbReady === true; channels.push(channelProbe("backend-core-api", "backend-core", backendCoreOk, "dispatch API, provider catalog, task polling, and database-backed CI state", { ok: responseOk(overview), dbReady: overviewBody?.dbReady ?? null, runnerDisposition: asRecord(overview)?.runnerDisposition ?? null, failureKind: asRecord(overview)?.failureKind ?? null, detail: backendCoreUnavailable(overview) ? overview : { status: asRecord(overview)?.status ?? null, body: overviewBody, }, })); channels.push(channelProbe("database", "database", backendCoreOk, "backend-core task dispatch, provider state, Tekton task polling, and source identity lookup", { dbReady: overviewBody?.dbReady ?? false, observedThrough: "backend-core /api/overview", })); const probeScript = [ "set -euo pipefail", ...d601K3sGuardShellLines(d601Kubeconfig), "printf 'provider_host_ssh=ok\\n'", "command -v bash >/dev/null", "command -v docker >/dev/null", "command -v kubectl >/dev/null", "test -S /var/run/docker.sock || test -S /run/docker.sock || true", ].join("\n"); const sshProbe = await transport.dispatchHostSsh(probeScript, 30_000, 15_000); channels.push(channelProbe("provider-dispatch", "provider", sshProbe.taskId !== null || sshProbe.ok, "backend-core /api/dispatch can create D601 host.ssh tasks", { taskId: sshProbe.taskId, status: sshProbe.status, ok: sshProbe.taskId !== null || sshProbe.ok, exitCode: sshProbe.exitCode, stderrTail: sshProbe.stderr.slice(-1200), })); channels.push(channelProbe("provider-host-ssh", "provider", sshProbe.ok, "D601 source export, registry checks, kubectl/Tekton submission, and artifact summary reads", { taskId: sshProbe.taskId, status: sshProbe.status, exitCode: sshProbe.exitCode, stdoutTail: sshProbe.stdout.slice(-1200), stderrTail: sshProbe.stderr.slice(-1200), raw: sshProbe.ok ? undefined : dispatchPreflightFailure("host.ssh readonly probe", sshProbe).raw, })); const registryOptions = parseArtifactRegistryOptions(["--provider-id", providerId]); const registryProbe = buildArtifactRegistryReadonlyProbe("health", registryOptions); const registryDispatch = await transport.dispatchHostSsh(registryProbe.script, Math.max(registryProbe.timeoutMs, 30_000), registryProbe.timeoutMs); const registryCommand = commandResultFromDispatch(transport.artifactRegistryCommand(registryProbe), transport.commandCwd, registryDispatch); const registry = artifactRegistryReadonlyResultFromCommand(registryProbe, registryCommand); const registryRecord = asRecord(registry); const registryOk = registryRecord?.ok === true || registryRecord?.runtimeApiHealthy === true; channels.push(channelProbe("artifact-registry", "registry", registryOk, "commit-pinned image push and later CD manifest checks", registry)); const missingChannels = channels.filter((item) => !item.ok).map((item) => item.channel); const controlChannels = summarizePublishControlChannels(channels); const missingControlChannels = controlChannels.filter((item) => !item.ok).map((item) => item.channel); const ready = missingChannels.length === 0; const failureClassification = ready ? null : classifyPublishPreflightFailure(transport, overview, sshProbe, registry); const recommendedAction = recommendedPublishPreflightAction(failureClassification, registry, missingControlChannels); return { ok: ready, runnerDisposition: ready ? "ready" : "infra-blocked", failureClassification, serviceId: options.serviceId, commit: options.commit, providerId, supportedArtifactPublish: true, missingChannels, missingControlChannels, controlChannels, channels, registry, controlPlane: publishPreflightControlPlane(transport, failureClassification), recommendedAction, remoteCommandShape: registryProbe.remoteCommandShape, next: ready ? [ `bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --wait-ms 1200000`, `later CD must consume ${plannedArtifact.imageRef}; CI itself must not deploy production`, ] : [ `Restore missing control channel(s): ${missingControlChannels.join(", ") || "unknown"}.`, failureClassification === "local-docker-required" ? "From a Code Queue runner, rerun this read-only preflight through the existing remote frontend transport: bun scripts/cli.ts --main-server-ip ci publish-user-service --service --commit --dry-run." : "Run from the main-server CLI or use remote frontend transport against a healthy frontend/backend-core path.", "Restore backend-core/database/provider-gateway/Host SSH connectivity before retrying artifact publication.", "Use bun scripts/cli.ts artifact-registry health --provider-id D601 to recheck registry reachability after the control bridge is restored.", ], boundary: "preflight is read-only: no D601 source export, no Tekton PipelineRun, no image push, no deploy apply, no service restart", }; } async function publishBackendCorePreflight( _config: UniDeskConfig, options: CiPublishBackendCoreOptions, plannedArtifact: ArtifactSummary, transport: PublishPreflightTransport, ): Promise { const providerId = d601ProviderId; const channels: PublishPreflightChannelProbe[] = []; const overview = await transport.coreFetch("/api/overview", { maxResponseBytes: 500_000 }); const overviewBody = coreBody(overview); const backendCoreOk = responseOk(overview) && overviewBody?.dbReady === true; channels.push(channelProbe("backend-core-api", "backend-core", backendCoreOk, "dispatch API, provider catalog, task polling, and database-backed CI state", { ok: responseOk(overview), dbReady: overviewBody?.dbReady ?? null, runnerDisposition: asRecord(overview)?.runnerDisposition ?? null, failureKind: asRecord(overview)?.failureKind ?? null, detail: backendCoreUnavailable(overview) ? overview : { status: asRecord(overview)?.status ?? null, body: overviewBody, }, })); channels.push(channelProbe("database", "database", backendCoreOk, "backend-core task dispatch, provider state, Tekton task polling, and source identity lookup", { dbReady: overviewBody?.dbReady ?? false, observedThrough: "backend-core /api/overview", })); const ciScript = ciRunnerPreflightScript(options.sourceHostPath); const ciProbe = await transport.dispatchHostSsh(ciScript, 30_000, 15_000); const ciRunnerReady = backendCoreCiRunnerReady(ciProbe); channels.push(channelProbe("provider-dispatch", "provider", ciProbe.taskId !== null || ciProbe.ok, "backend-core /api/dispatch can create D601 host.ssh tasks", { taskId: ciProbe.taskId, status: ciProbe.status, ok: ciProbe.taskId !== null || ciProbe.ok, exitCode: ciProbe.exitCode, stderrTail: ciProbe.stderr.slice(-1200), })); channels.push(channelProbe("provider-host-ssh", "provider", ciRunnerReady, "D601 host.ssh can observe Docker, kubectl, Tekton, PVC, and backend-core source parent without starting CI", { taskId: ciProbe.taskId, status: ciProbe.status, exitCode: ciProbe.exitCode, stdoutTail: ciProbe.stdout.slice(-1600), stderrTail: ciProbe.stderr.slice(-1200), raw: ciRunnerReady ? undefined : dispatchPreflightFailure("backend-core ci runner readonly probe", ciProbe).raw, })); const registryOptions = parseArtifactRegistryOptions(["--provider-id", providerId]); const registryProbe = buildArtifactRegistryReadonlyProbe("health", registryOptions); const registryDispatch = await transport.dispatchHostSsh(registryProbe.script, Math.max(registryProbe.timeoutMs, 30_000), registryProbe.timeoutMs); const registryCommand = commandResultFromDispatch(transport.artifactRegistryCommand(registryProbe), transport.commandCwd, registryDispatch); const registry = artifactRegistryReadonlyResultFromCommand(registryProbe, registryCommand); const registryRecord = asRecord(registry); const registryOk = registryRecord?.ok === true; channels.push(channelProbe("artifact-registry", "registry", registryOk, "backend-core commit-pinned image push and later dev CD manifest checks", registry)); const missingChannels = channels.filter((item) => !item.ok).map((item) => item.channel); const controlChannels = summarizePublishControlChannels(channels); const missingControlChannels = controlChannels.filter((item) => !item.ok).map((item) => item.channel); const ready = missingChannels.length === 0; const baseFailureClassification = ready ? null : classifyPublishPreflightFailure(transport, overview, ciProbe, registry); const failureClassification = !ready && baseFailureClassification === null ? "ci-runner-not-ready" : baseFailureClassification; const recommendedAction = ready ? `authorize and run: bun scripts/cli.ts ci publish-backend-core --commit ${options.commit} --wait-ms 1200000` : recommendedPublishPreflightAction(failureClassification, registry, missingControlChannels); return { ok: ready, runnerDisposition: ready ? "ready" : "infra-blocked", failureClassification, serviceId: "backend-core", commit: options.commit, providerId, supportedArtifactPublish: true, missingChannels, missingControlChannels, controlChannels, channels, registry, controlPlane: publishPreflightControlPlane(transport, failureClassification, backendCoreRemoteCommandShape(options.commit)), recommendedAction, remoteCommandShape: registryProbe.remoteCommandShape, next: ready ? [ `bun scripts/cli.ts ci publish-backend-core --commit ${options.commit} --wait-ms 1200000`, `verify ${plannedArtifact.imageRef} labels and digest before any dev artifact consumer apply`, `dev-only follow-up: bun scripts/cli.ts deploy apply --env dev --service backend-core --commit ${options.commit}`, ] : [ `Restore missing control channel(s): ${missingControlChannels.join(", ") || "unknown"}.`, failureClassification === "local-docker-required" ? `From a Code Queue runner, rerun this read-only preflight through the remote frontend transport: ${backendCoreRemoteCommandShape(options.commit)}.` : "Run from the main-server CLI or use remote frontend transport against a healthy frontend/backend-core path.", "Restore backend-core/database/provider-gateway/Host SSH/Tekton/registry readiness before retrying backend-core artifact publication.", "This preflight must remain read-only; do not start Tekton, compile Rust, push registry artifacts, deploy/apply, or restart services.", ], boundary: "preflight is read-only: no Tekton PipelineRun, no Rust compile, no source export, no image push, no registry write, no deploy apply, no service restart", }; } async function readArtifactSummaryFromPipelineRun(name: string, context: ArtifactSummaryContext): Promise { const result = await runRemoteKubectlRaw([ "set -euo pipefail", `kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o json`, ].join("\n"), 60_000, 45_000); if (result.ok && result.stdout.trim().length > 0) { try { const parsed = JSON.parse(result.stdout) as unknown; const fields = new Map(); const list = asRecord(parsed); const items = Array.isArray(list?.items) ? list.items : []; for (const item of items) { const taskRun = asRecord(item); const status = asRecord(taskRun?.status); const results = Array.isArray(status?.results) ? status.results : []; for (const rawResult of results) { const taskResult = asRecord(rawResult); const nameValue = asString(taskResult?.name); const value = asString(taskResult?.value); if (/^(user_service_artifact_[a-z_]+|backend_core_artifact_[a-z_]+)$/u.test(nameValue) && value.length > 0) { fields.set(nameValue, value); } } } if (fields.size > 0) { const fromResults = parseArtifactSummaryFromFields(fields, context); if (missingArtifactSummaryFields(fromResults).length === 0) return fromResults; const completedFromResults = await completeArtifactSummaryFromRegistry(fromResults, context); if (missingArtifactSummaryFields(completedFromResults).length === 0) return completedFromResults; } } catch { // Fall back to pod logs below; a malformed diagnostic line must not mask a succeeded PipelineRun. } } return completeArtifactSummaryFromRegistry(parseArtifactSummaryFromOutput(await readPipelineRunLogText(name), context), context); } async function readPipelineRunLogText(name: string, target = ciTarget(null)): Promise { const result = await runRemoteKubectlRaw([ "set -euo pipefail", `kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`, `kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o wide`, `for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail=240 || true; done`, ].join("\n"), 60_000, 45_000, target); return `${result.stdout}\n${result.stderr}`.trim(); } async function run(options: CiOptions): Promise> { const name = await remoteCreatePipelineRun(pipelineRunManifest(options), options.target); const wait = await waitForPipelineRun(name, options.waitMs, options.target); const condition = wait === null ? null : await readPipelineRunCondition(name, options.target); const waitSucceeded = pipelineRunWaitSucceeded(wait, condition); return { ok: waitSucceeded, providerId: options.target.providerId, pipelineRun: name, namespace: "unidesk-ci", repoUrl: options.repoUrl, revision: options.revision, wait: wait === null ? null : { ok: waitSucceeded, dispatchOk: wait.ok, dispatchStatus: wait.status, dispatchExitCode: wait.exitCode, stdoutTail: wait.stdout.slice(-6000), stderrTail: wait.stderr.slice(-6000), }, condition, next: [ `bun scripts/cli.ts ci logs ${name} --provider-id ${options.target.providerId}`, `bun scripts/cli.ts ci status --provider-id ${options.target.providerId}`, ], }; } async function publishBackendCoreArtifact(config: UniDeskConfig, options: CiPublishBackendCoreOptions): Promise> { const summaryContext: ArtifactSummaryContext = { serviceId: "backend-core", commit: options.commit, repoUrl: options.repoUrl, dockerfile: options.dockerfile, imageRepository: options.imageRepository, }; const plannedArtifact = artifactSummaryDefaults(summaryContext); if (options.dryRun) { return runCiPublishBackendCoreDryRunPreflight(config, [ "publish-backend-core", "--commit", options.commit, "--dry-run", ...(options.waitMs > 0 ? ["--wait-ms", String(options.waitMs)] : []), ], localPublishPreflightTransport, options); } const source = await prepareBackendCoreArtifactSource(config, options); const name = await remoteCreatePipelineRun(backendCoreArtifactPipelineRunManifest(options)); const wait = await waitForPipelineRun(name, options.waitMs); const condition = wait === null ? null : await readPipelineRunCondition(name); const waitSucceeded = pipelineRunWaitSucceeded(wait, condition); const artifact = waitSucceeded && wait !== null ? await readArtifactSummaryFromPipelineRun(name, summaryContext) : plannedArtifact; if (waitSucceeded && wait !== null) assertArtifactSummaryComplete(artifact, name); return { ok: waitSucceeded, pipelineRun: name, namespace: "unidesk-ci", repoUrl: options.repoUrl, commit: options.commit, source, artifact: artifact.imageRef, artifactSummary: artifact, boundary: "CI publishes the image to D601 registry; CD must pull it and must not build backend-core", wait: wait === null ? null : { ok: waitSucceeded, dispatchOk: wait.ok, dispatchStatus: wait.status, dispatchExitCode: wait.exitCode, stdoutTail: wait.stdout.slice(-6000), stderrTail: wait.stderr.slice(-6000), }, condition, next: [ `bun scripts/cli.ts ci logs ${name}`, `bun scripts/cli.ts deploy apply --env prod --service backend-core --commit ${options.commit}`, ], }; } async function publishUserServiceArtifact(config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions): Promise> { const summaryContext: ArtifactSummaryContext = { serviceId: options.serviceId, dockerfile: options.dockerfile, commit: options.commit, repoUrl: options.repoUrl, imageRepository: options.imageRepository, }; const plannedArtifact = artifactSummaryDefaults(summaryContext); const plannedRepoFetchUrl = repoSshUrl(options.repoUrl); if (options.dryRun) { const preflight = await publishUserServicePreflight(config, options, plannedArtifact, localPublishPreflightTransport); return { ok: preflight.ok, mode: "dry-run-preflight", runnerDisposition: preflight.runnerDisposition, pipeline: "unidesk-user-service-artifact-publish", namespace: "unidesk-ci", repoUrl: options.repoUrl, commit: options.commit, serviceId: options.serviceId, supportedArtifactPublish: preflight.supportedArtifactPublish, missingChannels: preflight.missingChannels, missingControlChannels: preflight.missingControlChannels, controlChannels: preflight.controlChannels, channels: preflight.channels, failureClassification: preflight.failureClassification, failedScopes: publishPreflightFailedScopes(preflight), recommendedAction: preflight.recommendedAction, remoteCommandShape: preflight.remoteCommandShape, controlPlane: preflight.controlPlane, registry: preflight.registry, sourceHostPath: options.sourceHostPath, source: { ok: preflight.ok, mode: "planned-only", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl: plannedRepoFetchUrl, repoProbeUrl: repoConnectivityProbeUrl(plannedRepoFetchUrl), commit: options.commit, serviceId: options.serviceId, dockerfile: options.dockerfile, imageRepository: options.imageRepository, sourceHostPath: options.sourceHostPath, ...(options.serviceId === "claudeqq" ? { overlay: "UniDesk claudeqq Dockerfile and unidesk-adapter.cjs are injected before Tekton build" } : {}), }, artifact: plannedArtifact.imageRef, artifactSummary: plannedArtifact, controlledPublish: { environment: "D601", namespace: "unidesk-ci", pipeline: "unidesk-user-service-artifact-publish", command: `bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --wait-ms 1200000 --transport ${options.transport}`, requiresReadyControlChannels: publishPreflightControlChannelOrder, }, boundary: preflight.boundary, next: preflight.next, }; } if (options.transport === "direct-docker" || (options.transport === "auto" && options.serviceId === "code-queue" && options.repoUrl === "https://github.com/pikasTech/unidesk")) { return publishUserServiceArtifactDirectDocker(options, summaryContext); } const source = options.serviceId === "claudeqq" ? await prepareClaudeqqArtifactSource(config, options) : await prepareUserServiceArtifactSource(config, options); const name = await remoteCreatePipelineRun(userServiceArtifactPipelineRunManifest(options)); const wait = await waitForPipelineRun(name, options.waitMs); const condition = wait === null ? null : await readPipelineRunCondition(name); const waitSucceeded = pipelineRunWaitSucceeded(wait, condition); const artifact = waitSucceeded && wait !== null ? await readArtifactSummaryFromPipelineRun(name, summaryContext) : plannedArtifact; if (waitSucceeded && wait !== null) assertArtifactSummaryComplete(artifact, name); return { ok: waitSucceeded, pipelineRun: name, namespace: "unidesk-ci", repoUrl: options.repoUrl, commit: options.commit, serviceId: options.serviceId, source, artifact: artifact.imageRef, artifactSummary: artifact, boundary: "CI publishes the user-service image to the D601 registry only; it must not deploy production or mutate the production namespace", wait: wait === null ? null : { ok: waitSucceeded, dispatchOk: wait.ok, dispatchStatus: wait.status, dispatchExitCode: wait.exitCode, stdoutTail: wait.stdout.slice(-6000), stderrTail: wait.stderr.slice(-6000), }, condition, next: [ `bun scripts/cli.ts ci logs ${name}`, "use artifactSummary.imageRef or artifactSummary.digestRef as later dev/prod deployment input", ], }; } export async function runCiPublishUserServiceDryRunPreflight( config: UniDeskConfig, args: string[], transport: PublishPreflightTransport, ): Promise> { const serviceId = requireServiceId(stringOption(args, "--service") ?? stringOption(args, "--service-id")); const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); if (!args.includes("--dry-run")) throw new Error("publish-user-service preflight requires --dry-run"); if (stringOption(args, "--repo") !== null || stringOption(args, "--repo-url") !== null) { throw new Error("ci publish-user-service reads source repo from CI.json; edit CI.json instead of using --repo"); } const artifact = resolveCatalogArtifact(serviceId); if (artifact.kind === "source-build" && artifact.serviceId === "backend-core") { throw new Error("backend-core uses ci publish-backend-core; publish-user-service is for registered user services"); } if (artifact.kind === "upstream-image") { return blockedArtifactResult(artifact, commit, artifact.blockedReason); } if (artifact.status === "blocked") { return blockedArtifactResult(artifact, commit, blockedReason(artifact)); } const dockerfile = requireRepoRelativePath(artifact.source.dockerfile, `CI.json.artifacts.${serviceId}.source.dockerfile`); const boundaryBlock = userServicePublishBoundaryBlock(config, serviceId, commit, artifact); if (boundaryBlock !== null) return boundaryBlock; const summaryContext: ArtifactSummaryContext = { serviceId, dockerfile, commit, repoUrl: artifact.source.repo, imageRepository: artifact.image.repository, }; const plannedArtifact = artifactSummaryDefaults(summaryContext); const options: CiPublishUserServiceArtifactOptions = { repoUrl: artifact.source.repo, commit, waitMs: numberOption(args, "--wait-ms", 0), serviceId, dockerfile, imageRepository: artifact.image.repository, sourceHostPath: userServiceArtifactSourceHostPath(serviceId, commit), dryRun: true, transport: publishTransportOption(stringOption(args, "--transport")), }; const preflight = await publishUserServicePreflight(config, options, plannedArtifact, transport); const plannedRepoFetchUrl = repoSshUrl(options.repoUrl); return { ok: preflight.ok, mode: "dry-run-preflight", runnerDisposition: preflight.runnerDisposition, pipeline: "unidesk-user-service-artifact-publish", namespace: "unidesk-ci", repoUrl: options.repoUrl, commit: options.commit, serviceId: options.serviceId, supportedArtifactPublish: preflight.supportedArtifactPublish, missingChannels: preflight.missingChannels, missingControlChannels: preflight.missingControlChannels, controlChannels: preflight.controlChannels, channels: preflight.channels, failureClassification: preflight.failureClassification, failedScopes: publishPreflightFailedScopes(preflight), recommendedAction: preflight.recommendedAction, remoteCommandShape: preflight.remoteCommandShape, controlPlane: preflight.controlPlane, registry: preflight.registry, sourceHostPath: options.sourceHostPath, source: { ok: preflight.ok, mode: "planned-only", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl: plannedRepoFetchUrl, repoProbeUrl: repoConnectivityProbeUrl(plannedRepoFetchUrl), commit: options.commit, serviceId: options.serviceId, dockerfile: options.dockerfile, imageRepository: options.imageRepository, sourceHostPath: options.sourceHostPath, ...(options.serviceId === "claudeqq" ? { overlay: "UniDesk claudeqq Dockerfile and unidesk-adapter.cjs are injected before Tekton build" } : {}), }, artifact: plannedArtifact.imageRef, artifactSummary: plannedArtifact, controlledPublish: { environment: "D601", namespace: "unidesk-ci", pipeline: "unidesk-user-service-artifact-publish", command: `bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --wait-ms 1200000 --transport ${options.transport}`, requiresReadyControlChannels: publishPreflightControlChannelOrder, }, boundary: preflight.boundary, next: preflight.next, }; } export async function runCiPublishBackendCoreDryRunPreflight( config: UniDeskConfig, args: string[], transport: PublishPreflightTransport, resolvedOptions?: CiPublishBackendCoreOptions, ): Promise> { const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); if (!args.includes("--dry-run")) throw new Error("publish-backend-core preflight requires --dry-run"); if (stringOption(args, "--repo") !== null || stringOption(args, "--repo-url") !== null) { throw new Error("ci publish-backend-core reads source repo from CI.json; edit CI.json instead of using --repo"); } const artifact = resolveCatalogArtifact("backend-core"); if (artifact.kind !== "source-build") throw new Error("backend-core must be modeled as a source-build artifact in CI.json"); if (artifact.status === "blocked") return blockedArtifactResult(artifact, commit, blockedReason(artifact)); const dockerfile = requireRepoRelativePath(artifact.source.dockerfile, "CI.json.artifacts.backend-core.source.dockerfile"); const options: CiPublishBackendCoreOptions = resolvedOptions ?? { repoUrl: artifact.source.repo, commit, waitMs: numberOption(args, "--wait-ms", 0), sourceHostPath: backendCoreArtifactSourceHostPath(commit), dockerfile, imageRepository: artifact.image.repository, dryRun: true, }; const summaryContext: ArtifactSummaryContext = { serviceId: "backend-core", commit, repoUrl: options.repoUrl, dockerfile: options.dockerfile, imageRepository: options.imageRepository, }; const plannedArtifact = artifactSummaryDefaults(summaryContext); const preflight = await publishBackendCorePreflight(config, options, plannedArtifact, transport); const plannedRepoFetchUrl = repoSshUrl(options.repoUrl); const blockedScopes = publishPreflightFailedScopes(preflight); return { ok: preflight.ok, mode: "dry-run-preflight", runnerDisposition: preflight.runnerDisposition, pipeline: "unidesk-backend-core-artifact-publish", namespace: "unidesk-ci", repoUrl: options.repoUrl, commit: options.commit, targetCommit: options.commit, sourceRepo: options.repoUrl, serviceId: "backend-core", providerId: d601ProviderId, ciRunner: { providerId: d601ProviderId, environment: "D601", namespace: "unidesk-ci", pipeline: "unidesk-backend-core-artifact-publish", wouldBuildOnD601: true, realBuildRequiresAuthorization: true, }, registryTarget: plannedArtifact.repository, wouldBuildOnD601: true, dryRunBuildStarted: false, supportedArtifactPublish: preflight.supportedArtifactPublish, missingChannels: preflight.missingChannels, missingControlChannels: preflight.missingControlChannels, controlChannels: preflight.controlChannels, channels: preflight.channels, failureClassification: preflight.failureClassification, failedScopes: blockedScopes, blockedScopes, recommendedAction: preflight.recommendedAction, remoteCommandShape: preflight.remoteCommandShape, remoteControlPlaneCandidate: preflight.controlPlane, controlPlane: preflight.controlPlane, registry: preflight.registry, sourceHostPath: options.sourceHostPath, source: { ok: preflight.ok, mode: "planned-only", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl: plannedRepoFetchUrl, repoProbeUrl: repoConnectivityProbeUrl(plannedRepoFetchUrl), commit: options.commit, serviceId: "backend-core", dockerfile: options.dockerfile, imageRepository: options.imageRepository, sourceHostPath: options.sourceHostPath, }, sourceAuth: { repoFetchUrl: plannedRepoFetchUrl, egressProxy: providerGatewayWsEgressProxyUrl, gitSshProxy: repoNeedsGithubSshIdentity(plannedRepoFetchUrl), identityRequired: repoNeedsGithubSshIdentity(plannedRepoFetchUrl), }, d601Ci: { providerId: d601ProviderId, namespace: "unidesk-ci", pipeline: "unidesk-backend-core-artifact-publish", wouldBuildOnD601: true, dryRunWillStartTekton: false, dryRunWillCompileRust: false, dryRunWillPushRegistry: false, }, artifact: plannedArtifact.imageRef, artifactSummary: plannedArtifact, artifactRequirements: backendCoreArtifactRequirements(options, plannedArtifact), controlledPublish: { environment: "D601", namespace: "unidesk-ci", pipeline: "unidesk-backend-core-artifact-publish", command: `bun scripts/cli.ts ci publish-backend-core --commit ${options.commit} --wait-ms 1200000`, requiresReadyControlChannels: publishPreflightControlChannelOrder, requiresHumanAuthorization: true, }, devApplyPath: backendCoreDevApplyPath(options, plannedArtifact), boundary: preflight.boundary, next: preflight.next, }; } async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise { const scriptTimeoutMs = Math.max(options.scriptTimeoutMs, options.waitMs, 60_000); const remoteTimeoutMs = 45_000; const command = [ "set -euo pipefail", `run_id=${shellQuote(options.runId)}`, `repo_url=${shellQuote(options.scriptRepo)}`, `commit=${shellQuote(options.deployCommit)}`, `script_path=${shellQuote(options.scriptPath)}`, `desired_ref=${shellQuote(options.desiredRef)}`, `environment=${shellQuote(options.environment)}`, `keep_namespace=${shellQuote(options.keepNamespace ? "true" : "false")}`, `timeout_ms=${shellQuote(String(scriptTimeoutMs))}`, "work_dir=\"/tmp/unidesk-ci/$run_id\"", "result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"", "mkdir -p \"$work_dir\" \"$result_dir\"", "launcher_log=\"$result_dir/launcher.log\"", "case \"$script_path\" in scripts/ci/*.sh) ;; *) echo \"invalid_script_path=$script_path\" >&2; exit 2 ;; esac", "(", "set -euo pipefail", "trap '' HUP", "exec >> \"$launcher_log\" 2>&1", "echo \"launcher_run_id=$run_id\"", "echo \"launcher_repo=$repo_url\"", "echo \"launcher_commit=$commit\"", "echo \"launcher_script_path=$script_path\"", "export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config", "mkdir -p \"$DOCKER_CONFIG\"", "printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"", `build_proxy=${shellQuote(providerGatewayWsEgressProxyUrl)}`, "export HTTP_PROXY=\"$build_proxy\" HTTPS_PROXY=\"$build_proxy\" ALL_PROXY=\"$build_proxy\"", "export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"", "if ! curl -fsSI --max-time 20 -x \"$build_proxy\" https://github.com >/dev/null; then", " echo \"ci_provider_egress_proxy_unavailable=$build_proxy\" >&2", " exit 1", "fi", "echo \"ci_provider_egress_proxy=provider-gateway-ws-egress:$build_proxy\"", "repo_fetch_url=\"$repo_url\"", "case \"$repo_fetch_url\" in", " https://github.com/*)", " repo_path=\"${repo_fetch_url#https://github.com/}\"", " repo_path=\"${repo_path%.git}\"", " repo_fetch_url=\"git@github.com:$repo_path.git\"", " ;;", "esac", "export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=nc -X connect -x 127.0.0.1:18789 %h %p'\"", "echo \"launcher_repo_fetch_url=$repo_fetch_url\"", "repo_dir=\"$work_dir/repo\"", "if [ ! -d \"$repo_dir/.git\" ]; then", " git clone --no-checkout \"$repo_fetch_url\" \"$repo_dir\"", "fi", "git -C \"$repo_dir\" remote set-url origin \"$repo_fetch_url\"", "git -C \"$repo_dir\" fetch --no-tags origin \"$commit\" || git -C \"$repo_dir\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git -C \"$repo_dir\" rev-parse --verify \"$commit^{commit}\")", "test \"$resolved\" = \"$commit\" || { echo \"resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }", "git -C \"$repo_dir\" cat-file -e \"$resolved:$script_path\"", "git -C \"$repo_dir\" show \"$resolved:$script_path\" > \"$work_dir/runner.sh\"", "git -C \"$repo_dir\" show \"$resolved:deploy.json\" > \"$work_dir/deploy.json\"", "chmod 700 \"$work_dir/runner.sh\"", "echo \"runner_script_ready=$work_dir/runner.sh\"", "runner_args=(", " --run-id \"$run_id\"", " --repo-url \"$repo_url\"", " --desired-ref \"$desired_ref\"", " --manifest-commit \"$commit\"", " --manifest-file \"$work_dir/deploy.json\"", " --environment \"$environment\"", " --result-dir \"$result_dir\"", " --timeout-ms \"$timeout_ms\"", ")", "if [ \"$keep_namespace\" = \"true\" ]; then runner_args+=(--keep-namespace); fi", "bash \"$work_dir/runner.sh\" \"${runner_args[@]}\"", ") &", "launcher_pid=$!", "disown \"$launcher_pid\" 2>/dev/null || true", "printf 'launcher_background_pid=%s\\nresult_dir=%s\\n' \"$launcher_pid\" \"$result_dir\"", ].join("\n"); return dispatchSsh(command, 30_000, remoteTimeoutMs); } async function waitForDevE2EResult(runId: string, waitMs: number): Promise { if (waitMs <= 0) return null; const deadline = Date.now() + waitMs; let latest: DispatchResult | null = null; while (Date.now() < deadline) { const result = await dispatchSsh([ "set -euo pipefail", `run_id=${shellQuote(runId)}`, "result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"", "if [ -f \"$result_dir/result.json\" ]; then cat \"$result_dir/result.json\"; exit 0; fi", "printf 'RUNNING result_dir=%s\\n' \"$result_dir\"", "tail -n 40 \"$result_dir/launcher.log\" 2>/dev/null || true", "tail -n 80 \"$result_dir/runner.log\" 2>/dev/null || true", ].join("\n"), 30_000, 20_000); latest = result; const stdout = result.stdout.trimStart(); if (stdout.startsWith("{")) { const parsed = JSON.parse(stdout) as { ok?: boolean; status?: string }; return { ...result, ok: parsed.ok === true, status: parsed.status ?? (parsed.ok === true ? "succeeded" : "failed"), exitCode: parsed.ok === true ? 0 : 1, }; } await Bun.sleep(5_000); } return { ok: false, taskId: latest?.taskId ?? null, status: "timeout", stdout: latest?.stdout ?? "", stderr: `dev e2e result did not finish within ${waitMs}ms`, exitCode: 124, raw: latest?.raw ?? null, }; } function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary { const remoteRef = `refs/remotes/origin/${desiredRef}`; const fetch = runCommand(["git", "fetch", "--quiet", "origin", `+refs/heads/${desiredRef}:${remoteRef}`], repoRoot); if (fetch.exitCode !== 0) throw new Error(`failed to fetch origin/${desiredRef}: ${fetch.stderr || fetch.stdout}`); const deployCommitResult = runCommand(["git", "rev-parse", `origin/${desiredRef}`], repoRoot); if (deployCommitResult.exitCode !== 0) throw new Error(`failed to resolve origin/${desiredRef}: ${deployCommitResult.stderr || deployCommitResult.stdout}`); const show = runCommand(["git", "show", `origin/${desiredRef}:deploy.json`], repoRoot); if (show.exitCode !== 0) throw new Error(`failed to read deploy.json from origin/${desiredRef}: ${show.stderr || show.stdout}`); const parsed = JSON.parse(show.stdout) as unknown; const record = asRecord(parsed); if (record?.schemaVersion !== 2) throw new Error(`origin/${desiredRef}:deploy.json must use schemaVersion=2`); const environments = asRecord(record.environments); const dev = asRecord(environments?.dev); const ci = asRecord(dev?.ci); if (ci === null) throw new Error(`origin/${desiredRef}:deploy.json must contain environments.dev.ci`); const rawServices = Array.isArray(dev?.services) ? dev.services : []; const services = rawServices.map((item) => { const service = asRecord(item); return { id: asString(service?.id), commitId: asString(service?.commitId).toLowerCase(), repo: asString(service?.repo), }; }).filter((service) => service.id.length > 0 && service.commitId.length > 0); if (services.length === 0) throw new Error(`origin/${desiredRef}:deploy.json has no environments.dev services with commitId`); const codeQueueService = services.find((service) => service.id === "code-queue"); if (codeQueueService === undefined) { throw new Error(`origin/${desiredRef}:deploy.json environments.dev.services must include code-queue for ci run-dev-e2e`); } if (!/^[0-9a-f]{40}$/u.test(codeQueueService.commitId)) { throw new Error(`origin/${desiredRef}:deploy.json environments.dev.services code-queue commitId must be a full 40-character SHA`); } return { deployCommit: deployCommitResult.stdout.trim(), desiredRef, environment: "dev", ci: { repo: requireManifestString(ci.repo, "environments.dev.ci.repo"), scriptPath: requireCiScriptPath(ci.scriptPath), timeoutMs: positiveManifestNumber(ci.timeoutMs, 1_800_000, "environments.dev.ci.timeoutMs"), }, services, }; } function makeRunId(deployCommit: string): string { const stamp = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase(); return `${stamp}-${deployCommit.slice(0, 8).toLowerCase()}`.replace(/[^a-z0-9-]/gu, "-").slice(0, 48); } async function runDevE2E(options: CiDevE2EOptions): Promise> { const result = await runRemoteDevE2ELauncher(options); const wait = result.ok ? await waitForDevE2EResult(options.runId, options.waitMs) : null; const ok = result.ok && (result.exitCode === null || result.exitCode === 0) && (wait === null || wait.ok); return { ok, runId: options.runId, namespace: "unidesk-ci", temporaryNamespace: `unidesk-ci-e2e-${options.runId}`, repoUrl: options.repoUrl, desiredRef: options.desiredRef, deployCommit: options.deployCommit, scriptRepo: options.scriptRepo, scriptPath: options.scriptPath, environment: options.environment, services: options.services, keepNamespace: options.keepNamespace, triggerMode: "commit-pinned-ssh-launcher", launcher: { taskId: result.taskId, status: result.status, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-6000), stderrTail: result.stderr.slice(-6000), }, wait: wait === null ? null : { status: wait.status, exitCode: wait.exitCode, stdoutTail: wait.stdout.slice(-6000), stderrTail: wait.stderr.slice(-6000), }, resultDir: `/home/ubuntu/.unidesk/runs/${options.runId}`, next: [ `bun scripts/cli.ts ci logs ${options.runId}`, "bun scripts/cli.ts ci status", ], }; } function remoteRunLogsScript(name: string, target: CiTarget, options: CiLogsOptions): string { return [ "set -eu", `run_id=${shellQuote(name)}`, `result_root=${shellQuote(`${target.homeDir}/.unidesk/runs`)}`, `tail_lines=${shellQuote(String(options.tailLines))}`, "result_dir=\"$result_root/$run_id\"", "printf 'result_dir=%s\\n' \"$result_dir\"", "found=0", "if [ -f \"$result_dir/result.json\" ]; then found=1; echo '===== result.json'; cat \"$result_dir/result.json\"; fi", "if [ -f \"$result_dir/launcher.log\" ]; then found=1; echo '===== launcher.log'; tail -n \"$tail_lines\" \"$result_dir/launcher.log\"; fi", "if [ -f \"$result_dir/runner.log\" ]; then found=1; echo '===== runner.log'; tail -n \"$tail_lines\" \"$result_dir/runner.log\"; fi", "if [ -f \"$result_dir/pods.log\" ]; then found=1; echo '===== pods.log'; tail -n \"$tail_lines\" \"$result_dir/pods.log\"; fi", "if [ \"$found\" = \"0\" ]; then echo \"no_run_files=$result_dir\" >&2; exit 42; fi", ].join("\n"); } function pipelineRunLogsScript(name: string, options: CiLogsOptions): string { return [ "set -eu", `tail_lines=${shellQuote(String(options.tailLines))}`, `kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`, `kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o wide`, "echo '===== pipelinerun conditions'", `kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o jsonpath='{range .status.conditions[*]}condition={.type} status={.status} reason={.reason} message={.message}{"\\n"}{end}' || true`, "echo '===== taskrun conditions'", `kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o jsonpath='{range .items[*]}taskrun={.metadata.name} phase={.status.conditions[0].type} status={.status.conditions[0].status} reason={.status.conditions[0].reason} message={.status.conditions[0].message}{"\\n"}{end}' || true`, "echo '===== pod log tails'", `for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail="$tail_lines" || true; done`, ].join("\n"); } function ciLogsResultFromCapture(args: { ok: boolean; providerId: string; kind: "run" | "pipelinerun"; name: string; options: CiLogsOptions; stdout: string; stderr: string; exitCode: number | null; dispatchTaskId?: string | null; }): Record { const combined = `${args.stdout}\n${args.stderr}`.trim(); const nextName = args.kind === "run" ? "runId" : "pipelineRun"; return { ok: args.ok, providerId: args.providerId, [nextName]: args.name, capture: args.options.capture, tailLines: args.options.tailLines, lineCount: countTextLines(combined), outputTail: boundedHeadTail(tailTextLines(args.stdout, args.options.tailLines * 4), 2400, 2400), stderrTail: boundedHeadTail(tailTextLines(args.stderr, Math.min(args.options.tailLines, 80)), 1200, 1200), conditionHints: extractCiLogConditionHints(combined), failureHints: extractCiLogFailureHints(combined), exitCode: args.exitCode, ...(args.dispatchTaskId === undefined ? {} : { dispatchTaskId: args.dispatchTaskId }), fullOutputAvailable: args.options.capture === "ssh-stream", next: [ `bun scripts/cli.ts ci logs ${args.name} --provider-id ${args.providerId} --tail-lines ${Math.min(args.options.tailLines * 2, 2000)}`, `bun scripts/cli.ts ci status --provider-id ${args.providerId}`, ], }; } async function logs(config: UniDeskConfig, name: string, target = ciTarget(null), options: CiLogsOptions = { tailLines: 160, capture: "ssh-stream" }): Promise> { if (name.length === 0) throw new Error("ci logs requires run id or PipelineRun name"); if (/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(name)) { const runScript = remoteRunLogsScript(name, target, options); const result = options.capture === "ssh-stream" ? await runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], runScript) : await dispatchSsh(runScript, 60_000, 45_000, true, target); const resultOk = "ok" in result ? result.ok : result.exitCode === 0; if (resultOk || (result.exitCode !== 42 && !result.stderr.includes("no_run_files="))) { return ciLogsResultFromCapture({ ok: resultOk, providerId: target.providerId, kind: "run", name, options, stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, dispatchTaskId: "taskId" in result ? result.taskId : undefined, }); } } const script = pipelineRunLogsScript(name, options); const result = options.capture === "ssh-stream" ? await runSshCommandCapture(config, `${target.providerId}:k3s`, ["sh"], script) : await runRemoteKubectl(script, 60_000, 45_000, target); return ciLogsResultFromCapture({ ok: result.exitCode === 0, providerId: target.providerId, kind: "pipelinerun", name, options, stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, dispatchTaskId: "taskId" in result ? result.taskId : undefined, }); } function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record { if (artifact.kind === "source-build") { return { serviceId: artifact.serviceId, kind: artifact.kind, status: artifact.status, producer: artifact.producer, source: artifact.source, image: artifact.image, ...(artifact.notes === undefined ? {} : { notes: artifact.notes }), ...(artifact.blockedReason === undefined ? {} : { blockedReason: artifact.blockedReason }), }; } return { serviceId: artifact.serviceId, kind: artifact.kind, status: artifact.status, producer: artifact.producer, upstream: artifact.upstream, blockedReason: artifact.blockedReason, ...(artifact.notes === undefined ? {} : { notes: artifact.notes }), }; } export function ciHelp(): Record { const catalog = loadCiCatalog(); return { command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs|cleanup-runs|cleanup-failed-pods", description: "Manage native k3s Tekton CI on D601 or G14. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.", examples: [ "bun scripts/cli.ts ci install", "bun scripts/cli.ts ci install --skip-prewarm", "bun scripts/cli.ts ci install --skip-prewarm --skip-tekton-install", "bun scripts/cli.ts ci install-status latest", "bun scripts/cli.ts ci install --wait --skip-prewarm --skip-tekton-install", "bun scripts/cli.ts ci install --provider-id G14", "bun scripts/cli.ts ci run --revision ", "bun scripts/cli.ts ci run --provider-id G14 --revision ", "bun scripts/cli.ts ci publish-backend-core --commit ", "bun scripts/cli.ts ci publish-user-service --service baidu-netdisk --commit ", "bun scripts/cli.ts ci publish-user-service --service mdtodo --commit ", "bun scripts/cli.ts ci publish-user-service --service claudeqq --commit ", "bun scripts/cli.ts ci publish-user-service --service code-queue --commit ", "bun scripts/cli.ts ci publish-user-service --service decision-center --commit ", "bun scripts/cli.ts ci publish-user-service --service frontend --commit ", "bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000", "bun scripts/cli.ts ci logs [--provider-id G14] [--tail-lines 80]", "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --dry-run", "bun scripts/cli.ts ci cleanup-runs --provider-id D601 --min-age-minutes 60 --limit 50 --confirm", "bun scripts/cli.ts ci cleanup-failed-pods --provider-id D601 --namespace unidesk-ci --min-age-minutes 60 --dry-run", "bun scripts/cli.ts ci cleanup-failed-pods --provider-id D601 --namespace unidesk-ci --min-age-minutes 60 --confirm", ], tekton: { pipelineVersion: tektonPipelineVersion, triggersVersion: tektonTriggersVersion, sources: { pipeline: tektonPipelineReleaseUrl, triggers: tektonTriggersReleaseUrl, interceptors: tektonTriggersInterceptorsUrl, }, targets: { default: "D601", g14: { providerId: "G14", pipelineManifest: g14CiPipelineManifest, nodeSelector: "unidesk.ai/node-id=G14", }, }, }, install: { defaultMode: "async-job", waitFlag: "Use --wait only for explicit synchronous debugging; the default returns a .state/jobs job immediately with install-status follow-up.", statusCommand: "bun scripts/cli.ts ci install-status ", prewarmDefault: true, skipPrewarm: "Use --skip-prewarm only to refresh Tekton/CI manifests when runtime images are already present or prewarm is blocked by provider root/containerd permissions.", skipTektonInstall: "Use --skip-tekton-install with --skip-prewarm only when Tekton is already installed and only UniDesk CI manifests/triggers need refreshing.", }, backendCoreArtifact: { producer: "D601 CI", registry: "127.0.0.1:5000/unidesk/backend-core:", cdCommand: "bun scripts/cli.ts deploy apply --env prod --service backend-core --commit ", }, userServiceArtifact: { producer: "D601 CI", command: "bun scripts/cli.ts ci publish-user-service --service --commit ", transports: { auto: "uses direct-docker for repo-owned code-queue artifacts and Tekton for the remaining services", tekton: "D601 Tekton PipelineRun through backend-core/provider control plane", directDocker: "repo-owned Docker artifact publish without backend-core/database dispatch; no deploy apply, rollout, restart, or active-task mutation", }, supportedServices: supportedSourceBuildArtifactIds().filter((serviceId) => serviceId !== "backend-core"), blockedServices: blockedCatalogArtifactIds(), registry: "127.0.0.1:5000/unidesk/:", outputFields: ["serviceId", "sourceCommit", "sourceRepo", "dockerfile", "imageRef", "tag", "digest", "digestRef"], summaryContract: catalog.summaryContract, catalogSummary: catalogSummary(), catalog: catalog.artifacts.map(catalogArtifactDescriptor), boundary: "artifact producer only; no prod deploy and no production namespace mutation", frontendNext: [ "bun scripts/cli.ts deploy apply --env dev --service frontend --commit ", "bun scripts/cli.ts deploy apply --env prod --service frontend --commit ", ], }, runDevE2E: { defaultTriggerMode: "commit-pinned-ssh-launcher", desiredState: "origin/master:deploy.json#environments.dev", scriptSource: "origin/master:deploy.json#environments.dev.ci", }, logs: { defaultCapture: "ssh-stream", dispatchTaskFallback: "--dispatch-task", tailLines: "1..2000", reason: "stream capture avoids backend-core task-result compactJson truncating CI log text before the CLI can extract failure hints", }, cleanupRuns: { defaultMode: "dry-run", namespace: "unidesk-ci", selector: "status.phase in Succeeded,Failed", guard: "--confirm required for deletion; Running/Pending pods are never selected", }, cleanupFailedPods: { defaultNamespace: "unidesk-ci", defaultMinAgeMinutes: 60, defaultLimit: 50, scope: "Kubernetes pods with status.phase=Failed only; does not touch Running/Pending pods, workloads, PVCs, Secrets, or Tekton resources", defaultMode: "dry-run", }, }; } function requireRunId(value: string): string { if (!/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(value)) { throw new Error("ci run-dev-e2e run id must be DNS-safe lowercase alnum/dash, max 48 chars"); } return value; } export async function runCiCommand(config: UniDeskConfig, args: string[]): Promise> { const [action = "status", nameArg] = args; if (isHelpArg(action) || args.slice(1).some(isHelpArg)) return ciHelp(); if (action === "install") { const options = { target: ciTarget(providerIdOption(args)), skipPrewarm: boolFlag(args, "--skip-prewarm"), skipTektonInstall: boolFlag(args, "--skip-tekton-install"), wait: boolFlag(args, "--wait"), }; return options.wait ? install(config, options) : installAsync(options); } if (action === "install-status") return installStatus(nameArg ?? "latest"); if (action === "status") return status(ciTarget(providerIdOption(args))); if (action === "run") { const target = ciTarget(providerIdOption(args)); const repoUrl = stringOption(args, "--repo") ?? stringOption(args, "--repo-url") ?? "https://github.com/pikasTech/unidesk"; const revision = requireRevision(stringOption(args, "--revision") ?? stringOption(args, "--commit")); const waitMs = numberOption(args, "--wait-ms", 0); return run({ repoUrl, revision, waitMs, target }); } if (action === "publish-backend-core") { if (stringOption(args, "--repo") !== null || stringOption(args, "--repo-url") !== null) { throw new Error("ci publish-backend-core reads source repo from CI.json; edit CI.json instead of using --repo"); } const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); const waitMs = numberOption(args, "--wait-ms", 0); const dryRun = boolFlag(args, "--dry-run"); const artifact = resolveCatalogArtifact("backend-core"); if (artifact.kind !== "source-build") throw new Error("backend-core must be modeled as a source-build artifact in CI.json"); if (artifact.status === "blocked") return blockedArtifactResult(artifact, commit, blockedReason(artifact)); return publishBackendCoreArtifact(config, { repoUrl: artifact.source.repo, commit, waitMs, sourceHostPath: backendCoreArtifactSourceHostPath(commit), dockerfile: artifact.source.dockerfile, imageRepository: artifact.image.repository, dryRun, }); } if (action === "publish-user-service") { const serviceId = requireServiceId(stringOption(args, "--service") ?? stringOption(args, "--service-id")); const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); const waitMs = numberOption(args, "--wait-ms", 0); const dryRun = boolFlag(args, "--dry-run"); const transport = publishTransportOption(stringOption(args, "--transport")); if (stringOption(args, "--repo") !== null || stringOption(args, "--repo-url") !== null) { throw new Error("ci publish-user-service reads source repo from CI.json; edit CI.json instead of using --repo"); } const artifact = resolveCatalogArtifact(serviceId); if (artifact.kind === "source-build" && artifact.serviceId === "backend-core") { throw new Error("backend-core uses ci publish-backend-core; publish-user-service is for registered user services"); } if (artifact.kind === "upstream-image") { return blockedArtifactResult(artifact, commit, artifact.blockedReason); } if (artifact.status === "blocked") { return blockedArtifactResult(artifact, commit, blockedReason(artifact)); } const repoUrl = artifact.source.repo; const dockerfile = requireRepoRelativePath(artifact.source.dockerfile, `CI.json.artifacts.${serviceId}.source.dockerfile`); const boundaryBlock = userServicePublishBoundaryBlock(config, serviceId, commit, artifact); if (boundaryBlock !== null) return boundaryBlock; return publishUserServiceArtifact(config, { repoUrl, commit, waitMs, serviceId, dockerfile, imageRepository: artifact.image.repository, sourceHostPath: userServiceArtifactSourceHostPath(serviceId, commit), dryRun, transport, }); } if (action === "run-dev-e2e") { const repoUrl = stringOption(args, "--repo") ?? stringOption(args, "--repo-url") ?? "https://github.com/pikasTech/unidesk"; const desiredRef = requireDesiredRef(stringOption(args, "--desired-ref") ?? stringOption(args, "--deploy-branch")); const manifest = resolveDeployDevManifest(desiredRef); const waitMs = numberOption(args, "--wait-ms", 0); const runId = requireRunId(stringOption(args, "--run-id") ?? makeRunId(manifest.deployCommit)); return runDevE2E({ repoUrl, desiredRef, deployCommit: manifest.deployCommit, environment: manifest.environment, scriptRepo: manifest.ci.repo, scriptPath: manifest.ci.scriptPath, scriptTimeoutMs: manifest.ci.timeoutMs, services: manifest.services, runId, keepNamespace: boolFlag(args, "--keep-namespace"), waitMs, }); } if (action === "logs") return logs(config, nameArg ?? "", ciTarget(providerIdOption(args)), ciLogsOptions(args)); if (action === "cleanup-runs") return cleanupRuns(config, ciCleanupRunsOptions(args)); if (action === "cleanup-failed-pods") return cleanupFailedPods(config, ciCleanupFailedPodsOptions(args)); throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs, cleanup-runs, cleanup-failed-pods"); } export function startCiInstallJob(providerId = d601ProviderId): Record { return installAsync({ target: ciTarget(providerId), skipPrewarm: false, skipTektonInstall: false, wait: false, }); }