Codex
284 lines
16 KiB
TypeScript
284 lines
16 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. publish-preflight module for scripts/src/ci.ts.
|
|
// SPEC: PJ2026-01060308 cicd-yaml-targets draft-2026-06-25-cicd-yaml-targets.
|
|
|
|
// Moved mechanically from scripts/src/ci.ts:2317-2567 for #903.
|
|
|
|
import { randomUUID } from "node:crypto";
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, posix as posixPath } from "node:path";
|
|
import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "../ci-catalog";
|
|
import { runCommand } from "../command";
|
|
import { type UniDeskConfig, repoRoot, rootPath } from "../config";
|
|
import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "../deploy-ssh-identity";
|
|
import { jobWithTail, listJobs, readJob, startJob } from "../jobs";
|
|
import { coreInternalFetch } from "../microservices";
|
|
import {
|
|
artifactRegistryReadonlyResultFromCommand,
|
|
buildArtifactRegistryReadonlyProbe,
|
|
parseArtifactRegistryOptions,
|
|
type ArtifactRegistryReadonlyProbe,
|
|
} from "../artifact-registry";
|
|
import { k3sGuardShellLines, defaultNativeKubeconfig } from "../k3s-target-guard";
|
|
import { runSshCommandCapture } from "../ssh";
|
|
|
|
import type { ArtifactSummary, ArtifactSummaryContext, CiPublishBackendCoreOptions, CiPublishUserServiceArtifactOptions, PublishPreflight, PublishPreflightChannelProbe, PublishPreflightTransport } from "./types";
|
|
import { completeArtifactSummaryFromRegistry, missingArtifactSummaryFields, parseArtifactSummaryFromFields, parseArtifactSummaryFromOutput } from "./artifact-summary";
|
|
import { status } from "./cleanup";
|
|
import { shellQuote } from "./options";
|
|
import { asRecord, asString, backendCoreUnavailable, channelProbe, classifyPublishPreflightFailure, coreBody, publishPreflightControlPlane, recommendedPublishPreflightAction, responseOk, summarizePublishControlChannels } from "./preflight";
|
|
import { runRemoteKubectlRaw } from "./remote";
|
|
import { backendCoreCiRunnerReady, backendCoreRemoteCommandShape, ciRunnerPreflightScript, commandResultFromDispatch, dispatchPreflightFailure } from "./runner-preflight";
|
|
import { ciTarget, defaultCiKubeconfig, defaultCiProviderId } from "./types";
|
|
|
|
export function assertArtifactSummaryComplete(artifact: ArtifactSummary, pipelineRun: string): void {
|
|
const missing = missingArtifactSummaryFields(artifact);
|
|
if (missing.length > 0) {
|
|
throw new Error(`artifact summary for ${pipelineRun} is missing required field(s): ${missing.join(", ")}`);
|
|
}
|
|
}
|
|
|
|
export async function publishUserServicePreflight(
|
|
_config: UniDeskConfig,
|
|
options: CiPublishUserServiceArtifactOptions,
|
|
plannedArtifact: ArtifactSummary,
|
|
transport: PublishPreflightTransport,
|
|
): Promise<PublishPreflight> {
|
|
const providerId = defaultCiProviderId;
|
|
const channels: PublishPreflightChannelProbe[] = [];
|
|
const overview = await transport.coreFetch("/api/overview", { maxResponseBytes: 500_000 });
|
|
const overviewBody = coreBody(overview);
|
|
const backendCoreOk = responseOk(overview) && overviewBody?.dbReady === true;
|
|
channels.push(channelProbe("backend-core-api", "backend-core", backendCoreOk, "dispatch API, provider catalog, task polling, and database-backed CI state", {
|
|
ok: responseOk(overview),
|
|
dbReady: overviewBody?.dbReady ?? null,
|
|
runnerDisposition: asRecord(overview)?.runnerDisposition ?? null,
|
|
failureKind: asRecord(overview)?.failureKind ?? null,
|
|
detail: backendCoreUnavailable(overview) ? overview : {
|
|
status: asRecord(overview)?.status ?? null,
|
|
body: overviewBody,
|
|
},
|
|
}));
|
|
channels.push(channelProbe("database", "database", backendCoreOk, "backend-core task dispatch, provider state, Tekton task polling, and source identity lookup", {
|
|
dbReady: overviewBody?.dbReady ?? false,
|
|
observedThrough: "backend-core /api/overview",
|
|
}));
|
|
|
|
const probeScript = [
|
|
"set -euo pipefail",
|
|
...k3sGuardShellLines(defaultCiKubeconfig),
|
|
"printf 'provider_host_ssh=ok\\n'",
|
|
"command -v bash >/dev/null",
|
|
"command -v docker >/dev/null",
|
|
"command -v kubectl >/dev/null",
|
|
"test -S /var/run/docker.sock || test -S /run/docker.sock || true",
|
|
].join("\n");
|
|
const sshProbe = await transport.dispatchHostSsh(probeScript, 30_000, 15_000);
|
|
channels.push(channelProbe("provider-dispatch", "provider", sshProbe.taskId !== null || sshProbe.ok, "backend-core /api/dispatch can create D601 host.ssh tasks", {
|
|
taskId: sshProbe.taskId,
|
|
status: sshProbe.status,
|
|
ok: sshProbe.taskId !== null || sshProbe.ok,
|
|
exitCode: sshProbe.exitCode,
|
|
stderrTail: sshProbe.stderr.slice(-1200),
|
|
}));
|
|
channels.push(channelProbe("provider-host-ssh", "provider", sshProbe.ok, "D601 source export, registry checks, kubectl/Tekton submission, and artifact summary reads", {
|
|
taskId: sshProbe.taskId,
|
|
status: sshProbe.status,
|
|
exitCode: sshProbe.exitCode,
|
|
stdoutTail: sshProbe.stdout.slice(-1200),
|
|
stderrTail: sshProbe.stderr.slice(-1200),
|
|
raw: sshProbe.ok ? undefined : dispatchPreflightFailure("host.ssh readonly probe", sshProbe).raw,
|
|
}));
|
|
|
|
const registryOptions = parseArtifactRegistryOptions(["--target", providerId]);
|
|
const registryProbe = buildArtifactRegistryReadonlyProbe("health", registryOptions);
|
|
const registryDispatch = await transport.dispatchHostSsh(registryProbe.script, Math.max(registryProbe.timeoutMs, 30_000), registryProbe.timeoutMs);
|
|
const registryCommand = commandResultFromDispatch(transport.artifactRegistryCommand(registryProbe), transport.commandCwd, registryDispatch);
|
|
const registry = artifactRegistryReadonlyResultFromCommand(registryProbe, registryCommand);
|
|
const registryRecord = asRecord(registry);
|
|
const registryOk = registryRecord?.ok === true || registryRecord?.runtimeApiHealthy === true;
|
|
channels.push(channelProbe("artifact-registry", "registry", registryOk, "commit-pinned image push and later CD manifest checks", registry));
|
|
|
|
const missingChannels = channels.filter((item) => !item.ok).map((item) => item.channel);
|
|
const controlChannels = summarizePublishControlChannels(channels);
|
|
const missingControlChannels = controlChannels.filter((item) => !item.ok).map((item) => item.channel);
|
|
const ready = missingChannels.length === 0;
|
|
const failureClassification = ready ? null : classifyPublishPreflightFailure(transport, overview, sshProbe, registry);
|
|
const recommendedAction = recommendedPublishPreflightAction(failureClassification, registry, missingControlChannels);
|
|
return {
|
|
ok: ready,
|
|
runnerDisposition: ready ? "ready" : "infra-blocked",
|
|
failureClassification,
|
|
serviceId: options.serviceId,
|
|
commit: options.commit,
|
|
providerId,
|
|
supportedArtifactPublish: true,
|
|
missingChannels,
|
|
missingControlChannels,
|
|
controlChannels,
|
|
channels,
|
|
registry,
|
|
controlPlane: publishPreflightControlPlane(transport, failureClassification),
|
|
recommendedAction,
|
|
remoteCommandShape: registryProbe.remoteCommandShape,
|
|
next: ready
|
|
? [
|
|
`bun scripts/cli.ts ci publish-user-service --service ${options.serviceId} --commit ${options.commit} --wait-ms 1200000`,
|
|
`later CD must consume ${plannedArtifact.imageRef}; CI itself must not deploy production`,
|
|
]
|
|
: [
|
|
`Restore missing control channel(s): ${missingControlChannels.join(", ") || "unknown"}.`,
|
|
failureClassification === "local-docker-required"
|
|
? "From a Code Queue runner, rerun this read-only preflight through the existing remote frontend transport: bun scripts/cli.ts --main-server-ip <host> ci publish-user-service --service <id> --commit <full-sha> --dry-run."
|
|
: "Run from the main-server CLI or use remote frontend transport against a healthy frontend/backend-core path.",
|
|
"Restore backend-core/database/provider-gateway/Host SSH connectivity before retrying artifact publication.",
|
|
"Use bun scripts/cli.ts artifact-registry health --target D601 to recheck registry reachability after the control bridge is restored.",
|
|
],
|
|
boundary: "preflight is read-only: no D601 source export, no Tekton PipelineRun, no image push, no deploy apply, no service restart",
|
|
};
|
|
}
|
|
|
|
export async function publishBackendCorePreflight(
|
|
_config: UniDeskConfig,
|
|
options: CiPublishBackendCoreOptions,
|
|
plannedArtifact: ArtifactSummary,
|
|
transport: PublishPreflightTransport,
|
|
): Promise<PublishPreflight> {
|
|
const providerId = defaultCiProviderId;
|
|
const channels: PublishPreflightChannelProbe[] = [];
|
|
const overview = await transport.coreFetch("/api/overview", { maxResponseBytes: 500_000 });
|
|
const overviewBody = coreBody(overview);
|
|
const backendCoreOk = responseOk(overview) && overviewBody?.dbReady === true;
|
|
channels.push(channelProbe("backend-core-api", "backend-core", backendCoreOk, "dispatch API, provider catalog, task polling, and database-backed CI state", {
|
|
ok: responseOk(overview),
|
|
dbReady: overviewBody?.dbReady ?? null,
|
|
runnerDisposition: asRecord(overview)?.runnerDisposition ?? null,
|
|
failureKind: asRecord(overview)?.failureKind ?? null,
|
|
detail: backendCoreUnavailable(overview) ? overview : {
|
|
status: asRecord(overview)?.status ?? null,
|
|
body: overviewBody,
|
|
},
|
|
}));
|
|
channels.push(channelProbe("database", "database", backendCoreOk, "backend-core task dispatch, provider state, Tekton task polling, and source identity lookup", {
|
|
dbReady: overviewBody?.dbReady ?? false,
|
|
observedThrough: "backend-core /api/overview",
|
|
}));
|
|
|
|
const ciScript = ciRunnerPreflightScript(options.sourceHostPath);
|
|
const ciProbe = await transport.dispatchHostSsh(ciScript, 30_000, 15_000);
|
|
const ciRunnerReady = backendCoreCiRunnerReady(ciProbe);
|
|
channels.push(channelProbe("provider-dispatch", "provider", ciProbe.taskId !== null || ciProbe.ok, "backend-core /api/dispatch can create D601 host.ssh tasks", {
|
|
taskId: ciProbe.taskId,
|
|
status: ciProbe.status,
|
|
ok: ciProbe.taskId !== null || ciProbe.ok,
|
|
exitCode: ciProbe.exitCode,
|
|
stderrTail: ciProbe.stderr.slice(-1200),
|
|
}));
|
|
channels.push(channelProbe("provider-host-ssh", "provider", ciRunnerReady, "D601 host.ssh can observe Docker, kubectl, Tekton, PVC, and backend-core source parent without starting CI", {
|
|
taskId: ciProbe.taskId,
|
|
status: ciProbe.status,
|
|
exitCode: ciProbe.exitCode,
|
|
stdoutTail: ciProbe.stdout.slice(-1600),
|
|
stderrTail: ciProbe.stderr.slice(-1200),
|
|
raw: ciRunnerReady ? undefined : dispatchPreflightFailure("backend-core ci runner readonly probe", ciProbe).raw,
|
|
}));
|
|
|
|
const registryOptions = parseArtifactRegistryOptions(["--target", providerId]);
|
|
const registryProbe = buildArtifactRegistryReadonlyProbe("health", registryOptions);
|
|
const registryDispatch = await transport.dispatchHostSsh(registryProbe.script, Math.max(registryProbe.timeoutMs, 30_000), registryProbe.timeoutMs);
|
|
const registryCommand = commandResultFromDispatch(transport.artifactRegistryCommand(registryProbe), transport.commandCwd, registryDispatch);
|
|
const registry = artifactRegistryReadonlyResultFromCommand(registryProbe, registryCommand);
|
|
const registryRecord = asRecord(registry);
|
|
const registryOk = registryRecord?.ok === true;
|
|
channels.push(channelProbe("artifact-registry", "registry", registryOk, "backend-core commit-pinned image push and later dev CD manifest checks", registry));
|
|
|
|
const missingChannels = channels.filter((item) => !item.ok).map((item) => item.channel);
|
|
const controlChannels = summarizePublishControlChannels(channels);
|
|
const missingControlChannels = controlChannels.filter((item) => !item.ok).map((item) => item.channel);
|
|
const ready = missingChannels.length === 0;
|
|
const baseFailureClassification = ready ? null : classifyPublishPreflightFailure(transport, overview, ciProbe, registry);
|
|
const failureClassification = !ready && baseFailureClassification === null ? "ci-runner-not-ready" : baseFailureClassification;
|
|
const recommendedAction = ready
|
|
? `authorize and run: bun scripts/cli.ts ci publish-backend-core --commit ${options.commit} --wait-ms 1200000`
|
|
: recommendedPublishPreflightAction(failureClassification, registry, missingControlChannels);
|
|
return {
|
|
ok: ready,
|
|
runnerDisposition: ready ? "ready" : "infra-blocked",
|
|
failureClassification,
|
|
serviceId: "backend-core",
|
|
commit: options.commit,
|
|
providerId,
|
|
supportedArtifactPublish: true,
|
|
missingChannels,
|
|
missingControlChannels,
|
|
controlChannels,
|
|
channels,
|
|
registry,
|
|
controlPlane: publishPreflightControlPlane(transport, failureClassification, backendCoreRemoteCommandShape(options.commit)),
|
|
recommendedAction,
|
|
remoteCommandShape: registryProbe.remoteCommandShape,
|
|
next: ready
|
|
? [
|
|
`bun scripts/cli.ts ci publish-backend-core --commit ${options.commit} --wait-ms 1200000`,
|
|
`verify ${plannedArtifact.imageRef} labels and digest before any dev artifact consumer apply`,
|
|
`dev-only follow-up: bun scripts/cli.ts deploy apply --env dev --service backend-core --commit ${options.commit}`,
|
|
]
|
|
: [
|
|
`Restore missing control channel(s): ${missingControlChannels.join(", ") || "unknown"}.`,
|
|
failureClassification === "local-docker-required"
|
|
? `From a Code Queue runner, rerun this read-only preflight through the remote frontend transport: ${backendCoreRemoteCommandShape(options.commit)}.`
|
|
: "Run from the main-server CLI or use remote frontend transport against a healthy frontend/backend-core path.",
|
|
"Restore backend-core/database/provider-gateway/Host SSH/Tekton/registry readiness before retrying backend-core artifact publication.",
|
|
"This preflight must remain read-only; do not start Tekton, compile Rust, push registry artifacts, deploy/apply, or restart services.",
|
|
],
|
|
boundary: "preflight is read-only: no Tekton PipelineRun, no Rust compile, no source export, no image push, no registry write, no deploy apply, no service restart",
|
|
};
|
|
}
|
|
|
|
export async function readArtifactSummaryFromPipelineRun(name: string, context: ArtifactSummaryContext): Promise<ArtifactSummary> {
|
|
const result = await runRemoteKubectlRaw([
|
|
"set -euo pipefail",
|
|
`kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o json`,
|
|
].join("\n"), 60_000, 45_000);
|
|
if (result.ok && result.stdout.trim().length > 0) {
|
|
try {
|
|
const parsed = JSON.parse(result.stdout) as unknown;
|
|
const fields = new Map<string, string>();
|
|
const list = asRecord(parsed);
|
|
const items = Array.isArray(list?.items) ? list.items : [];
|
|
for (const item of items) {
|
|
const taskRun = asRecord(item);
|
|
const status = asRecord(taskRun?.status);
|
|
const results = Array.isArray(status?.results) ? status.results : [];
|
|
for (const rawResult of results) {
|
|
const taskResult = asRecord(rawResult);
|
|
const nameValue = asString(taskResult?.name);
|
|
const value = asString(taskResult?.value);
|
|
if (/^(user_service_artifact_[a-z_]+|backend_core_artifact_[a-z_]+)$/u.test(nameValue) && value.length > 0) {
|
|
fields.set(nameValue, value);
|
|
}
|
|
}
|
|
}
|
|
if (fields.size > 0) {
|
|
const fromResults = parseArtifactSummaryFromFields(fields, context);
|
|
if (missingArtifactSummaryFields(fromResults).length === 0) return fromResults;
|
|
const completedFromResults = await completeArtifactSummaryFromRegistry(fromResults, context);
|
|
if (missingArtifactSummaryFields(completedFromResults).length === 0) return completedFromResults;
|
|
}
|
|
} catch {
|
|
// Fall back to pod logs below; a malformed diagnostic line must not mask a succeeded PipelineRun.
|
|
}
|
|
}
|
|
return completeArtifactSummaryFromRegistry(parseArtifactSummaryFromOutput(await readPipelineRunLogText(name), context), context);
|
|
}
|
|
|
|
export async function readPipelineRunLogText(name: string, target = ciTarget(null)): Promise<string> {
|
|
const result = await runRemoteKubectlRaw([
|
|
"set -euo pipefail",
|
|
`kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`,
|
|
`kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o wide`,
|
|
`for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail=240 || true; done`,
|
|
].join("\n"), 60_000, 45_000, target);
|
|
return `${result.stdout}\n${result.stderr}`.trim();
|
|
}
|