feat: add devops-controlled dev ci flow

This commit is contained in:
Codex
2026-05-18 06:59:51 +00:00
parent 348c644fde
commit b265274750
29 changed files with 2504 additions and 366 deletions
+116 -74
View File
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { runCommand } from "./command";
@@ -18,7 +18,7 @@ interface DeployManifestService {
}
interface DeployManifest {
schemaVersion: 1;
schemaVersion: 1 | 2;
environment: DeployEnvironment | null;
services: DeployManifestService[];
}
@@ -131,13 +131,14 @@ const nativeK3sInstallVersion = "v1.34.1+k3s1";
const nativeK3sImage = "rancher/k3s:v1.34.1-k3s1";
const nativeK3sCtrAddress = "/run/k3s/containerd/containerd.sock";
const unideskRepoUrl = "https://github.com/pikasTech/unidesk";
const devApplySupportedServiceIds = new Set(["backend-core", "frontend", "code-queue"]);
const d601MaintenanceDeployAllowedServiceIds = new Set(["devops"]);
const devApplySupportedServiceIds = d601MaintenanceDeployAllowedServiceIds;
const deployEnvironmentTargets: Record<DeployEnvironment, DeployEnvironmentTarget> = {
dev: {
environment: "dev",
remote: "origin",
branch: "deploy/dev",
gitRef: "origin/deploy/dev",
branch: "master",
gitRef: "origin/master",
namespace: "unidesk-dev",
runtimeScope: "d601-k3s-dev",
database: {
@@ -155,8 +156,8 @@ const deployEnvironmentTargets: Record<DeployEnvironment, DeployEnvironmentTarge
prod: {
environment: "prod",
remote: "origin",
branch: "deploy/prod",
gitRef: "origin/deploy/prod",
branch: "master",
gitRef: "origin/master",
namespace: "unidesk",
runtimeScope: "prod-main-server-compose-and-d601-k3s",
database: {
@@ -194,7 +195,7 @@ function deployHelp(action: string | undefined = undefined): Record<string, unkn
},
options: [
{ name: "--file <path>", default: defaultDeployFile, description: "Desired-state manifest path relative to the repo root. JSON and ESM JS manifests are supported, for example deploy.json or develop.js." },
{ name: "--env <dev|prod>", description: "Read deploy.json from the fixed environment ref: dev=origin/deploy/dev, prod=origin/deploy/prod. Apply is currently enabled for supported dev services only." },
{ name: "--env <dev|prod>", description: "Read the named environment from origin/master:deploy.json. Direct D601 apply is enabled only for DevOps bootstrap/repair." },
{ name: "--service <id>", description: "Limit reconcile to one service from the manifest." },
{ name: "--dry-run", description: "Prepare and validate without mutating the target service." },
{ name: "--force", description: "Redeploy even when the live commit appears up to date." },
@@ -474,23 +475,42 @@ function parseOptions(args: string[]): DeployOptions {
};
}
function positionalArgs(args: string[]): string[] {
const result: string[] = [];
for (let index = 0; index < args.length; index += 1) {
const value = args[index] ?? "";
if (value.startsWith("--")) {
if (!["--run-now", "--force"].includes(value)) index += 1;
continue;
}
result.push(value);
}
return result;
function parseDeployManifestService(item: unknown, index: number, seen: Set<string>, path: string): DeployManifestService {
const service = asRecord(item);
if (service === null) throw new Error(`deploy manifest ${path}[${index}] must be an object`);
const id = asString(service.id);
const repo = asString(service.repo);
const commitId = asString(service.commitId).toLowerCase();
if (id.length === 0) throw new Error(`deploy manifest ${path}[${index}].id must be a non-empty string`);
if (repo.length === 0) throw new Error(`deploy manifest ${path}[${index}].repo must be a non-empty string`);
if (!/^[0-9a-f]{7,40}$/iu.test(commitId)) throw new Error(`deploy manifest ${path}[${index}].commitId must be a 7-40 char git SHA`);
if (seen.has(id)) throw new Error(`duplicate deploy manifest service id: ${id}`);
seen.add(id);
return { id, repo, commitId };
}
function parseDeployManifestServices(value: unknown, path: string): DeployManifestService[] {
if (!Array.isArray(value)) throw new Error(`deploy manifest ${path} must be an array`);
const seen = new Set<string>();
return value.map((item, index) => parseDeployManifestService(item, index, seen, path));
}
function parseDeployManifest(parsed: unknown, source: string, expectedEnvironment: DeployEnvironment | null): DeployManifest {
const record = asRecord(parsed);
if (record === null) throw new Error(`deploy manifest ${source} must be an object`);
if (record.schemaVersion !== 1) throw new Error("deploy manifest schemaVersion must be 1");
if (record.schemaVersion !== 1 && record.schemaVersion !== 2) throw new Error("deploy manifest schemaVersion must be 1 or 2");
if (record.schemaVersion === 2) {
const environments = asRecord(record.environments);
if (environments === null) throw new Error("deploy manifest schemaVersion=2 requires environments");
if (expectedEnvironment === null) {
const prod = asRecord(environments.prod);
if (prod === null) throw new Error("deploy manifest schemaVersion=2 requires environments.prod for local compatibility mode");
return { schemaVersion: 2, environment: "prod", services: parseDeployManifestServices(prod.services, "environments.prod.services") };
}
const envRecord = asRecord(environments[expectedEnvironment]);
if (envRecord === null) throw new Error(`deploy manifest ${source} does not contain environments.${expectedEnvironment}`);
return { schemaVersion: 2, environment: expectedEnvironment, services: parseDeployManifestServices(envRecord.services, `environments.${expectedEnvironment}.services`) };
}
const rawEnvironment = record.environment;
let environment: DeployEnvironment | null = null;
if (rawEnvironment !== undefined) {
@@ -505,22 +525,7 @@ function parseDeployManifest(parsed: unknown, source: string, expectedEnvironmen
throw new Error(`deploy manifest ${source} declares environment=${environment}, refusing --env ${expectedEnvironment}`);
}
}
if (!Array.isArray(record.services)) throw new Error("deploy manifest services must be an array");
const seen = new Set<string>();
const services = record.services.map((item, index) => {
const service = asRecord(item);
if (service === null) throw new Error(`deploy manifest services[${index}] must be an object`);
const id = asString(service.id);
const repo = asString(service.repo);
const commitId = asString(service.commitId).toLowerCase();
if (id.length === 0) throw new Error(`deploy manifest services[${index}].id must be a non-empty string`);
if (repo.length === 0) throw new Error(`deploy manifest services[${index}].repo must be a non-empty string`);
if (!/^[0-9a-f]{7,40}$/iu.test(commitId)) throw new Error(`deploy manifest services[${index}].commitId must be a 7-40 char git SHA`);
if (seen.has(id)) throw new Error(`duplicate deploy manifest service id: ${id}`);
seen.add(id);
return { id, repo, commitId };
});
return { schemaVersion: 1, environment, services };
return { schemaVersion: 1, environment, services: parseDeployManifestServices(record.services, "services") };
}
function readEnvironmentDeployManifest(environment: DeployEnvironment): { manifest: DeployManifest; source: DeployEnvironmentManifestSource } {
@@ -535,8 +540,9 @@ function readEnvironmentDeployManifest(environment: DeployEnvironment): { manife
const blob = runGitOrThrow(["rev-parse", `${target.gitRef}:deploy.json`], repoRoot, `failed to resolve ${target.gitRef}:deploy.json`).stdout.trim().toLowerCase();
if (!/^[0-9a-f]{40}$/iu.test(blob)) throw new Error(`failed to resolve a deploy.json blob for ${target.gitRef}`);
const raw = runGitOrThrow(["show", `${target.gitRef}:deploy.json`], repoRoot, `failed to read ${target.gitRef}:deploy.json`).stdout;
const manifest = parseDeployManifest(JSON.parse(raw) as unknown, `${target.gitRef}:deploy.json#environments.${environment}`, environment);
return {
manifest: parseDeployManifest(JSON.parse(raw) as unknown, `${target.gitRef}:deploy.json`, environment),
manifest,
source: {
source: "git-ref",
path: "deploy.json",
@@ -659,6 +665,20 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
allowedMethods: ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"],
allowedPathPrefixes: ["/", "/api/", "/logs"],
},
devops: {
name: "UniDesk DevOps Control",
description: "D601 k3s-managed DevOps control plane for normal CI trigger/status/log paths.",
dockerfile: "src/components/microservices/devops/Dockerfile",
composeFile: "src/components/microservices/k3sctl-adapter/k3s/devops.k3s.json",
composeService: "devops",
containerName: "k3s:devops",
nodeBaseUrl: "k3s://devops",
nodePort: 4286,
healthPath: "/health",
route: "/devops",
allowedMethods: ["GET", "HEAD", "POST"],
allowedPathPrefixes: ["/health", "/live", "/logs", "/api/"],
},
};
const spec = specs[id];
if (spec === undefined) return undefined;
@@ -669,7 +689,7 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
description: spec.description,
repository: {
url: unideskRepoUrl,
commitId: "deploy-dev",
commitId: "deploy-env",
dockerfile: spec.dockerfile,
composeFile: spec.composeFile,
composeService: spec.composeService,
@@ -677,9 +697,9 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
},
backend: {
nodeBaseUrl: spec.nodeBaseUrl,
nodeBindHost: `k3s://unidesk-dev/${spec.composeService}`,
nodeBindHost: `k3s://${id === "devops" ? "unidesk-ci" : "unidesk-dev"}/${spec.composeService}`,
nodePort: spec.nodePort,
proxyMode: "dev-k3s-direct",
proxyMode: id === "devops" ? "k3sctl-adapter-http" : "dev-k3s-direct",
frontendOnly: true,
public: false,
allowedMethods: spec.allowedMethods,
@@ -691,14 +711,16 @@ function devK3sDeployService(id: string): UniDeskMicroserviceConfig | undefined
mode: "k3sctl-managed",
adapterServiceId: "k3sctl-adapter",
k3sServiceId: spec.composeService,
namespace: "unidesk-dev",
namespace: id === "devops" ? "unidesk-ci" : "unidesk-dev",
expectedNodeIds: ["D601"],
activeNodeId: "D601",
},
development: {
providerId: "D601",
sshPassthrough: true,
worktreePath: id === "code-queue"
worktreePath: id === "devops"
? "/home/ubuntu/.unidesk/devops-deploy"
: id === "code-queue"
? "/home/ubuntu/unidesk-dev-code-queue-deploy/code-queue"
: `/home/ubuntu/unidesk-dev-core-deploy/${id}`,
},
@@ -721,10 +743,15 @@ function isCoreDeployService(service: UniDeskMicroserviceConfig): boolean {
function isDevK3sDeployService(service: UniDeskMicroserviceConfig): boolean {
return service.deployment.mode === "k3sctl-managed"
&& service.deployment.namespace === deployEnvironmentTargets.dev.namespace
&& devApplySupportedServiceIds.has(service.id);
}
function isD601MaintenanceDeployBlocked(service: UniDeskMicroserviceConfig): boolean {
if (targetIsMain(service)) return false;
if (service.providerId !== "D601") return false;
return !d601MaintenanceDeployAllowedServiceIds.has(service.id);
}
function isDirectComposeDeployMode(service: UniDeskMicroserviceConfig): boolean {
return service.deployment.mode === "unidesk-direct" || service.deployment.mode === "internal-sidecar";
}
@@ -992,6 +1019,12 @@ function dockerBuildTimeoutMs(service: UniDeskMicroserviceConfig, options: Deplo
return Math.min(options.timeoutMs, maxBuildMs);
}
function devK3sPrepullImages(service: UniDeskMicroserviceConfig): string[] {
if (!isDevK3sDeployService(service)) return [];
if (service.id === "devops") return ["golang:1.23-bookworm", "debian:bookworm-slim"];
return ["oven/bun:1-alpine"];
}
function prepareSourceScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, exportDir: string): string {
if (targetIsMain(service) && isUnideskRepo(desired.repo)) {
return [
@@ -1172,11 +1205,11 @@ function buildImageScript(service: UniDeskMicroserviceConfig, desired: DeployMan
"if ! docker buildx version >/dev/null 2>&1; then",
" if ! docker buildx inspect default >/dev/null 2>&1; then echo target_build_builder=missing >&2; exit 1; fi",
"fi",
...(isDevK3sDeployService(service) ? [
"if ! docker image inspect oven/bun:1-alpine >/dev/null 2>&1; then",
" docker pull oven/bun:1-alpine",
...devK3sPrepullImages(service).flatMap((imageName) => [
`if ! docker image inspect ${shellQuote(imageName)} >/dev/null 2>&1; then`,
` docker pull ${shellQuote(imageName)}`,
"fi",
] : []),
]),
"builder_args=()",
"if docker buildx inspect --builder default >/dev/null 2>&1; then builder_args=(--builder default); echo target_build_builder=default; else echo target_build_builder=implicit; fi",
"docker buildx inspect \"${builder_args[@]}\" --bootstrap || true",
@@ -1218,9 +1251,9 @@ function patchDevK3sManifestScript(service: UniDeskMicroserviceConfig, desired:
" raise SystemExit(f'dev service {service_id}/{deployment_name} not found in {path}')",
"patched = []",
"for segment in kept:",
" segment = segment.replace('unidesk.ai/image-source: deploy-dev-commit', 'unidesk.ai/image-source: deploy-env-commit')",
" segment = segment.replace('unidesk.ai/image-source: deploy-env-commit', 'unidesk.ai/image-source: deploy-env-commit')",
" segment = re.sub(r'image: unidesk-[^\\n]+:dev-placeholder', f'image: {image}', segment)",
" segment = re.sub(r'value: replace-with-deploy-dev-commit', f'value: {commit}', segment)",
" segment = re.sub(r'value: replace-with-deploy-env-commit', f'value: {commit}', segment)",
" segment = segment.replace('value: https://github.com/pikasTech/unidesk', f'value: {repo}')",
" patched.append(segment.strip() + '\\n')",
"out = '---\\n'.join(patched)",
@@ -2148,6 +2181,15 @@ async function ensureGithubSshIdentityStep(config: UniDeskConfig, service: UniDe
async function applyOneService(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService, options: DeployOptions): Promise<Record<string, unknown>> {
const steps: StepResult[] = [];
const startedAt = nowIso();
if (!options.dryRun && isD601MaintenanceDeployBlocked(service)) {
return {
ok: false,
serviceId: service.id,
skipped: true,
reason: `D601 maintenance-channel direct deployment is allowed only for ${[...d601MaintenanceDeployAllowedServiceIds].join(", ")} bootstrap/repair. Deploy ${service.id} through the DevOps control plane instead.`,
steps,
};
}
const reason = unsupportedReason(service);
if (reason !== null) return { ok: false, serviceId: service.id, skipped: true, reason, steps };
const before = await readRuntimeState(config, service, desired);
@@ -2274,6 +2316,7 @@ function environmentDryRunPlan(
mutatesRuntime: false,
environment,
gitRef: source.ref,
environmentPath: `environments.${environment}`,
manifest: {
source: source.source,
path: source.path,
@@ -2283,6 +2326,7 @@ function environmentDryRunPlan(
commit: source.commit,
blob: source.blob,
environment: manifest.environment,
environmentPath: `environments.${environment}`,
fetchedAt: source.fetchedAt,
},
target: environmentTargetSummary(target),
@@ -2309,6 +2353,17 @@ function unsupportedDevApplyServices(manifest: DeployManifest, serviceId: string
return services.map((service) => service.id).filter((id) => !devApplySupportedServiceIds.has(id));
}
function blockedD601MaintenanceDeployServices(config: UniDeskConfig, manifest: DeployManifest, serviceId: string | null): string[] {
return selectServices(config, manifest, serviceId)
.map((item) => item.config)
.filter(isD601MaintenanceDeployBlocked)
.map((service) => service.id);
}
function d601MaintenanceDeployBlockMessage(blocked: string[]): string {
return `D601 maintenance-channel direct deployment is allowed only for ${[...d601MaintenanceDeployAllowedServiceIds].join(", ")} bootstrap/repair; blocked services: ${blocked.join(", ")}. Use the DevOps control plane for other direct/managed microservices.`;
}
async function runApplyNow(config: UniDeskConfig, manifest: DeployManifest, options: DeployOptions): Promise<Record<string, unknown>> {
const selected = selectServices(config, manifest, options.serviceId);
const startedAt = nowIso();
@@ -2333,7 +2388,7 @@ async function runApplyNow(config: UniDeskConfig, manifest: DeployManifest, opti
function applyJob(config: UniDeskConfig, args: string[], options: DeployOptions): Record<string, unknown> {
const runArgs = args.includes("--run-now") ? args : [...args, "--run-now"];
const command = [process.execPath, rootPath("scripts", "cli.ts"), "deploy", ...runArgs];
const source = options.environment === null ? options.file : `${deployEnvironmentTargets[options.environment].gitRef}:deploy.json`;
const source = options.environment === null ? options.file : `${deployEnvironmentTargets[options.environment].gitRef}:deploy.json#environments.${options.environment}`;
const job = startJob("deploy_apply", command, `Reconcile services from ${source}${options.serviceId === null ? "" : ` service=${options.serviceId}`}`);
return {
ok: true,
@@ -2361,40 +2416,27 @@ export async function runDeployCommand(config: UniDeskConfig | null, args: strin
throw new Error(`deploy apply --env dev currently supports only ${[...devApplySupportedServiceIds].join(", ")}; unsupported selected services: ${unsupported.join(", ")}`);
}
if (config === null) throw new Error("deploy apply --env dev requires config.json");
if (!options.dryRun) {
const blocked = blockedD601MaintenanceDeployServices(config, manifest, options.serviceId);
if (blocked.length > 0) throw new Error(d601MaintenanceDeployBlockMessage(blocked));
}
if (!options.runNow) return applyJob(config, args, options);
return await runApplyNow(config, manifest, options);
}
if (config === null) throw new Error("deploy local manifest mode requires config.json");
const manifest = resolveManifestCommits(await readDeployManifest(options.file), options.serviceId);
if (action === "check" || action === "plan") return await checkOrPlan(config, manifest, options, action);
if (!options.dryRun) {
const blocked = blockedD601MaintenanceDeployServices(config, manifest, options.serviceId);
if (blocked.length > 0) throw new Error(d601MaintenanceDeployBlockMessage(blocked));
}
if (!options.runNow) return applyJob(config, args, options);
return await runApplyNow(config, manifest, options);
}
export async function runCodeQueueDeployCompatCommand(config: UniDeskConfig, args: string[]): Promise<unknown> {
if (args.includes("--skip-build")) throw new Error("codex deploy no longer supports --skip-build; target-side Docker build is mandatory");
export async function runCodeQueueDeployCompatCommand(_config: UniDeskConfig, args: string[]): Promise<unknown> {
if (args.includes("--skip-build")) throw new Error("codex deploy is disabled; --skip-build is not supported");
const providerId = optionValue(args, ["--provider-id", "--provider"]) ?? "D601";
if (providerId !== "D601") throw new Error(`codex deploy compatibility path only supports D601; got ${providerId}`);
const commitId = optionValue(args, ["--commit", "--commit-id"]) ?? positionalArgs(args)[0] ?? "";
if (!/^[0-9a-f]{7,40}$/iu.test(commitId)) throw new Error("codex deploy requires a 7-40 char commit id");
const service = config.microservices.find((item) => item.id === "code-queue");
if (service === undefined) throw new Error("config.json does not contain microservice id=code-queue");
const manifestRelDir = join(".state", "deploy", "manifests");
mkdirSync(rootPath(manifestRelDir), { recursive: true });
const manifestRelPath = join(manifestRelDir, `code-queue-${commitId.toLowerCase()}.json`);
writeFileSync(rootPath(manifestRelPath), `${JSON.stringify({
schemaVersion: 1,
services: [{ id: "code-queue", repo: service.repository.url, commitId: commitId.toLowerCase() }],
}, null, 2)}\n`, "utf8");
const deployArgs = [
"apply",
"--file",
manifestRelPath,
"--service",
"code-queue",
...(args.includes("--run-now") ? ["--run-now"] : []),
...(args.includes("--force") ? ["--force"] : []),
...(optionValue(args, ["--timeout-ms"]) === undefined ? [] : ["--timeout-ms", optionValue(args, ["--timeout-ms"]) as string]),
];
return await runDeployCommand(config, deployArgs);
throw new Error("codex deploy is disabled because D601 maintenance-channel direct deployment is now reserved for DevOps bootstrap/repair. Use the DevOps control plane for Code Queue deployment.");
}