feat: add desired-state deploy reconciler

This commit is contained in:
Codex
2026-05-16 07:58:39 +00:00
parent 0c3cdb4ee0
commit e841c929cf
11 changed files with 1209 additions and 691 deletions
+12 -3
View File
@@ -9,7 +9,7 @@ import { runSsh } from "./src/ssh";
import { extractRemoteCliOptions, runRemoteCli } from "./src/remote";
import { runMicroserviceCommand } from "./src/microservices";
import { runCodeQueueCommand } from "./src/code-queue";
import { runCodexDeployCommand } from "./src/codex-deploy";
import { runCodeQueueDeployCompatCommand, runDeployCommand } from "./src/deploy";
import { runProviderCommand } from "./src/provider-attach";
import { runScheduleCommand } from "./src/schedules";
@@ -43,9 +43,10 @@ function help(): unknown {
{ command: "microservice status <id>", description: "Show one user service config, repository reference, backend mapping, and runtime status." },
{ command: "microservice health <id>", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy." },
{ command: "microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--raw] [--max-body-bytes N]", description: "Access a private user-service backend path through the same frontend-only proxy used by WebUI; large bodies are summarized unless --raw is set." },
{ command: "deploy check|plan|apply [--file deploy.json] [--service id] [--dry-run] [--force]", description: "Reconcile services from a repo+commit manifest using target-side build and live commit verification." },
{ command: "schedule list|get|runs|run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N." },
{ command: "schedule upsert-pgdata-backup [--time HH:MM] [--remote-base /SERVER_DATA/UNIDESK_PG_DATA]", description: "Create or update the daily PGDATA physical backup task that uploads monthly rotated archives to Baidu Netdisk." },
{ command: "codex deploy <commitId> [--provider-id D601] [--timeout-ms N] [--skip-build]", description: "Start an async D601 v3s/k8s Code Queue deployment job from a specific remote git commit." },
{ command: "codex deploy <commitId> [--provider-id D601] [--timeout-ms N]", description: "Compatibility wrapper for deploy apply --service code-queue with a temporary repo+commit manifest." },
{ command: "codex task <taskId> [--trace --tail|--from-start|--after-seq N|--before-seq N --limit N] [--full]", description: "Fetch a compact Code Queue task summary; trace rows are opt-in and paged with next/previous commands to avoid output explosion." },
{ command: "codex output <taskId> [--tail|--from-start|--after-seq N|--before-seq N --limit N] [--full-text]", description: "Fetch paged raw Code Queue output records by seq when a trace row has omitted command/output text." },
{ command: "codex judge <taskId> --attempt N [--dry-run] [--include-prompt]", description: "Replay one stored Code Queue attempt through the same judge context builder and MiniMax judge call path used by the live queue worker." },
@@ -179,6 +180,14 @@ async function main(): Promise<void> {
return;
}
if (top === "deploy") {
const result = await runDeployCommand(config, args.slice(1));
const ok = (result as { ok?: unknown }).ok !== false;
emitJson(commandName, result, ok);
if (!ok) process.exitCode = 1;
return;
}
if (top === "provider") {
emitJson(commandName, await runProviderCommand(config, args.slice(1)));
return;
@@ -191,7 +200,7 @@ async function main(): Promise<void> {
if (top === "codex") {
if (sub === "deploy") {
const result = await runCodexDeployCommand(config, args.slice(2));
const result = await runCodeQueueDeployCompatCommand(config, args.slice(2));
const ok = (result as { ok?: unknown }).ok !== false;
emitJson(commandName, result, ok);
if (!ok) process.exitCode = 1;
+1 -1
View File
@@ -70,7 +70,7 @@ export function runChecks(config: UniDeskConfig): { ok: boolean; items: CheckIte
fileItem("src/components/microservices/oa-event-flow/src/index.ts"),
fileItem("src/components/microservices/v3sctl-adapter/src/index.ts"),
fileItem("src/components/microservices/mdtodo/src/index.ts"),
fileItem("scripts/src/codex-deploy.ts"),
fileItem("scripts/src/deploy.ts"),
fileItem("scripts/src/e2e.ts"),
unifiedLogRotationItem(),
commandItem("bun:version", ["bun", "--version"]),
-679
View File
@@ -1,679 +0,0 @@
import { startJob, type JobRecord } from "./jobs";
import { coreInternalFetch } from "./microservices";
import { type UniDeskConfig, rootPath } from "./config";
const defaultProviderId = "D601";
const defaultTimeoutMs = 900_000;
const pollIntervalMs = 5_000;
const shortDispatchWaitMs = 25_000;
const shortRemoteTimeoutMs = 20_000;
const defaultSourceRepoDir = "/home/ubuntu/unidesk";
const defaultDeployDir = "/home/ubuntu/cq-deploy";
const defaultKubeconfigPath = "/home/ubuntu/cq-deploy/.state/v8s/kubeconfig";
const defaultK3sContainer = "unidesk-v8s-server";
const defaultImageTag = "unidesk-code-queue:d601";
const k8sNamespace = "unidesk";
const k8sDeployment = "code-queue";
const tcpEgressDeployment = "d601-tcp-egress-gateway";
const k8sManifestRelPath = "src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml";
interface DeployCliOptions {
commitId: string;
providerId: string;
repoUrl: string;
sourceRepoDir: string;
deployDir: string;
kubeconfigPath: string;
k3sContainer: string;
imageTag: string;
timeoutMs: number;
skipBuild: boolean;
runNow: boolean;
}
interface StepResult {
step: string;
ok: boolean;
detail: string;
startedAt: string;
finishedAt: string;
raw?: unknown;
}
interface DispatchResult {
ok: boolean;
taskId: string | null;
status: string | null;
stdout: string;
stderr: string;
exitCode: number | null;
raw: unknown;
}
interface BackgroundPoll {
done: boolean;
exitCode: number | null;
logTail: string;
raw: unknown;
}
function asRecord(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function asString(value: unknown): string {
return typeof value === "string" ? value : "";
}
function optionValue(args: string[], names: string[]): string | undefined {
for (const name of names) {
const index = args.indexOf(name);
if (index === -1) continue;
const raw = args[index + 1];
if (raw === undefined || raw.length === 0) throw new Error(`${name} requires a non-empty value`);
return raw;
}
return undefined;
}
function positiveIntegerOption(args: string[], names: string[], defaultValue: number): number {
const raw = optionValue(args, names);
if (raw === undefined) return defaultValue;
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0) throw new Error(`${names[0]} must be a positive integer`);
return value;
}
function positionalArgs(args: string[]): string[] {
const result: string[] = [];
for (let index = 0; index < args.length; index += 1) {
const value = args[index] ?? "";
if (value.startsWith("--")) {
if (!["--skip-build", "--run-now"].includes(value)) index += 1;
continue;
}
result.push(value);
}
return result;
}
function codeQueueRepoUrl(config: UniDeskConfig): string {
const service = config.microservices.find((item) => item.id === "code-queue");
if (service === undefined) throw new Error("config.json does not contain microservice id=code-queue");
return service.repository.url;
}
function parseOptions(config: UniDeskConfig, args: string[]): DeployCliOptions {
const commitId = optionValue(args, ["--commit", "--commit-id"]) ?? positionalArgs(args)[0] ?? "";
if (commitId.length === 0) {
throw new Error("codex deploy requires a commit ID: codex deploy <commitId>");
}
return {
commitId,
providerId: optionValue(args, ["--provider-id", "--provider"]) ?? defaultProviderId,
repoUrl: optionValue(args, ["--repo-url"]) ?? codeQueueRepoUrl(config),
sourceRepoDir: optionValue(args, ["--source-repo-dir"]) ?? defaultSourceRepoDir,
deployDir: optionValue(args, ["--deploy-dir"]) ?? defaultDeployDir,
kubeconfigPath: optionValue(args, ["--kubeconfig"]) ?? defaultKubeconfigPath,
k3sContainer: optionValue(args, ["--k3s-container"]) ?? defaultK3sContainer,
imageTag: optionValue(args, ["--image"]) ?? defaultImageTag,
timeoutMs: positiveIntegerOption(args, ["--timeout-ms"], defaultTimeoutMs),
skipBuild: args.includes("--skip-build"),
runNow: args.includes("--run-now"),
};
}
function validateOptions(options: DeployCliOptions): void {
if (!/^[0-9a-f]{7,40}$/iu.test(options.commitId)) {
throw new Error(`commit id must be a 7-40 character hex SHA, got: ${options.commitId}`);
}
if (!/^[A-Za-z0-9_.-]+$/u.test(options.providerId)) throw new Error(`invalid provider id: ${options.providerId}`);
if (options.providerId !== defaultProviderId) {
throw new Error(`codex deploy currently supports only ${defaultProviderId}; got ${options.providerId}`);
}
if (!/^https?:\/\//u.test(options.repoUrl) && !/^[A-Za-z0-9_.-]+@/u.test(options.repoUrl)) {
throw new Error(`repo url must be an http(s) or ssh git URL, got: ${options.repoUrl}`);
}
}
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
function nowIso(): string {
return new Date().toISOString();
}
function elapsedMs(startedAt: number): number {
return Math.max(0, Date.now() - startedAt);
}
function compactTail(text: string, maxChars = 900): string {
return text.length > maxChars ? text.slice(text.length - maxChars) : text;
}
function progressLine(step: string, message: string, detail?: unknown): void {
const payload = detail === undefined
? { at: nowIso(), step, message }
: { at: nowIso(), step, message, detail };
process.stderr.write(`${JSON.stringify(payload)}\n`);
}
function coreBody(response: unknown): Record<string, unknown> | null {
const record = asRecord(response);
return asRecord(record?.body);
}
function dispatchStdout(raw: unknown): string {
const task = asRecord(raw);
const result = asRecord(task?.result);
return asString(result?.stdout);
}
function parseFullCommit(value: string): string {
const match = value.match(/\b[0-9a-f]{40}\b/iu);
return match?.[0]?.toLowerCase() ?? "";
}
async function dispatchSsh(
config: UniDeskConfig,
providerId: string,
command: string,
cwd: string | null,
waitMs = shortDispatchWaitMs,
remoteTimeoutMs = shortRemoteTimeoutMs,
): Promise<DispatchResult> {
const dispatchResponse = coreInternalFetch("/api/dispatch", {
method: "POST",
body: {
providerId,
command: "host.ssh",
payload: {
source: "codex-deploy",
mode: "exec",
command,
timeoutMs: remoteTimeoutMs,
...(cwd === null ? {} : { cwd }),
},
},
});
const dispatchBody = coreBody(dispatchResponse);
const taskId = asString(dispatchBody?.taskId);
if (dispatchBody?.ok !== true || taskId.length === 0) {
return {
ok: false,
taskId: taskId || null,
status: null,
stdout: "",
stderr: asString(dispatchBody?.error) || "dispatch did not return a task id",
exitCode: null,
raw: dispatchResponse,
};
}
const deadline = Date.now() + waitMs;
let latest: unknown = null;
while (Date.now() < deadline) {
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`);
const task = asRecord(coreBody(latest)?.task);
const status = asString(task?.status);
if (status === "succeeded" || status === "failed") {
const result = asRecord(task?.result);
const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null;
const stdout = asString(result?.stdout);
const stderr = asString(result?.stderr);
return {
ok: status === "succeeded" && (exitCode === null || exitCode === 0),
taskId,
status,
stdout,
stderr,
exitCode,
raw: task,
};
}
await Bun.sleep(500);
}
return {
ok: false,
taskId,
status: "timeout",
stdout: "",
stderr: `host.ssh task ${taskId} did not finish within ${waitMs}ms`,
exitCode: null,
raw: latest,
};
}
async function launchBackground(
config: UniDeskConfig,
providerId: string,
shellScript: string,
cwd: string,
logFile: string,
sentinelFile: string,
): Promise<{ ok: boolean; pid: string; raw: unknown; error: string }> {
const wrapped = [
`bash -lc ${shellQuote(shellScript)}`,
"code=$?",
`printf '%s\\n' "$code" > ${shellQuote(sentinelFile)}`,
"exit \"$code\"",
].join("; ");
const command = [
`rm -f ${shellQuote(sentinelFile)} ${shellQuote(logFile)}`,
`nohup bash -lc ${shellQuote(wrapped)} > ${shellQuote(logFile)} 2>&1 < /dev/null & echo $!`,
].join("; ");
const result = await dispatchSsh(config, providerId, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
const pid = result.stdout.trim().split("\n").pop()?.trim() ?? "";
if (!result.ok || !/^\d+$/u.test(pid)) {
return { ok: false, pid: "", raw: result.raw, error: result.stderr || result.stdout || "failed to launch background command" };
}
return { ok: true, pid, raw: result.raw, error: "" };
}
async function pollBackground(
config: UniDeskConfig,
providerId: string,
cwd: string,
logFile: string,
sentinelFile: string,
): Promise<BackgroundPoll> {
const command = [
`if [ -f ${shellQuote(sentinelFile)} ]; then printf 'SENTINEL:%s\\n' "$(cat ${shellQuote(sentinelFile)} 2>/dev/null || true)"; else echo RUNNING; fi`,
`tail -n 80 ${shellQuote(logFile)} 2>/dev/null || true`,
].join("; ");
const result = await dispatchSsh(config, providerId, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
const stdout = result.stdout.trimEnd();
const [head = "", ...rest] = stdout.split("\n");
if (head.startsWith("SENTINEL:")) {
const rawExitCode = head.slice("SENTINEL:".length).trim();
const exitCode = /^\d+$/u.test(rawExitCode) ? Number(rawExitCode) : null;
return { done: true, exitCode, logTail: rest.join("\n").trim(), raw: result.raw };
}
return { done: false, exitCode: null, logTail: rest.join("\n").trim(), raw: result.raw };
}
async function backgroundStep(
config: UniDeskConfig,
options: DeployCliOptions,
step: string,
shellScript: string,
timeoutMs: number,
cwd: string | null = options.deployDir,
): Promise<StepResult> {
const startedAt = nowIso();
const startedMs = Date.now();
const runId = `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
const logFile = `/tmp/unidesk-codex-deploy-${step}-${runId}.log`;
const sentinelFile = `/tmp/unidesk-codex-deploy-${step}-${runId}.done`;
progressLine(step, "launching remote background step", { logFile, sentinelFile, timeoutMs });
const launch = await launchBackground(config, options.providerId, shellScript, cwd ?? "/home/ubuntu", logFile, sentinelFile);
if (!launch.ok) {
return { step, ok: false, detail: launch.error, startedAt, finishedAt: nowIso(), raw: launch.raw };
}
progressLine(step, "remote background step started", { pid: launch.pid, logFile });
const deadline = Date.now() + timeoutMs;
let lastTail = "";
while (Date.now() < deadline) {
await Bun.sleep(pollIntervalMs);
const poll = await pollBackground(config, options.providerId, cwd ?? "/home/ubuntu", logFile, sentinelFile);
const tail = compactTail(poll.logTail, 1200);
if (tail.length > 0 && tail !== lastTail) {
lastTail = tail;
progressLine(step, "remote log tail", { elapsedMs: elapsedMs(startedMs), tail });
}
if (poll.done) {
const ok = poll.exitCode === 0;
return {
step,
ok,
detail: ok
? `completed in ${elapsedMs(startedMs)}ms; log=${logFile}`
: `failed with exit ${poll.exitCode}; log=${logFile}; tail=${compactTail(poll.logTail)}`,
startedAt,
finishedAt: nowIso(),
raw: poll.raw,
};
}
}
return { step, ok: false, detail: `timed out after ${timeoutMs}ms; log=${logFile}`, startedAt, finishedAt: nowIso(), raw: null };
}
function bashScriptCommand(script: string): string {
return `bash -lc ${shellQuote(script)}`;
}
async function directStep(
config: UniDeskConfig,
options: DeployCliOptions,
step: string,
command: string,
cwd: string | null,
waitMs = 60_000,
remoteTimeoutMs = 45_000,
): Promise<StepResult> {
const startedAt = nowIso();
const startedMs = Date.now();
progressLine(step, "running remote command");
const result = await dispatchSsh(config, options.providerId, command, cwd, waitMs, remoteTimeoutMs);
const detail = compactTail([result.stdout, result.stderr].filter(Boolean).join("\n"), 1200);
return {
step,
ok: result.ok,
detail: result.ok ? detail || `completed in ${elapsedMs(startedMs)}ms` : detail || `failed with status=${result.status} exit=${result.exitCode}`,
startedAt,
finishedAt: nowIso(),
raw: result.raw,
};
}
function prepareSourceScript(options: DeployCliOptions, exportDir: string): string {
return [
"set -euo pipefail",
`repo=${shellQuote(options.sourceRepoDir)}`,
`repo_url=${shellQuote(options.repoUrl)}`,
`commit=${shellQuote(options.commitId)}`,
`export_dir=${shellQuote(exportDir)}`,
"mkdir -p \"$(dirname \"$repo\")\"",
"if [ ! -d \"$repo/.git\" ]; then rm -rf \"$repo\"; git clone --no-checkout \"$repo_url\" \"$repo\"; fi",
"cd \"$repo\"",
"git remote get-url origin >/dev/null",
"git fetch --no-tags origin \"$commit\" || git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
"resolved=$(git rev-parse --verify \"$commit^{commit}\")",
"rm -rf \"$export_dir\"",
"mkdir -p \"$export_dir\"",
"git archive --format=tar \"$resolved\" | tar -xf - -C \"$export_dir\"",
"printf 'resolved_commit=%s\\nexport_dir=%s\\n' \"$resolved\" \"$export_dir\"",
].join("\n");
}
function resolveCommitScript(options: DeployCliOptions): string {
return [
"set -euo pipefail",
`repo=${shellQuote(options.sourceRepoDir)}`,
`commit=${shellQuote(options.commitId)}`,
"git -C \"$repo\" rev-parse --verify \"$commit^{commit}\"",
].join("\n");
}
function syncDeployScript(options: DeployCliOptions, exportDir: string): string {
return [
"set -euo pipefail",
`export_dir=${shellQuote(exportDir)}`,
`deploy_dir=${shellQuote(options.deployDir)}`,
"test -f \"$export_dir/src/components/microservices/code-queue/Dockerfile\"",
"mkdir -p \"$deploy_dir\"",
[
"rsync -a --delete",
"--exclude '.git/'",
"--exclude '.state/'",
"--exclude 'logs/'",
"--exclude '**/node_modules/'",
"--exclude '**/dist/'",
"\"$export_dir/\"",
"\"$deploy_dir/\"",
].join(" "),
"test -f \"$deploy_dir/src/components/microservices/code-queue/Dockerfile\"",
"test -f \"$deploy_dir/src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml\"",
"printf 'synced deploy tree to %s\\n' \"$deploy_dir\"",
].join("\n");
}
function buildImageScript(options: DeployCliOptions): string {
return [
"set -euo pipefail",
`deploy_dir=${shellQuote(options.deployDir)}`,
`image=${shellQuote(options.imageTag)}`,
"cd \"$deploy_dir\"",
"docker build -t \"$image\" -f src/components/microservices/code-queue/Dockerfile .",
"docker image inspect \"$image\" --format 'image_id={{.Id}} repo_tags={{json .RepoTags}}'",
].join("\n");
}
function importImageScript(options: DeployCliOptions): string {
return [
"set -euo pipefail",
`image=${shellQuote(options.imageTag)}`,
`k3s_container=${shellQuote(options.k3sContainer)}`,
"docker image inspect \"$image\" >/dev/null",
"docker ps --format '{{.Names}}' | grep -Fx \"$k3s_container\" >/dev/null",
"docker save \"$image\" | docker exec -i \"$k3s_container\" ctr -n k8s.io images import -",
"docker exec \"$k3s_container\" ctr -n k8s.io images ls | grep -F \"$image\" || true",
].join("\n");
}
function applyManifestCommand(options: DeployCliOptions): string {
const manifest = `${options.deployDir}/${k8sManifestRelPath}`;
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl apply -f ${shellQuote(manifest)}`;
}
function stampDeployCommitScript(options: DeployCliOptions, resolvedCommit: string): string {
return [
"set -euo pipefail",
`kubeconfig=${shellQuote(options.kubeconfigPath)}`,
`namespace=${shellQuote(k8sNamespace)}`,
`deployment=${shellQuote(k8sDeployment)}`,
`tcp_deployment=${shellQuote(tcpEgressDeployment)}`,
`resolved_commit=${shellQuote(resolvedCommit)}`,
`requested_commit=${shellQuote(options.commitId)}`,
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" set env \"deployment/$tcp_deployment\" \"deployment/$deployment\" CODE_QUEUE_DEPLOY_COMMIT=\"$resolved_commit\" CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=\"$requested_commit\"",
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" annotate \"deployment/$tcp_deployment\" \"deployment/$deployment\" unidesk.ai/deploy-commit=\"$resolved_commit\" unidesk.ai/deploy-requested-commit=\"$requested_commit\" --overwrite",
"current=$(KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"CODE_QUEUE_DEPLOY_COMMIT\")].value}')",
"test \"$current\" = \"$resolved_commit\"",
"printf 'deployment_commit=%s\\nrequested_commit=%s\\n' \"$current\" \"$requested_commit\"",
].join("\n");
}
function rolloutRestartCommand(options: DeployCliOptions): string {
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl -n ${shellQuote(k8sNamespace)} rollout restart deployment/${shellQuote(tcpEgressDeployment)} deployment/${shellQuote(k8sDeployment)}`;
}
function rolloutStatusScript(options: DeployCliOptions): string {
return [
"set -euo pipefail",
`kubeconfig=${shellQuote(options.kubeconfigPath)}`,
`namespace=${shellQuote(k8sNamespace)}`,
`deployment=${shellQuote(k8sDeployment)}`,
`tcp_deployment=${shellQuote(tcpEgressDeployment)}`,
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" rollout status \"deployment/$tcp_deployment\" --timeout=120s",
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" rollout status \"deployment/$deployment\" --timeout=180s",
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get deploy \"$tcp_deployment\" \"$deployment\" -o wide",
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get pods -l app.kubernetes.io/name=code-queue,unidesk.ai/instance-id=D601 -o wide",
].join("\n");
}
function healthDeployCommit(body: Record<string, unknown> | null): string {
const deploy = asRecord(body?.deploy);
return asString(deploy?.commit).toLowerCase();
}
function healthProbeSummary(body: Record<string, unknown> | null): Record<string, unknown> | null {
if (body === null) return null;
return {
ok: body.ok,
service: body.service,
instanceId: body.instanceId,
deploy: body.deploy,
status: body.status,
databaseReady: body.databaseReady,
startedAt: body.startedAt,
};
}
async function microserviceHealthStep(config: UniDeskConfig, expectedCommit: string, timeoutMs: number): Promise<StepResult> {
const step = "unidesk-health";
const startedAt = nowIso();
const startedMs = Date.now();
const deadline = Date.now() + timeoutMs;
let latest: unknown = null;
let latestCommit = "";
while (Date.now() < deadline) {
latest = coreInternalFetch("/api/microservices/code-queue/health");
const record = asRecord(latest);
const body = asRecord(record?.body);
latestCommit = healthDeployCommit(body);
const commitMatches = latestCommit === expectedCommit;
const ok = record?.ok === true && body?.ok !== false && commitMatches;
progressLine(step, "health probe", { ok, status: record?.status ?? null, expectedCommit, deployedCommit: latestCommit || null, commitMatches, body: healthProbeSummary(body) });
if (ok) {
return {
step,
ok: true,
detail: `Code Queue health passed with deployed commit ${latestCommit} in ${elapsedMs(startedMs)}ms`,
startedAt,
finishedAt: nowIso(),
raw: latest,
};
}
await Bun.sleep(pollIntervalMs);
}
return {
step,
ok: false,
detail: `Code Queue health did not expose expected commit ${expectedCommit} within ${timeoutMs}ms; latest deployed commit=${latestCommit || "none"}`,
startedAt,
finishedAt: nowIso(),
raw: latest,
};
}
function remainingTimeout(deadline: number, fallbackMs: number): number {
return Math.max(30_000, Math.min(fallbackMs, deadline - Date.now()));
}
export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptions): Promise<unknown> {
validateOptions(options);
const startedAt = nowIso();
const deadline = Date.now() + options.timeoutMs;
const exportDir = `/tmp/unidesk-codex-deploy-src-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
const steps: StepResult[] = [];
const pushStep = (step: StepResult): boolean => {
steps.push(step);
progressLine(step.step, step.ok ? "step succeeded" : "step failed", { detail: step.detail });
return step.ok;
};
progressLine("deploy", "starting Code Queue deployment", {
commitId: options.commitId,
providerId: options.providerId,
repoUrl: options.repoUrl,
sourceRepoDir: options.sourceRepoDir,
deployDir: options.deployDir,
imageTag: options.imageTag,
timeoutMs: options.timeoutMs,
skipBuild: options.skipBuild,
});
const prepare = await backgroundStep(config, options, "prepare-source", prepareSourceScript(options, exportDir), remainingTimeout(deadline, 180_000), null);
if (!pushStep(prepare)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const resolveCommit = await directStep(config, options, "resolve-commit", bashScriptCommand(resolveCommitScript(options)), null, 60_000, 45_000);
if (!pushStep(resolveCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const resolvedCommit = parseFullCommit(dispatchStdout(resolveCommit.raw) || resolveCommit.detail);
if (resolvedCommit.length !== 40) {
const parseFailure = {
step: "resolve-commit-parse",
ok: false,
detail: `remote commit did not resolve to a full 40-character SHA; output=${resolveCommit.detail}`,
startedAt: nowIso(),
finishedAt: nowIso(),
};
pushStep(parseFailure);
return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
}
progressLine("resolve-commit", "resolved requested commit", { requestedCommit: options.commitId, resolvedCommit });
const sync = await directStep(config, options, "sync-deploy-tree", bashScriptCommand(syncDeployScript(options, exportDir)), null, 60_000, 45_000);
if (!pushStep(sync)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
if (options.skipBuild) {
steps.push({ step: "docker-build", ok: true, detail: "skipped by --skip-build", startedAt: nowIso(), finishedAt: nowIso() });
steps.push({ step: "import-k3s-image", ok: true, detail: "skipped by --skip-build", startedAt: nowIso(), finishedAt: nowIso() });
} else {
const build = await backgroundStep(config, options, "docker-build", buildImageScript(options), remainingTimeout(deadline, 540_000));
if (!pushStep(build)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const imageImport = await backgroundStep(config, options, "import-k3s-image", importImageScript(options), remainingTimeout(deadline, 180_000));
if (!pushStep(imageImport)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
}
const apply = await directStep(config, options, "kubectl-apply", applyManifestCommand(options), options.deployDir, 60_000, 45_000);
if (!pushStep(apply)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const stampCommit = await directStep(config, options, "stamp-deploy-commit", bashScriptCommand(stampDeployCommitScript(options, resolvedCommit)), options.deployDir, 60_000, 45_000);
if (!pushStep(stampCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, resolvedCommit, steps };
const restart = await directStep(config, options, "rollout-restart", rolloutRestartCommand(options), options.deployDir, 60_000, 45_000);
if (!pushStep(restart)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const rollout = await backgroundStep(config, options, "rollout-status", rolloutStatusScript(options), remainingTimeout(deadline, 240_000));
if (!pushStep(rollout)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
const health = await microserviceHealthStep(config, resolvedCommit, remainingTimeout(deadline, 90_000));
pushStep(health);
return {
ok: health.ok,
startedAt,
finishedAt: nowIso(),
options,
resolvedCommit,
steps,
statusCommands: {
health: "bun scripts/cli.ts microservice health code-queue",
overview: "bun scripts/cli.ts microservice proxy code-queue '/api/tasks/overview?limit=5&transcriptLimit=1&compact=1&afterSeq=0&preferId='",
},
};
}
function deployJobCommand(options: DeployCliOptions): string[] {
const command = [
process.execPath,
rootPath("scripts", "cli.ts"),
"codex",
"deploy",
options.commitId,
"--run-now",
"--provider-id",
options.providerId,
"--repo-url",
options.repoUrl,
"--source-repo-dir",
options.sourceRepoDir,
"--deploy-dir",
options.deployDir,
"--kubeconfig",
options.kubeconfigPath,
"--k3s-container",
options.k3sContainer,
"--image",
options.imageTag,
"--timeout-ms",
String(options.timeoutMs),
];
if (options.skipBuild) command.push("--skip-build");
return command;
}
function startDeployJob(options: DeployCliOptions): { ok: true; mode: "async-job"; job: JobRecord; commitId: string; providerId: string; statusCommand: string; tailCommand: string; note: string } {
const command = deployJobCommand(options);
const job = startJob("codex_deploy", command, `Deploy Code Queue ${options.commitId} to ${options.providerId} v3s/k8s`);
return {
ok: true,
mode: "async-job",
job,
commitId: options.commitId,
providerId: options.providerId,
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
tailCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`,
note: "Deployment continues in the background: fetch remote commit, export tracked files, sync D601 deploy tree, build/import image, apply k8s manifest, restart rollout, then verify Code Queue health.",
};
}
export async function runCodexDeployCommand(config: UniDeskConfig, args: string[]): Promise<unknown> {
const options = parseOptions(config, args);
validateOptions(options);
if (!options.runNow) return startDeployJob(options);
return codexDeploy(config, options);
}
File diff suppressed because it is too large Load Diff