feat: add desired-state deploy reconciler
This commit is contained in:
+12
-3
@@ -9,7 +9,7 @@ import { runSsh } from "./src/ssh";
|
||||
import { extractRemoteCliOptions, runRemoteCli } from "./src/remote";
|
||||
import { runMicroserviceCommand } from "./src/microservices";
|
||||
import { runCodeQueueCommand } from "./src/code-queue";
|
||||
import { runCodexDeployCommand } from "./src/codex-deploy";
|
||||
import { runCodeQueueDeployCompatCommand, runDeployCommand } from "./src/deploy";
|
||||
import { runProviderCommand } from "./src/provider-attach";
|
||||
import { runScheduleCommand } from "./src/schedules";
|
||||
|
||||
@@ -43,9 +43,10 @@ function help(): unknown {
|
||||
{ command: "microservice status <id>", description: "Show one user service config, repository reference, backend mapping, and runtime status." },
|
||||
{ command: "microservice health <id>", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy." },
|
||||
{ command: "microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--raw] [--max-body-bytes N]", description: "Access a private user-service backend path through the same frontend-only proxy used by WebUI; large bodies are summarized unless --raw is set." },
|
||||
{ command: "deploy check|plan|apply [--file deploy.json] [--service id] [--dry-run] [--force]", description: "Reconcile services from a repo+commit manifest using target-side build and live commit verification." },
|
||||
{ command: "schedule list|get|runs|run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N." },
|
||||
{ command: "schedule upsert-pgdata-backup [--time HH:MM] [--remote-base /SERVER_DATA/UNIDESK_PG_DATA]", description: "Create or update the daily PGDATA physical backup task that uploads monthly rotated archives to Baidu Netdisk." },
|
||||
{ command: "codex deploy <commitId> [--provider-id D601] [--timeout-ms N] [--skip-build]", description: "Start an async D601 v3s/k8s Code Queue deployment job from a specific remote git commit." },
|
||||
{ command: "codex deploy <commitId> [--provider-id D601] [--timeout-ms N]", description: "Compatibility wrapper for deploy apply --service code-queue with a temporary repo+commit manifest." },
|
||||
{ command: "codex task <taskId> [--trace --tail|--from-start|--after-seq N|--before-seq N --limit N] [--full]", description: "Fetch a compact Code Queue task summary; trace rows are opt-in and paged with next/previous commands to avoid output explosion." },
|
||||
{ command: "codex output <taskId> [--tail|--from-start|--after-seq N|--before-seq N --limit N] [--full-text]", description: "Fetch paged raw Code Queue output records by seq when a trace row has omitted command/output text." },
|
||||
{ command: "codex judge <taskId> --attempt N [--dry-run] [--include-prompt]", description: "Replay one stored Code Queue attempt through the same judge context builder and MiniMax judge call path used by the live queue worker." },
|
||||
@@ -179,6 +180,14 @@ async function main(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
if (top === "deploy") {
|
||||
const result = await runDeployCommand(config, args.slice(1));
|
||||
const ok = (result as { ok?: unknown }).ok !== false;
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (top === "provider") {
|
||||
emitJson(commandName, await runProviderCommand(config, args.slice(1)));
|
||||
return;
|
||||
@@ -191,7 +200,7 @@ async function main(): Promise<void> {
|
||||
|
||||
if (top === "codex") {
|
||||
if (sub === "deploy") {
|
||||
const result = await runCodexDeployCommand(config, args.slice(2));
|
||||
const result = await runCodeQueueDeployCompatCommand(config, args.slice(2));
|
||||
const ok = (result as { ok?: unknown }).ok !== false;
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
|
||||
@@ -70,7 +70,7 @@ export function runChecks(config: UniDeskConfig): { ok: boolean; items: CheckIte
|
||||
fileItem("src/components/microservices/oa-event-flow/src/index.ts"),
|
||||
fileItem("src/components/microservices/v3sctl-adapter/src/index.ts"),
|
||||
fileItem("src/components/microservices/mdtodo/src/index.ts"),
|
||||
fileItem("scripts/src/codex-deploy.ts"),
|
||||
fileItem("scripts/src/deploy.ts"),
|
||||
fileItem("scripts/src/e2e.ts"),
|
||||
unifiedLogRotationItem(),
|
||||
commandItem("bun:version", ["bun", "--version"]),
|
||||
|
||||
@@ -1,679 +0,0 @@
|
||||
import { startJob, type JobRecord } from "./jobs";
|
||||
import { coreInternalFetch } from "./microservices";
|
||||
import { type UniDeskConfig, rootPath } from "./config";
|
||||
|
||||
const defaultProviderId = "D601";
|
||||
const defaultTimeoutMs = 900_000;
|
||||
const pollIntervalMs = 5_000;
|
||||
const shortDispatchWaitMs = 25_000;
|
||||
const shortRemoteTimeoutMs = 20_000;
|
||||
|
||||
const defaultSourceRepoDir = "/home/ubuntu/unidesk";
|
||||
const defaultDeployDir = "/home/ubuntu/cq-deploy";
|
||||
const defaultKubeconfigPath = "/home/ubuntu/cq-deploy/.state/v8s/kubeconfig";
|
||||
const defaultK3sContainer = "unidesk-v8s-server";
|
||||
const defaultImageTag = "unidesk-code-queue:d601";
|
||||
const k8sNamespace = "unidesk";
|
||||
const k8sDeployment = "code-queue";
|
||||
const tcpEgressDeployment = "d601-tcp-egress-gateway";
|
||||
const k8sManifestRelPath = "src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml";
|
||||
|
||||
interface DeployCliOptions {
|
||||
commitId: string;
|
||||
providerId: string;
|
||||
repoUrl: string;
|
||||
sourceRepoDir: string;
|
||||
deployDir: string;
|
||||
kubeconfigPath: string;
|
||||
k3sContainer: string;
|
||||
imageTag: string;
|
||||
timeoutMs: number;
|
||||
skipBuild: boolean;
|
||||
runNow: boolean;
|
||||
}
|
||||
|
||||
interface StepResult {
|
||||
step: string;
|
||||
ok: boolean;
|
||||
detail: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
raw?: unknown;
|
||||
}
|
||||
|
||||
interface DispatchResult {
|
||||
ok: boolean;
|
||||
taskId: string | null;
|
||||
status: string | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
exitCode: number | null;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
interface BackgroundPoll {
|
||||
done: boolean;
|
||||
exitCode: number | null;
|
||||
logTail: string;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function asString(value: unknown): string {
|
||||
return typeof value === "string" ? value : "";
|
||||
}
|
||||
|
||||
function optionValue(args: string[], names: string[]): string | undefined {
|
||||
for (const name of names) {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) continue;
|
||||
const raw = args[index + 1];
|
||||
if (raw === undefined || raw.length === 0) throw new Error(`${name} requires a non-empty value`);
|
||||
return raw;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function positiveIntegerOption(args: string[], names: string[], defaultValue: number): number {
|
||||
const raw = optionValue(args, names);
|
||||
if (raw === undefined) return defaultValue;
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value <= 0) throw new Error(`${names[0]} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function positionalArgs(args: string[]): string[] {
|
||||
const result: string[] = [];
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const value = args[index] ?? "";
|
||||
if (value.startsWith("--")) {
|
||||
if (!["--skip-build", "--run-now"].includes(value)) index += 1;
|
||||
continue;
|
||||
}
|
||||
result.push(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function codeQueueRepoUrl(config: UniDeskConfig): string {
|
||||
const service = config.microservices.find((item) => item.id === "code-queue");
|
||||
if (service === undefined) throw new Error("config.json does not contain microservice id=code-queue");
|
||||
return service.repository.url;
|
||||
}
|
||||
|
||||
function parseOptions(config: UniDeskConfig, args: string[]): DeployCliOptions {
|
||||
const commitId = optionValue(args, ["--commit", "--commit-id"]) ?? positionalArgs(args)[0] ?? "";
|
||||
if (commitId.length === 0) {
|
||||
throw new Error("codex deploy requires a commit ID: codex deploy <commitId>");
|
||||
}
|
||||
return {
|
||||
commitId,
|
||||
providerId: optionValue(args, ["--provider-id", "--provider"]) ?? defaultProviderId,
|
||||
repoUrl: optionValue(args, ["--repo-url"]) ?? codeQueueRepoUrl(config),
|
||||
sourceRepoDir: optionValue(args, ["--source-repo-dir"]) ?? defaultSourceRepoDir,
|
||||
deployDir: optionValue(args, ["--deploy-dir"]) ?? defaultDeployDir,
|
||||
kubeconfigPath: optionValue(args, ["--kubeconfig"]) ?? defaultKubeconfigPath,
|
||||
k3sContainer: optionValue(args, ["--k3s-container"]) ?? defaultK3sContainer,
|
||||
imageTag: optionValue(args, ["--image"]) ?? defaultImageTag,
|
||||
timeoutMs: positiveIntegerOption(args, ["--timeout-ms"], defaultTimeoutMs),
|
||||
skipBuild: args.includes("--skip-build"),
|
||||
runNow: args.includes("--run-now"),
|
||||
};
|
||||
}
|
||||
|
||||
function validateOptions(options: DeployCliOptions): void {
|
||||
if (!/^[0-9a-f]{7,40}$/iu.test(options.commitId)) {
|
||||
throw new Error(`commit id must be a 7-40 character hex SHA, got: ${options.commitId}`);
|
||||
}
|
||||
if (!/^[A-Za-z0-9_.-]+$/u.test(options.providerId)) throw new Error(`invalid provider id: ${options.providerId}`);
|
||||
if (options.providerId !== defaultProviderId) {
|
||||
throw new Error(`codex deploy currently supports only ${defaultProviderId}; got ${options.providerId}`);
|
||||
}
|
||||
if (!/^https?:\/\//u.test(options.repoUrl) && !/^[A-Za-z0-9_.-]+@/u.test(options.repoUrl)) {
|
||||
throw new Error(`repo url must be an http(s) or ssh git URL, got: ${options.repoUrl}`);
|
||||
}
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function elapsedMs(startedAt: number): number {
|
||||
return Math.max(0, Date.now() - startedAt);
|
||||
}
|
||||
|
||||
function compactTail(text: string, maxChars = 900): string {
|
||||
return text.length > maxChars ? text.slice(text.length - maxChars) : text;
|
||||
}
|
||||
|
||||
function progressLine(step: string, message: string, detail?: unknown): void {
|
||||
const payload = detail === undefined
|
||||
? { at: nowIso(), step, message }
|
||||
: { at: nowIso(), step, message, detail };
|
||||
process.stderr.write(`${JSON.stringify(payload)}\n`);
|
||||
}
|
||||
|
||||
function coreBody(response: unknown): Record<string, unknown> | null {
|
||||
const record = asRecord(response);
|
||||
return asRecord(record?.body);
|
||||
}
|
||||
|
||||
function dispatchStdout(raw: unknown): string {
|
||||
const task = asRecord(raw);
|
||||
const result = asRecord(task?.result);
|
||||
return asString(result?.stdout);
|
||||
}
|
||||
|
||||
function parseFullCommit(value: string): string {
|
||||
const match = value.match(/\b[0-9a-f]{40}\b/iu);
|
||||
return match?.[0]?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
async function dispatchSsh(
|
||||
config: UniDeskConfig,
|
||||
providerId: string,
|
||||
command: string,
|
||||
cwd: string | null,
|
||||
waitMs = shortDispatchWaitMs,
|
||||
remoteTimeoutMs = shortRemoteTimeoutMs,
|
||||
): Promise<DispatchResult> {
|
||||
const dispatchResponse = coreInternalFetch("/api/dispatch", {
|
||||
method: "POST",
|
||||
body: {
|
||||
providerId,
|
||||
command: "host.ssh",
|
||||
payload: {
|
||||
source: "codex-deploy",
|
||||
mode: "exec",
|
||||
command,
|
||||
timeoutMs: remoteTimeoutMs,
|
||||
...(cwd === null ? {} : { cwd }),
|
||||
},
|
||||
},
|
||||
});
|
||||
const dispatchBody = coreBody(dispatchResponse);
|
||||
const taskId = asString(dispatchBody?.taskId);
|
||||
if (dispatchBody?.ok !== true || taskId.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
taskId: taskId || null,
|
||||
status: null,
|
||||
stdout: "",
|
||||
stderr: asString(dispatchBody?.error) || "dispatch did not return a task id",
|
||||
exitCode: null,
|
||||
raw: dispatchResponse,
|
||||
};
|
||||
}
|
||||
|
||||
const deadline = Date.now() + waitMs;
|
||||
let latest: unknown = null;
|
||||
while (Date.now() < deadline) {
|
||||
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`);
|
||||
const task = asRecord(coreBody(latest)?.task);
|
||||
const status = asString(task?.status);
|
||||
if (status === "succeeded" || status === "failed") {
|
||||
const result = asRecord(task?.result);
|
||||
const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null;
|
||||
const stdout = asString(result?.stdout);
|
||||
const stderr = asString(result?.stderr);
|
||||
return {
|
||||
ok: status === "succeeded" && (exitCode === null || exitCode === 0),
|
||||
taskId,
|
||||
status,
|
||||
stdout,
|
||||
stderr,
|
||||
exitCode,
|
||||
raw: task,
|
||||
};
|
||||
}
|
||||
await Bun.sleep(500);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
taskId,
|
||||
status: "timeout",
|
||||
stdout: "",
|
||||
stderr: `host.ssh task ${taskId} did not finish within ${waitMs}ms`,
|
||||
exitCode: null,
|
||||
raw: latest,
|
||||
};
|
||||
}
|
||||
|
||||
async function launchBackground(
|
||||
config: UniDeskConfig,
|
||||
providerId: string,
|
||||
shellScript: string,
|
||||
cwd: string,
|
||||
logFile: string,
|
||||
sentinelFile: string,
|
||||
): Promise<{ ok: boolean; pid: string; raw: unknown; error: string }> {
|
||||
const wrapped = [
|
||||
`bash -lc ${shellQuote(shellScript)}`,
|
||||
"code=$?",
|
||||
`printf '%s\\n' "$code" > ${shellQuote(sentinelFile)}`,
|
||||
"exit \"$code\"",
|
||||
].join("; ");
|
||||
const command = [
|
||||
`rm -f ${shellQuote(sentinelFile)} ${shellQuote(logFile)}`,
|
||||
`nohup bash -lc ${shellQuote(wrapped)} > ${shellQuote(logFile)} 2>&1 < /dev/null & echo $!`,
|
||||
].join("; ");
|
||||
const result = await dispatchSsh(config, providerId, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
||||
const pid = result.stdout.trim().split("\n").pop()?.trim() ?? "";
|
||||
if (!result.ok || !/^\d+$/u.test(pid)) {
|
||||
return { ok: false, pid: "", raw: result.raw, error: result.stderr || result.stdout || "failed to launch background command" };
|
||||
}
|
||||
return { ok: true, pid, raw: result.raw, error: "" };
|
||||
}
|
||||
|
||||
async function pollBackground(
|
||||
config: UniDeskConfig,
|
||||
providerId: string,
|
||||
cwd: string,
|
||||
logFile: string,
|
||||
sentinelFile: string,
|
||||
): Promise<BackgroundPoll> {
|
||||
const command = [
|
||||
`if [ -f ${shellQuote(sentinelFile)} ]; then printf 'SENTINEL:%s\\n' "$(cat ${shellQuote(sentinelFile)} 2>/dev/null || true)"; else echo RUNNING; fi`,
|
||||
`tail -n 80 ${shellQuote(logFile)} 2>/dev/null || true`,
|
||||
].join("; ");
|
||||
const result = await dispatchSsh(config, providerId, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
||||
const stdout = result.stdout.trimEnd();
|
||||
const [head = "", ...rest] = stdout.split("\n");
|
||||
if (head.startsWith("SENTINEL:")) {
|
||||
const rawExitCode = head.slice("SENTINEL:".length).trim();
|
||||
const exitCode = /^\d+$/u.test(rawExitCode) ? Number(rawExitCode) : null;
|
||||
return { done: true, exitCode, logTail: rest.join("\n").trim(), raw: result.raw };
|
||||
}
|
||||
return { done: false, exitCode: null, logTail: rest.join("\n").trim(), raw: result.raw };
|
||||
}
|
||||
|
||||
async function backgroundStep(
|
||||
config: UniDeskConfig,
|
||||
options: DeployCliOptions,
|
||||
step: string,
|
||||
shellScript: string,
|
||||
timeoutMs: number,
|
||||
cwd: string | null = options.deployDir,
|
||||
): Promise<StepResult> {
|
||||
const startedAt = nowIso();
|
||||
const startedMs = Date.now();
|
||||
const runId = `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
||||
const logFile = `/tmp/unidesk-codex-deploy-${step}-${runId}.log`;
|
||||
const sentinelFile = `/tmp/unidesk-codex-deploy-${step}-${runId}.done`;
|
||||
progressLine(step, "launching remote background step", { logFile, sentinelFile, timeoutMs });
|
||||
const launch = await launchBackground(config, options.providerId, shellScript, cwd ?? "/home/ubuntu", logFile, sentinelFile);
|
||||
if (!launch.ok) {
|
||||
return { step, ok: false, detail: launch.error, startedAt, finishedAt: nowIso(), raw: launch.raw };
|
||||
}
|
||||
progressLine(step, "remote background step started", { pid: launch.pid, logFile });
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastTail = "";
|
||||
while (Date.now() < deadline) {
|
||||
await Bun.sleep(pollIntervalMs);
|
||||
const poll = await pollBackground(config, options.providerId, cwd ?? "/home/ubuntu", logFile, sentinelFile);
|
||||
const tail = compactTail(poll.logTail, 1200);
|
||||
if (tail.length > 0 && tail !== lastTail) {
|
||||
lastTail = tail;
|
||||
progressLine(step, "remote log tail", { elapsedMs: elapsedMs(startedMs), tail });
|
||||
}
|
||||
if (poll.done) {
|
||||
const ok = poll.exitCode === 0;
|
||||
return {
|
||||
step,
|
||||
ok,
|
||||
detail: ok
|
||||
? `completed in ${elapsedMs(startedMs)}ms; log=${logFile}`
|
||||
: `failed with exit ${poll.exitCode}; log=${logFile}; tail=${compactTail(poll.logTail)}`,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
raw: poll.raw,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { step, ok: false, detail: `timed out after ${timeoutMs}ms; log=${logFile}`, startedAt, finishedAt: nowIso(), raw: null };
|
||||
}
|
||||
|
||||
function bashScriptCommand(script: string): string {
|
||||
return `bash -lc ${shellQuote(script)}`;
|
||||
}
|
||||
|
||||
async function directStep(
|
||||
config: UniDeskConfig,
|
||||
options: DeployCliOptions,
|
||||
step: string,
|
||||
command: string,
|
||||
cwd: string | null,
|
||||
waitMs = 60_000,
|
||||
remoteTimeoutMs = 45_000,
|
||||
): Promise<StepResult> {
|
||||
const startedAt = nowIso();
|
||||
const startedMs = Date.now();
|
||||
progressLine(step, "running remote command");
|
||||
const result = await dispatchSsh(config, options.providerId, command, cwd, waitMs, remoteTimeoutMs);
|
||||
const detail = compactTail([result.stdout, result.stderr].filter(Boolean).join("\n"), 1200);
|
||||
return {
|
||||
step,
|
||||
ok: result.ok,
|
||||
detail: result.ok ? detail || `completed in ${elapsedMs(startedMs)}ms` : detail || `failed with status=${result.status} exit=${result.exitCode}`,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
raw: result.raw,
|
||||
};
|
||||
}
|
||||
|
||||
function prepareSourceScript(options: DeployCliOptions, exportDir: string): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`repo=${shellQuote(options.sourceRepoDir)}`,
|
||||
`repo_url=${shellQuote(options.repoUrl)}`,
|
||||
`commit=${shellQuote(options.commitId)}`,
|
||||
`export_dir=${shellQuote(exportDir)}`,
|
||||
"mkdir -p \"$(dirname \"$repo\")\"",
|
||||
"if [ ! -d \"$repo/.git\" ]; then rm -rf \"$repo\"; git clone --no-checkout \"$repo_url\" \"$repo\"; fi",
|
||||
"cd \"$repo\"",
|
||||
"git remote get-url origin >/dev/null",
|
||||
"git fetch --no-tags origin \"$commit\" || git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
|
||||
"resolved=$(git rev-parse --verify \"$commit^{commit}\")",
|
||||
"rm -rf \"$export_dir\"",
|
||||
"mkdir -p \"$export_dir\"",
|
||||
"git archive --format=tar \"$resolved\" | tar -xf - -C \"$export_dir\"",
|
||||
"printf 'resolved_commit=%s\\nexport_dir=%s\\n' \"$resolved\" \"$export_dir\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function resolveCommitScript(options: DeployCliOptions): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`repo=${shellQuote(options.sourceRepoDir)}`,
|
||||
`commit=${shellQuote(options.commitId)}`,
|
||||
"git -C \"$repo\" rev-parse --verify \"$commit^{commit}\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function syncDeployScript(options: DeployCliOptions, exportDir: string): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`export_dir=${shellQuote(exportDir)}`,
|
||||
`deploy_dir=${shellQuote(options.deployDir)}`,
|
||||
"test -f \"$export_dir/src/components/microservices/code-queue/Dockerfile\"",
|
||||
"mkdir -p \"$deploy_dir\"",
|
||||
[
|
||||
"rsync -a --delete",
|
||||
"--exclude '.git/'",
|
||||
"--exclude '.state/'",
|
||||
"--exclude 'logs/'",
|
||||
"--exclude '**/node_modules/'",
|
||||
"--exclude '**/dist/'",
|
||||
"\"$export_dir/\"",
|
||||
"\"$deploy_dir/\"",
|
||||
].join(" "),
|
||||
"test -f \"$deploy_dir/src/components/microservices/code-queue/Dockerfile\"",
|
||||
"test -f \"$deploy_dir/src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml\"",
|
||||
"printf 'synced deploy tree to %s\\n' \"$deploy_dir\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function buildImageScript(options: DeployCliOptions): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`deploy_dir=${shellQuote(options.deployDir)}`,
|
||||
`image=${shellQuote(options.imageTag)}`,
|
||||
"cd \"$deploy_dir\"",
|
||||
"docker build -t \"$image\" -f src/components/microservices/code-queue/Dockerfile .",
|
||||
"docker image inspect \"$image\" --format 'image_id={{.Id}} repo_tags={{json .RepoTags}}'",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function importImageScript(options: DeployCliOptions): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`image=${shellQuote(options.imageTag)}`,
|
||||
`k3s_container=${shellQuote(options.k3sContainer)}`,
|
||||
"docker image inspect \"$image\" >/dev/null",
|
||||
"docker ps --format '{{.Names}}' | grep -Fx \"$k3s_container\" >/dev/null",
|
||||
"docker save \"$image\" | docker exec -i \"$k3s_container\" ctr -n k8s.io images import -",
|
||||
"docker exec \"$k3s_container\" ctr -n k8s.io images ls | grep -F \"$image\" || true",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function applyManifestCommand(options: DeployCliOptions): string {
|
||||
const manifest = `${options.deployDir}/${k8sManifestRelPath}`;
|
||||
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl apply -f ${shellQuote(manifest)}`;
|
||||
}
|
||||
|
||||
function stampDeployCommitScript(options: DeployCliOptions, resolvedCommit: string): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`kubeconfig=${shellQuote(options.kubeconfigPath)}`,
|
||||
`namespace=${shellQuote(k8sNamespace)}`,
|
||||
`deployment=${shellQuote(k8sDeployment)}`,
|
||||
`tcp_deployment=${shellQuote(tcpEgressDeployment)}`,
|
||||
`resolved_commit=${shellQuote(resolvedCommit)}`,
|
||||
`requested_commit=${shellQuote(options.commitId)}`,
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" set env \"deployment/$tcp_deployment\" \"deployment/$deployment\" CODE_QUEUE_DEPLOY_COMMIT=\"$resolved_commit\" CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=\"$requested_commit\"",
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" annotate \"deployment/$tcp_deployment\" \"deployment/$deployment\" unidesk.ai/deploy-commit=\"$resolved_commit\" unidesk.ai/deploy-requested-commit=\"$requested_commit\" --overwrite",
|
||||
"current=$(KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"CODE_QUEUE_DEPLOY_COMMIT\")].value}')",
|
||||
"test \"$current\" = \"$resolved_commit\"",
|
||||
"printf 'deployment_commit=%s\\nrequested_commit=%s\\n' \"$current\" \"$requested_commit\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function rolloutRestartCommand(options: DeployCliOptions): string {
|
||||
return `KUBECONFIG=${shellQuote(options.kubeconfigPath)} kubectl -n ${shellQuote(k8sNamespace)} rollout restart deployment/${shellQuote(tcpEgressDeployment)} deployment/${shellQuote(k8sDeployment)}`;
|
||||
}
|
||||
|
||||
function rolloutStatusScript(options: DeployCliOptions): string {
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
`kubeconfig=${shellQuote(options.kubeconfigPath)}`,
|
||||
`namespace=${shellQuote(k8sNamespace)}`,
|
||||
`deployment=${shellQuote(k8sDeployment)}`,
|
||||
`tcp_deployment=${shellQuote(tcpEgressDeployment)}`,
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" rollout status \"deployment/$tcp_deployment\" --timeout=120s",
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" rollout status \"deployment/$deployment\" --timeout=180s",
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get deploy \"$tcp_deployment\" \"$deployment\" -o wide",
|
||||
"KUBECONFIG=\"$kubeconfig\" kubectl -n \"$namespace\" get pods -l app.kubernetes.io/name=code-queue,unidesk.ai/instance-id=D601 -o wide",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function healthDeployCommit(body: Record<string, unknown> | null): string {
|
||||
const deploy = asRecord(body?.deploy);
|
||||
return asString(deploy?.commit).toLowerCase();
|
||||
}
|
||||
|
||||
function healthProbeSummary(body: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
if (body === null) return null;
|
||||
return {
|
||||
ok: body.ok,
|
||||
service: body.service,
|
||||
instanceId: body.instanceId,
|
||||
deploy: body.deploy,
|
||||
status: body.status,
|
||||
databaseReady: body.databaseReady,
|
||||
startedAt: body.startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function microserviceHealthStep(config: UniDeskConfig, expectedCommit: string, timeoutMs: number): Promise<StepResult> {
|
||||
const step = "unidesk-health";
|
||||
const startedAt = nowIso();
|
||||
const startedMs = Date.now();
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let latest: unknown = null;
|
||||
let latestCommit = "";
|
||||
while (Date.now() < deadline) {
|
||||
latest = coreInternalFetch("/api/microservices/code-queue/health");
|
||||
const record = asRecord(latest);
|
||||
const body = asRecord(record?.body);
|
||||
latestCommit = healthDeployCommit(body);
|
||||
const commitMatches = latestCommit === expectedCommit;
|
||||
const ok = record?.ok === true && body?.ok !== false && commitMatches;
|
||||
progressLine(step, "health probe", { ok, status: record?.status ?? null, expectedCommit, deployedCommit: latestCommit || null, commitMatches, body: healthProbeSummary(body) });
|
||||
if (ok) {
|
||||
return {
|
||||
step,
|
||||
ok: true,
|
||||
detail: `Code Queue health passed with deployed commit ${latestCommit} in ${elapsedMs(startedMs)}ms`,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
raw: latest,
|
||||
};
|
||||
}
|
||||
await Bun.sleep(pollIntervalMs);
|
||||
}
|
||||
return {
|
||||
step,
|
||||
ok: false,
|
||||
detail: `Code Queue health did not expose expected commit ${expectedCommit} within ${timeoutMs}ms; latest deployed commit=${latestCommit || "none"}`,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
raw: latest,
|
||||
};
|
||||
}
|
||||
|
||||
function remainingTimeout(deadline: number, fallbackMs: number): number {
|
||||
return Math.max(30_000, Math.min(fallbackMs, deadline - Date.now()));
|
||||
}
|
||||
|
||||
export async function codexDeploy(config: UniDeskConfig, options: DeployCliOptions): Promise<unknown> {
|
||||
validateOptions(options);
|
||||
const startedAt = nowIso();
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const exportDir = `/tmp/unidesk-codex-deploy-src-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
||||
const steps: StepResult[] = [];
|
||||
const pushStep = (step: StepResult): boolean => {
|
||||
steps.push(step);
|
||||
progressLine(step.step, step.ok ? "step succeeded" : "step failed", { detail: step.detail });
|
||||
return step.ok;
|
||||
};
|
||||
|
||||
progressLine("deploy", "starting Code Queue deployment", {
|
||||
commitId: options.commitId,
|
||||
providerId: options.providerId,
|
||||
repoUrl: options.repoUrl,
|
||||
sourceRepoDir: options.sourceRepoDir,
|
||||
deployDir: options.deployDir,
|
||||
imageTag: options.imageTag,
|
||||
timeoutMs: options.timeoutMs,
|
||||
skipBuild: options.skipBuild,
|
||||
});
|
||||
|
||||
const prepare = await backgroundStep(config, options, "prepare-source", prepareSourceScript(options, exportDir), remainingTimeout(deadline, 180_000), null);
|
||||
if (!pushStep(prepare)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const resolveCommit = await directStep(config, options, "resolve-commit", bashScriptCommand(resolveCommitScript(options)), null, 60_000, 45_000);
|
||||
if (!pushStep(resolveCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
const resolvedCommit = parseFullCommit(dispatchStdout(resolveCommit.raw) || resolveCommit.detail);
|
||||
if (resolvedCommit.length !== 40) {
|
||||
const parseFailure = {
|
||||
step: "resolve-commit-parse",
|
||||
ok: false,
|
||||
detail: `remote commit did not resolve to a full 40-character SHA; output=${resolveCommit.detail}`,
|
||||
startedAt: nowIso(),
|
||||
finishedAt: nowIso(),
|
||||
};
|
||||
pushStep(parseFailure);
|
||||
return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
}
|
||||
progressLine("resolve-commit", "resolved requested commit", { requestedCommit: options.commitId, resolvedCommit });
|
||||
|
||||
const sync = await directStep(config, options, "sync-deploy-tree", bashScriptCommand(syncDeployScript(options, exportDir)), null, 60_000, 45_000);
|
||||
if (!pushStep(sync)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
if (options.skipBuild) {
|
||||
steps.push({ step: "docker-build", ok: true, detail: "skipped by --skip-build", startedAt: nowIso(), finishedAt: nowIso() });
|
||||
steps.push({ step: "import-k3s-image", ok: true, detail: "skipped by --skip-build", startedAt: nowIso(), finishedAt: nowIso() });
|
||||
} else {
|
||||
const build = await backgroundStep(config, options, "docker-build", buildImageScript(options), remainingTimeout(deadline, 540_000));
|
||||
if (!pushStep(build)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
const imageImport = await backgroundStep(config, options, "import-k3s-image", importImageScript(options), remainingTimeout(deadline, 180_000));
|
||||
if (!pushStep(imageImport)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
}
|
||||
|
||||
const apply = await directStep(config, options, "kubectl-apply", applyManifestCommand(options), options.deployDir, 60_000, 45_000);
|
||||
if (!pushStep(apply)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const stampCommit = await directStep(config, options, "stamp-deploy-commit", bashScriptCommand(stampDeployCommitScript(options, resolvedCommit)), options.deployDir, 60_000, 45_000);
|
||||
if (!pushStep(stampCommit)) return { ok: false, startedAt, finishedAt: nowIso(), options, resolvedCommit, steps };
|
||||
|
||||
const restart = await directStep(config, options, "rollout-restart", rolloutRestartCommand(options), options.deployDir, 60_000, 45_000);
|
||||
if (!pushStep(restart)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const rollout = await backgroundStep(config, options, "rollout-status", rolloutStatusScript(options), remainingTimeout(deadline, 240_000));
|
||||
if (!pushStep(rollout)) return { ok: false, startedAt, finishedAt: nowIso(), options, steps };
|
||||
|
||||
const health = await microserviceHealthStep(config, resolvedCommit, remainingTimeout(deadline, 90_000));
|
||||
pushStep(health);
|
||||
|
||||
return {
|
||||
ok: health.ok,
|
||||
startedAt,
|
||||
finishedAt: nowIso(),
|
||||
options,
|
||||
resolvedCommit,
|
||||
steps,
|
||||
statusCommands: {
|
||||
health: "bun scripts/cli.ts microservice health code-queue",
|
||||
overview: "bun scripts/cli.ts microservice proxy code-queue '/api/tasks/overview?limit=5&transcriptLimit=1&compact=1&afterSeq=0&preferId='",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function deployJobCommand(options: DeployCliOptions): string[] {
|
||||
const command = [
|
||||
process.execPath,
|
||||
rootPath("scripts", "cli.ts"),
|
||||
"codex",
|
||||
"deploy",
|
||||
options.commitId,
|
||||
"--run-now",
|
||||
"--provider-id",
|
||||
options.providerId,
|
||||
"--repo-url",
|
||||
options.repoUrl,
|
||||
"--source-repo-dir",
|
||||
options.sourceRepoDir,
|
||||
"--deploy-dir",
|
||||
options.deployDir,
|
||||
"--kubeconfig",
|
||||
options.kubeconfigPath,
|
||||
"--k3s-container",
|
||||
options.k3sContainer,
|
||||
"--image",
|
||||
options.imageTag,
|
||||
"--timeout-ms",
|
||||
String(options.timeoutMs),
|
||||
];
|
||||
if (options.skipBuild) command.push("--skip-build");
|
||||
return command;
|
||||
}
|
||||
|
||||
function startDeployJob(options: DeployCliOptions): { ok: true; mode: "async-job"; job: JobRecord; commitId: string; providerId: string; statusCommand: string; tailCommand: string; note: string } {
|
||||
const command = deployJobCommand(options);
|
||||
const job = startJob("codex_deploy", command, `Deploy Code Queue ${options.commitId} to ${options.providerId} v3s/k8s`);
|
||||
return {
|
||||
ok: true,
|
||||
mode: "async-job",
|
||||
job,
|
||||
commitId: options.commitId,
|
||||
providerId: options.providerId,
|
||||
statusCommand: `bun scripts/cli.ts job status ${job.id}`,
|
||||
tailCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`,
|
||||
note: "Deployment continues in the background: fetch remote commit, export tracked files, sync D601 deploy tree, build/import image, apply k8s manifest, restart rollout, then verify Code Queue health.",
|
||||
};
|
||||
}
|
||||
|
||||
export async function runCodexDeployCommand(config: UniDeskConfig, args: string[]): Promise<unknown> {
|
||||
const options = parseOptions(config, args);
|
||||
validateOptions(options);
|
||||
if (!options.runNow) return startDeployJob(options);
|
||||
return codexDeploy(config, options);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user