Codex
351 lines
16 KiB
TypeScript
351 lines
16 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. remote module for scripts/src/deploy.ts.
|
|
|
|
// Moved mechanically from scripts/src/deploy.ts:2526-2839 for #903.
|
|
|
|
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
import { join, resolve } from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { runCommand } from "../command";
|
|
import { type UniDeskConfig, type UniDeskMicroserviceConfig, repoRoot, rootPath } from "../config";
|
|
import { ensureGithubSshIdentityForProvider } from "../deploy-ssh-identity";
|
|
import { baiduNetdiskRuntimeSecretRequirements, runtimeSecretContractFromEnvText, type RuntimeSecretContract, runArtifactRegistryCommand } from "../artifact-registry";
|
|
import { startJob } from "../jobs";
|
|
import { coreInternalFetch } from "../microservices";
|
|
import { codeQueueSourceImportPreflight, codeQueueSourceSubdir } from "../code-queue-source-guard";
|
|
import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard";
|
|
import { composeRuntimeEnvValue } from "../runtime-env";
|
|
import {
|
|
compareDeployJsonExecutorMirrors,
|
|
deployJsonCommitImage,
|
|
deployJsonDriftResult,
|
|
deployJsonSourceOfTruth,
|
|
encodeDeployJsonServiceContract,
|
|
hasDeployJsonExecutorContract,
|
|
k3sManifestExecutorMirror,
|
|
parseDeployJsonServiceContract,
|
|
type DeployJsonExecutorMirror,
|
|
type DeployJsonServiceContract,
|
|
} from "../deploy-json-contract";
|
|
|
|
import type { BackgroundPoll, DeployEnvironment, DeployManifestService, DispatchResult, RemoteScriptUpload, ServiceRuntimeState, StepResult } from "./types";
|
|
import { coreBody, dockerCommitProbeScript, healthDeployCommit, healthDeployRequestedCommit, healthSummary, k8sCommitProbeScript, runtimeCommitVerified, runtimeCurrentCommit, serviceHealth } from "./health";
|
|
import { asRecord, asString, compactTail, elapsedMs, nowIso, parseFullCommit, progressLine, safeId, shellQuote } from "./options";
|
|
import { k8sManifestPath, targetIsMain } from "./paths";
|
|
import { isDevK3sDeployService, unsupportedReason } from "./service-plan";
|
|
import { pollIntervalMs, providerDispatchCompletionLagMs, shortDispatchWaitMs, shortRemoteTimeoutMs } from "./types";
|
|
|
|
export async function dispatchSsh(
|
|
config: UniDeskConfig,
|
|
providerId: string,
|
|
command: string,
|
|
cwd: string | null,
|
|
waitMs = shortDispatchWaitMs,
|
|
remoteTimeoutMs = shortRemoteTimeoutMs,
|
|
): Promise<DispatchResult> {
|
|
const dispatchResponse = coreInternalFetch("/api/dispatch", {
|
|
method: "POST",
|
|
body: {
|
|
providerId,
|
|
command: "host.ssh",
|
|
payload: {
|
|
source: "deploy-reconciler",
|
|
mode: "exec",
|
|
command,
|
|
timeoutMs: remoteTimeoutMs,
|
|
...(cwd === null ? {} : { cwd }),
|
|
},
|
|
},
|
|
});
|
|
const dispatchBody = coreBody(dispatchResponse);
|
|
const taskId = asString(dispatchBody?.taskId);
|
|
if (dispatchBody?.ok !== true || taskId.length === 0) {
|
|
return {
|
|
ok: false,
|
|
taskId: taskId || null,
|
|
status: null,
|
|
stdout: "",
|
|
stderr: asString(dispatchBody?.error) || "dispatch did not return a task id",
|
|
exitCode: null,
|
|
raw: dispatchResponse,
|
|
};
|
|
}
|
|
const effectiveWaitMs = Math.max(waitMs, Math.min(remoteTimeoutMs + providerDispatchCompletionLagMs, 120_000));
|
|
const deadline = Date.now() + effectiveWaitMs;
|
|
let latest: unknown = null;
|
|
while (Date.now() < deadline) {
|
|
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`);
|
|
const task = asRecord(coreBody(latest)?.task);
|
|
const status = asString(task?.status);
|
|
if (status === "succeeded" || status === "failed") {
|
|
const result = asRecord(task?.result);
|
|
const exitCode = typeof result?.exitCode === "number" ? result.exitCode : null;
|
|
const stdout = asString(result?.stdout);
|
|
const stderr = asString(result?.stderr);
|
|
return {
|
|
ok: status === "succeeded" && (exitCode === null || exitCode === 0),
|
|
taskId,
|
|
status,
|
|
stdout,
|
|
stderr,
|
|
exitCode,
|
|
raw: task,
|
|
};
|
|
}
|
|
await Bun.sleep(500);
|
|
}
|
|
return {
|
|
ok: false,
|
|
taskId,
|
|
status: "timeout",
|
|
stdout: "",
|
|
stderr: `host.ssh task ${taskId} did not finish within ${effectiveWaitMs}ms`,
|
|
exitCode: null,
|
|
raw: latest,
|
|
};
|
|
}
|
|
|
|
export async function runTargetCommand(config: UniDeskConfig, service: UniDeskMicroserviceConfig, command: string, cwd: string | null, waitMs = 60_000, remoteTimeoutMs = 45_000): Promise<DispatchResult> {
|
|
if (targetIsMain(service)) {
|
|
const result = runCommand(["bash", "-lc", command], cwd ?? repoRoot);
|
|
return {
|
|
ok: result.exitCode === 0,
|
|
taskId: null,
|
|
status: result.exitCode === 0 ? "succeeded" : "failed",
|
|
stdout: result.stdout,
|
|
stderr: result.stderr,
|
|
exitCode: result.exitCode,
|
|
raw: result,
|
|
};
|
|
}
|
|
return await dispatchSsh(config, service.providerId, command, cwd, waitMs, remoteTimeoutMs);
|
|
}
|
|
|
|
export function splitFixed(value: string, size: number): string[] {
|
|
const chunks: string[] = [];
|
|
for (let index = 0; index < value.length; index += size) chunks.push(value.slice(index, index + size));
|
|
return chunks;
|
|
}
|
|
|
|
export async function uploadRemoteShellScript(
|
|
config: UniDeskConfig,
|
|
service: UniDeskMicroserviceConfig,
|
|
script: string,
|
|
cwd: string,
|
|
name: string,
|
|
): Promise<RemoteScriptUpload> {
|
|
const remotePath = `/tmp/unidesk-deploy-${safeId(service.id)}-${safeId(name)}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}.sh`;
|
|
const b64Path = `${remotePath}.b64`;
|
|
const raw: unknown[] = [];
|
|
const init = await runTargetCommand(config, service, `umask 077; rm -f ${shellQuote(remotePath)} ${shellQuote(b64Path)}; : > ${shellQuote(b64Path)}`, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
|
raw.push(init.raw);
|
|
if (!init.ok) return { ok: false, path: remotePath, error: init.stderr || init.stdout || "failed to initialize remote script upload", raw };
|
|
const encoded = Buffer.from(script, "utf8").toString("base64");
|
|
for (const chunk of splitFixed(encoded, 2400)) {
|
|
const append = await runTargetCommand(config, service, `printf %s ${shellQuote(chunk)} >> ${shellQuote(b64Path)}`, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
|
raw.push(append.raw);
|
|
if (!append.ok) return { ok: false, path: remotePath, error: append.stderr || append.stdout || "failed to append remote script chunk", raw };
|
|
}
|
|
const finalize = await runTargetCommand(config, service, `base64 -d ${shellQuote(b64Path)} > ${shellQuote(remotePath)}; chmod 700 ${shellQuote(remotePath)}; rm -f ${shellQuote(b64Path)}; wc -c ${shellQuote(remotePath)}`, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
|
raw.push(finalize.raw);
|
|
if (!finalize.ok) return { ok: false, path: remotePath, error: finalize.stderr || finalize.stdout || "failed to finalize remote script upload", raw };
|
|
return { ok: true, path: remotePath, error: "", raw };
|
|
}
|
|
|
|
export async function launchRemoteBackground(
|
|
config: UniDeskConfig,
|
|
service: UniDeskMicroserviceConfig,
|
|
shellScript: string,
|
|
cwd: string,
|
|
logFile: string,
|
|
sentinelFile: string,
|
|
): Promise<{ ok: boolean; pid: string; raw: unknown; error: string }> {
|
|
const wrapped = [
|
|
`bash -lc ${shellQuote(shellScript)}`,
|
|
"code=$?",
|
|
`printf '%s\\n' "$code" > ${shellQuote(sentinelFile)}`,
|
|
"exit \"$code\"",
|
|
].join("; ");
|
|
const command = [
|
|
`rm -f ${shellQuote(sentinelFile)} ${shellQuote(logFile)}`,
|
|
`nohup bash -lc ${shellQuote(wrapped)} > ${shellQuote(logFile)} 2>&1 < /dev/null & echo $!`,
|
|
].join("; ");
|
|
const result = await runTargetCommand(config, service, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
|
const pid = result.stdout.trim().split("\n").pop()?.trim() ?? "";
|
|
if (!result.ok || !/^\d+$/u.test(pid)) {
|
|
return { ok: false, pid: "", raw: result.raw, error: result.stderr || result.stdout || "failed to launch background command" };
|
|
}
|
|
return { ok: true, pid, raw: result.raw, error: "" };
|
|
}
|
|
|
|
export async function pollRemoteBackground(config: UniDeskConfig, service: UniDeskMicroserviceConfig, cwd: string, logFile: string, sentinelFile: string): Promise<BackgroundPoll> {
|
|
const command = [
|
|
`if [ -f ${shellQuote(sentinelFile)} ]; then printf 'SENTINEL:%s\\n' "$(cat ${shellQuote(sentinelFile)} 2>/dev/null || true)"; else echo RUNNING; fi`,
|
|
`tail -n 100 ${shellQuote(logFile)} 2>/dev/null | tr -d '\\000' | LC_ALL=C sed 's/[^[:print:]\t]//g' || true`,
|
|
].join("; ");
|
|
const result = await runTargetCommand(config, service, command, cwd, shortDispatchWaitMs, shortRemoteTimeoutMs);
|
|
const stdout = result.stdout.trimEnd();
|
|
const [head = "", ...rest] = stdout.split("\n");
|
|
if (head.startsWith("SENTINEL:")) {
|
|
const rawExitCode = head.slice("SENTINEL:".length).trim();
|
|
const exitCode = /^\d+$/u.test(rawExitCode) ? Number(rawExitCode) : null;
|
|
return { done: true, exitCode, logTail: rest.join("\n").trim(), raw: result.raw };
|
|
}
|
|
return { done: false, exitCode: null, logTail: rest.join("\n").trim(), raw: result.raw };
|
|
}
|
|
|
|
export async function step(
|
|
config: UniDeskConfig,
|
|
service: UniDeskMicroserviceConfig,
|
|
name: string,
|
|
command: string,
|
|
cwd: string | null,
|
|
timeoutMs: number,
|
|
background = false,
|
|
): Promise<StepResult> {
|
|
const startedAt = nowIso();
|
|
const startedMs = Date.now();
|
|
progressLine(name, "start", { serviceId: service.id, providerId: service.providerId, cwd });
|
|
if (!background || targetIsMain(service)) {
|
|
const result = await runTargetCommand(config, service, command, cwd, timeoutMs, timeoutMs);
|
|
const detail = compactTail([result.stdout, result.stderr].filter(Boolean).join("\n"), 2000);
|
|
const ok = result.ok;
|
|
progressLine(name, ok ? "succeeded" : "failed", { serviceId: service.id, elapsedMs: elapsedMs(startedMs), detail });
|
|
return { step: name, ok, detail: ok ? detail || `completed in ${elapsedMs(startedMs)}ms` : detail || "command failed", startedAt, finishedAt: nowIso(), raw: result.raw };
|
|
}
|
|
|
|
const runId = `${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
|
const logFile = `/tmp/unidesk-deploy-${safeId(service.id)}-${name}-${runId}.log`;
|
|
const sentinelFile = `/tmp/unidesk-deploy-${safeId(service.id)}-${name}-${runId}.done`;
|
|
let launchCommand = command;
|
|
if (launchCommand.length > 1800) {
|
|
const upload = await uploadRemoteShellScript(config, service, command, cwd ?? "/home/ubuntu", name);
|
|
if (!upload.ok) return { step: name, ok: false, detail: upload.error, startedAt, finishedAt: nowIso(), raw: upload.raw };
|
|
launchCommand = `bash ${shellQuote(upload.path)}`;
|
|
progressLine(name, "remote script uploaded", { serviceId: service.id, path: upload.path, bytes: command.length });
|
|
}
|
|
const launch = await launchRemoteBackground(config, service, launchCommand, cwd ?? "/home/ubuntu", logFile, sentinelFile);
|
|
if (!launch.ok) return { step: name, ok: false, detail: launch.error, startedAt, finishedAt: nowIso(), raw: launch.raw };
|
|
progressLine(name, "remote background started", { serviceId: service.id, pid: launch.pid, logFile });
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastTail = "";
|
|
while (Date.now() < deadline) {
|
|
await Bun.sleep(pollIntervalMs);
|
|
const poll = await pollRemoteBackground(config, service, cwd ?? "/home/ubuntu", logFile, sentinelFile);
|
|
const tail = compactTail(poll.logTail, 1800);
|
|
if (tail.length > 0 && tail !== lastTail) {
|
|
lastTail = tail;
|
|
progressLine(name, "remote log tail", { serviceId: service.id, elapsedMs: elapsedMs(startedMs), tail });
|
|
}
|
|
if (poll.done) {
|
|
const ok = poll.exitCode === 0;
|
|
const detail = ok
|
|
? `completed in ${elapsedMs(startedMs)}ms; log=${logFile}`
|
|
: `failed with exit ${poll.exitCode}; log=${logFile}; tail=${compactTail(poll.logTail)}`;
|
|
progressLine(name, ok ? "succeeded" : "failed", { serviceId: service.id, detail });
|
|
return { step: name, ok, detail, startedAt, finishedAt: nowIso(), raw: poll.raw };
|
|
}
|
|
}
|
|
return { step: name, ok: false, detail: `timed out after ${timeoutMs}ms`, startedAt, finishedAt: nowIso(), raw: null };
|
|
}
|
|
|
|
export async function readDockerImageCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<string | null> {
|
|
if (isDevK3sDeployService(service)) return null;
|
|
const result = await runTargetCommand(config, service, dockerCommitProbeScript(service), targetIsMain(service) ? repoRoot : "/home/ubuntu", 30_000, 20_000);
|
|
const commit = parseFullCommit(result.stdout);
|
|
return commit.length > 0 ? commit : null;
|
|
}
|
|
|
|
export async function readK8sCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<string | null> {
|
|
if (service.deployment.mode !== "k3sctl-managed") return null;
|
|
const result = await runTargetCommand(config, service, k8sCommitProbeScript(service), "/home/ubuntu", 30_000, 20_000);
|
|
const commit = parseFullCommit(result.stdout);
|
|
return commit.length > 0 ? commit : null;
|
|
}
|
|
|
|
export function manifestEnvironmentForService(service: UniDeskMicroserviceConfig): DeployEnvironment {
|
|
return service.deployment.namespace === "unidesk-dev" ? "dev" : "prod";
|
|
}
|
|
|
|
export function serviceDeployRef(service: UniDeskMicroserviceConfig): string {
|
|
return `deploy.json#environments.${manifestEnvironmentForService(service)}.services.${service.id}`;
|
|
}
|
|
|
|
export function k3sDeploymentManifestPath(service: UniDeskMicroserviceConfig): string {
|
|
if (service.deployment.namespace === "unidesk-dev") return k8sManifestPath(service);
|
|
if (service.id === "decision-center") {
|
|
return "src/components/microservices/k3sctl-adapter/k3s/decision-center.k8s.yaml";
|
|
}
|
|
return k8sManifestPath(service);
|
|
}
|
|
|
|
export async function readRuntimeState(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService): Promise<ServiceRuntimeState> {
|
|
const reason = unsupportedReason(service);
|
|
const health = await serviceHealth(config, service);
|
|
const healthBody = coreBody(health);
|
|
const healthCommit = healthDeployCommit(healthBody);
|
|
const healthRequestedCommit = healthDeployRequestedCommit(healthBody);
|
|
const healthRecord = asRecord(health);
|
|
const healthOk = healthRecord?.ok === true && healthBody?.ok !== false;
|
|
const [imageCommit, orchestratorCommit] = await Promise.all([
|
|
readDockerImageCommit(config, service).catch(() => null),
|
|
readK8sCommit(config, service).catch(() => null),
|
|
]);
|
|
const currentCommit = runtimeCurrentCommit(service, healthCommit, healthRequestedCommit, imageCommit, orchestratorCommit);
|
|
return {
|
|
serviceId: service.id,
|
|
ok: reason === null,
|
|
supported: reason === null,
|
|
reason,
|
|
desiredRepo: desired.repo,
|
|
desiredCommit: desired.commitId,
|
|
providerId: service.providerId,
|
|
deploymentMode: service.deployment.mode,
|
|
currentCommit,
|
|
healthCommit,
|
|
healthRequestedCommit,
|
|
imageCommit,
|
|
orchestratorCommit,
|
|
healthOk,
|
|
upToDate: reason === null && healthOk && runtimeCommitVerified(service, healthCommit, healthRequestedCommit, imageCommit, orchestratorCommit, desired.commitId),
|
|
raw: { health: healthSummary(health) },
|
|
};
|
|
}
|
|
|
|
export async function healthVerify(config: UniDeskConfig, service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string, timeoutMs: number): Promise<StepResult> {
|
|
const startedAt = nowIso();
|
|
const startedMs = Date.now();
|
|
const deadline = Date.now() + timeoutMs;
|
|
let latest: ServiceRuntimeState | null = null;
|
|
while (Date.now() < deadline) {
|
|
latest = await readRuntimeState(config, service, { ...desired, commitId: resolvedCommit });
|
|
const commitOk = runtimeCommitVerified(service, latest.healthCommit, latest.healthRequestedCommit, latest.imageCommit, latest.orchestratorCommit, resolvedCommit);
|
|
const ok = latest.healthOk && commitOk;
|
|
progressLine("live-health", "probe", { serviceId: service.id, ok, healthOk: latest.healthOk, expectedCommit: resolvedCommit, healthCommit: latest.healthCommit, healthRequestedCommit: latest.healthRequestedCommit, imageCommit: latest.imageCommit, orchestratorCommit: latest.orchestratorCommit });
|
|
if (ok) {
|
|
return {
|
|
step: "live-health",
|
|
ok: true,
|
|
detail: `service ${service.id} health passed with deployed commit ${resolvedCommit} in ${elapsedMs(startedMs)}ms`,
|
|
startedAt,
|
|
finishedAt: nowIso(),
|
|
raw: latest,
|
|
};
|
|
}
|
|
await Bun.sleep(3_000);
|
|
}
|
|
return {
|
|
step: "live-health",
|
|
ok: false,
|
|
detail: `service ${service.id} did not report expected commit ${resolvedCommit} within ${timeoutMs}ms`,
|
|
startedAt,
|
|
finishedAt: nowIso(),
|
|
raw: latest,
|
|
};
|
|
}
|
|
|
|
export function pushStep(steps: StepResult[], result: StepResult): boolean {
|
|
steps.push(result);
|
|
return result.ok;
|
|
}
|