Codex
344 lines
15 KiB
TypeScript
344 lines
15 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. health module for scripts/src/deploy.ts.
|
|
|
|
// Moved mechanically from scripts/src/deploy.ts:2219-2525 for #903.
|
|
|
|
import { createHash } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
import { join, resolve } from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { runCommand } from "../command";
|
|
import { type UniDeskConfig, type UniDeskMicroserviceConfig, repoRoot, rootPath } from "../config";
|
|
import { ensureGithubSshIdentityForProvider } from "../deploy-ssh-identity";
|
|
import { baiduNetdiskRuntimeSecretRequirements, runtimeSecretContractFromEnvText, type RuntimeSecretContract, runArtifactRegistryCommand } from "../artifact-registry";
|
|
import { startJob } from "../jobs";
|
|
import { coreInternalFetch } from "../microservices";
|
|
import { codeQueueSourceImportPreflight, codeQueueSourceSubdir } from "../code-queue-source-guard";
|
|
import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard";
|
|
import { composeRuntimeEnvValue } from "../runtime-env";
|
|
import {
|
|
compareDeployJsonExecutorMirrors,
|
|
deployJsonCommitImage,
|
|
deployJsonDriftResult,
|
|
deployJsonSourceOfTruth,
|
|
encodeDeployJsonServiceContract,
|
|
hasDeployJsonExecutorContract,
|
|
k3sManifestExecutorMirror,
|
|
parseDeployJsonServiceContract,
|
|
type DeployJsonExecutorMirror,
|
|
type DeployJsonServiceContract,
|
|
} from "../deploy-json-contract";
|
|
|
|
import type { DeployManifestService } from "./types";
|
|
import { asRecord, asString, d601K3sGuardScript, parseJsonObjectFromText, shellQuote } from "./options";
|
|
import { buildImageTag, k8sManifestPath, targetWorkDir } from "./paths";
|
|
import { runTargetCommand } from "./remote";
|
|
import { isCoreDeployService, isDevK3sDeployService } from "./service-plan";
|
|
import { k8sKubeconfig, k8sNamespace } from "./types";
|
|
|
|
export function cleanupLegacyDirectCodeQueueScript(service: UniDeskMicroserviceConfig): string {
|
|
if (service.id !== "code-queue") return "";
|
|
if (isDevK3sDeployService(service)) return "";
|
|
return [
|
|
"set -euo pipefail",
|
|
"container=code-queue-backend",
|
|
"if docker ps -a --format '{{.Names}}' | grep -Fx \"$container\" >/dev/null; then",
|
|
" docker update --restart=no \"$container\" >/dev/null 2>&1 || true",
|
|
` compose_file=${shellQuote(`${targetWorkDir(service)}/src/components/microservices/code-queue/docker-compose.d601.yml`)}`,
|
|
" if [ -f \"$compose_file\" ]; then",
|
|
" docker compose -p code-queue -f \"$compose_file\" down --remove-orphans",
|
|
" else",
|
|
" docker rm -f \"$container\"",
|
|
" fi",
|
|
" echo stopped_legacy_direct_code_queue=$container",
|
|
"else",
|
|
" echo stopped_legacy_direct_code_queue=not-present",
|
|
"fi",
|
|
"if docker ps --format '{{.Names}}' | grep -Fx \"$container\" >/dev/null; then",
|
|
" echo legacy_direct_code_queue_still_running=$container >&2",
|
|
" exit 1",
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function k8sNamespaceForService(service: UniDeskMicroserviceConfig): string {
|
|
return service.deployment.namespace ?? k8sNamespace;
|
|
}
|
|
|
|
export function k8sDeploymentsForService(service: UniDeskMicroserviceConfig): string[] {
|
|
if (isDevK3sDeployService(service) && service.id === "code-queue") {
|
|
return ["d601-dev-provider-egress-proxy", "code-queue-scheduler-dev", "code-queue-read-dev", "code-queue-write-dev"];
|
|
}
|
|
if (service.id === "code-queue") return ["d601-provider-egress-proxy", "d601-tcp-egress-gateway", "code-queue", "code-queue-read", "code-queue-write"];
|
|
return [service.repository.composeService];
|
|
}
|
|
|
|
export function applyK8sScript(service: UniDeskMicroserviceConfig): string {
|
|
const manifest = `${targetWorkDir(service)}/${k8sManifestPath(service)}`;
|
|
const namespace = k8sNamespaceForService(service);
|
|
const cleanup = service.id === "code-queue" && !isDevK3sDeployService(service)
|
|
? [
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} delete endpointslice d601-provider-egress-proxy --ignore-not-found`,
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} delete deployment code-queue-d518 --ignore-not-found`,
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} delete service code-queue-d518 --ignore-not-found`,
|
|
].join("\n")
|
|
: "";
|
|
return [
|
|
"set -euo pipefail",
|
|
d601K3sGuardScript(),
|
|
cleanup,
|
|
`kubectl apply -f ${shellQuote(manifest)}`,
|
|
].filter(Boolean).join("\n");
|
|
}
|
|
|
|
export function verifyK8sImagesScript(service: UniDeskMicroserviceConfig): string {
|
|
const namespace = k8sNamespaceForService(service);
|
|
const image = buildImageTag(service);
|
|
const deployments = k8sDeploymentsForService(service);
|
|
return [
|
|
"set -euo pipefail",
|
|
`image=${shellQuote(image)}`,
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`deployments=(${deployments.map(shellQuote).join(" ")})`,
|
|
"for deployment in \"${deployments[@]}\"; do",
|
|
` actual=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}')`,
|
|
" if [ -z \"$actual\" ]; then",
|
|
" echo \"deployment_image_missing=$deployment\" >&2",
|
|
" exit 1",
|
|
" fi",
|
|
" while IFS= read -r actual_image; do",
|
|
" [ -z \"$actual_image\" ] && continue",
|
|
" if [ \"$actual_image\" != \"$image\" ]; then",
|
|
" printf 'deployment_image_mismatch deployment=%s expected=%s actual=%s\\n' \"$deployment\" \"$image\" \"$actual_image\" >&2",
|
|
" exit 1",
|
|
" fi",
|
|
" done <<EOF",
|
|
"$actual",
|
|
"EOF",
|
|
" printf 'deployment_image_ok deployment=%s image=%s\\n' \"$deployment\" \"$image\"",
|
|
"done",
|
|
].join("\n");
|
|
}
|
|
|
|
export function stampK8sScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, resolvedCommit: string): string {
|
|
const deployments = k8sDeploymentsForService(service).map((name) => `deployment/${name}`);
|
|
const namespace = k8sNamespaceForService(service);
|
|
const envPairs = [
|
|
`UNIDESK_DEPLOY_SERVICE_ID=${service.id}`,
|
|
`UNIDESK_DEPLOY_REPO=${desired.repo}`,
|
|
`UNIDESK_DEPLOY_COMMIT=${resolvedCommit}`,
|
|
`UNIDESK_DEPLOY_REQUESTED_COMMIT=${desired.commitId}`,
|
|
...(service.id === "code-queue" ? [
|
|
`CODE_QUEUE_DEPLOY_COMMIT=${resolvedCommit}`,
|
|
`CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=${desired.commitId}`,
|
|
] : []),
|
|
];
|
|
const annotatePairs = [
|
|
`unidesk.ai/deploy-service-id=${service.id}`,
|
|
`unidesk.ai/deploy-repo=${desired.repo}`,
|
|
`unidesk.ai/deploy-commit=${resolvedCommit}`,
|
|
`unidesk.ai/deploy-requested-commit=${desired.commitId}`,
|
|
];
|
|
return [
|
|
"set -euo pipefail",
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} set env ${deployments.map(shellQuote).join(" ")} ${envPairs.map(shellQuote).join(" ")}`,
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} annotate ${deployments.map(shellQuote).join(" ")} ${annotatePairs.map(shellQuote).join(" ")} --overwrite`,
|
|
`actual=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} get deployment ${shellQuote(service.repository.composeService)} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}')`,
|
|
`test "$actual" = ${shellQuote(resolvedCommit)}`,
|
|
"printf 'k8s_deploy_commit=%s\\n' \"$actual\"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function rolloutK8sScript(service: UniDeskMicroserviceConfig): string {
|
|
const deployments = k8sDeploymentsForService(service);
|
|
const namespace = k8sNamespaceForService(service);
|
|
return [
|
|
"set -euo pipefail",
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} rollout restart ${deployments.map((name) => shellQuote(`deployment/${name}`)).join(" ")}`,
|
|
...deployments.map((name) => `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} rollout status ${shellQuote(`deployment/${name}`)} --timeout=180s`),
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} get deploy ${deployments.map(shellQuote).join(" ")} -o wide`,
|
|
].join("\n");
|
|
}
|
|
|
|
export function k8sCommitProbeScript(service: UniDeskMicroserviceConfig): string {
|
|
const namespace = k8sNamespaceForService(service);
|
|
return [
|
|
"set -euo pipefail",
|
|
`commit=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} get deployment ${shellQuote(service.repository.composeService)} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}' 2>/dev/null || true)`,
|
|
"printf '%s\\n' \"$commit\"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function dockerCommitProbeScript(service: UniDeskMicroserviceConfig): string {
|
|
return [
|
|
"set -euo pipefail",
|
|
`container=${shellQuote(service.repository.containerName)}`,
|
|
"cid=$(docker ps -q -f name=\"^/${container}$\" | head -1)",
|
|
"if [ -z \"$cid\" ]; then exit 0; fi",
|
|
"image_id=$(docker inspect -f '{{.Image}}' \"$cid\")",
|
|
"docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\" 2>/dev/null || true",
|
|
].join("\n");
|
|
}
|
|
|
|
export function healthDeployCommit(body: Record<string, unknown> | null): string | null {
|
|
const deploy = asRecord(body?.deploy);
|
|
const commit = asString(deploy?.commit).toLowerCase();
|
|
return commit.length > 0 ? commit : null;
|
|
}
|
|
|
|
export function healthDeployRequestedCommit(body: Record<string, unknown> | null): string | null {
|
|
const deploy = asRecord(body?.deploy);
|
|
const commit = asString(deploy?.requestedCommit).toLowerCase();
|
|
return commit.length > 0 ? commit : null;
|
|
}
|
|
|
|
export function healthSummary(response: unknown): Record<string, unknown> {
|
|
const record = asRecord(response);
|
|
const body = asRecord(record?.body);
|
|
return {
|
|
ok: record?.ok ?? false,
|
|
status: record?.status ?? null,
|
|
body: body === null
|
|
? null
|
|
: {
|
|
ok: body.ok ?? null,
|
|
service: body.service ?? null,
|
|
instanceId: body.instanceId ?? null,
|
|
deploy: body.deploy ?? null,
|
|
status: body.status ?? null,
|
|
startedAt: body.startedAt ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function directHttpJson(url: string, timeoutMs: number): Promise<unknown> {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetch(url, { signal: controller.signal });
|
|
const text = await response.text();
|
|
let body: unknown = null;
|
|
try {
|
|
body = text.length > 0 ? JSON.parse(text) : null;
|
|
} catch {
|
|
body = { text };
|
|
}
|
|
return { ok: response.ok, status: response.status, body };
|
|
} catch (error) {
|
|
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export function devK3sServiceHealthScript(service: UniDeskMicroserviceConfig): string {
|
|
const namespace = k8sNamespaceForService(service);
|
|
const serviceName = service.repository.composeService;
|
|
const path = service.backend.healthPath;
|
|
const port = service.backend.nodePort;
|
|
return [
|
|
"set -euo pipefail",
|
|
`namespace=${shellQuote(namespace)}`,
|
|
`service_name=${shellQuote(serviceName)}`,
|
|
`path=${shellQuote(path)}`,
|
|
`port=${shellQuote(String(port))}`,
|
|
"case \"$path\" in /*) ;; *) path=\"/$path\" ;; esac",
|
|
"proxy_path=\"/api/v1/namespaces/$namespace/services/http:$service_name:$port/proxy$path\"",
|
|
"tmp_health=$(mktemp)",
|
|
"trap 'rm -f \"$tmp_health\"' EXIT",
|
|
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get --raw "$proxy_path" > "$tmp_health"`,
|
|
"python3 - \"$tmp_health\" <<'PY'",
|
|
"import json",
|
|
"import sys",
|
|
"",
|
|
"text = open(sys.argv[1], encoding='utf-8', errors='replace').read()",
|
|
"try:",
|
|
" body = json.loads(text)",
|
|
"except Exception:",
|
|
" print(text[:4000])",
|
|
" raise SystemExit(0)",
|
|
"",
|
|
"def compact(mapping, keys):",
|
|
" if not isinstance(mapping, dict):",
|
|
" return None",
|
|
" return {key: mapping.get(key) for key in keys if key in mapping}",
|
|
"",
|
|
"summary = compact(body, ['ok', 'service', 'instanceId', 'role', 'environment', 'namespace', 'databaseName', 'serviceId', 'status', 'startedAt', 'deployRef']) or {}",
|
|
"deploy = compact(body.get('deploy'), ['repo', 'commit', 'requestedCommit', 'serviceId'])",
|
|
"if deploy is not None:",
|
|
" summary['deploy'] = deploy",
|
|
"queue = body.get('queue')",
|
|
"if isinstance(queue, dict):",
|
|
" queue_summary = compact(queue, ['total', 'queueCount', 'defaultQueueId', 'processing']) or {}",
|
|
" storage = compact(queue.get('storage'), ['primary', 'postgresReady'])",
|
|
" if storage is not None:",
|
|
" queue_summary['storage'] = storage",
|
|
" summary['queue'] = queue_summary",
|
|
"print(json.dumps(summary, ensure_ascii=False, separators=(',', ':')))",
|
|
"PY",
|
|
].join("\n");
|
|
}
|
|
|
|
export async function serviceHealth(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<unknown> {
|
|
if (isCoreDeployService(service)) {
|
|
return await directHttpJson(`${service.backend.nodeBaseUrl}${service.backend.healthPath}`, service.backend.timeoutMs);
|
|
}
|
|
if (isDevK3sDeployService(service)) {
|
|
const result = await runTargetCommand(config, service, devK3sServiceHealthScript(service), "/home/ubuntu", 30_000, 20_000);
|
|
const stdout = result.stdout.trim();
|
|
let body: unknown = null;
|
|
try {
|
|
body = stdout.length > 0 ? parseJsonObjectFromText(stdout) : null;
|
|
} catch {
|
|
body = null;
|
|
}
|
|
if (body === null && stdout.length > 0) body = { text: stdout };
|
|
return {
|
|
ok: result.ok,
|
|
status: result.ok ? 200 : 502,
|
|
body,
|
|
raw: result.raw,
|
|
error: result.ok ? undefined : result.stderr || result.stdout || "dev k3s service health failed",
|
|
};
|
|
}
|
|
return coreInternalFetch(`/api/microservices/${encodeURIComponent(service.id)}/health`);
|
|
}
|
|
|
|
export function commitMatches(actual: string | null, desired: string): boolean {
|
|
if (actual === null || actual.length === 0) return false;
|
|
const normalized = actual.toLowerCase();
|
|
return normalized === desired.toLowerCase() || (desired.length < 40 && normalized.startsWith(desired.toLowerCase()));
|
|
}
|
|
|
|
export function runtimeCommitVerified(
|
|
service: UniDeskMicroserviceConfig,
|
|
healthCommit: string | null,
|
|
healthRequestedCommit: string | null,
|
|
imageCommit: string | null,
|
|
orchestratorCommit: string | null,
|
|
desired: string,
|
|
): boolean {
|
|
if (service.deployment.mode === "k3sctl-managed") {
|
|
if (healthRequestedCommit !== null && healthRequestedCommit.length > 0 && !commitMatches(healthRequestedCommit, desired)) return false;
|
|
return commitMatches(orchestratorCommit, desired);
|
|
}
|
|
if (healthCommit !== null && healthCommit.length > 0 && !commitMatches(healthCommit, desired)) return false;
|
|
if (healthRequestedCommit !== null && healthRequestedCommit.length > 0 && !commitMatches(healthRequestedCommit, desired)) return false;
|
|
return commitMatches(imageCommit, desired);
|
|
}
|
|
|
|
export function runtimeCurrentCommit(
|
|
service: UniDeskMicroserviceConfig,
|
|
healthCommit: string | null,
|
|
healthRequestedCommit: string | null,
|
|
imageCommit: string | null,
|
|
orchestratorCommit: string | null,
|
|
): string | null {
|
|
if (service.deployment.mode === "k3sctl-managed") return orchestratorCommit ?? healthCommit ?? healthRequestedCommit ?? imageCommit;
|
|
return healthCommit ?? healthRequestedCommit ?? imageCommit ?? orchestratorCommit;
|
|
}
|
|
|
|
export function coreBody(response: unknown): Record<string, unknown> | null {
|
|
const record = asRecord(response);
|
|
return asRecord(record?.body);
|
|
}
|