// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. health module for scripts/src/deploy.ts. // Moved mechanically from scripts/src/deploy.ts:2219-2525 for #903. import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { runCommand } from "../command"; import { type UniDeskConfig, type UniDeskMicroserviceConfig, repoRoot, rootPath } from "../config"; import { ensureGithubSshIdentityForProvider } from "../deploy-ssh-identity"; import { baiduNetdiskRuntimeSecretRequirements, runtimeSecretContractFromEnvText, type RuntimeSecretContract, runArtifactRegistryCommand } from "../artifact-registry"; import { startJob } from "../jobs"; import { coreInternalFetch } from "../microservices"; import { codeQueueSourceImportPreflight, codeQueueSourceSubdir } from "../code-queue-source-guard"; import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard"; import { composeRuntimeEnvValue } from "../runtime-env"; import { compareDeployJsonExecutorMirrors, deployJsonCommitImage, deployJsonDriftResult, deployJsonSourceOfTruth, encodeDeployJsonServiceContract, hasDeployJsonExecutorContract, k3sManifestExecutorMirror, parseDeployJsonServiceContract, type DeployJsonExecutorMirror, type DeployJsonServiceContract, } from "../deploy-json-contract"; import type { DeployManifestService } from "./types"; import { asRecord, asString, d601K3sGuardScript, parseJsonObjectFromText, shellQuote } from "./options"; import { buildImageTag, k8sManifestPath, targetWorkDir } from "./paths"; import { runTargetCommand } from "./remote"; import { isCoreDeployService, isDevK3sDeployService } from "./service-plan"; import { k8sKubeconfig, k8sNamespace } from "./types"; export function cleanupLegacyDirectCodeQueueScript(service: UniDeskMicroserviceConfig): string { if (service.id !== "code-queue") return ""; if (isDevK3sDeployService(service)) return ""; return [ "set -euo pipefail", "container=code-queue-backend", "if docker ps -a --format '{{.Names}}' | grep -Fx \"$container\" >/dev/null; then", " docker update --restart=no \"$container\" >/dev/null 2>&1 || true", ` compose_file=${shellQuote(`${targetWorkDir(service)}/src/components/microservices/code-queue/docker-compose.d601.yml`)}`, " if [ -f \"$compose_file\" ]; then", " docker compose -p code-queue -f \"$compose_file\" down --remove-orphans", " else", " docker rm -f \"$container\"", " fi", " echo stopped_legacy_direct_code_queue=$container", "else", " echo stopped_legacy_direct_code_queue=not-present", "fi", "if docker ps --format '{{.Names}}' | grep -Fx \"$container\" >/dev/null; then", " echo legacy_direct_code_queue_still_running=$container >&2", " exit 1", "fi", ].join("\n"); } export function k8sNamespaceForService(service: UniDeskMicroserviceConfig): string { return service.deployment.namespace ?? k8sNamespace; } export function k8sDeploymentsForService(service: UniDeskMicroserviceConfig): string[] { if (isDevK3sDeployService(service) && service.id === "code-queue") { return ["d601-dev-provider-egress-proxy", "code-queue-scheduler-dev", "code-queue-read-dev", "code-queue-write-dev"]; } if (service.id === "code-queue") return ["d601-provider-egress-proxy", "d601-tcp-egress-gateway", "code-queue", "code-queue-read", "code-queue-write"]; return [service.repository.composeService]; } export function applyK8sScript(service: UniDeskMicroserviceConfig): string { const manifest = `${targetWorkDir(service)}/${k8sManifestPath(service)}`; const namespace = k8sNamespaceForService(service); const cleanup = service.id === "code-queue" && !isDevK3sDeployService(service) ? [ `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} delete endpointslice d601-provider-egress-proxy --ignore-not-found`, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} delete deployment code-queue-d518 --ignore-not-found`, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} delete service code-queue-d518 --ignore-not-found`, ].join("\n") : ""; return [ "set -euo pipefail", d601K3sGuardScript(), cleanup, `kubectl apply -f ${shellQuote(manifest)}`, ].filter(Boolean).join("\n"); } export function verifyK8sImagesScript(service: UniDeskMicroserviceConfig): string { const namespace = k8sNamespaceForService(service); const image = buildImageTag(service); const deployments = k8sDeploymentsForService(service); return [ "set -euo pipefail", `image=${shellQuote(image)}`, `namespace=${shellQuote(namespace)}`, `deployments=(${deployments.map(shellQuote).join(" ")})`, "for deployment in \"${deployments[@]}\"; do", ` actual=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n "$namespace" get deployment "$deployment" -o jsonpath='{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}')`, " if [ -z \"$actual\" ]; then", " echo \"deployment_image_missing=$deployment\" >&2", " exit 1", " fi", " while IFS= read -r actual_image; do", " [ -z \"$actual_image\" ] && continue", " if [ \"$actual_image\" != \"$image\" ]; then", " printf 'deployment_image_mismatch deployment=%s expected=%s actual=%s\\n' \"$deployment\" \"$image\" \"$actual_image\" >&2", " exit 1", " fi", " done < `deployment/${name}`); const namespace = k8sNamespaceForService(service); const envPairs = [ `UNIDESK_DEPLOY_SERVICE_ID=${service.id}`, `UNIDESK_DEPLOY_REPO=${desired.repo}`, `UNIDESK_DEPLOY_COMMIT=${resolvedCommit}`, `UNIDESK_DEPLOY_REQUESTED_COMMIT=${desired.commitId}`, ...(service.id === "code-queue" ? [ `CODE_QUEUE_DEPLOY_COMMIT=${resolvedCommit}`, `CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=${desired.commitId}`, ] : []), ]; const annotatePairs = [ `unidesk.ai/deploy-service-id=${service.id}`, `unidesk.ai/deploy-repo=${desired.repo}`, `unidesk.ai/deploy-commit=${resolvedCommit}`, `unidesk.ai/deploy-requested-commit=${desired.commitId}`, ]; return [ "set -euo pipefail", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} set env ${deployments.map(shellQuote).join(" ")} ${envPairs.map(shellQuote).join(" ")}`, `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} annotate ${deployments.map(shellQuote).join(" ")} ${annotatePairs.map(shellQuote).join(" ")} --overwrite`, `actual=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} get deployment ${shellQuote(service.repository.composeService)} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}')`, `test "$actual" = ${shellQuote(resolvedCommit)}`, "printf 'k8s_deploy_commit=%s\\n' \"$actual\"", ].join("\n"); } export function rolloutK8sScript(service: UniDeskMicroserviceConfig): string { const deployments = k8sDeploymentsForService(service); const namespace = k8sNamespaceForService(service); return [ "set -euo pipefail", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} rollout restart ${deployments.map((name) => shellQuote(`deployment/${name}`)).join(" ")}`, ...deployments.map((name) => `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} rollout status ${shellQuote(`deployment/${name}`)} --timeout=180s`), `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} get deploy ${deployments.map(shellQuote).join(" ")} -o wide`, ].join("\n"); } export function k8sCommitProbeScript(service: UniDeskMicroserviceConfig): string { const namespace = k8sNamespaceForService(service); return [ "set -euo pipefail", `commit=$(KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n ${shellQuote(namespace)} get deployment ${shellQuote(service.repository.composeService)} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}' 2>/dev/null || true)`, "printf '%s\\n' \"$commit\"", ].join("\n"); } export function dockerCommitProbeScript(service: UniDeskMicroserviceConfig): string { return [ "set -euo pipefail", `container=${shellQuote(service.repository.containerName)}`, "cid=$(docker ps -q -f name=\"^/${container}$\" | head -1)", "if [ -z \"$cid\" ]; then exit 0; fi", "image_id=$(docker inspect -f '{{.Image}}' \"$cid\")", "docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\" 2>/dev/null || true", ].join("\n"); } export function healthDeployCommit(body: Record | null): string | null { const deploy = asRecord(body?.deploy); const commit = asString(deploy?.commit).toLowerCase(); return commit.length > 0 ? commit : null; } export function healthDeployRequestedCommit(body: Record | null): string | null { const deploy = asRecord(body?.deploy); const commit = asString(deploy?.requestedCommit).toLowerCase(); return commit.length > 0 ? commit : null; } export function healthSummary(response: unknown): Record { const record = asRecord(response); const body = asRecord(record?.body); return { ok: record?.ok ?? false, status: record?.status ?? null, body: body === null ? null : { ok: body.ok ?? null, service: body.service ?? null, instanceId: body.instanceId ?? null, deploy: body.deploy ?? null, status: body.status ?? null, startedAt: body.startedAt ?? null, }, }; } export async function directHttpJson(url: string, timeoutMs: number): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { signal: controller.signal }); const text = await response.text(); let body: unknown = null; try { body = text.length > 0 ? JSON.parse(text) : null; } catch { body = { text }; } return { ok: response.ok, status: response.status, body }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; } finally { clearTimeout(timer); } } export function devK3sServiceHealthScript(service: UniDeskMicroserviceConfig): string { const namespace = k8sNamespaceForService(service); const serviceName = service.repository.composeService; const path = service.backend.healthPath; const port = service.backend.nodePort; return [ "set -euo pipefail", `namespace=${shellQuote(namespace)}`, `service_name=${shellQuote(serviceName)}`, `path=${shellQuote(path)}`, `port=${shellQuote(String(port))}`, "case \"$path\" in /*) ;; *) path=\"/$path\" ;; esac", "proxy_path=\"/api/v1/namespaces/$namespace/services/http:$service_name:$port/proxy$path\"", "tmp_health=$(mktemp)", "trap 'rm -f \"$tmp_health\"' EXIT", `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get --raw "$proxy_path" > "$tmp_health"`, "python3 - \"$tmp_health\" <<'PY'", "import json", "import sys", "", "text = open(sys.argv[1], encoding='utf-8', errors='replace').read()", "try:", " body = json.loads(text)", "except Exception:", " print(text[:4000])", " raise SystemExit(0)", "", "def compact(mapping, keys):", " if not isinstance(mapping, dict):", " return None", " return {key: mapping.get(key) for key in keys if key in mapping}", "", "summary = compact(body, ['ok', 'service', 'instanceId', 'role', 'environment', 'namespace', 'databaseName', 'serviceId', 'status', 'startedAt', 'deployRef']) or {}", "deploy = compact(body.get('deploy'), ['repo', 'commit', 'requestedCommit', 'serviceId'])", "if deploy is not None:", " summary['deploy'] = deploy", "queue = body.get('queue')", "if isinstance(queue, dict):", " queue_summary = compact(queue, ['total', 'queueCount', 'defaultQueueId', 'processing']) or {}", " storage = compact(queue.get('storage'), ['primary', 'postgresReady'])", " if storage is not None:", " queue_summary['storage'] = storage", " summary['queue'] = queue_summary", "print(json.dumps(summary, ensure_ascii=False, separators=(',', ':')))", "PY", ].join("\n"); } export async function serviceHealth(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise { if (isCoreDeployService(service)) { return await directHttpJson(`${service.backend.nodeBaseUrl}${service.backend.healthPath}`, service.backend.timeoutMs); } if (isDevK3sDeployService(service)) { const result = await runTargetCommand(config, service, devK3sServiceHealthScript(service), "/home/ubuntu", 30_000, 20_000); const stdout = result.stdout.trim(); let body: unknown = null; try { body = stdout.length > 0 ? parseJsonObjectFromText(stdout) : null; } catch { body = null; } if (body === null && stdout.length > 0) body = { text: stdout }; return { ok: result.ok, status: result.ok ? 200 : 502, body, raw: result.raw, error: result.ok ? undefined : result.stderr || result.stdout || "dev k3s service health failed", }; } return coreInternalFetch(`/api/microservices/${encodeURIComponent(service.id)}/health`); } export function commitMatches(actual: string | null, desired: string): boolean { if (actual === null || actual.length === 0) return false; const normalized = actual.toLowerCase(); return normalized === desired.toLowerCase() || (desired.length < 40 && normalized.startsWith(desired.toLowerCase())); } export function runtimeCommitVerified( service: UniDeskMicroserviceConfig, healthCommit: string | null, healthRequestedCommit: string | null, imageCommit: string | null, orchestratorCommit: string | null, desired: string, ): boolean { if (service.deployment.mode === "k3sctl-managed") { if (healthRequestedCommit !== null && healthRequestedCommit.length > 0 && !commitMatches(healthRequestedCommit, desired)) return false; return commitMatches(orchestratorCommit, desired); } if (healthCommit !== null && healthCommit.length > 0 && !commitMatches(healthCommit, desired)) return false; if (healthRequestedCommit !== null && healthRequestedCommit.length > 0 && !commitMatches(healthRequestedCommit, desired)) return false; return commitMatches(imageCommit, desired); } export function runtimeCurrentCommit( service: UniDeskMicroserviceConfig, healthCommit: string | null, healthRequestedCommit: string | null, imageCommit: string | null, orchestratorCommit: string | null, ): string | null { if (service.deployment.mode === "k3sctl-managed") return orchestratorCommit ?? healthCommit ?? healthRequestedCommit ?? imageCommit; return healthCommit ?? healthRequestedCommit ?? imageCommit ?? orchestratorCommit; } export function coreBody(response: unknown): Record | null { const record = asRecord(response); return asRecord(record?.body); }