Files
pikasTech-unidesk/scripts/src/artifact-registry/k3s-deploy.ts
T
2026-06-25 16:16:25 +00:00

326 lines
20 KiB
TypeScript

// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. k3s-deploy module for scripts/src/artifact-registry.ts.
// Moved mechanically from scripts/src/artifact-registry.ts:3154-3447 for #903.
import { createHash } from "node:crypto";
import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { runCommand, type CommandResult } from "../command";
import { readConfig, type UniDeskConfig, repoRoot, rootPath } from "../config";
import { startJob } from "../jobs";
import {
compareDeployJsonExecutorMirrors,
deployJsonCommitImage,
deployJsonDriftResult,
deployJsonSourceOfTruth,
hasDeployJsonExecutorContract,
k3sManifestExecutorMirror,
parseDeployJsonServiceContractBase64,
readDeployJsonServiceContractFromFile,
type DeployJsonExecutorMirror,
type DeployJsonServiceContract,
} from "../deploy-json-contract";
import { d601K3sGuardShellLines } from "../d601-k3s-guard";
import { composeRuntimeEnvValue } from "../runtime-env";
import type { ArtifactConsumerSpec, ArtifactConsumerTarget, ArtifactDeployEnvironment, ArtifactRegistryOptions } from "./types";
import { d601DevFrontendAuthPatchScript, registryArtifactMissingMessage, registryArtifactProbeScript } from "./artifact-probe";
import { d601K3sGuardScript, rootExecPrelude, shellQuote } from "./bundle";
import { artifactImageRef, commandTail, deployRefFor, sourceRepoFor } from "./consumer";
import { runReadonlyStatus } from "./readonly";
import { runRemoteScript, runRemoteScriptBackground } from "./remote";
export function rollbackInfo(spec: ArtifactConsumerSpec, target: ArtifactConsumerTarget, environment: ArtifactDeployEnvironment, commit: string): Record<string, unknown> {
if (spec.kind === "compose" || spec.kind === "d601-compose") {
const compose = target.compose;
return {
type: spec.kind === "d601-compose" ? "d601-compose-retag-recreate" : "compose-retag-recreate",
composeService: compose?.serviceName,
containerName: compose?.containerName,
workDir: compose?.workDir,
composeFile: compose?.composeFile,
previousImageHint: `Use docker image ls / docker inspect to find the previous labeled ${spec.serviceId} image id; Compose volumes are unchanged.`,
commandShape: `bun scripts/cli.ts deploy apply --env ${environment} --service ${spec.serviceId}${environment === "prod" ? " --commit <previous-full-sha>" : ""}`,
};
}
return {
type: "d601-k3s-previous-commit",
serviceId: spec.serviceId,
environment,
currentCommit: commit,
discovery: `KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n ${target.k3s?.namespace} rollout history deployment/${target.k3s?.deploymentName} && KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n ${target.k3s?.namespace} get deployment ${target.k3s?.deploymentName} -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-previous-commit}'`,
commandShape: `bun scripts/cli.ts deploy apply --env ${environment} --service ${spec.serviceId} --commit <previous-full-sha>`,
note: "Rollback is exposed as the same artifact consumer pointed at a previous commit-pinned image that still exists in D601 registry.",
};
}
export function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec, target: ArtifactConsumerTarget, commit: string): string {
const environment = options.environment ?? "prod";
if (target.k3s === undefined) throw new Error(`${spec.serviceId} missing k3s artifact consumer config for ${environment}`);
const manifestText = readFileSync(rootPath(target.k3s.manifestRepoPath), "utf8");
const manifestBase64 = Buffer.from(manifestText, "utf8").toString("base64");
const sourceImage = artifactImageRef(options, spec, commit);
const sourceRepo = sourceRepoFor(options, spec);
const commitImage = target.targetCommitImage(commit);
const k3s = target.k3s;
const deploymentSpecs = [
{ name: k3s.deploymentName, containerName: k3s.containerName },
...(k3s.extraDeployments ?? []).map((deployment) => typeof deployment === "string"
? { name: deployment, containerName: k3s.containerName }
: { name: deployment.name, containerName: deployment.containerName ?? k3s.containerName }),
];
const deploymentSpecsBase64 = Buffer.from(JSON.stringify(deploymentSpecs), "utf8").toString("base64");
const labels = [
`unidesk.ai/deploy-service-id=${spec.serviceId}`,
`unidesk.ai/deploy-ref=${deployRefFor(options, spec)}`,
`unidesk.ai/deploy-repo=${sourceRepo}`,
`unidesk.ai/deploy-commit=${commit}`,
`unidesk.ai/deploy-requested-commit=${commit}`,
`unidesk.ai/image-source=${sourceImage}`,
`unidesk.ai/deploy-environment=${environment}`,
];
return [
"set -euo pipefail",
rootExecPrelude(),
`registry_image=${shellQuote(sourceImage)}`,
`stable_image=${shellQuote(target.targetImage)}`,
`commit_image=${shellQuote(commitImage)}`,
`environment=${shellQuote(environment)}`,
`service_id=${shellQuote(spec.serviceId)}`,
`source_repo=${shellQuote(sourceRepo)}`,
`deploy_ref=${shellQuote(deployRefFor(options, spec))}`,
`commit=${shellQuote(commit)}`,
`dockerfile=${shellQuote(spec.dockerfile)}`,
`namespace=${shellQuote(k3s.namespace)}`,
`deployment=${shellQuote(k3s.deploymentName)}`,
`container_name=${shellQuote(k3s.containerName)}`,
`deployment_specs_b64=${shellQuote(deploymentSpecsBase64)}`,
`service_name=${shellQuote(k3s.serviceName)}`,
`service_port=${shellQuote(String(k3s.servicePort))}`,
`health_path=${shellQuote(k3s.healthPath)}`,
`apply_selector=${shellQuote(k3s.applySelector ?? "")}`,
`pod_selector=${shellQuote(k3s.podLabelSelector ?? `app.kubernetes.io/name=${k3s.deploymentName}`)}`,
"archive=",
"manifest=",
"deployment_specs=",
"health_tmp=",
`manifest_repo_path=${shellQuote(k3s.manifestRepoPath)}`,
`manifest_b64=${shellQuote(manifestBase64)}`,
"export DOCKER_CONFIG=$(mktemp -d /tmp/unidesk-artifact-docker-config.XXXXXX)",
"printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"",
"cleanup_artifact_cd() {",
" [ -n \"${archive:-}\" ] && rm -f \"$archive\"",
" [ -n \"${manifest:-}\" ] && rm -f \"$manifest\"",
" [ -n \"${deployment_specs:-}\" ] && rm -f \"$deployment_specs\"",
" [ -n \"${health_tmp:-}\" ] && rm -f \"$health_tmp\"",
" [ -n \"${DOCKER_CONFIG:-}\" ] && rm -rf \"$DOCKER_CONFIG\"",
"}",
"trap cleanup_artifact_cd EXIT",
"command -v docker >/dev/null",
"command -v kubectl >/dev/null",
"command -v ctr >/dev/null",
"test -S /run/k3s/containerd/containerd.sock",
d601K3sGuardScript(),
`curl -fsSI -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' ${shellQuote(`http://127.0.0.1:${options.port}/v2/${spec.registryRepository}/manifests/${commit}`)} >/dev/null`,
"docker pull -q \"$registry_image\" >/dev/null",
"label_commit=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}')",
"label_service=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/service-id\" }}')",
"label_repo=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/source-repo\" }}')",
"label_dockerfile=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/dockerfile\" }}')",
"test \"$label_commit\" = \"$commit\"",
"test \"$label_service\" = \"$service_id\"",
"test \"$label_repo\" = \"$source_repo\"",
"test \"$label_dockerfile\" = \"$dockerfile\"",
"docker tag \"$registry_image\" \"$stable_image\"",
"docker tag \"$registry_image\" \"$commit_image\"",
"archive=$(mktemp /tmp/unidesk-artifact-k3s-image.XXXXXX.tar)",
"manifest=$(mktemp /tmp/unidesk-artifact-k3s-manifest.XXXXXX.yaml)",
"docker save \"$commit_image\" \"$stable_image\" -o \"$archive\"",
"root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import \"$archive\" >/dev/null",
"containerd_image_ref() {",
" ref=\"$1\"",
" if root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images inspect \"$ref\" >/dev/null 2>&1; then printf '%s\\n' \"$ref\"; return; fi",
" suffix=\"${ref#*/}\"",
" actual_ref=$(root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images ls -q | grep -F \"/$suffix\" | head -1 || true)",
" test -n \"$actual_ref\"",
" printf '%s\\n' \"$actual_ref\"",
"}",
"stable_containerd_ref=$(containerd_image_ref \"$stable_image\")",
"commit_containerd_ref=$(containerd_image_ref \"$commit_image\")",
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest\"",
"deployment_specs=$(mktemp /tmp/unidesk-artifact-k3s-deployments.XXXXXX.json)",
"printf '%s' \"$deployment_specs_b64\" | base64 -d > \"$deployment_specs\"",
"python3 - \"$deployment_specs\" \"$manifest\" <<'PY'",
"import json, sys",
"specs = json.load(open(sys.argv[1], encoding='utf-8'))",
"manifest = open(sys.argv[2], encoding='utf-8').read()",
"missing = [item['name'] for item in specs if f\"name: {item['name']}\" not in manifest]",
"if missing:",
" raise SystemExit('manifest missing deployment(s): ' + ','.join(missing))",
"PY",
"if [ -n \"$apply_selector\" ]; then kubectl apply -f \"$manifest\" -l \"$apply_selector\"; else kubectl apply -f \"$manifest\"; fi",
...(environment === "dev" && spec.serviceId === "frontend" ? [d601DevFrontendAuthPatchScript(readConfig())] : []),
"previous_commit=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}' 2>/dev/null || true)",
"python3 - \"$deployment_specs\" <<'PY' | while IFS=$'\\t' read -r rollout_deployment rollout_container; do",
"import json, sys",
"for item in json.load(open(sys.argv[1], encoding='utf-8')):",
" print(f\"{item['name']}\\t{item['containerName']}\")",
"PY",
" kubectl -n \"$namespace\" set image \"deployment/$rollout_deployment\" \"$rollout_container=$commit_image\"",
" kubectl -n \"$namespace\" set env \"deployment/$rollout_deployment\" \"UNIDESK_DEPLOY_SERVICE_ID=$service_id\" \"UNIDESK_DEPLOY_REF=$deploy_ref\" \"UNIDESK_DEPLOY_REPO=$source_repo\" \"UNIDESK_DEPLOY_COMMIT=$commit\" \"UNIDESK_DEPLOY_REQUESTED_COMMIT=$commit\" \"CODE_QUEUE_DEPLOY_COMMIT=$commit\" \"CODE_QUEUE_DEPLOY_REQUESTED_COMMIT=$commit\"",
` kubectl -n "$namespace" annotate "deployment/$rollout_deployment" ${labels.map(shellQuote).join(" ")} --overwrite`,
" if [ -n \"$previous_commit\" ] && [ \"$previous_commit\" != \"$commit\" ]; then kubectl -n \"$namespace\" annotate \"deployment/$rollout_deployment\" \"unidesk.ai/deploy-previous-commit=$previous_commit\" --overwrite; fi",
"done",
"python3 - \"$deployment_specs\" <<'PY' | while IFS= read -r rollout_deployment; do",
"import json, sys",
"for item in json.load(open(sys.argv[1], encoding='utf-8')):",
" print(item['name'])",
"PY",
" kubectl -n \"$namespace\" rollout status \"deployment/$rollout_deployment\" --timeout=180s",
"done",
"deployment_commit=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}')",
"deployment_requested_commit=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-requested-commit}')",
"test \"$deployment_commit\" = \"$commit\"",
"test \"$deployment_requested_commit\" = \"$commit\"",
"deployment_image=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o jsonpath='{.spec.template.spec.containers[?(@.name==\"'\"$container_name\"'\")].image}')",
"test \"$deployment_image\" = \"$commit_image\"",
"containerd_config_label() {",
" ref=\"$1\"",
" key=\"$2\"",
" actual_ref=$(containerd_image_ref \"$ref\")",
" target_digest=$(root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images inspect \"$actual_ref\" | python3 -c 'import re,sys; m=re.search(r\"@(sha256:[0-9a-f]+)\", sys.stdin.read()); print(m.group(1) if m else \"\")')",
" test -n \"$target_digest\"",
" digest=\"$target_digest\"",
" config_digest=",
" while [ -n \"$digest\" ]; do",
" content_file=$(mktemp /tmp/unidesk-artifact-content.XXXXXX.json)",
" root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io content get \"$digest\" > \"$content_file\"",
" next=$(python3 - \"$content_file\" <<'PY'",
"import json, sys",
"obj = json.load(open(sys.argv[1], encoding='utf-8'))",
"manifests = obj.get('manifests')",
"if isinstance(manifests, list):",
" for item in manifests:",
" platform = item.get('platform') or {}",
" if platform.get('architecture') in ('amd64', '') or not platform:",
" print('manifest:' + str(item.get('digest') or ''))",
" raise SystemExit(0)",
" print('')",
" raise SystemExit(0)",
"config = obj.get('config') or {}",
"print('config:' + str(config.get('digest') or ''))",
"PY",
" )",
" rm -f \"$content_file\"",
" case \"$next\" in",
" manifest:*) digest=\"${next#manifest:}\" ;;",
" config:*) config_digest=\"${next#config:}\"; break ;;",
" *) digest= ;;",
" esac",
" done",
" test -n \"$config_digest\"",
" config_file=$(mktemp /tmp/unidesk-artifact-config.XXXXXX.json)",
" root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io content get \"$config_digest\" > \"$config_file\"",
" python3 - \"$config_file\" \"$key\" <<'PY'",
"import json, sys",
"cfg = json.load(open(sys.argv[1], encoding='utf-8'))",
"key = sys.argv[2]",
"print(str(((cfg.get('config') or {}).get('Labels') or {}).get(key) or ''))",
"PY",
" rm -f \"$config_file\"",
"}",
"image_label_commit=$(containerd_config_label \"$commit_image\" 'unidesk.ai/source-commit' 2>/tmp/unidesk-artifact-containerd-label.err || true)",
"image_label_service=$(containerd_config_label \"$commit_image\" 'unidesk.ai/service-id' 2>>/tmp/unidesk-artifact-containerd-label.err || true)",
"if [ \"$image_label_commit\" != \"$commit\" ] || [ \"$image_label_service\" != \"$service_id\" ]; then",
" echo \"artifact_cd_containerd_label_fallback=docker-label imageLabelCommit=${image_label_commit:-missing} imageLabelService=${image_label_service:-missing}\"",
" image_label_commit=\"$label_commit\"",
" image_label_service=\"$label_service\"",
"fi",
"test \"$image_label_commit\" = \"$commit\"",
"test \"$image_label_service\" = \"$service_id\"",
"pod=$(kubectl -n \"$namespace\" get pod -l \"$pod_selector\" -o jsonpath='{.items[0].metadata.name}')",
"test -n \"$pod\"",
"pod_image=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.containers[?(@.name==\"'\"$container_name\"'\")].image}')",
"test \"$pod_image\" = \"$commit_image\"",
"pod_image_id=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.containerStatuses[?(@.name==\"'\"$container_name\"'\")].imageID}')",
"if [ -z \"$pod_image_id\" ]; then pod_image_id=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.containerStatuses[0].imageID}'); fi",
"case \"$health_path\" in /*) ;; *) health_path=\"/$health_path\" ;; esac",
"proxy_path=\"/api/v1/namespaces/$namespace/services/http:$service_name:$service_port/proxy$health_path\"",
"health_tmp=$(mktemp /tmp/unidesk-artifact-health.XXXXXX.json)",
"for attempt in $(seq 1 60); do",
" if kubectl get --raw \"$proxy_path\" > \"$health_tmp\" 2>/tmp/unidesk-artifact-health.err; then",
" health_commit=$(python3 -c 'import json,sys; print(((json.load(open(sys.argv[1])).get(\"deploy\") or {}).get(\"commit\") or \"\"))' \"$health_tmp\" 2>/dev/null || true)",
" health_requested_commit=$(python3 -c 'import json,sys; print(((json.load(open(sys.argv[1])).get(\"deploy\") or {}).get(\"requestedCommit\") or \"\"))' \"$health_tmp\" 2>/dev/null || true)",
" health_ok=$(python3 -c 'import json,sys; print(\"true\" if json.load(open(sys.argv[1])).get(\"ok\") is True else \"false\")' \"$health_tmp\" 2>/dev/null || true)",
" echo \"artifact_cd_health_probe attempt=$attempt ok=$health_ok commit=$health_commit requestedCommit=$health_requested_commit\"",
" if [ \"$health_ok\" = \"true\" ] && [ \"$health_commit\" = \"$commit\" ] && [ \"$health_requested_commit\" = \"$commit\" ]; then break; fi",
" else",
" echo \"artifact_cd_health_probe attempt=$attempt request=failed\"",
" fi",
" if [ \"$attempt\" = \"60\" ]; then echo artifact_cd_health_failed >&2; cat \"$health_tmp\" >&2 || true; exit 1; fi",
" sleep 2",
"done",
"printf 'artifact_cd_service=%s\\nartifact_cd_environment=%s\\nartifact_cd_manifest_repo_path=%s\\nartifact_cd_source_image=%s\\nartifact_cd_stable_image=%s\\nartifact_cd_runtime_image=%s\\nartifact_cd_commit=%s\\nartifact_cd_requested_commit=%s\\nartifact_cd_previous_commit=%s\\nartifact_cd_pod=%s\\nartifact_cd_pod_image=%s\\nartifact_cd_pod_image_id=%s\\nartifact_cd_image_label_commit=%s\\nartifact_cd_health_commit=%s\\nartifact_cd_health_requested_commit=%s\\n' \"$service_id\" \"$environment\" \"$manifest_repo_path\" \"$registry_image\" \"$stable_image\" \"$commit_image\" \"$commit\" \"$deployment_requested_commit\" \"$previous_commit\" \"$pod\" \"$pod_image\" \"$pod_image_id\" \"$image_label_commit\" \"$health_commit\" \"$health_requested_commit\"",
].join("\n");
}
export async function deployD601K3sArtifactNow(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec, target: ArtifactConsumerTarget): Promise<Record<string, unknown>> {
const environment = options.environment ?? "prod";
const commit = options.commit;
if (commit === null) throw new Error("artifact-registry deploy-service requires --commit <full-sha>");
const health = runReadonlyStatus(options, true);
if (health.ok !== true) return { ok: false, serviceId: spec.serviceId, error: "D601 artifact registry is not healthy", health };
const sourceImage = artifactImageRef(options, spec, commit);
const registryProbe = runRemoteScript(options, registryArtifactProbeScript(options, spec, commit), Math.max(options.timeoutMs, 120_000));
if (registryProbe.exitCode !== 0 || registryProbe.timedOut) {
return {
ok: false,
supported: true,
serviceId: spec.serviceId,
step: "registry-artifact-check",
error: registryArtifactMissingMessage(spec),
sourceImage,
registryProbe: commandTail(registryProbe),
};
}
const deployScript = d601K3sArtifactDeployScript(options, spec, target, commit);
const deploy = await runRemoteScriptBackground(options, deployScript, Math.max(options.timeoutMs, 420_000), "d601-k3s-deploy");
if (deploy.exitCode !== 0 || deploy.timedOut) {
return {
ok: false,
supported: true,
serviceId: spec.serviceId,
step: "d601-k3s-artifact-deploy",
sourceImage,
registryProbe: commandTail(registryProbe),
deploy: commandTail(deploy),
rollback: rollbackInfo(spec, target, environment, commit),
};
}
return {
ok: true,
supported: true,
serviceId: spec.serviceId,
environment,
commit,
providerId: options.providerId,
sourceRepo: sourceRepoFor(options, spec),
deployRef: deployRefFor(options, spec),
sourceImage,
stableImage: target.targetImage,
runtimeImage: target.targetCommitImage(commit),
manifestRepoPath: target.k3s?.manifestRepoPath,
registryProbe: commandTail(registryProbe),
deploy: commandTail(deploy),
validation: {
liveCommit: commit,
liveRequestedCommit: commit,
imageLabelCommit: commit,
serviceHealthCommit: commit,
serviceHealthRequestedCommit: commit,
healthyOldVersionAccepted: false,
},
rollback: rollbackInfo(spec, target, environment, commit),
};
}