feat: add baidu netdisk artifact delivery

This commit is contained in:
Codex
2026-05-19 15:22:56 +00:00
parent 11d7b217d9
commit c8561d392b
17 changed files with 302 additions and 103 deletions
+137 -42
View File
@@ -22,9 +22,10 @@ interface ArtifactRegistryOptions {
dryRun: boolean;
runNow: boolean;
commit: string | null;
targetImage: string;
targetImage: string | null;
serviceId: string | null;
sourceRepo: string;
deployRef: string | null;
}
interface RenderedFile {
@@ -60,11 +61,12 @@ const defaultOptions: ArtifactRegistryOptions = {
dryRun: false,
runNow: false,
commit: null,
targetImage: "unidesk-backend-core",
targetImage: null,
serviceId: null,
sourceRepo: "https://github.com/pikasTech/unidesk",
deployRef: null,
};
const supportedArtifactConsumerServices = ["backend-core", "decision-center"] as const;
const supportedArtifactConsumerServices = ["backend-core", "baidu-netdisk", "decision-center"] as const;
type SupportedArtifactConsumerService = typeof supportedArtifactConsumerServices[number];
interface ArtifactConsumerSpec {
@@ -75,6 +77,13 @@ interface ArtifactConsumerSpec {
targetImage: string;
targetCommitImage: (commit: string) => string;
deployRef: string;
compose?: {
serviceName: string;
containerName: string;
deployEnvPrefix: string;
healthProbeCommand: string;
requireHealthCommit: boolean;
};
k3s?: {
namespace: string;
manifestPath: string;
@@ -95,6 +104,29 @@ const artifactConsumerSpecs: Record<SupportedArtifactConsumerService, ArtifactCo
targetImage: "unidesk-backend-core",
targetCommitImage: (commit: string) => `unidesk-backend-core:${commit}`,
deployRef: "deploy.json#environments.prod.services.backend-core",
compose: {
serviceName: "backend-core",
containerName: "unidesk-backend-core",
deployEnvPrefix: "UNIDESK_DEPLOY",
healthProbeCommand: "if command -v backend-core >/dev/null 2>&1; then backend-core --fetch-json http://127.0.0.1:8080/health --require-ok; elif command -v bun >/dev/null 2>&1; then bun -e \"fetch('http://127.0.0.1:8080/health').then(async r=>{const text=await r.text(); console.log(text); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\"; else exit 1; fi",
requireHealthCommit: false,
},
},
"baidu-netdisk": {
serviceId: "baidu-netdisk",
kind: "compose",
registryRepository: "unidesk/baidu-netdisk",
dockerfile: "src/components/microservices/baidu-netdisk/Dockerfile",
targetImage: "baidu-netdisk",
targetCommitImage: (commit: string) => `baidu-netdisk:${commit}`,
deployRef: "deploy.json#environments.prod.services.baidu-netdisk",
compose: {
serviceName: "baidu-netdisk",
containerName: "baidu-netdisk-backend",
deployEnvPrefix: "UNIDESK_BAIDU_NETDISK_DEPLOY",
healthProbeCommand: "bun -e \"fetch('http://127.0.0.1:4244/health').then(async r=>{const text=await r.text(); console.log(text); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\"",
requireHealthCommit: true,
},
},
"decision-center": {
serviceId: "decision-center",
@@ -185,6 +217,9 @@ function parseOptions(args: string[]): ArtifactRegistryOptions {
} else if (arg === "--source-repo") {
options.sourceRepo = requireValue(args, index, arg);
index += 1;
} else if (arg === "--deploy-ref") {
options.deployRef = requireValue(args, index, arg);
index += 1;
} else {
throw new Error(`unknown artifact-registry option: ${arg}`);
}
@@ -202,6 +237,10 @@ function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
function safeName(value: string): string {
return value.replace(/[^A-Za-z0-9_.-]/gu, "-");
}
function rootExecPrelude(): string {
return [
"root_exec() {",
@@ -328,16 +367,17 @@ function plan(options: ArtifactRegistryOptions): Record<string, unknown> {
"listen only on D601 host loopback 127.0.0.1:5000",
"do not expose a public port, NodePort, hostPort, or third-party registry",
"do not run inside k3s; keep the registry outside the native k3s failure domain",
"CI builds and publishes backend-core artifacts on D601",
"CD only pulls, retags, recreates backend-core, and verifies live commit",
"master server CD must not compile Rust or run docker compose build backend-core",
"CI builds and publishes backend-core and reviewed user-service artifacts on D601",
"CD only pulls, retags, recreates/imports artifacts, and verifies live commit",
"master server CD must not compile Rust or run docker compose build for artifact consumers",
],
renderedPaths: bundle.paths,
backendCoreArtifactFlow: [
"D601 CI builds unidesk/backend-core:<commit>",
"D601 CI pushes 127.0.0.1:5000/unidesk/backend-core:<commit>",
"main server pulls via a controlled short-lived localhost relay",
"prod CD retags, recreates backend-core, and verifies live commit metadata",
artifactConsumerFlow: [
"D601 CI builds unidesk/<service-id>:<commit>",
"D601 CI pushes 127.0.0.1:5000/unidesk/<service-id>:<commit>",
"Compose consumers pull via a controlled provider-gateway SSH image stream",
"k3s consumers pull on D601 and import into native k3s containerd",
"CD retags/recreates or updates Deployment images and verifies live commit metadata",
],
};
}
@@ -471,6 +511,10 @@ function artifactImageRef(options: ArtifactRegistryOptions, spec: ArtifactConsum
return `127.0.0.1:${options.port}/${spec.registryRepository}:${commit}`;
}
function deployRefFor(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec): string {
return options.deployRef ?? spec.deployRef;
}
function runRemoteScript(options: ArtifactRegistryOptions, script: string, timeoutMs = options.timeoutMs): CommandResult {
const command = [process.execPath, "scripts/cli.ts", "ssh", options.providerId, "argv", "bash", "-lc", script];
return runCommand(command, repoRoot, { timeoutMs });
@@ -742,21 +786,34 @@ function verifyLocalArtifactLabels(
};
}
async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise<Record<string, unknown>> {
function composeArtifactEnvValues(spec: ArtifactConsumerSpec, options: ArtifactRegistryOptions, commit: string): Record<string, string> {
if (spec.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`);
const prefix = spec.compose.deployEnvPrefix;
return {
[`${prefix}_SERVICE_ID`]: spec.serviceId,
[`${prefix}_REF`]: deployRefFor(options, spec),
[`${prefix}_REPO`]: options.sourceRepo,
[`${prefix}_COMMIT`]: commit,
[`${prefix}_REQUESTED_COMMIT`]: commit,
};
}
async function deployComposeArtifactNow(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec): Promise<Record<string, unknown>> {
const commit = options.commit;
if (commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit <full-sha>");
const spec = artifactConsumerSpecs["backend-core"];
if (commit === null) throw new Error("artifact-registry deploy-service requires --commit <full-sha>");
if (spec.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`);
const health = runReadonlyStatus(options, true);
if (health.ok !== true) {
return { ok: false, error: "D601 artifact registry is not healthy", health };
return { ok: false, serviceId: spec.serviceId, error: "D601 artifact registry is not healthy", health };
}
const sourceImage = artifactImageRef(options, spec, commit);
const composeImage = options.targetImage === defaultOptions.targetImage ? spec.targetImage : options.targetImage;
const commitImage = `${composeImage}:${commit}`;
const composeImage = options.targetImage ?? spec.targetImage;
const commitImage = options.targetImage === null ? spec.targetCommitImage(commit) : `${options.targetImage}:${commit}`;
const registryProbe = runRemoteScript(options, registryArtifactProbeScript(options, spec, commit), Math.max(options.timeoutMs, 120_000));
if (registryProbe.exitCode !== 0 || registryProbe.timedOut) {
return {
ok: false,
serviceId: spec.serviceId,
step: "registry-artifact-check",
error: registryArtifactMissingMessage(spec),
sourceImage,
@@ -767,6 +824,7 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise<R
if (pull.exitCode !== 0 || pull.timedOut) {
return {
ok: false,
serviceId: spec.serviceId,
step: "docker-load",
sourceImage,
registryProbe: commandTail(registryProbe),
@@ -787,41 +845,46 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise<R
const config = readConfig();
const runtimeEnv = writeComposeEnv(config, false);
upsertEnvFileValues(runtimeEnv.envFile, {
UNIDESK_DEPLOY_REF: spec.deployRef,
UNIDESK_DEPLOY_SERVICE_ID: "backend-core",
UNIDESK_DEPLOY_REPO: options.sourceRepo,
UNIDESK_DEPLOY_COMMIT: commit,
UNIDESK_DEPLOY_REQUESTED_COMMIT: commit,
...composeArtifactEnvValues(spec, options, commit),
});
const compose = resolveComposeCommand(config, runtimeEnv.envFile);
const serviceName = spec.compose.serviceName;
const containerName = spec.compose.containerName;
const serviceLogPrefix = spec.serviceId.replace(/-/gu, "_");
const upScript = [
"set -euo pipefail",
`echo ${shellQuote(`backend_core_artifact_cd source=${sourceImage} target=${composeImage}`)}`,
`${compose.map(shellQuote).join(" ")} up -d --no-build --no-deps --force-recreate backend-core`,
`echo ${shellQuote(`${serviceLogPrefix}_artifact_cd source=${sourceImage} target=${composeImage}`)}`,
`${compose.map(shellQuote).join(" ")} up -d --no-build --no-deps --force-recreate ${shellQuote(serviceName)}`,
"ready=0",
"for attempt in $(seq 1 60); do",
" cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=backend-core --filter label=com.docker.compose.oneoff=False | head -1 || true)",
` cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=${shellQuote(serviceName)} --filter label=com.docker.compose.oneoff=False | head -1 || true)`,
" if [ -n \"$cid\" ]; then",
" health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' \"$cid\" 2>/dev/null || true)",
" echo \"backend_core_container_probe attempt=$attempt cid=$cid health=$health\"",
` echo "${serviceLogPrefix}_container_probe attempt=$attempt cid=$cid health=$health"`,
" if [ \"$health\" = \"healthy\" ] || [ \"$health\" = \"running\" ]; then ready=1; break; fi",
" else",
" echo \"backend_core_container_probe attempt=$attempt cid=missing\"",
` echo "${serviceLogPrefix}_container_probe attempt=$attempt cid=missing"`,
" fi",
" sleep 1",
"done",
"test \"$ready\" = \"1\"",
"cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=backend-core --filter label=com.docker.compose.oneoff=False | head -1)",
`cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=${shellQuote(serviceName)} --filter label=com.docker.compose.oneoff=False | head -1)`,
"image_id=$(docker inspect -f '{{.Image}}' \"$cid\")",
"actual_commit=$(docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\")",
`test "$actual_commit" = ${shellQuote(commit)}`,
"docker exec \"$cid\" bun -e \"fetch('http://127.0.0.1:8080/health').then(async r=>{const text=await r.text(); console.log(text); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\" >/tmp/unidesk-backend-core-health.json",
"cat /tmp/unidesk-backend-core-health.json",
`health_json=/tmp/unidesk-${safeName(spec.serviceId)}-health.json`,
`docker exec "$cid" sh -lc ${shellQuote(spec.compose.healthProbeCommand)} > "$health_json"`,
"cat \"$health_json\"",
...(spec.compose.requireHealthCommit ? [
"health_commit=$(python3 -c 'import json,sys; print(((json.load(open(sys.argv[1])).get(\"deploy\") or {}).get(\"commit\") or \"\"))' \"$health_json\")",
`test "$health_commit" = ${shellQuote(commit)}`,
] : []),
].join("\n");
const deploy = runCommand(["bash", "-lc", composeLockScript(upScript)], repoRoot, { timeoutMs: Math.max(options.timeoutMs, 300_000) });
if (deploy.exitCode !== 0 || deploy.timedOut) {
return {
ok: false,
serviceId: spec.serviceId,
step: "compose-recreate",
sourceImage,
targetImage: composeImage,
@@ -829,9 +892,11 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise<R
deploy: commandTail(deploy),
};
}
const running = runCommand(["docker", "inspect", "unidesk-backend-core", "--format", "image={{.Config.Image}} imageId={{.Image}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}"], repoRoot);
const running = runCommand(["docker", "inspect", containerName, "--format", "image={{.Config.Image}} imageId={{.Image}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}"], repoRoot);
return {
ok: true,
supported: true,
serviceId: spec.serviceId,
commit,
providerId: options.providerId,
sourceImage,
@@ -842,13 +907,24 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise<R
pull: commandTail(pull),
deploy: commandTail(deploy),
running: running.stdout.trim(),
validation: {
liveCommit: commit,
imageLabelCommit: commit,
serviceHealthCommit: spec.compose.requireHealthCommit ? commit : "not-required",
healthyOldVersionAccepted: false,
},
rollback: {
previousImageHint: "Use docker image ls and docker inspect logs for the previous backend-core image id; Compose named volumes were not changed.",
commandShape: `${compose.map(shellQuote).join(" ")} up -d --no-build --no-deps --force-recreate backend-core`,
previousImageHint: `Use docker image ls and docker inspect logs for the previous ${spec.serviceId} image id; Compose named volumes were not changed.`,
commandShape: `${compose.map(shellQuote).join(" ")} up -d --no-build --no-deps --force-recreate ${serviceName}`,
},
};
}
async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise<Record<string, unknown>> {
if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit <full-sha>");
return deployComposeArtifactNow(options, artifactConsumerSpecs["backend-core"]);
}
function deployBackendCoreJob(args: string[], options: ArtifactRegistryOptions): Record<string, unknown> {
if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit <full-sha>");
const runArgs = args.includes("--run-now") ? args : [...args, "--run-now"];
@@ -875,6 +951,7 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti
serviceId: spec.serviceId,
commit,
sourceRepo: options.sourceRepo,
deployRef: deployRefFor(options, spec),
sourceImage,
requiredLabels: {
"unidesk.ai/service-id": spec.serviceId,
@@ -885,23 +962,30 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti
method: "HEAD",
url: `http://127.0.0.1:${options.port}/v2/${spec.registryRepository}/manifests/${commit}`,
},
boundary: "prod CD is artifact-consumer only: verify commit-pinned registry image, pull/import, deploy, then verify live commit/image/health; it never builds source on prod",
boundary: "artifact CD is artifact-consumer only: verify commit-pinned registry image, pull/import, deploy, then verify live commit/image/health; it never builds source on the runtime target",
};
if (spec.kind === "compose") {
if (spec.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`);
return {
...common,
target: {
kind: "compose",
composeService: "backend-core",
targetImage: options.targetImage === defaultOptions.targetImage ? spec.targetImage : options.targetImage,
deployCommandShape: "docker compose up -d --no-build --no-deps --force-recreate backend-core",
composeService: spec.compose.serviceName,
containerName: spec.compose.containerName,
targetImage: options.targetImage ?? spec.targetImage,
runtimeImage: spec.targetCommitImage(commit),
deployEnvPrefix: spec.compose.deployEnvPrefix,
deployCommandShape: `docker compose up -d --no-build --no-deps --force-recreate ${spec.compose.serviceName}`,
},
validation: [
"D601 registry /v2 manifest exists for the commit tag",
"loaded image labels match service id, source commit, and Dockerfile",
"running Compose container image label matches the requested commit",
"backend-core /health succeeds for the recreated container",
spec.compose.requireHealthCommit
? `${spec.serviceId} /health succeeds and reports deploy.commit matching the artifact commit`
: `${spec.serviceId} /health succeeds for the recreated container`,
],
rollback: rollbackInfo(spec, commit),
};
}
return {
@@ -929,9 +1013,12 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti
function rollbackInfo(spec: ArtifactConsumerSpec, commit: string): Record<string, unknown> {
if (spec.kind === "compose") {
const compose = spec.compose;
return {
type: "compose-retag-recreate",
previousImageHint: "Use docker image ls / docker inspect to find the previous labeled backend-core image id; Compose volumes are unchanged.",
composeService: compose?.serviceName,
previousImageHint: `Use docker image ls / docker inspect to find the previous labeled ${spec.serviceId} image id; Compose volumes are unchanged.`,
commandShape: `bun scripts/cli.ts deploy apply --env prod --service ${spec.serviceId} --commit <previous-full-sha>`,
};
}
return {
@@ -951,6 +1038,7 @@ function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: Art
const k3s = spec.k3s;
const labels = [
`unidesk.ai/deploy-service-id=${spec.serviceId}`,
`unidesk.ai/deploy-ref=${deployRefFor(options, spec)}`,
`unidesk.ai/deploy-repo=${options.sourceRepo}`,
`unidesk.ai/deploy-commit=${commit}`,
`unidesk.ai/deploy-requested-commit=${commit}`,
@@ -964,6 +1052,7 @@ function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: Art
`commit_image=${shellQuote(commitImage)}`,
`service_id=${shellQuote(spec.serviceId)}`,
`source_repo=${shellQuote(options.sourceRepo)}`,
`deploy_ref=${shellQuote(deployRefFor(options, spec))}`,
`commit=${shellQuote(commit)}`,
`dockerfile=${shellQuote(spec.dockerfile)}`,
`namespace=${shellQuote(k3s.namespace)}`,
@@ -997,7 +1086,7 @@ function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: Art
"if [ -f \"$manifest\" ]; then kubectl apply -f \"$manifest\"; else echo artifact_cd_manifest_missing=$manifest; fi",
"previous_commit=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}' 2>/dev/null || true)",
"kubectl -n \"$namespace\" set image \"deployment/$deployment\" \"$container_name=$commit_image\"",
"kubectl -n \"$namespace\" set env \"deployment/$deployment\" \"UNIDESK_DEPLOY_SERVICE_ID=$service_id\" \"UNIDESK_DEPLOY_REPO=$source_repo\" \"UNIDESK_DEPLOY_COMMIT=$commit\" \"UNIDESK_DEPLOY_REQUESTED_COMMIT=$commit\"",
"kubectl -n \"$namespace\" set env \"deployment/$deployment\" \"UNIDESK_DEPLOY_SERVICE_ID=$service_id\" \"UNIDESK_DEPLOY_REF=$deploy_ref\" \"UNIDESK_DEPLOY_REPO=$source_repo\" \"UNIDESK_DEPLOY_COMMIT=$commit\" \"UNIDESK_DEPLOY_REQUESTED_COMMIT=$commit\"",
`kubectl -n "$namespace" annotate "deployment/$deployment" ${labels.map(shellQuote).join(" ")} --overwrite`,
"if [ -n \"$previous_commit\" ] && [ \"$previous_commit\" != \"$commit\" ]; then kubectl -n \"$namespace\" annotate \"deployment/$deployment\" \"unidesk.ai/deploy-previous-commit=$previous_commit\" --overwrite; fi",
"kubectl -n \"$namespace\" rollout status \"deployment/$deployment\" --timeout=180s",
@@ -1109,6 +1198,7 @@ async function deployD601K3sArtifactNow(options: ArtifactRegistryOptions, spec:
commit,
providerId: options.providerId,
sourceRepo: options.sourceRepo,
deployRef: deployRefFor(options, spec),
sourceImage,
stableImage: spec.targetImage,
runtimeImage: spec.targetCommitImage(commit),
@@ -1130,7 +1220,7 @@ async function deployServiceNow(options: ArtifactRegistryOptions): Promise<Recor
const spec = artifactConsumerSpec(options.serviceId);
if (spec === null) return unsupportedService(options.serviceId, options);
if (options.dryRun) return dryRunArtifactConsumerPlan(options, spec, options.commit);
if (spec.kind === "compose") return deployBackendCoreNow({ ...options, targetImage: options.targetImage || spec.targetImage });
if (spec.kind === "compose") return deployComposeArtifactNow(options, spec);
return deployD601K3sArtifactNow(options, spec);
}
@@ -1171,13 +1261,18 @@ function localHelp(): Record<string, unknown> {
"bun scripts/cli.ts artifact-registry health [--provider-id D601]",
"bun scripts/cli.ts artifact-registry install [--provider-id D601]",
"bun scripts/cli.ts artifact-registry deploy-backend-core --commit <full-sha> [--run-now] [--provider-id D601]",
"bun scripts/cli.ts artifact-registry deploy-service --service baidu-netdisk --commit <full-sha> [--dry-run] [--run-now] [--provider-id D601]",
"bun scripts/cli.ts artifact-registry deploy-service --service decision-center --commit <full-sha> [--dry-run] [--run-now] [--provider-id D601]",
],
firstStage: "install now writes the rendered systemd/Compose/config files and starts the registry",
artifactConsumers: {
supportedServices: supportedArtifactConsumerServices,
unsupportedPolicy: "return structured unsupported; never fall back to legacy maintenance-channel source builds",
prodCommand: "bun scripts/cli.ts deploy apply --env prod --service decision-center",
prodCommands: [
"bun scripts/cli.ts deploy apply --env prod --service backend-core",
"bun scripts/cli.ts deploy apply --env prod --service baidu-netdisk",
"bun scripts/cli.ts deploy apply --env prod --service decision-center",
],
rollbackShape: "rerun the same artifact consumer with a previous commit-pinned image",
},
defaults: defaultOptions,