Codex
259 lines
14 KiB
TypeScript
259 lines
14 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. source-prepare module for scripts/src/ci.ts.
|
|
|
|
// Moved mechanically from scripts/src/ci.ts:1668-1898 for #903.
|
|
|
|
import { randomUUID } from "node:crypto";
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, posix as posixPath } from "node:path";
|
|
import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "../ci-catalog";
|
|
import { runCommand } from "../command";
|
|
import { type UniDeskConfig, repoRoot, rootPath } from "../config";
|
|
import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "../deploy-ssh-identity";
|
|
import { jobWithTail, listJobs, readJob, startJob } from "../jobs";
|
|
import { coreInternalFetch } from "../microservices";
|
|
import {
|
|
artifactRegistryReadonlyResultFromCommand,
|
|
buildArtifactRegistryReadonlyProbe,
|
|
parseArtifactRegistryOptions,
|
|
type ArtifactRegistryReadonlyProbe,
|
|
} from "../artifact-registry";
|
|
import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard";
|
|
import { runSshCommandCapture } from "../ssh";
|
|
|
|
import type { CiPublishBackendCoreOptions, CiPublishUserServiceArtifactOptions } from "./types";
|
|
import { repoConnectivityProbeUrl, repoNeedsGithubSshIdentity, repoSshUrl, requireRepoRelativePath, safePathToken, shellQuote } from "./options";
|
|
import { runRemoteBackground } from "./remote";
|
|
import { d601ProviderId, providerGatewayWsEgressProxyUrl } from "./types";
|
|
|
|
export async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: CiPublishBackendCoreOptions): Promise<Record<string, unknown>> {
|
|
const sourceRoot = "/home/ubuntu/.unidesk/ci/backend-core-artifacts";
|
|
const sourceHostPath = options.sourceHostPath;
|
|
const repoCache = "/home/ubuntu/.unidesk/ci/git/unidesk.git";
|
|
const repoFetchUrl = repoSshUrl(options.repoUrl);
|
|
const sshIdentity = repoNeedsGithubSshIdentity(repoFetchUrl) ? await ensureGithubSshIdentityForProvider(config, d601ProviderId) : null;
|
|
if (sshIdentity !== null && !sshIdentity.ok) throw new Error(sshIdentity.detail);
|
|
const proxyPython = gitSshHttpConnectProxySource();
|
|
const dockerfile = requireRepoRelativePath(options.dockerfile, "CI.json.artifacts.backend-core.source.dockerfile");
|
|
const script = [
|
|
"set -euo pipefail",
|
|
`commit=${shellQuote(options.commit)}`,
|
|
`repo_url=${shellQuote(options.repoUrl)}`,
|
|
`repo_fetch_url=${shellQuote(repoFetchUrl)}`,
|
|
`dockerfile=${shellQuote(dockerfile)}`,
|
|
`source_root=${shellQuote(sourceRoot)}`,
|
|
`source_dir=${shellQuote(sourceHostPath)}`,
|
|
`repo_cache=${shellQuote(repoCache)}`,
|
|
`proxy_url=${shellQuote(providerGatewayWsEgressProxyUrl)}`,
|
|
"mkdir -p \"$(dirname \"$repo_cache\")\" \"$source_root\"",
|
|
"export HTTP_PROXY=\"$proxy_url\" HTTPS_PROXY=\"$proxy_url\" ALL_PROXY=\"$proxy_url\"",
|
|
"export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"",
|
|
"curl -fsSI --max-time 20 -x \"$proxy_url\" https://github.com >/dev/null",
|
|
"git_ssh_proxy=/tmp/unidesk-git-ssh-http-connect.py",
|
|
"cat > \"$git_ssh_proxy\" <<'UNIDESK_GIT_SSH_PROXY'",
|
|
proxyPython,
|
|
"UNIDESK_GIT_SSH_PROXY",
|
|
"chmod 700 \"$git_ssh_proxy\"",
|
|
"export UNIDESK_GIT_SSH_HTTP_PROXY=\"$proxy_url\"",
|
|
"export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=$git_ssh_proxy %h %p'\"",
|
|
"echo backend_core_artifact_source_proxy=provider-gateway-ws-egress:$proxy_url",
|
|
"echo backend_core_artifact_repo_fetch_url=$repo_fetch_url",
|
|
"if [ ! -d \"$repo_cache\" ]; then git clone --mirror \"$repo_fetch_url\" \"$repo_cache\"; fi",
|
|
"git -C \"$repo_cache\" remote set-url origin \"$repo_fetch_url\"",
|
|
"git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
|
|
"resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")",
|
|
"test \"$resolved\" = \"$commit\" || { echo \"backend_core_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
|
|
"git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile\"",
|
|
"git -C \"$repo_cache\" cat-file -e \"$commit:src/components/backend-core/src\"",
|
|
"tmp_dir=\"$source_root/.tmp-$commit-$$\"",
|
|
"rm -rf \"$tmp_dir\"",
|
|
"mkdir -p \"$tmp_dir\"",
|
|
"git -C \"$repo_cache\" archive \"$commit\" | tar -x -C \"$tmp_dir\"",
|
|
"printf '%s\\n' \"$commit\" > \"$tmp_dir/.unidesk-source-commit\"",
|
|
"printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"",
|
|
"rm -rf \"$source_dir\"",
|
|
"mv \"$tmp_dir\" \"$source_dir\"",
|
|
"test -f \"$source_dir/$dockerfile\"",
|
|
"test -d \"$source_dir/src/components/backend-core/src\"",
|
|
"echo backend_core_artifact_source_host_path=$source_dir",
|
|
].join("\n");
|
|
const result = await runRemoteBackground("prepare-backend-core-source", script, 300_000);
|
|
if (!result.ok) throw new Error(`failed to prepare backend-core source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`);
|
|
return {
|
|
ok: true,
|
|
mode: repoNeedsGithubSshIdentity(repoFetchUrl) ? "d601-host-git-ssh-export" : "d601-host-git-https-export",
|
|
providerId: d601ProviderId,
|
|
repoUrl: options.repoUrl,
|
|
repoFetchUrl,
|
|
commit: options.commit,
|
|
sourceHostPath,
|
|
dockerfile,
|
|
identity: sshIdentity === null ? null : {
|
|
fingerprint: sshIdentity.fingerprint,
|
|
seededFromLocal: sshIdentity.seededFromLocal,
|
|
},
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
};
|
|
}
|
|
|
|
export async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions): Promise<Record<string, unknown>> {
|
|
const sourceRoot = `/home/ubuntu/.unidesk/ci/user-service-artifacts/${options.serviceId}`;
|
|
const sourceHostPath = options.sourceHostPath;
|
|
const repoCache = `/home/ubuntu/.unidesk/ci/git/${safePathToken(options.serviceId)}.git`;
|
|
const repoFetchUrl = repoSshUrl(options.repoUrl);
|
|
const repoProbeUrl = repoConnectivityProbeUrl(repoFetchUrl);
|
|
const sshIdentity = repoNeedsGithubSshIdentity(repoFetchUrl) ? await ensureGithubSshIdentityForProvider(config, d601ProviderId) : null;
|
|
if (sshIdentity !== null && !sshIdentity.ok) throw new Error(sshIdentity.detail);
|
|
const proxyPython = gitSshHttpConnectProxySource();
|
|
const script = [
|
|
"set -euo pipefail",
|
|
`service_id=${shellQuote(options.serviceId)}`,
|
|
`commit=${shellQuote(options.commit)}`,
|
|
`repo_url=${shellQuote(options.repoUrl)}`,
|
|
`repo_fetch_url=${shellQuote(repoFetchUrl)}`,
|
|
`repo_probe_url=${shellQuote(repoProbeUrl)}`,
|
|
`dockerfile=${shellQuote(options.dockerfile)}`,
|
|
`source_root=${shellQuote(sourceRoot)}`,
|
|
`source_dir=${shellQuote(sourceHostPath)}`,
|
|
`repo_cache=${shellQuote(repoCache)}`,
|
|
`proxy_url=${shellQuote(providerGatewayWsEgressProxyUrl)}`,
|
|
"mkdir -p \"$(dirname \"$repo_cache\")\" \"$source_root\"",
|
|
"export HTTP_PROXY=\"$proxy_url\" HTTPS_PROXY=\"$proxy_url\" ALL_PROXY=\"$proxy_url\"",
|
|
"export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"",
|
|
"curl -fsSI --max-time 20 -x \"$proxy_url\" \"$repo_probe_url\" >/dev/null",
|
|
"git_ssh_proxy=/tmp/unidesk-git-ssh-http-connect.py",
|
|
"cat > \"$git_ssh_proxy\" <<'UNIDESK_GIT_SSH_PROXY'",
|
|
proxyPython,
|
|
"UNIDESK_GIT_SSH_PROXY",
|
|
"chmod 700 \"$git_ssh_proxy\"",
|
|
"export UNIDESK_GIT_SSH_HTTP_PROXY=\"$proxy_url\"",
|
|
"export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=$git_ssh_proxy %h %p'\"",
|
|
"echo user_service_artifact_source_proxy=provider-gateway-ws-egress:$proxy_url",
|
|
"echo user_service_artifact_repo_fetch_url=$repo_fetch_url",
|
|
"echo user_service_artifact_repo_probe_url=$repo_probe_url",
|
|
"echo user_service_artifact_service_id=$service_id",
|
|
"if [ ! -d \"$repo_cache\" ]; then git clone --mirror \"$repo_fetch_url\" \"$repo_cache\"; fi",
|
|
"git -C \"$repo_cache\" remote set-url origin \"$repo_fetch_url\"",
|
|
"git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
|
|
"resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")",
|
|
"test \"$resolved\" = \"$commit\" || { echo \"user_service_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
|
|
"git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile\"",
|
|
"tmp_dir=\"$source_root/.tmp-$commit-$$\"",
|
|
"rm -rf \"$tmp_dir\"",
|
|
"mkdir -p \"$tmp_dir\"",
|
|
"git -C \"$repo_cache\" archive \"$commit\" | tar -x -C \"$tmp_dir\"",
|
|
"printf '%s\\n' \"$commit\" > \"$tmp_dir/.unidesk-source-commit\"",
|
|
"printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"",
|
|
"printf '%s\\n' \"$service_id\" > \"$tmp_dir/.unidesk-service-id\"",
|
|
"printf '%s\\n' \"$dockerfile\" > \"$tmp_dir/.unidesk-dockerfile\"",
|
|
"rm -rf \"$source_dir\"",
|
|
"mv \"$tmp_dir\" \"$source_dir\"",
|
|
"test -f \"$source_dir/$dockerfile\"",
|
|
"echo user_service_artifact_source_host_path=$source_dir",
|
|
].join("\n");
|
|
const result = await runRemoteBackground(`prepare-${options.serviceId}-source`, script, 300_000);
|
|
if (!result.ok) throw new Error(`failed to prepare ${options.serviceId} source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`);
|
|
return {
|
|
ok: true,
|
|
mode: repoNeedsGithubSshIdentity(repoFetchUrl) ? "d601-host-git-ssh-export" : "d601-host-git-https-export",
|
|
providerId: d601ProviderId,
|
|
repoUrl: options.repoUrl,
|
|
repoFetchUrl,
|
|
repoProbeUrl,
|
|
commit: options.commit,
|
|
serviceId: options.serviceId,
|
|
dockerfile: options.dockerfile,
|
|
sourceHostPath,
|
|
identity: sshIdentity === null ? null : {
|
|
fingerprint: sshIdentity.fingerprint,
|
|
seededFromLocal: sshIdentity.seededFromLocal,
|
|
},
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
};
|
|
}
|
|
|
|
export async function prepareClaudeqqArtifactSource(config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions): Promise<Record<string, unknown>> {
|
|
const sourceRoot = `/home/ubuntu/.unidesk/ci/user-service-artifacts/${options.serviceId}`;
|
|
const sourceHostPath = options.sourceHostPath;
|
|
const repoCache = "/home/ubuntu/.unidesk/ci/git/claudeqq-agent-skills.git";
|
|
const repoFetchUrl = options.repoUrl;
|
|
const assets = [
|
|
{
|
|
relativePath: "claudeqq/Dockerfile",
|
|
sourcePath: rootPath("src/components/microservices/claudeqq/Dockerfile"),
|
|
label: "Dockerfile",
|
|
},
|
|
{
|
|
relativePath: "claudeqq/unidesk-adapter.cjs",
|
|
sourcePath: rootPath("src/components/microservices/claudeqq/adapter.js"),
|
|
label: "unidesk-adapter.cjs",
|
|
},
|
|
];
|
|
for (const asset of assets) {
|
|
if (!existsSync(asset.sourcePath)) throw new Error(`claudeqq artifact asset missing: ${asset.sourcePath}`);
|
|
}
|
|
const overlayCommands = assets.flatMap((asset) => {
|
|
const encoded = Buffer.from(readFileSync(asset.sourcePath, "utf8"), "utf8").toString("base64");
|
|
return [
|
|
`mkdir -p "$tmp_dir/$(dirname ${shellQuote(asset.relativePath)})"`,
|
|
`printf %s ${shellQuote(encoded)} | base64 -d > "$tmp_dir/${asset.relativePath}"`,
|
|
`printf 'user_service_artifact_overlay=%s\\n' ${shellQuote(asset.label)}`,
|
|
];
|
|
});
|
|
const script = [
|
|
"set -euo pipefail",
|
|
`service_id=${shellQuote(options.serviceId)}`,
|
|
`commit=${shellQuote(options.commit)}`,
|
|
`repo_url=${shellQuote(options.repoUrl)}`,
|
|
`repo_fetch_url=${shellQuote(repoFetchUrl)}`,
|
|
`dockerfile=${shellQuote(options.dockerfile)}`,
|
|
`source_root=${shellQuote(sourceRoot)}`,
|
|
`source_dir=${shellQuote(sourceHostPath)}`,
|
|
`repo_cache=${shellQuote(repoCache)}`,
|
|
`proxy_url=${shellQuote(providerGatewayWsEgressProxyUrl)}`,
|
|
"mkdir -p \"$(dirname \"$repo_cache\")\" \"$source_root\"",
|
|
"export HTTP_PROXY=\"$proxy_url\" HTTPS_PROXY=\"$proxy_url\" ALL_PROXY=\"$proxy_url\"",
|
|
"export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"",
|
|
"curl -fsSI --max-time 20 -x \"$proxy_url\" https://gitee.com >/dev/null",
|
|
"echo user_service_artifact_source_proxy=provider-gateway-ws-egress:$proxy_url",
|
|
"echo user_service_artifact_repo_fetch_url=$repo_fetch_url",
|
|
"echo user_service_artifact_service_id=$service_id",
|
|
"if [ ! -d \"$repo_cache\" ]; then git clone --mirror \"$repo_fetch_url\" \"$repo_cache\"; fi",
|
|
"git -C \"$repo_cache\" remote set-url origin \"$repo_fetch_url\"",
|
|
"git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
|
|
"resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")",
|
|
"test \"$resolved\" = \"$commit\" || { echo \"user_service_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
|
|
"git -C \"$repo_cache\" cat-file -e \"$commit:claudeqq/scripts/src/server_ts/package.json\"",
|
|
"git -C \"$repo_cache\" cat-file -e \"$commit:claudeqq/scripts/src/server_ts/src\"",
|
|
"tmp_dir=\"$source_root/.tmp-$commit-$$\"",
|
|
"rm -rf \"$tmp_dir\"",
|
|
"mkdir -p \"$tmp_dir\"",
|
|
"git -C \"$repo_cache\" archive \"$commit\" claudeqq | tar -x -C \"$tmp_dir\"",
|
|
...overlayCommands,
|
|
"printf '%s\\n' \"$commit\" > \"$tmp_dir/.unidesk-source-commit\"",
|
|
"printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"",
|
|
"printf '%s\\n' \"$service_id\" > \"$tmp_dir/.unidesk-service-id\"",
|
|
"printf '%s\\n' \"$dockerfile\" > \"$tmp_dir/.unidesk-dockerfile\"",
|
|
"rm -rf \"$source_dir\"",
|
|
"mv \"$tmp_dir\" \"$source_dir\"",
|
|
"test -f \"$source_dir/$dockerfile\"",
|
|
"test -f \"$source_dir/claudeqq/unidesk-adapter.cjs\"",
|
|
"test -d \"$source_dir/claudeqq/scripts/src/server_ts/src\"",
|
|
"echo user_service_artifact_source_host_path=$source_dir",
|
|
].join("\n");
|
|
const result = await runRemoteBackground(`prepare-${options.serviceId}-source`, script, 300_000);
|
|
if (!result.ok) throw new Error(`failed to prepare ${options.serviceId} source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`);
|
|
return {
|
|
ok: true,
|
|
mode: "d601-host-gitee-https-export-with-unidesk-overlay",
|
|
providerId: d601ProviderId,
|
|
repoUrl: options.repoUrl,
|
|
repoFetchUrl,
|
|
commit: options.commit,
|
|
serviceId: options.serviceId,
|
|
dockerfile: options.dockerfile,
|
|
sourceHostPath,
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
};
|
|
}
|