Codex
237 lines
11 KiB
TypeScript
237 lines
11 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. dev-e2e module for scripts/src/ci.ts.
|
|
|
|
// Moved mechanically from scripts/src/ci.ts:2960-3167 for #903.
|
|
|
|
import { randomUUID } from "node:crypto";
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join, posix as posixPath } from "node:path";
|
|
import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "../ci-catalog";
|
|
import { runCommand } from "../command";
|
|
import { type UniDeskConfig, repoRoot, rootPath } from "../config";
|
|
import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "../deploy-ssh-identity";
|
|
import { jobWithTail, listJobs, readJob, startJob } from "../jobs";
|
|
import { coreInternalFetch } from "../microservices";
|
|
import {
|
|
artifactRegistryReadonlyResultFromCommand,
|
|
buildArtifactRegistryReadonlyProbe,
|
|
parseArtifactRegistryOptions,
|
|
type ArtifactRegistryReadonlyProbe,
|
|
} from "../artifact-registry";
|
|
import { d601K3sGuardShellLines, d601NativeKubeconfig } from "../d601-k3s-guard";
|
|
import { runSshCommandCapture } from "../ssh";
|
|
|
|
import type { CiDevE2EOptions, DeployDevManifestSummary, DispatchResult } from "./types";
|
|
import { status } from "./cleanup";
|
|
import { shellQuote } from "./options";
|
|
import { asRecord, asString } from "./preflight";
|
|
import { dispatchSsh, positiveManifestNumber, requireCiScriptPath, requireManifestString } from "./remote";
|
|
import { providerGatewayWsEgressProxyUrl } from "./types";
|
|
|
|
export async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise<DispatchResult> {
|
|
const scriptTimeoutMs = Math.max(options.scriptTimeoutMs, options.waitMs, 60_000);
|
|
const remoteTimeoutMs = 45_000;
|
|
const command = [
|
|
"set -euo pipefail",
|
|
`run_id=${shellQuote(options.runId)}`,
|
|
`repo_url=${shellQuote(options.scriptRepo)}`,
|
|
`commit=${shellQuote(options.deployCommit)}`,
|
|
`script_path=${shellQuote(options.scriptPath)}`,
|
|
`desired_ref=${shellQuote(options.desiredRef)}`,
|
|
`environment=${shellQuote(options.environment)}`,
|
|
`keep_namespace=${shellQuote(options.keepNamespace ? "true" : "false")}`,
|
|
`timeout_ms=${shellQuote(String(scriptTimeoutMs))}`,
|
|
"work_dir=\"/tmp/unidesk-ci/$run_id\"",
|
|
"result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"",
|
|
"mkdir -p \"$work_dir\" \"$result_dir\"",
|
|
"launcher_log=\"$result_dir/launcher.log\"",
|
|
"case \"$script_path\" in scripts/ci/*.sh) ;; *) echo \"invalid_script_path=$script_path\" >&2; exit 2 ;; esac",
|
|
"(",
|
|
"set -euo pipefail",
|
|
"trap '' HUP",
|
|
"exec >> \"$launcher_log\" 2>&1",
|
|
"echo \"launcher_run_id=$run_id\"",
|
|
"echo \"launcher_repo=$repo_url\"",
|
|
"echo \"launcher_commit=$commit\"",
|
|
"echo \"launcher_script_path=$script_path\"",
|
|
"export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config",
|
|
"mkdir -p \"$DOCKER_CONFIG\"",
|
|
"printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"",
|
|
`build_proxy=${shellQuote(providerGatewayWsEgressProxyUrl)}`,
|
|
"export HTTP_PROXY=\"$build_proxy\" HTTPS_PROXY=\"$build_proxy\" ALL_PROXY=\"$build_proxy\"",
|
|
"export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"",
|
|
"if ! curl -fsSI --max-time 20 -x \"$build_proxy\" https://github.com >/dev/null; then",
|
|
" echo \"ci_provider_egress_proxy_unavailable=$build_proxy\" >&2",
|
|
" exit 1",
|
|
"fi",
|
|
"echo \"ci_provider_egress_proxy=provider-gateway-ws-egress:$build_proxy\"",
|
|
"repo_fetch_url=\"$repo_url\"",
|
|
"case \"$repo_fetch_url\" in",
|
|
" https://github.com/*)",
|
|
" repo_path=\"${repo_fetch_url#https://github.com/}\"",
|
|
" repo_path=\"${repo_path%.git}\"",
|
|
" repo_fetch_url=\"git@github.com:$repo_path.git\"",
|
|
" ;;",
|
|
"esac",
|
|
"export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=nc -X connect -x 127.0.0.1:18789 %h %p'\"",
|
|
"echo \"launcher_repo_fetch_url=$repo_fetch_url\"",
|
|
"repo_dir=\"$work_dir/repo\"",
|
|
"if [ ! -d \"$repo_dir/.git\" ]; then",
|
|
" git clone --no-checkout \"$repo_fetch_url\" \"$repo_dir\"",
|
|
"fi",
|
|
"git -C \"$repo_dir\" remote set-url origin \"$repo_fetch_url\"",
|
|
"git -C \"$repo_dir\" fetch --no-tags origin \"$commit\" || git -C \"$repo_dir\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
|
|
"resolved=$(git -C \"$repo_dir\" rev-parse --verify \"$commit^{commit}\")",
|
|
"test \"$resolved\" = \"$commit\" || { echo \"resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
|
|
"git -C \"$repo_dir\" cat-file -e \"$resolved:$script_path\"",
|
|
"git -C \"$repo_dir\" show \"$resolved:$script_path\" > \"$work_dir/runner.sh\"",
|
|
"git -C \"$repo_dir\" show \"$resolved:deploy.json\" > \"$work_dir/deploy.json\"",
|
|
"chmod 700 \"$work_dir/runner.sh\"",
|
|
"echo \"runner_script_ready=$work_dir/runner.sh\"",
|
|
"runner_args=(",
|
|
" --run-id \"$run_id\"",
|
|
" --repo-url \"$repo_url\"",
|
|
" --desired-ref \"$desired_ref\"",
|
|
" --manifest-commit \"$commit\"",
|
|
" --manifest-file \"$work_dir/deploy.json\"",
|
|
" --environment \"$environment\"",
|
|
" --result-dir \"$result_dir\"",
|
|
" --timeout-ms \"$timeout_ms\"",
|
|
")",
|
|
"if [ \"$keep_namespace\" = \"true\" ]; then runner_args+=(--keep-namespace); fi",
|
|
"bash \"$work_dir/runner.sh\" \"${runner_args[@]}\"",
|
|
") &",
|
|
"launcher_pid=$!",
|
|
"disown \"$launcher_pid\" 2>/dev/null || true",
|
|
"printf 'launcher_background_pid=%s\\nresult_dir=%s\\n' \"$launcher_pid\" \"$result_dir\"",
|
|
].join("\n");
|
|
return dispatchSsh(command, 30_000, remoteTimeoutMs);
|
|
}
|
|
|
|
export async function waitForDevE2EResult(runId: string, waitMs: number): Promise<DispatchResult | null> {
|
|
if (waitMs <= 0) return null;
|
|
const deadline = Date.now() + waitMs;
|
|
let latest: DispatchResult | null = null;
|
|
while (Date.now() < deadline) {
|
|
const result = await dispatchSsh([
|
|
"set -euo pipefail",
|
|
`run_id=${shellQuote(runId)}`,
|
|
"result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"",
|
|
"if [ -f \"$result_dir/result.json\" ]; then cat \"$result_dir/result.json\"; exit 0; fi",
|
|
"printf 'RUNNING result_dir=%s\\n' \"$result_dir\"",
|
|
"tail -n 40 \"$result_dir/launcher.log\" 2>/dev/null || true",
|
|
"tail -n 80 \"$result_dir/runner.log\" 2>/dev/null || true",
|
|
].join("\n"), 30_000, 20_000);
|
|
latest = result;
|
|
const stdout = result.stdout.trimStart();
|
|
if (stdout.startsWith("{")) {
|
|
const parsed = JSON.parse(stdout) as { ok?: boolean; status?: string };
|
|
return {
|
|
...result,
|
|
ok: parsed.ok === true,
|
|
status: parsed.status ?? (parsed.ok === true ? "succeeded" : "failed"),
|
|
exitCode: parsed.ok === true ? 0 : 1,
|
|
};
|
|
}
|
|
await Bun.sleep(5_000);
|
|
}
|
|
return {
|
|
ok: false,
|
|
taskId: latest?.taskId ?? null,
|
|
status: "timeout",
|
|
stdout: latest?.stdout ?? "",
|
|
stderr: `dev e2e result did not finish within ${waitMs}ms`,
|
|
exitCode: 124,
|
|
raw: latest?.raw ?? null,
|
|
};
|
|
}
|
|
|
|
export function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary {
|
|
const remoteRef = `refs/remotes/origin/${desiredRef}`;
|
|
const fetch = runCommand(["git", "fetch", "--quiet", "origin", `+refs/heads/${desiredRef}:${remoteRef}`], repoRoot);
|
|
if (fetch.exitCode !== 0) throw new Error(`failed to fetch origin/${desiredRef}: ${fetch.stderr || fetch.stdout}`);
|
|
const deployCommitResult = runCommand(["git", "rev-parse", `origin/${desiredRef}`], repoRoot);
|
|
if (deployCommitResult.exitCode !== 0) throw new Error(`failed to resolve origin/${desiredRef}: ${deployCommitResult.stderr || deployCommitResult.stdout}`);
|
|
const show = runCommand(["git", "show", `origin/${desiredRef}:deploy.json`], repoRoot);
|
|
if (show.exitCode !== 0) throw new Error(`failed to read deploy.json from origin/${desiredRef}: ${show.stderr || show.stdout}`);
|
|
const parsed = JSON.parse(show.stdout) as unknown;
|
|
const record = asRecord(parsed);
|
|
if (record?.schemaVersion !== 2) throw new Error(`origin/${desiredRef}:deploy.json must use schemaVersion=2`);
|
|
const environments = asRecord(record.environments);
|
|
const dev = asRecord(environments?.dev);
|
|
const ci = asRecord(dev?.ci);
|
|
if (ci === null) throw new Error(`origin/${desiredRef}:deploy.json must contain environments.dev.ci`);
|
|
const rawServices = Array.isArray(dev?.services) ? dev.services : [];
|
|
const services = rawServices.map((item) => {
|
|
const service = asRecord(item);
|
|
return {
|
|
id: asString(service?.id),
|
|
commitId: asString(service?.commitId).toLowerCase(),
|
|
repo: asString(service?.repo),
|
|
};
|
|
}).filter((service) => service.id.length > 0 && service.commitId.length > 0);
|
|
if (services.length === 0) throw new Error(`origin/${desiredRef}:deploy.json has no environments.dev services with commitId`);
|
|
const codeQueueService = services.find((service) => service.id === "code-queue");
|
|
if (codeQueueService === undefined) {
|
|
throw new Error(`origin/${desiredRef}:deploy.json environments.dev.services must include code-queue for ci run-dev-e2e`);
|
|
}
|
|
if (!/^[0-9a-f]{40}$/u.test(codeQueueService.commitId)) {
|
|
throw new Error(`origin/${desiredRef}:deploy.json environments.dev.services code-queue commitId must be a full 40-character SHA`);
|
|
}
|
|
return {
|
|
deployCommit: deployCommitResult.stdout.trim(),
|
|
desiredRef,
|
|
environment: "dev",
|
|
ci: {
|
|
repo: requireManifestString(ci.repo, "environments.dev.ci.repo"),
|
|
scriptPath: requireCiScriptPath(ci.scriptPath),
|
|
timeoutMs: positiveManifestNumber(ci.timeoutMs, 1_800_000, "environments.dev.ci.timeoutMs"),
|
|
},
|
|
services,
|
|
};
|
|
}
|
|
|
|
export function makeRunId(deployCommit: string): string {
|
|
const stamp = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase();
|
|
return `${stamp}-${deployCommit.slice(0, 8).toLowerCase()}`.replace(/[^a-z0-9-]/gu, "-").slice(0, 48);
|
|
}
|
|
|
|
export async function runDevE2E(options: CiDevE2EOptions): Promise<Record<string, unknown>> {
|
|
const result = await runRemoteDevE2ELauncher(options);
|
|
const wait = result.ok ? await waitForDevE2EResult(options.runId, options.waitMs) : null;
|
|
const ok = result.ok && (result.exitCode === null || result.exitCode === 0) && (wait === null || wait.ok);
|
|
return {
|
|
ok,
|
|
runId: options.runId,
|
|
namespace: "unidesk-ci",
|
|
temporaryNamespace: `unidesk-ci-e2e-${options.runId}`,
|
|
repoUrl: options.repoUrl,
|
|
desiredRef: options.desiredRef,
|
|
deployCommit: options.deployCommit,
|
|
scriptRepo: options.scriptRepo,
|
|
scriptPath: options.scriptPath,
|
|
environment: options.environment,
|
|
services: options.services,
|
|
keepNamespace: options.keepNamespace,
|
|
triggerMode: "commit-pinned-ssh-launcher",
|
|
launcher: {
|
|
taskId: result.taskId,
|
|
status: result.status,
|
|
exitCode: result.exitCode,
|
|
stdoutTail: result.stdout.slice(-6000),
|
|
stderrTail: result.stderr.slice(-6000),
|
|
},
|
|
wait: wait === null ? null : {
|
|
status: wait.status,
|
|
exitCode: wait.exitCode,
|
|
stdoutTail: wait.stdout.slice(-6000),
|
|
stderrTail: wait.stderr.slice(-6000),
|
|
},
|
|
resultDir: `/home/ubuntu/.unidesk/runs/${options.runId}`,
|
|
next: [
|
|
`bun scripts/cli.ts ci logs ${options.runId}`,
|
|
"bun scripts/cli.ts ci status",
|
|
],
|
|
};
|
|
}
|