refactor: use git-controlled dev ci runner
This commit is contained in:
Codex
+153
-95
@@ -39,11 +39,13 @@ interface CiDevE2EOptions {
|
||||
desiredRef: string;
|
||||
deployCommit: string;
|
||||
environment: "dev";
|
||||
scriptRepo: string;
|
||||
scriptPath: string;
|
||||
scriptTimeoutMs: number;
|
||||
services: Array<{ id: string; commitId: string; repo: string }>;
|
||||
runId: string;
|
||||
keepNamespace: boolean;
|
||||
waitMs: number;
|
||||
direct: boolean;
|
||||
}
|
||||
|
||||
interface DispatchResult {
|
||||
@@ -60,6 +62,11 @@ interface DeployDevManifestSummary {
|
||||
deployCommit: string;
|
||||
desiredRef: string;
|
||||
environment: "dev";
|
||||
ci: {
|
||||
repo: string;
|
||||
scriptPath: string;
|
||||
timeoutMs: number;
|
||||
};
|
||||
services: Array<{ id: string; commitId: string; repo: string }>;
|
||||
}
|
||||
|
||||
@@ -121,13 +128,26 @@ function coreBody(response: unknown): Record<string, unknown> | null {
|
||||
return asRecord(asRecord(response)?.body);
|
||||
}
|
||||
|
||||
function proxyBody(response: unknown): Record<string, unknown> | null {
|
||||
const body = coreBody(response);
|
||||
const nested = asRecord(body?.body);
|
||||
return nested ?? body;
|
||||
function positiveManifestNumber(value: unknown, fallback: number, path: string): number {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
||||
return value;
|
||||
}
|
||||
|
||||
async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number): Promise<DispatchResult> {
|
||||
function requireManifestString(value: unknown, path: string): string {
|
||||
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireCiScriptPath(value: unknown): string {
|
||||
const scriptPath = requireManifestString(value, "environments.dev.ci.scriptPath");
|
||||
if (!scriptPath.startsWith("scripts/ci/") || scriptPath.includes("..") || scriptPath.startsWith("/") || !scriptPath.endsWith(".sh")) {
|
||||
throw new Error("environments.dev.ci.scriptPath must be a repo-relative scripts/ci/*.sh path");
|
||||
}
|
||||
return scriptPath;
|
||||
}
|
||||
|
||||
async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: number, pollCompletion = true): Promise<DispatchResult> {
|
||||
const dispatchResponse = coreInternalFetch("/api/dispatch", {
|
||||
method: "POST",
|
||||
body: {
|
||||
@@ -155,7 +175,18 @@ async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: num
|
||||
raw: dispatchResponse,
|
||||
};
|
||||
}
|
||||
const deadline = Date.now() + Math.max(waitMs, remoteTimeoutMs + 10_000);
|
||||
if (!pollCompletion) {
|
||||
return {
|
||||
ok: true,
|
||||
taskId,
|
||||
status: "submitted",
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
exitCode: null,
|
||||
raw: dispatchBody,
|
||||
};
|
||||
}
|
||||
const deadline = Date.now() + Math.max(waitMs, 1_000);
|
||||
let latest: unknown = null;
|
||||
while (Date.now() < deadline) {
|
||||
latest = coreInternalFetch(`/api/tasks/${encodeURIComponent(taskId)}`, { maxResponseBytes: 3_000_000 });
|
||||
@@ -183,7 +214,7 @@ async function dispatchSsh(command: string, waitMs: number, remoteTimeoutMs: num
|
||||
taskId,
|
||||
status: "timeout",
|
||||
stdout: "",
|
||||
stderr: `host.ssh task ${taskId} did not finish within ${Math.max(waitMs, remoteTimeoutMs + 10_000)}ms`,
|
||||
stderr: `host.ssh task ${taskId} did not finish within ${Math.max(waitMs, 1_000)}ms`,
|
||||
exitCode: null,
|
||||
raw: latest,
|
||||
};
|
||||
@@ -439,44 +470,6 @@ spec:
|
||||
`;
|
||||
}
|
||||
|
||||
function devE2EPipelineRunManifest(options: CiDevE2EOptions): string {
|
||||
const deployRevisionLabel = options.deployCommit.slice(0, 40);
|
||||
return `apiVersion: tekton.dev/v1
|
||||
kind: PipelineRun
|
||||
metadata:
|
||||
generateName: unidesk-dev-e2e-${options.runId}-
|
||||
namespace: unidesk-ci
|
||||
labels:
|
||||
app.kubernetes.io/name: unidesk-dev-namespace-e2e
|
||||
app.kubernetes.io/part-of: unidesk
|
||||
unidesk.ai/ci-kind: dev-namespace-e2e
|
||||
unidesk.ai/deploy-ref: master-deploy-json-dev
|
||||
unidesk.ai/deploy-commit: ${JSON.stringify(deployRevisionLabel)}
|
||||
spec:
|
||||
pipelineRef:
|
||||
name: unidesk-dev-namespace-e2e
|
||||
taskRunTemplate:
|
||||
serviceAccountName: unidesk-ci-runner
|
||||
params:
|
||||
- name: repo-url
|
||||
value: ${JSON.stringify(options.repoUrl)}
|
||||
- name: desired-ref
|
||||
value: ${JSON.stringify(options.desiredRef)}
|
||||
- name: deploy-commit
|
||||
value: ${JSON.stringify(options.deployCommit)}
|
||||
- name: environment
|
||||
value: ${JSON.stringify(options.environment)}
|
||||
- name: run-id
|
||||
value: ${JSON.stringify(options.runId)}
|
||||
- name: keep-namespace
|
||||
value: ${JSON.stringify(options.keepNamespace ? "true" : "false")}
|
||||
workspaces:
|
||||
- name: shared-workspace
|
||||
persistentVolumeClaim:
|
||||
claimName: unidesk-ci-cache
|
||||
`;
|
||||
}
|
||||
|
||||
async function remoteCreatePipelineRun(manifest: string): Promise<string> {
|
||||
const encoded = Buffer.from(manifest, "utf8").toString("base64");
|
||||
const token = randomUUID().replace(/-/gu, "").slice(0, 12);
|
||||
@@ -543,6 +536,68 @@ async function run(options: CiOptions): Promise<Record<string, unknown>> {
|
||||
};
|
||||
}
|
||||
|
||||
async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise<DispatchResult> {
|
||||
const scriptTimeoutMs = Math.max(options.scriptTimeoutMs, options.waitMs, 60_000);
|
||||
const remoteTimeoutMs = scriptTimeoutMs + 120_000;
|
||||
const waitMs = options.waitMs > 0 ? options.waitMs + 30_000 : 0;
|
||||
const command = [
|
||||
"set -euo pipefail",
|
||||
`run_id=${shellQuote(options.runId)}`,
|
||||
`repo_url=${shellQuote(options.scriptRepo)}`,
|
||||
`commit=${shellQuote(options.deployCommit)}`,
|
||||
`script_path=${shellQuote(options.scriptPath)}`,
|
||||
`desired_ref=${shellQuote(options.desiredRef)}`,
|
||||
`environment=${shellQuote(options.environment)}`,
|
||||
`keep_namespace=${shellQuote(options.keepNamespace ? "true" : "false")}`,
|
||||
`timeout_ms=${shellQuote(String(scriptTimeoutMs))}`,
|
||||
"work_dir=\"/tmp/unidesk-ci/$run_id\"",
|
||||
"result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"",
|
||||
"mkdir -p \"$work_dir\" \"$result_dir\"",
|
||||
"launcher_log=\"$result_dir/launcher.log\"",
|
||||
"exec > >(tee -a \"$launcher_log\") 2>&1",
|
||||
"echo \"launcher_run_id=$run_id\"",
|
||||
"echo \"launcher_repo=$repo_url\"",
|
||||
"echo \"launcher_commit=$commit\"",
|
||||
"echo \"launcher_script_path=$script_path\"",
|
||||
"case \"$script_path\" in scripts/ci/*.sh) ;; *) echo \"invalid_script_path=$script_path\" >&2; exit 2 ;; esac",
|
||||
"export DOCKER_CONFIG=/tmp/unidesk-ci-docker-config",
|
||||
"mkdir -p \"$DOCKER_CONFIG\"",
|
||||
"printf '{}\\n' > \"$DOCKER_CONFIG/config.json\"",
|
||||
`build_proxy=${shellQuote(providerGatewayWsEgressProxyUrl)}`,
|
||||
"export HTTP_PROXY=\"$build_proxy\" HTTPS_PROXY=\"$build_proxy\" ALL_PROXY=\"$build_proxy\"",
|
||||
"export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal,.svc,.cluster.local,kubernetes.default.svc\"",
|
||||
"if ! curl -fsSI --max-time 20 -x \"$build_proxy\" https://github.com >/dev/null; then",
|
||||
" echo \"ci_provider_egress_proxy_unavailable=$build_proxy\" >&2",
|
||||
" exit 1",
|
||||
"fi",
|
||||
"echo \"ci_provider_egress_proxy=provider-gateway-ws-egress:$build_proxy\"",
|
||||
"repo_dir=\"$work_dir/repo\"",
|
||||
"if [ ! -d \"$repo_dir/.git\" ]; then",
|
||||
" git clone --no-checkout \"$repo_url\" \"$repo_dir\"",
|
||||
"fi",
|
||||
"git -C \"$repo_dir\" remote set-url origin \"$repo_url\"",
|
||||
"git -C \"$repo_dir\" fetch --no-tags origin \"$commit\" || git -C \"$repo_dir\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'",
|
||||
"resolved=$(git -C \"$repo_dir\" rev-parse --verify \"$commit^{commit}\")",
|
||||
"test \"$resolved\" = \"$commit\" || { echo \"resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
|
||||
"git -C \"$repo_dir\" cat-file -e \"$resolved:$script_path\"",
|
||||
"git -C \"$repo_dir\" show \"$resolved:$script_path\" > \"$work_dir/runner.sh\"",
|
||||
"chmod 700 \"$work_dir/runner.sh\"",
|
||||
"echo \"runner_script_ready=$work_dir/runner.sh\"",
|
||||
"runner_args=(",
|
||||
" --run-id \"$run_id\"",
|
||||
" --repo-url \"$repo_url\"",
|
||||
" --desired-ref \"$desired_ref\"",
|
||||
" --manifest-commit \"$commit\"",
|
||||
" --environment \"$environment\"",
|
||||
" --result-dir \"$result_dir\"",
|
||||
" --timeout-ms \"$timeout_ms\"",
|
||||
")",
|
||||
"if [ \"$keep_namespace\" = \"true\" ]; then runner_args+=(--keep-namespace); fi",
|
||||
"bash \"$work_dir/runner.sh\" \"${runner_args[@]}\"",
|
||||
].join("\n");
|
||||
return dispatchSsh(command, waitMs, remoteTimeoutMs, options.waitMs > 0);
|
||||
}
|
||||
|
||||
function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary {
|
||||
const remoteRef = `refs/remotes/origin/${desiredRef}`;
|
||||
const fetch = runCommand(["git", "fetch", "--quiet", "origin", `+refs/heads/${desiredRef}:${remoteRef}`], repoRoot);
|
||||
@@ -556,6 +611,8 @@ function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary
|
||||
if (record?.schemaVersion !== 2) throw new Error(`origin/${desiredRef}:deploy.json must use schemaVersion=2`);
|
||||
const environments = asRecord(record.environments);
|
||||
const dev = asRecord(environments?.dev);
|
||||
const ci = asRecord(dev?.ci);
|
||||
if (ci === null) throw new Error(`origin/${desiredRef}:deploy.json must contain environments.dev.ci`);
|
||||
const rawServices = Array.isArray(dev?.services) ? dev.services : [];
|
||||
const services = rawServices.map((item) => {
|
||||
const service = asRecord(item);
|
||||
@@ -570,6 +627,11 @@ function resolveDeployDevManifest(desiredRef: string): DeployDevManifestSummary
|
||||
deployCommit: deployCommitResult.stdout.trim(),
|
||||
desiredRef,
|
||||
environment: "dev",
|
||||
ci: {
|
||||
repo: requireManifestString(ci.repo, "environments.dev.ci.repo"),
|
||||
scriptPath: requireCiScriptPath(ci.scriptPath),
|
||||
timeoutMs: positiveManifestNumber(ci.timeoutMs, 1_800_000, "environments.dev.ci.timeoutMs"),
|
||||
},
|
||||
services,
|
||||
};
|
||||
}
|
||||
@@ -580,68 +642,62 @@ function makeRunId(deployCommit: string): string {
|
||||
}
|
||||
|
||||
async function runDevE2E(options: CiDevE2EOptions): Promise<Record<string, unknown>> {
|
||||
if (!options.direct) {
|
||||
const devopsResponse = coreInternalFetch("/api/microservices/devops/proxy/api/ci/dev-e2e/run", {
|
||||
method: "POST",
|
||||
body: {
|
||||
repoUrl: options.repoUrl,
|
||||
desiredRef: options.desiredRef,
|
||||
environment: options.environment,
|
||||
runId: options.runId,
|
||||
keepNamespace: options.keepNamespace,
|
||||
},
|
||||
maxResponseBytes: 2_000_000,
|
||||
});
|
||||
const devopsBody = proxyBody(devopsResponse);
|
||||
if (devopsBody?.ok === true) {
|
||||
const pipelineRun = asString(devopsBody.pipelineRun);
|
||||
const wait = pipelineRun.length > 0 ? await waitForPipelineRun(pipelineRun, options.waitMs) : null;
|
||||
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
|
||||
return {
|
||||
...devopsBody,
|
||||
ok: waitSucceeded,
|
||||
triggerMode: "devops-service",
|
||||
wait: wait === null ? null : {
|
||||
stdoutTail: wait.stdout.slice(-6000),
|
||||
stderrTail: wait.stderr.slice(-6000),
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
triggerMode: "devops-service",
|
||||
error: "DevOps service trigger failed or did not return ok=true; use --direct only for CI bootstrap/recovery, never as a service deployment path.",
|
||||
devopsResponse,
|
||||
};
|
||||
}
|
||||
const name = await remoteCreatePipelineRun(devE2EPipelineRunManifest(options));
|
||||
const wait = await waitForPipelineRun(name, options.waitMs);
|
||||
const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t");
|
||||
const result = await runRemoteDevE2ELauncher(options);
|
||||
const ok = result.ok && (result.exitCode === null || result.exitCode === 0);
|
||||
return {
|
||||
ok: waitSucceeded,
|
||||
pipelineRun: name,
|
||||
ok,
|
||||
runId: options.runId,
|
||||
namespace: "unidesk-ci",
|
||||
temporaryNamespace: `unidesk-ci-e2e-${options.runId}`,
|
||||
repoUrl: options.repoUrl,
|
||||
desiredRef: options.desiredRef,
|
||||
deployCommit: options.deployCommit,
|
||||
scriptRepo: options.scriptRepo,
|
||||
scriptPath: options.scriptPath,
|
||||
environment: options.environment,
|
||||
services: options.services,
|
||||
keepNamespace: options.keepNamespace,
|
||||
triggerMode: "direct-maintenance",
|
||||
wait: wait === null ? null : {
|
||||
stdoutTail: wait.stdout.slice(-6000),
|
||||
stderrTail: wait.stderr.slice(-6000),
|
||||
triggerMode: "commit-pinned-ssh-launcher",
|
||||
launcher: {
|
||||
taskId: result.taskId,
|
||||
status: result.status,
|
||||
exitCode: result.exitCode,
|
||||
stdoutTail: result.stdout.slice(-6000),
|
||||
stderrTail: result.stderr.slice(-6000),
|
||||
},
|
||||
resultDir: `/home/ubuntu/.unidesk/runs/${options.runId}`,
|
||||
next: [
|
||||
`bun scripts/cli.ts ci logs ${name}`,
|
||||
`bun scripts/cli.ts ci logs ${options.runId}`,
|
||||
"bun scripts/cli.ts ci status",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function logs(name: string): Promise<Record<string, unknown>> {
|
||||
if (name.length === 0) throw new Error("ci logs requires PipelineRun name");
|
||||
if (name.length === 0) throw new Error("ci logs requires run id or PipelineRun name");
|
||||
if (/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(name)) {
|
||||
const result = await dispatchSsh([
|
||||
"set -euo pipefail",
|
||||
`run_id=${shellQuote(name)}`,
|
||||
"result_dir=\"/home/ubuntu/.unidesk/runs/$run_id\"",
|
||||
"printf 'result_dir=%s\\n' \"$result_dir\"",
|
||||
"found=0",
|
||||
"if [ -f \"$result_dir/result.json\" ]; then found=1; echo '===== result.json'; cat \"$result_dir/result.json\"; fi",
|
||||
"if [ -f \"$result_dir/launcher.log\" ]; then found=1; echo '===== launcher.log'; tail -n 160 \"$result_dir/launcher.log\"; fi",
|
||||
"if [ -f \"$result_dir/runner.log\" ]; then found=1; echo '===== runner.log'; tail -n 240 \"$result_dir/runner.log\"; fi",
|
||||
"if [ -f \"$result_dir/pods.log\" ]; then found=1; echo '===== pods.log'; tail -n 240 \"$result_dir/pods.log\"; fi",
|
||||
"if [ \"$found\" = \"0\" ]; then echo \"no_run_files=$result_dir\" >&2; exit 42; fi",
|
||||
].join("\n"), 60_000, 45_000);
|
||||
if (result.ok || result.exitCode !== 42) {
|
||||
return {
|
||||
ok: result.ok,
|
||||
runId: name,
|
||||
output: result.stdout,
|
||||
stderr: result.stderr,
|
||||
exitCode: result.exitCode,
|
||||
};
|
||||
}
|
||||
}
|
||||
const result = await runRemoteKubectl([
|
||||
"set -euo pipefail",
|
||||
`kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`,
|
||||
@@ -664,7 +720,7 @@ function help(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts ci install",
|
||||
"bun scripts/cli.ts ci run --revision <commit>",
|
||||
"bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000",
|
||||
"bun scripts/cli.ts ci logs <pipelineRun>",
|
||||
"bun scripts/cli.ts ci logs <runId>",
|
||||
],
|
||||
tekton: {
|
||||
pipelineVersion: tektonPipelineVersion,
|
||||
@@ -676,9 +732,9 @@ function help(): Record<string, unknown> {
|
||||
},
|
||||
},
|
||||
runDevE2E: {
|
||||
defaultTriggerMode: "devops-service",
|
||||
directMaintenanceFlag: "--direct",
|
||||
defaultTriggerMode: "commit-pinned-ssh-launcher",
|
||||
desiredState: "origin/master:deploy.json#environments.dev",
|
||||
scriptSource: "origin/master:deploy.json#environments.dev.ci",
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -712,11 +768,13 @@ export async function runCiCommand(_config: UniDeskConfig, args: string[]): Prom
|
||||
desiredRef,
|
||||
deployCommit: manifest.deployCommit,
|
||||
environment: manifest.environment,
|
||||
scriptRepo: manifest.ci.repo,
|
||||
scriptPath: manifest.ci.scriptPath,
|
||||
scriptTimeoutMs: manifest.ci.timeoutMs,
|
||||
services: manifest.services,
|
||||
runId,
|
||||
keepNamespace: boolFlag(args, "--keep-namespace"),
|
||||
waitMs,
|
||||
direct: boolFlag(args, "--direct"),
|
||||
});
|
||||
}
|
||||
if (action === "logs") return logs(nameArg ?? "");
|
||||
|
||||
Reference in New Issue
Block a user