mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: keep dev e2e repo fetch on host

Browse Source
This commit is contained in:
Codex
2026-05-18 09:26:06 +00:00
parent 613e5a1742
commit 8511792601
5 changed files with 132 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/reference/dev-ci-runner.md
+4 -3
View File
@@ -47,10 +47,10 @@ The automatic path is intentionally single and narrow:
3. CLI sends a short launcher through backend-core `/api/dispatch` using the existing `host.ssh` provider capability for D601.
4. D601 creates `/tmp/unidesk-ci/<runId>` and `/home/ubuntu/.unidesk/runs/<runId>`.
5. D601 fetches the manifest commit from GitHub through the node-local provider-gateway WS egress proxy at `http://127.0.0.1:18789`.
6. D601 extracts the runner with `git show <commit>:<scriptPath> > /tmp/unidesk-ci/<runId>/runner.sh` and executes it.
7. The runner creates the Tekton PipelineRun in `unidesk-ci`, waits for completion when requested, and writes `result.json`, `launcher.log`, `runner.log`, PipelineRun JSON and pod logs under `/home/ubuntu/.unidesk/runs/<runId>/`.
6. D601 extracts the runner with `git show <commit>:<scriptPath> > /tmp/unidesk-ci/<runId>/runner.sh` and the desired-state blob with `git show <commit>:deploy.json > /tmp/unidesk-ci/<runId>/deploy.json`.
7. The runner parses the host-fetched `deploy.json`, creates the Tekton PipelineRun in `unidesk-ci`, passes the required dev service commits as PipelineRun params, waits for completion when requested, and writes `result.json`, `launcher.log`, `runner.log`, PipelineRun JSON and pod logs under `/home/ubuntu/.unidesk/runs/<runId>/`.
The CLI must not upload the runner script body. The submitted launcher may contain only repo, full commit, script path, run id, environment, timeout and keep-namespace settings plus the fixed fetch/execute wrapper. If k3s, Tekton or the provider egress proxy is unavailable, the run fails with visible logs; it must not fall back to an alternate deployment path.
The CLI must not upload the runner script body. Tekton dev e2e must not clone the private UniDesk repo itself; repo access and desired-state extraction happen once in the D601 host launcher under the manifest commit. The submitted launcher may contain only repo, full commit, script path, run id, environment, timeout, keep-namespace and fixed workspace path settings plus the fixed fetch/execute wrapper. If k3s, Tekton or the provider egress proxy is unavailable, the run fails with visible logs; it must not fall back to an alternate deployment path.
## Runner Contract
@@ -62,6 +62,7 @@ scripts/ci/dev-e2e.sh \
--repo-url <repo> \
--desired-ref master \
--manifest-commit <full-sha> \
--manifest-file /tmp/unidesk-ci/<runId>/deploy.json \
--environment dev \
--result-dir /home/ubuntu/.unidesk/runs/<runId> \
--timeout-ms <ms> \
scripts/ci/dev-e2e.sh
+55 -1
View File
@@ -9,11 +9,12 @@ environment="dev"
result_dir=""
timeout_ms="1800000"
keep_namespace="false"
manifest_file=""
usage() {
cat <<'EOF'
Usage:
dev-e2e.sh --run-id ID --manifest-commit COMMIT --result-dir DIR [--repo-url URL] [--desired-ref master] [--environment dev] [--timeout-ms MS] [--keep-namespace]
dev-e2e.sh --run-id ID --manifest-commit COMMIT --manifest-file FILE --result-dir DIR [--repo-url URL] [--desired-ref master] [--environment dev] [--timeout-ms MS] [--keep-namespace]
This script runs the D601 dev namespace e2e harness from a Git-controlled blob.
It must be launched by the CLI with a short command; do not paste this script
@@ -47,6 +48,10 @@ while [ "$#" -gt 0 ]; do
result_dir="${2:-}"
shift 2
;;
--manifest-file)
manifest_file="${2:-}"
shift 2
;;
--timeout-ms)
timeout_ms="${2:-}"
shift 2
@@ -86,6 +91,10 @@ fi
if [ -z "$result_dir" ]; then
result_dir="/home/ubuntu/.unidesk/runs/$run_id"
fi
if [ -z "$manifest_file" ] || [ ! -f "$manifest_file" ]; then
echo "--manifest-file must point to the commit-pinned deploy.json fetched by the launcher" >&2
exit 2
fi
mkdir -p "$result_dir"
runner_log="$result_dir/runner.log"
@@ -144,6 +153,43 @@ log_json runner_started run_id "$run_id" manifest_commit "$manifest_commit"
kubectl get pipeline/unidesk-dev-namespace-e2e -n unidesk-ci >/dev/null
kubectl get pvc/unidesk-ci-cache -n unidesk-ci >/dev/null
service_env="$result_dir/dev-e2e-service-commits.env"
python3 - "$manifest_file" "$service_env" "$environment" <<'PY'
import json
import re
import sys
manifest_path, out_path, environment = sys.argv[1:]
with open(manifest_path, "r", encoding="utf-8") as handle:
manifest = json.load(handle)
if manifest.get("schemaVersion") != 2:
raise SystemExit("deploy.json must use schemaVersion=2")
env = manifest.get("environments", {}).get(environment)
if not isinstance(env, dict):
raise SystemExit(f"deploy.json must contain environments.{environment}")
services = env.get("services")
if not isinstance(services, list) or not services:
raise SystemExit(f"deploy.json environments.{environment}.services must contain services")
lines = []
summary = []
for service in services:
if not isinstance(service, dict) or not service.get("id") or not service.get("repo") or not service.get("commitId"):
raise SystemExit(f"each deploy.json environments.{environment} service must contain id, repo and commitId")
key = re.sub(r"[^A-Z0-9]", "_", str(service["id"]).upper())
commit = str(service["commitId"])
lines.append(f"{key}_COMMIT={commit}")
summary.append({"id": service["id"], "commitId": commit})
with open(out_path, "w", encoding="utf-8") as handle:
handle.write("\n".join(lines) + "\n")
print(json.dumps({"ok": True, "environment": environment, "services": summary}, ensure_ascii=False))
PY
# shellcheck disable=SC1090
source "$service_env"
backend_commit="${BACKEND_CORE_COMMIT:-unknown}"
frontend_commit="${FRONTEND_COMMIT:-unknown}"
code_queue_commit="${CODE_QUEUE_COMMIT:-unknown}"
deploy_json_b64="$(base64 -w0 "$manifest_file")"
pipeline_manifest="$result_dir/pipelinerun.yaml"
cat >"$pipeline_manifest" <<YAML
apiVersion: tekton.dev/v1
@@ -175,6 +221,14 @@ spec:
value: "$run_id"
- name: keep-namespace
value: "$keep_namespace"
- name: backend-core-commit
value: "$backend_commit"
- name: frontend-commit
value: "$frontend_commit"
- name: code-queue-commit
value: "$code_queue_commit"
- name: deploy-json-b64
value: "$deploy_json_b64"
workspaces:
- name: shared-workspace
persistentVolumeClaim:
scripts/cli.ts
+31
View File
@@ -81,6 +81,32 @@ function help(): unknown {
};
}
function isHelpToken(value: string | undefined): boolean {
return value === "help" || value === "--help" || value === "-h";
}
function sshHelp(): unknown {
return {
command: "ssh",
output: "json",
description: "Open a Host SSH / WSL SSH maintenance session through the provider-gateway bridge.",
usage: [
"bun scripts/cli.ts ssh <providerId>",
"bun scripts/cli.ts ssh <providerId> argv <command> [args...]",
"bun scripts/cli.ts ssh <providerId> apply-patch < patch.diff",
"bun scripts/cli.ts ssh <providerId> py [script-args...] < script.py",
"bun scripts/cli.ts ssh <providerId> skills [--scope all|wsl|windows] [--limit N]",
"bun scripts/cli.ts ssh <providerId> find <path...> [--contains TEXT] [--limit N]",
"bun scripts/cli.ts ssh <providerId> glob [--root DIR] [--pattern PATTERN]",
],
notes: [
"ssh --help and ssh <providerId> --help print this JSON help and never open an interactive session.",
"Use argv when nested shell quoting would be fragile.",
"Use -- before a remote command that intentionally starts with a dash.",
],
};
}
function numberOption(name: string, defaultValue: number): number {
const index = args.indexOf(name);
if (index === -1) return defaultValue;
@@ -147,6 +173,11 @@ async function main(): Promise<void> {
return;
}
if (top === "ssh" && (sub === undefined || isHelpToken(sub) || (isHelpToken(third) && args.length === 3))) {
emitJson(commandName, sshHelp());
return;
}
if (top === "internal" && sub === "run-job") {
if (!third) throw new Error("internal run-job requires job id");
emitJson(commandName, await runJob(third));
scripts/src/ci.ts
+5 -2
View File
@@ -593,6 +593,7 @@ async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise<Dispat
"test \"$resolved\" = \"$commit\" || { echo \"resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }",
"git -C \"$repo_dir\" cat-file -e \"$resolved:$script_path\"",
"git -C \"$repo_dir\" show \"$resolved:$script_path\" > \"$work_dir/runner.sh\"",
"git -C \"$repo_dir\" show \"$resolved:deploy.json\" > \"$work_dir/deploy.json\"",
"chmod 700 \"$work_dir/runner.sh\"",
"echo \"runner_script_ready=$work_dir/runner.sh\"",
"runner_args=(",
@@ -600,6 +601,7 @@ async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise<Dispat
" --repo-url \"$repo_url\"",
" --desired-ref \"$desired_ref\"",
" --manifest-commit \"$commit\"",
" --manifest-file \"$work_dir/deploy.json\"",
" --environment \"$environment\"",
" --result-dir \"$result_dir\"",
" --timeout-ms \"$timeout_ms\"",
@@ -629,8 +631,9 @@ async function waitForDevE2EResult(runId: string, waitMs: number): Promise<Dispa
"tail -n 80 \"$result_dir/runner.log\" 2>/dev/null || true",
].join("\n"), 30_000, 20_000);
latest = result;
if (result.ok && result.stdout.trimStart().startsWith("{")) {
const parsed = JSON.parse(result.stdout) as { ok?: boolean; status?: string };
const stdout = result.stdout.trimStart();
if (stdout.startsWith("{")) {
const parsed = JSON.parse(stdout) as { ok?: boolean; status?: string };
return {
...result,
ok: parsed.ok === true,
src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml
+37 -65
View File
@@ -654,45 +654,21 @@ spec:
- name: app-image
type: string
default: unidesk-code-queue:dev
- name: backend-core-commit
type: string
default: unknown
- name: frontend-commit
type: string
default: unknown
- name: code-queue-commit
type: string
default: unknown
- name: deploy-json-b64
type: string
default: ""
workspaces:
- name: source
steps:
- name: clone-deploy-manifest
image: alpine/git:2.45.2
env:
- name: HTTP_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: HTTPS_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: ALL_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: NO_PROXY
value: "localhost,127.0.0.1,::1,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local"
- name: http_proxy
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: https_proxy
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: all_proxy
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: no_proxy
value: "localhost,127.0.0.1,::1,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local"
script: |
#!/bin/sh
set -eu
run_dir="$(workspaces.source.path)/dev-e2e-$(params.run-id)"
rm -rf "$run_dir"
mkdir -p "$run_dir"
git clone --filter=blob:none --no-checkout "$(params.repo-url)" "$run_dir/repo"
cd "$run_dir/repo"
git fetch --depth=1 origin "$(params.desired-ref)"
actual="$(git rev-parse FETCH_HEAD)"
expected="$(params.deploy-commit)"
if [ "$actual" != "$expected" ]; then
echo "desired ref commit mismatch actual=$actual expected=$expected" >&2
exit 1
fi
git show FETCH_HEAD:deploy.json >"$run_dir/deploy.json"
printf '%s\n' "$actual" >"$run_dir/deploy-commit.txt"
- name: namespace-smoke-e2e
image: "$(params.app-image)"
imagePullPolicy: Never
@@ -719,8 +695,12 @@ spec:
ns="unidesk-ci-e2e-$(params.run-id)"
keep="$(params.keep-namespace)"
run_dir="$(workspaces.source.path)/dev-e2e-$(params.run-id)"
deploy_json="$run_dir/deploy.json"
mkdir -p "$run_dir"
result_json="$run_dir/dev-e2e-result.json"
deploy_json_b64="$(params.deploy-json-b64)"
backend_commit="$(params.backend-core-commit)"
frontend_commit="$(params.frontend-commit)"
code_queue_commit="$(params.code-queue-commit)"
kube_api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
kube_token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
kube_ca="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
@@ -749,33 +729,6 @@ spec:
}
trap cleanup EXIT
bun - "$deploy_json" "$run_dir/dev-e2e-service-commits.env" "$(params.environment)" <<'BUN'
const [deployPath, outPath, environment] = process.argv.slice(2);
const manifest = await Bun.file(deployPath).json();
if (!manifest || manifest.schemaVersion !== 2 || !manifest.environments?.[environment]) {
throw new Error(`deploy.json must contain schemaVersion=2 environments.${environment}`);
}
const services = manifest.environments[environment].services;
if (!Array.isArray(services) || services.length === 0) {
throw new Error(`deploy.json environments.${environment}.services must contain services`);
}
const lines = [];
for (const service of services) {
if (!service?.id || !service?.commitId || !service?.repo) {
throw new Error(`each deploy.json environments.${environment} service must contain id, repo and commitId`);
}
const key = String(service.id).toUpperCase().replace(/[^A-Z0-9]/g, "_");
lines.push(`${key}_COMMIT=${service.commitId}`);
}
await Bun.write(outPath, lines.join("\n") + "\n");
console.log(JSON.stringify({ ok: true, environment, services: services.map((service) => ({ id: service.id, commitId: service.commitId })) }));
BUN
# shellcheck disable=SC1091
source "$run_dir/dev-e2e-service-commits.env"
backend_commit="${BACKEND_CORE_COMMIT:-unknown}"
frontend_commit="${FRONTEND_COMMIT:-unknown}"
code_queue_commit="${CODE_QUEUE_COMMIT:-unknown}"
delete_path "api/v1/namespaces/$ns"
cat >/tmp/dev-e2e-namespace.yaml <<YAML
apiVersion: v1
@@ -792,7 +745,6 @@ spec:
--data-binary @/tmp/dev-e2e-namespace.yaml \
"$kube_api/api/v1/namespaces/$ns?fieldManager=unidesk-ci&force=true" >/dev/null
deploy_json_b64="$(base64 -w0 "$deploy_json")"
cat >/tmp/dev-e2e-configmap.yaml <<YAML
apiVersion: v1
kind: ConfigMap
@@ -1002,6 +954,18 @@ spec:
- name: app-image
type: string
default: unidesk-code-queue:dev
- name: backend-core-commit
type: string
default: unknown
- name: frontend-commit
type: string
default: unknown
- name: code-queue-commit
type: string
default: unknown
- name: deploy-json-b64
type: string
default: ""
workspaces:
- name: shared-workspace
tasks:
@@ -1023,6 +987,14 @@ spec:
value: "$(params.keep-namespace)"
- name: app-image
value: "$(params.app-image)"
- name: backend-core-commit
value: "$(params.backend-core-commit)"
- name: frontend-commit
value: "$(params.frontend-commit)"
- name: code-queue-commit
value: "$(params.code-queue-commit)"
- name: deploy-json-b64
value: "$(params.deploy-json-b64)"
workspaces:
- name: source
workspace: shared-workspace