From 8511792601205ac4f03acc680916aae7a7aadc83 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 18 May 2026 09:26:06 +0000 Subject: [PATCH] fix: keep dev e2e repo fetch on host --- docs/reference/dev-ci-runner.md | 7 +- scripts/ci/dev-e2e.sh | 56 +++++++++- scripts/cli.ts | 31 ++++++ scripts/src/ci.ts | 7 +- .../k3s/ci/unidesk-ci.pipeline.yaml | 102 +++++++----------- 5 files changed, 132 insertions(+), 71 deletions(-) diff --git a/docs/reference/dev-ci-runner.md b/docs/reference/dev-ci-runner.md index 9b34a21b..0ef752fa 100644 --- a/docs/reference/dev-ci-runner.md +++ b/docs/reference/dev-ci-runner.md @@ -47,10 +47,10 @@ The automatic path is intentionally single and narrow: 3. CLI sends a short launcher through backend-core `/api/dispatch` using the existing `host.ssh` provider capability for D601. 4. D601 creates `/tmp/unidesk-ci/` and `/home/ubuntu/.unidesk/runs/`. 5. D601 fetches the manifest commit from GitHub through the node-local provider-gateway WS egress proxy at `http://127.0.0.1:18789`. -6. D601 extracts the runner with `git show : > /tmp/unidesk-ci//runner.sh` and executes it. -7. The runner creates the Tekton PipelineRun in `unidesk-ci`, waits for completion when requested, and writes `result.json`, `launcher.log`, `runner.log`, PipelineRun JSON and pod logs under `/home/ubuntu/.unidesk/runs//`. +6. D601 extracts the runner with `git show : > /tmp/unidesk-ci//runner.sh` and the desired-state blob with `git show :deploy.json > /tmp/unidesk-ci//deploy.json`. +7. The runner parses the host-fetched `deploy.json`, creates the Tekton PipelineRun in `unidesk-ci`, passes the required dev service commits as PipelineRun params, waits for completion when requested, and writes `result.json`, `launcher.log`, `runner.log`, PipelineRun JSON and pod logs under `/home/ubuntu/.unidesk/runs//`. -The CLI must not upload the runner script body. The submitted launcher may contain only repo, full commit, script path, run id, environment, timeout and keep-namespace settings plus the fixed fetch/execute wrapper. If k3s, Tekton or the provider egress proxy is unavailable, the run fails with visible logs; it must not fall back to an alternate deployment path. +The CLI must not upload the runner script body. Tekton dev e2e must not clone the private UniDesk repo itself; repo access and desired-state extraction happen once in the D601 host launcher under the manifest commit. The submitted launcher may contain only repo, full commit, script path, run id, environment, timeout, keep-namespace and fixed workspace path settings plus the fixed fetch/execute wrapper. If k3s, Tekton or the provider egress proxy is unavailable, the run fails with visible logs; it must not fall back to an alternate deployment path. ## Runner Contract @@ -62,6 +62,7 @@ scripts/ci/dev-e2e.sh \ --repo-url \ --desired-ref master \ --manifest-commit \ + --manifest-file /tmp/unidesk-ci//deploy.json \ --environment dev \ --result-dir /home/ubuntu/.unidesk/runs/ \ --timeout-ms \ diff --git a/scripts/ci/dev-e2e.sh b/scripts/ci/dev-e2e.sh index d33a17af..0f9f8763 100755 --- a/scripts/ci/dev-e2e.sh +++ b/scripts/ci/dev-e2e.sh @@ -9,11 +9,12 @@ environment="dev" result_dir="" timeout_ms="1800000" keep_namespace="false" +manifest_file="" usage() { cat <<'EOF' Usage: - dev-e2e.sh --run-id ID --manifest-commit COMMIT --result-dir DIR [--repo-url URL] [--desired-ref master] [--environment dev] [--timeout-ms MS] [--keep-namespace] + dev-e2e.sh --run-id ID --manifest-commit COMMIT --manifest-file FILE --result-dir DIR [--repo-url URL] [--desired-ref master] [--environment dev] [--timeout-ms MS] [--keep-namespace] This script runs the D601 dev namespace e2e harness from a Git-controlled blob. It must be launched by the CLI with a short command; do not paste this script @@ -47,6 +48,10 @@ while [ "$#" -gt 0 ]; do result_dir="${2:-}" shift 2 ;; + --manifest-file) + manifest_file="${2:-}" + shift 2 + ;; --timeout-ms) timeout_ms="${2:-}" shift 2 @@ -86,6 +91,10 @@ fi if [ -z "$result_dir" ]; then result_dir="/home/ubuntu/.unidesk/runs/$run_id" fi +if [ -z "$manifest_file" ] || [ ! -f "$manifest_file" ]; then + echo "--manifest-file must point to the commit-pinned deploy.json fetched by the launcher" >&2 + exit 2 +fi mkdir -p "$result_dir" runner_log="$result_dir/runner.log" @@ -144,6 +153,43 @@ log_json runner_started run_id "$run_id" manifest_commit "$manifest_commit" kubectl get pipeline/unidesk-dev-namespace-e2e -n unidesk-ci >/dev/null kubectl get pvc/unidesk-ci-cache -n unidesk-ci >/dev/null +service_env="$result_dir/dev-e2e-service-commits.env" +python3 - "$manifest_file" "$service_env" "$environment" <<'PY' +import json +import re +import sys + +manifest_path, out_path, environment = sys.argv[1:] +with open(manifest_path, "r", encoding="utf-8") as handle: + manifest = json.load(handle) +if manifest.get("schemaVersion") != 2: + raise SystemExit("deploy.json must use schemaVersion=2") +env = manifest.get("environments", {}).get(environment) +if not isinstance(env, dict): + raise SystemExit(f"deploy.json must contain environments.{environment}") +services = env.get("services") +if not isinstance(services, list) or not services: + raise SystemExit(f"deploy.json environments.{environment}.services must contain services") +lines = [] +summary = [] +for service in services: + if not isinstance(service, dict) or not service.get("id") or not service.get("repo") or not service.get("commitId"): + raise SystemExit(f"each deploy.json environments.{environment} service must contain id, repo and commitId") + key = re.sub(r"[^A-Z0-9]", "_", str(service["id"]).upper()) + commit = str(service["commitId"]) + lines.append(f"{key}_COMMIT={commit}") + summary.append({"id": service["id"], "commitId": commit}) +with open(out_path, "w", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") +print(json.dumps({"ok": True, "environment": environment, "services": summary}, ensure_ascii=False)) +PY +# shellcheck disable=SC1090 +source "$service_env" +backend_commit="${BACKEND_CORE_COMMIT:-unknown}" +frontend_commit="${FRONTEND_COMMIT:-unknown}" +code_queue_commit="${CODE_QUEUE_COMMIT:-unknown}" +deploy_json_b64="$(base64 -w0 "$manifest_file")" + pipeline_manifest="$result_dir/pipelinerun.yaml" cat >"$pipeline_manifest" <", + "bun scripts/cli.ts ssh argv [args...]", + "bun scripts/cli.ts ssh apply-patch < patch.diff", + "bun scripts/cli.ts ssh py [script-args...] < script.py", + "bun scripts/cli.ts ssh skills [--scope all|wsl|windows] [--limit N]", + "bun scripts/cli.ts ssh find [--contains TEXT] [--limit N]", + "bun scripts/cli.ts ssh glob [--root DIR] [--pattern PATTERN]", + ], + notes: [ + "ssh --help and ssh --help print this JSON help and never open an interactive session.", + "Use argv when nested shell quoting would be fragile.", + "Use -- before a remote command that intentionally starts with a dash.", + ], + }; +} + function numberOption(name: string, defaultValue: number): number { const index = args.indexOf(name); if (index === -1) return defaultValue; @@ -147,6 +173,11 @@ async function main(): Promise { return; } + if (top === "ssh" && (sub === undefined || isHelpToken(sub) || (isHelpToken(third) && args.length === 3))) { + emitJson(commandName, sshHelp()); + return; + } + if (top === "internal" && sub === "run-job") { if (!third) throw new Error("internal run-job requires job id"); emitJson(commandName, await runJob(third)); diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index 918afa2c..a55ca97b 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -593,6 +593,7 @@ async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise&2; exit 1; }", "git -C \"$repo_dir\" cat-file -e \"$resolved:$script_path\"", "git -C \"$repo_dir\" show \"$resolved:$script_path\" > \"$work_dir/runner.sh\"", + "git -C \"$repo_dir\" show \"$resolved:deploy.json\" > \"$work_dir/deploy.json\"", "chmod 700 \"$work_dir/runner.sh\"", "echo \"runner_script_ready=$work_dir/runner.sh\"", "runner_args=(", @@ -600,6 +601,7 @@ async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise/dev/null || true", ].join("\n"), 30_000, 20_000); latest = result; - if (result.ok && result.stdout.trimStart().startsWith("{")) { - const parsed = JSON.parse(result.stdout) as { ok?: boolean; status?: string }; + const stdout = result.stdout.trimStart(); + if (stdout.startsWith("{")) { + const parsed = JSON.parse(stdout) as { ok?: boolean; status?: string }; return { ...result, ok: parsed.ok === true, diff --git a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml index a6a6ebe8..eebfd8d6 100644 --- a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml +++ b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml @@ -654,45 +654,21 @@ spec: - name: app-image type: string default: unidesk-code-queue:dev + - name: backend-core-commit + type: string + default: unknown + - name: frontend-commit + type: string + default: unknown + - name: code-queue-commit + type: string + default: unknown + - name: deploy-json-b64 + type: string + default: "" workspaces: - name: source steps: - - name: clone-deploy-manifest - image: alpine/git:2.45.2 - env: - - name: HTTP_PROXY - value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" - - name: HTTPS_PROXY - value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" - - name: ALL_PROXY - value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" - - name: NO_PROXY - value: "localhost,127.0.0.1,::1,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local" - - name: http_proxy - value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" - - name: https_proxy - value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" - - name: all_proxy - value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" - - name: no_proxy - value: "localhost,127.0.0.1,::1,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local" - script: | - #!/bin/sh - set -eu - run_dir="$(workspaces.source.path)/dev-e2e-$(params.run-id)" - rm -rf "$run_dir" - mkdir -p "$run_dir" - git clone --filter=blob:none --no-checkout "$(params.repo-url)" "$run_dir/repo" - cd "$run_dir/repo" - git fetch --depth=1 origin "$(params.desired-ref)" - actual="$(git rev-parse FETCH_HEAD)" - expected="$(params.deploy-commit)" - if [ "$actual" != "$expected" ]; then - echo "desired ref commit mismatch actual=$actual expected=$expected" >&2 - exit 1 - fi - git show FETCH_HEAD:deploy.json >"$run_dir/deploy.json" - printf '%s\n' "$actual" >"$run_dir/deploy-commit.txt" - name: namespace-smoke-e2e image: "$(params.app-image)" imagePullPolicy: Never @@ -719,8 +695,12 @@ spec: ns="unidesk-ci-e2e-$(params.run-id)" keep="$(params.keep-namespace)" run_dir="$(workspaces.source.path)/dev-e2e-$(params.run-id)" - deploy_json="$run_dir/deploy.json" + mkdir -p "$run_dir" result_json="$run_dir/dev-e2e-result.json" + deploy_json_b64="$(params.deploy-json-b64)" + backend_commit="$(params.backend-core-commit)" + frontend_commit="$(params.frontend-commit)" + code_queue_commit="$(params.code-queue-commit)" kube_api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}" kube_token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" kube_ca="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" @@ -749,33 +729,6 @@ spec: } trap cleanup EXIT - bun - "$deploy_json" "$run_dir/dev-e2e-service-commits.env" "$(params.environment)" <<'BUN' - const [deployPath, outPath, environment] = process.argv.slice(2); - const manifest = await Bun.file(deployPath).json(); - if (!manifest || manifest.schemaVersion !== 2 || !manifest.environments?.[environment]) { - throw new Error(`deploy.json must contain schemaVersion=2 environments.${environment}`); - } - const services = manifest.environments[environment].services; - if (!Array.isArray(services) || services.length === 0) { - throw new Error(`deploy.json environments.${environment}.services must contain services`); - } - const lines = []; - for (const service of services) { - if (!service?.id || !service?.commitId || !service?.repo) { - throw new Error(`each deploy.json environments.${environment} service must contain id, repo and commitId`); - } - const key = String(service.id).toUpperCase().replace(/[^A-Z0-9]/g, "_"); - lines.push(`${key}_COMMIT=${service.commitId}`); - } - await Bun.write(outPath, lines.join("\n") + "\n"); - console.log(JSON.stringify({ ok: true, environment, services: services.map((service) => ({ id: service.id, commitId: service.commitId })) })); - BUN - # shellcheck disable=SC1091 - source "$run_dir/dev-e2e-service-commits.env" - backend_commit="${BACKEND_CORE_COMMIT:-unknown}" - frontend_commit="${FRONTEND_COMMIT:-unknown}" - code_queue_commit="${CODE_QUEUE_COMMIT:-unknown}" - delete_path "api/v1/namespaces/$ns" cat >/tmp/dev-e2e-namespace.yaml </dev/null - deploy_json_b64="$(base64 -w0 "$deploy_json")" cat >/tmp/dev-e2e-configmap.yaml <