From 0c0c979a564195973cca8b2d7662788b0b0d5ccd Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 18 May 2026 13:06:14 +0000 Subject: [PATCH] chore: pin dev frontend auth fix --- deploy.json | 4 +- docs/reference/ci.md | 3 +- docs/reference/dev-ci-runner.md | 12 +- docs/reference/dev-environment.md | 8 +- scripts/ci/dev-e2e.sh | 106 ++- scripts/src/ci.ts | 7 + scripts/src/code-queue.ts | 10 +- scripts/src/e2e.ts | 174 +++- src/components/frontend/src/code-queue.tsx | 21 +- .../code-queue-mgr/src-rs/main.rs | 38 +- .../microservices/code-queue-mgr/src/index.ts | 18 +- .../k3s/ci/unidesk-ci.pipeline.yaml | 752 +++++++++++++++--- 12 files changed, 993 insertions(+), 160 deletions(-) diff --git a/deploy.json b/deploy.json index 96062f34..5a0c1968 100644 --- a/deploy.json +++ b/deploy.json @@ -80,12 +80,12 @@ { "id": "backend-core", "repo": "https://github.com/pikasTech/unidesk", - "commitId": "51c1576aa3c535210635bb5d602eb774c8b1822e" + "commitId": "c09beb09e8f7e72cfe8dc7c1379d39f7facbfb3a" }, { "id": "frontend", "repo": "https://github.com/pikasTech/unidesk", - "commitId": "51c1576aa3c535210635bb5d602eb774c8b1822e" + "commitId": "c09beb09e8f7e72cfe8dc7c1379d39f7facbfb3a" } ] } diff --git a/docs/reference/ci.md b/docs/reference/ci.md index 5bce9328..f0cb4e37 100644 --- a/docs/reference/ci.md +++ b/docs/reference/ci.md @@ -21,6 +21,7 @@ Each commit CI run performs: - `UNIDESK_D601_RUST_CHECK=1 bun scripts/cli.ts check --full --rust`, so Rust backend-core is checked only inside the D601 CI execution boundary. - Temporary `code-queue-ci-read` Deployment and ClusterIP Service in `unidesk-ci`. - Code Queue read performance checks against the production PostgreSQL through `d601-tcp-egress-gateway`. +- Manual dev desired-state smoke for Code Queue via `ci run-dev-e2e`, using the Git-pinned `code-queue` service commit from `origin/master:deploy.json#environments.dev`. `ci install` also prewarms the D601 k3s containerd runtime with the Tekton entrypoint/workingdir helper images, `oven/bun:1-debian`, `alpine/git:2.45.2` and `unidesk-code-queue:dev`. Missing images are pulled through the node-local provider-gateway WS egress proxy and then imported into native k3s containerd with digests preserved, so PipelineRun pods do not hang on external registry pulls. Sustained pull throughput below 1 MB/s is treated as a provider/main-server network or proxy degradation first, not as a Dockerfile or application failure. @@ -48,7 +49,7 @@ This means the CI service can read existing tasks, Trace summaries, Trace steps `ci run-dev-e2e` is the manual dev desired-state smoke flow. The single authoritative reference for its Git-controlled runner script, short launcher, result directory and no-CD boundary is `docs/reference/dev-ci-runner.md`. -The current dev namespace e2e is a harness and smoke gate, not a full frontend/backend/code-queue stack rollout. Full-stack temporary namespace deployment can be added behind the same command only after image build/import and per-run database bootstrap are promoted into a controlled deployment design. +The current dev namespace e2e is a harness and smoke gate, not a full frontend/backend stack rollout. It does include a controlled Code Queue slice: D601 builds or reuses the `environments.dev.services[].id=code-queue` commit, imports the image into native k3s containerd, starts temporary PostgreSQL plus Code Queue scheduler/read/write Services in `unidesk-ci-e2e-`, and verifies the HTTP API through the Kubernetes API service proxy. This remains CI-only and must not deploy persistent `unidesk-dev` or production resources. ## Performance Gate diff --git a/docs/reference/dev-ci-runner.md b/docs/reference/dev-ci-runner.md index ac1b15fb..dc24b7cb 100644 --- a/docs/reference/dev-ci-runner.md +++ b/docs/reference/dev-ci-runner.md @@ -15,6 +15,7 @@ The runner exists to prove the dev desired state without interrupting production - D601 execution: Git fetch, Tekton PipelineRun creation, Kubernetes polling and e2e log collection happen on D601, not on the main master. - CLI observability: the submit command returns a `runId`, result directory and next commands; `ci logs ` can recover status after the local CLI exits. - CI only: the flow may create CI-owned temporary resources, but it must not deploy backend-core, frontend, Code Queue, Decision Center, k3sctl-adapter or any other direct/managed service. +- Code Queue reproducibility: the runner must use the `code-queue` commit from `environments.dev.services`, build or reuse a labeled image from that Git commit on D601, import it into native k3s containerd, and validate the HTTP API inside a temporary namespace. ## Design Boundary @@ -54,6 +55,11 @@ Do not add a long-lived DevOps service, run broker, webhook listener or second d "id": "frontend", "repo": "https://github.com/pikasTech/unidesk", "commitId": "" + }, + { + "id": "code-queue", + "repo": "https://github.com/pikasTech/unidesk", + "commitId": "" } ] } @@ -61,7 +67,7 @@ Do not add a long-lived DevOps service, run broker, webhook listener or second d } ``` -`scriptPath` must be a repo-relative `scripts/ci/*.sh` path. Inline shell bodies, arbitrary script paths, local dirty scripts and separate `develop.json` or CI manifest files are forbidden. The script is fetched from the same full 40-character manifest commit that supplied `deploy.json`, so the runner logic is auditable and rollbackable with the desired state. Persistent dev rollout service scope is owned by `docs/reference/dev-environment.md`; this runner only consumes the dev service list for smoke verification and must not deploy it. +`scriptPath` must be a repo-relative `scripts/ci/*.sh` path. Inline shell bodies, arbitrary script paths, local dirty scripts and separate `develop.json` or CI manifest files are forbidden. The script is fetched from the same full 40-character manifest commit that supplied `deploy.json`, so the runner logic is auditable and rollbackable with the desired state. Persistent dev rollout service scope is owned by `docs/reference/dev-environment.md`; this runner only consumes the dev service list for smoke verification and must not deploy it. `code-queue` is required in the dev service list for this smoke runner, but that does not enable `deploy apply --env dev --service code-queue`. ## Execution Path @@ -73,7 +79,7 @@ The automatic path is intentionally single and narrow: 4. D601 creates `/tmp/unidesk-ci/` and `/home/ubuntu/.unidesk/runs/`. 5. D601 fetches the manifest commit from GitHub through the node-local provider-gateway WS egress proxy at `http://127.0.0.1:18789`. 6. D601 extracts the runner with `git show : > /tmp/unidesk-ci//runner.sh` and the desired-state blob with `git show :deploy.json > /tmp/unidesk-ci//deploy.json`. -7. The runner parses the host-fetched `deploy.json`, creates the Tekton PipelineRun in `unidesk-ci`, passes the required dev service commits as PipelineRun params, waits for completion when requested, and writes `result.json`, `launcher.log`, `runner.log`, PipelineRun JSON and pod logs under `/home/ubuntu/.unidesk/runs//`. +7. The runner parses the host-fetched `deploy.json`, requires a full-SHA `code-queue` service commit, builds or reuses a D601 Docker image for that commit, imports the image and `postgres:16-alpine` into native k3s containerd, creates the Tekton PipelineRun in `unidesk-ci`, passes the required dev service commits and Code Queue image tag as PipelineRun params, waits for completion when requested, and writes `result.json`, `launcher.log`, `runner.log`, PipelineRun JSON and pod logs under `/home/ubuntu/.unidesk/runs//`. The CLI must not upload the runner script body. Tekton dev e2e must not clone the private UniDesk repo itself; repo access and desired-state extraction happen once in the D601 host launcher under the manifest commit. The submitted launcher may contain only repo, full commit, script path, run id, environment, timeout, keep-namespace and fixed workspace path settings plus the fixed fetch/execute wrapper. If k3s, Tekton or the provider egress proxy is unavailable, the run fails with visible logs; it must not fall back to an alternate deployment path. @@ -96,6 +102,8 @@ scripts/ci/dev-e2e.sh \ The current script creates a Tekton `PipelineRun` for `pipeline/unidesk-dev-namespace-e2e`, stores the generated PipelineRun name in `pipelinerun.txt`, and writes a final `result.json` with `ok`, `status`, `runId`, `manifestCommit`, `pipelineRun`, `temporaryNamespace` and `finishedAt`. +The Tekton task creates a temporary namespace `unidesk-ci-e2e-` and may create only CI-owned smoke resources there: `postgres-dev`, `code-queue-scheduler-dev`, `code-queue-read-dev`, `code-queue-write-dev`, their ClusterIP Services and a per-run Secret/ConfigMap. It must not mutate `unidesk` or persistent `unidesk-dev`. Code Queue API validation must use ClusterIP Services and the Kubernetes API `services/.../proxy` subresource; NodePort, D601 host ports and direct public service exposure are forbidden. The smoke currently proves `/health`, `/live` and `/api/workdirs` GET/POST/DELETE on read/write/scheduler roles, giving follow-up Code Queue API fixes a reproducible CI target before production rollout. + ## Commands Start a run and return after dispatch: diff --git a/docs/reference/dev-environment.md b/docs/reference/dev-environment.md index 52d9f521..b907fad6 100644 --- a/docs/reference/dev-environment.md +++ b/docs/reference/dev-environment.md @@ -30,14 +30,14 @@ The unrestricted public network entries are therefore production frontend, dev f ## Desired State -`deploy.json` remains the only version intent file. Dev entries live under `environments.dev` and are read from `origin/master:deploy.json`, never from a dirty local file, when using `--env dev`. +`deploy.json` remains the only version intent file. Dev entries live under `environments.dev` and are read from `origin/master:deploy.json`, never from a dirty local file, when using `--env dev` or `ci run-dev-e2e`. The persistent dev rollout currently supports only: - `backend-core` - `frontend` -`code-queue`, Decision Center, k3sctl-adapter and other D601 services are not part of persistent dev apply yet. Their smoke validation stays under `ci run-dev-e2e` or service-specific future designs. The `environments.dev.ci` declaration and short launcher runner are owned by `docs/reference/dev-ci-runner.md`. +`code-queue` is present in `environments.dev.services` only so `ci run-dev-e2e` can build a Git-pinned Code Queue image and run a temporary namespace smoke. It is not part of persistent dev apply: `deploy apply --env dev --service code-queue` must still be rejected. Decision Center, k3sctl-adapter and other D601 services are not part of persistent dev apply yet. Their smoke validation stays under `ci run-dev-e2e` or service-specific future designs. The `environments.dev.ci` declaration and short launcher runner are owned by `docs/reference/dev-ci-runner.md`. ## Rust Backend-Core Boundary @@ -94,7 +94,7 @@ Use this sequence for backend-core Rust and frontend dev work: 7. If the dev service catalog changes, deploy the pushed `k3sctl-adapter` commit through the controlled local manifest exception, then verify `/api/control-plane` lists `k3s/dev/unidesk-dev-core.k3s.json`. 8. Rebuild or verify `dev-frontend-proxy` on the main server with `bun scripts/cli.ts server rebuild dev-frontend-proxy` when the proxy config or port changes. 9. Manually test `http://74.48.78.17:18083/` and the dev health endpoints. -10. Run D601 CI for the commit and the dev smoke runner: `bun scripts/cli.ts ci run --revision --wait-ms ` and `bun scripts/cli.ts ci run-dev-e2e --wait-ms `. +10. Run D601 CI for the commit and the dev smoke runner: `bun scripts/cli.ts ci run --revision --wait-ms ` and `bun scripts/cli.ts ci run-dev-e2e --wait-ms `. When Code Queue behavior changes, update the `code-queue` entry in `environments.dev.services` to the pushed commit before running the dev smoke; do not use `deploy apply --env dev --service code-queue`. ## Validation Commands @@ -105,6 +105,8 @@ bun scripts/cli.ts server status bun scripts/cli.ts deploy plan --env dev bun scripts/cli.ts deploy plan --env dev --service backend-core bun scripts/cli.ts dev-env validate --manifest src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-core.k8s.yaml +bun scripts/cli.ts dev-env validate --manifest src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-code-queue.k8s.yaml +bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000 bun scripts/cli.ts microservice proxy k3sctl-adapter /api/services/backend-core-dev/proxy/health --raw --full bun scripts/cli.ts microservice proxy k3sctl-adapter /api/services/frontend-dev/proxy/health --raw --full curl -fsS http://127.0.0.1:18083/health diff --git a/scripts/ci/dev-e2e.sh b/scripts/ci/dev-e2e.sh index 0f9f8763..d453073b 100755 --- a/scripts/ci/dev-e2e.sh +++ b/scripts/ci/dev-e2e.sh @@ -116,12 +116,12 @@ write_result() { local ok="$1" local status="$2" local detail="$3" - python3 - "$result_json" "$ok" "$status" "$detail" "$run_id" "$repo_url" "$desired_ref" "$manifest_commit" "$environment" "$pipeline_run" "$temporary_namespace" <<'PY' + python3 - "$result_json" "$ok" "$status" "$detail" "$run_id" "$repo_url" "$desired_ref" "$manifest_commit" "$environment" "$pipeline_run" "$temporary_namespace" "$code_queue_image" <<'PY' import json import sys from datetime import datetime, timezone -path, ok, status, detail, run_id, repo, desired_ref, commit, environment, pipeline_run, temporary_namespace = sys.argv[1:] +path, ok, status, detail, run_id, repo, desired_ref, commit, environment, pipeline_run, temporary_namespace, code_queue_image = sys.argv[1:] record = { "ok": ok == "true", "status": status, @@ -133,6 +133,7 @@ record = { "environment": environment, "pipelineRun": pipeline_run or None, "temporaryNamespace": temporary_namespace or None, + "codeQueueImage": code_queue_image or None, "finishedAt": datetime.now(timezone.utc).isoformat(), } with open(path, "w", encoding="utf-8") as handle: @@ -144,6 +145,7 @@ PY pipeline_run="" temporary_namespace="unidesk-ci-e2e-$run_id" +code_queue_image="" trap 'code=$?; if [ "$code" -ne 0 ] && [ ! -f "$result_json" ]; then write_result false failed "runner exited with code $code" || true; fi' EXIT export KUBECONFIG=/etc/rancher/k3s/k3s.yaml @@ -188,8 +190,106 @@ source "$service_env" backend_commit="${BACKEND_CORE_COMMIT:-unknown}" frontend_commit="${FRONTEND_COMMIT:-unknown}" code_queue_commit="${CODE_QUEUE_COMMIT:-unknown}" +if ! [[ "$code_queue_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "deploy.json environments.$environment.services must include code-queue with a full 40 character commitId; got: $code_queue_commit" >&2 + exit 2 +fi deploy_json_b64="$(base64 -w0 "$manifest_file")" +work_dir="$(dirname "$manifest_file")" +repo_dir="$work_dir/repo" +if [ ! -d "$repo_dir/.git" ]; then + echo "launcher repo checkout is missing: $repo_dir" >&2 + exit 2 +fi + +root_exec() { + if [ "$(id -u)" = "0" ]; then + "$@" + return + fi + if sudo -n true >/dev/null 2>&1; then + sudo -n "$@" + return + fi + if [ -x /mnt/c/Windows/System32/wsl.exe ]; then + /mnt/c/Windows/System32/wsl.exe -u root -- "$@" + return + fi + echo "dev_e2e_native_k3s_root_access=missing" >&2 + return 1 +} + +import_image_to_k3s() { + local image="$1" + local archive="/tmp/unidesk-ci-image-${run_id}-${image//[^A-Za-z0-9_.-]/-}.tar" + rm -f "$archive" + docker save "$image" -o "$archive" + root_exec ctr --address /run/k3s/containerd/containerd.sock -n k8s.io images import --digests --all-platforms "$archive" + rm -f "$archive" + echo "dev_e2e_image_imported=$image" +} + +ensure_runtime_image() { + local image="$1" + if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "dev_e2e_runtime_image_pull=$image" + docker pull --platform linux/amd64 "$image" + else + echo "dev_e2e_runtime_image_cached=$image" + fi + import_image_to_k3s "$image" +} + +build_code_queue_image() { + local short="${code_queue_commit:0:12}" + local commit_image="unidesk-code-queue:ci-$short" + local run_image="unidesk-code-queue:ci-dev-e2e-$run_id" + local source_dir="$work_dir/code-queue-src-$short" + local resolved + git -C "$repo_dir" fetch --no-tags origin "$code_queue_commit" || git -C "$repo_dir" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' + resolved="$(git -C "$repo_dir" rev-parse --verify "$code_queue_commit^{commit}")" + if [ "$resolved" != "$code_queue_commit" ]; then + echo "code_queue_commit_mismatch resolved=$resolved expected=$code_queue_commit" >&2 + exit 1 + fi + local existing_rev + existing_rev="$(docker image inspect "$commit_image" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' 2>/dev/null || true)" + if [ "$existing_rev" != "$resolved" ]; then + rm -rf "$source_dir" + mkdir -p "$source_dir" + git -C "$repo_dir" archive "$resolved" | tar -x -C "$source_dir" + local base_args=() + if docker image inspect unidesk-code-queue:d601-build-base >/dev/null 2>&1; then + base_args=(--build-arg CODE_QUEUE_BASE_IMAGE=unidesk-code-queue:d601-build-base) + elif docker image inspect unidesk-code-queue:d601 >/dev/null 2>&1; then + base_args=(--build-arg CODE_QUEUE_BASE_IMAGE=unidesk-code-queue:d601) + fi + echo "dev_e2e_code_queue_image_build=$commit_image commit=$resolved" + docker build \ + "${base_args[@]}" \ + --build-arg HTTP_PROXY="${HTTP_PROXY:-}" \ + --build-arg HTTPS_PROXY="${HTTPS_PROXY:-}" \ + --build-arg ALL_PROXY="${ALL_PROXY:-}" \ + --build-arg NO_PROXY="${NO_PROXY:-}" \ + --label "org.opencontainers.image.source=$repo_url" \ + --label "org.opencontainers.image.revision=$resolved" \ + --label "unidesk.ai/ci-run-id=$run_id" \ + --label "unidesk.ai/ci-kind=dev-e2e" \ + -t "$commit_image" \ + -f "$source_dir/src/components/microservices/code-queue/Dockerfile" \ + "$source_dir" + else + echo "dev_e2e_code_queue_image_cached=$commit_image commit=$resolved" + fi + docker tag "$commit_image" "$run_image" + import_image_to_k3s "$run_image" + code_queue_image="$run_image" +} + +ensure_runtime_image "postgres:16-alpine" +build_code_queue_image + pipeline_manifest="$result_dir/pipelinerun.yaml" cat >"$pipeline_manifest" < service.id.length > 0 && service.commitId.length > 0); if (services.length === 0) throw new Error(`origin/${desiredRef}:deploy.json has no environments.dev services with commitId`); + const codeQueueService = services.find((service) => service.id === "code-queue"); + if (codeQueueService === undefined) { + throw new Error(`origin/${desiredRef}:deploy.json environments.dev.services must include code-queue for ci run-dev-e2e`); + } + if (!/^[0-9a-f]{40}$/u.test(codeQueueService.commitId)) { + throw new Error(`origin/${desiredRef}:deploy.json environments.dev.services code-queue commitId must be a full 40-character SHA`); + } return { deployCommit: deployCommitResult.stdout.trim(), desiredRef, diff --git a/scripts/src/code-queue.ts b/scripts/src/code-queue.ts index 5086be2d..6a2c9723 100644 --- a/scripts/src/code-queue.ts +++ b/scripts/src/code-queue.ts @@ -228,6 +228,8 @@ function compactSummary(summary: unknown, options: CodexTaskOptions, taskId: str const transcriptCount = asNumber(record.transcriptCount, 0); const transcriptMaxSeq = transcriptCount > 0 ? record.transcriptMaxSeq ?? null : null; const initialPrompt = asString(record.initialPrompt ?? record.prompt); + const initialPromptView = textView(initialPrompt, options.full, 3000); + const basePromptView = textView(asString(record.basePrompt), options.full, 2000); return { id: record.id ?? taskId, queueId: record.queueId ?? null, @@ -256,8 +258,9 @@ function compactSummary(summary: unknown, options: CodexTaskOptions, taskId: str startedAt: record.startedAt ?? null, updatedAt: record.updatedAt ?? null, finishedAt: record.finishedAt ?? null, - initialPrompt: textView(initialPrompt, options.full, 3000), - basePrompt: textView(asString(record.basePrompt), options.full, 2000), + ...(options.full + ? { initialPrompt: initialPromptView, basePrompt: basePromptView } + : { initialPromptPreview: initialPromptView, basePromptPreview: basePromptView }), referenceTaskIds: record.referenceTaskIds ?? [], referenceInjection: record.referenceInjection ?? null, lastAssistantMessage: compactLastAssistant(record.lastAssistantMessage, options.full), @@ -808,6 +811,7 @@ function compactTaskMutationResponse(task: unknown, options: CompactTaskMutation const record = asRecord(task) ?? {}; const taskId = asString(record.id); const prompt = asString(record.displayPrompt ?? record.basePrompt ?? record.prompt); + const promptView = textView(prompt, options.fullPrompt === true, 1200); return { id: taskId || null, queueId: record.queueId ?? null, @@ -825,7 +829,7 @@ function compactTaskMutationResponse(task: unknown, options: CompactTaskMutation startedAt: record.startedAt ?? null, updatedAt: record.updatedAt ?? null, finishedAt: record.finishedAt ?? null, - prompt: textView(prompt, options.fullPrompt === true, 1200), + ...(options.fullPrompt === true ? { prompt: promptView } : { promptPreview: promptView }), commands: taskId.length === 0 ? null : { show: `bun scripts/cli.ts codex task ${taskId}`, trace: `bun scripts/cli.ts codex task ${taskId} --trace --tail --limit ${defaultTraceLimit}`, diff --git a/scripts/src/e2e.ts b/scripts/src/e2e.ts index 0b6b3f22..5043d4f9 100644 --- a/scripts/src/e2e.ts +++ b/scripts/src/e2e.ts @@ -139,6 +139,7 @@ const FRONTEND_CHECK_NAMES = [ "frontend:code-queue-integrated-visible", "frontend:code-queue-enqueue-await-smoke", "frontend:code-queue-summary-mobile-wrap", + "frontend:code-queue-long-prompt-observation", "frontend:code-queue-initial-prompt-full-expand", "frontend:code-queue-trace-full-load", "frontend:code-queue-judge-wrap", @@ -486,8 +487,10 @@ function wantsPrefix(options: E2ERunOptions, prefix: string): boolean { } function publicUrls(config: UniDeskConfig): PublicUrls { + const frontendTarget = String(process.env.UNIDESK_E2E_FRONTEND || "prod").trim().toLowerCase(); + const frontendPort = frontendTarget === "dev" ? config.network.devFrontend.port : config.network.frontend.port; return { - frontendUrl: `http://${config.network.publicHost}:${config.network.frontend.port}`, + frontendUrl: `http://${config.network.publicHost}:${frontendPort}`, providerIngressHealthUrl: `http://${config.network.publicHost}:${config.network.providerIngress.port}/health`, providerIngressWsUrl: `ws://${config.network.publicHost}:${config.network.providerIngress.port}/ws/provider`, blockedCoreUrl: `http://${config.network.publicHost}:${config.network.core.port}`, @@ -750,6 +753,149 @@ async function runCodeQueueEnqueueAwaitSmoke(page: Page): Promise { } } +async function runCodeQueueLongPromptObservation(page: Page): Promise { + const marker = `E2E_LONG_PROMPT_TAIL_${Date.now()}_${Math.random().toString(16).slice(2, 8)}`; + const submitQueueId = "e2e-long-prompt"; + const filler = Array.from({ length: 76 }, (_, index) => `long-prompt-line-${String(index + 1).padStart(2, "0")} ${"abcdefghijklmnopqrstuvwxyz0123456789".repeat(2)}`).join("\n"); + const prompt = [ + "Code Queue long prompt observation regression", + "", + filler, + "", + "验收标准：", + `必须在所有原始 prompt 观察层保留这个唯一 tail marker: ${marker}`, + ].join("\n"); + const promptChars = prompt.length; + if (promptChars <= 2500) throw new Error(`long prompt fixture too short: ${promptChars}`); + const apiFetch = async (path: string, init: RequestInit = {}): Promise => { + const response = await fetch(path, { + credentials: "same-origin", + headers: { "content-type": "application/json", ...(init.headers || {}) }, + ...init, + }); + const text = await response.text(); + let body: any = null; + try { body = text.length > 0 ? JSON.parse(text) : null; } catch { body = { text }; } + return { ok: response.ok, status: response.status, body }; + }; + + await page.getByTestId("code-queue-filter-select").selectOption("__all__").catch(() => undefined); + await page.getByTestId("code-queue-id-select").selectOption(submitQueueId).catch(async () => { + page.once("dialog", (dialog) => { void dialog.accept(submitQueueId); }); + await page.getByTestId("codex-create-queue-button").click(); + await page.waitForFunction((queueId) => { + const select = document.querySelector('[data-testid="code-queue-id-select"]') as HTMLSelectElement | null; + return Array.from(select?.options || []).some((option) => option.value === queueId); + }, submitQueueId, { timeout: 10000 }); + await page.getByTestId("code-queue-id-select").selectOption(submitQueueId); + }); + await page.getByTestId("codex-max-attempts-input").fill("1"); + await page.getByTestId("codex-repeat-count-input").fill("1"); + await page.locator('[data-testid="code-queue-task-form"] textarea').fill(prompt); + await page.waitForFunction(() => { + const button = document.querySelector('[data-testid="codex-enqueue-button"]') as HTMLButtonElement | null; + return button !== null && !button.disabled; + }, undefined, { timeout: 5000 }); + const responsePromise = page.waitForResponse((response) => isCodeQueueTaskEnqueueRequest(response.url(), response.request().method()), { timeout: 30000 }); + await page.getByTestId("codex-enqueue-button").click(); + const response = await responsePromise; + const createBody = await response.json().catch((error: unknown) => ({ parseError: error instanceof Error ? error.message : String(error) })); + const createdTask = Array.isArray(createBody?.tasks) ? createBody.tasks[0] : null; + const taskId = String(createdTask?.id || ""); + if (!taskId) return { checked: true, marker, promptChars, createStatus: response.status(), taskId: "", error: "missing task id", createBody }; + await page.waitForSelector(`[data-testid="codex-task-${taskId}"]`, { timeout: 30000 }).catch(() => undefined); + + const overview = await page.evaluate(async (id) => { + const response = await fetch(`/api/microservices/code-queue/proxy/api/tasks/overview?limit=120&transcriptLimit=0&compact=1&selected=1&includeActive=1&stats=0&afterSeq=0&preferId=${encodeURIComponent(String(id))}`, { credentials: "same-origin" }); + const body = await response.json().catch(() => null); + return { ok: response.ok, status: response.status, body }; + }, taskId); + const meta = await page.evaluate(async (id) => { + const response = await fetch(`/api/microservices/code-queue/proxy/api/tasks/${encodeURIComponent(String(id))}?meta=1`, { credentials: "same-origin" }); + const body = await response.json().catch(() => null); + return { ok: response.ok, status: response.status, body }; + }, taskId); + const initialPrompt = await page.evaluate(async (id) => { + const response = await fetch(`/api/microservices/code-queue/proxy/api/tasks/${encodeURIComponent(String(id))}/prompt?part=initial`, { credentials: "same-origin" }); + const body = await response.json().catch(() => null); + return { ok: response.ok, status: response.status, body }; + }, taskId); + + await page.getByTestId(`codex-task-${taskId}`).click().catch(() => undefined); + await page.waitForSelector('[data-testid="codex-initial-prompt-base"]', { timeout: 15000 }).catch(() => undefined); + const ui = await page.evaluate(() => { + const base = document.querySelector('[data-testid="codex-initial-prompt-base"]') as HTMLElement | null; + return { + baseText: base?.innerText || "", + baseChars: (base?.innerText || "").length, + traceText: document.querySelector('.codex-output-panel')?.textContent || "", + }; + }); + + const overviewTasks = Array.isArray(overview.body?.tasks) ? overview.body.tasks : []; + const overviewTask = overviewTasks.find((task: any) => String(task?.id || "") === taskId) || null; + const selectedTask = overview.body?.selected?.task || null; + const metaTask = meta.body?.task || null; + const createPrompt = String(createdTask?.prompt || ""); + const createDisplayPrompt = String(createdTask?.displayPrompt || ""); + const createPreview = createdTask?.displayPromptPreview || createdTask?.promptPreview || null; + const overviewPrompt = String(overviewTask?.prompt || ""); + const overviewDisplayPrompt = String(overviewTask?.displayPrompt || ""); + const overviewPreview = overviewTask?.displayPromptPreview || overviewTask?.promptPreview || null; + const selectedDisplayPrompt = String(selectedTask?.displayPrompt || ""); + const selectedPreview = selectedTask?.displayPromptPreview || selectedTask?.promptPreview || null; + const metaPrompt = String(metaTask?.prompt || ""); + const metaBasePrompt = String(metaTask?.basePrompt || ""); + const metaDisplayPrompt = String(metaTask?.displayPrompt || ""); + const promptText = String(initialPrompt.body?.text || ""); + const previewHasTruncationMarker = [createPreview?.text, overviewPreview?.text, selectedPreview?.text].some((value) => String(value || "").includes("...")); + const originalValues = [createPrompt, createDisplayPrompt, overviewPrompt, overviewDisplayPrompt, selectedDisplayPrompt, metaPrompt, metaBasePrompt, metaDisplayPrompt, promptText]; + await apiFetch(`/api/microservices/code-queue/proxy/api/tasks/${encodeURIComponent(taskId)}/interrupt`, { method: "POST", body: "{}" }).catch(() => null); + return { + checked: true, + marker, + taskId, + promptChars, + createStatus: response.status(), + createOk: createBody?.ok === true, + createPromptChars: createPrompt.length, + createDisplayPromptChars: createDisplayPrompt.length, + overviewStatus: overview.status, + overviewTaskFound: overviewTask !== null, + overviewPromptChars: overviewPrompt.length, + overviewDisplayPromptChars: overviewDisplayPrompt.length, + selectedTaskFound: selectedTask !== null && String(selectedTask?.id || "") === taskId, + selectedDisplayPromptChars: selectedDisplayPrompt.length, + metaStatus: meta.status, + metaPromptChars: metaPrompt.length, + metaBasePromptChars: metaBasePrompt.length, + metaDisplayPromptChars: metaDisplayPrompt.length, + initialPromptStatus: initialPrompt.status, + initialPromptChars: Number(initialPrompt.body?.chars || promptText.length), + uiBaseChars: ui.baseChars, + preview: { + create: createPreview, + overview: overviewPreview, + selected: selectedPreview, + previewHasTruncationMarker, + createPromptPreviewFieldPresent: createdTask?.promptPreview !== undefined, + overviewPromptPreviewFieldPresent: overviewTask?.promptPreview !== undefined, + selectedPromptPreviewFieldPresent: selectedTask?.promptPreview !== undefined, + }, + checks: { + createOriginalHasTail: createPrompt.includes(marker) && createDisplayPrompt.includes(marker), + overviewOriginalHasTail: overviewPrompt.includes(marker) && overviewDisplayPrompt.includes(marker), + selectedOriginalHasTail: selectedDisplayPrompt.includes(marker), + metaOriginalHasTail: metaPrompt.includes(marker) && metaBasePrompt.includes(marker) && metaDisplayPrompt.includes(marker), + initialPromptHasTail: promptText.includes(marker), + uiFullPromptHasTail: String(ui.baseText || "").includes(marker), + originalsHaveNoTruncationMarker: originalValues.every((value) => !value.includes("...")), + previewFieldsExplicit: createdTask?.promptPreview !== undefined && overviewTask?.promptPreview !== undefined && selectedTask?.promptPreview !== undefined, + previewCanTruncate: Boolean(createPreview?.truncated || overviewPreview?.truncated || selectedPreview?.truncated), + }, + }; +} + function runPsql(config: UniDeskConfig, sql: string): { ok: boolean; stdout: string; stderr: string; exitCode: number | null } { const result = runCommand([ "docker", @@ -1450,6 +1596,7 @@ async function frontendCheck(config: UniDeskConfig, urls: PublicUrls, checks: E2 "frontend:code-queue-integrated-visible", "frontend:code-queue-enqueue-await-smoke", "frontend:code-queue-summary-mobile-wrap", + "frontend:code-queue-long-prompt-observation", "frontend:code-queue-initial-prompt-full-expand", "frontend:code-queue-trace-full-load", "frontend:code-queue-judge-wrap", @@ -1486,8 +1633,10 @@ async function frontendCheck(config: UniDeskConfig, urls: PublicUrls, checks: E2 page.on("dialog", (dialog) => dialog.accept()); await page.goto(urls.frontendUrl, { waitUntil: "domcontentloaded", timeout: 15000 }); await page.waitForSelector('[data-testid="login-screen"]', { timeout: 10000 }); - await page.fill('input[name="username"]', config.auth.username); - await page.fill('input[name="password"]', config.auth.password); + const frontendAuthUsername = process.env.UNIDESK_E2E_AUTH_USERNAME || config.auth.username; + const frontendAuthPassword = process.env.UNIDESK_E2E_AUTH_PASSWORD || config.auth.password; + await page.fill('input[name="username"]', frontendAuthUsername); + await page.fill('input[name="password"]', frontendAuthPassword); await page.click('button[type="submit"]'); await page.waitForSelector('[data-testid="app-shell"]', { timeout: 10000 }); await page.waitForFunction(() => document.querySelector('[data-testid="conn-text"]')?.textContent?.includes("核心在线"), undefined, { timeout: 15000 }); @@ -1547,6 +1696,7 @@ async function frontendCheck(config: UniDeskConfig, urls: PublicUrls, checks: E2 let codeQueueSidebarUpdateMetrics: any = { cardCount: 0, labels: [], hasRecentUpdateLabel: false }; let codeQueueHtmlGuard: any = { rootAttrMissing: false, sourceAttrMissing: false, sourceNoBasePrompt: false }; let codeQueueSummaryMobileMetrics: any = { checked: false, summaryCount: 0, ok: false }; + let codeQueueLongPromptMetrics: any = { checked: false }; let codeQueuePromptDefaultEmpty = false; let codeQueueSubmitGuard: any = { batchRowVisible: false, disabledBeforeConfirm: false, enabledAfterConfirm: false, waitElementMissingBeforeSubmit: false }; let codeQueueEnqueueAwaitSmoke: any = { checked: false }; @@ -1995,6 +2145,9 @@ async function frontendCheck(config: UniDeskConfig, urls: PublicUrls, checks: E2 if (wants("frontend:code-queue-enqueue-await-smoke")) { codeQueueEnqueueAwaitSmoke = await runCodeQueueEnqueueAwaitSmoke(page); } + if (wants("frontend:code-queue-long-prompt-observation")) { + codeQueueLongPromptMetrics = await runCodeQueueLongPromptObservation(page); + } codeQueueOptions = await page.locator('[data-testid="code-queue-filter-select"] option').evaluateAll((options) => options.map((option) => (option as HTMLOptionElement).textContent || "")); codeQueueSwitchMetrics = await page.locator('[data-testid="code-queue-filter-select"] option').evaluateAll((options) => ({ optionCount: options.length, @@ -3049,6 +3202,21 @@ async function frontendCheck(config: UniDeskConfig, urls: PublicUrls, checks: E2 && (codeQueueEnqueueAwaitSmoke.interrupt?.ok === true || codeQueueEnqueueAwaitSmoke.interrupt?.status === 409), { codeQueueEnqueueAwaitSmoke }); addSelectedCheck(checks, options, "frontend:code-queue-summary-mobile-wrap", codeQueueSummaryMobileMetrics.checked === true && (codeQueueSummaryMobileMetrics.summaryCount === 0 || codeQueueSummaryMobileMetrics.ok === true), { codeQueueSummaryMobileMetrics }); + addSelectedCheck(checks, options, "frontend:code-queue-long-prompt-observation", + codeQueueLongPromptMetrics.checked === true + && codeQueueLongPromptMetrics.createOk === true + && /^codex_\d+_[A-Za-z0-9_-]+$/u.test(String(codeQueueLongPromptMetrics.taskId || "")) + && Number(codeQueueLongPromptMetrics.promptChars || 0) > 2500 + && codeQueueLongPromptMetrics.checks?.createOriginalHasTail === true + && codeQueueLongPromptMetrics.checks?.overviewOriginalHasTail === true + && codeQueueLongPromptMetrics.checks?.selectedOriginalHasTail === true + && codeQueueLongPromptMetrics.checks?.metaOriginalHasTail === true + && codeQueueLongPromptMetrics.checks?.initialPromptHasTail === true + && codeQueueLongPromptMetrics.checks?.uiFullPromptHasTail === true + && codeQueueLongPromptMetrics.checks?.originalsHaveNoTruncationMarker === true + && codeQueueLongPromptMetrics.checks?.previewFieldsExplicit === true + && codeQueueLongPromptMetrics.checks?.previewCanTruncate === true, + { codeQueueLongPromptMetrics }); addSelectedCheck(checks, options, "frontend:code-queue-error-red-markers", codeQueueErrorHighlightMetrics.checked === true && codeQueueErrorHighlightMetrics.candidateFound === true && codeQueueErrorHighlightMetrics.ok === true, { codeQueueErrorHighlightMetrics }); addSelectedCheck(checks, options, "frontend:code-queue-initial-prompt-full-expand", codexInitialPromptFullMetrics.candidateFound === false diff --git a/src/components/frontend/src/code-queue.tsx b/src/components/frontend/src/code-queue.tsx index 6476653b..520c2474 100644 --- a/src/components/frontend/src/code-queue.tsx +++ b/src/components/frontend/src/code-queue.tsx @@ -535,7 +535,17 @@ function promptLineCount(text: string): number { return text.length > 0 ? text.split(/\r\n|\r|\n/u).length : 0; } -function taskDisplayPrompt(task: any): string { +function previewTextField(value: any): string { + if (typeof value === "string") return value; + if (value && typeof value === "object" && !Array.isArray(value) && typeof value.text === "string") return value.text; + return ""; +} + +function taskDisplayPrompt(task: any, options: AnyRecord = {}): string { + if (options.preview === true) { + const preview = previewTextField(task?.displayPromptPreview || task?.basePromptPreview || task?.promptPreview); + if (preview.length > 0) return preview; + } const explicit = String(task?.displayPrompt || ""); if (explicit.length > 0) return explicit; const prompt = String(task?.prompt || ""); @@ -640,7 +650,9 @@ function traceSummaryIsCurrent(task: any): boolean { function taskBasePromptText(task: any): string { const summaryPrompt = taskPromptSummary(task); const basePrompt = String(summaryPrompt.basePrompt || ""); - return basePrompt.length > 0 ? basePrompt : taskDisplayPrompt(task); + if (basePrompt.length > 0) return basePrompt; + const direct = String(task?.basePrompt || ""); + return direct.length > 0 ? direct : taskDisplayPrompt(task); } function taskFinalResponseText(task: any): string { @@ -1473,7 +1485,7 @@ function TaskCard({ task, selected, onSelect, onCopy, onReference, onMarkRead, c }, markingRead ? "标记中" : "标为已读") : null, ), ), - h("strong", null, shortText(taskDisplayPrompt(task), 120) || "空任务"), + h("strong", null, shortText(taskDisplayPrompt(task, { preview: true }), 120) || "空任务"), h("div", { className: "codex-task-meta" }, h("span", null, `queue=${taskQueueLabel(task)}`), h("span", null, `provider=${task?.providerId || "D601"}`), @@ -2229,6 +2241,9 @@ export function CodeQueuePage({ microservices, onRaw, apiBaseUrl = "/api", initi task?.providerId, task?.model, task?.cwd, + previewTextField(task?.displayPromptPreview), + previewTextField(task?.basePromptPreview), + previewTextField(task?.promptPreview), task?.displayPrompt, task?.basePrompt, task?.prompt, diff --git a/src/components/microservices/code-queue-mgr/src-rs/main.rs b/src/components/microservices/code-queue-mgr/src-rs/main.rs index e4ed5375..3283f15a 100644 --- a/src/components/microservices/code-queue-mgr/src-rs/main.rs +++ b/src/components/microservices/code-queue-mgr/src-rs/main.rs @@ -497,16 +497,6 @@ fn preview(text: &str, max_chars: usize) -> String { result } -fn prefix_preview(text: &str, max_chars: usize) -> String { - if text.chars().count() <= max_chars { - return text.to_string(); - } - let take = max_chars.saturating_sub(1); - let mut result = text.chars().take(take).collect::(); - result.push('…'); - result -} - fn strip_after_marker(text: &str, marker: &str) -> Option { text.find(marker).map(|index| text[index + marker.len()..].trim_start().to_string()) } @@ -611,6 +601,17 @@ fn final_response(task: &TaskMeta) -> String { .to_string() } +fn text_preview(value: &str, max_chars: usize) -> Value { + let chars = value.chars().count(); + let truncated = chars > max_chars; + json!({ + "text": if truncated { preview(value, max_chars) } else { value.to_string() }, + "chars": chars, + "truncated": truncated, + "omittedChars": if truncated { chars.saturating_sub(max_chars.saturating_sub(20)) } else { 0 } + }) +} + fn text_from_output(item: &Value) -> String { item.get("text").and_then(Value::as_str).unwrap_or("").to_string() } @@ -664,13 +665,17 @@ fn task_list_response(_state: &AppState, task: &TaskMeta, lite: bool) -> Value { }; let final_text = final_response(task); let agent_port = code_agent_port(&task.model); + let preview_limit = if lite { 360 } else { 2000 }; json!({ "id": task.id, "queueId": task.queue_id, "queueEnteredAt": task.task_json.as_ref().and_then(|value| value.get("queueEnteredAt")).and_then(Value::as_str).unwrap_or(&task.created_at), - "prompt": if lite { prefix_preview(&display_prompt, 360) } else { preview(&task.prompt, 2000) }, - "basePrompt": if lite { prefix_preview(&task.base_prompt, 360) } else { preview(&task.base_prompt, 2000) }, - "displayPrompt": if lite { prefix_preview(&display_prompt, 360) } else { preview(&display_prompt, 2000) }, + "prompt": task.prompt, + "basePrompt": task.base_prompt, + "displayPrompt": display_prompt, + "promptPreview": text_preview(&task.prompt, preview_limit), + "basePromptPreview": text_preview(&task.base_prompt, preview_limit), + "displayPromptPreview": text_preview(&display_prompt, preview_limit), "promptChars": task.prompt.chars().count(), "basePromptChars": task.base_prompt.chars().count(), "displayPromptChars": display_prompt.chars().count(), @@ -729,8 +734,15 @@ fn task_meta_response(state: &AppState, task: &TaskMeta) -> Value { let map = base.as_object_mut().expect("task response object"); let output_count = json_array_len(task.task_json.as_ref(), "output", task.output_count); let prompt_history_count = json_array_len(task.task_json.as_ref(), "promptHistory", 0); + let display_prompt = if task.base_prompt.is_empty() { + user_prompt_for_display(&task.prompt) + } else { + task.base_prompt.clone() + }; map.insert("prompt".to_string(), json!(task.prompt)); map.insert("basePrompt".to_string(), json!(task.base_prompt)); + map.insert("displayPrompt".to_string(), json!(display_prompt)); + map.insert("initialPrompt".to_string(), json!(task.prompt)); map.insert("finalResponse".to_string(), json!(final_response(task))); map.insert( "promptHistory".to_string(), diff --git a/src/components/microservices/code-queue-mgr/src/index.ts b/src/components/microservices/code-queue-mgr/src/index.ts index daf56e86..984c95bd 100644 --- a/src/components/microservices/code-queue-mgr/src/index.ts +++ b/src/components/microservices/code-queue-mgr/src/index.ts @@ -1,5 +1,6 @@ import postgres from "postgres"; import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../../shared/src/rotating-jsonl"; +import { codeQueuePromptResponseFields, safePreview } from "./prompt-observation"; type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; type JsonRecord = Record; @@ -434,12 +435,6 @@ function timestampMs(value: string | Date | null | undefined): number | null { return Number.isFinite(ms) ? ms : null; } -function safePreview(value: unknown, maxChars: number): string { - const text = String(value ?? ""); - if (text.length <= maxChars) return text; - return `${text.slice(0, Math.max(0, maxChars - 20))}\n...`; -} - function prefixPreview(value: unknown, maxChars: number): string { const text = String(value ?? ""); return text.length <= maxChars ? text : `${text.slice(0, Math.max(0, maxChars - 1))}…`; @@ -1635,17 +1630,12 @@ function taskStatisticsSummary(tasks: QueueTask[], days = 14): JsonRecord { } function taskListResponse(task: QueueTask, lite = true): JsonRecord { - const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt); + const promptFields = codeQueuePromptResponseFields(task, { lite, userPromptForDisplay }); return { id: task.id, queueId: queueIdOf(task), queueEnteredAt: task.queueEnteredAt, - prompt: lite ? prefixPreview(displayPrompt, 360) : safePreview(displayPrompt, 2000), - basePrompt: lite ? prefixPreview(task.basePrompt, 360) : safePreview(task.basePrompt, 2000), - displayPrompt: lite ? prefixPreview(displayPrompt, 360) : safePreview(displayPrompt, 2000), - promptChars: task.prompt.length, - basePromptChars: task.basePrompt.length, - displayPromptChars: displayPrompt.length, + ...promptFields, promptEditable: queuedTaskPromptEditable(task), finalResponseChars: task.finalResponse.length, stepCount: numberField(task.stepCount ?? task.llmStepCount, 0), @@ -1695,6 +1685,8 @@ function taskMetaResponse(task: QueueTask): JsonRecord { ...taskListResponse(task, false), prompt: task.prompt, basePrompt: task.basePrompt, + displayPrompt: task.basePrompt || userPromptForDisplay(task.prompt), + initialPrompt: task.prompt, finalResponse: task.finalResponse, promptHistory: toJsonValue(task.promptHistory), attempts: toJsonValue(task.attempts), diff --git a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml index 86dc5763..3caef69f 100644 --- a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml +++ b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml @@ -88,7 +88,7 @@ rules: resources: ["namespaces"] verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: [""] - resources: ["configmaps", "services", "pods", "pods/log"] + resources: ["configmaps", "secrets", "services", "services/proxy", "pods", "pods/log"] verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: ["apps"] resources: ["deployments"] @@ -701,6 +701,8 @@ spec: backend_commit="$(params.backend-core-commit)" frontend_commit="$(params.frontend-commit)" code_queue_commit="$(params.code-queue-commit)" + app_image="$(params.app-image)" + database_url="postgres://unidesk_ci:unidesk_ci_password@postgres-dev.$ns.svc.cluster.local:5432/unidesk_ci" kube_api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}" kube_token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" kube_ca="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" @@ -719,6 +721,48 @@ spec: cat /tmp/unidesk-dev-e2e-delete-response >&2 return 1 } + proxy_get() { + local service_name="$1" + local path="$2" + curl -fsS --cacert "$kube_ca" -H "Authorization: Bearer $kube_token" \ + "$kube_api/api/v1/namespaces/$ns/services/${service_name}:4222/proxy$path" + } + proxy_json() { + local method="$1" + local service_name="$2" + local path="$3" + local body="${4:-}" + if [ -n "$body" ]; then + curl -fsS --cacert "$kube_ca" -H "Authorization: Bearer $kube_token" -H "Content-Type: application/json" \ + -X "$method" --data "$body" \ + "$kube_api/api/v1/namespaces/$ns/services/${service_name}:4222/proxy$path" + else + curl -fsS --cacert "$kube_ca" -H "Authorization: Bearer $kube_token" \ + -X "$method" \ + "$kube_api/api/v1/namespaces/$ns/services/${service_name}:4222/proxy$path" + fi + } + wait_deployment_available() { + local deployment="$1" + local timeout_seconds="$2" + local deadline=$((SECONDS + timeout_seconds)) + while [ "$SECONDS" -lt "$deadline" ]; do + status="$(kube GET "$kube_api/apis/apps/v1/namespaces/$ns/deployments/$deployment")" + replicas="$(printf '%s' "$status" | jq -r '.spec.replicas // 1')" + available="$(printf '%s' "$status" | jq -r '.status.availableReplicas // 0')" + updated="$(printf '%s' "$status" | jq -r '.status.updatedReplicas // 0')" + observed="$(printf '%s' "$status" | jq -r '.status.observedGeneration // 0')" + generation="$(printf '%s' "$status" | jq -r '.metadata.generation // 0')" + if [ "$available" -ge "$replicas" ] && [ "$updated" -ge "$replicas" ] && [ "$observed" -ge "$generation" ]; then + echo "dev_e2e_rollout=available deployment=$deployment namespace=$ns replicas=$available generation=$generation" + return 0 + fi + sleep 2 + done + echo "dev_e2e_rollout=timeout deployment=$deployment namespace=$ns" >&2 + kube GET "$kube_api/apis/apps/v1/namespaces/$ns/deployments/$deployment" >&2 + return 1 + } cleanup() { if [ "$keep" = "true" ]; then echo "dev_e2e_namespace_retained=$ns" @@ -767,162 +811,640 @@ spec: --data-binary @/tmp/dev-e2e-configmap.yaml \ "$kube_api/api/v1/namespaces/$ns/configmaps/desired-manifest?fieldManager=unidesk-ci&force=true" >/dev/null - cat >/tmp/dev-e2e-target.yaml </tmp/dev-e2e-secret.yaml </dev/null + + cat >/tmp/dev-e2e-postgres.yaml < {}); ports: - - name: http - containerPort: 8080 + - name: postgres + containerPort: 5432 env: - - name: PORT - value: "8080" - - name: CI_E2E_NAMESPACE - value: "$ns" - - name: CI_E2E_DEPLOY_COMMIT - value: "$(params.deploy-commit)" - - name: BACKEND_CORE_COMMIT - value: "$backend_commit" - - name: FRONTEND_COMMIT - value: "$frontend_commit" - - name: CODE_QUEUE_COMMIT - value: "$code_queue_commit" + - name: POSTGRES_USER + value: unidesk_ci + - name: POSTGRES_PASSWORD + value: unidesk_ci_password + - name: POSTGRES_DB + value: unidesk_ci readinessProbe: - httpGet: - path: /health - port: http - periodSeconds: 3 - timeoutSeconds: 2 - failureThreshold: 20 + exec: + command: + - pg_isready + - -U + - unidesk_ci + - -d + - unidesk_ci + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 24 resources: requests: - cpu: 20m - memory: 64Mi + cpu: 50m + memory: 128Mi limits: - memory: 256Mi + cpu: 250m + memory: 512Mi + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumes: + - name: data + emptyDir: {} --- apiVersion: v1 kind: Service metadata: - name: dev-e2e-target + name: postgres-dev namespace: $ns labels: - app.kubernetes.io/name: unidesk-dev-namespace-e2e - app.kubernetes.io/component: smoke-target + app.kubernetes.io/name: postgres-dev + app.kubernetes.io/component: database app.kubernetes.io/part-of: unidesk spec: type: ClusterIP selector: - app.kubernetes.io/name: unidesk-dev-namespace-e2e - app.kubernetes.io/component: smoke-target + app.kubernetes.io/name: postgres-dev + app.kubernetes.io/component: database + ports: + - name: postgres + port: 5432 + targetPort: postgres + YAML + csplit -s -f /tmp/dev-e2e-postgres- /tmp/dev-e2e-postgres.yaml '/^---$/' '{*}' + kube PATCH \ + -H "Content-Type: application/apply-patch+yaml" \ + --data-binary @/tmp/dev-e2e-postgres-00 \ + "$kube_api/apis/apps/v1/namespaces/$ns/deployments/postgres-dev?fieldManager=unidesk-ci&force=true" >/dev/null + kube PATCH \ + -H "Content-Type: application/apply-patch+yaml" \ + --data-binary @/tmp/dev-e2e-postgres-01 \ + "$kube_api/api/v1/namespaces/$ns/services/postgres-dev?fieldManager=unidesk-ci&force=true" >/dev/null + wait_deployment_available postgres-dev 180 + + cat >/tmp/dev-e2e-code-queue.yaml </dev/null - kube PATCH \ - -H "Content-Type: application/apply-patch+yaml" \ - --data-binary @/tmp/dev-e2e-target-01 \ - "$kube_api/api/v1/namespaces/$ns/services/dev-e2e-target?fieldManager=unidesk-ci&force=true" >/dev/null + csplit -s -f /tmp/dev-e2e-code-queue- /tmp/dev-e2e-code-queue.yaml '/^---$/' '{*}' + kube PATCH -H "Content-Type: application/apply-patch+yaml" --data-binary @/tmp/dev-e2e-code-queue-00 "$kube_api/apis/apps/v1/namespaces/$ns/deployments/code-queue-scheduler-dev?fieldManager=unidesk-ci&force=true" >/dev/null + kube PATCH -H "Content-Type: application/apply-patch+yaml" --data-binary @/tmp/dev-e2e-code-queue-01 "$kube_api/api/v1/namespaces/$ns/services/code-queue-scheduler-dev?fieldManager=unidesk-ci&force=true" >/dev/null + kube PATCH -H "Content-Type: application/apply-patch+yaml" --data-binary @/tmp/dev-e2e-code-queue-02 "$kube_api/apis/apps/v1/namespaces/$ns/deployments/code-queue-read-dev?fieldManager=unidesk-ci&force=true" >/dev/null + kube PATCH -H "Content-Type: application/apply-patch+yaml" --data-binary @/tmp/dev-e2e-code-queue-03 "$kube_api/api/v1/namespaces/$ns/services/code-queue-read-dev?fieldManager=unidesk-ci&force=true" >/dev/null + kube PATCH -H "Content-Type: application/apply-patch+yaml" --data-binary @/tmp/dev-e2e-code-queue-04 "$kube_api/apis/apps/v1/namespaces/$ns/deployments/code-queue-write-dev?fieldManager=unidesk-ci&force=true" >/dev/null + kube PATCH -H "Content-Type: application/apply-patch+yaml" --data-binary @/tmp/dev-e2e-code-queue-05 "$kube_api/api/v1/namespaces/$ns/services/code-queue-write-dev?fieldManager=unidesk-ci&force=true" >/dev/null + wait_deployment_available code-queue-scheduler-dev 420 + wait_deployment_available code-queue-read-dev 420 + wait_deployment_available code-queue-write-dev 420 - deadline=$((SECONDS + 180)) - while [ "$SECONDS" -lt "$deadline" ]; do - status="$(kube GET "$kube_api/apis/apps/v1/namespaces/$ns/deployments/dev-e2e-target")" - available="$(printf '%s' "$status" | jq -r '.status.availableReplicas // 0')" - observed="$(printf '%s' "$status" | jq -r '.status.observedGeneration // 0')" - generation="$(printf '%s' "$status" | jq -r '.metadata.generation // 0')" - if [ "$available" -ge 1 ] && [ "$observed" -ge "$generation" ]; then - echo "dev_e2e_target_rollout=available namespace=$ns" - break - fi - sleep 2 - done - if [ "$SECONDS" -ge "$deadline" ]; then - echo "dev_e2e_target_rollout=timeout namespace=$ns" >&2 - kube GET "$kube_api/apis/apps/v1/namespaces/$ns/deployments/dev-e2e-target" >&2 - exit 1 - fi + proxy_get code-queue-write-dev /health >"$run_dir/code-queue-write-health.json" + proxy_get code-queue-scheduler-dev /health >"$run_dir/code-queue-scheduler-health.json" + proxy_get code-queue-read-dev /live >"$run_dir/code-queue-read-live.json" + proxy_get code-queue-read-dev /api/workdirs >"$run_dir/code-queue-read-workdirs.json" + proxy_json POST code-queue-write-dev /api/workdirs '{"providerId":"D601-dev","executionMode":"default","path":"/home/ubuntu/unidesk-dev-workspace/ci-workdirs-smoke"}' >"$run_dir/code-queue-workdir-created.json" + proxy_get code-queue-write-dev '/api/workdirs?providerId=D601-dev&executionMode=default' >"$run_dir/code-queue-write-workdirs.json" + proxy_json DELETE code-queue-write-dev /api/workdirs/D601-dev/default/%2Fhome%2Fubuntu%2Funidesk-dev-workspace%2Fci-workdirs-smoke >"$run_dir/code-queue-workdir-deleted.json" - bun - "$ns" "$(params.deploy-commit)" "$backend_commit" "$frontend_commit" "$code_queue_commit" "$result_json" <<'BUN' - const [ns, deployCommit, backendCommit, frontendCommit, codeQueueCommit, resultPath] = process.argv.slice(2); - const url = `http://dev-e2e-target.${ns}.svc.cluster.local:8080/health`; - const started = performance.now(); - const response = await fetch(url); - const elapsedMs = Math.round(performance.now() - started); - const body = await response.json(); - const checks = [ - response.ok, - body.ok === true, - body.environment === "dev", - body.namespace === ns, - body.deployCommit === deployCommit, - body.backendCoreCommit === backendCommit, - body.frontendCommit === frontendCommit, - body.codeQueueCommit === codeQueueCommit - ]; - const result = { ok: checks.every(Boolean), elapsedMs, url, body }; - await Bun.write(resultPath, JSON.stringify(result, null, 2) + "\n"); - console.log(JSON.stringify(result)); - if (!result.ok) process.exit(1); - BUN + python3 - "$ns" "$(params.deploy-commit)" "$backend_commit" "$frontend_commit" "$code_queue_commit" "$app_image" "$result_json" "$run_dir" <<'PY' +import json +import sys +from pathlib import Path + +ns, deploy_commit, backend_commit, frontend_commit, code_queue_commit, app_image, result_path, run_dir = sys.argv[1:] +run_path = Path(run_dir) + +def read_json(name): + with (run_path / name).open("r", encoding="utf-8") as handle: + return json.load(handle) + +health = read_json("code-queue-write-health.json") +scheduler = read_json("code-queue-scheduler-health.json") +read_live = read_json("code-queue-read-live.json") +initial_workdirs = read_json("code-queue-read-workdirs.json") +created = read_json("code-queue-workdir-created.json") +listed = read_json("code-queue-write-workdirs.json") +deleted = read_json("code-queue-workdir-deleted.json") +checks = [ + health.get("ok") is True and health.get("role") == "write" and health.get("deploy", {}).get("commit") == code_queue_commit, + scheduler.get("ok") is True and scheduler.get("role") == "scheduler" and scheduler.get("schedulerEnabled") is True, + read_live.get("ok") is True and read_live.get("role") == "read", + initial_workdirs.get("ok") is True and isinstance(initial_workdirs.get("workdirs"), list), + created.get("ok") is True and created.get("workdir", {}).get("providerId") == "D601-dev" and created.get("workdir", {}).get("path") == "/home/ubuntu/unidesk-dev-workspace/ci-workdirs-smoke", + listed.get("ok") is True and any(item.get("path") == "/home/ubuntu/unidesk-dev-workspace/ci-workdirs-smoke" for item in listed.get("workdirs", [])), + deleted.get("ok") is True, +] +result = { + "ok": all(checks), + "namespace": ns, + "deployCommit": deploy_commit, + "backendCoreCommit": backend_commit, + "frontendCommit": frontend_commit, + "codeQueueCommit": code_queue_commit, + "codeQueueImage": app_image, + "accessPath": "kubernetes-api-service-proxy", + "services": ["code-queue-scheduler-dev", "code-queue-read-dev", "code-queue-write-dev"], + "health": health, + "scheduler": scheduler, + "readLive": read_live, + "initialWorkdirs": initial_workdirs, + "created": created, + "listed": listed, + "deleted": deleted, +} +Path(result_path).write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +print(json.dumps(result, ensure_ascii=False)) +if not result["ok"]: + raise SystemExit(1) +PY --- apiVersion: tekton.dev/v1 kind: Pipeline