merge: code queue claim move race fix

# Conflicts:
#	AGENTS.md
#	TEST.md
#	config.json
#	deploy.json
#	docs/reference/cli.md
#	docs/reference/deploy.md
#	docs/reference/microservices.md
#	scripts/cli.ts
#	scripts/src/check.ts
#	scripts/src/e2e.ts
#	scripts/src/remote.ts
#	src/components/microservices/code-queue/src/index.ts
#	src/components/microservices/code-queue/src/queue-api.ts
#	src/components/microservices/decision-center/Dockerfile
#	src/components/microservices/decision-center/src/index.ts

src/components/backend-core/src/egress-tcp.ts

@@ -93,3 +93,11 @@ export function closeEgressTcpConnectionsForProvider(providerId: string): void {
     connection.socket.destroy();
   }
 }
+
+export function closeEgressTcpConnectionsForSocket(provider: ProviderSocket): void {
+  for (const [key, connection] of ctx.activeEgressTcpConnections) {
+    if (connection.provider !== provider) continue;
+    ctx.activeEgressTcpConnections.delete(key);
+    connection.socket.destroy();
+  }
+}

src/components/backend-core/src/index.ts

@@ -9,7 +9,7 @@ import { recordRequestPerformance, withPerformanceOperation, getPerformance } fr
 import { handleProviderMessage, markProviderOffline, markStaleProvidersOffline } from "./provider-registry";
 import { markStaleTasksFailed, dispatchTask } from "./task-dispatcher";
 import { handleSshClientMessage, sshRoute } from "./ssh-bridge";
-import { closeEgressTcpConnectionsForProvider } from "./egress-tcp";
+import { closeEgressTcpConnectionsForProvider, closeEgressTcpConnectionsForSocket } from "./egress-tcp";
 import { scheduledTaskRoute, runDueScheduledTasks, recoverScheduledRuns } from "./scheduler";
 import { microserviceRoute, getMicroservices } from "./microservice-proxy";
 import { getOverview, codexQueueLoadTest } from "./overview";
@@ -171,17 +171,18 @@ const providerServer = Bun.serve<WsData>({
       const providerId = ws.data.providerId;
       logger("warn", "provider_socket_close", { providerId: providerId ?? null });
       if (providerId !== undefined) {
+        if (ctx.activeProviders.get(providerId) !== ws) {
+          closeEgressTcpConnectionsForSocket(ws);
+          logger("info", "provider_socket_close_ignored_replaced", { providerId });
+          return;
+        }
         closeEgressTcpConnectionsForProvider(providerId);
         for (const [requestId, waiter] of ctx.httpTunnelWaiters) {
           if (requestId.startsWith(`${providerId}:`)) {
             ctx.httpTunnelWaiters.delete(requestId);
-            waiter(null);
+            waiter(null, "provider-disconnected");
           }
         }
-        if (ctx.activeProviders.get(providerId) !== ws) {
-          logger("info", "provider_socket_close_ignored_replaced", { providerId });
-          return;
-        }
         markProviderOffline(providerId).catch((error) => logger("error", "provider_offline_mark_failed", { providerId, error: errorToJson(error) }));
       }
     },

src/components/backend-core/src/microservice-proxy.ts

@@ -1,6 +1,6 @@
 import type { JsonValue } from "../../shared/src/index";
 import { ctx, config, logger } from "./context";
-import type { MicroserviceConfig, MicroserviceProxyCacheEntry, MicroserviceHealthAssessment, MicroserviceAvailabilityEntry, RawTaskRow } from "./types";
+import type { HttpTunnelFailureReason, MicroserviceConfig, MicroserviceProxyCacheEntry, MicroserviceHealthAssessment, MicroserviceAvailabilityEntry, RawTaskRow } from "./types";
 import { jsonResponse, errorToJson, compactJson, isPlainRecord, truncateText } from "./http";
 import { createAndSendTask, waitForTaskTerminal, providerSupports } from "./task-dispatcher";
 import { getNodes, getNodeDockerStatuses } from "./db";
@@ -12,6 +12,7 @@ import { getNodes, getNodeDockerStatuses } from "./db";
 const microserviceProxyMaxBodyTextLength = 8 * 1024 * 1024;
 const microserviceAvailabilityTtlMs = 30_000;
 const codeQueueOverviewPathFallbackStaleMs = 30_000;
+const providerHttpTunnelMaxAttempts = 3;
 const microserviceForwardRequestHeaders = [
   "accept",
   "content-type",
@@ -468,6 +469,13 @@ function responseFromMicroserviceCache(entry: MicroserviceProxyCacheEntry, state
   });
 }
 
+function isMicroserviceTransientFailureResponse(response: Response): boolean {
+  if (response.status !== 502 && response.status !== 503 && response.status !== 504) return false;
+  return response.headers.get("x-unidesk-transient-error") === "true"
+    || response.headers.get("x-unidesk-tunnel-error") !== null
+    || response.headers.get("x-unidesk-upstream-proxy-mode") === "provider-gateway-http-fetch";
+}
+
 function readMicroserviceCache(key: string): Response | null {
   const entry = ctx.microserviceProxyCache.get(key);
   if (entry === undefined) return null;
@@ -638,43 +646,248 @@ async function k3sctlAdapterMicroserviceResponse(
   return fetchMicroserviceUpstreamResponse(adapter, method, adapterTargetPath, proxyOptions, requestHeaders, bodyText, abortSignal);
 }
 
+async function k3sctlManagedDiagnosticsResponse(service: MicroserviceConfig): Promise<Response> {
+  const adapterServiceId = service.deployment.adapterServiceId ?? "k3sctl-adapter";
+  const adapter = microserviceById(adapterServiceId);
+  const checkedAt = new Date().toISOString();
+  const providerId = adapter?.providerId ?? service.providerId;
+  const providerOnline = ctx.activeProviders.has(providerId);
+  const providerTunnelCapable = await providerSupports(providerId, "microservice.http.tunnel");
+  if (adapter === null) {
+    return jsonResponse({
+      ok: false,
+      serviceId: service.id,
+      checkedAt,
+      requestPath: "/diagnostics",
+      checks: {
+        providerGateway: { ok: providerOnline, providerId, online: providerOnline },
+        httpTunnel: { ok: providerTunnelCapable, providerId, capable: providerTunnelCapable },
+        k3sctlAdapter: { ok: false, serviceId: adapterServiceId, error: `k3sctl adapter microservice not found: ${adapterServiceId}` },
+        kubernetesApiServiceProxy: { ok: false, skipped: true },
+        targetService: { ok: false, skipped: true },
+      },
+    }, 502);
+  }
+
+  const k3sServiceId = service.id === "code-queue"
+    ? codeQueueK3sServiceIdForRequest("GET", service.backend.healthPath)
+    : service.deployment.k3sServiceId ?? service.id;
+  const adapterPath = `/api/services/${encodeURIComponent(k3sServiceId)}/diagnostics`;
+  const response = await fetchMicroserviceUpstreamResponse(
+    adapter,
+    "GET",
+    adapterPath,
+    { query: "", jsonArrayLimits: {} },
+    { accept: "application/json" },
+    "",
+  );
+  const contentType = response.headers.get("content-type") ?? "application/json; charset=utf-8";
+  const bodyText = await response.text();
+  let adapterBody: JsonValue = bodyText;
+  try {
+    adapterBody = JSON.parse(bodyText) as JsonValue;
+  } catch {
+    adapterBody = bodyText.slice(0, 4000);
+  }
+  const bodyRecord = isPlainRecord(adapterBody) ? adapterBody : {};
+  const adapterChecks = isPlainRecord(bodyRecord.checks) ? bodyRecord.checks : {};
+  const checks = {
+    providerGateway: {
+      ok: providerOnline,
+      providerId,
+      online: providerOnline,
+      activeSocketCount: ctx.activeProviders.size,
+    },
+    httpTunnel: {
+      ok: response.ok && response.headers.get("x-unidesk-proxy-mode") === "provider-ws-http-tunnel",
+      providerId,
+      capable: providerTunnelCapable,
+      requestId: response.headers.get("x-unidesk-request-id") ?? null,
+      attempts: response.headers.get("x-unidesk-http-tunnel-attempts") ?? null,
+      upstreamProxyMode: response.headers.get("x-unidesk-upstream-proxy-mode") ?? null,
+      proxyStatus: response.status,
+    },
+    k3sctlAdapter: {
+      ok: response.ok,
+      serviceId: adapter.id,
+      providerId: adapter.providerId,
+      status: response.status,
+      contentType,
+    },
+    kubernetesApiServiceProxy: compactJson(adapterChecks.kubernetesApiServiceProxy ?? { ok: false, skipped: true }),
+    targetService: compactJson(adapterChecks.targetService ?? adapterChecks.managedService ?? { ok: false, skipped: true }),
+  } satisfies Record<string, JsonValue>;
+  const httpTunnelCheck = checks.httpTunnel as Record<string, JsonValue>;
+  return jsonResponse({
+    ok: response.ok && providerOnline && httpTunnelCheck.ok === true,
+    serviceId: service.id,
+    k3sServiceId,
+    checkedAt,
+    path: service.backend.healthPath,
+    chain: "CLI/frontend -> backend-core -> provider-gateway HTTP tunnel -> k3sctl-adapter -> Kubernetes API service proxy -> k3s Service",
+    checks,
+    adapter: adapterBody,
+  }, response.ok ? 200 : response.status);
+}
+
+async function microserviceTunnelSelfTestResponse(service: MicroserviceConfig): Promise<Response> {
+  const tunnelService = isK3sctlManagedMicroservice(service)
+    ? microserviceById(service.deployment.adapterServiceId ?? "k3sctl-adapter")
+    : service;
+  if (tunnelService === null) {
+    return jsonResponse({ ok: false, serviceId: service.id, error: "tunnel service not found", adapterServiceId: service.deployment.adapterServiceId ?? null }, 502);
+  }
+  if (!(await providerSupports(tunnelService.providerId, "microservice.http.tunnel"))) {
+    return jsonResponse({
+      ok: false,
+      serviceId: service.id,
+      providerId: tunnelService.providerId,
+      error: `provider does not declare microservice.http.tunnel capability: ${tunnelService.providerId}`,
+    }, 409);
+  }
+  const probeService = {
+    ...tunnelService,
+    backend: {
+      ...tunnelService.backend,
+      nodeBaseUrl: "http://127.0.0.1:1",
+      timeoutMs: 1000,
+    },
+  };
+  const response = await providerHttpTunnelMicroserviceResponse(
+    probeService,
+    "GET",
+    "/",
+    { query: "", jsonArrayLimits: {} },
+    { accept: "application/json" },
+    "",
+  );
+  const headers = {
+    requestId: response.headers.get("x-unidesk-request-id"),
+    tunnelError: response.headers.get("x-unidesk-tunnel-error"),
+    providerId: response.headers.get("x-unidesk-provider-id"),
+    serviceId: response.headers.get("x-unidesk-service-id"),
+    transient: response.headers.get("x-unidesk-transient-error"),
+  };
+  const bodyText = await response.text();
+  let body: JsonValue = bodyText.slice(0, 4000);
+  try {
+    body = JSON.parse(bodyText) as JsonValue;
+  } catch {
+    // Keep bounded text body for malformed JSON diagnostics.
+  }
+  const bodyRecord = isPlainRecord(body) ? body : {};
+  const hasRequestId = typeof bodyRecord.requestId === "string" && bodyRecord.requestId.length > 0;
+  const hasStage = typeof bodyRecord.stage === "string" && bodyRecord.stage.length > 0;
+  const ok = response.status === 502 && hasRequestId && hasStage && headers.requestId === bodyRecord.requestId;
+  return jsonResponse({
+    ok,
+    serviceId: service.id,
+    tunnelServiceId: tunnelService.id,
+    providerId: tunnelService.providerId,
+    expectedFailure: true,
+    status: response.status,
+    checks: {
+      expectedStatus: response.status === 502,
+      bodyHasRequestId: hasRequestId,
+      bodyHasStage: hasStage,
+      headerHasRequestId: typeof headers.requestId === "string" && headers.requestId.length > 0,
+      headerHasTunnelError: typeof headers.tunnelError === "string" && headers.tunnelError.length > 0,
+    },
+    headers,
+    body,
+  }, ok ? 200 : 502);
+}
+
 function providerHttpTunnelRequestId(providerId: string): string {
   return `${providerId}:http_${Date.now()}_${Math.random().toString(16).slice(2)}`;
 }
 
+function canRetryProviderHttpTunnel(method: string, targetPath: string): boolean {
+  const normalizedMethod = method.toUpperCase();
+  if (normalizedMethod !== "GET" && normalizedMethod !== "HEAD") return false;
+  return !targetPath.endsWith("/stream");
+}
+
+function providerHttpTunnelWaitMs(service: MicroserviceConfig, attempt: number, retryable: boolean): number {
+  const baseTimeoutMs = Math.max(1000, service.backend.timeoutMs);
+  if (!retryable) return baseTimeoutMs + 3000;
+  if (attempt === 1) return Math.min(baseTimeoutMs + 3000, Math.max(5000, Math.floor(baseTimeoutMs * 0.45)));
+  if (attempt === 2) return Math.min(baseTimeoutMs + 3000, Math.max(6000, Math.floor(baseTimeoutMs * 0.7)));
+  return baseTimeoutMs + 3000;
+}
+
+function tunnelErrorBody(
+  service: MicroserviceConfig,
+  requestId: string,
+  error: string,
+  stage: string,
+  status: number,
+  extra: Record<string, JsonValue> = {},
+): Response {
+  const response = jsonResponse({
+    ok: false,
+    error,
+    stage,
+    providerId: service.providerId,
+    serviceId: service.id,
+    requestId,
+    ...extra,
+  }, status);
+  response.headers.set("x-unidesk-request-id", requestId);
+  response.headers.set("x-unidesk-provider-id", service.providerId);
+  response.headers.set("x-unidesk-service-id", service.id);
+  response.headers.set("x-unidesk-tunnel-error", stage);
+  if (status === 502 || status === 503 || status === 504) response.headers.set("x-unidesk-transient-error", "true");
+  return response;
+}
+
+function providerHttpTunnelFailureStatus(reason: HttpTunnelFailureReason | null): number {
+  if (reason === "aborted") return 499;
+  if (reason === "provider-disconnected") return 503;
+  if (reason === "send-failed") return 502;
+  return 504;
+}
+
+function tunnelFailureRetryable(reason: HttpTunnelFailureReason | null): boolean {
+  return reason === "timeout" || reason === "provider-disconnected" || reason === "send-failed";
+}
+
 async function waitForProviderHttpTunnelResponse(
   providerId: string,
   requestId: string,
   timeoutMs: number,
   abortSignal?: AbortSignal,
-): Promise<{ providerId: string; requestId: string; ok: boolean; result: JsonValue } | null> {
+): Promise<{ message: { providerId: string; requestId: string; ok: boolean; result: JsonValue } | null; reason: HttpTunnelFailureReason | null }> {
   return await new Promise((resolve) => {
     let settled = false;
     let abortHandler: (() => void) | null = null;
-    const timer = setTimeout(() => settle(null), Math.max(1, timeoutMs));
-    const settle = (message: { providerId: string; requestId: string; ok: boolean; result: JsonValue } | null): void => {
+    const timer = setTimeout(() => settle(null, "timeout"), Math.max(1, timeoutMs));
+    const settle = (
+      message: { providerId: string; requestId: string; ok: boolean; result: JsonValue } | null,
+      reason: HttpTunnelFailureReason | null = null,
+    ): void => {
       if (settled) return;
       settled = true;
       clearTimeout(timer);
       if (abortHandler !== null) abortSignal?.removeEventListener("abort", abortHandler);
       ctx.httpTunnelWaiters.delete(requestId);
-      resolve(message);
+      resolve({ message, reason });
     };
-    abortHandler = () => settle(null);
+    abortHandler = () => settle(null, "aborted");
     if (abortSignal !== undefined) {
       if (abortSignal.aborted) {
-        settle(null);
+        settle(null, "aborted");
         return;
       }
       abortSignal.addEventListener("abort", abortHandler, { once: true });
     }
-    ctx.httpTunnelWaiters.set(requestId, (message) => {
+    ctx.httpTunnelWaiters.set(requestId, (message, reason) => {
       if (message !== null && message.providerId !== providerId) {
         logger("warn", "http_tunnel_provider_mismatch", { requestId, expectedProviderId: providerId, actualProviderId: message.providerId });
-        settle(null);
+        settle(null, "provider-mismatch");
         return;
       }
-      settle(message);
+      settle(message, reason ?? null);
     });
   });
 }
@@ -688,32 +901,116 @@ async function providerHttpTunnelMicroserviceResponse(
   bodyText: string,
   abortSignal?: AbortSignal,
 ): Promise<Response> {
-  const socket = ctx.activeProviders.get(service.providerId);
-  if (socket === undefined) return jsonResponse({ ok: false, error: `provider is offline: ${service.providerId}` }, 503);
-  const requestId = providerHttpTunnelRequestId(service.providerId);
-  const timeoutMs = service.backend.timeoutMs + 3000;
-  const waiter = waitForProviderHttpTunnelResponse(service.providerId, requestId, timeoutMs, abortSignal);
-  socket.send(JSON.stringify({
-    type: "http_tunnel_request",
-    requestId,
-    payload: {
-      source: "microservice-frontend-proxy",
-      serviceId: service.id,
-      method,
-      targetBaseUrl: service.backend.nodeBaseUrl,
-      path: targetPath,
-      query: proxyOptions.query,
-      requestHeaders,
-      bodyText,
-      jsonArrayLimits: proxyOptions.jsonArrayLimits,
-      timeoutMs: service.backend.timeoutMs,
-      cacheTtlMs: providerMicroserviceCacheTtlMs(service.id, targetPath),
-    },
-  }));
-  const message = await waiter;
-  if (message === null) return jsonResponse({ ok: false, error: "provider HTTP tunnel timed out or disconnected", providerId: service.providerId, requestId }, 504);
-  if (!message.ok) return jsonResponse({ ok: false, error: "provider HTTP tunnel failed", providerId: service.providerId, requestId, result: message.result }, 502);
-  return responseFromProviderMicroserviceResult(dockerStatusRecord(message.result), "provider-ws-http-tunnel");
+  const retryable = canRetryProviderHttpTunnel(method, targetPath);
+  const maxAttempts = retryable ? providerHttpTunnelMaxAttempts : 1;
+  const attempts: JsonValue[] = [];
+  let lastRequestId = "";
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    const socket = ctx.activeProviders.get(service.providerId);
+    const requestId = providerHttpTunnelRequestId(service.providerId);
+    lastRequestId = requestId;
+    if (socket === undefined) {
+      attempts.push({ attempt, requestId, ok: false, reason: "provider-offline" });
+      return tunnelErrorBody(service, requestId, `provider is offline: ${service.providerId}`, "provider-gateway-online", 503, {
+        retryable,
+        attempts,
+      });
+    }
+    const timeoutMs = providerHttpTunnelWaitMs(service, attempt, retryable);
+    const startedAt = Date.now();
+    const waiter = waitForProviderHttpTunnelResponse(service.providerId, requestId, timeoutMs, abortSignal);
+    try {
+      socket.send(JSON.stringify({
+        type: "http_tunnel_request",
+        requestId,
+        payload: {
+          source: "microservice-frontend-proxy",
+          serviceId: service.id,
+          method,
+          targetBaseUrl: service.backend.nodeBaseUrl,
+          path: targetPath,
+          query: proxyOptions.query,
+          requestHeaders,
+          bodyText,
+          jsonArrayLimits: proxyOptions.jsonArrayLimits,
+          timeoutMs: service.backend.timeoutMs,
+          cacheTtlMs: providerMicroserviceCacheTtlMs(service.id, targetPath),
+        },
+      }));
+    } catch (error) {
+      ctx.httpTunnelWaiters.get(requestId)?.(null, "send-failed");
+      const durationMs = Date.now() - startedAt;
+      attempts.push({ attempt, requestId, ok: false, reason: "send-failed", durationMs, error: errorToJson(error) });
+      if (attempt < maxAttempts) {
+        logger("warn", "http_tunnel_send_retry", { providerId: service.providerId, serviceId: service.id, requestId, attempt, maxAttempts, error: errorToJson(error) });
+        await Bun.sleep(Math.min(500, 75 * attempt));
+        continue;
+      }
+      return tunnelErrorBody(service, requestId, "provider HTTP tunnel send failed", "http-tunnel-send", 502, {
+        retryable,
+        attempts,
+        detail: errorToJson(error),
+      });
+    }
+    const { message, reason } = await waiter;
+    const durationMs = Date.now() - startedAt;
+    if (message === null) {
+      attempts.push({ attempt, requestId, ok: false, reason: reason ?? "timeout", durationMs, timeoutMs });
+      if (retryable && tunnelFailureRetryable(reason) && attempt < maxAttempts) {
+        logger("warn", "http_tunnel_retry", {
+          providerId: service.providerId,
+          serviceId: service.id,
+          requestId,
+          attempt,
+          maxAttempts,
+          reason: reason ?? "timeout",
+          durationMs,
+          timeoutMs,
+        });
+        await Bun.sleep(Math.min(750, 100 * attempt));
+        continue;
+      }
+      return tunnelErrorBody(
+        service,
+        requestId,
+        "provider HTTP tunnel timed out or disconnected",
+        reason === "provider-disconnected" ? "http-tunnel-provider-disconnected" : reason === "aborted" ? "client-aborted" : "http-tunnel-wait",
+        providerHttpTunnelFailureStatus(reason),
+        { retryable, attempts, timeoutMs, failureReason: reason ?? "timeout" },
+      );
+    }
+    attempts.push({ attempt, requestId, ok: message.ok, durationMs });
+    if (!message.ok) {
+      const result = dockerStatusRecord(message.result);
+      const resultError = typeof result.error === "string" ? result.error : "provider HTTP tunnel failed";
+      logger("warn", "http_tunnel_provider_error", {
+        providerId: service.providerId,
+        serviceId: service.id,
+        requestId,
+        attempt,
+        maxAttempts,
+        durationMs,
+        result: compactJson(result),
+      });
+      if (retryable && attempt < maxAttempts) {
+        await Bun.sleep(Math.min(750, 100 * attempt));
+        continue;
+      }
+      return tunnelErrorBody(service, requestId, "provider HTTP tunnel failed", "provider-gateway-http-fetch", 502, {
+        retryable,
+        attempts,
+        result: message.result,
+        providerError: resultError,
+      });
+    }
+    const response = responseFromProviderMicroserviceResult(dockerStatusRecord(message.result), "provider-ws-http-tunnel");
+    response.headers.set("x-unidesk-request-id", requestId);
+    response.headers.set("x-unidesk-http-tunnel-attempt", String(attempt));
+    response.headers.set("x-unidesk-http-tunnel-attempts", String(attempts.length));
+    response.headers.set("x-unidesk-provider-id", service.providerId);
+    return response;
+  }
+  return tunnelErrorBody(service, lastRequestId, "provider HTTP tunnel exhausted attempts", "http-tunnel-wait", 504, { retryable, attempts });
 }
 
 async function fetchMicroserviceUpstreamResponse(
@@ -918,14 +1215,29 @@ export async function microserviceRoute(req: Request, url: URL): Promise<Respons
       ? "/"
       : suffix.startsWith(`${proxyPrefix}/`)
         ? `/${suffix.slice(proxyPrefix.length + 1)}`
+        : suffix === "diagnostics"
+          ? "/diagnostics"
+          : suffix === "tunnel-self-test"
+            ? "/tunnel-self-test"
         : "";
-  if (targetPath.length === 0) return jsonResponse({ ok: false, error: "microservice route must be /status, /health, or /proxy/<path>" }, 404);
+  if (targetPath.length === 0) return jsonResponse({ ok: false, error: "microservice route must be /status, /health, /diagnostics, /tunnel-self-test, or /proxy/<path>" }, 404);
   if (suffix === "health" && method !== "GET" && method !== "HEAD") {
     return jsonResponse({ ok: false, error: "microservice health only supports GET/HEAD" }, 405);
   }
+  if (suffix === "diagnostics" && method !== "GET" && method !== "HEAD") {
+    return jsonResponse({ ok: false, error: "microservice diagnostics only supports GET/HEAD" }, 405);
+  }
+  if (suffix === "tunnel-self-test" && method !== "GET" && method !== "HEAD") {
+    return jsonResponse({ ok: false, error: "microservice tunnel self-test only supports GET/HEAD" }, 405);
+  }
   if (!isMicroserviceMethodAllowed(service, method)) {
     return jsonResponse({ ok: false, error: "microservice method is not allowed", serviceId, method, allowedMethods: service.backend.allowedMethods }, 405);
   }
+  if (suffix === "diagnostics") {
+    if (!isK3sctlManagedMicroservice(service)) return strictMicroserviceHealthResponse(service, method === "HEAD");
+    return k3sctlManagedDiagnosticsResponse(service);
+  }
+  if (suffix === "tunnel-self-test") return microserviceTunnelSelfTestResponse(service);
   if (!isMicroservicePathAllowed(service, targetPath)) {
     return jsonResponse({ ok: false, error: "microservice path is not allowed", serviceId, targetPath }, 403);
   }
@@ -968,6 +1280,14 @@ export async function microserviceRoute(req: Request, url: URL): Promise<Respons
     }
   }
   const response = await fetchMicroserviceUpstreamResponse(service, method, targetPath, proxyOptions, requestHeaders, bodyText, req.signal);
+  if ((method === "GET" || method === "HEAD") && isMicroserviceTransientFailureResponse(response)) {
+    const stale = readStaleMicroserviceCache(cacheKey) ?? readMicroservicePathFallback(service, method, targetPath);
+    if (stale !== null) {
+      stale.headers.set("x-unidesk-cache", "stale-on-transient-failure");
+      stale.headers.set("x-unidesk-stale-reason", String(response.status));
+      return stale;
+    }
+  }
   if ((method === "GET" || method === "HEAD") && cacheTtlMs > 0) {
     const snapshot = await cacheableResponseSnapshot(response);
     rememberMicroserviceCache(cacheKey, cacheTtlMs, snapshot);

src/components/backend-core/src/types.ts

@@ -168,6 +168,13 @@ export interface EgressTcpConnection {
   provider: ProviderSocket;
 }
 
+export type HttpTunnelFailureReason =
+  | "timeout"
+  | "aborted"
+  | "provider-disconnected"
+  | "provider-mismatch"
+  | "send-failed";
+
 export interface MicroserviceProxyCacheEntry {
   expiresAt: number;
   staleExpiresAt: number;
@@ -193,6 +200,6 @@ export type HttpTunnelWaiter = (message: {
   requestId: string;
   ok: boolean;
   result: JsonValue;
-} | null) => void;
+} | null, reason?: HttpTunnelFailureReason) => void;
 
 export type LoggerFn = (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue) => void;

src/components/microservices/k3sctl-adapter/k3s/ci/tekton-install.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: unidesk-tekton-install
+  namespace: unidesk
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+data:
+  pipelineVersion: "v1.12.0"
+  triggersVersion: "v0.34.0"
+  pipelineReleaseUrl: "https://infra.tekton.dev/tekton-releases/pipeline/previous/v1.12.0/release.yaml"
+  triggersReleaseUrl: "https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/release.yaml"
+  triggersInterceptorsReleaseUrl: "https://infra.tekton.dev/tekton-releases/triggers/previous/v0.34.0/interceptors.yaml"

src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml

@@ -0,0 +1,593 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: unidesk-ci
+  labels:
+    app.kubernetes.io/part-of: unidesk
+    unidesk.ai/purpose: ci
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: unidesk-ci-runner
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: unidesk-ci-runner
+  namespace: unidesk-ci
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "services"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
+  - apiGroups: ["tekton.dev"]
+    resources: ["pipelineruns", "taskruns"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
+  - apiGroups: ["triggers.tekton.dev"]
+    resources: ["eventlisteners", "triggers", "triggerbindings", "triggertemplates", "interceptors"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: unidesk-ci-runner
+  namespace: unidesk-ci
+subjects:
+  - kind: ServiceAccount
+    name: unidesk-ci-runner
+    namespace: unidesk-ci
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: unidesk-ci-runner
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: unidesk-ci-trigger-reader
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+rules:
+  - apiGroups: ["triggers.tekton.dev"]
+    resources: ["clusterinterceptors", "clustertriggerbindings"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: unidesk-ci-trigger-reader
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+subjects:
+  - kind: ServiceAccount
+    name: unidesk-ci-runner
+    namespace: unidesk-ci
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: unidesk-ci-trigger-reader
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: unidesk-ci-cross-namespace
+  namespace: unidesk
+rules:
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: unidesk-ci-cross-namespace
+  namespace: unidesk
+subjects:
+  - kind: ServiceAccount
+    name: unidesk-ci-runner
+    namespace: unidesk-ci
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: unidesk-ci-cross-namespace
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: unidesk-ci-cache
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: unidesk-ci-budgets
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+data:
+  firstPaintMs: "10000"
+  traceSummaryMs: "10000"
+  traceStepsMs: "20000"
+  traceStepDetailMs: "20000"
+  overviewP95Ms: "20000"
+---
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: unidesk-repo-check
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/component: repo-check
+spec:
+  params:
+    - name: repo-url
+      type: string
+    - name: revision
+      type: string
+    - name: image
+      type: string
+      default: unidesk-code-queue:d601
+  workspaces:
+    - name: source
+  volumes:
+    - name: docker-sock
+      hostPath:
+        path: /var/run/docker.sock
+        type: Socket
+  steps:
+    - name: clone
+      image: alpine/git:2.45.2
+      env:
+        - name: HTTP_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: HTTPS_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: ALL_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: NO_PROXY
+          value: "localhost,127.0.0.1,::1,ci-git-mirror,ci-git-mirror.unidesk-ci,ci-git-mirror.unidesk-ci.svc,ci-git-mirror.unidesk-ci.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local"
+        - name: http_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: https_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: all_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: no_proxy
+          value: "localhost,127.0.0.1,::1,ci-git-mirror,ci-git-mirror.unidesk-ci,ci-git-mirror.unidesk-ci.svc,ci-git-mirror.unidesk-ci.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local"
+      script: |
+        #!/bin/sh
+        set -eu
+        rm -rf "$(workspaces.source.path)/repo"
+        git clone --filter=blob:none "$(params.repo-url)" "$(workspaces.source.path)/repo"
+        cd "$(workspaces.source.path)/repo"
+        git fetch --depth=1 origin "$(params.revision)"
+        git checkout --detach FETCH_HEAD
+        git rev-parse HEAD | tee "$(workspaces.source.path)/commit.txt"
+    - name: install-and-check
+      image: "$(params.image)"
+      env:
+        - name: DOCKER_HOST
+          value: unix:///var/run/docker.sock
+        - name: BUN_INSTALL_CACHE_DIR
+          value: "$(workspaces.source.path)/cache/bun"
+        - name: HTTP_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: HTTPS_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: ALL_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: NO_PROXY
+          value: "localhost,127.0.0.1,::1,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local"
+        - name: http_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: https_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: all_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: no_proxy
+          value: "localhost,127.0.0.1,::1,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local"
+      volumeMounts:
+        - name: docker-sock
+          mountPath: /var/run/docker.sock
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+        cd "$(workspaces.source.path)/repo"
+        command -v bun
+        command -v git
+        command -v docker
+        docker compose version >/dev/null
+        bun install --frozen-lockfile
+        (cd src && bun install --frozen-lockfile)
+        bun scripts/cli.ts check
+---
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: unidesk-code-queue-read-perf
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/component: code-queue-performance
+spec:
+  params:
+    - name: revision
+      type: string
+    - name: app-image
+      type: string
+      default: unidesk-code-queue:d601
+  workspaces:
+    - name: source
+  steps:
+    - name: start-read-service
+      image: "$(params.app-image)"
+      env:
+        - name: HTTP_PROXY
+          value: ""
+        - name: HTTPS_PROXY
+          value: ""
+        - name: ALL_PROXY
+          value: ""
+        - name: NO_PROXY
+          value: "localhost,127.0.0.1,::1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local"
+        - name: http_proxy
+          value: ""
+        - name: https_proxy
+          value: ""
+        - name: all_proxy
+          value: ""
+        - name: no_proxy
+          value: "localhost,127.0.0.1,::1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local"
+      script: |
+        #!/bin/bash
+        set -euo pipefail
+        kube_api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
+        kube_token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+        kube_ca="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        kube_namespace="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)"
+        kube() {
+          local method="$1"
+          shift
+          curl -fsS --cacert "$kube_ca" -H "Authorization: Bearer $kube_token" -X "$method" "$@"
+        }
+        cat >/tmp/code-queue-ci-read-deployment.yaml <<YAML
+        apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: code-queue-ci-read
+          namespace: unidesk-ci
+          labels:
+            app.kubernetes.io/name: code-queue
+            app.kubernetes.io/component: ci-read
+            app.kubernetes.io/part-of: unidesk
+            unidesk.ai/ci-revision: "$(params.revision)"
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app.kubernetes.io/name: code-queue
+              app.kubernetes.io/component: ci-read
+          template:
+            metadata:
+              labels:
+                app.kubernetes.io/name: code-queue
+                app.kubernetes.io/component: ci-read
+                app.kubernetes.io/part-of: unidesk
+                unidesk.ai/node-id: D601
+                unidesk.ai/ci-task-run: "$(context.taskRun.name)"
+            spec:
+              nodeSelector:
+                unidesk.ai/node-id: D601
+              terminationGracePeriodSeconds: 10
+              containers:
+                - name: code-queue
+                  image: "$(params.app-image)"
+                  imagePullPolicy: IfNotPresent
+                  ports:
+                    - name: http
+                      containerPort: 4222
+                  envFrom:
+                    - secretRef:
+                        name: code-queue-env
+                        optional: true
+                  env:
+                    - name: HOST
+                      value: "0.0.0.0"
+                    - name: PORT
+                      value: "4222"
+                    - name: DATABASE_URL
+                      value: "postgres://unidesk:unidesk_dev_password@d601-tcp-egress-gateway.unidesk.svc.cluster.local:15432/unidesk"
+                    - name: CODE_QUEUE_INSTANCE_ID
+                      value: "CI-read"
+                    - name: CODE_QUEUE_SERVICE_ROLE
+                      value: "read"
+                    - name: CODE_QUEUE_SCHEDULER_ENABLED
+                      value: "false"
+                    - name: CODE_QUEUE_STARTUP_OA_BACKFILL_ENABLED
+                      value: "false"
+                    - name: CODE_QUEUE_NOTIFY_CLAUDEQQ_ENABLED
+                      value: "false"
+                    - name: CODE_QUEUE_CODEX_SQLITE_LOG_EXPORT_ENABLED
+                      value: "false"
+                    - name: CODE_QUEUE_EGRESS_PROXY_ENABLED
+                      value: "true"
+                    - name: CODE_QUEUE_EGRESS_PROXY_URL
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: CODE_QUEUE_EGRESS_PROXY_NO_PROXY
+                      value: "localhost,127.0.0.1,::1,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,backend-core,oa-event-flow,database,hyueapi.com,.hyueapi.com"
+                    - name: HTTP_PROXY
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: HTTPS_PROXY
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: ALL_PROXY
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: NO_PROXY
+                      value: "localhost,127.0.0.1,::1,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,backend-core,oa-event-flow,database,hyueapi.com,.hyueapi.com"
+                    - name: http_proxy
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: https_proxy
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: all_proxy
+                      value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+                    - name: no_proxy
+                      value: "localhost,127.0.0.1,::1,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,backend-core,oa-event-flow,database,hyueapi.com,.hyueapi.com"
+                    - name: CODE_QUEUE_DATA_DIR
+                      value: "/var/lib/unidesk/code-queue-ci"
+                    - name: CODE_QUEUE_WORKDIR
+                      value: "/workspace"
+                    - name: CODE_QUEUE_CODEX_HOME
+                      value: "/var/lib/unidesk/code-queue-ci/codex-home"
+                    - name: CODE_QUEUE_OPENCODE_XDG_DIR
+                      value: "/var/lib/unidesk/code-queue-ci/opencode-xdg"
+                    - name: CODE_QUEUE_DEFAULT_MODEL
+                      value: "gpt-5.5"
+                    - name: CODE_QUEUE_MODELS
+                      value: "gpt-5.5,gpt-5.4-mini,gpt-5.4,minimax-m2.7"
+                    - name: CODE_QUEUE_DATABASE_POOL_MAX
+                      value: "4"
+                    - name: CODE_QUEUE_IN_MEMORY_OUTPUT_RECORDS
+                      value: "5"
+                    - name: CODE_QUEUE_IN_MEMORY_EVENT_RECORDS
+                      value: "5"
+                    - name: OA_EVENT_FLOW_BASE_URL
+                      value: "http://d601-tcp-egress-gateway.unidesk.svc.cluster.local:4255"
+                    - name: LOG_FILE
+                      value: "/var/log/unidesk/code-queue-ci-read.jsonl"
+                    - name: NODE_OPTIONS
+                      value: "--max-old-space-size=512"
+                  readinessProbe:
+                    httpGet:
+                      path: /live
+                      port: http
+                    periodSeconds: 5
+                    timeoutSeconds: 3
+                    failureThreshold: 20
+                  livenessProbe:
+                    httpGet:
+                      path: /live
+                      port: http
+                    periodSeconds: 10
+                    timeoutSeconds: 3
+                    failureThreshold: 6
+                  resources:
+                    requests:
+                      cpu: 100m
+                      memory: 256Mi
+                    limits:
+                      memory: 1Gi
+                  volumeMounts:
+                    - name: state
+                      mountPath: /var/lib/unidesk/code-queue-ci
+                    - name: logs
+                      mountPath: /var/log/unidesk
+              volumes:
+                - name: state
+                  emptyDir: {}
+                - name: logs
+                  emptyDir: {}
+        ---
+        apiVersion: v1
+        kind: Service
+        metadata:
+          name: code-queue-ci-read
+          namespace: unidesk-ci
+          labels:
+            app.kubernetes.io/name: code-queue
+            app.kubernetes.io/component: ci-read
+            app.kubernetes.io/part-of: unidesk
+        spec:
+          type: ClusterIP
+          selector:
+            app.kubernetes.io/name: code-queue
+            app.kubernetes.io/component: ci-read
+          ports:
+            - name: http
+              port: 4222
+              targetPort: http
+        YAML
+        csplit -s -f /tmp/code-queue-ci-read- /tmp/code-queue-ci-read-deployment.yaml '/^---$/' '{*}'
+        kube PATCH \
+          -H "Content-Type: application/apply-patch+yaml" \
+          --data-binary @/tmp/code-queue-ci-read-00 \
+          "$kube_api/apis/apps/v1/namespaces/$kube_namespace/deployments/code-queue-ci-read?fieldManager=unidesk-ci&force=true" >/dev/null
+        kube PATCH \
+          -H "Content-Type: application/apply-patch+yaml" \
+          --data-binary @/tmp/code-queue-ci-read-01 \
+          "$kube_api/api/v1/namespaces/$kube_namespace/services/code-queue-ci-read?fieldManager=unidesk-ci&force=true" >/dev/null
+        deadline=$((SECONDS + 180))
+        while [ "$SECONDS" -lt "$deadline" ]; do
+          status="$(kube GET "$kube_api/apis/apps/v1/namespaces/$kube_namespace/deployments/code-queue-ci-read")"
+          replicas="$(printf '%s' "$status" | jq -r '.spec.replicas // 1')"
+          available="$(printf '%s' "$status" | jq -r '.status.availableReplicas // 0')"
+          updated="$(printf '%s' "$status" | jq -r '.status.updatedReplicas // 0')"
+          observed="$(printf '%s' "$status" | jq -r '.status.observedGeneration // 0')"
+          generation="$(printf '%s' "$status" | jq -r '.metadata.generation // 0')"
+          if [ "$available" -ge "$replicas" ] && [ "$updated" -ge "$replicas" ] && [ "$observed" -ge "$generation" ]; then
+            echo "code_queue_ci_read_rollout=available replicas=$available generation=$generation"
+            exit 0
+          fi
+          sleep 2
+        done
+        echo "code_queue_ci_read_rollout=timeout" >&2
+        kube GET "$kube_api/apis/apps/v1/namespaces/$kube_namespace/deployments/code-queue-ci-read" >&2
+        exit 1
+    - name: measure
+      image: "$(params.app-image)"
+      workingDir: "$(workspaces.source.path)/repo"
+      env:
+        - name: CI_CODE_QUEUE_URL
+          value: "http://code-queue-ci-read.unidesk-ci.svc.cluster.local:4222"
+        - name: HTTP_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: HTTPS_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: ALL_PROXY
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: NO_PROXY
+          value: "localhost,127.0.0.1,::1,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local"
+        - name: http_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: https_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: all_proxy
+          value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
+        - name: no_proxy
+          value: "localhost,127.0.0.1,::1,code-queue-ci-read,code-queue-ci-read.unidesk-ci,code-queue-ci-read.unidesk-ci.svc,code-queue-ci-read.unidesk-ci.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local"
+        - name: FIRST_PAINT_BUDGET_MS
+          valueFrom:
+            configMapKeyRef:
+              name: unidesk-ci-budgets
+              key: firstPaintMs
+        - name: TRACE_SUMMARY_BUDGET_MS
+          valueFrom:
+            configMapKeyRef:
+              name: unidesk-ci-budgets
+              key: traceSummaryMs
+        - name: TRACE_STEPS_BUDGET_MS
+          valueFrom:
+            configMapKeyRef:
+              name: unidesk-ci-budgets
+              key: traceStepsMs
+        - name: TRACE_STEP_DETAIL_BUDGET_MS
+          valueFrom:
+            configMapKeyRef:
+              name: unidesk-ci-budgets
+              key: traceStepDetailMs
+        - name: OVERVIEW_P95_BUDGET_MS
+          valueFrom:
+            configMapKeyRef:
+              name: unidesk-ci-budgets
+              key: overviewP95Ms
+      script: |
+        #!/usr/bin/env bash
+        set -euo pipefail
+        bun scripts/ci-code-queue-read-perf.ts
+    - name: cleanup
+      image: "$(params.app-image)"
+      env:
+        - name: HTTP_PROXY
+          value: ""
+        - name: HTTPS_PROXY
+          value: ""
+        - name: ALL_PROXY
+          value: ""
+        - name: NO_PROXY
+          value: "localhost,127.0.0.1,::1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local"
+        - name: http_proxy
+          value: ""
+        - name: https_proxy
+          value: ""
+        - name: all_proxy
+          value: ""
+        - name: no_proxy
+          value: "localhost,127.0.0.1,::1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local"
+      script: |
+        #!/bin/bash
+        set -euo pipefail
+        kube_api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
+        kube_token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+        kube_ca="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+        kube_namespace="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)"
+        delete_resource() {
+          local path="$1"
+          local code
+          code="$(curl -sS -o /tmp/unidesk-ci-delete-response -w "%{http_code}" --cacert "$kube_ca" -H "Authorization: Bearer $kube_token" -X DELETE "$kube_api/$path")"
+          if [ "$code" = "200" ] || [ "$code" = "202" ] || [ "$code" = "404" ]; then
+            return 0
+          fi
+          cat /tmp/unidesk-ci-delete-response >&2
+          return 1
+        }
+        delete_resource "apis/apps/v1/namespaces/$kube_namespace/deployments/code-queue-ci-read"
+        delete_resource "api/v1/namespaces/$kube_namespace/services/code-queue-ci-read"
+---
+apiVersion: tekton.dev/v1
+kind: Pipeline
+metadata:
+  name: unidesk-ci
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/part-of: unidesk
+spec:
+  params:
+    - name: repo-url
+      type: string
+      default: https://github.com/pikasTech/unidesk
+    - name: revision
+      type: string
+    - name: check-image
+      type: string
+      default: unidesk-code-queue:d601
+    - name: code-queue-image
+      type: string
+      default: unidesk-code-queue:d601
+  workspaces:
+    - name: shared-workspace
+  tasks:
+    - name: repo-check
+      taskRef:
+        name: unidesk-repo-check
+      params:
+        - name: repo-url
+          value: "$(params.repo-url)"
+        - name: revision
+          value: "$(params.revision)"
+        - name: image
+          value: "$(params.check-image)"
+      workspaces:
+        - name: source
+          workspace: shared-workspace
+    - name: code-queue-read-perf
+      runAfter:
+        - repo-check
+      taskRef:
+        name: unidesk-code-queue-read-perf
+      params:
+        - name: revision
+          value: "$(params.revision)"
+        - name: app-image
+          value: "$(params.code-queue-image)"
+      workspaces:
+        - name: source
+          workspace: shared-workspace

src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.triggers.yaml

@@ -0,0 +1,80 @@
+apiVersion: triggers.tekton.dev/v1beta1
+kind: EventListener
+metadata:
+  name: unidesk-ci
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/component: triggers
+    app.kubernetes.io/part-of: unidesk
+spec:
+  serviceAccountName: unidesk-ci-runner
+  triggers:
+    - name: github-push
+      interceptors:
+        - ref:
+            name: cel
+          params:
+            - name: filter
+              value: >-
+                body.ref.startsWith('refs/heads/') &&
+                body.after.matches('^[0-9a-f]{40}$')
+      bindings:
+        - ref: unidesk-ci-github-push
+      template:
+        ref: unidesk-ci
+---
+apiVersion: triggers.tekton.dev/v1beta1
+kind: TriggerBinding
+metadata:
+  name: unidesk-ci-github-push
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/component: triggers
+    app.kubernetes.io/part-of: unidesk
+spec:
+  params:
+    - name: repo-url
+      value: $(body.repository.clone_url)
+    - name: revision
+      value: $(body.after)
+---
+apiVersion: triggers.tekton.dev/v1beta1
+kind: TriggerTemplate
+metadata:
+  name: unidesk-ci
+  namespace: unidesk-ci
+  labels:
+    app.kubernetes.io/name: unidesk-ci
+    app.kubernetes.io/component: triggers
+    app.kubernetes.io/part-of: unidesk
+spec:
+  params:
+    - name: repo-url
+      default: https://github.com/pikasTech/unidesk
+    - name: revision
+  resourcetemplates:
+    - apiVersion: tekton.dev/v1
+      kind: PipelineRun
+      metadata:
+        generateName: unidesk-ci-
+        namespace: unidesk-ci
+        labels:
+          app.kubernetes.io/name: unidesk-ci
+          app.kubernetes.io/part-of: unidesk
+          unidesk.ai/revision: $(tt.params.revision)
+      spec:
+        pipelineRef:
+          name: unidesk-ci
+        taskRunTemplate:
+          serviceAccountName: unidesk-ci-runner
+        params:
+          - name: repo-url
+            value: $(tt.params.repo-url)
+          - name: revision
+            value: $(tt.params.revision)
+        workspaces:
+          - name: shared-workspace
+            persistentVolumeClaim:
+              claimName: unidesk-ci-cache

src/components/microservices/k3sctl-adapter/src/index.ts

@@ -723,6 +723,30 @@ function parseCurlHeaderBody(output: Buffer): { status: number; contentType: str
   return { status: Number.isFinite(status) ? status : 502, contentType, bodyText };
 }
 
+function bodyPreview(bodyText: string, contentType: string): JsonValue {
+  if (contentType.toLowerCase().includes("json")) {
+    try {
+      return JSON.parse(bodyText) as JsonValue;
+    } catch {
+      return bodyText.slice(0, 2000);
+    }
+  }
+  return bodyText.slice(0, 2000);
+}
+
+function responseProbeRecord(response: Response, bodyText: string, startedAt: number): JsonRecord {
+  const contentType = response.headers.get("content-type") ?? "application/octet-stream";
+  return {
+    ok: response.ok,
+    status: response.ok ? "healthy" : "unhealthy",
+    upstreamStatus: response.status,
+    contentType,
+    proxyMode: response.headers.get("x-unidesk-proxy-mode") ?? "",
+    durationMs: Date.now() - startedAt,
+    body: bodyPreview(bodyText, contentType),
+  };
+}
+
 async function kubeApiServiceProxyResponse(
   service: ManagedService,
   req: Request,
@@ -733,6 +757,25 @@ async function kubeApiServiceProxyResponse(
   return kubeApiProxyResponse(service, req, serviceProxyApiPath(service, targetPath), query, timeoutMs);
 }
 
+async function kubeApiServiceProxyProbe(service: ManagedService, targetPath: string, timeoutMs: number): Promise<JsonRecord> {
+  const startedAt = Date.now();
+  try {
+    const request = new Request("http://k3sctl-adapter.local/diagnostics", { method: "GET", headers: { accept: "application/json" } });
+    const response = await kubeApiServiceProxyResponse(service, request, targetPath, "", timeoutMs);
+    const bodyText = await response.text();
+    return responseProbeRecord(response, bodyText, startedAt);
+  } catch (error) {
+    return {
+      ok: false,
+      status: "unhealthy",
+      upstreamStatus: null,
+      proxyMode: "kubernetes-api-service-proxy",
+      durationMs: Date.now() - startedAt,
+      error: errorToJson(error),
+    };
+  }
+}
+
 async function nativeServiceProxyResponse(
   service: ManagedService,
   req: Request,
@@ -1116,6 +1159,74 @@ async function controlPlaneSnapshot(): Promise<JsonRecord> {
   };
 }
 
+async function serviceDiagnostics(service: ManagedService): Promise<JsonRecord> {
+  const checkedAt = new Date().toISOString();
+  const healthPath = activeEndpoint(service).healthPath;
+  const healthTimeoutMs = Math.max(500, Math.min(config.healthTimeoutMs, 5000));
+  const kubernetesApiServiceProxy = await kubeApiServiceProxyProbe(service, healthPath, healthTimeoutMs);
+  const targetServiceStartedAt = Date.now();
+  let targetService: JsonRecord;
+  try {
+    const nativeRequest = new Request("http://k3sctl-adapter.local/diagnostics", { method: "GET", headers: { accept: "application/json" } });
+    const native = await nativeServiceProxyResponse(service, nativeRequest.clone(), healthPath, "", healthTimeoutMs);
+    const response = native ?? await kubeApiServiceProxyResponse(service, nativeRequest, healthPath, "", healthTimeoutMs);
+    const bodyText = await response.text();
+    targetService = responseProbeRecord(response, bodyText, targetServiceStartedAt);
+  } catch (error) {
+    targetService = {
+      ok: false,
+      status: "unhealthy",
+      upstreamStatus: null,
+      proxyMode: "k3sctl-service-health",
+      durationMs: Date.now() - targetServiceStartedAt,
+      error: errorToJson(error),
+    };
+  }
+  const managedService = await serviceStatus(service).then((status) => ({
+    ok: status.healthy === true,
+    status: String(status.status ?? ""),
+    servingHealthy: status.servingHealthy === true,
+    topologyComplete: status.topologyComplete === true,
+    activeInstanceId: String(status.activeInstanceId ?? ""),
+    active: status.active ?? null,
+    missingNodeIds: Array.isArray(status.missingNodeIds) ? status.missingNodeIds as JsonValue : [],
+  } satisfies JsonRecord)).catch((error) => ({
+    ok: false,
+    status: "unhealthy",
+    error: errorToJson(error),
+  } satisfies JsonRecord));
+  const kubernetesApiServiceProxyOk = kubernetesApiServiceProxy.ok === true;
+  const targetServiceOk = targetService.ok === true;
+  const checks = {
+    k3sctlAdapter: {
+      ok: true,
+      nodeId: config.nodeId,
+      clusterId: config.clusterId,
+      startedAt,
+    },
+    kubernetesApiServiceProxy: {
+      ...kubernetesApiServiceProxy,
+      configured: kubeClient !== null,
+      kubeconfigPath: config.kubeconfigPath,
+      connectHost: config.kubeApiConnectHost,
+    },
+    targetService,
+    managedService,
+  } satisfies Record<string, JsonValue>;
+  const ok = kubernetesApiServiceProxyOk && targetServiceOk;
+  return {
+    ok,
+    service: "k3sctl-adapter",
+    serviceId: service.id,
+    namespace: service.namespace,
+    checkedAt,
+    healthPath,
+    route: service.route,
+    noFallback: true,
+    checks,
+  };
+}
+
 function forwardHeaders(request: Request): Headers {
   const headers = new Headers();
   for (const name of ["accept", "content-type", "x-requested-with"]) {
@@ -1165,6 +1276,13 @@ async function route(req: Request): Promise<Response> {
       const status = await serviceStatus(service);
       return req.method === "HEAD" ? new Response(null, { status: status.healthy === true ? 200 : 503 }) : jsonResponse({ ok: status.healthy === true, managedService: status }, status.healthy === true ? 200 : 503);
     }
+    const diagnosticsMatch = url.pathname.match(/^\/api\/services\/([^/]+)\/diagnostics$/u);
+    if (diagnosticsMatch !== null && (req.method === "GET" || req.method === "HEAD")) {
+      const service = serviceById(decodeURIComponent(diagnosticsMatch[1] ?? ""));
+      if (service === null) return jsonResponse({ ok: false, error: "managed service not found" }, 404);
+      const diagnostics = await serviceDiagnostics(service);
+      return req.method === "HEAD" ? new Response(null, { status: diagnostics.ok === true ? 200 : 503 }) : jsonResponse(diagnostics, diagnostics.ok === true ? 200 : 503);
+    }
     const proxyMatch = url.pathname.match(/^\/api\/services\/([^/]+)\/proxy(\/.*)$/u);
     if (proxyMatch !== null) {
       const service = serviceById(decodeURIComponent(proxyMatch[1] ?? ""));