fix: migrate code queue control plane to k3s

This commit is contained in:
Codex
2026-05-16 12:01:09 +00:00
parent 83ed99f659
commit 111ec9f1eb
40 changed files with 419 additions and 366 deletions
@@ -0,0 +1,23 @@
ARG K3SCTL_ADAPTER_BASE_IMAGE=oven/bun:1-debian
FROM ${K3SCTL_ADAPTER_BASE_IMAGE}
# Never build the adapter FROM a service image: inherited Docker Desktop labels
# can silently republish old Code Queue ports and mounts.
RUN test -z "${CODE_QUEUE_DATA_DIR:-}" && test "${PORT:-}" != "4222"
ENTRYPOINT []
RUN (command -v curl >/dev/null 2>&1 && command -v ssh >/dev/null 2>&1 && command -v ps >/dev/null 2>&1) \
|| (apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates curl openssh-client procps \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*)
WORKDIR /app/src/components/microservices/k3sctl-adapter
COPY src/components/shared /app/src/components/shared
COPY src/components/microservices/k3sctl-adapter/package.json ./package.json
COPY src/components/microservices/k3sctl-adapter/tsconfig.json ./tsconfig.json
COPY src/components/microservices/k3sctl-adapter/src ./src
COPY src/components/microservices/k3sctl-adapter/k3s ./k3s
EXPOSE 4266
CMD ["bun", "--smol", "run", "src/index.ts"]
@@ -0,0 +1,46 @@
services:
k3sctl-adapter:
image: unidesk-k3sctl-adapter:d601
build:
context: ../../../..
dockerfile: src/components/microservices/k3sctl-adapter/Dockerfile
args:
K3SCTL_ADAPTER_BASE_IMAGE: ${K3SCTL_ADAPTER_BASE_IMAGE:-oven/bun:1-debian}
container_name: k3sctl-adapter
restart: unless-stopped
env_file:
- path: ${K3SCTL_ADAPTER_ENV_FILE:-../../../../.state/k3sctl-adapter-d601.env}
required: false
ports:
- "127.0.0.1:${K3SCTL_ADAPTER_HOST_PORT:-4266}:4266"
environment:
HOST: "0.0.0.0"
PORT: "4266"
LOG_FILE: "/var/log/unidesk/k3sctl-adapter.jsonl"
K3SCTL_CLUSTER_ID: "${K3SCTL_CLUSTER_ID:-unidesk-k3s}"
K3SCTL_NODE_ID: "${K3SCTL_NODE_ID:-D601}"
K3SCTL_KUBECTL_ENABLED: "${K3SCTL_KUBECTL_ENABLED:-false}"
K3SCTL_KUBE_API_PROXY_ENABLED: "${K3SCTL_KUBE_API_PROXY_ENABLED:-true}"
K3SCTL_KUBECONFIG_PATH: "/var/lib/unidesk/k3s/kubeconfig"
K3SCTL_KUBE_API_CONNECT_HOST: "${K3SCTL_KUBE_API_CONNECT_HOST:-host.docker.internal}"
K3SCTL_MANIFEST_PATHS: "${K3SCTL_MANIFEST_PATHS:-k3s/code-queue.k3s.json,k3s/mdtodo.k3s.json}"
K3SCTL_SERVICES_JSON: "${K3SCTL_SERVICES_JSON:-[]}"
UNIDESK_LOG_RETENTION_BYTES: "${UNIDESK_LOG_RETENTION_BYTES:-512MiB}"
volumes:
- ${K3SCTL_ADAPTER_LOG_DIR:-../../../../.state/k3sctl-adapter/logs}:/var/log/unidesk
- ${K3SCTL_KUBECONFIG_HOST_PATH:-/etc/rancher/k3s/k3s.yaml}:/var/lib/unidesk/k3s/kubeconfig:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- default
- provider-gateway
healthcheck:
test: ["CMD-SHELL", "curl -fsS --max-time 2 http://127.0.0.1:4266/health >/dev/null"]
interval: 5s
timeout: 3s
retries: 20
networks:
provider-gateway:
external: true
name: ${K3SCTL_PROVIDER_GATEWAY_NETWORK:-unidesk-provider-d601_default}
@@ -0,0 +1,46 @@
{
"apiVersion": "unidesk.ai/k3s/v1",
"kind": "ManagedKubernetesService",
"metadata": {
"name": "code-queue",
"namespace": "unidesk"
},
"spec": {
"adapterServiceId": "k3sctl-adapter",
"controlPlane": {
"type": "kubernetes",
"cluster": "unidesk-k3s",
"context": "unidesk-k3s"
},
"route": {
"kind": "kubernetes-service",
"serviceName": "code-queue",
"servicePort": 4222
},
"activeInstanceId": "D601",
"singleWriter": true,
"expectedNodeIds": [
"D601",
"D518"
],
"instances": [
{
"id": "D601",
"nodeId": "D601",
"role": "primary",
"baseUrl": "kubernetes://unidesk/services/code-queue:4222",
"healthPath": "/health",
"healthMode": "service-proxy"
},
{
"id": "D518",
"nodeId": "D518",
"role": "standby",
"baseUrl": "kubernetes://unidesk/services/code-queue-d518:4222",
"healthPath": "/health",
"healthMode": "pod-ready"
}
],
"requireAllInstancesHealthy": false
}
}
@@ -0,0 +1,738 @@
apiVersion: v1
kind: Namespace
metadata:
name: unidesk
labels:
app.kubernetes.io/part-of: unidesk
unidesk.ai/k3s-cluster: unidesk-k3s
---
apiVersion: v1
kind: Service
metadata:
name: d601-provider-egress-proxy
namespace: unidesk
labels:
app.kubernetes.io/name: provider-egress-proxy
app.kubernetes.io/part-of: unidesk
unidesk.ai/provider-id: D601
spec:
type: ClusterIP
ports:
- name: http
port: 18789
targetPort: 18789
protocol: TCP
---
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
name: d601-provider-egress-proxy
namespace: unidesk
labels:
kubernetes.io/service-name: d601-provider-egress-proxy
app.kubernetes.io/name: provider-egress-proxy
app.kubernetes.io/part-of: unidesk
unidesk.ai/provider-id: D601
addressType: IPv4
ports:
- name: http
protocol: TCP
port: 18789
endpoints:
- addresses:
- "172.25.0.3"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: d601-tcp-egress-gateway
namespace: unidesk
labels:
app.kubernetes.io/name: tcp-egress-gateway
app.kubernetes.io/part-of: unidesk
unidesk.ai/provider-id: D601
data:
tcp-egress-gateway.js: |
const net = require("node:net");
const proxyUrl = new URL(process.env.TCP_EGRESS_HTTP_PROXY || "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789");
const proxyHost = proxyUrl.hostname;
const proxyPort = Number(proxyUrl.port || 80);
const healthPort = Number(process.env.TCP_EGRESS_HEALTH_PORT || 18080);
const rawRoutes = process.env.TCP_EGRESS_ROUTES || "";
const startedAt = new Date().toISOString();
function parseRoute(text) {
const parts = text.trim().split("=");
if (parts.length !== 3) throw new Error(`invalid route: ${text}`);
const [name, listenPortText, target] = parts;
const [targetHost, targetPortText] = target.split(":");
const listenPort = Number(listenPortText);
const targetPort = Number(targetPortText);
if (!name || !targetHost || !Number.isInteger(listenPort) || !Number.isInteger(targetPort)) throw new Error(`invalid route: ${text}`);
return { name, listenPort, targetHost, targetPort };
}
const routes = rawRoutes.split(",").map((item) => item.trim()).filter(Boolean).map(parseRoute);
if (routes.length === 0) throw new Error("TCP_EGRESS_ROUTES must define at least one route");
function findHeaderEnd(buffer) {
for (let index = 0; index <= buffer.length - 4; index += 1) {
if (buffer[index] === 13 && buffer[index + 1] === 10 && buffer[index + 2] === 13 && buffer[index + 3] === 10) return index + 4;
}
return -1;
}
function log(level, message, data = {}) {
console.log(JSON.stringify({ ts: new Date().toISOString(), service: "tcp-egress-gateway", level, message, data }));
}
function pipeViaHttpConnect(client, route) {
const proxy = net.connect({ host: proxyHost, port: proxyPort });
const earlyClientChunks = [];
const proxyHeaderChunks = [];
let tunnelReady = false;
let closed = false;
const closeBoth = () => {
if (closed) return;
closed = true;
client.destroy();
proxy.destroy();
};
client.on("data", (chunk) => {
if (tunnelReady) proxy.write(chunk);
else earlyClientChunks.push(chunk);
});
client.on("error", closeBoth);
client.on("close", closeBoth);
proxy.on("connect", () => {
proxy.write(`CONNECT ${route.targetHost}:${route.targetPort} HTTP/1.1\r\nHost: ${route.targetHost}:${route.targetPort}\r\n\r\n`);
});
proxy.on("data", (chunk) => {
if (tunnelReady) {
client.write(chunk);
return;
}
proxyHeaderChunks.push(chunk);
const headerBuffer = Buffer.concat(proxyHeaderChunks);
const headerEnd = findHeaderEnd(headerBuffer);
if (headerEnd < 0) return;
const header = headerBuffer.subarray(0, headerEnd).toString("latin1");
if (!/^HTTP\/1\.[01] 200\b/u.test(header)) {
log("warn", "connect_rejected", { route: route.name, header: header.slice(0, 200) });
closeBoth();
return;
}
tunnelReady = true;
const remaining = headerBuffer.subarray(headerEnd);
if (remaining.length > 0) client.write(remaining);
for (const early of earlyClientChunks) proxy.write(early);
earlyClientChunks.length = 0;
});
proxy.on("error", (error) => {
log("warn", "proxy_error", { route: route.name, error: String(error?.message || error) });
closeBoth();
});
proxy.on("close", closeBoth);
}
for (const route of routes) {
net.createServer((client) => pipeViaHttpConnect(client, route)).listen(route.listenPort, "0.0.0.0", () => {
log("info", "route_listening", { route: route.name, listenPort: route.listenPort, target: `${route.targetHost}:${route.targetPort}`, proxy: `${proxyHost}:${proxyPort}` });
});
}
Bun.serve({
hostname: "0.0.0.0",
port: healthPort,
fetch() {
return Response.json({ ok: true, service: "tcp-egress-gateway", startedAt, proxy: `${proxyHost}:${proxyPort}`, routes });
},
});
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: d601-tcp-egress-gateway
namespace: unidesk
labels:
app.kubernetes.io/name: tcp-egress-gateway
app.kubernetes.io/part-of: unidesk
unidesk.ai/provider-id: D601
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: tcp-egress-gateway
unidesk.ai/provider-id: D601
template:
metadata:
labels:
app.kubernetes.io/name: tcp-egress-gateway
app.kubernetes.io/part-of: unidesk
unidesk.ai/provider-id: D601
unidesk.ai/node-id: D601
spec:
nodeSelector:
unidesk.ai/node-id: D601
containers:
- name: tcp-egress-gateway
image: unidesk-code-queue:d601
imagePullPolicy: IfNotPresent
command:
- bun
- /etc/unidesk-tcp-egress/tcp-egress-gateway.js
ports:
- name: pg
containerPort: 15432
- name: oa
containerPort: 4255
- name: health
containerPort: 18080
env:
- name: TCP_EGRESS_HTTP_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: TCP_EGRESS_ROUTES
value: "postgres=15432=74.48.78.17:15432,oa-event-flow=4255=74.48.78.17:4255"
- name: TCP_EGRESS_HEALTH_PORT
value: "18080"
volumeMounts:
- name: script
mountPath: /etc/unidesk-tcp-egress
readOnly: true
readinessProbe:
httpGet:
path: /health
port: health
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 12
livenessProbe:
httpGet:
path: /health
port: health
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 6
volumes:
- name: script
configMap:
name: d601-tcp-egress-gateway
---
apiVersion: v1
kind: Service
metadata:
name: d601-tcp-egress-gateway
namespace: unidesk
labels:
app.kubernetes.io/name: tcp-egress-gateway
app.kubernetes.io/part-of: unidesk
unidesk.ai/provider-id: D601
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: tcp-egress-gateway
unidesk.ai/provider-id: D601
ports:
- name: pg
port: 15432
targetPort: pg
- name: oa
port: 4255
targetPort: oa
- name: health
port: 18080
targetPort: health
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: code-queue
namespace: unidesk
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: code-queue
unidesk.ai/instance-id: D601
template:
metadata:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
unidesk.ai/node-id: D601
spec:
nodeSelector:
unidesk.ai/node-id: D601
terminationGracePeriodSeconds: 30
containers:
- name: code-queue
image: unidesk-code-queue:d601
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 4222
envFrom:
- secretRef:
name: code-queue-env
optional: true
env:
- name: HOST
value: "0.0.0.0"
- name: PORT
value: "4222"
- name: DATABASE_URL
value: "postgres://unidesk:unidesk_dev_password@d601-tcp-egress-gateway.unidesk.svc.cluster.local:15432/unidesk"
- name: CODE_QUEUE_INSTANCE_ID
value: "D601"
- name: CODE_QUEUE_SCHEDULER_ENABLED
value: "true"
- name: CODE_QUEUE_STARTUP_OA_BACKFILL_ENABLED
value: "false"
- name: CODE_QUEUE_DATA_DIR
value: "/var/lib/unidesk/code-queue"
- name: CODE_QUEUE_WORKDIR
value: "/workspace"
- name: CODE_QUEUE_CODEX_HOME
value: "/var/lib/unidesk/code-queue/codex-home"
- name: CODE_QUEUE_OPENCODE_XDG_DIR
value: "/var/lib/unidesk/code-queue/opencode-xdg"
- name: CODE_QUEUE_SOURCE_CODEX_CONFIG
value: "/root/.codex/config.toml"
- name: CODE_QUEUE_DEFAULT_MODEL
value: "gpt-5.5"
- name: CODE_QUEUE_MODELS
value: "gpt-5.5,gpt-5.4-mini,gpt-5.4,minimax-m2.7"
- name: CODE_QUEUE_MODEL_REASONING_EFFORTS
value: "gpt-5.5=xhigh"
- name: CODE_QUEUE_SANDBOX
value: "danger-full-access"
- name: CODE_QUEUE_APPROVAL_POLICY
value: "never"
- name: CODE_QUEUE_MAX_ACTIVE_QUEUES
value: "0"
- name: CODE_QUEUE_DATABASE_POOL_MAX
value: "2"
- name: NODE_OPTIONS
value: "--max-old-space-size=1024"
- name: GIT_CONFIG_COUNT
value: "1"
- name: GIT_CONFIG_KEY_0
value: "safe.directory"
- name: GIT_CONFIG_VALUE_0
value: "*"
- name: CODE_QUEUE_IN_MEMORY_OUTPUT_RECORDS
value: "10"
- name: CODE_QUEUE_IN_MEMORY_EVENT_RECORDS
value: "10"
- name: CODE_QUEUE_MAIN_PROVIDER_ID
value: "D601"
- name: CODE_QUEUE_REMOTE_WORKDIR
value: "/home/ubuntu"
- name: CODE_QUEUE_EXECUTION_PROVIDER_IDS
value: "D601"
- name: CODE_QUEUE_DEV_CONTAINER_MASTER_HOST
value: "74.48.78.17"
- name: CODE_QUEUE_DEV_CONTAINER_DEFAULT_PROVIDER_ID
value: "D601"
- name: CODE_QUEUE_DEV_CONTAINER_WORKDIR
value: "/home/ubuntu"
- name: CODE_QUEUE_EGRESS_PROXY_ENABLED
value: "true"
- name: CODE_QUEUE_EGRESS_PROXY_URL
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: CODE_QUEUE_EGRESS_PROXY_NO_PROXY
value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,172.25.0.3,unidesk-provider-gateway-D601,backend-core,oa-event-flow,database,hyueapi.com,.hyueapi.com"
- name: HTTP_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: HTTPS_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: ALL_PROXY
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: http_proxy
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: https_proxy
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: all_proxy
value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789"
- name: NO_PROXY
value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,172.25.0.3,unidesk-provider-gateway-D601,backend-core,oa-event-flow,database,hyueapi.com,.hyueapi.com"
- name: no_proxy
value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,172.25.0.3,unidesk-provider-gateway-D601,backend-core,oa-event-flow,database,hyueapi.com,.hyueapi.com"
- name: OA_EVENT_FLOW_BASE_URL
value: "http://d601-tcp-egress-gateway.unidesk.svc.cluster.local:4255"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_ENABLED
value: "true"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_BASE_URL
value: "http://host.docker.internal:3290"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_TARGET_TYPE
value: "private"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_USER_ID
value: "645275593"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_MAX_RESPONSE_CHARS
value: "12000"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_TIMEOUT_MS
value: "15000"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_SEND_ATTEMPTS
value: "3"
- name: LOG_FILE
value: "/var/log/unidesk/code-queue.jsonl"
- name: UNIDESK_LOG_RETENTION_BYTES
value: "1GiB"
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
- name: workspace
mountPath: /workspace
- name: repo
mountPath: /root/unidesk
- name: repo
mountPath: /app
- name: codex-config
mountPath: /root/.codex/config.toml
readOnly: true
- name: codex-auth
mountPath: /root/.codex/auth.json
readOnly: true
- name: ssh-dir
mountPath: /root/.ssh
readOnly: true
- name: logs
mountPath: /var/log/unidesk
- name: state
mountPath: /var/lib/unidesk/code-queue
readinessProbe:
httpGet:
path: /health
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 20
livenessProbe:
httpGet:
path: /live
port: http
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 6
startupProbe:
httpGet:
path: /live
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
memory: 4Gi
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
type: Socket
- name: workspace
hostPath:
path: /home/ubuntu
type: Directory
- name: repo
hostPath:
path: /home/ubuntu/cq-deploy
type: Directory
- name: codex-config
hostPath:
path: /home/ubuntu/.codex/config.toml
type: File
- name: codex-auth
hostPath:
path: /home/ubuntu/.codex/auth.json
type: File
- name: ssh-dir
hostPath:
path: /home/ubuntu/.ssh
type: Directory
- name: logs
hostPath:
path: /home/ubuntu/cq-deploy/.state/code-queue/logs
type: DirectoryOrCreate
- name: state
hostPath:
path: /home/ubuntu/cq-deploy/.state/code-queue
type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
name: code-queue
namespace: unidesk
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: code-queue
unidesk.ai/instance-id: D601
ports:
- name: http
port: 4222
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: code-queue-d518
namespace: unidesk
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D518
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: code-queue
unidesk.ai/instance-id: D518
template:
metadata:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D518
unidesk.ai/node-id: D518
spec:
nodeSelector:
unidesk.ai/node-id: D518
terminationGracePeriodSeconds: 30
containers:
- name: code-queue
image: unidesk-code-queue:d601
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 4222
envFrom:
- secretRef:
name: code-queue-env
optional: true
env:
- name: HOST
value: "0.0.0.0"
- name: PORT
value: "4222"
- name: DATABASE_URL
value: "postgres://unidesk:unidesk_dev_password@d601-tcp-egress-gateway.unidesk.svc.cluster.local:15432/unidesk"
- name: CODE_QUEUE_INSTANCE_ID
value: "D518"
- name: CODE_QUEUE_SCHEDULER_ENABLED
value: "false"
- name: CODE_QUEUE_STARTUP_OA_BACKFILL_ENABLED
value: "false"
- name: CODE_QUEUE_DATA_DIR
value: "/var/lib/unidesk/code-queue"
- name: CODE_QUEUE_WORKDIR
value: "/workspace"
- name: CODE_QUEUE_CODEX_HOME
value: "/var/lib/unidesk/code-queue/codex-home"
- name: CODE_QUEUE_OPENCODE_XDG_DIR
value: "/var/lib/unidesk/code-queue/opencode-xdg"
- name: CODE_QUEUE_SOURCE_CODEX_CONFIG
value: "/root/.codex/config.toml"
- name: CODE_QUEUE_DEFAULT_MODEL
value: "gpt-5.5"
- name: CODE_QUEUE_MODELS
value: "gpt-5.5,gpt-5.4-mini,gpt-5.4,minimax-m2.7"
- name: CODE_QUEUE_MODEL_REASONING_EFFORTS
value: "gpt-5.5=xhigh"
- name: CODE_QUEUE_SANDBOX
value: "danger-full-access"
- name: CODE_QUEUE_APPROVAL_POLICY
value: "never"
- name: CODE_QUEUE_MAX_ACTIVE_QUEUES
value: "0"
- name: CODE_QUEUE_DATABASE_POOL_MAX
value: "2"
- name: NODE_OPTIONS
value: "--max-old-space-size=1024"
- name: GIT_CONFIG_COUNT
value: "1"
- name: GIT_CONFIG_KEY_0
value: "safe.directory"
- name: GIT_CONFIG_VALUE_0
value: "*"
- name: CODE_QUEUE_IN_MEMORY_OUTPUT_RECORDS
value: "10"
- name: CODE_QUEUE_IN_MEMORY_EVENT_RECORDS
value: "10"
- name: CODE_QUEUE_MAIN_PROVIDER_ID
value: "D518"
- name: CODE_QUEUE_REMOTE_WORKDIR
value: "/home/ubuntu"
- name: CODE_QUEUE_EXECUTION_PROVIDER_IDS
value: "D518"
- name: CODE_QUEUE_DEV_CONTAINER_MASTER_HOST
value: "74.48.78.17"
- name: CODE_QUEUE_DEV_CONTAINER_DEFAULT_PROVIDER_ID
value: "D518"
- name: CODE_QUEUE_DEV_CONTAINER_WORKDIR
value: "/home/ubuntu"
- name: CODE_QUEUE_EGRESS_PROXY_ENABLED
value: "false"
- name: CODE_QUEUE_EGRESS_PROXY_URL
value: ""
- name: CODE_QUEUE_EGRESS_PROXY_NO_PROXY
value: "localhost,127.0.0.1,::1,host.docker.internal,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,backend-core,oa-event-flow,database"
- name: HTTP_PROXY
value: ""
- name: HTTPS_PROXY
value: ""
- name: ALL_PROXY
value: ""
- name: http_proxy
value: ""
- name: https_proxy
value: ""
- name: all_proxy
value: ""
- name: NO_PROXY
value: "localhost,127.0.0.1,::1,host.docker.internal,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,backend-core,oa-event-flow,database"
- name: no_proxy
value: "localhost,127.0.0.1,::1,host.docker.internal,d601-tcp-egress-gateway,d601-tcp-egress-gateway.unidesk,d601-tcp-egress-gateway.unidesk.svc,d601-tcp-egress-gateway.unidesk.svc.cluster.local,backend-core,oa-event-flow,database"
- name: OA_EVENT_FLOW_BASE_URL
value: "http://d601-tcp-egress-gateway.unidesk.svc.cluster.local:4255"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_ENABLED
value: "false"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_BASE_URL
value: ""
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_TARGET_TYPE
value: "private"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_USER_ID
value: "645275593"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_MAX_RESPONSE_CHARS
value: "12000"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_TIMEOUT_MS
value: "15000"
- name: CODE_QUEUE_NOTIFY_CLAUDEQQ_SEND_ATTEMPTS
value: "3"
- name: LOG_FILE
value: "/var/log/unidesk/code-queue-d518.jsonl"
- name: UNIDESK_LOG_RETENTION_BYTES
value: "1GiB"
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
- name: workspace
mountPath: /workspace
- name: repo
mountPath: /root/unidesk
- name: repo
mountPath: /app
- name: codex-config
mountPath: /root/.codex/config.toml
readOnly: true
- name: codex-auth
mountPath: /root/.codex/auth.json
readOnly: true
- name: ssh-dir
mountPath: /root/.ssh
readOnly: true
- name: logs
mountPath: /var/log/unidesk
- name: state
mountPath: /var/lib/unidesk/code-queue
readinessProbe:
httpGet:
path: /health
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 20
livenessProbe:
httpGet:
path: /health
port: http
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 6
startupProbe:
httpGet:
path: /health
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
memory: 4Gi
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
type: Socket
- name: workspace
hostPath:
path: /home/ubuntu
type: Directory
- name: repo
hostPath:
path: /home/ubuntu/cq-deploy
type: Directory
- name: codex-config
hostPath:
path: /home/ubuntu/.codex/config.toml
type: File
- name: codex-auth
hostPath:
path: /home/ubuntu/.codex/auth.json
type: File
- name: ssh-dir
hostPath:
path: /home/ubuntu/.ssh
type: Directory
- name: logs
hostPath:
path: /home/ubuntu/cq-deploy/.state/code-queue/logs
type: DirectoryOrCreate
- name: state
hostPath:
path: /home/ubuntu/cq-deploy/.state/code-queue
type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
name: code-queue-d518
namespace: unidesk
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D518
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: code-queue
unidesk.ai/instance-id: D518
ports:
- name: http
port: 4222
targetPort: http
@@ -0,0 +1,37 @@
{
"apiVersion": "unidesk.ai/k3s/v1",
"kind": "ManagedKubernetesService",
"metadata": {
"name": "mdtodo",
"namespace": "unidesk"
},
"spec": {
"adapterServiceId": "k3sctl-adapter",
"controlPlane": {
"type": "kubernetes",
"cluster": "unidesk-k3s",
"context": "unidesk-k3s"
},
"route": {
"kind": "kubernetes-service",
"serviceName": "mdtodo",
"servicePort": 4267
},
"activeInstanceId": "D601",
"singleWriter": true,
"expectedNodeIds": [
"D601"
],
"instances": [
{
"id": "D601",
"nodeId": "D601",
"role": "primary",
"baseUrl": "kubernetes://unidesk/services/mdtodo:4267",
"healthPath": "/health",
"healthMode": "service-proxy"
}
],
"requireAllInstancesHealthy": true
}
}
@@ -0,0 +1,106 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: mdtodo
namespace: unidesk
labels:
app.kubernetes.io/name: mdtodo
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: mdtodo
unidesk.ai/instance-id: D601
template:
metadata:
labels:
app.kubernetes.io/name: mdtodo
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
unidesk.ai/node-id: D601
spec:
nodeSelector:
unidesk.ai/node-id: D601
terminationGracePeriodSeconds: 15
containers:
- name: mdtodo
image: unidesk-mdtodo:d601
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 4267
env:
- name: HOST
value: "0.0.0.0"
- name: PORT
value: "4267"
- name: MDTODO_ROOT_DIR
value: "/workspace"
- name: LOG_FILE
value: "/var/log/unidesk/mdtodo.jsonl"
- name: UNIDESK_LOG_RETENTION_BYTES
value: "512MiB"
volumeMounts:
- name: workspace
mountPath: /workspace
- name: logs
mountPath: /var/log/unidesk
readinessProbe:
httpGet:
path: /health
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 12
livenessProbe:
httpGet:
path: /live
port: http
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 6
startupProbe:
httpGet:
path: /live
port: http
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 30
resources:
requests:
cpu: 50m
memory: 96Mi
limits:
memory: 512Mi
volumes:
- name: workspace
hostPath:
path: /home/ubuntu/cq-deploy/.state/mdtodo-workspace
type: Directory
- name: logs
hostPath:
path: /home/ubuntu/cq-deploy/.state/mdtodo/logs
type: DirectoryOrCreate
---
apiVersion: v1
kind: Service
metadata:
name: mdtodo
namespace: unidesk
labels:
app.kubernetes.io/name: mdtodo
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: k3sctl-managed
spec:
type: ClusterIP
selector:
app.kubernetes.io/name: mdtodo
unidesk.ai/instance-id: D601
ports:
- name: http
port: 4267
targetPort: http
@@ -0,0 +1,9 @@
{
"name": "@unidesk/k3sctl-adapter",
"private": true,
"type": "module",
"scripts": {
"start": "bun run src/index.ts",
"check": "tsc -p tsconfig.json --noEmit"
}
}
@@ -0,0 +1,840 @@
import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../../shared/src/rotating-jsonl";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, isAbsolute, join } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
type JsonRecord = Record<string, JsonValue>;
type InstanceRole = "primary" | "standby" | "worker";
type EndpointHealthMode = "service-proxy" | "pod-ready";
interface ManagedEndpoint {
id: string;
nodeId: string;
role: InstanceRole;
baseUrl: string;
healthPath: string;
healthMode: EndpointHealthMode;
}
interface ManagedService {
id: string;
namespace: string;
kind: string;
controlPlane: JsonRecord;
route: JsonRecord;
activeInstanceId: string;
singleWriter: boolean;
requireAllInstancesHealthy: boolean;
expectedNodeIds: string[];
endpoints: ManagedEndpoint[];
}
interface RuntimeConfig {
host: string;
port: number;
logFile: string;
manifestPaths: string[];
clusterId: string;
nodeId: string;
kubectlEnabled: boolean;
kubectlContext: string;
kubeApiProxyEnabled: boolean;
kubeconfigPath: string;
kubeApiConnectHost: string;
requestTimeoutMs: number;
healthTimeoutMs: number;
services: ManagedService[];
}
interface KubeApiClient {
serverUrl: URL;
connectHost: string;
caFile: string;
certFile: string;
keyFile: string;
}
const recentLogs: JsonRecord[] = [];
const startedAt = new Date().toISOString();
const adapterRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
const config = readConfig();
const logWriter = config.logFile
? createHourlyJsonlWriter({
baseLogFile: config.logFile,
service: "k3sctl-adapter",
maxBytes: logRetentionBytesForService("k3sctl-adapter"),
})
: null;
const kubeClient = loadKubeApiClient();
logWriter?.prune();
function envString(name: string, fallback: string): string {
const value = process.env[name];
return value === undefined || value.length === 0 ? fallback : value;
}
function envNumber(name: string, fallback: number): number {
const raw = process.env[name];
if (raw === undefined || raw.trim().length === 0) return fallback;
const value = Number(raw);
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
}
function envBool(name: string, fallback: boolean): boolean {
const raw = process.env[name];
if (raw === undefined || raw.trim().length === 0) return fallback;
const normalized = raw.trim().toLowerCase();
if (["1", "true", "yes", "on"].includes(normalized)) return true;
if (["0", "false", "no", "off"].includes(normalized)) return false;
return fallback;
}
function asRecord(value: unknown, path: string): Record<string, unknown> {
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`);
return value as Record<string, unknown>;
}
function stringField(value: Record<string, unknown>, key: string, path: string): string {
const field = value[key];
if (typeof field !== "string" || field.length === 0) throw new Error(`${path}.${key} must be a non-empty string`);
return field;
}
function optionalStringField(value: Record<string, unknown>, key: string, fallback: string): string {
const field = value[key];
if (field === undefined || field === null || field === "") return fallback;
if (typeof field !== "string") throw new Error(`${key} must be a string`);
return field;
}
function optionalBoolField(value: Record<string, unknown>, key: string, fallback: boolean): boolean {
const field = value[key];
if (field === undefined || field === null) return fallback;
if (typeof field !== "boolean") throw new Error(`${key} must be a boolean`);
return field;
}
function isJsonValue(value: unknown): value is JsonValue {
if (value === null) return true;
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return true;
if (Array.isArray(value)) return value.every(isJsonValue);
if (typeof value === "object") return Object.values(value as Record<string, unknown>).every(isJsonValue);
return false;
}
function manifestJsonRecord(value: unknown, path: string): JsonRecord {
if (value === undefined || value === null) return {};
if (typeof value !== "object" || Array.isArray(value) || !isJsonValue(value)) throw new Error(`${path} must be a JSON object`);
return value as JsonRecord;
}
function stringArrayField(value: Record<string, unknown>, key: string, fallback: string[]): string[] {
const field = value[key];
if (field === undefined || field === null) return fallback;
if (!Array.isArray(field) || field.some((item) => typeof item !== "string" || item.length === 0)) {
throw new Error(`${key} must be an array of non-empty strings`);
}
return field;
}
function normalizeRole(value: string): InstanceRole {
if (value === "primary" || value === "standby" || value === "worker") return value;
return "worker";
}
function normalizeHealthMode(value: string): EndpointHealthMode {
if (value === "service-proxy" || value === "pod-ready") return value;
return "service-proxy";
}
function parseEndpoint(value: unknown, index: number, ownerPath = "endpoint"): ManagedEndpoint {
const path = `${ownerPath}[${index}]`;
const item = asRecord(value, path);
const id = stringField(item, "id", path);
const nodeId = optionalStringField(item, "nodeId", id);
return {
id,
nodeId,
role: normalizeRole(optionalStringField(item, "role", id === "D601" ? "primary" : "standby")),
baseUrl: stringField(item, "baseUrl", path).replace(/\/+$/u, ""),
healthPath: optionalStringField(item, "healthPath", "/health"),
healthMode: normalizeHealthMode(optionalStringField(item, "healthMode", "service-proxy")),
};
}
function parseManagedKubernetesManifest(value: unknown, index: number, ownerPath = "manifests"): ManagedService {
const path = `${ownerPath}[${index}]`;
const manifest = asRecord(value, path);
const metadata = asRecord(manifest.metadata, `${path}.metadata`);
const spec = asRecord(manifest.spec, `${path}.spec`);
const kind = optionalStringField(manifest, "kind", "ManagedKubernetesService");
const serviceId = stringField(metadata, "name", `${path}.metadata`);
if (kind !== "ManagedKubernetesService") throw new Error(`${path}.kind must be ManagedKubernetesService; direct ManagedHttpService manifests are not allowed in pure k3s mode`);
const instancesRaw = spec.instances;
if (!Array.isArray(instancesRaw) || instancesRaw.length === 0) throw new Error(`${path}.spec.instances must be a non-empty array`);
const endpoints = instancesRaw.map((endpoint, endpointIndex) => parseEndpoint(endpoint, endpointIndex, `${path}.spec.instances`));
const activeInstanceId = optionalStringField(spec, "activeInstanceId", endpoints[0]?.id ?? serviceId);
if (!endpoints.some((endpoint) => endpoint.id === activeInstanceId)) throw new Error(`${path}.spec.activeInstanceId must match one instance id`);
const controlPlane = manifestJsonRecord(spec.controlPlane, `${path}.spec.controlPlane`);
const route = manifestJsonRecord(spec.route, `${path}.spec.route`);
if (String(route.kind ?? "") !== "kubernetes-service") throw new Error(`${path}.spec.route.kind must be kubernetes-service`);
return {
id: serviceId,
namespace: optionalStringField(metadata, "namespace", optionalStringField(spec, "namespace", "unidesk")),
kind,
controlPlane,
route,
activeInstanceId,
singleWriter: optionalBoolField(spec, "singleWriter", true),
requireAllInstancesHealthy: optionalBoolField(spec, "requireAllInstancesHealthy", false),
expectedNodeIds: stringArrayField(spec, "expectedNodeIds", endpoints.map((endpoint) => endpoint.nodeId)),
endpoints,
};
}
function parseServiceOrManifest(value: unknown, index: number, ownerPath = "services"): ManagedService {
const item = asRecord(value, `${ownerPath}[${index}]`);
if (typeof item.kind === "string" || item.metadata !== undefined || item.spec !== undefined) {
return parseManagedKubernetesManifest(item, index, ownerPath);
}
throw new Error(`${ownerPath}[${index}] must be a ManagedKubernetesService manifest; static HTTP service declarations are not allowed in pure k3s mode`);
}
function parseServices(raw: string, ownerPath = "K3SCTL_SERVICES_JSON"): ManagedService[] {
const value = raw.trim().length === 0 ? [] : JSON.parse(raw) as unknown;
if (!Array.isArray(value)) throw new Error("K3SCTL_SERVICES_JSON must be an array");
return value.map((item, index) => parseServiceOrManifest(item, index, ownerPath));
}
function manifestPaths(raw: string): string[] {
return raw.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
}
function resolveManifestPath(path: string): string {
if (isAbsolute(path)) return path;
const adapterRelative = join(adapterRoot, path);
if (existsSync(adapterRelative)) return adapterRelative;
return path;
}
function readManifestServices(paths: string[]): ManagedService[] {
const services: ManagedService[] = [];
for (const path of paths) {
const resolved = resolveManifestPath(path);
const parsed = JSON.parse(readFileSync(resolved, "utf8")) as unknown;
const records = Array.isArray(parsed) ? parsed : [parsed];
for (const [index, record] of records.entries()) {
services.push(parseServiceOrManifest(record, index, `manifest:${path}`));
}
}
return services;
}
function mergeServices(services: ManagedService[]): ManagedService[] {
const seen = new Set<string>();
for (const service of services) {
const key = `${service.namespace}/${service.id}`;
if (seen.has(key)) throw new Error(`duplicate k3s managed service: ${key}`);
seen.add(key);
}
return services;
}
function readConfig(): RuntimeConfig {
const paths = manifestPaths(envString("K3SCTL_MANIFEST_PATHS", "k3s/code-queue.k3s.json"));
const inlineServices = parseServices(envString("K3SCTL_SERVICES_JSON", "[]"));
const manifestServices = readManifestServices(paths);
return {
host: envString("HOST", "0.0.0.0"),
port: envNumber("PORT", 4266),
logFile: envString("LOG_FILE", "/var/log/unidesk/k3sctl-adapter.jsonl"),
manifestPaths: paths,
clusterId: envString("K3SCTL_CLUSTER_ID", "unidesk-k3s"),
nodeId: envString("K3SCTL_NODE_ID", "D601"),
kubectlEnabled: envBool("K3SCTL_KUBECTL_ENABLED", false),
kubectlContext: envString("K3SCTL_KUBECTL_CONTEXT", ""),
kubeApiProxyEnabled: envBool("K3SCTL_KUBE_API_PROXY_ENABLED", true),
kubeconfigPath: envString("K3SCTL_KUBECONFIG_PATH", "/var/lib/unidesk/k3s/kubeconfig"),
kubeApiConnectHost: envString("K3SCTL_KUBE_API_CONNECT_HOST", "host.docker.internal"),
requestTimeoutMs: Math.max(1000, Math.min(120_000, envNumber("K3SCTL_REQUEST_TIMEOUT_MS", 30_000))),
healthTimeoutMs: Math.max(500, Math.min(30_000, envNumber("K3SCTL_HEALTH_TIMEOUT_MS", 2500))),
services: mergeServices([...manifestServices, ...inlineServices]),
};
}
function log(level: "debug" | "info" | "warn" | "error", event: string, detail: JsonRecord = {}): void {
const record: JsonRecord = { at: new Date().toISOString(), service: "k3sctl-adapter", level, event, ...detail };
recentLogs.push(record);
while (recentLogs.length > 500) recentLogs.shift();
try {
logWriter?.appendJson(record, new Date(String(record.at)));
} catch {
// Logging must never break proxying.
}
const line = JSON.stringify(record);
const writer = level === "error" ? console.error : level === "warn" ? console.warn : console.log;
writer(line);
}
function kubeconfigScalar(text: string, key: string): string {
const escaped = key.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const match = text.match(new RegExp(`^\\s*${escaped}:\\s*([^\\s]+)\\s*$`, "mu"));
return match?.[1] ?? "";
}
function writeKubeSecretFile(dir: string, name: string, base64Value: string): string {
const path = join(dir, name);
writeFileSync(path, Buffer.from(base64Value, "base64"), { mode: 0o600 });
return path;
}
function loadKubeApiClient(): KubeApiClient | null {
if (!config.kubeApiProxyEnabled) return null;
if (!existsSync(config.kubeconfigPath)) {
log("warn", "kubeconfig_missing", { path: config.kubeconfigPath });
return null;
}
try {
const raw = readFileSync(config.kubeconfigPath, "utf8");
const server = kubeconfigScalar(raw, "server");
const ca = kubeconfigScalar(raw, "certificate-authority-data");
const cert = kubeconfigScalar(raw, "client-certificate-data");
const key = kubeconfigScalar(raw, "client-key-data");
if (server.length === 0 || ca.length === 0 || cert.length === 0 || key.length === 0) throw new Error("kubeconfig must include server, CA, client certificate, and client key data");
const dir = join(tmpdir(), `unidesk-k3sctl-kube-${config.clusterId}`);
mkdirSync(dir, { recursive: true, mode: 0o700 });
const client: KubeApiClient = {
serverUrl: new URL(server),
connectHost: config.kubeApiConnectHost,
caFile: writeKubeSecretFile(dir, "ca.crt", ca),
certFile: writeKubeSecretFile(dir, "client.crt", cert),
keyFile: writeKubeSecretFile(dir, "client.key", key),
};
log("info", "kube_api_client_loaded", { kubeconfigPath: config.kubeconfigPath, serverHost: client.serverUrl.hostname, connectHost: client.connectHost });
return client;
} catch (error) {
log("error", "kube_api_client_failed", { kubeconfigPath: config.kubeconfigPath, error: errorToJson(error) });
return null;
}
}
function jsonResponse(body: unknown, status = 200, headers: Record<string, string> = {}): Response {
return new Response(JSON.stringify(body), {
status,
headers: { "content-type": "application/json; charset=utf-8", ...headers },
});
}
function errorToJson(error: unknown): JsonRecord {
if (error instanceof Error) return { name: error.name, message: error.message, stack: error.stack ?? "" };
return { message: String(error) };
}
function serviceById(id: string): ManagedService | null {
return config.services.find((service) => service.id === id) ?? null;
}
function activeEndpoint(service: ManagedService): ManagedEndpoint {
const endpoint = service.endpoints.find((item) => item.id === service.activeInstanceId);
if (endpoint === undefined) throw new Error(`active endpoint not found for service ${service.id}: ${service.activeInstanceId}`);
return endpoint;
}
function endpointUrl(endpoint: ManagedEndpoint, targetPath: string, query = ""): string {
const base = new URL(endpoint.baseUrl);
const upstream = new URL(targetPath, base);
upstream.search = query;
return upstream.toString();
}
async function boundedText(response: Response, maxBytes = 1_000_000): Promise<{ text: string; truncated: boolean }> {
const reader = response.body?.getReader();
if (reader === undefined) return { text: "", truncated: false };
const chunks: Uint8Array[] = [];
let total = 0;
let truncated = false;
while (true) {
const item = await reader.read();
if (item.done) break;
total += item.value.byteLength;
if (total <= maxBytes) {
chunks.push(item.value);
} else {
truncated = true;
const remaining = Math.max(0, maxBytes - (total - item.value.byteLength));
if (remaining > 0) chunks.push(item.value.slice(0, remaining));
break;
}
}
return { text: Buffer.concat(chunks).toString("utf8"), truncated };
}
function routeString(service: ManagedService, key: string, fallback: string): string {
const value = service.route[key];
return typeof value === "string" && value.length > 0 ? value : fallback;
}
function routeNumber(service: ManagedService, key: string, fallback: number): number {
const value = service.route[key];
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
}
function isKubernetesServiceRoute(service: ManagedService): boolean {
return String(service.route.kind ?? "") === "kubernetes-service";
}
function serviceProxyApiPath(service: ManagedService, targetPath: string): string {
const serviceName = routeString(service, "serviceName", service.id);
const servicePort = routeNumber(service, "servicePort", 80);
const safeTargetPath = targetPath.startsWith("/") ? targetPath : `/${targetPath}`;
return `/api/v1/namespaces/${encodeURIComponent(service.namespace)}/services/${encodeURIComponent(`${serviceName}:${servicePort}`)}/proxy${safeTargetPath}`;
}
function endpointProxyApiPath(service: ManagedService, endpoint: ManagedEndpoint, targetPath: string): string {
const { namespace, serviceRef } = kubernetesEndpointServiceRef(service, endpoint);
const safeTargetPath = targetPath.startsWith("/") ? targetPath : `/${targetPath}`;
return `/api/v1/namespaces/${encodeURIComponent(namespace)}/services/${encodeURIComponent(serviceRef)}/proxy${safeTargetPath}`;
}
function kubernetesEndpointServiceRef(service: ManagedService, endpoint: ManagedEndpoint): { namespace: string; serviceRef: string } {
const base = new URL(endpoint.baseUrl);
if (base.protocol !== "kubernetes:") throw new Error(`endpoint ${endpoint.id} must use kubernetes:// baseUrl`);
const namespace = base.hostname || service.namespace;
const parts = base.pathname.split("/").filter(Boolean);
if (parts.length !== 2 || parts[0] !== "services" || parts[1].length === 0) {
throw new Error(`endpoint ${endpoint.id} baseUrl must be kubernetes://<namespace>/services/<service>:<port>`);
}
return { namespace, serviceRef: parts[1] };
}
function kubeProxyCurlArgs(client: KubeApiClient, method: string, url: URL, headers: Headers, hasBody: boolean, timeoutMs: number): string[] {
const args = [
"-sS",
"--show-error",
"--location",
"--max-time", String(Math.max(1, Math.ceil(timeoutMs / 1000))),
"--request", method,
"--cacert", client.caFile,
"--cert", client.certFile,
"--key", client.keyFile,
"--dump-header", "-",
];
const port = url.port || (url.protocol === "https:" ? "443" : "80");
if ((url.hostname === "127.0.0.1" || url.hostname === "localhost") && client.connectHost.length > 0) {
args.push("--connect-to", `${url.hostname}:${port}:${client.connectHost}:${port}`);
}
for (const [name, value] of headers.entries()) args.push("--header", `${name}: ${value}`);
if (hasBody) args.push("--data-binary", "@-");
args.push(url.toString());
return args;
}
function parseCurlHeaderBody(output: Buffer): { status: number; contentType: string; bodyText: string } {
const text = output.toString("utf8");
const separator = text.indexOf("\r\n\r\n") >= 0 ? "\r\n\r\n" : "\n\n";
const index = text.indexOf(separator);
if (index < 0) return { status: 502, contentType: "text/plain; charset=utf-8", bodyText: text };
let headerText = text.slice(0, index);
let bodyText = text.slice(index + separator.length);
while (/^HTTP\/\d(?:\.\d)?\s+1\d\d\b/mu.test(headerText)) {
const nextIndex = bodyText.indexOf(separator);
if (nextIndex < 0) break;
headerText = bodyText.slice(0, nextIndex);
bodyText = bodyText.slice(nextIndex + separator.length);
}
const status = Number(headerText.match(/^HTTP\/\d(?:\.\d)?\s+(\d+)/mu)?.[1] ?? 502);
const contentType = headerText.match(/^content-type:\s*(.+)$/imu)?.[1]?.trim() || "application/octet-stream";
return { status: Number.isFinite(status) ? status : 502, contentType, bodyText };
}
async function kubeApiServiceProxyResponse(
service: ManagedService,
req: Request,
targetPath: string,
query: string,
timeoutMs: number,
): Promise<Response> {
return kubeApiProxyResponse(service, req, serviceProxyApiPath(service, targetPath), query, timeoutMs);
}
async function kubeApiEndpointProxyResponse(
service: ManagedService,
endpoint: ManagedEndpoint,
req: Request,
targetPath: string,
query: string,
timeoutMs: number,
): Promise<Response> {
return kubeApiProxyResponse(service, req, endpointProxyApiPath(service, endpoint, targetPath), query, timeoutMs);
}
async function kubeApiProxyResponse(
service: ManagedService,
req: Request,
apiPath: string,
query: string,
timeoutMs: number,
): Promise<Response> {
if (kubeClient === null) {
return jsonResponse({ ok: false, error: "kubernetes api proxy is not configured", serviceId: service.id, kubeconfigPath: config.kubeconfigPath, noFallback: true }, 502);
}
const upstreamUrl = new URL(apiPath, kubeClient.serverUrl);
upstreamUrl.search = query;
const headers = forwardHeaders(req);
const bodyText = req.method === "GET" || req.method === "HEAD" ? "" : await req.text();
const args = kubeProxyCurlArgs(kubeClient, req.method, upstreamUrl, headers, bodyText.length > 0, timeoutMs);
const proc = Bun.spawn(["curl", ...args], {
stdin: bodyText.length > 0 ? "pipe" : "ignore",
stdout: "pipe",
stderr: "pipe",
});
if (bodyText.length > 0) {
proc.stdin?.write(bodyText);
proc.stdin?.end();
}
const [stdout, stderr, exitCode] = await Promise.all([
new Response(proc.stdout).arrayBuffer(),
new Response(proc.stderr).text(),
proc.exited,
]);
if (exitCode !== 0) {
log("error", "kube_api_proxy_failed", { serviceId: service.id, apiPath, exitCode, stderr: stderr.slice(0, 2000), noFallback: true });
return jsonResponse({ ok: false, error: "kubernetes api service proxy failed", serviceId: service.id, detail: stderr.slice(0, 4000), noFallback: true }, 502);
}
const parsed = parseCurlHeaderBody(Buffer.from(stdout));
return new Response(parsed.bodyText, {
status: parsed.status,
headers: {
"content-type": parsed.contentType,
"x-unidesk-proxy-mode": "kubernetes-api-service-proxy",
"x-unidesk-k3s-service": service.id,
"x-unidesk-response-truncated": "false",
},
});
}
async function probeEndpoint(endpoint: ManagedEndpoint): Promise<JsonRecord> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), config.healthTimeoutMs);
const checkedAt = new Date().toISOString();
try {
const response = await fetch(endpointUrl(endpoint, endpoint.healthPath), {
method: "GET",
headers: { accept: "application/json" },
signal: controller.signal,
});
const contentType = response.headers.get("content-type") ?? "application/octet-stream";
const bodyText = await response.text();
let body: JsonValue = bodyText.slice(0, 2000);
try {
body = JSON.parse(bodyText) as JsonValue;
} catch {
// Keep text preview for non-JSON health endpoints.
}
return {
id: endpoint.id,
nodeId: endpoint.nodeId,
role: endpoint.role,
baseUrl: endpoint.baseUrl,
healthPath: endpoint.healthPath,
healthy: response.ok,
status: response.ok ? "healthy" : "unhealthy",
upstreamStatus: response.status,
contentType,
checkedAt,
body,
};
} catch (error) {
return {
id: endpoint.id,
nodeId: endpoint.nodeId,
role: endpoint.role,
baseUrl: endpoint.baseUrl,
healthPath: endpoint.healthPath,
healthy: false,
status: "unhealthy",
upstreamStatus: null,
contentType: null,
checkedAt,
error: error instanceof Error ? error.message : String(error),
};
} finally {
clearTimeout(timer);
}
}
async function probeKubernetesServiceActive(service: ManagedService): Promise<JsonRecord> {
const endpoint = activeEndpoint(service);
return probeKubernetesEndpoint(service, endpoint, true);
}
async function probeKubernetesEndpoint(service: ManagedService, endpoint: ManagedEndpoint, active = false): Promise<JsonRecord> {
if (!active && endpoint.healthMode === "pod-ready") return await probeKubernetesPodReady(service, endpoint);
const checkedAt = new Date().toISOString();
const response = active
? await kubeApiServiceProxyResponse(
service,
new Request("http://k3sctl-adapter.local/health", { method: "GET", headers: { accept: "application/json" } }),
endpoint.healthPath,
"",
config.healthTimeoutMs,
)
: await kubeApiEndpointProxyResponse(
service,
endpoint,
new Request("http://k3sctl-adapter.local/health", { method: "GET", headers: { accept: "application/json" } }),
endpoint.healthPath,
"",
config.healthTimeoutMs,
);
const contentType = response.headers.get("content-type") ?? "application/octet-stream";
const bodyText = await response.text();
let body: JsonValue = bodyText.slice(0, 2000);
try {
body = JSON.parse(bodyText) as JsonValue;
} catch {
// Health endpoint may return text.
}
return {
id: endpoint.id,
nodeId: endpoint.nodeId,
role: endpoint.role,
baseUrl: endpoint.baseUrl,
healthPath: endpoint.healthPath,
healthMode: endpoint.healthMode,
proxyMode: "kubernetes-api-service-proxy",
route: service.route,
healthy: response.ok,
status: response.ok ? "healthy" : "unhealthy",
upstreamStatus: response.status,
contentType,
checkedAt,
body,
};
}
function jsonAtPath(value: unknown, path: string): unknown {
return path.split(".").reduce((current, key) => {
if (typeof current !== "object" || current === null) return undefined;
return (current as Record<string, unknown>)[key];
}, value);
}
function podReady(item: unknown): boolean {
const conditions = jsonAtPath(item, "status.conditions");
return Array.isArray(conditions) && conditions.some((condition) => {
const record = typeof condition === "object" && condition !== null ? condition as Record<string, unknown> : {};
return record.type === "Ready" && record.status === "True";
});
}
function podSummary(item: unknown): JsonRecord {
const metadata = typeof jsonAtPath(item, "metadata") === "object" && jsonAtPath(item, "metadata") !== null ? jsonAtPath(item, "metadata") as Record<string, unknown> : {};
return {
name: typeof metadata.name === "string" ? metadata.name : "",
nodeName: typeof jsonAtPath(item, "spec.nodeName") === "string" ? jsonAtPath(item, "spec.nodeName") as string : "",
phase: typeof jsonAtPath(item, "status.phase") === "string" ? jsonAtPath(item, "status.phase") as string : "",
podIP: typeof jsonAtPath(item, "status.podIP") === "string" ? jsonAtPath(item, "status.podIP") as string : "",
ready: podReady(item),
};
}
async function probeKubernetesPodReady(service: ManagedService, endpoint: ManagedEndpoint): Promise<JsonRecord> {
const checkedAt = new Date().toISOString();
const { namespace } = kubernetesEndpointServiceRef(service, endpoint);
const labelSelector = new URLSearchParams({
labelSelector: `app.kubernetes.io/name=${service.id},unidesk.ai/instance-id=${endpoint.id}`,
}).toString();
const response = await kubeApiProxyResponse(
service,
new Request("http://k3sctl-adapter.local/api/pods", { method: "GET", headers: { accept: "application/json" } }),
`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods`,
`?${labelSelector}`,
config.healthTimeoutMs,
);
const contentType = response.headers.get("content-type") ?? "application/octet-stream";
const bodyText = await response.text();
let body: JsonValue = bodyText.slice(0, 2000);
let pods: JsonRecord[] = [];
try {
const parsed = JSON.parse(bodyText) as JsonRecord;
const items = Array.isArray(parsed.items) ? parsed.items : [];
pods = items.map(podSummary);
body = { itemCount: items.length, pods };
} catch {
// Keep the raw text preview below.
}
const healthy = response.ok && pods.some((pod) => pod.ready === true);
return {
id: endpoint.id,
nodeId: endpoint.nodeId,
role: endpoint.role,
baseUrl: endpoint.baseUrl,
healthPath: endpoint.healthPath,
healthMode: endpoint.healthMode,
proxyMode: "kubernetes-api-pod-readiness",
route: service.route,
healthy,
status: healthy ? "healthy" : "unhealthy",
upstreamStatus: response.status,
contentType,
checkedAt,
body,
};
}
async function serviceStatus(service: ManagedService): Promise<JsonRecord> {
const instances = isKubernetesServiceRoute(service)
? await Promise.all(service.endpoints.map((endpoint) => endpoint.id === service.activeInstanceId ? probeKubernetesServiceActive(service) : probeKubernetesEndpoint(service, endpoint)))
: [{
id: service.activeInstanceId,
nodeId: activeEndpoint(service).nodeId,
role: activeEndpoint(service).role,
baseUrl: activeEndpoint(service).baseUrl,
healthPath: activeEndpoint(service).healthPath,
healthy: false,
status: "invalid-route",
upstreamStatus: null,
contentType: null,
checkedAt: new Date().toISOString(),
error: "k3s managed service route must be kubernetes-service",
noFallback: true,
}];
const active = instances.find((item) => item.id === service.activeInstanceId) ?? null;
const activeHealthy = active?.healthy === true;
const allInstancesHealthy = instances.every((item) => item.healthy === true);
const expectedNodeIds = service.expectedNodeIds;
const presentNodeIds = Array.from(new Set(instances.filter((item) => item.healthy === true).map((item) => String(item.nodeId))));
const missingNodeIds = expectedNodeIds.filter((nodeId) => !presentNodeIds.includes(nodeId));
const topologyComplete = missingNodeIds.length === 0;
const requiredTopologyHealthy = !service.requireAllInstancesHealthy || (topologyComplete && allInstancesHealthy);
const healthy = activeHealthy && requiredTopologyHealthy;
return {
id: service.id,
namespace: service.namespace,
kind: service.kind,
controlPlane: service.controlPlane,
route: service.route,
activeInstanceId: service.activeInstanceId,
singleWriter: service.singleWriter,
expectedNodeIds,
presentNodeIds,
missingNodeIds,
topologyComplete,
topologyHealthy: topologyComplete && allInstancesHealthy,
servingHealthy: activeHealthy,
healthy,
status: healthy ? (topologyComplete ? "healthy" : "degraded") : "unhealthy",
active,
instances,
};
}
async function kubectlSnapshot(): Promise<JsonValue> {
if (!config.kubectlEnabled) return { enabled: false };
const args = ["get", "nodes", "-o", "json"];
if (config.kubectlContext.length > 0) args.unshift("--context", config.kubectlContext);
const proc = Bun.spawn(["kubectl", ...args], { stdout: "pipe", stderr: "pipe" });
const [stdout, stderr, exitCode] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
if (exitCode !== 0) return { enabled: true, ok: false, exitCode, stderr: stderr.slice(0, 4000) };
try {
const parsed = JSON.parse(stdout) as JsonRecord;
const items = Array.isArray(parsed.items) ? parsed.items : [];
return { enabled: true, ok: true, nodeCount: items.length, nodes: items.slice(0, 20) as JsonValue };
} catch (error) {
return { enabled: true, ok: false, error: error instanceof Error ? error.message : String(error), stdout: stdout.slice(0, 4000) };
}
}
async function controlPlaneSnapshot(): Promise<JsonRecord> {
const services = await Promise.all(config.services.map(serviceStatus));
const managedServicesHealthy = services.every((service) => service.healthy === true);
return {
ok: true,
service: "k3sctl-adapter",
clusterId: config.clusterId,
nodeId: config.nodeId,
startedAt,
manifestPaths: config.manifestPaths,
managedServicesHealthy,
noFallback: true,
runtimePath: "frontend -> backend-core -> k3sctl-adapter -> kubernetes api service proxy -> k3s service",
kubeApiProxy: {
enabled: config.kubeApiProxyEnabled,
configured: kubeClient !== null,
kubeconfigPath: config.kubeconfigPath,
connectHost: config.kubeApiConnectHost,
serverHost: kubeClient?.serverUrl.hostname ?? null,
mode: "kubernetes-api-service-proxy",
},
services,
kubectl: await kubectlSnapshot(),
};
}
function forwardHeaders(request: Request): Headers {
const headers = new Headers();
for (const name of ["accept", "content-type", "x-requested-with"]) {
const value = request.headers.get(name);
if (value !== null) headers.set(name, value);
}
return headers;
}
async function proxyToService(service: ManagedService, req: Request, targetPath: string, query: string): Promise<Response> {
if (isKubernetesServiceRoute(service)) {
return kubeApiServiceProxyResponse(service, req, targetPath, query, config.requestTimeoutMs);
}
log("error", "k3sctl_route_not_kubernetes_service", { serviceId: service.id, route: service.route, noFallback: true });
return jsonResponse({ ok: false, error: "k3s managed service route must be kubernetes-service", serviceId: service.id, route: service.route, noFallback: true }, 500);
}
async function route(req: Request): Promise<Response> {
const url = new URL(req.url);
if (req.method === "OPTIONS") return jsonResponse({ ok: true });
try {
if (url.pathname === "/" || url.pathname === "/health") {
const snapshot = await controlPlaneSnapshot();
return jsonResponse({
ok: true,
service: "k3sctl-adapter",
clusterId: config.clusterId,
nodeId: config.nodeId,
startedAt,
managedServiceCount: config.services.length,
managedServicesHealthy: snapshot.managedServicesHealthy,
kubeApiProxyConfigured: kubeClient !== null,
noFallback: true,
});
}
if (url.pathname === "/logs" && req.method === "GET") return jsonResponse({ ok: true, logs: recentLogs.slice(-100) });
if (url.pathname === "/api/services" && req.method === "GET") {
return jsonResponse({ ok: true, clusterId: config.clusterId, services: await Promise.all(config.services.map(serviceStatus)) });
}
if (url.pathname === "/api/control-plane" && req.method === "GET") return jsonResponse(await controlPlaneSnapshot());
const healthMatch = url.pathname.match(/^\/api\/services\/([^/]+)\/health$/u);
if (healthMatch !== null && (req.method === "GET" || req.method === "HEAD")) {
const service = serviceById(decodeURIComponent(healthMatch[1] ?? ""));
if (service === null) return jsonResponse({ ok: false, error: "managed service not found" }, 404);
const status = await serviceStatus(service);
return req.method === "HEAD" ? new Response(null, { status: status.healthy === true ? 200 : 503 }) : jsonResponse({ ok: status.healthy === true, managedService: status }, status.healthy === true ? 200 : 503);
}
const proxyMatch = url.pathname.match(/^\/api\/services\/([^/]+)\/proxy(\/.*)$/u);
if (proxyMatch !== null) {
const service = serviceById(decodeURIComponent(proxyMatch[1] ?? ""));
if (service === null) return jsonResponse({ ok: false, error: "managed service not found" }, 404);
const targetPath = proxyMatch[2] ?? "/";
return await proxyToService(service, req, targetPath, url.search);
}
return jsonResponse({ ok: false, error: "not found" }, 404);
} catch (error) {
log("error", "request_failed", { path: url.pathname, error: errorToJson(error) });
return jsonResponse({ ok: false, error: error instanceof Error ? error.message : String(error) }, 500);
}
}
Bun.serve({ hostname: config.host, port: config.port, idleTimeout: 120, fetch: route });
log("info", "service_started", { port: config.port, clusterId: config.clusterId, nodeId: config.nodeId, managedServiceCount: config.services.length });
@@ -0,0 +1,18 @@
{
"compilerOptions": {
"composite": true,
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "Bundler",
"types": ["bun", "node"],
"strict": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"declaration": true,
"emitDeclarationOnly": true,
"outDir": "dist",
"skipLibCheck": true
},
"include": ["src/**/*.ts"],
"references": [{ "path": "../../shared" }]
}