fix: migrate code queue control plane to k3s

This commit is contained in:
Codex
2026-05-16 12:01:09 +00:00
parent 83ed99f659
commit 111ec9f1eb
40 changed files with 419 additions and 366 deletions
@@ -3109,7 +3109,7 @@ function queuedStatusReason(task: QueueTask, tasks: QueueTask[] = state.tasks):
return queuedReason("service", "SERVICE", "Code Queue is still starting and has not enabled scheduling yet.");
}
if (!config.schedulerEnabled) {
return queuedReason("scheduler_disabled", "STANDBY", "This Code Queue instance is a v3s-managed standby and does not start queued work.");
return queuedReason("scheduler_disabled", "STANDBY", "This Code Queue instance is a k3s-managed standby and does not start queued work.");
}
if (mergingQueues.has(queueId)) {
return queuedReason("queue_merge", "MERGING", "Queue merge is rewriting queue membership; scheduling will resume immediately after it finishes.");
@@ -1,5 +1,5 @@
ARG V3SCTL_ADAPTER_BASE_IMAGE=oven/bun:1-debian
FROM ${V3SCTL_ADAPTER_BASE_IMAGE}
ARG K3SCTL_ADAPTER_BASE_IMAGE=oven/bun:1-debian
FROM ${K3SCTL_ADAPTER_BASE_IMAGE}
# Never build the adapter FROM a service image: inherited Docker Desktop labels
# can silently republish old Code Queue ports and mounts.
@@ -12,12 +12,12 @@ RUN (command -v curl >/dev/null 2>&1 && command -v ssh >/dev/null 2>&1 && comman
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*)
WORKDIR /app/src/components/microservices/v3sctl-adapter
WORKDIR /app/src/components/microservices/k3sctl-adapter
COPY src/components/shared /app/src/components/shared
COPY src/components/microservices/v3sctl-adapter/package.json ./package.json
COPY src/components/microservices/v3sctl-adapter/tsconfig.json ./tsconfig.json
COPY src/components/microservices/v3sctl-adapter/src ./src
COPY src/components/microservices/v3sctl-adapter/v3s ./v3s
COPY src/components/microservices/k3sctl-adapter/package.json ./package.json
COPY src/components/microservices/k3sctl-adapter/tsconfig.json ./tsconfig.json
COPY src/components/microservices/k3sctl-adapter/src ./src
COPY src/components/microservices/k3sctl-adapter/k3s ./k3s
EXPOSE 4266
CMD ["bun", "--smol", "run", "src/index.ts"]
@@ -0,0 +1,46 @@
services:
k3sctl-adapter:
image: unidesk-k3sctl-adapter:d601
build:
context: ../../../..
dockerfile: src/components/microservices/k3sctl-adapter/Dockerfile
args:
K3SCTL_ADAPTER_BASE_IMAGE: ${K3SCTL_ADAPTER_BASE_IMAGE:-oven/bun:1-debian}
container_name: k3sctl-adapter
restart: unless-stopped
env_file:
- path: ${K3SCTL_ADAPTER_ENV_FILE:-../../../../.state/k3sctl-adapter-d601.env}
required: false
ports:
- "127.0.0.1:${K3SCTL_ADAPTER_HOST_PORT:-4266}:4266"
environment:
HOST: "0.0.0.0"
PORT: "4266"
LOG_FILE: "/var/log/unidesk/k3sctl-adapter.jsonl"
K3SCTL_CLUSTER_ID: "${K3SCTL_CLUSTER_ID:-unidesk-k3s}"
K3SCTL_NODE_ID: "${K3SCTL_NODE_ID:-D601}"
K3SCTL_KUBECTL_ENABLED: "${K3SCTL_KUBECTL_ENABLED:-false}"
K3SCTL_KUBE_API_PROXY_ENABLED: "${K3SCTL_KUBE_API_PROXY_ENABLED:-true}"
K3SCTL_KUBECONFIG_PATH: "/var/lib/unidesk/k3s/kubeconfig"
K3SCTL_KUBE_API_CONNECT_HOST: "${K3SCTL_KUBE_API_CONNECT_HOST:-host.docker.internal}"
K3SCTL_MANIFEST_PATHS: "${K3SCTL_MANIFEST_PATHS:-k3s/code-queue.k3s.json,k3s/mdtodo.k3s.json}"
K3SCTL_SERVICES_JSON: "${K3SCTL_SERVICES_JSON:-[]}"
UNIDESK_LOG_RETENTION_BYTES: "${UNIDESK_LOG_RETENTION_BYTES:-512MiB}"
volumes:
- ${K3SCTL_ADAPTER_LOG_DIR:-../../../../.state/k3sctl-adapter/logs}:/var/log/unidesk
- ${K3SCTL_KUBECONFIG_HOST_PATH:-/etc/rancher/k3s/k3s.yaml}:/var/lib/unidesk/k3s/kubeconfig:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- default
- provider-gateway
healthcheck:
test: ["CMD-SHELL", "curl -fsS --max-time 2 http://127.0.0.1:4266/health >/dev/null"]
interval: 5s
timeout: 3s
retries: 20
networks:
provider-gateway:
external: true
name: ${K3SCTL_PROVIDER_GATEWAY_NETWORK:-unidesk-provider-d601_default}
@@ -1,16 +1,16 @@
{
"apiVersion": "unidesk.ai/v3s/v1",
"apiVersion": "unidesk.ai/k3s/v1",
"kind": "ManagedKubernetesService",
"metadata": {
"name": "code-queue",
"namespace": "unidesk"
},
"spec": {
"adapterServiceId": "v3sctl-adapter",
"adapterServiceId": "k3sctl-adapter",
"controlPlane": {
"type": "kubernetes",
"cluster": "unidesk-v8s",
"context": "unidesk-v8s"
"cluster": "unidesk-k3s",
"context": "unidesk-k3s"
},
"route": {
"kind": "kubernetes-service",
@@ -4,7 +4,7 @@ metadata:
name: unidesk
labels:
app.kubernetes.io/part-of: unidesk
unidesk.ai/v3s-cluster: unidesk-v8s
unidesk.ai/k3s-cluster: unidesk-k3s
---
apiVersion: v1
kind: Service
@@ -255,7 +255,7 @@ metadata:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
spec:
replicas: 1
@@ -268,7 +268,7 @@ spec:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
unidesk.ai/node-id: D601
spec:
@@ -480,7 +480,7 @@ metadata:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
spec:
type: ClusterIP
selector:
@@ -500,7 +500,7 @@ metadata:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D518
spec:
replicas: 1
@@ -513,7 +513,7 @@ spec:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D518
unidesk.ai/node-id: D518
spec:
@@ -725,7 +725,7 @@ metadata:
labels:
app.kubernetes.io/name: code-queue
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D518
spec:
type: ClusterIP
@@ -1,16 +1,16 @@
{
"apiVersion": "unidesk.ai/v3s/v1",
"apiVersion": "unidesk.ai/k3s/v1",
"kind": "ManagedKubernetesService",
"metadata": {
"name": "mdtodo",
"namespace": "unidesk"
},
"spec": {
"adapterServiceId": "v3sctl-adapter",
"adapterServiceId": "k3sctl-adapter",
"controlPlane": {
"type": "kubernetes",
"cluster": "unidesk-v8s",
"context": "unidesk-v8s"
"cluster": "unidesk-k3s",
"context": "unidesk-k3s"
},
"route": {
"kind": "kubernetes-service",
@@ -6,7 +6,7 @@ metadata:
labels:
app.kubernetes.io/name: mdtodo
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
spec:
replicas: 1
@@ -19,7 +19,7 @@ spec:
labels:
app.kubernetes.io/name: mdtodo
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
unidesk.ai/instance-id: D601
unidesk.ai/node-id: D601
spec:
@@ -94,7 +94,7 @@ metadata:
labels:
app.kubernetes.io/name: mdtodo
app.kubernetes.io/part-of: unidesk
unidesk.ai/deployment-mode: v3sctl-managed
unidesk.ai/deployment-mode: k3sctl-managed
spec:
type: ClusterIP
selector:
@@ -1,5 +1,5 @@
{
"name": "@unidesk/v3sctl-adapter",
"name": "@unidesk/k3sctl-adapter",
"private": true,
"type": "module",
"scripts": {
@@ -64,8 +64,8 @@ const config = readConfig();
const logWriter = config.logFile
? createHourlyJsonlWriter({
baseLogFile: config.logFile,
service: "v3sctl-adapter",
maxBytes: logRetentionBytesForService("v3sctl-adapter"),
service: "k3sctl-adapter",
maxBytes: logRetentionBytesForService("k3sctl-adapter"),
})
: null;
const kubeClient = loadKubeApiClient();
@@ -172,7 +172,7 @@ function parseManagedKubernetesManifest(value: unknown, index: number, ownerPath
const spec = asRecord(manifest.spec, `${path}.spec`);
const kind = optionalStringField(manifest, "kind", "ManagedKubernetesService");
const serviceId = stringField(metadata, "name", `${path}.metadata`);
if (kind !== "ManagedKubernetesService") throw new Error(`${path}.kind must be ManagedKubernetesService; direct ManagedHttpService manifests are not allowed in pure v3s mode`);
if (kind !== "ManagedKubernetesService") throw new Error(`${path}.kind must be ManagedKubernetesService; direct ManagedHttpService manifests are not allowed in pure k3s mode`);
const instancesRaw = spec.instances;
if (!Array.isArray(instancesRaw) || instancesRaw.length === 0) throw new Error(`${path}.spec.instances must be a non-empty array`);
const endpoints = instancesRaw.map((endpoint, endpointIndex) => parseEndpoint(endpoint, endpointIndex, `${path}.spec.instances`));
@@ -200,12 +200,12 @@ function parseServiceOrManifest(value: unknown, index: number, ownerPath = "serv
if (typeof item.kind === "string" || item.metadata !== undefined || item.spec !== undefined) {
return parseManagedKubernetesManifest(item, index, ownerPath);
}
throw new Error(`${ownerPath}[${index}] must be a ManagedKubernetesService manifest; static HTTP service declarations are not allowed in pure v3s mode`);
throw new Error(`${ownerPath}[${index}] must be a ManagedKubernetesService manifest; static HTTP service declarations are not allowed in pure k3s mode`);
}
function parseServices(raw: string, ownerPath = "V3SCTL_SERVICES_JSON"): ManagedService[] {
function parseServices(raw: string, ownerPath = "K3SCTL_SERVICES_JSON"): ManagedService[] {
const value = raw.trim().length === 0 ? [] : JSON.parse(raw) as unknown;
if (!Array.isArray(value)) throw new Error("V3SCTL_SERVICES_JSON must be an array");
if (!Array.isArray(value)) throw new Error("K3SCTL_SERVICES_JSON must be an array");
return value.map((item, index) => parseServiceOrManifest(item, index, ownerPath));
}
@@ -237,36 +237,36 @@ function mergeServices(services: ManagedService[]): ManagedService[] {
const seen = new Set<string>();
for (const service of services) {
const key = `${service.namespace}/${service.id}`;
if (seen.has(key)) throw new Error(`duplicate v3s managed service: ${key}`);
if (seen.has(key)) throw new Error(`duplicate k3s managed service: ${key}`);
seen.add(key);
}
return services;
}
function readConfig(): RuntimeConfig {
const paths = manifestPaths(envString("V3SCTL_MANIFEST_PATHS", "v3s/code-queue.v3s.json"));
const inlineServices = parseServices(envString("V3SCTL_SERVICES_JSON", "[]"));
const paths = manifestPaths(envString("K3SCTL_MANIFEST_PATHS", "k3s/code-queue.k3s.json"));
const inlineServices = parseServices(envString("K3SCTL_SERVICES_JSON", "[]"));
const manifestServices = readManifestServices(paths);
return {
host: envString("HOST", "0.0.0.0"),
port: envNumber("PORT", 4266),
logFile: envString("LOG_FILE", "/var/log/unidesk/v3sctl-adapter.jsonl"),
logFile: envString("LOG_FILE", "/var/log/unidesk/k3sctl-adapter.jsonl"),
manifestPaths: paths,
clusterId: envString("V3SCTL_CLUSTER_ID", "unidesk-v8s"),
nodeId: envString("V3SCTL_NODE_ID", "D601"),
kubectlEnabled: envBool("V3SCTL_KUBECTL_ENABLED", false),
kubectlContext: envString("V3SCTL_KUBECTL_CONTEXT", ""),
kubeApiProxyEnabled: envBool("V3SCTL_KUBE_API_PROXY_ENABLED", true),
kubeconfigPath: envString("V3SCTL_KUBECONFIG_PATH", "/var/lib/unidesk/v8s/kubeconfig"),
kubeApiConnectHost: envString("V3SCTL_KUBE_API_CONNECT_HOST", "host.docker.internal"),
requestTimeoutMs: Math.max(1000, Math.min(120_000, envNumber("V3SCTL_REQUEST_TIMEOUT_MS", 30_000))),
healthTimeoutMs: Math.max(500, Math.min(30_000, envNumber("V3SCTL_HEALTH_TIMEOUT_MS", 2500))),
clusterId: envString("K3SCTL_CLUSTER_ID", "unidesk-k3s"),
nodeId: envString("K3SCTL_NODE_ID", "D601"),
kubectlEnabled: envBool("K3SCTL_KUBECTL_ENABLED", false),
kubectlContext: envString("K3SCTL_KUBECTL_CONTEXT", ""),
kubeApiProxyEnabled: envBool("K3SCTL_KUBE_API_PROXY_ENABLED", true),
kubeconfigPath: envString("K3SCTL_KUBECONFIG_PATH", "/var/lib/unidesk/k3s/kubeconfig"),
kubeApiConnectHost: envString("K3SCTL_KUBE_API_CONNECT_HOST", "host.docker.internal"),
requestTimeoutMs: Math.max(1000, Math.min(120_000, envNumber("K3SCTL_REQUEST_TIMEOUT_MS", 30_000))),
healthTimeoutMs: Math.max(500, Math.min(30_000, envNumber("K3SCTL_HEALTH_TIMEOUT_MS", 2500))),
services: mergeServices([...manifestServices, ...inlineServices]),
};
}
function log(level: "debug" | "info" | "warn" | "error", event: string, detail: JsonRecord = {}): void {
const record: JsonRecord = { at: new Date().toISOString(), service: "v3sctl-adapter", level, event, ...detail };
const record: JsonRecord = { at: new Date().toISOString(), service: "k3sctl-adapter", level, event, ...detail };
recentLogs.push(record);
while (recentLogs.length > 500) recentLogs.shift();
try {
@@ -304,7 +304,7 @@ function loadKubeApiClient(): KubeApiClient | null {
const cert = kubeconfigScalar(raw, "client-certificate-data");
const key = kubeconfigScalar(raw, "client-key-data");
if (server.length === 0 || ca.length === 0 || cert.length === 0 || key.length === 0) throw new Error("kubeconfig must include server, CA, client certificate, and client key data");
const dir = join(tmpdir(), `unidesk-v3sctl-kube-${config.clusterId}`);
const dir = join(tmpdir(), `unidesk-k3sctl-kube-${config.clusterId}`);
mkdirSync(dir, { recursive: true, mode: 0o700 });
const client: KubeApiClient = {
serverUrl: new URL(server),
@@ -510,7 +510,7 @@ async function kubeApiProxyResponse(
headers: {
"content-type": parsed.contentType,
"x-unidesk-proxy-mode": "kubernetes-api-service-proxy",
"x-unidesk-v3s-service": service.id,
"x-unidesk-k3s-service": service.id,
"x-unidesk-response-truncated": "false",
},
});
@@ -577,7 +577,7 @@ async function probeKubernetesEndpoint(service: ManagedService, endpoint: Manage
const response = active
? await kubeApiServiceProxyResponse(
service,
new Request("http://v3sctl-adapter.local/health", { method: "GET", headers: { accept: "application/json" } }),
new Request("http://k3sctl-adapter.local/health", { method: "GET", headers: { accept: "application/json" } }),
endpoint.healthPath,
"",
config.healthTimeoutMs,
@@ -585,7 +585,7 @@ async function probeKubernetesEndpoint(service: ManagedService, endpoint: Manage
: await kubeApiEndpointProxyResponse(
service,
endpoint,
new Request("http://v3sctl-adapter.local/health", { method: "GET", headers: { accept: "application/json" } }),
new Request("http://k3sctl-adapter.local/health", { method: "GET", headers: { accept: "application/json" } }),
endpoint.healthPath,
"",
config.healthTimeoutMs,
@@ -650,7 +650,7 @@ async function probeKubernetesPodReady(service: ManagedService, endpoint: Manage
}).toString();
const response = await kubeApiProxyResponse(
service,
new Request("http://v3sctl-adapter.local/api/pods", { method: "GET", headers: { accept: "application/json" } }),
new Request("http://k3sctl-adapter.local/api/pods", { method: "GET", headers: { accept: "application/json" } }),
`/api/v1/namespaces/${encodeURIComponent(namespace)}/pods`,
`?${labelSelector}`,
config.healthTimeoutMs,
@@ -700,7 +700,7 @@ async function serviceStatus(service: ManagedService): Promise<JsonRecord> {
upstreamStatus: null,
contentType: null,
checkedAt: new Date().toISOString(),
error: "v3s managed service route must be kubernetes-service",
error: "k3s managed service route must be kubernetes-service",
noFallback: true,
}];
const active = instances.find((item) => item.id === service.activeInstanceId) ?? null;
@@ -754,14 +754,14 @@ async function controlPlaneSnapshot(): Promise<JsonRecord> {
const managedServicesHealthy = services.every((service) => service.healthy === true);
return {
ok: true,
service: "v3sctl-adapter",
service: "k3sctl-adapter",
clusterId: config.clusterId,
nodeId: config.nodeId,
startedAt,
manifestPaths: config.manifestPaths,
managedServicesHealthy,
noFallback: true,
runtimePath: "frontend -> backend-core -> v3sctl-adapter -> kubernetes api service proxy -> v3s service",
runtimePath: "frontend -> backend-core -> k3sctl-adapter -> kubernetes api service proxy -> k3s service",
kubeApiProxy: {
enabled: config.kubeApiProxyEnabled,
configured: kubeClient !== null,
@@ -788,8 +788,8 @@ async function proxyToService(service: ManagedService, req: Request, targetPath:
if (isKubernetesServiceRoute(service)) {
return kubeApiServiceProxyResponse(service, req, targetPath, query, config.requestTimeoutMs);
}
log("error", "v3sctl_route_not_kubernetes_service", { serviceId: service.id, route: service.route, noFallback: true });
return jsonResponse({ ok: false, error: "v3s managed service route must be kubernetes-service", serviceId: service.id, route: service.route, noFallback: true }, 500);
log("error", "k3sctl_route_not_kubernetes_service", { serviceId: service.id, route: service.route, noFallback: true });
return jsonResponse({ ok: false, error: "k3s managed service route must be kubernetes-service", serviceId: service.id, route: service.route, noFallback: true }, 500);
}
async function route(req: Request): Promise<Response> {
@@ -800,7 +800,7 @@ async function route(req: Request): Promise<Response> {
const snapshot = await controlPlaneSnapshot();
return jsonResponse({
ok: true,
service: "v3sctl-adapter",
service: "k3sctl-adapter",
clusterId: config.clusterId,
nodeId: config.nodeId,
startedAt,
@@ -1,46 +0,0 @@
services:
v3sctl-adapter:
image: unidesk-v3sctl-adapter:d601
build:
context: ../../../..
dockerfile: src/components/microservices/v3sctl-adapter/Dockerfile
args:
V3SCTL_ADAPTER_BASE_IMAGE: ${V3SCTL_ADAPTER_BASE_IMAGE:-oven/bun:1-debian}
container_name: v3sctl-adapter
restart: unless-stopped
env_file:
- path: ${V3SCTL_ADAPTER_ENV_FILE:-../../../../.state/v3sctl-adapter-d601.env}
required: false
ports:
- "127.0.0.1:${V3SCTL_ADAPTER_HOST_PORT:-4266}:4266"
environment:
HOST: "0.0.0.0"
PORT: "4266"
LOG_FILE: "/var/log/unidesk/v3sctl-adapter.jsonl"
V3SCTL_CLUSTER_ID: "${V3SCTL_CLUSTER_ID:-unidesk-v8s}"
V3SCTL_NODE_ID: "${V3SCTL_NODE_ID:-D601}"
V3SCTL_KUBECTL_ENABLED: "${V3SCTL_KUBECTL_ENABLED:-false}"
V3SCTL_KUBE_API_PROXY_ENABLED: "${V3SCTL_KUBE_API_PROXY_ENABLED:-true}"
V3SCTL_KUBECONFIG_PATH: "/var/lib/unidesk/v8s/kubeconfig"
V3SCTL_KUBE_API_CONNECT_HOST: "${V3SCTL_KUBE_API_CONNECT_HOST:-host.docker.internal}"
V3SCTL_MANIFEST_PATHS: "${V3SCTL_MANIFEST_PATHS:-v3s/code-queue.v3s.json,v3s/mdtodo.v3s.json}"
V3SCTL_SERVICES_JSON: "${V3SCTL_SERVICES_JSON:-[]}"
UNIDESK_LOG_RETENTION_BYTES: "${UNIDESK_LOG_RETENTION_BYTES:-512MiB}"
volumes:
- ${V3SCTL_ADAPTER_LOG_DIR:-../../../../.state/v3sctl-adapter/logs}:/var/log/unidesk
- ${V3SCTL_KUBECONFIG_HOST_PATH:-../../../../.state/v8s/kubeconfig}:/var/lib/unidesk/v8s/kubeconfig:ro
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
- default
- provider-gateway
healthcheck:
test: ["CMD-SHELL", "curl -fsS --max-time 2 http://127.0.0.1:4266/health >/dev/null"]
interval: 5s
timeout: 3s
retries: 20
networks:
provider-gateway:
external: true
name: ${V3SCTL_PROVIDER_GATEWAY_NETWORK:-unidesk-provider-d601_default}