mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: migrate code queue control plane to k3s

Browse Source
This commit is contained in:
Codex
2026-05-16 12:01:09 +00:00
parent 83ed99f659
commit 111ec9f1eb
40 changed files with 419 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/check.ts
+2 -2
View File
@@ -34,7 +34,7 @@ function unifiedLogRotationItem(): CheckItem {
"src/components/frontend/src/index.ts",
"src/components/provider-gateway/src/index.ts",
"src/components/microservices/code-queue/src/index.ts",
"src/components/microservices/v3sctl-adapter/src/index.ts",
"src/components/microservices/k3sctl-adapter/src/index.ts",
"src/components/microservices/mdtodo/src/index.ts",
"src/components/microservices/project-manager/src/index.ts",
"src/components/microservices/baidu-netdisk/src/index.ts",
@@ -68,7 +68,7 @@ export function runChecks(config: UniDeskConfig): { ok: boolean; items: CheckIte
fileItem("src/components/frontend/src/index.ts"),
fileItem("src/components/provider-gateway/src/index.ts"),
fileItem("src/components/microservices/oa-event-flow/src/index.ts"),
fileItem("src/components/microservices/v3sctl-adapter/src/index.ts"),
fileItem("src/components/microservices/k3sctl-adapter/src/index.ts"),
fileItem("src/components/microservices/mdtodo/src/index.ts"),
fileItem("scripts/src/deploy.ts"),
fileItem("scripts/src/e2e.ts"),
scripts/src/config.ts
+5 -5
View File
@@ -67,9 +67,9 @@ export interface UniDeskMicroserviceConfig {
timeoutMs: number;
};
deployment: {
mode: "unidesk-direct" | "v3sctl-managed";
mode: "unidesk-direct" | "k3sctl-managed";
adapterServiceId?: string;
v3sServiceId?: string;
k3sServiceId?: string;
namespace?: string;
expectedNodeIds?: string[];
activeNodeId?: string;
@@ -169,8 +169,8 @@ function microserviceConfig(item: Record<string, unknown>, index: number): UniDe
const frontend = asRecord(item.frontend, `${path}.frontend`);
const deployment = item.deployment === undefined ? undefined : asRecord(item.deployment, `${path}.deployment`);
const deploymentMode = deployment === undefined ? "unidesk-direct" : stringField(deployment, "mode", `${path}.deployment`);
if (deploymentMode !== "unidesk-direct" && deploymentMode !== "v3sctl-managed") {
throw new Error(`${path}.deployment.mode must be unidesk-direct or v3sctl-managed`);
if (deploymentMode !== "unidesk-direct" && deploymentMode !== "k3sctl-managed") {
throw new Error(`${path}.deployment.mode must be unidesk-direct or k3sctl-managed`);
}
return {
id: stringField(item, "id", path),
@@ -202,7 +202,7 @@ function microserviceConfig(item: Record<string, unknown>, index: number): UniDe
: {
mode: deploymentMode,
adapterServiceId: optionalStringField(deployment, "adapterServiceId", `${path}.deployment`),
v3sServiceId: optionalStringField(deployment, "v3sServiceId", `${path}.deployment`),
k3sServiceId: optionalStringField(deployment, "k3sServiceId", `${path}.deployment`),
namespace: optionalStringField(deployment, "namespace", `${path}.deployment`),
expectedNodeIds: optionalStringArrayField(deployment, "expectedNodeIds", `${path}.deployment`),
activeNodeId: optionalStringField(deployment, "activeNodeId", `${path}.deployment`),
scripts/src/deploy.ts
+67 -14
View File
@@ -79,10 +79,10 @@ const shortRemoteTimeoutMs = 20_000;
const pollIntervalMs = 5_000;
const remoteDeployRoot = "/home/ubuntu/.unidesk/deploy";
const k8sNamespace = "unidesk";
const k8sContainer = "unidesk-v8s-server";
const k8sKubeconfig = "/home/ubuntu/cq-deploy/.state/v8s/kubeconfig";
const v3sDeployDir = "/home/ubuntu/cq-deploy";
const k8sKubeconfig = "/etc/rancher/k3s/k3s.yaml";
const k3sDeployDir = "/home/ubuntu/cq-deploy";
const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789";
const nativeK3sInstallVersion = "v1.34.1+k3s1";
function nowIso(): string {
return new Date().toISOString();
@@ -232,7 +232,7 @@ function targetExportDir(service: UniDeskMicroserviceConfig, runId: string): str
}
function targetWorkDir(service: UniDeskMicroserviceConfig): string {
if (service.deployment.mode === "v3sctl-managed") return v3sDeployDir;
if (service.deployment.mode === "k3sctl-managed") return k3sDeployDir;
if (targetIsMain(service) && service.repository.url === "https://github.com/pikasTech/unidesk") {
return rootPath(".state", "deploy", "work", safeId(service.id));
}
@@ -240,7 +240,7 @@ function targetWorkDir(service: UniDeskMicroserviceConfig): string {
}
function buildImageTag(service: UniDeskMicroserviceConfig): string {
if (service.deployment.mode === "v3sctl-managed") return `unidesk-${service.id}:d601`;
if (service.deployment.mode === "k3sctl-managed") return `unidesk-${service.id}:d601`;
if (targetIsMain(service)) {
if (["project-manager", "baidu-netdisk", "oa-event-flow"].includes(service.repository.composeService)) return service.repository.composeService;
return `unidesk-${service.repository.composeService}`;
@@ -270,8 +270,8 @@ function directDockerfileOverride(service: UniDeskMicroserviceConfig): string {
function k8sManifestPath(service: UniDeskMicroserviceConfig): string {
const composeFile = service.repository.composeFile;
if (!composeFile.endsWith(".v3s.json")) throw new Error(`${service.id} v3s service composeFile must point to *.v3s.json`);
return composeFile.replace(/\.v3s\.json$/u, ".k8s.yaml");
if (!composeFile.endsWith(".k3s.json")) throw new Error(`${service.id} k3s service composeFile must point to *.k3s.json`);
return composeFile.replace(/\.k3s\.json$/u, ".k8s.yaml");
}
function sourceProxyPrelude(service: UniDeskMicroserviceConfig): string {
@@ -510,16 +510,67 @@ function writeComposeEnvFallbackPath(): string {
return rootPath(".state", "docker-compose.env");
}
function rootAccessPrelude(): string[] {
return [
"root_exec() {",
" if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return; fi",
" if sudo -n true >/dev/null 2>&1; then sudo -n \"$@\"; return; fi",
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return; fi",
" echo 'native_k3s_root_access=missing' >&2",
" return 1",
"}",
"root_shell() {",
" script=\"$1\"",
" if [ \"$(id -u)\" = \"0\" ]; then bash -lc \"$script\"; return; fi",
" if sudo -n true >/dev/null 2>&1; then sudo -n bash -lc \"$script\"; return; fi",
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- bash -lc \"$script\"; return; fi",
" echo 'native_k3s_root_access=missing' >&2",
" return 1",
"}",
];
}
function ensureNativeK3sScript(): string {
const installCommand = [
`INSTALL_K3S_VERSION=${nativeK3sInstallVersion}`,
"INSTALL_K3S_EXEC=\"server --disable traefik --disable servicelb --disable metrics-server --node-name D601 --node-label unidesk.ai/node-id=D601 --node-label unidesk.ai/provider-id=D601 --tls-san 127.0.0.1 --tls-san host.docker.internal --write-kubeconfig-mode 644\"",
"sh /tmp/unidesk-install-k3s.sh",
].join(" ");
return [
"set -euo pipefail",
...rootAccessPrelude(),
"if ! command -v k3s >/dev/null 2>&1; then",
" curl -fsSL --max-time 60 https://get.k3s.io -o /tmp/unidesk-install-k3s.sh",
` root_shell ${shellQuote(installCommand)}`,
"fi",
"if ! systemctl is-active --quiet k3s; then",
" root_exec systemctl enable --now k3s",
"fi",
"for attempt in $(seq 1 60); do",
` if KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes >/dev/null 2>&1; then break; fi`,
" sleep 2",
"done",
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -l unidesk.ai/node-id=D601 --no-headers | grep -q .`,
"legacy_k3s_containers=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1}')",
"for container in $legacy_k3s_containers; do",
" docker update --restart=no \"$container\" >/dev/null 2>&1 || true",
" docker stop \"$container\" >/dev/null",
" echo stopped_containerized_k3s=$container",
"done",
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -o wide`,
"printf 'native_k3s=ready kubeconfig=%s\\n' /etc/rancher/k3s/k3s.yaml",
].join("\n");
}
function importK3sImageScript(service: UniDeskMicroserviceConfig): string {
const image = buildImageTag(service);
return [
"set -euo pipefail",
...rootAccessPrelude(),
`image=${shellQuote(image)}`,
`k3s_container=${shellQuote(k8sContainer)}`,
"docker image inspect \"$image\" >/dev/null",
"docker ps --format '{{.Names}}' | grep -Fx \"$k3s_container\" >/dev/null",
"docker save \"$image\" | docker exec -i \"$k3s_container\" ctr -n k8s.io images import -",
"docker exec \"$k3s_container\" ctr -n k8s.io images ls | grep -F \"$image\" || true",
"docker save \"$image\" | root_exec k3s ctr -n k8s.io images import -",
"root_exec k3s ctr -n k8s.io images ls | grep -F \"$image\" || true",
].join("\n");
}
@@ -629,7 +680,7 @@ function runtimeCommitVerified(
desired: string,
): boolean {
if (healthCommit !== null && healthCommit.length > 0 && !commitMatches(healthCommit, desired)) return false;
if (service.deployment.mode === "v3sctl-managed") return commitMatches(orchestratorCommit, desired);
if (service.deployment.mode === "k3sctl-managed") return commitMatches(orchestratorCommit, desired);
return commitMatches(imageCommit, desired);
}
@@ -820,7 +871,7 @@ async function readDockerImageCommit(config: UniDeskConfig, service: UniDeskMicr
}
async function readK8sCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<string | null> {
if (service.deployment.mode !== "v3sctl-managed") return null;
if (service.deployment.mode !== "k3sctl-managed") return null;
const result = await runTargetCommand(config, service, k8sCommitProbeScript(service), "/home/ubuntu", 30_000, 20_000);
const commit = parseFullCommit(result.stdout);
return commit.length > 0 ? commit : null;
@@ -961,7 +1012,9 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi
const build = await step(config, service, "docker-build", buildScript, targetWorkDir(service), Math.min(options.timeoutMs, 540_000), !targetIsMain(service));
if (!pushStep(steps, build)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
if (service.deployment.mode === "v3sctl-managed") {
if (service.deployment.mode === "k3sctl-managed") {
const nativeK3s = await step(config, service, "ensure-native-k3s", ensureNativeK3sScript(), "/home/ubuntu", 180_000, true);
if (!pushStep(steps, nativeK3s)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
const imageImport = await step(config, service, "import-k3s-image", importK3sImageScript(service), targetWorkDir(service), 180_000, true);
if (!pushStep(steps, imageImport)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
const apply = await step(config, service, "kubectl-apply", applyK8sScript(service), targetWorkDir(service), 60_000, false);
scripts/src/e2e.ts
+37 -37
View File
@@ -61,8 +61,8 @@ const SERVICE_CHECK_NAMES = [
"microservice:catalog-todo-note",
"microservice:catalog-oa-event-flow",
"microservice:catalog-code-queue",
"microservice:v3sctl-adapter-status",
"microservice:v3sctl-control-plane",
"microservice:k3sctl-adapter-status",
"microservice:k3sctl-control-plane",
"microservice:catalog-filebrowser",
"microservice:filebrowser-health",
"microservice:filebrowser-webui",
@@ -1028,8 +1028,8 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
const oaEventFlowEvents = dockerCoreJson("/api/microservices/oa-event-flow/proxy/api/events?limit=10");
const oaEventFlowPipelineEvents = dockerCoreJson("/api/microservices/oa-event-flow/proxy/api/events?tags=service:pipeline&limit=10");
const oaEventFlowStats = dockerCoreJson("/api/microservices/oa-event-flow/proxy/api/stats/trace?limit=10");
const v3sctlStatus = dockerCoreJson("/api/microservices/v3sctl-adapter/status");
const v3sctlControlPlane = dockerCoreJson("/api/microservices/v3sctl-adapter/proxy/api/control-plane");
const k3sctlStatus = dockerCoreJson("/api/microservices/k3sctl-adapter/status");
const k3sctlControlPlane = dockerCoreJson("/api/microservices/k3sctl-adapter/proxy/api/control-plane");
const codeQueueStatus = dockerCoreJson("/api/microservices/code-queue/status");
const codeQueueHealth = dockerCoreJson("/api/microservices/code-queue/health");
const codeQueueTasks = dockerCoreJson("/api/microservices/code-queue/proxy/api/tasks/overview?limit=5&transcriptLimit=1&compact=1&afterSeq=0&preferId=");
@@ -1106,7 +1106,7 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
const oaEventFlowStatsBody = (oaEventFlowStats as { body?: { ok?: boolean; stats?: unknown[]; returned?: number } }).body;
const codeQueueHealthBody = (codeQueueHealth as { body?: { ok?: boolean; egressProxy?: { connected?: boolean }; queue?: { defaultModel?: string; judgeConfigured?: boolean; modelReasoningEfforts?: Record<string, string> } } }).body;
const codeQueueTasksBody = (codeQueueTasks as { body?: { ok?: boolean; queue?: { defaultModel?: string; modelReasoningEfforts?: Record<string, string> }; tasks?: unknown[] } }).body;
const v3sctlControlPlaneBody = (v3sctlControlPlane as { body?: {
const k3sctlControlPlaneBody = (k3sctlControlPlane as { body?: {
ok?: boolean;
clusterId?: string;
noFallback?: boolean;
@@ -1123,8 +1123,8 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
instances?: Array<{ id?: string; healthy?: boolean; proxyMode?: string }>;
}>;
} }).body;
const v3sctlCodeQueueService = v3sctlControlPlaneBody?.services?.find((service) => service.id === "code-queue");
const v3sctlD518Instance = v3sctlCodeQueueService?.instances?.find((instance) => instance.id === "D518");
const k3sctlCodeQueueService = k3sctlControlPlaneBody?.services?.find((service) => service.id === "code-queue");
const k3sctlD518Instance = k3sctlCodeQueueService?.instances?.find((instance) => instance.id === "D518");
const filebrowserHealthBody = (filebrowserHealth as { body?: { status?: string } }).body;
const filebrowserD601HealthBody = (filebrowserD601Health as { body?: { status?: string } }).body;
const filebrowserWebuiText = String((filebrowserWebui as { body?: { text?: string } }).body?.text || "");
@@ -1159,39 +1159,39 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
(microservices as { ok?: boolean }).ok === true
&& codeQueue?.providerId === "D601"
&& codeQueue.backend?.public === false
&& codeQueue.backend?.proxyMode === "v3sctl-adapter-http"
&& codeQueue.deployment?.mode === "v3sctl-managed"
&& codeQueue.runtime?.orchestrator === "v3sctl"
&& codeQueue.backend?.proxyMode === "k3sctl-adapter-http"
&& codeQueue.deployment?.mode === "k3sctl-managed"
&& codeQueue.runtime?.orchestrator === "k3sctl"
&& codeQueue.runtime?.container === null,
{ microservices });
addSelectedCheck(checks, options, "microservice:v3sctl-adapter-status",
(v3sctlStatus as { ok?: boolean; body?: { microservice?: { id?: string; providerId?: string } } }).ok === true
&& (v3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.id === "v3sctl-adapter"
&& (v3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.providerId === "D601",
v3sctlStatus);
addSelectedCheck(checks, options, "microservice:v3sctl-control-plane",
(v3sctlControlPlane as { ok?: boolean }).ok === true
&& v3sctlControlPlaneBody?.ok === true
&& v3sctlControlPlaneBody.clusterId === "unidesk-v8s"
&& v3sctlControlPlaneBody.noFallback === true
&& v3sctlControlPlaneBody.managedServicesHealthy === true
&& v3sctlControlPlaneBody.kubeApiProxy?.mode === "kubernetes-api-service-proxy"
&& v3sctlCodeQueueService?.status === "healthy"
&& v3sctlCodeQueueService?.topologyComplete === true
&& v3sctlCodeQueueService?.servingHealthy === true
&& v3sctlCodeQueueService?.active?.id === "D601"
&& v3sctlCodeQueueService?.active?.healthy === true
&& (v3sctlCodeQueueService?.presentNodeIds ?? []).includes("D601")
&& (v3sctlCodeQueueService?.presentNodeIds ?? []).includes("D518")
&& (v3sctlCodeQueueService?.missingNodeIds ?? []).length === 0
&& v3sctlD518Instance?.healthy === true
&& v3sctlD518Instance?.proxyMode === "kubernetes-api-pod-readiness",
addSelectedCheck(checks, options, "microservice:k3sctl-adapter-status",
(k3sctlStatus as { ok?: boolean; body?: { microservice?: { id?: string; providerId?: string } } }).ok === true
&& (k3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.id === "k3sctl-adapter"
&& (k3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.providerId === "D601",
k3sctlStatus);
addSelectedCheck(checks, options, "microservice:k3sctl-control-plane",
(k3sctlControlPlane as { ok?: boolean }).ok === true
&& k3sctlControlPlaneBody?.ok === true
&& k3sctlControlPlaneBody.clusterId === "unidesk-k3s"
&& k3sctlControlPlaneBody.noFallback === true
&& k3sctlControlPlaneBody.managedServicesHealthy === true
&& k3sctlControlPlaneBody.kubeApiProxy?.mode === "kubernetes-api-service-proxy"
&& k3sctlCodeQueueService?.status === "healthy"
&& k3sctlCodeQueueService?.topologyComplete === true
&& k3sctlCodeQueueService?.servingHealthy === true
&& k3sctlCodeQueueService?.active?.id === "D601"
&& k3sctlCodeQueueService?.active?.healthy === true
&& (k3sctlCodeQueueService?.presentNodeIds ?? []).includes("D601")
&& (k3sctlCodeQueueService?.presentNodeIds ?? []).includes("D518")
&& (k3sctlCodeQueueService?.missingNodeIds ?? []).length === 0
&& k3sctlD518Instance?.healthy === true
&& k3sctlD518Instance?.proxyMode === "kubernetes-api-pod-readiness",
{
ok: (v3sctlControlPlane as { ok?: boolean }).ok,
clusterId: v3sctlControlPlaneBody?.clusterId,
noFallback: v3sctlControlPlaneBody?.noFallback,
kubeApiProxy: v3sctlControlPlaneBody?.kubeApiProxy,
service: v3sctlCodeQueueService,
ok: (k3sctlControlPlane as { ok?: boolean }).ok,
clusterId: k3sctlControlPlaneBody?.clusterId,
noFallback: k3sctlControlPlaneBody?.noFallback,
kubeApiProxy: k3sctlControlPlaneBody?.kubeApiProxy,
service: k3sctlCodeQueueService,
});
addSelectedCheck(checks, options, "microservice:catalog-filebrowser", (microservices as { ok?: boolean }).ok === true
&& filebrowser?.providerId === "D518"