fix: migrate code queue control plane to k3s
This commit is contained in:
Codex
@@ -34,7 +34,7 @@ function unifiedLogRotationItem(): CheckItem {
|
||||
"src/components/frontend/src/index.ts",
|
||||
"src/components/provider-gateway/src/index.ts",
|
||||
"src/components/microservices/code-queue/src/index.ts",
|
||||
"src/components/microservices/v3sctl-adapter/src/index.ts",
|
||||
"src/components/microservices/k3sctl-adapter/src/index.ts",
|
||||
"src/components/microservices/mdtodo/src/index.ts",
|
||||
"src/components/microservices/project-manager/src/index.ts",
|
||||
"src/components/microservices/baidu-netdisk/src/index.ts",
|
||||
@@ -68,7 +68,7 @@ export function runChecks(config: UniDeskConfig): { ok: boolean; items: CheckIte
|
||||
fileItem("src/components/frontend/src/index.ts"),
|
||||
fileItem("src/components/provider-gateway/src/index.ts"),
|
||||
fileItem("src/components/microservices/oa-event-flow/src/index.ts"),
|
||||
fileItem("src/components/microservices/v3sctl-adapter/src/index.ts"),
|
||||
fileItem("src/components/microservices/k3sctl-adapter/src/index.ts"),
|
||||
fileItem("src/components/microservices/mdtodo/src/index.ts"),
|
||||
fileItem("scripts/src/deploy.ts"),
|
||||
fileItem("scripts/src/e2e.ts"),
|
||||
|
||||
@@ -67,9 +67,9 @@ export interface UniDeskMicroserviceConfig {
|
||||
timeoutMs: number;
|
||||
};
|
||||
deployment: {
|
||||
mode: "unidesk-direct" | "v3sctl-managed";
|
||||
mode: "unidesk-direct" | "k3sctl-managed";
|
||||
adapterServiceId?: string;
|
||||
v3sServiceId?: string;
|
||||
k3sServiceId?: string;
|
||||
namespace?: string;
|
||||
expectedNodeIds?: string[];
|
||||
activeNodeId?: string;
|
||||
@@ -169,8 +169,8 @@ function microserviceConfig(item: Record<string, unknown>, index: number): UniDe
|
||||
const frontend = asRecord(item.frontend, `${path}.frontend`);
|
||||
const deployment = item.deployment === undefined ? undefined : asRecord(item.deployment, `${path}.deployment`);
|
||||
const deploymentMode = deployment === undefined ? "unidesk-direct" : stringField(deployment, "mode", `${path}.deployment`);
|
||||
if (deploymentMode !== "unidesk-direct" && deploymentMode !== "v3sctl-managed") {
|
||||
throw new Error(`${path}.deployment.mode must be unidesk-direct or v3sctl-managed`);
|
||||
if (deploymentMode !== "unidesk-direct" && deploymentMode !== "k3sctl-managed") {
|
||||
throw new Error(`${path}.deployment.mode must be unidesk-direct or k3sctl-managed`);
|
||||
}
|
||||
return {
|
||||
id: stringField(item, "id", path),
|
||||
@@ -202,7 +202,7 @@ function microserviceConfig(item: Record<string, unknown>, index: number): UniDe
|
||||
: {
|
||||
mode: deploymentMode,
|
||||
adapterServiceId: optionalStringField(deployment, "adapterServiceId", `${path}.deployment`),
|
||||
v3sServiceId: optionalStringField(deployment, "v3sServiceId", `${path}.deployment`),
|
||||
k3sServiceId: optionalStringField(deployment, "k3sServiceId", `${path}.deployment`),
|
||||
namespace: optionalStringField(deployment, "namespace", `${path}.deployment`),
|
||||
expectedNodeIds: optionalStringArrayField(deployment, "expectedNodeIds", `${path}.deployment`),
|
||||
activeNodeId: optionalStringField(deployment, "activeNodeId", `${path}.deployment`),
|
||||
|
||||
+67
-14
@@ -79,10 +79,10 @@ const shortRemoteTimeoutMs = 20_000;
|
||||
const pollIntervalMs = 5_000;
|
||||
const remoteDeployRoot = "/home/ubuntu/.unidesk/deploy";
|
||||
const k8sNamespace = "unidesk";
|
||||
const k8sContainer = "unidesk-v8s-server";
|
||||
const k8sKubeconfig = "/home/ubuntu/cq-deploy/.state/v8s/kubeconfig";
|
||||
const v3sDeployDir = "/home/ubuntu/cq-deploy";
|
||||
const k8sKubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
const k3sDeployDir = "/home/ubuntu/cq-deploy";
|
||||
const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789";
|
||||
const nativeK3sInstallVersion = "v1.34.1+k3s1";
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
@@ -232,7 +232,7 @@ function targetExportDir(service: UniDeskMicroserviceConfig, runId: string): str
|
||||
}
|
||||
|
||||
function targetWorkDir(service: UniDeskMicroserviceConfig): string {
|
||||
if (service.deployment.mode === "v3sctl-managed") return v3sDeployDir;
|
||||
if (service.deployment.mode === "k3sctl-managed") return k3sDeployDir;
|
||||
if (targetIsMain(service) && service.repository.url === "https://github.com/pikasTech/unidesk") {
|
||||
return rootPath(".state", "deploy", "work", safeId(service.id));
|
||||
}
|
||||
@@ -240,7 +240,7 @@ function targetWorkDir(service: UniDeskMicroserviceConfig): string {
|
||||
}
|
||||
|
||||
function buildImageTag(service: UniDeskMicroserviceConfig): string {
|
||||
if (service.deployment.mode === "v3sctl-managed") return `unidesk-${service.id}:d601`;
|
||||
if (service.deployment.mode === "k3sctl-managed") return `unidesk-${service.id}:d601`;
|
||||
if (targetIsMain(service)) {
|
||||
if (["project-manager", "baidu-netdisk", "oa-event-flow"].includes(service.repository.composeService)) return service.repository.composeService;
|
||||
return `unidesk-${service.repository.composeService}`;
|
||||
@@ -270,8 +270,8 @@ function directDockerfileOverride(service: UniDeskMicroserviceConfig): string {
|
||||
|
||||
function k8sManifestPath(service: UniDeskMicroserviceConfig): string {
|
||||
const composeFile = service.repository.composeFile;
|
||||
if (!composeFile.endsWith(".v3s.json")) throw new Error(`${service.id} v3s service composeFile must point to *.v3s.json`);
|
||||
return composeFile.replace(/\.v3s\.json$/u, ".k8s.yaml");
|
||||
if (!composeFile.endsWith(".k3s.json")) throw new Error(`${service.id} k3s service composeFile must point to *.k3s.json`);
|
||||
return composeFile.replace(/\.k3s\.json$/u, ".k8s.yaml");
|
||||
}
|
||||
|
||||
function sourceProxyPrelude(service: UniDeskMicroserviceConfig): string {
|
||||
@@ -510,16 +510,67 @@ function writeComposeEnvFallbackPath(): string {
|
||||
return rootPath(".state", "docker-compose.env");
|
||||
}
|
||||
|
||||
function rootAccessPrelude(): string[] {
|
||||
return [
|
||||
"root_exec() {",
|
||||
" if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return; fi",
|
||||
" if sudo -n true >/dev/null 2>&1; then sudo -n \"$@\"; return; fi",
|
||||
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return; fi",
|
||||
" echo 'native_k3s_root_access=missing' >&2",
|
||||
" return 1",
|
||||
"}",
|
||||
"root_shell() {",
|
||||
" script=\"$1\"",
|
||||
" if [ \"$(id -u)\" = \"0\" ]; then bash -lc \"$script\"; return; fi",
|
||||
" if sudo -n true >/dev/null 2>&1; then sudo -n bash -lc \"$script\"; return; fi",
|
||||
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- bash -lc \"$script\"; return; fi",
|
||||
" echo 'native_k3s_root_access=missing' >&2",
|
||||
" return 1",
|
||||
"}",
|
||||
];
|
||||
}
|
||||
|
||||
function ensureNativeK3sScript(): string {
|
||||
const installCommand = [
|
||||
`INSTALL_K3S_VERSION=${nativeK3sInstallVersion}`,
|
||||
"INSTALL_K3S_EXEC=\"server --disable traefik --disable servicelb --disable metrics-server --node-name D601 --node-label unidesk.ai/node-id=D601 --node-label unidesk.ai/provider-id=D601 --tls-san 127.0.0.1 --tls-san host.docker.internal --write-kubeconfig-mode 644\"",
|
||||
"sh /tmp/unidesk-install-k3s.sh",
|
||||
].join(" ");
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
...rootAccessPrelude(),
|
||||
"if ! command -v k3s >/dev/null 2>&1; then",
|
||||
" curl -fsSL --max-time 60 https://get.k3s.io -o /tmp/unidesk-install-k3s.sh",
|
||||
` root_shell ${shellQuote(installCommand)}`,
|
||||
"fi",
|
||||
"if ! systemctl is-active --quiet k3s; then",
|
||||
" root_exec systemctl enable --now k3s",
|
||||
"fi",
|
||||
"for attempt in $(seq 1 60); do",
|
||||
` if KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes >/dev/null 2>&1; then break; fi`,
|
||||
" sleep 2",
|
||||
"done",
|
||||
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -l unidesk.ai/node-id=D601 --no-headers | grep -q .`,
|
||||
"legacy_k3s_containers=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1}')",
|
||||
"for container in $legacy_k3s_containers; do",
|
||||
" docker update --restart=no \"$container\" >/dev/null 2>&1 || true",
|
||||
" docker stop \"$container\" >/dev/null",
|
||||
" echo stopped_containerized_k3s=$container",
|
||||
"done",
|
||||
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -o wide`,
|
||||
"printf 'native_k3s=ready kubeconfig=%s\\n' /etc/rancher/k3s/k3s.yaml",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function importK3sImageScript(service: UniDeskMicroserviceConfig): string {
|
||||
const image = buildImageTag(service);
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
...rootAccessPrelude(),
|
||||
`image=${shellQuote(image)}`,
|
||||
`k3s_container=${shellQuote(k8sContainer)}`,
|
||||
"docker image inspect \"$image\" >/dev/null",
|
||||
"docker ps --format '{{.Names}}' | grep -Fx \"$k3s_container\" >/dev/null",
|
||||
"docker save \"$image\" | docker exec -i \"$k3s_container\" ctr -n k8s.io images import -",
|
||||
"docker exec \"$k3s_container\" ctr -n k8s.io images ls | grep -F \"$image\" || true",
|
||||
"docker save \"$image\" | root_exec k3s ctr -n k8s.io images import -",
|
||||
"root_exec k3s ctr -n k8s.io images ls | grep -F \"$image\" || true",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -629,7 +680,7 @@ function runtimeCommitVerified(
|
||||
desired: string,
|
||||
): boolean {
|
||||
if (healthCommit !== null && healthCommit.length > 0 && !commitMatches(healthCommit, desired)) return false;
|
||||
if (service.deployment.mode === "v3sctl-managed") return commitMatches(orchestratorCommit, desired);
|
||||
if (service.deployment.mode === "k3sctl-managed") return commitMatches(orchestratorCommit, desired);
|
||||
return commitMatches(imageCommit, desired);
|
||||
}
|
||||
|
||||
@@ -820,7 +871,7 @@ async function readDockerImageCommit(config: UniDeskConfig, service: UniDeskMicr
|
||||
}
|
||||
|
||||
async function readK8sCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<string | null> {
|
||||
if (service.deployment.mode !== "v3sctl-managed") return null;
|
||||
if (service.deployment.mode !== "k3sctl-managed") return null;
|
||||
const result = await runTargetCommand(config, service, k8sCommitProbeScript(service), "/home/ubuntu", 30_000, 20_000);
|
||||
const commit = parseFullCommit(result.stdout);
|
||||
return commit.length > 0 ? commit : null;
|
||||
@@ -961,7 +1012,9 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi
|
||||
const build = await step(config, service, "docker-build", buildScript, targetWorkDir(service), Math.min(options.timeoutMs, 540_000), !targetIsMain(service));
|
||||
if (!pushStep(steps, build)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
|
||||
if (service.deployment.mode === "v3sctl-managed") {
|
||||
if (service.deployment.mode === "k3sctl-managed") {
|
||||
const nativeK3s = await step(config, service, "ensure-native-k3s", ensureNativeK3sScript(), "/home/ubuntu", 180_000, true);
|
||||
if (!pushStep(steps, nativeK3s)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const imageImport = await step(config, service, "import-k3s-image", importK3sImageScript(service), targetWorkDir(service), 180_000, true);
|
||||
if (!pushStep(steps, imageImport)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const apply = await step(config, service, "kubectl-apply", applyK8sScript(service), targetWorkDir(service), 60_000, false);
|
||||
|
||||
+37
-37
@@ -61,8 +61,8 @@ const SERVICE_CHECK_NAMES = [
|
||||
"microservice:catalog-todo-note",
|
||||
"microservice:catalog-oa-event-flow",
|
||||
"microservice:catalog-code-queue",
|
||||
"microservice:v3sctl-adapter-status",
|
||||
"microservice:v3sctl-control-plane",
|
||||
"microservice:k3sctl-adapter-status",
|
||||
"microservice:k3sctl-control-plane",
|
||||
"microservice:catalog-filebrowser",
|
||||
"microservice:filebrowser-health",
|
||||
"microservice:filebrowser-webui",
|
||||
@@ -1028,8 +1028,8 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
|
||||
const oaEventFlowEvents = dockerCoreJson("/api/microservices/oa-event-flow/proxy/api/events?limit=10");
|
||||
const oaEventFlowPipelineEvents = dockerCoreJson("/api/microservices/oa-event-flow/proxy/api/events?tags=service:pipeline&limit=10");
|
||||
const oaEventFlowStats = dockerCoreJson("/api/microservices/oa-event-flow/proxy/api/stats/trace?limit=10");
|
||||
const v3sctlStatus = dockerCoreJson("/api/microservices/v3sctl-adapter/status");
|
||||
const v3sctlControlPlane = dockerCoreJson("/api/microservices/v3sctl-adapter/proxy/api/control-plane");
|
||||
const k3sctlStatus = dockerCoreJson("/api/microservices/k3sctl-adapter/status");
|
||||
const k3sctlControlPlane = dockerCoreJson("/api/microservices/k3sctl-adapter/proxy/api/control-plane");
|
||||
const codeQueueStatus = dockerCoreJson("/api/microservices/code-queue/status");
|
||||
const codeQueueHealth = dockerCoreJson("/api/microservices/code-queue/health");
|
||||
const codeQueueTasks = dockerCoreJson("/api/microservices/code-queue/proxy/api/tasks/overview?limit=5&transcriptLimit=1&compact=1&afterSeq=0&preferId=");
|
||||
@@ -1106,7 +1106,7 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
|
||||
const oaEventFlowStatsBody = (oaEventFlowStats as { body?: { ok?: boolean; stats?: unknown[]; returned?: number } }).body;
|
||||
const codeQueueHealthBody = (codeQueueHealth as { body?: { ok?: boolean; egressProxy?: { connected?: boolean }; queue?: { defaultModel?: string; judgeConfigured?: boolean; modelReasoningEfforts?: Record<string, string> } } }).body;
|
||||
const codeQueueTasksBody = (codeQueueTasks as { body?: { ok?: boolean; queue?: { defaultModel?: string; modelReasoningEfforts?: Record<string, string> }; tasks?: unknown[] } }).body;
|
||||
const v3sctlControlPlaneBody = (v3sctlControlPlane as { body?: {
|
||||
const k3sctlControlPlaneBody = (k3sctlControlPlane as { body?: {
|
||||
ok?: boolean;
|
||||
clusterId?: string;
|
||||
noFallback?: boolean;
|
||||
@@ -1123,8 +1123,8 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
|
||||
instances?: Array<{ id?: string; healthy?: boolean; proxyMode?: string }>;
|
||||
}>;
|
||||
} }).body;
|
||||
const v3sctlCodeQueueService = v3sctlControlPlaneBody?.services?.find((service) => service.id === "code-queue");
|
||||
const v3sctlD518Instance = v3sctlCodeQueueService?.instances?.find((instance) => instance.id === "D518");
|
||||
const k3sctlCodeQueueService = k3sctlControlPlaneBody?.services?.find((service) => service.id === "code-queue");
|
||||
const k3sctlD518Instance = k3sctlCodeQueueService?.instances?.find((instance) => instance.id === "D518");
|
||||
const filebrowserHealthBody = (filebrowserHealth as { body?: { status?: string } }).body;
|
||||
const filebrowserD601HealthBody = (filebrowserD601Health as { body?: { status?: string } }).body;
|
||||
const filebrowserWebuiText = String((filebrowserWebui as { body?: { text?: string } }).body?.text || "");
|
||||
@@ -1159,39 +1159,39 @@ async function serviceChecks(config: UniDeskConfig, urls: PublicUrls, checks: E2
|
||||
(microservices as { ok?: boolean }).ok === true
|
||||
&& codeQueue?.providerId === "D601"
|
||||
&& codeQueue.backend?.public === false
|
||||
&& codeQueue.backend?.proxyMode === "v3sctl-adapter-http"
|
||||
&& codeQueue.deployment?.mode === "v3sctl-managed"
|
||||
&& codeQueue.runtime?.orchestrator === "v3sctl"
|
||||
&& codeQueue.backend?.proxyMode === "k3sctl-adapter-http"
|
||||
&& codeQueue.deployment?.mode === "k3sctl-managed"
|
||||
&& codeQueue.runtime?.orchestrator === "k3sctl"
|
||||
&& codeQueue.runtime?.container === null,
|
||||
{ microservices });
|
||||
addSelectedCheck(checks, options, "microservice:v3sctl-adapter-status",
|
||||
(v3sctlStatus as { ok?: boolean; body?: { microservice?: { id?: string; providerId?: string } } }).ok === true
|
||||
&& (v3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.id === "v3sctl-adapter"
|
||||
&& (v3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.providerId === "D601",
|
||||
v3sctlStatus);
|
||||
addSelectedCheck(checks, options, "microservice:v3sctl-control-plane",
|
||||
(v3sctlControlPlane as { ok?: boolean }).ok === true
|
||||
&& v3sctlControlPlaneBody?.ok === true
|
||||
&& v3sctlControlPlaneBody.clusterId === "unidesk-v8s"
|
||||
&& v3sctlControlPlaneBody.noFallback === true
|
||||
&& v3sctlControlPlaneBody.managedServicesHealthy === true
|
||||
&& v3sctlControlPlaneBody.kubeApiProxy?.mode === "kubernetes-api-service-proxy"
|
||||
&& v3sctlCodeQueueService?.status === "healthy"
|
||||
&& v3sctlCodeQueueService?.topologyComplete === true
|
||||
&& v3sctlCodeQueueService?.servingHealthy === true
|
||||
&& v3sctlCodeQueueService?.active?.id === "D601"
|
||||
&& v3sctlCodeQueueService?.active?.healthy === true
|
||||
&& (v3sctlCodeQueueService?.presentNodeIds ?? []).includes("D601")
|
||||
&& (v3sctlCodeQueueService?.presentNodeIds ?? []).includes("D518")
|
||||
&& (v3sctlCodeQueueService?.missingNodeIds ?? []).length === 0
|
||||
&& v3sctlD518Instance?.healthy === true
|
||||
&& v3sctlD518Instance?.proxyMode === "kubernetes-api-pod-readiness",
|
||||
addSelectedCheck(checks, options, "microservice:k3sctl-adapter-status",
|
||||
(k3sctlStatus as { ok?: boolean; body?: { microservice?: { id?: string; providerId?: string } } }).ok === true
|
||||
&& (k3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.id === "k3sctl-adapter"
|
||||
&& (k3sctlStatus as { body?: { microservice?: { id?: string; providerId?: string } } }).body?.microservice?.providerId === "D601",
|
||||
k3sctlStatus);
|
||||
addSelectedCheck(checks, options, "microservice:k3sctl-control-plane",
|
||||
(k3sctlControlPlane as { ok?: boolean }).ok === true
|
||||
&& k3sctlControlPlaneBody?.ok === true
|
||||
&& k3sctlControlPlaneBody.clusterId === "unidesk-k3s"
|
||||
&& k3sctlControlPlaneBody.noFallback === true
|
||||
&& k3sctlControlPlaneBody.managedServicesHealthy === true
|
||||
&& k3sctlControlPlaneBody.kubeApiProxy?.mode === "kubernetes-api-service-proxy"
|
||||
&& k3sctlCodeQueueService?.status === "healthy"
|
||||
&& k3sctlCodeQueueService?.topologyComplete === true
|
||||
&& k3sctlCodeQueueService?.servingHealthy === true
|
||||
&& k3sctlCodeQueueService?.active?.id === "D601"
|
||||
&& k3sctlCodeQueueService?.active?.healthy === true
|
||||
&& (k3sctlCodeQueueService?.presentNodeIds ?? []).includes("D601")
|
||||
&& (k3sctlCodeQueueService?.presentNodeIds ?? []).includes("D518")
|
||||
&& (k3sctlCodeQueueService?.missingNodeIds ?? []).length === 0
|
||||
&& k3sctlD518Instance?.healthy === true
|
||||
&& k3sctlD518Instance?.proxyMode === "kubernetes-api-pod-readiness",
|
||||
{
|
||||
ok: (v3sctlControlPlane as { ok?: boolean }).ok,
|
||||
clusterId: v3sctlControlPlaneBody?.clusterId,
|
||||
noFallback: v3sctlControlPlaneBody?.noFallback,
|
||||
kubeApiProxy: v3sctlControlPlaneBody?.kubeApiProxy,
|
||||
service: v3sctlCodeQueueService,
|
||||
ok: (k3sctlControlPlane as { ok?: boolean }).ok,
|
||||
clusterId: k3sctlControlPlaneBody?.clusterId,
|
||||
noFallback: k3sctlControlPlaneBody?.noFallback,
|
||||
kubeApiProxy: k3sctlControlPlaneBody?.kubeApiProxy,
|
||||
service: k3sctlCodeQueueService,
|
||||
});
|
||||
addSelectedCheck(checks, options, "microservice:catalog-filebrowser", (microservices as { ok?: boolean }).ok === true
|
||||
&& filebrowser?.providerId === "D518"
|
||||
|
||||
Reference in New Issue
Block a user