fix: migrate code queue control plane to k3s
This commit is contained in:
Codex
+67
-14
@@ -79,10 +79,10 @@ const shortRemoteTimeoutMs = 20_000;
|
||||
const pollIntervalMs = 5_000;
|
||||
const remoteDeployRoot = "/home/ubuntu/.unidesk/deploy";
|
||||
const k8sNamespace = "unidesk";
|
||||
const k8sContainer = "unidesk-v8s-server";
|
||||
const k8sKubeconfig = "/home/ubuntu/cq-deploy/.state/v8s/kubeconfig";
|
||||
const v3sDeployDir = "/home/ubuntu/cq-deploy";
|
||||
const k8sKubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
const k3sDeployDir = "/home/ubuntu/cq-deploy";
|
||||
const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789";
|
||||
const nativeK3sInstallVersion = "v1.34.1+k3s1";
|
||||
|
||||
function nowIso(): string {
|
||||
return new Date().toISOString();
|
||||
@@ -232,7 +232,7 @@ function targetExportDir(service: UniDeskMicroserviceConfig, runId: string): str
|
||||
}
|
||||
|
||||
function targetWorkDir(service: UniDeskMicroserviceConfig): string {
|
||||
if (service.deployment.mode === "v3sctl-managed") return v3sDeployDir;
|
||||
if (service.deployment.mode === "k3sctl-managed") return k3sDeployDir;
|
||||
if (targetIsMain(service) && service.repository.url === "https://github.com/pikasTech/unidesk") {
|
||||
return rootPath(".state", "deploy", "work", safeId(service.id));
|
||||
}
|
||||
@@ -240,7 +240,7 @@ function targetWorkDir(service: UniDeskMicroserviceConfig): string {
|
||||
}
|
||||
|
||||
function buildImageTag(service: UniDeskMicroserviceConfig): string {
|
||||
if (service.deployment.mode === "v3sctl-managed") return `unidesk-${service.id}:d601`;
|
||||
if (service.deployment.mode === "k3sctl-managed") return `unidesk-${service.id}:d601`;
|
||||
if (targetIsMain(service)) {
|
||||
if (["project-manager", "baidu-netdisk", "oa-event-flow"].includes(service.repository.composeService)) return service.repository.composeService;
|
||||
return `unidesk-${service.repository.composeService}`;
|
||||
@@ -270,8 +270,8 @@ function directDockerfileOverride(service: UniDeskMicroserviceConfig): string {
|
||||
|
||||
function k8sManifestPath(service: UniDeskMicroserviceConfig): string {
|
||||
const composeFile = service.repository.composeFile;
|
||||
if (!composeFile.endsWith(".v3s.json")) throw new Error(`${service.id} v3s service composeFile must point to *.v3s.json`);
|
||||
return composeFile.replace(/\.v3s\.json$/u, ".k8s.yaml");
|
||||
if (!composeFile.endsWith(".k3s.json")) throw new Error(`${service.id} k3s service composeFile must point to *.k3s.json`);
|
||||
return composeFile.replace(/\.k3s\.json$/u, ".k8s.yaml");
|
||||
}
|
||||
|
||||
function sourceProxyPrelude(service: UniDeskMicroserviceConfig): string {
|
||||
@@ -510,16 +510,67 @@ function writeComposeEnvFallbackPath(): string {
|
||||
return rootPath(".state", "docker-compose.env");
|
||||
}
|
||||
|
||||
function rootAccessPrelude(): string[] {
|
||||
return [
|
||||
"root_exec() {",
|
||||
" if [ \"$(id -u)\" = \"0\" ]; then \"$@\"; return; fi",
|
||||
" if sudo -n true >/dev/null 2>&1; then sudo -n \"$@\"; return; fi",
|
||||
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- \"$@\"; return; fi",
|
||||
" echo 'native_k3s_root_access=missing' >&2",
|
||||
" return 1",
|
||||
"}",
|
||||
"root_shell() {",
|
||||
" script=\"$1\"",
|
||||
" if [ \"$(id -u)\" = \"0\" ]; then bash -lc \"$script\"; return; fi",
|
||||
" if sudo -n true >/dev/null 2>&1; then sudo -n bash -lc \"$script\"; return; fi",
|
||||
" if [ -x /mnt/c/Windows/System32/wsl.exe ]; then /mnt/c/Windows/System32/wsl.exe -u root -- bash -lc \"$script\"; return; fi",
|
||||
" echo 'native_k3s_root_access=missing' >&2",
|
||||
" return 1",
|
||||
"}",
|
||||
];
|
||||
}
|
||||
|
||||
function ensureNativeK3sScript(): string {
|
||||
const installCommand = [
|
||||
`INSTALL_K3S_VERSION=${nativeK3sInstallVersion}`,
|
||||
"INSTALL_K3S_EXEC=\"server --disable traefik --disable servicelb --disable metrics-server --node-name D601 --node-label unidesk.ai/node-id=D601 --node-label unidesk.ai/provider-id=D601 --tls-san 127.0.0.1 --tls-san host.docker.internal --write-kubeconfig-mode 644\"",
|
||||
"sh /tmp/unidesk-install-k3s.sh",
|
||||
].join(" ");
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
...rootAccessPrelude(),
|
||||
"if ! command -v k3s >/dev/null 2>&1; then",
|
||||
" curl -fsSL --max-time 60 https://get.k3s.io -o /tmp/unidesk-install-k3s.sh",
|
||||
` root_shell ${shellQuote(installCommand)}`,
|
||||
"fi",
|
||||
"if ! systemctl is-active --quiet k3s; then",
|
||||
" root_exec systemctl enable --now k3s",
|
||||
"fi",
|
||||
"for attempt in $(seq 1 60); do",
|
||||
` if KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes >/dev/null 2>&1; then break; fi`,
|
||||
" sleep 2",
|
||||
"done",
|
||||
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -l unidesk.ai/node-id=D601 --no-headers | grep -q .`,
|
||||
"legacy_k3s_containers=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1}')",
|
||||
"for container in $legacy_k3s_containers; do",
|
||||
" docker update --restart=no \"$container\" >/dev/null 2>&1 || true",
|
||||
" docker stop \"$container\" >/dev/null",
|
||||
" echo stopped_containerized_k3s=$container",
|
||||
"done",
|
||||
`KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -o wide`,
|
||||
"printf 'native_k3s=ready kubeconfig=%s\\n' /etc/rancher/k3s/k3s.yaml",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function importK3sImageScript(service: UniDeskMicroserviceConfig): string {
|
||||
const image = buildImageTag(service);
|
||||
return [
|
||||
"set -euo pipefail",
|
||||
...rootAccessPrelude(),
|
||||
`image=${shellQuote(image)}`,
|
||||
`k3s_container=${shellQuote(k8sContainer)}`,
|
||||
"docker image inspect \"$image\" >/dev/null",
|
||||
"docker ps --format '{{.Names}}' | grep -Fx \"$k3s_container\" >/dev/null",
|
||||
"docker save \"$image\" | docker exec -i \"$k3s_container\" ctr -n k8s.io images import -",
|
||||
"docker exec \"$k3s_container\" ctr -n k8s.io images ls | grep -F \"$image\" || true",
|
||||
"docker save \"$image\" | root_exec k3s ctr -n k8s.io images import -",
|
||||
"root_exec k3s ctr -n k8s.io images ls | grep -F \"$image\" || true",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -629,7 +680,7 @@ function runtimeCommitVerified(
|
||||
desired: string,
|
||||
): boolean {
|
||||
if (healthCommit !== null && healthCommit.length > 0 && !commitMatches(healthCommit, desired)) return false;
|
||||
if (service.deployment.mode === "v3sctl-managed") return commitMatches(orchestratorCommit, desired);
|
||||
if (service.deployment.mode === "k3sctl-managed") return commitMatches(orchestratorCommit, desired);
|
||||
return commitMatches(imageCommit, desired);
|
||||
}
|
||||
|
||||
@@ -820,7 +871,7 @@ async function readDockerImageCommit(config: UniDeskConfig, service: UniDeskMicr
|
||||
}
|
||||
|
||||
async function readK8sCommit(config: UniDeskConfig, service: UniDeskMicroserviceConfig): Promise<string | null> {
|
||||
if (service.deployment.mode !== "v3sctl-managed") return null;
|
||||
if (service.deployment.mode !== "k3sctl-managed") return null;
|
||||
const result = await runTargetCommand(config, service, k8sCommitProbeScript(service), "/home/ubuntu", 30_000, 20_000);
|
||||
const commit = parseFullCommit(result.stdout);
|
||||
return commit.length > 0 ? commit : null;
|
||||
@@ -961,7 +1012,9 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi
|
||||
const build = await step(config, service, "docker-build", buildScript, targetWorkDir(service), Math.min(options.timeoutMs, 540_000), !targetIsMain(service));
|
||||
if (!pushStep(steps, build)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
|
||||
if (service.deployment.mode === "v3sctl-managed") {
|
||||
if (service.deployment.mode === "k3sctl-managed") {
|
||||
const nativeK3s = await step(config, service, "ensure-native-k3s", ensureNativeK3sScript(), "/home/ubuntu", 180_000, true);
|
||||
if (!pushStep(steps, nativeK3s)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const imageImport = await step(config, service, "import-k3s-image", importK3sImageScript(service), targetWorkDir(service), 180_000, true);
|
||||
if (!pushStep(steps, imageImport)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
|
||||
const apply = await step(config, service, "kubectl-apply", applyK8sScript(service), targetWorkDir(service), 60_000, false);
|
||||
|
||||
Reference in New Issue
Block a user