fix: enforce native k3s runtime

scripts/src/deploy.ts

@@ -83,6 +83,8 @@ const k8sKubeconfig = "/etc/rancher/k3s/k3s.yaml";
 const k3sDeployDir = "/home/ubuntu/cq-deploy";
 const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789";
 const nativeK3sInstallVersion = "v1.34.1+k3s1";
+const nativeK3sImage = "rancher/k3s:v1.34.1-k3s1";
+const nativeK3sCtrAddress = "/run/k3s/containerd/containerd.sock";
 
 function nowIso(): string {
   return new Date().toISOString();
@@ -532,6 +534,7 @@ function rootAccessPrelude(): string[] {
 
 function ensureNativeK3sScript(): string {
   const installCommand = [
+    "INSTALL_K3S_SKIP_DOWNLOAD=true",
     `INSTALL_K3S_VERSION=${nativeK3sInstallVersion}`,
     "INSTALL_K3S_EXEC=\"server --disable traefik --disable servicelb --disable metrics-server --node-name D601 --node-label unidesk.ai/node-id=D601 --node-label unidesk.ai/provider-id=D601 --tls-san 127.0.0.1 --tls-san host.docker.internal --write-kubeconfig-mode 644\"",
     "sh /tmp/unidesk-install-k3s.sh",
@@ -539,7 +542,60 @@ function ensureNativeK3sScript(): string {
   return [
     "set -euo pipefail",
     ...rootAccessPrelude(),
-    "if ! command -v k3s >/dev/null 2>&1; then",
+    `native_k3s_image=${shellQuote(nativeK3sImage)}`,
+    `native_ctr_address=${shellQuote(nativeK3sCtrAddress)}`,
+    "install_native_k3s_binaries() {",
+    "  missing=0",
+    "  for binary in k3s ctr containerd containerd-shim-runc-v2 crictl runc cni flannel bridge host-local loopback portmap bandwidth firewall; do",
+    "    command -v \"$binary\" >/dev/null 2>&1 || missing=1",
+    "  done",
+    "  if [ \"$missing\" = \"0\" ]; then return; fi",
+    "  docker image inspect \"$native_k3s_image\" >/dev/null 2>&1 || docker pull \"$native_k3s_image\"",
+    "  tmp_dir=$(mktemp -d)",
+    "  tmp_container=$(docker create \"$native_k3s_image\" sh -lc true)",
+    "  docker cp \"$tmp_container:/bin/.\" \"$tmp_dir/bin\"",
+    "  docker rm \"$tmp_container\" >/dev/null",
+    "  for binary in k3s ctr containerd containerd-shim-runc-v2 crictl runc cni flannel bridge host-local loopback portmap bandwidth firewall; do",
+    "    if [ -e \"$tmp_dir/bin/$binary\" ]; then",
+    "      root_exec install -m 755 \"$tmp_dir/bin/$binary\" \"/usr/local/bin/$binary\"",
+    "      echo native_k3s_binary_installed=$binary",
+    "    fi",
+    "  done",
+    "  rm -rf \"$tmp_dir\"",
+    "}",
+    "unmount_invalid_wsl_kubelet_mounts() {",
+    "  if awk 'NF != 6 && $0 ~ /\\/Docker\\/host/ {found=1} END {exit found ? 0 : 1}' /proc/mounts; then",
+    "    root_exec umount /Docker/host >/dev/null 2>&1 || root_exec umount -l /Docker/host >/dev/null 2>&1 || true",
+    "    echo native_k3s_unmounted_invalid_mount=/Docker/host",
+    "  fi",
+    "  awk 'NF != 6 {print \"native_k3s_invalid_mount_line=\" NR \":\" $0; bad=1} END {exit bad}' /proc/mounts",
+    "}",
+    "install_system_images_from_legacy_k3s() {",
+    "  legacy_container=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1; exit}')",
+    "  if [ -n \"$legacy_container\" ]; then",
+    "    if docker exec \"$legacy_container\" ctr -n k8s.io images export /tmp/unidesk-k3s-system-images.tar docker.io/rancher/local-path-provisioner:v0.0.32 docker.io/rancher/mirrored-coredns-coredns:1.12.3 >/dev/null 2>&1; then",
+    "      docker cp \"$legacy_container:/tmp/unidesk-k3s-system-images.tar\" /tmp/unidesk-k3s-system-images.tar >/dev/null",
+    "      root_exec ctr --address \"$native_ctr_address\" -n k8s.io images import /tmp/unidesk-k3s-system-images.tar >/dev/null",
+    "      echo native_k3s_imported_legacy_system_images=$legacy_container",
+    "    fi",
+    "  fi",
+    "}",
+    "install_pause_image() {",
+    "  if root_exec ctr --address \"$native_ctr_address\" -n k8s.io images ls | grep -q 'docker.io/rancher/mirrored-pause:3.6'; then return; fi",
+    "  docker image inspect rancher/mirrored-pause:3.6 >/dev/null 2>&1 || {",
+    "    docker image inspect \"$native_k3s_image\" >/dev/null 2>&1 || docker pull \"$native_k3s_image\"",
+    "    tmp_dir=$(mktemp -d)",
+    "    printf '%s\\n' \"FROM $native_k3s_image\" 'ENTRYPOINT [\"/bin/sleep\", \"365d\"]' > \"$tmp_dir/Dockerfile\"",
+    "    docker build -q -t rancher/mirrored-pause:3.6 \"$tmp_dir\" >/dev/null",
+    "    rm -rf \"$tmp_dir\"",
+    "  }",
+    "  docker save rancher/mirrored-pause:3.6 -o /tmp/unidesk-k3s-pause.tar",
+    "  root_exec ctr --address \"$native_ctr_address\" -n k8s.io images import /tmp/unidesk-k3s-pause.tar >/dev/null",
+    "  echo native_k3s_imported_pause_image=rancher/mirrored-pause:3.6",
+    "}",
+    "install_native_k3s_binaries",
+    "unmount_invalid_wsl_kubelet_mounts",
+    "if ! systemctl cat k3s.service >/dev/null 2>&1; then",
     "  curl -fsSL --max-time 60 https://get.k3s.io -o /tmp/unidesk-install-k3s.sh",
     `  root_shell ${shellQuote(installCommand)}`,
     "fi",
@@ -551,6 +607,11 @@ function ensureNativeK3sScript(): string {
     "  sleep 2",
     "done",
     `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl get nodes -l unidesk.ai/node-id=D601 --no-headers | grep -q .`,
+    `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl wait --for=condition=Ready node -l unidesk.ai/node-id=D601 --timeout=180s`,
+    "install_system_images_from_legacy_k3s",
+    "install_pause_image",
+    `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n kube-system delete pod -l k8s-app=kube-dns --ignore-not-found >/dev/null 2>&1 || true`,
+    `KUBECONFIG=${shellQuote(k8sKubeconfig)} kubectl -n kube-system delete pod -l app=local-path-provisioner --ignore-not-found >/dev/null 2>&1 || true`,
     "legacy_k3s_containers=$(docker ps --format '{{.Names}} {{.Image}}' | awk '$2 ~ /^rancher\\/k3s:/ {print $1}')",
     "for container in $legacy_k3s_containers; do",
     "  docker update --restart=no \"$container\" >/dev/null 2>&1 || true",
@@ -569,8 +630,8 @@ function importK3sImageScript(service: UniDeskMicroserviceConfig): string {
     ...rootAccessPrelude(),
     `image=${shellQuote(image)}`,
     "docker image inspect \"$image\" >/dev/null",
-    "docker save \"$image\" | root_exec k3s ctr -n k8s.io images import -",
-    "root_exec k3s ctr -n k8s.io images ls | grep -F \"$image\" || true",
+    `docker save "$image" | root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images import -`,
+    `root_exec ctr --address ${shellQuote(nativeK3sCtrAddress)} -n k8s.io images ls | grep -F "$image" || true`,
   ].join("\n");
 }
 
@@ -1013,7 +1074,7 @@ async function applyOneService(config: UniDeskConfig, service: UniDeskMicroservi
   if (!pushStep(steps, build)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
 
   if (service.deployment.mode === "k3sctl-managed") {
-    const nativeK3s = await step(config, service, "ensure-native-k3s", ensureNativeK3sScript(), "/home/ubuntu", 180_000, true);
+    const nativeK3s = await step(config, service, "ensure-native-k3s", ensureNativeK3sScript(), "/home/ubuntu", 600_000, true);
     if (!pushStep(steps, nativeK3s)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };
     const imageImport = await step(config, service, "import-k3s-image", importK3sImageScript(service), targetWorkDir(service), 180_000, true);
     if (!pushStep(steps, imageImport)) return { ok: false, serviceId: service.id, startedAt, finishedAt: nowIso(), resolvedCommit, before, steps };