fix: make HWLAB public exposure restart observable

scripts/src/hwlab-node.ts

@@ -1909,6 +1909,9 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
     "      }",
     "      if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {",
     "        foundDeployment = true;",
+    "        item.spec = item.spec || {};",
+    "        const nextStrategy = { type: 'Recreate' };",
+    "        if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }",
     "        const podSpec = podSpecFor(item);",
     "        for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {",
     "          if (!isObject(container)) continue;",
@@ -2074,6 +2077,8 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
     "  }",
     "  const podSpec = podSpecFor(deployment);",
     "  const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];",
+    "  const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;",
+    "  if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });",
     "  const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));",
     "  const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;",
     "  const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;",
@@ -3144,16 +3149,19 @@ function runNodePublicExposure(options: NodePublicExposureOptions): Record<strin
       next: { fixSecretSource: `create .state/secrets/${exposure.tokenSourceRef} with ${exposure.tokenSourceKey}=<redacted>` },
     };
   }
-  const secretResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds);
-  const secretFields = keyValueLinesFromText(statusText(secretResult));
+  const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds);
+  let secretFields = keyValueLinesFromText(statusText(secretApplyResult));
+  let secretResult = secretApplyResult;
+  if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") {
+    const restartResult = runPublicExposureFrpcRecreate(options);
+    secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) };
+    secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult);
+  }
   const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds);
   const caddyFields = keyValueLinesFromText(statusText(caddyResult));
-  const secretOk = options.dryRun ? secretResult.exitCode === 0 : secretResult.exitCode === 0 && secretFields.afterSecretExists === "yes";
-  const ok = secretOk
-    && caddyResult.exitCode === 0
-    && caddyFields.afterBlockPresent === "yes"
-    && caddyFields.validateExitCode === "0"
-    && (options.dryRun || caddyFields.active === "active");
+  const secretStatus = publicExposureSecretStatus(secretFields, secretResult);
+  const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult);
+  const ok = secretStatus.ok === true && caddyStatus.ok === true;
   return {
     ok,
     command: "hwlab nodes control-plane public-exposure",
@@ -3171,8 +3179,8 @@ function runNodePublicExposure(options: NodePublicExposureOptions): Record<strin
       fingerprint: source.fingerprint,
       valueRedacted: true,
     },
-    secret: publicExposureSecretStatus(secretFields, secretResult),
-    caddy: publicExposureCaddyStatus(caddyFields, caddyResult),
+    secret: secretStatus,
+    caddy: caddyStatus,
     valuesRedacted: true,
     next: ok
       ? {
@@ -3211,26 +3219,54 @@ function publicExposureTokenSourcePaths(exposure: HwlabRuntimePublicExposureSpec
 function publicExposureSecretStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
   const dryRun = fields.dryRun === "true";
   return {
-    ok: result.exitCode === 0 && (dryRun || fields.afterSecretExists === "yes"),
+    ok: result.exitCode === 0 && (dryRun || (
+      fields.afterSecretExists === "yes"
+      && fields.strategyPatchExitCode === "0"
+      && fields.rolloutRestartExitCode === "0"
+      && fields.rolloutStatusExitCode === "0"
+    )),
     dryRun,
     mutation: fields.mutation === "true",
     namespace: fields.namespace || null,
     secret: fields.secret || null,
+    deployment: fields.deployment || null,
     beforeExists: fields.beforeSecretExists === "yes",
     afterExists: fields.afterSecretExists === "yes",
     tokenBytes: numericField(fields.tokenBytes),
     applyExitCode: numericField(fields.applyExitCode),
+    restartMode: fields.restartMode || null,
+    strategyPatchExitCode: numericField(fields.strategyPatchExitCode),
     rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode),
+    scaleDownExitCode: numericField(fields.scaleDownExitCode),
+    scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode),
+    scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount),
+    scaleUpExitCode: numericField(fields.scaleUpExitCode),
     rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode),
+    readyReplicas: numericField(fields.readyReplicas),
+    availableReplicas: numericField(fields.availableReplicas),
+    desiredReplicas: numericField(fields.desiredReplicas),
+    strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null,
+    restartErrorPreview: fields.restartErrorPreview || null,
+    scaleDownErrorPreview: fields.scaleDownErrorPreview || null,
+    scaleUpErrorPreview: fields.scaleUpErrorPreview || null,
+    readyErrorPreview: fields.readyErrorPreview || null,
     exitCode: result.exitCode,
     stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
   };
 }
 
 function publicExposureCaddyStatus(fields: Record<string, string>, result: CommandResult): Record<string, unknown> {
+  const dryRun = fields.dryRun === "true";
   return {
-    ok: result.exitCode === 0 && fields.afterBlockPresent === "yes" && fields.validateExitCode === "0",
-    dryRun: fields.dryRun === "true",
+    ok: result.exitCode === 0
+      && fields.afterBlockPresent === "yes"
+      && fields.validateExitCode === "0"
+      && (dryRun || (
+        fields.active === "active"
+        && fields.localWebStatus === "200"
+        && fields.localApiStatus === "200"
+      )),
+    dryRun,
     mutation: fields.mutation === "true",
     hostname: fields.hostname || null,
     configPath: fields.configPath || null,
@@ -3251,10 +3287,12 @@ function publicExposureCaddyStatus(fields: Record<string, string>, result: Comma
 }
 
 function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
+  const deployment = `${options.spec.runtimeNamespace}-frpc`;
   return [
     "set +e",
     `namespace=${shellQuote(options.spec.runtimeNamespace)}`,
     `secret=${shellQuote(exposure.secretName)}`,
+    `deployment=${shellQuote(deployment)}`,
     `token_key=${shellQuote(exposure.tokenKey)}`,
     `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
     "token=$(cat)",
@@ -3262,8 +3300,17 @@ function publicExposureSecretScript(options: NodePublicExposureOptions, exposure
     "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes",
     "token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')",
     "apply_exit=",
+    "restart_mode=recreate-rollout-restart",
+    "strategy_patch_exit=",
     "rollout_restart_exit=",
+    "scale_down_exit=",
+    "scale_down_wait_exit=",
+    "scale_down_wait_pod_count=",
+    "scale_up_exit=",
     "rollout_status_exit=",
+    "ready_replicas=",
+    "available_replicas=",
+    "desired_replicas=",
     "mutation=false",
     "if [ \"$dry_run\" = false ]; then",
     "  tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)",
@@ -3285,30 +3332,148 @@ function publicExposureSecretScript(options: NodePublicExposureOptions, exposure
     "  apply_exit=$?",
     "  rm -f \"$tmp\"",
     "  if [ \"$apply_exit\" = 0 ]; then mutation=true; fi",
-    "  kubectl -n \"$namespace\" rollout restart deploy/hwlab-v03-frpc >/tmp/hwlab-public-frpc-rollout-restart.out 2>/tmp/hwlab-public-frpc-rollout-restart.err",
-    "  rollout_restart_exit=$?",
-    "  if [ \"$rollout_restart_exit\" = 0 ]; then",
-    "    kubectl -n \"$namespace\" rollout status deploy/hwlab-v03-frpc --timeout=60s >/tmp/hwlab-public-frpc-rollout-status.out 2>/tmp/hwlab-public-frpc-rollout-status.err",
-    "    rollout_status_exit=$?",
-    "  fi",
     "fi",
     "after_secret_exists=no",
     "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes",
     "printf 'namespace\\t%s\\n' \"$namespace\"",
     "printf 'secret\\t%s\\n' \"$secret\"",
+    "printf 'deployment\\t%s\\n' \"$deployment\"",
     "printf 'dryRun\\t%s\\n' \"$dry_run\"",
     "printf 'mutation\\t%s\\n' \"$mutation\"",
     "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"",
     "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"",
     "printf 'tokenBytes\\t%s\\n' \"$token_bytes\"",
     "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"",
+    "printf 'restartMode\\t%s\\n' \"$restart_mode\"",
+    "printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"",
     "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"",
+    "printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"",
+    "printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"",
+    "printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"",
+    "printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"",
     "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"",
+    "printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"",
+    "printf 'availableReplicas\\t%s\\n' \"$available_replicas\"",
+    "printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"",
     "if [ \"$dry_run\" = true ]; then exit 0; fi",
-    "[ \"$after_secret_exists\" = yes ]",
+    "[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]",
   ].join("\n");
 }
 
+function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult {
+  const namespace = options.spec.runtimeNamespace;
+  const deployment = `${namespace}-frpc`;
+  const fields: Record<string, string> = {
+    deployment,
+    restartMode: "recreate-rollout-restart",
+    strategyPatchExitCode: "",
+    rolloutRestartExitCode: "",
+    scaleDownExitCode: "",
+    scaleDownWaitExitCode: "",
+    scaleDownWaitPodCount: "",
+    scaleUpExitCode: "",
+    rolloutStatusExitCode: "",
+    readyReplicas: "",
+    availableReplicas: "",
+    desiredReplicas: "",
+    strategyPatchErrorPreview: "",
+    restartErrorPreview: "",
+    scaleDownErrorPreview: "",
+    scaleUpErrorPreview: "",
+    readyErrorPreview: "",
+  };
+  const stdoutParts: string[] = [];
+  const stderrParts: string[] = [];
+  const addResult = (result: CommandResult): void => {
+    if (result.stdout.trim()) stdoutParts.push(result.stdout.trim());
+    if (result.stderr.trim()) stderrParts.push(result.stderr.trim());
+  };
+  const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55);
+  const strategyPatch = runTransScript(options.node, [
+    "set +e",
+    `namespace=${shellQuote(namespace)}`,
+    `deployment=${shellQuote(deployment)}`,
+    "kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err",
+    "code=$?",
+    "printf 'strategyPatchExitCode\\t%s\\n' \"$code\"",
+    "printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-strategy-patch.err | cut -c1-1000 || true)\"",
+    "exit \"$code\"",
+  ].join("\n"), "", shortTimeout);
+  addResult(strategyPatch);
+  Object.assign(fields, keyValueLinesFromText(statusText(strategyPatch)));
+
+  if (strategyPatch.exitCode === 0) {
+    const restart = runTransScript(options.node, [
+      "set +e",
+      `namespace=${shellQuote(namespace)}`,
+      `deployment=${shellQuote(deployment)}`,
+      "kubectl -n \"$namespace\" rollout restart deploy/\"$deployment\" >/tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err",
+      "code=$?",
+      "printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"",
+      "printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-restart.err | cut -c1-1000 || true)\"",
+      "exit \"$code\"",
+    ].join("\n"), "", shortTimeout);
+    addResult(restart);
+    Object.assign(fields, keyValueLinesFromText(statusText(restart)));
+  }
+
+  if (fields.rolloutRestartExitCode === "0") {
+    const deadline = Date.now() + 55_000;
+    while (Date.now() <= deadline) {
+      const probe = runTransScript(options.node, [
+        "set +e",
+        `namespace=${shellQuote(namespace)}`,
+        `deployment=${shellQuote(deployment)}`,
+        "kubectl -n \"$namespace\" get deploy \"$deployment\" -o jsonpath='{.spec.replicas}{\"\\t\"}{.status.readyReplicas}{\"\\t\"}{.status.availableReplicas}' >/tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err",
+        "code=$?",
+        "values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)",
+        "desired=$(printf '%s' \"$values\" | cut -f1)",
+        "ready=$(printf '%s' \"$values\" | cut -f2)",
+        "available=$(printf '%s' \"$values\" | cut -f3)",
+        "printf 'desiredReplicas\\t%s\\n' \"$desired\"",
+        "printf 'readyReplicas\\t%s\\n' \"$ready\"",
+        "printf 'availableReplicas\\t%s\\n' \"$available\"",
+        "printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' </tmp/hwlab-public-frpc-ready.err | cut -c1-1000 || true)\"",
+        "[ \"$code\" = 0 ] && [ \"$desired\" = 1 ] && [ \"$ready\" = 1 ] && [ \"$available\" = 1 ]",
+      ].join("\n"), "", shortTimeout);
+      addResult(probe);
+      Object.assign(fields, keyValueLinesFromText(statusText(probe)));
+      if (probe.exitCode === 0) {
+        fields.rolloutStatusExitCode = "0";
+        break;
+      }
+      runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
+    }
+  }
+  if (fields.rolloutStatusExitCode === "") fields.rolloutStatusExitCode = "1";
+
+  const ok = fields.strategyPatchExitCode === "0"
+    && fields.rolloutRestartExitCode === "0"
+    && fields.rolloutStatusExitCode === "0";
+  const stdout = Object.entries(fields).map(([key, value]) => `${key}\t${value}`).join("\n") + "\n";
+  return {
+    command: [transPath(), `${options.node}:k3s`, "script", "--", "<public-exposure-frpc-recreate>"],
+    cwd: repoRoot,
+    exitCode: ok ? 0 : 1,
+    stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n",
+    stderr: stderrParts.join("\n"),
+    signal: null,
+    timedOut: false,
+  };
+}
+
+function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult {
+  return {
+    command: [...first.command, "&&", ...second.command],
+    cwd: repoRoot,
+    exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode,
+    stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n",
+    stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"),
+    signal: second.signal ?? first.signal,
+    timedOut: first.timedOut || second.timedOut,
+  };
+}
+
 function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string {
   const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64");
   const marker = `unidesk managed ${exposure.hostname}`;