From 08544ae1f2aeb42c9518ea6824ffa8acc7c71e43 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 13 Jun 2026 00:30:41 +0000 Subject: [PATCH] fix: make HWLAB public exposure restart observable --- scripts/src/hwlab-node.ts | 205 ++++++++++++++++++++++++++++++++++---- 1 file changed, 185 insertions(+), 20 deletions(-) diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 890dd679..b987cfdf 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -1909,6 +1909,9 @@ function nodeRuntimePipelinePostprocessScript(): string[] { " }", " if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {", " foundDeployment = true;", + " item.spec = item.spec || {};", + " const nextStrategy = { type: 'Recreate' };", + " if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }", " const podSpec = podSpecFor(item);", " for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {", " if (!isObject(container)) continue;", @@ -2074,6 +2077,8 @@ function nodeRuntimePipelinePostprocessScript(): string[] { " }", " const podSpec = podSpecFor(deployment);", " const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];", + " const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;", + " if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });", " const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));", " const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;", " const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;", @@ -3144,16 +3149,19 @@ function runNodePublicExposure(options: NodePublicExposureOptions): Record` }, }; } - const secretResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds); - const secretFields = keyValueLinesFromText(statusText(secretResult)); + const secretApplyResult = runTransScript(options.node, publicExposureSecretScript(options, exposure), source.value ?? "", options.timeoutSeconds); + let secretFields = keyValueLinesFromText(statusText(secretApplyResult)); + let secretResult = secretApplyResult; + if (!options.dryRun && secretApplyResult.exitCode === 0 && secretFields.afterSecretExists === "yes") { + const restartResult = runPublicExposureFrpcRecreate(options); + secretFields = { ...secretFields, ...keyValueLinesFromText(statusText(restartResult)) }; + secretResult = combinePublicExposureCommandResults(secretApplyResult, restartResult); + } const caddyResult = runTransHostScript(exposure.caddyRoute, publicExposureCaddyScript(options, exposure), "", options.timeoutSeconds); const caddyFields = keyValueLinesFromText(statusText(caddyResult)); - const secretOk = options.dryRun ? secretResult.exitCode === 0 : secretResult.exitCode === 0 && secretFields.afterSecretExists === "yes"; - const ok = secretOk - && caddyResult.exitCode === 0 - && caddyFields.afterBlockPresent === "yes" - && caddyFields.validateExitCode === "0" - && (options.dryRun || caddyFields.active === "active"); + const secretStatus = publicExposureSecretStatus(secretFields, secretResult); + const caddyStatus = publicExposureCaddyStatus(caddyFields, caddyResult); + const ok = secretStatus.ok === true && caddyStatus.ok === true; return { ok, command: "hwlab nodes control-plane public-exposure", @@ -3171,8 +3179,8 @@ function runNodePublicExposure(options: NodePublicExposureOptions): Record, result: CommandResult): Record { const dryRun = fields.dryRun === "true"; return { - ok: result.exitCode === 0 && (dryRun || fields.afterSecretExists === "yes"), + ok: result.exitCode === 0 && (dryRun || ( + fields.afterSecretExists === "yes" + && fields.strategyPatchExitCode === "0" + && fields.rolloutRestartExitCode === "0" + && fields.rolloutStatusExitCode === "0" + )), dryRun, mutation: fields.mutation === "true", namespace: fields.namespace || null, secret: fields.secret || null, + deployment: fields.deployment || null, beforeExists: fields.beforeSecretExists === "yes", afterExists: fields.afterSecretExists === "yes", tokenBytes: numericField(fields.tokenBytes), applyExitCode: numericField(fields.applyExitCode), + restartMode: fields.restartMode || null, + strategyPatchExitCode: numericField(fields.strategyPatchExitCode), rolloutRestartExitCode: numericField(fields.rolloutRestartExitCode), + scaleDownExitCode: numericField(fields.scaleDownExitCode), + scaleDownWaitExitCode: numericField(fields.scaleDownWaitExitCode), + scaleDownWaitPodCount: numericField(fields.scaleDownWaitPodCount), + scaleUpExitCode: numericField(fields.scaleUpExitCode), rolloutStatusExitCode: numericField(fields.rolloutStatusExitCode), + readyReplicas: numericField(fields.readyReplicas), + availableReplicas: numericField(fields.availableReplicas), + desiredReplicas: numericField(fields.desiredReplicas), + strategyPatchErrorPreview: fields.strategyPatchErrorPreview || null, + restartErrorPreview: fields.restartErrorPreview || null, + scaleDownErrorPreview: fields.scaleDownErrorPreview || null, + scaleUpErrorPreview: fields.scaleUpErrorPreview || null, + readyErrorPreview: fields.readyErrorPreview || null, exitCode: result.exitCode, stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), }; } function publicExposureCaddyStatus(fields: Record, result: CommandResult): Record { + const dryRun = fields.dryRun === "true"; return { - ok: result.exitCode === 0 && fields.afterBlockPresent === "yes" && fields.validateExitCode === "0", - dryRun: fields.dryRun === "true", + ok: result.exitCode === 0 + && fields.afterBlockPresent === "yes" + && fields.validateExitCode === "0" + && (dryRun || ( + fields.active === "active" + && fields.localWebStatus === "200" + && fields.localApiStatus === "200" + )), + dryRun, mutation: fields.mutation === "true", hostname: fields.hostname || null, configPath: fields.configPath || null, @@ -3251,10 +3287,12 @@ function publicExposureCaddyStatus(fields: Record, result: Comma } function publicExposureSecretScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string { + const deployment = `${options.spec.runtimeNamespace}-frpc`; return [ "set +e", `namespace=${shellQuote(options.spec.runtimeNamespace)}`, `secret=${shellQuote(exposure.secretName)}`, + `deployment=${shellQuote(deployment)}`, `token_key=${shellQuote(exposure.tokenKey)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, "token=$(cat)", @@ -3262,8 +3300,17 @@ function publicExposureSecretScript(options: NodePublicExposureOptions, exposure "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && before_secret_exists=yes", "token_bytes=$(printf '%s' \"$token\" | wc -c | tr -d ' ')", "apply_exit=", + "restart_mode=recreate-rollout-restart", + "strategy_patch_exit=", "rollout_restart_exit=", + "scale_down_exit=", + "scale_down_wait_exit=", + "scale_down_wait_pod_count=", + "scale_up_exit=", "rollout_status_exit=", + "ready_replicas=", + "available_replicas=", + "desired_replicas=", "mutation=false", "if [ \"$dry_run\" = false ]; then", " tmp=$(mktemp /tmp/hwlab-public-frpc-secret.XXXXXX.yaml)", @@ -3285,30 +3332,148 @@ function publicExposureSecretScript(options: NodePublicExposureOptions, exposure " apply_exit=$?", " rm -f \"$tmp\"", " if [ \"$apply_exit\" = 0 ]; then mutation=true; fi", - " kubectl -n \"$namespace\" rollout restart deploy/hwlab-v03-frpc >/tmp/hwlab-public-frpc-rollout-restart.out 2>/tmp/hwlab-public-frpc-rollout-restart.err", - " rollout_restart_exit=$?", - " if [ \"$rollout_restart_exit\" = 0 ]; then", - " kubectl -n \"$namespace\" rollout status deploy/hwlab-v03-frpc --timeout=60s >/tmp/hwlab-public-frpc-rollout-status.out 2>/tmp/hwlab-public-frpc-rollout-status.err", - " rollout_status_exit=$?", - " fi", "fi", "after_secret_exists=no", "kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && after_secret_exists=yes", "printf 'namespace\\t%s\\n' \"$namespace\"", "printf 'secret\\t%s\\n' \"$secret\"", + "printf 'deployment\\t%s\\n' \"$deployment\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", "printf 'beforeSecretExists\\t%s\\n' \"$before_secret_exists\"", "printf 'afterSecretExists\\t%s\\n' \"$after_secret_exists\"", "printf 'tokenBytes\\t%s\\n' \"$token_bytes\"", "printf 'applyExitCode\\t%s\\n' \"$apply_exit\"", + "printf 'restartMode\\t%s\\n' \"$restart_mode\"", + "printf 'strategyPatchExitCode\\t%s\\n' \"$strategy_patch_exit\"", "printf 'rolloutRestartExitCode\\t%s\\n' \"$rollout_restart_exit\"", + "printf 'scaleDownExitCode\\t%s\\n' \"$scale_down_exit\"", + "printf 'scaleDownWaitExitCode\\t%s\\n' \"$scale_down_wait_exit\"", + "printf 'scaleDownWaitPodCount\\t%s\\n' \"$scale_down_wait_pod_count\"", + "printf 'scaleUpExitCode\\t%s\\n' \"$scale_up_exit\"", "printf 'rolloutStatusExitCode\\t%s\\n' \"$rollout_status_exit\"", + "printf 'readyReplicas\\t%s\\n' \"$ready_replicas\"", + "printf 'availableReplicas\\t%s\\n' \"$available_replicas\"", + "printf 'desiredReplicas\\t%s\\n' \"$desired_replicas\"", "if [ \"$dry_run\" = true ]; then exit 0; fi", - "[ \"$after_secret_exists\" = yes ]", + "[ \"$after_secret_exists\" = yes ] && [ \"$apply_exit\" = 0 ]", ].join("\n"); } +function runPublicExposureFrpcRecreate(options: NodePublicExposureOptions): CommandResult { + const namespace = options.spec.runtimeNamespace; + const deployment = `${namespace}-frpc`; + const fields: Record = { + deployment, + restartMode: "recreate-rollout-restart", + strategyPatchExitCode: "", + rolloutRestartExitCode: "", + scaleDownExitCode: "", + scaleDownWaitExitCode: "", + scaleDownWaitPodCount: "", + scaleUpExitCode: "", + rolloutStatusExitCode: "", + readyReplicas: "", + availableReplicas: "", + desiredReplicas: "", + strategyPatchErrorPreview: "", + restartErrorPreview: "", + scaleDownErrorPreview: "", + scaleUpErrorPreview: "", + readyErrorPreview: "", + }; + const stdoutParts: string[] = []; + const stderrParts: string[] = []; + const addResult = (result: CommandResult): void => { + if (result.stdout.trim()) stdoutParts.push(result.stdout.trim()); + if (result.stderr.trim()) stderrParts.push(result.stderr.trim()); + }; + const shortTimeout = Math.min(Math.max(20, options.timeoutSeconds), 55); + const strategyPatch = runTransScript(options.node, [ + "set +e", + `namespace=${shellQuote(namespace)}`, + `deployment=${shellQuote(deployment)}`, + "kubectl -n \"$namespace\" patch deploy/\"$deployment\" --type=merge -p '{\"spec\":{\"strategy\":{\"type\":\"Recreate\",\"rollingUpdate\":null}}}' >/tmp/hwlab-public-frpc-strategy-patch.out 2>/tmp/hwlab-public-frpc-strategy-patch.err", + "code=$?", + "printf 'strategyPatchExitCode\\t%s\\n' \"$code\"", + "printf 'strategyPatchErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-strategy-patch.err ] && tr '\\n' ';' /tmp/hwlab-public-frpc-restart.out 2>/tmp/hwlab-public-frpc-restart.err", + "code=$?", + "printf 'rolloutRestartExitCode\\t%s\\n' \"$code\"", + "printf 'restartErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-restart.err ] && tr '\\n' ';' /tmp/hwlab-public-frpc-ready.out 2>/tmp/hwlab-public-frpc-ready.err", + "code=$?", + "values=$(cat /tmp/hwlab-public-frpc-ready.out 2>/dev/null || true)", + "desired=$(printf '%s' \"$values\" | cut -f1)", + "ready=$(printf '%s' \"$values\" | cut -f2)", + "available=$(printf '%s' \"$values\" | cut -f3)", + "printf 'desiredReplicas\\t%s\\n' \"$desired\"", + "printf 'readyReplicas\\t%s\\n' \"$ready\"", + "printf 'availableReplicas\\t%s\\n' \"$available\"", + "printf 'readyErrorPreview\\t%s\\n' \"$([ -f /tmp/hwlab-public-frpc-ready.err ] && tr '\\n' ';' `${key}\t${value}`).join("\n") + "\n"; + return { + command: [transPath(), `${options.node}:k3s`, "script", "--", ""], + cwd: repoRoot, + exitCode: ok ? 0 : 1, + stdout: [stdout.trim(), ...stdoutParts].filter(Boolean).join("\n") + "\n", + stderr: stderrParts.join("\n"), + signal: null, + timedOut: false, + }; +} + +function combinePublicExposureCommandResults(first: CommandResult, second: CommandResult): CommandResult { + return { + command: [...first.command, "&&", ...second.command], + cwd: repoRoot, + exitCode: first.exitCode === 0 && second.exitCode === 0 ? 0 : second.exitCode ?? first.exitCode, + stdout: [first.stdout.trim(), second.stdout.trim()].filter(Boolean).join("\n") + "\n", + stderr: [first.stderr.trim(), second.stderr.trim()].filter(Boolean).join("\n"), + signal: second.signal ?? first.signal, + timedOut: first.timedOut || second.timedOut, + }; +} + function publicExposureCaddyScript(options: NodePublicExposureOptions, exposure: HwlabRuntimePublicExposureSpec): string { const blockB64 = Buffer.from(publicExposureCaddyBlock(exposure), "utf8").toString("base64"); const marker = `unidesk managed ${exposure.hostname}`;