From 1b3af54fba31fd420c653b3801cb929657a28505 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Tue, 23 Jun 2026 09:27:28 +0800 Subject: [PATCH] fix(hwlab): clean stale statefulset pods during node sync (#727) Co-authored-by: Codex --- .agents/skills/unidesk-cicd/SKILL.md | 3 ++ scripts/src/hwlab-node-help.ts | 1 + scripts/src/hwlab-node-impl.ts | 43 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index d26c12d8..ef660d40 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -93,12 +93,15 @@ bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lan bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run |--source-commit ] [--full|--raw] bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait +bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm ``` 从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。 `hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。 +`hwlab nodes control-plane sync --confirm` 是 Argo runtime 收敛修复入口：会终止卡住的 running Argo operation、删除失败 hook Job，并在 StatefulSet template 已更新但旧 controller-revision pod 因 `ImagePullBackOff` / `ErrImagePull` / `CrashLoopBackOff` 卡住时受控删除该旧 pod，让 StatefulSet 按最新 revision 重建。不要手工裸删 pod；需要解除这类死锁时走该入口。 + `hwlab nodes control-plane trigger-current --node --lane --confirm --wait` 是 node/lane CI/CD 一键入口：按 YAML 解析 source head，执行 git-mirror pre-sync/pre-flush，刷新 control-plane，创建或复用 commit-pinned PipelineRun，等待 PipelineRun 终态，并在终态成功后执行 post-flush。默认输出必须是低噪声 CICD 表格摘要；完整 JSON 只能通过 `--full` 或 `--raw` 展开。`--wait` 默认最多等待 120 秒；超过 120 秒时 CLI 返回 `pending` warning，并直接给出 env-reuse 检查命令 `control-plane status --full` 和 git-mirror 检查命令 `git-mirror status`，不继续长时间阻塞，也不把仍在运行误报为构建失败。 ### G14 v0.3 runtime base image diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 1ae872a2..8b7282ab 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -60,6 +60,7 @@ export function hwlabNodeHelp(): Record { notes: [ "`trigger-current` skips an already succeeded PipelineRun for the same HWLAB source commit by default.", "`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.", + "`control-plane sync --confirm` terminates a stale running Argo operation, deletes failed Argo hook Jobs, and recreates stale non-ready StatefulSet pods that are still pinned to an old controller revision with pull/backoff errors.", "`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.", "Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not." ], diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 55ce7f0a..89667148 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -2614,6 +2614,39 @@ function nodeRuntimeSync(scoped: ReturnType/dev/null || true); do", + " sts_name=${sts#statefulset.apps/}", + " current_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.currentRevision}' 2>/dev/null || true)", + " update_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.updateRevision}' 2>/dev/null || true)", + " if [ -z \"$current_revision\" ] || [ -z \"$update_revision\" ] || [ \"$current_revision\" = \"$update_revision\" ]; then continue; fi", + " for pod in $(kubectl -n \"$runtime_namespace\" get pod -l \"controller-revision-hash=$current_revision\" -o name 2>/dev/null || true); do", + " pod_name=${pod#pod/}", + " owner=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' 2>/dev/null || true)", + " if [ \"$owner\" != \"StatefulSet/$sts_name\" ]; then continue; fi", + " ready=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null || true)", + " waiting=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={range .status.containerStatuses[*]}{.state.waiting.reason}{\" \"}{end}' 2>/dev/null || true)", + " case \"$waiting\" in *ImagePullBackOff*|*ErrImagePull*|*CrashLoopBackOff*) ;; *) continue ;; esac", + " if [ \"$ready\" = True ]; then continue; fi", + " stale_statefulset_pod_count=$((stale_statefulset_pod_count + 1))", + " stale_item=\"$sts_name/$pod_name:$waiting\"", + " if [ -z \"$stale_statefulset_pods\" ]; then stale_statefulset_pods=$stale_item; else stale_statefulset_pods=$stale_statefulset_pods,$stale_item; fi", + " if [ \"$dry_run\" = false ]; then", + " delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$pod\" --wait=false 2>&1)", + " delete_code=$?", + " if [ \"$delete_code\" -eq 0 ]; then", + " deleted_stale_statefulset_pod_count=$((deleted_stale_statefulset_pod_count + 1))", + " if [ -z \"$deleted_stale_statefulset_pods\" ]; then deleted_stale_statefulset_pods=$pod_name; else deleted_stale_statefulset_pods=$deleted_stale_statefulset_pods,$pod_name; fi", + " else", + " if [ -z \"$stale_statefulset_pod_delete_errors\" ]; then stale_statefulset_pod_delete_errors=\"$pod_name:$delete_code\"; else stale_statefulset_pod_delete_errors=\"$stale_statefulset_pod_delete_errors,$pod_name:$delete_code\"; fi", + " fi", + " fi", + " done", + "done", "if [ \"$dry_run\" = true ]; then", " output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)", "else", @@ -2634,6 +2667,11 @@ function nodeRuntimeSync(scoped: ReturnType