fix(hwlab): clean stale statefulset pods during node sync (#727)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
@@ -93,12 +93,15 @@ bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lan
|
||||
bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm
|
||||
bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run <name>|--source-commit <sha>] [--full|--raw]
|
||||
bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait
|
||||
bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm
|
||||
```
|
||||
|
||||
从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。
|
||||
|
||||
`hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。
|
||||
|
||||
`hwlab nodes control-plane sync --confirm` 是 Argo runtime 收敛修复入口：会终止卡住的 running Argo operation、删除失败 hook Job，并在 StatefulSet template 已更新但旧 controller-revision pod 因 `ImagePullBackOff` / `ErrImagePull` / `CrashLoopBackOff` 卡住时受控删除该旧 pod，让 StatefulSet 按最新 revision 重建。不要手工裸删 pod；需要解除这类死锁时走该入口。
|
||||
|
||||
`hwlab nodes control-plane trigger-current --node <node> --lane <lane> --confirm --wait` 是 node/lane CI/CD 一键入口：按 YAML 解析 source head，执行 git-mirror pre-sync/pre-flush，刷新 control-plane，创建或复用 commit-pinned PipelineRun，等待 PipelineRun 终态，并在终态成功后执行 post-flush。默认输出必须是低噪声 CICD 表格摘要；完整 JSON 只能通过 `--full` 或 `--raw` 展开。`--wait` 默认最多等待 120 秒；超过 120 秒时 CLI 返回 `pending` warning，并直接给出 env-reuse 检查命令 `control-plane status --full` 和 git-mirror 检查命令 `git-mirror status`，不继续长时间阻塞，也不把仍在运行误报为构建失败。
|
||||
|
||||
### G14 v0.3 runtime base image
|
||||
|
||||
@@ -60,6 +60,7 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
notes: [
|
||||
"`trigger-current` skips an already succeeded PipelineRun for the same HWLAB source commit by default.",
|
||||
"`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.",
|
||||
"`control-plane sync --confirm` terminates a stale running Argo operation, deletes failed Argo hook Jobs, and recreates stale non-ready StatefulSet pods that are still pinned to an old controller revision with pull/backoff errors.",
|
||||
"`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.",
|
||||
"Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not."
|
||||
],
|
||||
|
||||
@@ -2614,6 +2614,39 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
|
||||
" fi",
|
||||
" fi",
|
||||
"done",
|
||||
"stale_statefulset_pod_count=0",
|
||||
"deleted_stale_statefulset_pod_count=0",
|
||||
"stale_statefulset_pods=",
|
||||
"deleted_stale_statefulset_pods=",
|
||||
"stale_statefulset_pod_delete_errors=",
|
||||
"for sts in $(kubectl -n \"$runtime_namespace\" get statefulset -o name 2>/dev/null || true); do",
|
||||
" sts_name=${sts#statefulset.apps/}",
|
||||
" current_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.currentRevision}' 2>/dev/null || true)",
|
||||
" update_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.updateRevision}' 2>/dev/null || true)",
|
||||
" if [ -z \"$current_revision\" ] || [ -z \"$update_revision\" ] || [ \"$current_revision\" = \"$update_revision\" ]; then continue; fi",
|
||||
" for pod in $(kubectl -n \"$runtime_namespace\" get pod -l \"controller-revision-hash=$current_revision\" -o name 2>/dev/null || true); do",
|
||||
" pod_name=${pod#pod/}",
|
||||
" owner=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' 2>/dev/null || true)",
|
||||
" if [ \"$owner\" != \"StatefulSet/$sts_name\" ]; then continue; fi",
|
||||
" ready=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null || true)",
|
||||
" waiting=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={range .status.containerStatuses[*]}{.state.waiting.reason}{\" \"}{end}' 2>/dev/null || true)",
|
||||
" case \"$waiting\" in *ImagePullBackOff*|*ErrImagePull*|*CrashLoopBackOff*) ;; *) continue ;; esac",
|
||||
" if [ \"$ready\" = True ]; then continue; fi",
|
||||
" stale_statefulset_pod_count=$((stale_statefulset_pod_count + 1))",
|
||||
" stale_item=\"$sts_name/$pod_name:$waiting\"",
|
||||
" if [ -z \"$stale_statefulset_pods\" ]; then stale_statefulset_pods=$stale_item; else stale_statefulset_pods=$stale_statefulset_pods,$stale_item; fi",
|
||||
" if [ \"$dry_run\" = false ]; then",
|
||||
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$pod\" --wait=false 2>&1)",
|
||||
" delete_code=$?",
|
||||
" if [ \"$delete_code\" -eq 0 ]; then",
|
||||
" deleted_stale_statefulset_pod_count=$((deleted_stale_statefulset_pod_count + 1))",
|
||||
" if [ -z \"$deleted_stale_statefulset_pods\" ]; then deleted_stale_statefulset_pods=$pod_name; else deleted_stale_statefulset_pods=$deleted_stale_statefulset_pods,$pod_name; fi",
|
||||
" else",
|
||||
" if [ -z \"$stale_statefulset_pod_delete_errors\" ]; then stale_statefulset_pod_delete_errors=\"$pod_name:$delete_code\"; else stale_statefulset_pod_delete_errors=\"$stale_statefulset_pod_delete_errors,$pod_name:$delete_code\"; fi",
|
||||
" fi",
|
||||
" fi",
|
||||
" done",
|
||||
"done",
|
||||
"if [ \"$dry_run\" = true ]; then",
|
||||
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)",
|
||||
"else",
|
||||
@@ -2634,6 +2667,11 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
|
||||
"printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"",
|
||||
"printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"",
|
||||
"printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"",
|
||||
"printf 'staleStatefulSetPodCount\\t%s\\n' \"$stale_statefulset_pod_count\"",
|
||||
"printf 'staleStatefulSetPods\\t%s\\n' \"$stale_statefulset_pods\"",
|
||||
"printf 'deletedStaleStatefulSetPodCount\\t%s\\n' \"$deleted_stale_statefulset_pod_count\"",
|
||||
"printf 'deletedStaleStatefulSetPods\\t%s\\n' \"$deleted_stale_statefulset_pods\"",
|
||||
"printf 'staleStatefulSetPodDeleteErrors\\t%s\\n' \"$stale_statefulset_pod_delete_errors\"",
|
||||
"printf 'patchExitCode\\t%s\\n' \"$code\"",
|
||||
"printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
|
||||
"rm -f \"$patch_file\" \"$terminate_patch_file\"",
|
||||
@@ -2666,6 +2704,11 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
|
||||
deletedHookCount: numericField(fields.deletedHookCount),
|
||||
deletedHooks: commaListField(fields.deletedHooks),
|
||||
hookDeleteErrors: fields.hookDeleteErrors || null,
|
||||
staleStatefulSetPodCount: numericField(fields.staleStatefulSetPodCount),
|
||||
staleStatefulSetPods: commaListField(fields.staleStatefulSetPods),
|
||||
deletedStaleStatefulSetPodCount: numericField(fields.deletedStaleStatefulSetPodCount),
|
||||
deletedStaleStatefulSetPods: commaListField(fields.deletedStaleStatefulSetPods),
|
||||
staleStatefulSetPodDeleteErrors: fields.staleStatefulSetPodDeleteErrors || null,
|
||||
},
|
||||
patchExitCode: numericField(fields.patchExitCode),
|
||||
patchOutput: fields.patchOutput || null,
|
||||
|
||||
Reference in New Issue
Block a user