mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hwlab): clean stale statefulset pods during node sync (#727)

Browse Source 
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
Lyon
2026-06-23 09:27:28 +08:00
committed by GitHub
parent c987df66f1
commit 1b3af54fba
3 changed files with 47 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+43
View File
@@ -2614,6 +2614,39 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
" fi",
" fi",
"done",
"stale_statefulset_pod_count=0",
"deleted_stale_statefulset_pod_count=0",
"stale_statefulset_pods=",
"deleted_stale_statefulset_pods=",
"stale_statefulset_pod_delete_errors=",
"for sts in $(kubectl -n \"$runtime_namespace\" get statefulset -o name 2>/dev/null || true); do",
" sts_name=${sts#statefulset.apps/}",
" current_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.currentRevision}' 2>/dev/null || true)",
" update_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.updateRevision}' 2>/dev/null || true)",
" if [ -z \"$current_revision\" ] || [ -z \"$update_revision\" ] || [ \"$current_revision\" = \"$update_revision\" ]; then continue; fi",
" for pod in $(kubectl -n \"$runtime_namespace\" get pod -l \"controller-revision-hash=$current_revision\" -o name 2>/dev/null || true); do",
" pod_name=${pod#pod/}",
" owner=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' 2>/dev/null || true)",
" if [ \"$owner\" != \"StatefulSet/$sts_name\" ]; then continue; fi",
" ready=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null || true)",
" waiting=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={range .status.containerStatuses[*]}{.state.waiting.reason}{\" \"}{end}' 2>/dev/null || true)",
" case \"$waiting\" in *ImagePullBackOff*|*ErrImagePull*|*CrashLoopBackOff*) ;; *) continue ;; esac",
" if [ \"$ready\" = True ]; then continue; fi",
" stale_statefulset_pod_count=$((stale_statefulset_pod_count + 1))",
" stale_item=\"$sts_name/$pod_name:$waiting\"",
" if [ -z \"$stale_statefulset_pods\" ]; then stale_statefulset_pods=$stale_item; else stale_statefulset_pods=$stale_statefulset_pods,$stale_item; fi",
" if [ \"$dry_run\" = false ]; then",
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$pod\" --wait=false 2>&1)",
" delete_code=$?",
" if [ \"$delete_code\" -eq 0 ]; then",
" deleted_stale_statefulset_pod_count=$((deleted_stale_statefulset_pod_count + 1))",
" if [ -z \"$deleted_stale_statefulset_pods\" ]; then deleted_stale_statefulset_pods=$pod_name; else deleted_stale_statefulset_pods=$deleted_stale_statefulset_pods,$pod_name; fi",
" else",
" if [ -z \"$stale_statefulset_pod_delete_errors\" ]; then stale_statefulset_pod_delete_errors=\"$pod_name:$delete_code\"; else stale_statefulset_pod_delete_errors=\"$stale_statefulset_pod_delete_errors,$pod_name:$delete_code\"; fi",
" fi",
" fi",
" done",
"done",
"if [ \"$dry_run\" = true ]; then",
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)",
"else",
@@ -2634,6 +2667,11 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
"printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"",
"printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"",
"printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"",
"printf 'staleStatefulSetPodCount\\t%s\\n' \"$stale_statefulset_pod_count\"",
"printf 'staleStatefulSetPods\\t%s\\n' \"$stale_statefulset_pods\"",
"printf 'deletedStaleStatefulSetPodCount\\t%s\\n' \"$deleted_stale_statefulset_pod_count\"",
"printf 'deletedStaleStatefulSetPods\\t%s\\n' \"$deleted_stale_statefulset_pods\"",
"printf 'staleStatefulSetPodDeleteErrors\\t%s\\n' \"$stale_statefulset_pod_delete_errors\"",
"printf 'patchExitCode\\t%s\\n' \"$code\"",
"printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"rm -f \"$patch_file\" \"$terminate_patch_file\"",
@@ -2666,6 +2704,11 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
deletedHookCount: numericField(fields.deletedHookCount),
deletedHooks: commaListField(fields.deletedHooks),
hookDeleteErrors: fields.hookDeleteErrors || null,
staleStatefulSetPodCount: numericField(fields.staleStatefulSetPodCount),
staleStatefulSetPods: commaListField(fields.staleStatefulSetPods),
deletedStaleStatefulSetPodCount: numericField(fields.deletedStaleStatefulSetPodCount),
deletedStaleStatefulSetPods: commaListField(fields.deletedStaleStatefulSetPods),
staleStatefulSetPodDeleteErrors: fields.staleStatefulSetPodDeleteErrors || null,
},
patchExitCode: numericField(fields.patchExitCode),
patchOutput: fields.patchOutput || null,