// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. cleanup module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:1886-2201 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { NodeRuntimeCleanupOptions, NodeRuntimeCleanupPipelineRunRow } from "./entry"; import { deleteNodeRuntimeCleanupRuns, nodeRuntimeApply, nodeRuntimeCiObjectCounts, nodeRuntimeCleanupOwnedPodsFromText, nodeRuntimeCleanupOwnedPvcsFromText, nodeRuntimeCleanupOwnedTaskRunsFromText, nodeRuntimeMigration, nodeRuntimeRefresh, nodeRuntimeSync } from "./control-actions"; import { HWLAB_CI_NAMESPACE } from "./entry"; import { nodeRuntimeTriggerCurrentOutput } from "./git-mirror"; import { parseNodeScopedDelegatedOptions } from "./plan"; import { compactRuntimeCommand, compactRuntimeCommandStats, nodeRuntimeUnsupportedAction, transPath } from "./runtime-common"; import { optionValue, positiveIntegerOption, shellQuote, statusText } from "./utils"; export function parseJsonObject(text: string): Record { try { const parsed = JSON.parse(text) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : {}; } catch { return {}; } } export function commandResultFromAsync(spec: HwlabRuntimeLaneSpec, payload: Record, statusPath: string, timedOut: boolean): CommandResult { return { command: [transPath(), spec.nodeRoute, "sh", "--", ""], cwd: repoRoot, exitCode: typeof payload.exitCode === "number" ? payload.exitCode : payload.ok === true ? 0 : 1, stdout: typeof payload.stdout === "string" ? payload.stdout : "", stderr: typeof payload.stderr === "string" ? payload.stderr : `remote async statusPath=${statusPath}`, signal: null, timedOut, }; } export function runNodeK3sArgs(spec: HwlabRuntimeLaneSpec, args: string[], timeoutSeconds: number): CommandResult { return runCommand([transPath(), spec.nodeKubeRoute, ...args], repoRoot, { timeoutMs: timeoutSeconds * 1000 }); } export function runNodeK3sScript(spec: HwlabRuntimeLaneSpec, script: string, timeoutSeconds: number, input = ""): CommandResult { return runCommand([transPath(), spec.nodeKubeRoute, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } export function isCommandSuccess(result: CommandResult): boolean { return result.exitCode === 0 && !result.timedOut; } export function shortSha(value: string): string { return value.slice(0, 12).toLowerCase(); } export function shortValue(value: unknown): string { if (typeof value !== "string" || value.length === 0) return "-"; if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value); return value.length > 30 ? `${value.slice(0, 27)}~` : value; } export function formatElapsedMs(value: unknown): string { const ms = Number(value); if (!Number.isFinite(ms) || ms < 0) return "-"; const seconds = Math.round(ms / 1000); const minutes = Math.floor(seconds / 60); const rest = seconds % 60; return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`; } export function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; } export function nodeRuntimeRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { return `${nodeRuntimePipelineRunName(spec, sourceCommit)}-r${Date.now().toString(36)}`.slice(0, 63); } export function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string { const suffix = `/${spec.runtimeRenderDir}`; if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length); return spec.gitopsRoot; } export function resolveNodeRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], repoRoot, { timeoutMs: 45_000 }); if (!isCommandSuccess(result)) return { sourceCommit: null, result }; const match = /[0-9a-f]{40}/iu.exec(statusText(result)); return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; } export function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { const gitRetries = Math.max(1, spec.downloadProfile.git.retries); const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds); return [ `cicd_repo=${shellQuote(spec.cicdRepo)}`, `cicd_url=${shellQuote(spec.gitUrl)}`, `cicd_branch=${shellQuote(spec.sourceBranch)}`, `cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`, `git_retries=${shellQuote(String(gitRetries))}`, `git_timeout=${shellQuote(String(gitTimeoutSeconds))}`, "run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }", "retry_git() {", " op=$1", " shift", " attempt=1", " while [ \"$attempt\" -le \"$git_retries\" ]; do", " echo \"phase=git-$op attempt=$attempt timeoutSeconds=$git_timeout\" >&2", " run_git \"$@\"", " code=$?", " if [ \"$code\" -eq 0 ]; then return 0; fi", " echo \"phase=git-$op attempt=$attempt exitCode=$code\" >&2", " attempt=$((attempt + 1))", " done", " return 1", "}", "mkdir -p \"$(dirname \"$cicd_repo\")\"", "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", " :", "elif [ -e \"$cicd_repo\" ]; then", " echo \"CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", " exit 41", "else", " retry_git clone-ci-cache clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", "fi", "git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"", "git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", "if ! retry_git fetch-ci-cache --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune; then", " rm -rf \"$cicd_repo\"", " retry_git clone-ci-cache-retry clone --bare --filter=blob:none --single-branch --branch \"$cicd_branch\" \"$cicd_url\" \"$cicd_repo\"", " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", " retry_git fetch-ci-cache-retry --git-dir=\"$cicd_repo\" fetch --filter=blob:none --depth=1 origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune", "fi", ].join("\n"); } export function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record | RenderedCliResult { if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped); if (scoped.action === "sync") return nodeRuntimeSync(scoped); if (scoped.action === "apply") return nodeRuntimeApply(scoped); if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped); if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped); if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped); return nodeRuntimeUnsupportedAction(scoped); } export function parseNodeRuntimeCleanupOptions(scoped: ReturnType): NodeRuntimeCleanupOptions { const sourceCommitRaw = optionValue(scoped.originalArgs, "--source-commit"); const pipelineRunRaw = optionValue(scoped.originalArgs, "--pipeline-run"); if (sourceCommitRaw !== undefined && pipelineRunRaw !== undefined) { throw new Error("control-plane cleanup-runs accepts only one of --source-commit or --pipeline-run"); } const sourceCommit = sourceCommitRaw?.toLowerCase(); if (sourceCommit !== undefined && !/^[0-9a-f]{40}$/u.test(sourceCommit)) { throw new Error("--source-commit must be a full 40-character git sha for cleanup-runs"); } const pipelineRun = pipelineRunRaw === undefined ? undefined : validateNodeRuntimePipelineRunName(scoped.spec, pipelineRunRaw); return { minAgeMinutes: positiveIntegerOption(scoped.originalArgs, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(scoped.originalArgs, "--limit", 20, 200), sourceCommit, pipelineRun, targetPipelineRun: pipelineRun ?? (sourceCommit === undefined ? undefined : nodeRuntimePipelineRunName(scoped.spec, sourceCommit)), includeActive: scoped.originalArgs.includes("--include-active"), dryRun: scoped.dryRun || !scoped.confirm, }; } export function validateNodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, value: string): string { const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}(?:-[a-z0-9][a-z0-9-]{0,24})?$`, "iu").test(value)) { throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}-[-rerun] PipelineRun name for --lane ${spec.lane}`); } return value.toLowerCase(); } export function nodeRuntimeCleanupRuns(scoped: ReturnType): Record { const options = parseNodeRuntimeCleanupOptions(scoped); const beforeCounts = nodeRuntimeCiObjectCounts(scoped.spec); const candidates = listNodeRuntimeCleanupPipelineRuns(scoped.spec, options); const selectedPipelineRuns = candidates .filter((item) => item.selected !== false) .map((item) => item.name); const ownedResources = listNodeRuntimeCleanupOwnedResources(scoped.spec, selectedPipelineRuns); const command = `hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane}`; if (options.dryRun) { return { ok: true, command, mode: "dry-run", node: scoped.node, lane: scoped.lane, namespace: HWLAB_CI_NAMESPACE, minAgeMinutes: options.minAgeMinutes, limit: options.limit, sourceCommit: options.sourceCommit, pipelineRun: options.pipelineRun, includeActive: options.includeActive, candidates, candidateCount: candidates.length, selectedPipelineRuns, selectedPipelineRunCount: selectedPipelineRuns.length, ownedResources, ciObjectCounts: beforeCounts, mutation: false, next: { confirm: [ command, `--min-age-minutes ${options.minAgeMinutes}`, `--limit ${options.limit}`, options.pipelineRun === undefined ? "" : `--pipeline-run ${options.pipelineRun}`, options.sourceCommit === undefined ? "" : `--source-commit ${options.sourceCommit}`, options.includeActive ? "--include-active" : "", "--confirm", "--wait", ].filter(Boolean).join(" "), }, }; } const deletion = deleteNodeRuntimeCleanupRuns(scoped.spec, selectedPipelineRuns, scoped.timeoutSeconds); const afterCounts = nodeRuntimeCiObjectCounts(scoped.spec); return { ok: isCommandSuccess(deletion), command, mode: "confirmed-cleanup", node: scoped.node, lane: scoped.lane, namespace: HWLAB_CI_NAMESPACE, minAgeMinutes: options.minAgeMinutes, limit: options.limit, sourceCommit: options.sourceCommit, pipelineRun: options.pipelineRun, includeActive: options.includeActive, deletedPipelineRuns: selectedPipelineRuns, deletedPipelineRunCount: selectedPipelineRuns.length, ownedResourcesBefore: ownedResources, ciObjectCountsBefore: beforeCounts, ciObjectCountsAfter: afterCounts, deletion: compactRuntimeCommand(deletion), mutation: isCommandSuccess(deletion), degradedReason: isCommandSuccess(deletion) ? undefined : "node-runtime-ci-cleanup-delete-failed", next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, rerunStatus: options.targetPipelineRun === undefined ? undefined : `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${options.targetPipelineRun}`, }, }; } export function listNodeRuntimeCleanupPipelineRuns(spec: HwlabRuntimeLaneSpec, options: NodeRuntimeCleanupOptions): NodeRuntimeCleanupPipelineRunRow[] { if (options.targetPipelineRun !== undefined) { const target = getNodeRuntimeCleanupPipelineRun(spec, options.targetPipelineRun); if (target === null) { return [{ name: options.targetPipelineRun, createdAt: null, ageMinutes: null, status: null, reason: "target-pipelinerun-not-found", selected: false, }]; } if (target.status !== "True" && target.status !== "False" && !options.includeActive) { return [{ ...target, selected: false, selectedReason: "target-pipelinerun-not-terminal" }]; } if ((target.status === "True" || target.status === "False") && (target.ageMinutes === null || target.ageMinutes < options.minAgeMinutes)) { return [{ ...target, selected: false, selectedReason: target.ageMinutes === null ? "missing-creation-timestamp" : "below-min-age" }]; } return [target]; } const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pipelinerun", "-o", 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}', ], 60); if (!isCommandSuccess(result)) throw new Error(`failed to list ${HWLAB_CI_NAMESPACE} PipelineRuns: ${result.stderr.trim().slice(0, 1000)}`); const rows = nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout); const prefix = `${spec.pipelineRunPrefix}-`; const terminalRuns = rows .filter((item) => item.name.startsWith(prefix)) .filter((item) => item.status === "True" || item.status === "False") .sort((left, right) => String(left.createdAt ?? "").localeCompare(String(right.createdAt ?? ""))); const protectedLatest = terminalRuns .slice() .sort((left, right) => String(right.createdAt ?? "").localeCompare(String(left.createdAt ?? "")))[0]?.name ?? null; return terminalRuns .filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes) .map((item) => item.name === protectedLatest ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" } : item) .slice(0, options.limit); } export function getNodeRuntimeCleanupPipelineRun(spec: HwlabRuntimeLaneSpec, pipelineRun: string): NodeRuntimeCleanupPipelineRunRow | null { const result = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pipelinerun", pipelineRun, "-o", 'jsonpath={.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}', ], 60); if (result.exitCode !== 0) return null; return nodeRuntimeCleanupPipelineRunRowsFromText(result.stdout)[0] ?? null; } export function nodeRuntimeCleanupPipelineRunRowsFromText(text: string): NodeRuntimeCleanupPipelineRunRow[] { const now = Date.now(); return text.split(/\r?\n/u).map((line) => { const [name = "", createdAtRaw = "", statusRaw = "", reasonRaw = ""] = line.trim().split("\t"); const createdAt = createdAtRaw.length > 0 ? createdAtRaw : null; const createdMs = createdAt === null ? NaN : Date.parse(createdAt); return { name, createdAt, ageMinutes: Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null, status: statusRaw || null, reason: reasonRaw || null, }; }).filter((item) => item.name.length > 0); } export function listNodeRuntimeCleanupOwnedResources(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[]): Record { const previewLimit = 24; if (pipelineRunNames.length === 0) { return { taskRunPreview: [], podPreview: [], pvcPreview: [], previewLimit, truncated: false, taskRunCount: 0, podCount: 0, pvcCount: 0, }; } const wanted = new Set(pipelineRunNames); const taskRunsResult = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "Succeeded"}}{{.status}}{{"\\t"}}{{.reason}}{{end}}{{end}}{{"\\n"}}{{end}}', ], 60); const podsResult = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\n"}}{{end}}', ], 60); const pvcsResult = runNodeK3sArgs(spec, [ "kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pvc", "-o", 'go-template={{range .items}}{{.metadata.name}}{{"\\t"}}{{.spec.volumeName}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{range .metadata.ownerReferences}}{{if eq .kind "PipelineRun"}}{{.kind}}{{"\\t"}}{{.name}}{{end}}{{end}}{{"\\t"}}{{.spec.resources.requests.storage}}{{"\\n"}}{{end}}', ], 60); const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimeCleanupOwnedTaskRunsFromText(taskRunsResult.stdout, wanted) : []; const pods = isCommandSuccess(podsResult) ? nodeRuntimeCleanupOwnedPodsFromText(podsResult.stdout, wanted) : []; const pvcs = isCommandSuccess(pvcsResult) ? nodeRuntimeCleanupOwnedPvcsFromText(pvcsResult.stdout, wanted) : []; return { taskRunPreview: taskRuns.slice(0, previewLimit), podPreview: pods.slice(0, previewLimit), pvcPreview: pvcs.slice(0, previewLimit), previewLimit, truncated: taskRuns.length > previewLimit || pods.length > previewLimit || pvcs.length > previewLimit, taskRunCount: taskRuns.length, podCount: pods.length, pvcCount: pvcs.length, query: { taskRuns: compactRuntimeCommandStats(taskRunsResult), pods: compactRuntimeCommandStats(podsResult), pvcs: compactRuntimeCommandStats(pvcsResult), }, }; }