// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. utils module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:14165-16000 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { RuntimeSecretSpec } from "./entry"; export function masterAdminApiKeyEnvPath(spec: RuntimeSecretSpec): string { return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`; } export function readMasterAdminApiKey(spec: RuntimeSecretSpec): { key: string; source: string } { const source = masterAdminApiKeyEnvPath(spec); if (!existsSync(source)) throw new Error(`HWLAB_API_KEY source missing: ${source}`); const content = readFileSync(source, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/m); const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, ""); if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${source}`); return { key: raw, source }; } export function optionValue(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index === -1) return undefined; const value = args[index + 1]; if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); return value; } export function requiredOption(args: string[], name: string): string { const value = optionValue(args, name); if (value === undefined) throw new Error(`${name} is required`); return value; } export function stripOption(args: string[], name: string): string[] { return stripOptions(args, [name]); } export function stripOptions(args: string[], names: readonly string[]): string[] { const remove = new Set(names); const without: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; if (remove.has(arg)) { if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1; continue; } without.push(arg); } return without; } export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const raw = optionValue(args, name); if (raw === undefined) return defaultValue; const value = Number(raw); if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`); return Math.min(value, maxValue); } export function assertLane(value: string): void { if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`); } export function assertNodeId(value: string): void { if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`); } export function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } export function nullableRecord(value: unknown): Record | null { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; } export function stringValue(value: unknown, path: string): string { if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`); return value; } export function optionalStringValue(value: unknown, path: string): string | null { if (value === undefined || value === null) return null; if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string when set`); return value; } export function positiveIntegerValue(value: unknown, path: string): number { if (!Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`); return value; } export function parseJsonObject(text: string): Record { const trimmed = text.trim(); if (trimmed.length === 0) return {}; try { return record(JSON.parse(trimmed) as unknown); } catch { const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start >= 0 && end > start) { try { return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown); } catch {} } } return {}; } export function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } export function statusText(result: CommandResult): string { return result.stdout || result.stderr; } export function keyValueLinesFromText(text: string): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const index = line.indexOf("\t"); if (index <= 0) continue; fields[line.slice(0, index)] = line.slice(index + 1); } return fields; } export function numericField(value: string | undefined): number | null { if (value === undefined || value === "") return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } export function commaListField(value: string | undefined): string[] { if (!value) return []; return value.split(",").map((item) => item.trim()).filter(Boolean); } export function splitWhitespaceField(value: string | undefined): string[] { if (!value) return []; return value.split(/\s+/u).filter(Boolean); } export function compactCommandResult(result: CommandResult): Record { return { command: compactCommand(result.command), exitCode: result.exitCode, stdoutBytes: result.stdout.length, stdoutTail: result.exitCode === 0 && !result.timedOut ? "" : result.stdout.trim().slice(-2000), stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000), timedOut: result.timedOut, }; } export function compactCommandResultWithStdoutTail(result: CommandResult): Record { const compact = compactCommandResult(result); compact.stdoutTail = result.stdout.trim().slice(-2000); return compact; } export function compactCommandResultRedacted(result: CommandResult, secrets: string[]): Record { const compact = compactCommandResult(result); if (typeof compact.stderr === "string" && compact.stderr.length > 0) { compact.stderr = redactKnownSecrets(compact.stderr, secrets); } return compact; } export function redactKnownSecrets(text: string, secrets: string[]): string { let next = text; for (const secret of secrets.filter((item) => item.length > 0)) { next = next.split(secret).join(""); } return next; } export function parseJsonObject(text: string): Record | null { const trimmed = text.trim(); if (trimmed.length === 0) return null; try { const parsed = JSON.parse(trimmed) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; } catch { const objectText = firstJsonObjectText(trimmed); if (objectText) { try { const parsed = JSON.parse(objectText) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; } catch {} } return null; } } export function firstJsonObjectText(text: string): string | null { const start = text.indexOf("{"); if (start < 0) return null; let depth = 0; let inString = false; let escaped = false; for (let index = start; index < text.length; index += 1) { const char = text[index]; if (inString) { if (escaped) { escaped = false; } else if (char === "\\") { escaped = true; } else if (char === "\"") { inString = false; } continue; } if (char === "\"") { inString = true; continue; } if (char === "{") depth += 1; else if (char === "}") { depth -= 1; if (depth === 0) return text.slice(start, index + 1); } } return null; } export function compactCommand(command: string[]): string[] { const scriptIndex = command.indexOf("--"); if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "