Codex
257 lines
11 KiB
TypeScript
257 lines
11 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. utils module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:14165-16000 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { RuntimeSecretSpec } from "./entry";
|
|
|
|
export function masterAdminApiKeyEnvPath(spec: RuntimeSecretSpec): string {
|
|
return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`;
|
|
}
|
|
|
|
export function readMasterAdminApiKey(spec: RuntimeSecretSpec): { key: string; source: string } {
|
|
const source = masterAdminApiKeyEnvPath(spec);
|
|
if (!existsSync(source)) throw new Error(`HWLAB_API_KEY source missing: ${source}`);
|
|
const content = readFileSync(source, "utf8");
|
|
const match = content.match(/^HWLAB_API_KEY=(.+)$/m);
|
|
const raw = (match?.[1] ?? "").trim().replace(/^['"]|['"]$/g, "");
|
|
if (!raw.startsWith("hwl_live_")) throw new Error(`HWLAB_API_KEY source invalid: ${source}`);
|
|
return { key: raw, source };
|
|
}
|
|
|
|
export function optionValue(args: string[], name: string): string | undefined {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return undefined;
|
|
const value = args[index + 1];
|
|
if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
export function requiredOption(args: string[], name: string): string {
|
|
const value = optionValue(args, name);
|
|
if (value === undefined) throw new Error(`${name} is required`);
|
|
return value;
|
|
}
|
|
|
|
export function stripOption(args: string[], name: string): string[] {
|
|
return stripOptions(args, [name]);
|
|
}
|
|
|
|
export function stripOptions(args: string[], names: readonly string[]): string[] {
|
|
const remove = new Set(names);
|
|
const without: string[] = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (remove.has(arg)) {
|
|
if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1;
|
|
continue;
|
|
}
|
|
without.push(arg);
|
|
}
|
|
return without;
|
|
}
|
|
|
|
export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const raw = optionValue(args, name);
|
|
if (raw === undefined) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
export function assertLane(value: string): void {
|
|
if (!/^v[0-9]{2,}$/u.test(value)) throw new Error(`--lane must look like v03/v04, got ${value}`);
|
|
}
|
|
|
|
export function assertNodeId(value: string): void {
|
|
if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`);
|
|
}
|
|
|
|
export function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
export function nullableRecord(value: unknown): Record<string, unknown> | null {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
|
}
|
|
|
|
export function stringValue(value: unknown, path: string): string {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
export function optionalStringValue(value: unknown, path: string): string | null {
|
|
if (value === undefined || value === null) return null;
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string when set`);
|
|
return value;
|
|
}
|
|
|
|
export function positiveIntegerValue(value: unknown, path: string): number {
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error(`${path} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
export function parseJsonObject(text: string): Record<string, unknown> {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return {};
|
|
try {
|
|
return record(JSON.parse(trimmed) as unknown);
|
|
} catch {
|
|
const start = trimmed.indexOf("{");
|
|
const end = trimmed.lastIndexOf("}");
|
|
if (start >= 0 && end > start) {
|
|
try {
|
|
return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown);
|
|
} catch {}
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
export function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
|
}
|
|
|
|
export function statusText(result: CommandResult): string {
|
|
return result.stdout || result.stderr;
|
|
}
|
|
|
|
export function keyValueLinesFromText(text: string): Record<string, string> {
|
|
const fields: Record<string, string> = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const index = line.indexOf("\t");
|
|
if (index <= 0) continue;
|
|
fields[line.slice(0, index)] = line.slice(index + 1);
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
export function numericField(value: string | undefined): number | null {
|
|
if (value === undefined || value === "") return null;
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
export function commaListField(value: string | undefined): string[] {
|
|
if (!value) return [];
|
|
return value.split(",").map((item) => item.trim()).filter(Boolean);
|
|
}
|
|
|
|
export function splitWhitespaceField(value: string | undefined): string[] {
|
|
if (!value) return [];
|
|
return value.split(/\s+/u).filter(Boolean);
|
|
}
|
|
|
|
export function compactCommandResult(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
command: compactCommand(result.command),
|
|
exitCode: result.exitCode,
|
|
stdoutBytes: result.stdout.length,
|
|
stdoutTail: result.exitCode === 0 && !result.timedOut ? "" : result.stdout.trim().slice(-2000),
|
|
stderr: result.exitCode === 0 ? "" : result.stderr.trim().slice(0, 2000),
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
export function compactCommandResultWithStdoutTail(result: CommandResult): Record<string, unknown> {
|
|
const compact = compactCommandResult(result);
|
|
compact.stdoutTail = result.stdout.trim().slice(-2000);
|
|
return compact;
|
|
}
|
|
|
|
export function compactCommandResultRedacted(result: CommandResult, secrets: string[]): Record<string, unknown> {
|
|
const compact = compactCommandResult(result);
|
|
if (typeof compact.stderr === "string" && compact.stderr.length > 0) {
|
|
compact.stderr = redactKnownSecrets(compact.stderr, secrets);
|
|
}
|
|
return compact;
|
|
}
|
|
|
|
export function redactKnownSecrets(text: string, secrets: string[]): string {
|
|
let next = text;
|
|
for (const secret of secrets.filter((item) => item.length > 0)) {
|
|
next = next.split(secret).join("<redacted>");
|
|
}
|
|
return next;
|
|
}
|
|
|
|
export function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {
|
|
const objectText = firstJsonObjectText(trimmed);
|
|
if (objectText) {
|
|
try {
|
|
const parsed = JSON.parse(objectText) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function firstJsonObjectText(text: string): string | null {
|
|
const start = text.indexOf("{");
|
|
if (start < 0) return null;
|
|
let depth = 0;
|
|
let inString = false;
|
|
let escaped = false;
|
|
for (let index = start; index < text.length; index += 1) {
|
|
const char = text[index];
|
|
if (inString) {
|
|
if (escaped) {
|
|
escaped = false;
|
|
} else if (char === "\\") {
|
|
escaped = true;
|
|
} else if (char === "\"") {
|
|
inString = false;
|
|
}
|
|
continue;
|
|
}
|
|
if (char === "\"") {
|
|
inString = true;
|
|
continue;
|
|
}
|
|
if (char === "{") depth += 1;
|
|
else if (char === "}") {
|
|
depth -= 1;
|
|
if (depth === 0) return text.slice(start, index + 1);
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function compactCommand(command: string[]): string[] {
|
|
const scriptIndex = command.indexOf("--");
|
|
if (scriptIndex >= 0 && scriptIndex + 1 < command.length) return [...command.slice(0, scriptIndex + 1), "<script omitted>"];
|
|
return command;
|
|
}
|