fix: 为 WebProbe 增加物理内存门禁

2026-07-13 07:10:26 +02:00
parent 95bf0b7de6
commit 0c22a05cee
18 changed files with 727 additions and 40 deletions
@@ -0,0 +1,174 @@
+// SPEC: PJ2026-01060508 Web 哨兵。WebProbe 启动前的 YAML-first 物理内存资格判定。
+import type { CommandResult } from "./command";
+import { runCommand } from "./command";
+import { resolveCliChildJsonCommandResult } from "./cli-child-json-recovery";
+import { repoRoot } from "./config";
+import type { HwlabRuntimeLaneSpec, HwlabRuntimeWebProbeMemoryStartGuardSpec } from "./hwlab-node-lanes";
+
+export const webProbeMemoryStartGuardConfigRef = "config/hwlab-node-lanes.yaml#templates.hwlabV03.webProbeWorkbench.resourcePolicy.memoryStartGuard";
+
+export type WebProbeMemoryGuardMode = "manual-start" | "sentinel-cadence";
+export type WebProbeMemoryGuardStatus = "allowed" | "blocked" | "skipped";
+
+type MemoryGuardExecutor = (
+  command: string[],
+  cwd: string,
+  options: { timeoutMs?: number },
+) => CommandResult;
+
+function record(value: unknown): Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+function memoryPolicy(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeMemoryStartGuardSpec | null {
+  return spec.webProbe?.resourcePolicy?.memoryStartGuard ?? null;
+}
+
+function humanBytes(value: number | null): string | null {
+  if (value === null || !Number.isFinite(value)) return null;
+  const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+  let current = value;
+  let index = 0;
+  while (Math.abs(current) >= 1024 && index < units.length - 1) {
+    current /= 1024;
+    index += 1;
+  }
+  return `${current.toFixed(index === 0 ? 0 : 2)} ${units[index]}`;
+}
+
+function controlledGcNext(node: string): Record<string, unknown> {
+  return {
+    automaticGc: false,
+    requiredOrder: ["plan", "run", "recheck"],
+    planCommand: `bun scripts/cli.ts gc remote ${node} plan`,
+    runCommand: `bun scripts/cli.ts gc remote ${node} run --confirm`,
+    recheckCommand: `bun scripts/cli.ts gc remote ${node} memory`,
+    recheckMetric: "MemAvailable",
+    recheckSource: "/proc/meminfo",
+    swapExcluded: true,
+    valuesRedacted: true,
+  };
+}
+
+function unavailableGuard(spec: HwlabRuntimeLaneSpec, mode: WebProbeMemoryGuardMode, reason: string): Record<string, unknown> {
+  const policy = memoryPolicy(spec);
+  const comparator = policy === null
+    ? null
+    : mode === "manual-start"
+      ? policy.manualComparator
+      : policy.sentinelComparator;
+  return {
+    ok: false,
+    status: "blocked" satisfies WebProbeMemoryGuardStatus,
+    eligible: false,
+    startBlocked: true,
+    decision: "do-not-start-observer",
+    reason,
+    mode,
+    policyConfigRef: webProbeMemoryStartGuardConfigRef,
+    memory: {
+      metric: policy?.metric ?? "MemAvailable",
+      source: policy?.source ?? "/proc/meminfo",
+      swapExcluded: policy?.swapExcluded ?? true,
+      availableBytes: null,
+      availableHuman: null,
+      thresholdBytes: policy?.thresholdBytes ?? null,
+      thresholdHuman: humanBytes(policy?.thresholdBytes ?? null),
+      comparator,
+      thresholdMatched: null,
+      valuesRedacted: true,
+    },
+    gcNext: controlledGcNext(spec.nodeId),
+    valuesRedacted: true,
+  };
+}
+
+export function evaluateWebProbeMemoryGuard(
+  spec: HwlabRuntimeLaneSpec,
+  mode: WebProbeMemoryGuardMode,
+  snapshotPayload: unknown,
+): Record<string, unknown> {
+  const policy = memoryPolicy(spec);
+  if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
+  const payload = record(snapshotPayload);
+  const memory = record(payload.memory);
+  const availableBytes = typeof memory.availableBytes === "number" && Number.isSafeInteger(memory.availableBytes) && memory.availableBytes >= 0
+    ? memory.availableBytes
+    : null;
+  const metadataMatches = payload.ok === true
+    && payload.action === "gc remote memory"
+    && payload.metric === policy.metric
+    && payload.source === policy.source
+    && payload.swapExcluded === true;
+  if (!metadataMatches || availableBytes === null) {
+    return unavailableGuard(spec, mode, metadataMatches ? "memavailable-unavailable" : "memory-snapshot-contract-invalid");
+  }
+  const comparator = mode === "manual-start" ? policy.manualComparator : policy.sentinelComparator;
+  const thresholdMatched = mode === "manual-start"
+    ? availableBytes <= policy.thresholdBytes
+    : availableBytes < policy.thresholdBytes;
+  const status: WebProbeMemoryGuardStatus = thresholdMatched
+    ? mode === "manual-start" ? "blocked" : "skipped"
+    : "allowed";
+  return {
+    ok: status === "allowed",
+    status,
+    eligible: status === "allowed",
+    startBlocked: status !== "allowed",
+    decision: status === "allowed" ? "start-observer" : mode === "manual-start" ? "gc-required" : "skip-observer",
+    reason: status === "allowed" ? null : "memavailable-threshold-matched",
+    mode,
+    policyConfigRef: webProbeMemoryStartGuardConfigRef,
+    memory: {
+      metric: policy.metric,
+      source: policy.source,
+      swapExcluded: policy.swapExcluded,
+      availableBytes,
+      availableHuman: humanBytes(availableBytes),
+      thresholdBytes: policy.thresholdBytes,
+      thresholdHuman: humanBytes(policy.thresholdBytes),
+      comparator,
+      thresholdMatched,
+      valuesRedacted: true,
+    },
+    gcNext: status === "blocked" && mode === "manual-start"
+      ? { ...controlledGcNext(spec.nodeId), requiredBeforeStart: true }
+      : { ...controlledGcNext(spec.nodeId), requiredBeforeStart: false },
+    valuesRedacted: true,
+  };
+}
+
+export function runWebProbeMemoryGuard(
+  spec: HwlabRuntimeLaneSpec,
+  mode: WebProbeMemoryGuardMode,
+  execute: MemoryGuardExecutor = runCommand,
+): Record<string, unknown> {
+  const policy = memoryPolicy(spec);
+  if (policy === null) return unavailableGuard(spec, mode, "memory-start-policy-missing");
+  const command = ["bun", "scripts/cli.ts", "gc", "remote", spec.nodeId, "memory"];
+  const result = execute(command, repoRoot, { timeoutMs: policy.snapshotTimeoutSeconds * 1000 });
+  const resolution = resolveCliChildJsonCommandResult({
+    result,
+    requestedStdoutType: `gc remote ${spec.nodeId} memory contract`,
+    acceptParsed: (value) => {
+      const payload = value.action === "gc remote memory" ? value : record(value.data);
+      return payload.action === "gc remote memory" && payload.providerId === spec.nodeId;
+    },
+  });
+  const resolvedPayload = resolution.parsed?.action === "gc remote memory"
+    ? resolution.parsed
+    : record(resolution.parsed?.data);
+  const evaluated = evaluateWebProbeMemoryGuard(spec, mode, resolvedPayload);
+  return {
+    ...evaluated,
+    snapshot: {
+      action: "gc remote memory",
+      command: command.join(" "),
+      exitCode: result.exitCode,
+      timedOut: result.timedOut,
+      parsed: resolution.parsed !== null,
+      diagnostics: resolution.diagnostics,
+      valuesRedacted: true,
+    },
+  };
+}