// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. observability module for scripts/src/hwlab-node-impl.ts.

// Moved mechanically from scripts/src/hwlab-node-impl.ts:568-1599 for #903.

// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
import { createHash, randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";

import type { NodeObservabilityOptions } from "./entry";
import { runNodeK3sArgs } from "./cleanup";
import { nodeRuntimeExpected, nodeRuntimeLocalPostgresExpectedAbsent, parseNodeScopedDelegatedOptions } from "./plan";
import { runTransWorkspaceStdinScript, runtimeSecretSpec } from "./public-exposure";
import { joinUrlPath, nullableInteger } from "./render";
import { compactPrometheusLines, compactRuntimeCommand, isWorkbenchBackendEventMetric, k8sServiceSelector, labelSelectorText, metricNameFromPrometheusLine, parseJsonRecordFromText, prometheusLineHasLabel, prometheusLineMatchesMetric, selectK8sPod } from "./runtime-common";
import { compactCommandResultRedacted, record, shellQuote } from "./utils";
import { readBootstrapAdminPasswordMaterial } from "./web-probe";
import { webProbeCredential } from "./web-probe-observe";
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";

export function nodeRuntimeControlPlanePlan(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(scoped.spec);
  return {
    ok: true,
    command: `hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
    mode: "plan",
    mutation: false,
    node: scoped.node,
    lane: scoped.lane,
    expected: nodeRuntimeExpected(scoped.spec),
    checks: {
      nodeScopedTargetConfigured: true,
      externalPostgresDeclared: scoped.spec.externalPostgres !== undefined,
      externalPostgresActive: activeExternalPostgres !== undefined,
      secretValuesPrinted: false,
      runtimeNamespace: scoped.spec.runtimeNamespace,
      localPostgresExpectedAbsent: nodeRuntimeLocalPostgresExpectedAbsent(scoped.spec),
      publicExposureDeclared: scoped.spec.publicExposure !== null,
    },
    next: {
      infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${scoped.node} --lane ${scoped.lane}`,
      status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
      platformDbStatus: activeExternalPostgres === undefined
        ? null
        : `bun scripts/cli.ts platform-db postgres status --config ${activeExternalPostgres.configRef}`,
      publicExposure: scoped.spec.publicExposure === null ? null : `bun scripts/cli.ts hwlab nodes control-plane public-exposure --node ${scoped.node} --lane ${scoped.lane} --confirm`,
    },
  };
}

export function runNodeObservability(options: NodeObservabilityOptions): Record<string, unknown> {
  if (options.action === "plan") return nodeObservabilityPlan(options);
  if (options.action === "apply") return nodeObservabilityApply(options);
  if (options.action === "status") return nodeObservabilityStatus(options);
  if (options.action === "performance-summary") return nodeObservabilityPerformanceSummary(options);
  return nodeObservabilityWorkbenchSummary(options);
}

export function nodeObservabilityPlan(options: NodeObservabilityOptions): Record<string, unknown> {
  const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
  return {
    ok: configProblem === null,
    command: `hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
    mode: "node-observability-plan",
    mutation: false,
    node: options.node,
    lane: options.lane,
    configPath: hwlabRuntimeLaneConfigPath(),
    expected: nodeObservabilityExpected(options.spec),
    renderPlan: nodeObservabilityRenderPlan(options.spec.observability),
    degradedReason: configProblem,
    next: {
      dryRun: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
      apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
      status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
      workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
      performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
    },
  };
}

export function nodeObservabilityApply(options: NodeObservabilityOptions): Record<string, unknown> {
  const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
  const applyPlan = nodeObservabilityApplyPlan(options.spec.observability);
  if (options.dryRun) {
    return {
      ok: configProblem === null && applyPlan.supported === true,
      command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --dry-run`,
      mode: "node-observability-apply-dry-run",
      mutation: false,
      node: options.node,
      lane: options.lane,
      configPath: hwlabRuntimeLaneConfigPath(),
      expected: nodeObservabilityExpected(options.spec),
      applyPlan,
      degradedReason: configProblem ?? (applyPlan.supported === true ? undefined : "node-observability-apply-mode-unsupported"),
    };
  }
  if (configProblem !== null || applyPlan.supported !== true) {
    return {
      ok: false,
      command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
      mode: "node-observability-apply",
      mutation: false,
      node: options.node,
      lane: options.lane,
      configPath: hwlabRuntimeLaneConfigPath(),
      applyPlan,
      degradedReason: configProblem ?? "node-observability-apply-mode-unsupported",
    };
  }
  const status = nodeObservabilityStatus({ ...options, full: true });
  return {
    ok: status.ok === true,
    command: `hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
    mode: "node-observability-apply",
    mutation: false,
    node: options.node,
    lane: options.lane,
    configPath: hwlabRuntimeLaneConfigPath(),
    applyPlan,
    appliedResources: [],
    applyResult: {
      mode: applyPlan.mode,
      reason: applyPlan.reason,
      runtimeValidationOnly: true,
    },
    status: options.full ? status : summarizeNodeObservabilityStatus(status),
    degradedReason: status.ok === true ? undefined : "node-observability-status-not-ready",
  };
}

export function nodeObservabilityStatus(options: NodeObservabilityOptions): Record<string, unknown> {
  const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
  const serviceStatus = nodeObservabilityServiceStatus(options);
  const publicRawMetrics = nodeObservabilityPublicRawMetricsProbe(options);
  const workbenchSummary = nodeObservabilityWorkbenchSummary(options, serviceStatus);
  const ok = configProblem === null && serviceStatus.ready === true && publicRawMetrics.ok === true && workbenchSummary.ok === true;
  const status = {
    ok,
    command: `hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
    mode: "node-observability-status",
    mutation: false,
    node: options.node,
    lane: options.lane,
    configPath: hwlabRuntimeLaneConfigPath(),
    expected: nodeObservabilityExpected(options.spec),
    service: serviceStatus,
    publicRawMetrics,
    workbenchSummary,
    degradedReason: configProblem
      ?? (serviceStatus.ready === true
        ? publicRawMetrics.ok === true
          ? workbenchSummary.ok === true ? undefined : "node-observability-workbench-metrics-not-ready"
          : "node-observability-public-raw-metrics-boundary-failed"
        : "node-observability-service-not-ready"),
    next: {
      plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
      workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
      performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
    },
  };
  return options.full ? status : summarizeNodeObservabilityStatus(status);
}

export function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, existingServiceStatus?: Record<string, unknown>): Record<string, unknown> {
  const configProblem = nodeObservabilityConfigProblem(options.spec.observability);
  const serviceStatus = existingServiceStatus ?? nodeObservabilityServiceStatus(options);
  const podName = typeof serviceStatus.podName === "string" && serviceStatus.podName.length > 0 ? serviceStatus.podName : null;
  const metricsProbe = podName === null ? null : nodeObservabilityMetricsProbe(options, podName);
  const metrics = metricsProbe?.summary ?? summarizeNodeObservabilityMetrics(options.spec.observability, "", null);
  const ok = configProblem === null && serviceStatus.ready === true && metricsProbe?.ok === true && metrics.ready === true;
  return {
    ok,
    command: `hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
    mode: "node-observability-workbench-summary",
    mutation: false,
    node: options.node,
    lane: options.lane,
    configPath: hwlabRuntimeLaneConfigPath(),
    metricsEndpoint: nodeObservabilityEndpointSummary(options.spec.observability),
    service: {
      ready: serviceStatus.ready === true,
      namespace: serviceStatus.namespace,
      serviceName: serviceStatus.serviceName,
      podName,
      containerName: serviceStatus.containerName,
    },
    recordingRules: nodeObservabilityRecordingRuleSummaries(options.spec.observability),
    warningAlerts: nodeObservabilityWarningAlertSummaries(options.spec.observability),
    metrics,
    probe: metricsProbe === null ? null : {
      ok: metricsProbe.ok,
      httpStatus: metricsProbe.httpStatus,
      bodyBytes: metricsProbe.bodyBytes,
      result: compactRuntimeCommand(metricsProbe.result),
      error: metricsProbe.error,
    },
    degradedReason: configProblem
      ?? (serviceStatus.ready === true
        ? metricsProbe?.ok === true
          ? metrics.ready === true ? undefined : "node-observability-required-series-missing"
          : "node-observability-metrics-endpoint-not-readable"
        : "node-observability-service-not-ready"),
  };
}

export function nodeObservabilityPerformanceSummary(options: NodeObservabilityOptions): Record<string, unknown> {
  const workbench = options.spec.observability.workbench;
  const summaryPath = workbench?.summaryPath ?? "/v1/web-performance/summary";
  const endpoint = joinUrlPath(options.spec.publicWebUrl, summaryPath);
  const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane });
  const material = readBootstrapAdminPasswordMaterial(secretSpec);
  const credential = webProbeCredential(secretSpec, material);
  const command = `hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`;
  if (material.ok !== true || material.password === null) {
    return {
      ok: false,
      command,
      mode: "node-observability-performance-summary",
      mutation: false,
      node: options.node,
      lane: options.lane,
      target: {
        workspace: options.spec.workspace,
        webBaseUrl: options.spec.publicWebUrl,
        endpoint,
      },
      credential,
      degradedReason: material.error ?? "web-login-secret-unavailable",
      next: {
        secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name bootstrap-admin`,
        webProbe: `bun scripts/cli.ts web-probe run --node ${options.node} --lane ${options.lane}`,
      },
    };
  }
  const timeoutSeconds = Math.max(10, Math.min(options.timeoutSeconds, 55));
  const script = nodePerformanceSummaryRemoteScript(options.spec.publicWebUrl, summaryPath, material.username ?? secretSpec.bootstrapAdminUsername, material.password);
  const result = runTransWorkspaceStdinScript(options.node, options.spec.workspace, script, timeoutSeconds);
  const report = parseJsonRecordFromText(result.stdout);
  const performance = record(report.performance);
  const rows = nodePerformanceRows(performance);
  const analysis = nodePerformanceAnalysis(performance, rows);
  const ok = result.exitCode === 0 && report.ok === true && performance.ok !== false;
  return {
    ok,
    command,
    mode: "node-observability-performance-summary",
    mutation: false,
    node: options.node,
    lane: options.lane,
    target: {
      workspace: options.spec.workspace,
      webBaseUrl: options.spec.publicWebUrl,
      endpoint,
      summaryPath,
      lowSampleThreshold: workbench?.lowSampleThreshold ?? null,
    },
    credential,
    auth: record(report.auth),
    httpStatus: report.httpStatus ?? null,
    fetchAttempts: report.fetchAttempts ?? null,
    summary: performance,
    analysis,
    probe: compactCommandResultRedacted(result, [material.password]),
    degradedReason: ok
      ? undefined
      : typeof report.error === "string" && report.error.length > 0
        ? report.error
        : result.exitCode === 0 ? "web-performance-summary-not-ready" : "web-performance-summary-probe-failed",
    next: {
      webProbe: `bun scripts/cli.ts web-probe run --node ${options.node} --lane ${options.lane}`,
      workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
      apiMetrics: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane} --full`,
    },
  };
}

export function nodePerformanceSummaryRemoteScript(baseUrl: string, summaryPath: string, username: string, password: string): string {
  const nodeScript = [
    "const baseUrl = process.env.HWLAB_WEB_BASE_URL;",
    "const summaryPath = process.env.HWLAB_PERFORMANCE_SUMMARY_PATH || '/v1/web-performance/summary';",
    "const username = process.env.HWLAB_WEB_USER;",
    "const password = process.env.HWLAB_WEB_PASS;",
    "function compactNumber(value) { const n = Number(value); return Number.isFinite(n) ? Math.round(n * 10) / 10 : null; }",
    "function compactSeconds(msValue, secondsValue) { const ms = Number(msValue); if (Number.isFinite(ms)) return Math.round((ms / 1000) * 10) / 10; return compactNumber(secondsValue); }",
    "function compactString(value) { return typeof value === 'string' && value.length > 0 ? value : null; }",
    "function compactRow(row) {",
    "  if (!row || typeof row !== 'object' || Array.isArray(row)) return null;",
    "  const route = compactString(row.route) || compactString(row.path) || compactString(row.name) || compactString(row.metric) || compactString(row.title);",
    "  return {",
    "    source: compactString(row.source) || compactString(row.kind) || compactString(row.category),",
    "    metric: compactString(row.metric) || compactString(row.eventKind) || compactString(row.phase),",
    "    route,",
    "    problem: compactString(row.problem) || compactString(row.diagnosis) || compactString(row.reason),",
    "    outcome: compactString(row.outcome) || compactString(row.result) || compactString(row.status),",
    "    tone: compactString(row.tone) || compactString(row.severity),",
    "    sampleState: compactString(row.sampleState) || compactString(row.sampleStatus),",
    "    count: compactNumber(row.count ?? row.sampleCount ?? row.samples),",
    "    durationUnit: 'seconds',",
    "    p50Seconds: compactSeconds(row.p50Ms, row.p50Seconds ?? row.p50 ?? row.medianSeconds ?? row.median),",
    "    p75Seconds: compactSeconds(row.p75Ms, row.p75Seconds ?? row.p75),",
    "    p95Seconds: compactSeconds(row.p95Ms, row.p95Seconds ?? row.p95),",
    "    maxSeconds: compactSeconds(row.maxMs, row.maxSeconds ?? row.max),",
    "  };",
    "}",
    "function rowHasSignal(row) { return Boolean(row && (row.source || row.metric || row.route || row.problem || row.outcome)); }",
    "function compactRows(value, limit = 16) { return Array.isArray(value) ? value.map(compactRow).filter(rowHasSignal).slice(0, limit) : []; }",
    "function compactTopN(value) {",
    "  if (!Array.isArray(value)) return [];",
    "  return value.slice(0, 8).map((group) => {",
    "    const rows = compactRows(group?.rows || group?.items || group?.data, 8);",
    "    return { id: compactString(group?.id) || compactString(group?.key), title: compactString(group?.title) || compactString(group?.label), rows };",
    "  }).filter((group) => group.rows.length > 0);",
    "}",
    "function compactCards(value) {",
    "  if (!Array.isArray(value)) return [];",
    "  return value.slice(0, 12).map((card) => ({ id: compactString(card?.id), title: compactString(card?.title), value: card?.value ?? null, tone: compactString(card?.tone) || compactString(card?.status), detail: compactString(card?.detail) || compactString(card?.hint) }));",
    "}",
    "function compactPerformance(body) {",
    "  if (!body || typeof body !== 'object' || Array.isArray(body)) return { ok: false, degradedReason: 'non-json-summary' };",
    "  const dashboard = body.dashboard && typeof body.dashboard === 'object' ? body.dashboard : {};",
    "  const summary = body.summary && typeof body.summary === 'object' ? body.summary : {};",
    "  const rows = compactRows(body.rows || body.tableRows || body.performanceRows, 24);",
    "  const workbenchRows = compactRows(body.workbenchRows || body.workbench || body.workbenchTableRows, 12);",
    "  const problems = compactRows(body.problems || body.problemRows || dashboard.problems, 24);",
    "  const displayRows = rows.length > 0 ? rows.slice(0, 16) : problems.slice(0, 16);",
    "  const displayProblems = rows.length > 0 ? [] : problems.slice(0, 16);",
    "  const topN = compactTopN(dashboard.topN || body.topN);",
    "  return {",
    "    ok: true,",
    "    schemaVersion: body.schemaVersion || dashboard.schemaVersion || null,",
    "    source: body.source || dashboard.source || null,",
    "    observedAt: body.observedAt || dashboard.observedAt || body.generatedAt || dashboard.generatedAt || null,",
    "    window: body.window || body.sampleWindow || dashboard.window || null,",
    "    freshness: body.freshness || dashboard.freshness || null,",
    "    sampleCount: summary.sampleCount ?? body.sampleCount ?? dashboard.sampleCount ?? null,",
    "    problemCount: summary.problemCount ?? body.problemCount ?? dashboard.problemCount ?? null,",
    "    status: summary.status || body.status || dashboard.status || null,",
    "    cards: compactCards(dashboard.cards || body.cards),",
    "    rows: displayRows,",
    "    workbenchRows,",
    "    problems: displayProblems,",
    "    topN,",
    "    rowCount: rows.length,",
    "    workbenchRowCount: workbenchRows.length,",
    "    problemRowCount: problems.length,",
    "  };",
    "}",
    "function splitSetCookie(raw) { return raw ? raw.split(/,(?=[^;,]+=)/g).map((item) => item.trim()).filter(Boolean) : []; }",
    "async function fetchWithTimeout(url, init, timeoutMs = 12000) {",
    "  const controller = new AbortController();",
    "  const timer = setTimeout(() => controller.abort(), timeoutMs);",
    "  try {",
    "    const response = await fetch(url, { ...init, signal: controller.signal });",
    "    const text = await response.text();",
    "    let body = null;",
    "    try { body = JSON.parse(text); } catch {}",
    "    return { response, text, body };",
    "  } finally { clearTimeout(timer); }",
    "}",
    "function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }",
    "async function fetchWithRetry(url, init, attempts = 3) {",
    "  let last = null;",
    "  for (let attempt = 1; attempt <= attempts; attempt += 1) {",
    "    try {",
    "      const result = await fetchWithTimeout(url, init);",
    "      result.attempt = attempt;",
    "      last = result;",
    "      if (![502, 503, 504].includes(result.response.status)) return result;",
    "    } catch (error) {",
    "      last = { attempt, error: error instanceof Error ? error.message : String(error), response: { status: 0, ok: false, headers: new Headers() }, body: null, text: '' };",
    "    }",
    "    if (attempt < attempts) await sleep(400 * attempt);",
    "  }",
    "  return last;",
    "}",
    "(async () => {",
    "  try {",
    "    const loginUrl = new URL('/auth/login', baseUrl).toString();",
    "    const login = await fetchWithRetry(loginUrl, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ username, password }) });",
    "    const setCookie = splitSetCookie(login.response.headers.get('set-cookie'));",
    "    const cookie = setCookie.map((item) => item.split(';')[0].trim()).filter(Boolean).join('; ');",
    "    const auth = { ok: login.response.ok && cookie.length > 0, httpStatus: login.response.status, cookiePresent: cookie.length > 0, attempts: login.attempt || 1 };",
    "    if (!auth.ok) {",
    "      console.log(JSON.stringify({ ok: false, auth, httpStatus: login.response.status, error: 'web-login-failed' }));",
    "      process.exitCode = 1;",
    "      return;",
    "    }",
    "    const summaryUrl = new URL(summaryPath, baseUrl).toString();",
    "    const summary = await fetchWithRetry(summaryUrl, { method: 'GET', headers: { accept: 'application/json', cookie } });",
    "    const performance = compactPerformance(summary.body);",
    "    const ok = summary.response.ok && performance.ok === true;",
    "    console.log(JSON.stringify({ ok, auth, httpStatus: summary.response.status, fetchAttempts: summary.attempt || 1, finalUrl: summaryUrl, performance, error: ok ? null : 'web-performance-summary-fetch-failed' }));",
    "    process.exitCode = ok ? 0 : 1;",
    "  } catch (error) {",
    "    console.log(JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }));",
    "    process.exitCode = 1;",
    "  }",
    "})();",
  ].join("\n");
  return [
    "set -eu",
    `HWLAB_WEB_BASE_URL=${shellQuote(baseUrl)} HWLAB_PERFORMANCE_SUMMARY_PATH=${shellQuote(summaryPath)} HWLAB_WEB_USER=${shellQuote(username)} HWLAB_WEB_PASS=${shellQuote(password)} node <<'NODE'`,
    nodeScript,
    "NODE",
  ].join("\n");
}

export function nodePerformanceRows(performance: Record<string, unknown>): Record<string, unknown>[] {
  const rows = [
    ...nodePerformanceArray(performance.problems),
    ...nodePerformanceArray(performance.rows),
    ...nodePerformanceArray(performance.workbenchRows),
  ];
  const topN = Array.isArray(performance.topN) ? performance.topN.map(record) : [];
  for (const group of topN) {
    for (const row of nodePerformanceArray(group.rows)) rows.push(row);
  }
  const seen = new Set<string>();
  return rows.filter((row) => {
    const key = [
      nodePerformanceString(row.source),
      nodePerformanceString(row.metric),
      nodePerformanceString(row.route),
      nodePerformanceString(row.outcome),
      nodePerformanceString(row.problem),
    ].join("|");
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  });
}

export function nodePerformanceArray(value: unknown): Record<string, unknown>[] {
  return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : [];
}

export function nodePerformanceAnalysis(performance: Record<string, unknown>, rows: Record<string, unknown>[]): Record<string, unknown> {
  const actionableRows = rows.filter(nodePerformanceActionableProblem);
  const networkErrors = actionableRows.filter((row) => nodePerformanceString(row.outcome) === "network_error" || nodePerformanceString(row.problem) === "network_error" || nodePerformanceString(row.problem) === "network error");
  const blockedRows = actionableRows.filter((row) => nodePerformanceString(row.tone) === "blocked" || nodePerformanceString(row.status) === "blocked");
  const slowApiRows = rows.filter((row) => {
    const source = nodePerformanceString(row.source) ?? "";
    const metric = nodePerformanceString(row.metric) ?? "";
    const p95Seconds = nodePerformanceNumber(row.p95Seconds);
    return nodePerformanceActionableProblem(row)
      && (source.includes("api") || metric.includes("api") || (nodePerformanceString(row.route) ?? "").startsWith("/v1") || (nodePerformanceString(row.route) ?? "").startsWith("/health"))
      && (p95Seconds === null || p95Seconds >= 5);
  });
  const lcpRows = actionableRows.filter((row) => {
    const metric = `${nodePerformanceString(row.metric) ?? ""} ${nodePerformanceString(row.problem) ?? ""}`.toLowerCase();
    return metric.includes("lcp") || metric.includes("largest contentful") || metric.includes("最大内容");
  });
  const maxP95Seconds = Math.max(0, ...rows.map((row) => nodePerformanceNumber(row.p95Seconds) ?? 0));
  const headline = networkErrors.length > 0
    ? "Public same-origin API has intermittent network_error samples; treat edge/proxy/API reachability as P0 before tuning page render."
    : lcpRows.length > 0
      ? "Workbench page LCP is blocked, likely by first-load API waterfalls and late visible content."
      : slowApiRows.length > 0
        ? "Same-origin API p95 is above the interactive budget; split edge wait, server work, and dependency latency."
        : "No blocking performance rows were returned by the current summary window.";
  const priorities = [
    networkErrors.length > 0
      ? {
        priority: "P0",
        area: "public-edge-api-reachability",
        action: "Correlate network_error rows with Caddy/FRP/ingress, cloud-api pod restarts, response-header timeout, and browser Resource Timing failure stage.",
        evidence: nodePerformanceEvidence(networkErrors),
      }
      : null,
    slowApiRows.length > 0
      ? {
        priority: "P1",
        area: "same-origin-api-latency",
        action: "Add server-side route histograms and dependency spans for slow /health, /v1/live-builds, /v1/workbench/sessions, trace events, hwpod specs, and hwpod-node-ops paths; then optimize the highest p95 route first.",
        evidence: nodePerformanceEvidence(slowApiRows),
      }
      : null,
    lcpRows.length > 0
      ? {
        priority: "P1",
        area: "workbench-lcp",
        action: "Render a stable first screen before slow session/detail API calls complete, and split LCP by route plus blocking API waterfall.",
        evidence: nodePerformanceEvidence(lcpRows),
      }
      : null,
    blockedRows.length > 0
      ? {
        priority: "P2",
        area: "alert-thresholds-and-sample-quality",
        action: "Keep low-sample rows visible but avoid closing incidents on count=1; require repeated windows or matching server metrics before labeling a regression fixed.",
        evidence: nodePerformanceEvidence(blockedRows),
      }
      : null,
  ].filter(Boolean);
  return {
    headline,
    sampleCount: performance.sampleCount ?? null,
    problemCount: performance.problemCount ?? null,
    rowCount: rows.length,
    blockedRowCount: blockedRows.length,
    networkErrorRowCount: networkErrors.length,
    slowApiRowCount: slowApiRows.length,
    lcpRowCount: lcpRows.length,
    maxP95Seconds,
    priorities,
  };
}

export function nodePerformanceActionableProblem(row: Record<string, unknown>): boolean {
  const tone = nodePerformanceString(row.tone) ?? "";
  const status = nodePerformanceString(row.status) ?? "";
  if (tone === "blocked" || tone === "warn" || status === "blocked" || status === "warn") return true;
  const sampleState = nodePerformanceString(row.sampleState) ?? "";
  if (sampleState && sampleState !== "ok") return false;
  const problem = (nodePerformanceString(row.problem) ?? "").trim().toLowerCase();
  return Boolean(problem && problem !== "ok" && problem !== "low-sample" && problem !== "no-sample");
}

export function nodePerformanceEvidence(rows: Record<string, unknown>[]): Record<string, unknown>[] {
  return rows.slice(0, 8).map((row) => ({
    source: nodePerformanceString(row.source),
    metric: nodePerformanceString(row.metric),
    route: nodePerformanceString(row.route),
    outcome: nodePerformanceString(row.outcome),
    problem: nodePerformanceString(row.problem),
    count: nodePerformanceNumber(row.count),
    durationUnit: nodePerformanceString(row.durationUnit) ?? "seconds",
    p50Seconds: nodePerformanceNumber(row.p50Seconds),
    p75Seconds: nodePerformanceNumber(row.p75Seconds),
    p95Seconds: nodePerformanceNumber(row.p95Seconds),
  }));
}

export function nodePerformanceString(value: unknown): string | null {
  return typeof value === "string" && value.length > 0 ? value : null;
}

export function nodePerformanceNumber(value: unknown): number | null {
  const parsed = Number(value);
  return Number.isFinite(parsed) ? parsed : null;
}

export function summarizeNodeObservabilityStatus(status: Record<string, unknown>): Record<string, unknown> {
  const service = record(status.service);
  const publicRawMetrics = record(status.publicRawMetrics);
  const workbenchSummary = record(status.workbenchSummary);
  const metrics = record(workbenchSummary.metrics);
  const recordingRules = Array.isArray(workbenchSummary.recordingRules) ? workbenchSummary.recordingRules : [];
  const warningAlerts = Array.isArray(workbenchSummary.warningAlerts) ? workbenchSummary.warningAlerts : [];
  return {
    ok: status.ok === true,
    command: status.command,
    mode: "node-observability-status-summary",
    mutation: false,
    node: status.node,
    lane: status.lane,
    degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
    service: {
      ready: service.ready === true,
      namespace: service.namespace ?? null,
      serviceName: service.serviceName ?? null,
      podName: service.podName ?? null,
      containerName: service.containerName ?? null,
    },
    publicRawMetrics: {
      ok: publicRawMetrics.ok === true,
      expected: publicRawMetrics.expected ?? null,
      httpStatus: publicRawMetrics.httpStatus ?? null,
      exposesPrometheus: publicRawMetrics.exposesPrometheus === true,
    },
    workbench: {
      ok: workbenchSummary.ok === true,
      httpStatus: metrics.httpStatus ?? null,
      bodyBytes: metrics.bodyBytes ?? null,
      recordingRuleCount: recordingRules.length,
      warningAlertCount: warningAlerts.length,
      seriesByPrefix: metrics.seriesByPrefix ?? {},
      missingSeries: metrics.missingSeries ?? [],
      topSlowDimensions: metrics.topSlowDimensions ?? [],
      deniedBackendEventLineCount: metrics.deniedBackendEventLineCount ?? null,
      maxDeniedBackendEventLines: metrics.maxDeniedBackendEventLines ?? null,
    },
    next: status.next,
  };
}

export function nodeObservabilityExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
  return {
    node: spec.nodeId,
    lane: spec.lane,
    namespace: spec.runtimeNamespace,
    publicApiUrl: spec.publicApiUrl,
    observability: spec.observability,
    sourceOfTruth: hwlabRuntimeLaneConfigPath(),
    runtimeAsSourceOfTruth: false,
  };
}

export function nodeObservabilityRenderPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
  const endpoint = observability.metricsEndpoint;
  return {
    prometheusOperator: observability.prometheusOperator,
    metricsEndpoint: nodeObservabilityEndpointSummary(observability),
    workbench: observability.workbench ?? null,
    recordingRules: nodeObservabilityRecordingRuleSummaries(observability),
    warningAlerts: nodeObservabilityWarningAlertSummaries(observability),
    clusterResources: observability.prometheusOperator
      ? {
        source: "HWLAB GitOps rendered manifests",
        serviceMonitor: "YAML-rendered only when declared by target lane",
        prometheusRule: "YAML-rendered only when declared by target lane",
      }
      : {
        source: "none",
        reason: endpoint?.scrapeMode === "pod-loopback" ? "target lane declares pod-loopback scrape mode" : "no YAML-declared scrape target",
      },
    publicRawMetrics: endpoint?.publicRawMetrics ?? null,
  };
}

export function nodeObservabilityApplyPlan(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> {
  const endpoint = observability.metricsEndpoint;
  if (endpoint === undefined) {
    return {
      supported: false,
      mode: "unconfigured",
      reason: "metricsEndpoint is missing from YAML",
    };
  }
  if (!observability.prometheusOperator && endpoint.scrapeMode === "pod-loopback") {
    return {
      supported: true,
      mode: "validated-noop",
      reason: "target lane declares pod-loopback metrics collection and no Prometheus Operator resources",
      resources: [],
      validates: ["kubernetes-service", "selected-pod", "pod-loopback-metrics", "public-raw-metrics-boundary", "workbench-required-series"],
    };
  }
  if (observability.prometheusOperator) {
    return {
      supported: false,
      mode: "prometheus-operator",
      reason: "Prometheus Operator object rendering must be declared in the target lane GitOps YAML before this node-scoped apply can own it",
    };
  }
  return {
    supported: false,
    mode: endpoint.scrapeMode,
    reason: "unsupported metricsEndpoint.scrapeMode",
  };
}

export function nodeObservabilityConfigProblem(observability: HwlabRuntimeObservabilitySpec): string | null {
  if (observability.metricsEndpoint === undefined) return "node-observability-metrics-endpoint-not-declared";
  if (observability.workbench === undefined) return "node-observability-workbench-not-declared";
  if (!observability.workbench.enabled) return "node-observability-workbench-disabled";
  return null;
}

export function nodeObservabilityEndpointSummary(observability: HwlabRuntimeObservabilitySpec): Record<string, unknown> | null {
  const endpoint = observability.metricsEndpoint;
  if (endpoint === undefined) return null;
  return {
    serviceName: endpoint.serviceName,
    containerName: endpoint.containerName,
    port: endpoint.port,
    scheme: endpoint.scheme,
    path: endpoint.path,
    scrapeMode: endpoint.scrapeMode,
    publicRawMetrics: endpoint.publicRawMetrics,
  };
}

export function nodeObservabilityServiceStatus(options: NodeObservabilityOptions): Record<string, unknown> {
  const endpoint = options.spec.observability.metricsEndpoint;
  if (endpoint === undefined) {
    return {
      ready: false,
      namespace: options.spec.runtimeNamespace,
      serviceName: null,
      podName: null,
      containerName: null,
      degradedReason: "node-observability-metrics-endpoint-not-declared",
    };
  }
  const namespace = runNodeK3sArgs(options.spec, ["kubectl", "get", "ns", options.spec.runtimeNamespace, "-o", "name"], 60);
  const namespaceExists = namespace.exitCode === 0;
  const service = namespaceExists
    ? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "svc", endpoint.serviceName, "-o", "json"], 60)
    : null;
  const serviceJson = service !== null && service.exitCode === 0 ? parseJsonRecordFromText(service.stdout) : {};
  const selector = k8sServiceSelector(serviceJson);
  const selectorText = labelSelectorText(selector);
  const pods = namespaceExists && service !== null && service.exitCode === 0 && selectorText.length > 0
    ? runNodeK3sArgs(options.spec, ["kubectl", "-n", options.spec.runtimeNamespace, "get", "pods", "-l", selectorText, "-o", "json"], 60)
    : null;
  const pod = pods !== null && pods.exitCode === 0 ? selectK8sPod(parseJsonRecordFromText(pods.stdout), endpoint.containerName) : null;
  const ready = namespaceExists && service !== null && service.exitCode === 0 && pod?.ready === true;
  return {
    ready,
    namespace: options.spec.runtimeNamespace,
    namespaceExists,
    serviceName: endpoint.serviceName,
    serviceExists: service !== null && service.exitCode === 0,
    selector,
    selectorText: selectorText.length > 0 ? selectorText : null,
    podName: pod?.name ?? null,
    podPhase: pod?.phase ?? null,
    podReady: pod?.ready ?? false,
    containerName: endpoint.containerName,
    containerReady: pod?.containerReady ?? false,
    probes: {
      namespace: compactRuntimeCommand(namespace),
      service: service === null ? null : compactRuntimeCommand(service),
      pods: pods === null ? null : compactRuntimeCommand(pods),
    },
    degradedReason: ready ? undefined : namespaceExists ? "node-observability-pod-or-service-not-ready" : "node-observability-namespace-missing",
  };
}

export interface NodeObservabilityMetricsProbeResult {
  readonly ok: boolean;
  readonly httpStatus: number | null;
  readonly bodyBytes: number;
  readonly summary: Record<string, unknown>;
  readonly result: CommandResult;
  readonly error: string | null;
}

export function nodeObservabilityMetricsProbe(options: NodeObservabilityOptions, podName: string): NodeObservabilityMetricsProbeResult {
  const endpoint = options.spec.observability.metricsEndpoint;
  if (endpoint === undefined) throw new Error("metricsEndpoint is required before probing metrics");
  const workbench = options.spec.observability.workbench;
  const summaryConfig = Buffer.from(JSON.stringify({
    metricPrefixes: workbench?.metricPrefixes ?? [],
    requiredSeries: workbench?.requiredSeries ?? [],
    backendLabelDenylist: workbench?.backendLabelDenylist ?? [],
    maxDeniedBackendEventLines: workbench?.maxUnknownEventLines ?? 0,
    lowSampleThreshold: workbench?.lowSampleThreshold ?? 0,
    histogramMetrics: [
      { id: "workbench_journey", metric: "hwlab_workbench_journey_duration_seconds", kind: "workbench_journey", dimensionLabels: ["namespace", "gitops_target", "journey", "route", "backend", "transport", "target_state", "cache", "source", "entry", "outcome"] },
      { id: "workbench_event_phase", metric: "hwlab_workbench_event_phase_duration_seconds", kind: "workbench_event_phase", dimensionLabels: ["namespace", "gitops_target", "phase", "event_type", "backend", "transport", "outcome"] },
      { id: "workbench_backend_event_visible", metric: "hwlab_workbench_backend_event_visible_latency_seconds", kind: "workbench_backend_event_visible", dimensionLabels: ["namespace", "gitops_target", "event_type", "backend", "transport", "outcome"] },
      { id: "workbench_projection_lag_events", metric: "hwlab_workbench_projection_lag_events", kind: "workbench_projection_lag_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
      { id: "workbench_projection_lag_seconds", metric: "hwlab_workbench_projection_lag_seconds", kind: "workbench_projection_lag_seconds", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
      { id: "workbench_terminal_projection_delay", metric: "hwlab_workbench_terminal_projection_delay_seconds", kind: "workbench_terminal_projection_delay", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "projection_status", "source", "status", "reason"] },
      { id: "workbench_turn_get", metric: "hwlab_workbench_turn_get_duration_seconds", kind: "workbench_turn_get", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "route", "status", "degraded_reason"] },
      { id: "agentrun_result_duration", metric: "hwlab_agentrun_result_duration_seconds", kind: "agentrun_result_duration", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "event_count_bucket", "status"] },
      { id: "agentrun_result_pages", metric: "hwlab_agentrun_result_pages_scanned", kind: "agentrun_result_pages", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
      { id: "agentrun_result_events", metric: "hwlab_agentrun_result_events_scanned", kind: "agentrun_result_events", dimensionLabels: ["namespace", "gitops_target", "node", "lane"] },
      { id: "workbench_projector_batch", metric: "hwlab_workbench_projector_batch_duration_seconds", kind: "workbench_projector_batch", dimensionLabels: ["namespace", "gitops_target", "node", "lane", "phase", "status"] },
    ],
  }), "utf8").toString("base64");
  const source = [
    "const http = require('node:http');",
    "const port = Number(process.argv[1]);",
    "const path = process.argv[2];",
    "const config = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
    "function metricNameFromLine(line) { const match = /^([A-Za-z_:][A-Za-z0-9_:]*)/.exec(line); return match ? match[1] : null; }",
    "function lineMatchesMetric(line, name) { if (!line.startsWith(name)) return false; const next = line.charAt(name.length); return next === '' || next === '{' || /\\s/.test(next); }",
    "function lineHasLabel(line, name, value) { return line.includes(`${name}=\\\"${String(value).replace(/\\\\/g, '\\\\\\\\').replace(/\\\"/g, '\\\\\\\"')}\\\"`); }",
    "function isBackendEventMetric(name) { return name.startsWith('hwlab_workbench_event_') || name.startsWith('hwlab_workbench_backend_event_'); }",
    "function compactLines(lines) { return lines.slice(0, 16).map((line) => line.length > 320 ? `${line.slice(0, 317)}...` : line); }",
    "function parseLabels(raw) { const labels = {}; for (const part of String(raw || '').split(',')) { const index = part.indexOf('='); if (index <= 0) continue; const key = part.slice(0, index); let value = part.slice(index + 1); if (value.startsWith('\\\"') && value.endsWith('\\\"')) value = value.slice(1, -1); labels[key] = value.replace(/\\\\\\\"/g, '\\\"').replace(/\\\\\\\\/g, '\\\\'); } return labels; }",
    "function labelKey(metric, labels) { const copy = { ...labels }; delete copy.le; return `${metric}|${Object.keys(copy).sort().map((key) => `${key}=${copy[key]}`).join('|')}`; }",
    "function pickLabels(labels, names) { return Object.fromEntries(names.map((name) => [name, labels[name] || 'unknown'])); }",
    "function quantile(row, q) { const count = Number(row.count || 0); if (count <= 0) return 0; const rank = Math.max(1, Math.ceil(count * q)); const buckets = row.buckets.filter((item) => item.le !== '+Inf').sort((left, right) => Number(left.le) - Number(right.le)); for (const bucket of buckets) { if (bucket.value >= rank) return Number(bucket.le); } return buckets.length ? Number(buckets[buckets.length - 1].le) : 0; }",
    "function summarizeHistograms(sampleLines) {",
    "  const metricConfig = new Map(config.histogramMetrics.map((item) => [item.metric, item]));",
    "  const series = new Map();",
    "  for (const line of sampleLines) {",
    "    const match = /^([A-Za-z_:][A-Za-z0-9_:]*?)_(bucket|sum|count)\\{([^}]*)\\}\\s+([0-9.eE+-]+)/.exec(line);",
    "    if (!match || !metricConfig.has(match[1])) continue;",
    "    const metric = match[1];",
    "    const suffix = match[2];",
    "    const labels = parseLabels(match[3]);",
    "    const value = Number(match[4]);",
    "    if (!Number.isFinite(value)) continue;",
    "    const key = labelKey(metric, labels);",
    "    const row = series.get(key) || { metric, labels: { ...labels }, buckets: [], sum: 0, count: 0 };",
    "    if (suffix === 'bucket') row.buckets.push({ le: labels.le || '+Inf', value });",
    "    if (suffix === 'sum') row.sum = value;",
    "    if (suffix === 'count') row.count = value;",
    "    series.set(key, row);",
    "  }",
    "  const lowSampleThreshold = Number(config.lowSampleThreshold || 0);",
    "  const groups = Object.fromEntries(config.histogramMetrics.map((item) => [item.id, []]));",
    "  const rows = [];",
    "  for (const row of series.values()) {",
    "    const cfg = metricConfig.get(row.metric);",
    "    const sampleCount = Number(row.count || 0);",
    "    const output = { kind: cfg.kind, metric: row.metric, sampleCount, average: sampleCount > 0 ? Number((row.sum / sampleCount).toFixed(4)) : 0, p50: quantile(row, 0.5), p75: quantile(row, 0.75), p95: quantile(row, 0.95), lowSample: sampleCount > 0 && sampleCount < lowSampleThreshold, sampleState: sampleCount <= 0 ? 'empty' : sampleCount < lowSampleThreshold ? 'low-sample' : 'ok', dimensions: pickLabels(row.labels, cfg.dimensionLabels) };",
    "    groups[cfg.id].push(output);",
    "    rows.push(output);",
    "  }",
    "  for (const key of Object.keys(groups)) groups[key].sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
    "  rows.sort((left, right) => right.p95 - left.p95 || right.sampleCount - left.sampleCount);",
    "  return { groups: Object.fromEntries(Object.entries(groups).map(([key, value]) => [key, value.slice(0, 24)])), topSlowDimensions: rows.slice(0, 12) };",
    "}",
    "function summarize(text, statusCode) {",
    "  const lines = text.split(/\\r?\\n/).map((line) => line.trim()).filter(Boolean);",
    "  const sampleLines = lines.filter((line) => !line.startsWith('#'));",
    "  const seriesByPrefix = Object.fromEntries(config.metricPrefixes.map((prefix) => [prefix, sampleLines.filter((line) => (metricNameFromLine(line) || '').startsWith(prefix)).length]));",
    "  const requiredSeries = config.requiredSeries.map((name) => { const matching = sampleLines.filter((line) => lineMatchesMetric(line, name)); return { name, present: matching.length > 0, sampleCount: matching.length }; });",
    "  const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);",
    "  const deniedBackendEventLines = sampleLines.filter((line) => { const name = metricNameFromLine(line) || ''; if (!isBackendEventMetric(name)) return false; return config.backendLabelDenylist.some((label) => lineHasLabel(line, 'backend', label)); });",
    "  const histogramSummary = summarizeHistograms(sampleLines);",
    "  const ready = statusCode >= 200 && statusCode < 300 && missingSeries.length === 0 && deniedBackendEventLines.length <= config.maxDeniedBackendEventLines;",
    "  return {",
    "    ready,",
    "    httpStatus: statusCode,",
    "    bodyBytes: Buffer.byteLength(text),",
    "    lineCount: lines.length,",
    "    sampleLineCount: sampleLines.length,",
    "    metricPrefixes: config.metricPrefixes,",
    "    seriesByPrefix,",
    "    requiredSeries,",
    "    missingSeries,",
    "    backendLabelDenylist: config.backendLabelDenylist,",
    "    lowSampleThreshold: config.lowSampleThreshold,",
    "    workbenchHistograms: histogramSummary.groups,",
    "    topSlowDimensions: histogramSummary.topSlowDimensions,",
    "    deniedBackendEventLineCount: deniedBackendEventLines.length,",
    "    maxDeniedBackendEventLines: config.maxDeniedBackendEventLines,",
    "    deniedBackendEventLines: compactLines(deniedBackendEventLines),",
    "    backendVisibleCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_backend_event_visible_latency_seconds_count'))),",
    "    phaseCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_event_phase_duration_seconds_count'))),",
    "    projectionLagCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projection_lag_events_count') || lineMatchesMetric(line, 'hwlab_workbench_projection_lag_seconds_count'))),",
    "    turnGetCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_turn_get_duration_seconds_count'))),",
    "    agentRunResultCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_agentrun_result_duration_seconds_count'))),",
    "    projectorBatchCountLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_batch_duration_seconds_count'))),",
    "    projectorLastSuccessLines: compactLines(sampleLines.filter((line) => lineMatchesMetric(line, 'hwlab_workbench_projector_last_success_unixtime'))),",
    "    degradedReason: ready ? undefined : missingSeries.length > 0 ? 'node-observability-required-series-missing' : 'node-observability-denied-backend-label-present',",
    "  };",
    "}",
    "let settled = false;",
    "const finish = (payload, code = 0) => { if (settled) return; settled = true; console.log(JSON.stringify(payload)); process.exitCode = code; };",
    "const req = http.get({ host: '127.0.0.1', port, path, timeout: 8000 }, (res) => {",
    "  let body = '';",
    "  res.setEncoding('utf8');",
    "  res.on('data', (chunk) => { body += chunk; if (body.length > 2000000) req.destroy(new Error('metrics body exceeded 2MB')); });",
    "  res.on('end', () => finish({ ok: res.statusCode >= 200 && res.statusCode < 300, statusCode: res.statusCode, bodyBytes: Buffer.byteLength(body), summary: summarize(body, res.statusCode || 0) }));",
    "});",
    "req.on('timeout', () => req.destroy(new Error('timeout')));",
    "req.on('error', (error) => finish({ ok: false, statusCode: null, bodyBytes: 0, summary: null, error: String(error && error.message || error) }, 1));",
  ].join("\n");
  const result = runNodeK3sArgs(options.spec, [
    "kubectl",
    "-n", options.spec.runtimeNamespace,
    "exec", podName,
    "-c", endpoint.containerName,
    "--",
    "node", "-e", source,
    String(endpoint.port),
    endpoint.path,
    summaryConfig,
  ], options.timeoutSeconds);
  const payload = parseJsonRecordFromText(result.stdout);
  const httpStatus = typeof payload.statusCode === "number" ? payload.statusCode : null;
  const error = typeof payload.error === "string" ? payload.error : null;
  const summary = record(payload.summary);
  return {
    ok: result.exitCode === 0 && payload.ok === true && httpStatus !== null && httpStatus >= 200 && httpStatus < 300,
    httpStatus,
    bodyBytes: typeof payload.bodyBytes === "number" ? payload.bodyBytes : 0,
    summary,
    result,
    error,
  };
}

export function nodeObservabilityPublicRawMetricsProbe(options: NodeObservabilityOptions): Record<string, unknown> {
  const endpoint = options.spec.observability.metricsEndpoint;
  if (endpoint === undefined) {
    return {
      ok: false,
      expected: null,
      degradedReason: "node-observability-metrics-endpoint-not-declared",
    };
  }
  const url = joinUrlPath(options.spec.publicApiUrl, endpoint.path);
  const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "-", "-w", "\nUNIDESK_HTTP_STATUS:%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
  const parsed = splitCurlBodyStatus(result.stdout);
  const exposesPrometheus = looksLikePrometheusMetrics(parsed.body, options.spec.observability.workbench?.metricPrefixes ?? []);
  const denied = endpoint.publicRawMetrics === "denied" && (parsed.httpStatus === null || parsed.httpStatus >= 400 || !exposesPrometheus);
  return {
    ok: result.exitCode === 0 && denied,
    expected: endpoint.publicRawMetrics,
    url,
    httpStatus: parsed.httpStatus,
    exposesPrometheus,
    bodyBytes: Buffer.byteLength(parsed.body),
    result: compactRuntimeCommand(result),
    degradedReason: denied ? undefined : "node-observability-public-raw-metrics-exposed",
  };
}

export function summarizeNodeObservabilityMetrics(observability: HwlabRuntimeObservabilitySpec, text: string, httpStatus: number | null): Record<string, unknown> {
  const workbench = observability.workbench;
  if (workbench === undefined) {
    return {
      ready: false,
      httpStatus,
      degradedReason: "node-observability-workbench-not-declared",
    };
  }
  const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
  const sampleLines = lines.filter((line) => !line.startsWith("#"));
  const seriesByPrefix = Object.fromEntries(workbench.metricPrefixes.map((prefix) => [
    prefix,
    sampleLines.filter((line) => (metricNameFromPrometheusLine(line) ?? "").startsWith(prefix)).length,
  ]));
  const requiredSeries = workbench.requiredSeries.map((name) => {
    const matching = sampleLines.filter((line) => prometheusLineMatchesMetric(line, name));
    return {
      name,
      present: matching.length > 0,
      sampleCount: matching.length,
    };
  });
  const missingSeries = requiredSeries.filter((series) => !series.present).map((series) => series.name);
  const deniedBackendEventLines = sampleLines.filter((line) => {
    const name = metricNameFromPrometheusLine(line) ?? "";
    if (!isWorkbenchBackendEventMetric(name)) return false;
    return workbench.backendLabelDenylist.some((label) => prometheusLineHasLabel(line, "backend", label));
  });
  const ready = httpStatus !== null
    && httpStatus >= 200
    && httpStatus < 300
    && missingSeries.length === 0
    && deniedBackendEventLines.length <= workbench.maxUnknownEventLines;
  return {
    ready,
    httpStatus,
    bodyBytes: Buffer.byteLength(text),
    lineCount: lines.length,
    sampleLineCount: sampleLines.length,
    metricPrefixes: workbench.metricPrefixes,
    seriesByPrefix,
    requiredSeries,
    missingSeries,
    backendLabelDenylist: workbench.backendLabelDenylist,
    deniedBackendEventLineCount: deniedBackendEventLines.length,
    maxDeniedBackendEventLines: workbench.maxUnknownEventLines,
    deniedBackendEventLines: compactPrometheusLines(deniedBackendEventLines),
    backendVisibleCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_backend_event_visible_latency_seconds_count"))),
    phaseCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_event_phase_duration_seconds_count"))),
    projectionLagCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_events_count") || prometheusLineMatchesMetric(line, "hwlab_workbench_projection_lag_seconds_count"))),
    turnGetCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_turn_get_duration_seconds_count"))),
    agentRunResultCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_agentrun_result_duration_seconds_count"))),
    projectorBatchCountLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_batch_duration_seconds_count"))),
    projectorLastSuccessLines: compactPrometheusLines(sampleLines.filter((line) => prometheusLineMatchesMetric(line, "hwlab_workbench_projector_last_success_unixtime"))),
    degradedReason: ready ? undefined : missingSeries.length > 0 ? "node-observability-required-series-missing" : "node-observability-denied-backend-label-present",
  };
}

export function splitCurlBodyStatus(stdout: string): { body: string; httpStatus: number | null } {
  const marker = "\nUNIDESK_HTTP_STATUS:";
  const index = stdout.lastIndexOf(marker);
  if (index < 0) return { body: stdout, httpStatus: null };
  const rawStatus = stdout.slice(index + marker.length).trim();
  const status = nullableInteger(rawStatus);
  return {
    body: stdout.slice(0, index),
    httpStatus: status,
  };
}

export function looksLikePrometheusMetrics(text: string, prefixes: readonly string[]): boolean {
  const lines = text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
  return lines.some((line) => {
    if (line.startsWith("# HELP ") || line.startsWith("# TYPE ")) return true;
    const name = metricNameFromPrometheusLine(line);
    return name !== null && (prefixes.length === 0 || prefixes.some((prefix) => name.startsWith(prefix)));
  });
}