fix: improve hwlab probe diagnostics

2026-06-20 03:59:48 +00:00
parent 7696fddccf
commit 6a88f38ca9
6 changed files with 317 additions and 5 deletions
@@ -3349,10 +3349,14 @@ function nullableInteger(value: string): number | null {
 function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
  const web = publicHttpProbe("web", spec.publicWebUrl);
  const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
+  const ready = web.ok === true && apiHealth.ok === true;
+  const targetHost = nodeRuntimeTargetHostPublicProbeStatus(spec);
  return {
-    ready: web.ok === true && apiHealth.ok === true,
+    ready,
    web,
    apiHealth,
+    targetHost,
+    diagnostic: nodeRuntimePublicProbeDiagnostic(ready, targetHost),
  };
 }

@@ -3368,6 +3372,103 @@ function publicHttpProbe(kind: string, url: string): Record<string, unknown> {
  };
 }

+function nodeRuntimeTargetHostPublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
+  const webUrl = spec.publicWebUrl;
+  const apiHealthUrl = joinUrlPath(spec.publicApiUrl, "/health/live");
+  const script = [
+    "set -eu",
+    `web_url=${shellQuote(webUrl)}`,
+    `api_url=${shellQuote(apiHealthUrl)}`,
+    "probe() {",
+    "  name=\"$1\"",
+    "  url=\"$2\"",
+    "  err_file=$(mktemp)",
+    "  set +e",
+    "  http_status=$(curl -k -sS --connect-timeout 5 --max-time 12 -o /dev/null -w '%{http_code}' \"$url\" 2>\"$err_file\")",
+    "  rc=$?",
+    "  set -e",
+    "  error_text=$(tr '\\r\\n\\t' '   ' <\"$err_file\" | tail -c 600)",
+    "  rm -f \"$err_file\"",
+    "  printf '%sUrl\\t%s\\n' \"$name\" \"$url\"",
+    "  printf '%sExitCode\\t%s\\n' \"$name\" \"$rc\"",
+    "  printf '%sHttpStatus\\t%s\\n' \"$name\" \"$http_status\"",
+    "  printf '%sError\\t%s\\n' \"$name\" \"$error_text\"",
+    "}",
+    "probe web \"$web_url\"",
+    "probe apiHealth \"$api_url\"",
+  ].join("\n");
+  const result = runTransHostScript(spec.nodeId, script, "", 35);
+  const fields = keyValueLinesFromText(result.stdout);
+  const web = targetHostPublicHttpProbeFromFields("web", fields, webUrl, result.exitCode === 0);
+  const apiHealth = targetHostPublicHttpProbeFromFields("apiHealth", fields, apiHealthUrl, result.exitCode === 0);
+  return {
+    node: spec.nodeId,
+    ready: web.ok === true && apiHealth.ok === true,
+    probeAvailable: result.exitCode === 0 && result.timedOut !== true,
+    web,
+    apiHealth,
+    result: compactRuntimeCommand(result),
+  };
+}
+
+function targetHostPublicHttpProbeFromFields(kind: string, fields: Record<string, string>, fallbackUrl: string, transportOk: boolean): Record<string, unknown> {
+  const exitCode = numericField(fields[`${kind}ExitCode`]);
+  const httpStatus = numericField(fields[`${kind}HttpStatus`]);
+  const error = fields[`${kind}Error`] ?? "";
+  return {
+    kind,
+    url: fields[`${kind}Url`] ?? fallbackUrl,
+    ok: transportOk && exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
+    httpStatus,
+    exitCode,
+    error: error.length > 0 ? error.slice(0, 600) : null,
+  };
+}
+
+function nodeRuntimePublicProbeDiagnostic(publicReady: boolean, targetHost: Record<string, unknown>): Record<string, unknown> {
+  const targetWeb = record(targetHost.web);
+  const targetApiHealth = record(targetHost.apiHealth);
+  const targetReady = targetHost.ready === true;
+  if (!publicReady) {
+    return {
+      kind: "public-entry-probe-failed",
+      affectsUserEntry: true,
+      targetHostReady: targetReady,
+      message: "control-plane public probe failed; treat this as a public endpoint readiness failure before using web-probe closeout evidence.",
+      nextAction: "run hwlab nodes web-probe run for the same node/lane after checking publicProbe.web and publicProbe.apiHealth",
+    };
+  }
+  if (targetHost.probeAvailable !== true) {
+    return {
+      kind: "target-host-public-probe-unavailable",
+      affectsUserEntry: false,
+      targetHostReady: false,
+      message: "control-plane public probe passed, but the target host diagnostic probe could not run; this does not invalidate user-entry evidence.",
+      nextAction: "use control-plane publicProbe and web-probe evidence for closeout; inspect target host SSH/trans health separately if host-side CLI must call the public URL",
+    };
+  }
+  if (!targetReady) {
+    return {
+      kind: "target-host-public-egress-mismatch",
+      affectsUserEntry: false,
+      targetHostReady: false,
+      failed: {
+        web: targetWeb.ok === true ? null : { httpStatus: targetWeb.httpStatus ?? null, exitCode: targetWeb.exitCode ?? null, error: targetWeb.error ?? null },
+        apiHealth: targetApiHealth.ok === true ? null : { httpStatus: targetApiHealth.httpStatus ?? null, exitCode: targetApiHealth.exitCode ?? null, error: targetApiHealth.error ?? null },
+      },
+      message: "control-plane public probe passed, but the target host cannot reach the same public URLs; classify this as target-host egress/hairpin diagnostics, not a public endpoint failure.",
+      nextAction: "use control-plane publicProbe and web-probe evidence for issue closeout; track host-side public URL access separately if hwlab-cli must run on that host",
+    };
+  }
+  return {
+    kind: "public-entry-and-target-host-ok",
+    affectsUserEntry: false,
+    targetHostReady: true,
+    message: "control-plane public probe and target-host public URL probe both passed.",
+    nextAction: null,
+  };
+}
+
 function joinUrlPath(baseUrl: string, suffix: string): string {
  return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`;
 }
@@ -3385,6 +3486,10 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
  const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length;
  const webProbe = record(publicProbes.web);
  const apiProbe = record(publicProbes.apiHealth);
+  const targetHostProbe = record(publicProbes.targetHost);
+  const targetHostWebProbe = record(targetHostProbe.web);
+  const targetHostApiProbe = record(targetHostProbe.apiHealth);
+  const publicProbeDiagnostic = record(publicProbes.diagnostic);
  return {
    ok: status.ok === true,
    command: status.command,
@@ -3437,6 +3542,26 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
      ready: publicProbes.ready === true,
      web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
      apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
+      targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
+        node: targetHostProbe.node ?? status.node ?? null,
+        ready: targetHostProbe.ready === true,
+        probeAvailable: targetHostProbe.probeAvailable === true,
+        web: {
+          url: targetHostWebProbe.url ?? null,
+          ok: targetHostWebProbe.ok === true,
+          httpStatus: targetHostWebProbe.httpStatus ?? null,
+          exitCode: targetHostWebProbe.exitCode ?? null,
+          error: targetHostWebProbe.error ?? null,
+        },
+        apiHealth: {
+          url: targetHostApiProbe.url ?? null,
+          ok: targetHostApiProbe.ok === true,
+          httpStatus: targetHostApiProbe.httpStatus ?? null,
+          exitCode: targetHostApiProbe.exitCode ?? null,
+          error: targetHostApiProbe.error ?? null,
+        },
+      },
+      diagnostic: Object.keys(publicProbeDiagnostic).length === 0 ? null : publicProbeDiagnostic,
    },
    gitMirror: {
      ready: gitMirror.ready === true,
@@ -6212,6 +6337,7 @@ function runNodeWebProbeScript(
    stderrTail: result.stderr.trim().slice(-2000),
    valuesRedacted: true,
  });
+  const issueEvidence = nullableRecord(report?.issueEvidence) ?? nullableRecord(effectiveSummary?.issueEvidence);
  const compactResult = compactCommandResultRedacted(result, [material.password ?? ""]);
  if (outputFailureKind !== null) {
    compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]);
@@ -6230,6 +6356,7 @@ function runNodeWebProbeScript(
    degradedReason,
    failureKind,
    summary: effectiveSummary,
+    issueEvidence,
    probe: report,
    reportLoad: stdoutReport !== null ? { source: "stdout", path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : {
      source: recoveredReport.source,