fix: improve hwlab probe diagnostics

This commit is contained in:
Codex
2026-06-20 03:59:48 +00:00
parent 7696fddccf
commit 6a88f38ca9
6 changed files with 317 additions and 5 deletions
+1 -1
View File
@@ -75,7 +75,7 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
},
notes: [
"Prefer --script-file for reusable probes; stdin heredocs remain supported for one-off probes.",
"Issue-ready summary is available under probe.summary; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.",
"Issue-ready evidence is available under issueEvidence and summary.issueEvidence; full script report is persisted under probe.reportPath with a SHA-256 fingerprint.",
"Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.",
"Use reloadStable(), gotoCurrentStable(), or safeReload() for bounded retries around page reload/current-URL navigation jitter such as ERR_NETWORK_CHANGED.",
"Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).",
+128 -1
View File
@@ -3349,10 +3349,14 @@ function nullableInteger(value: string): number | null {
function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const web = publicHttpProbe("web", spec.publicWebUrl);
const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
const ready = web.ok === true && apiHealth.ok === true;
const targetHost = nodeRuntimeTargetHostPublicProbeStatus(spec);
return {
ready: web.ok === true && apiHealth.ok === true,
ready,
web,
apiHealth,
targetHost,
diagnostic: nodeRuntimePublicProbeDiagnostic(ready, targetHost),
};
}
@@ -3368,6 +3372,103 @@ function publicHttpProbe(kind: string, url: string): Record<string, unknown> {
};
}
function nodeRuntimeTargetHostPublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const webUrl = spec.publicWebUrl;
const apiHealthUrl = joinUrlPath(spec.publicApiUrl, "/health/live");
const script = [
"set -eu",
`web_url=${shellQuote(webUrl)}`,
`api_url=${shellQuote(apiHealthUrl)}`,
"probe() {",
" name=\"$1\"",
" url=\"$2\"",
" err_file=$(mktemp)",
" set +e",
" http_status=$(curl -k -sS --connect-timeout 5 --max-time 12 -o /dev/null -w '%{http_code}' \"$url\" 2>\"$err_file\")",
" rc=$?",
" set -e",
" error_text=$(tr '\\r\\n\\t' ' ' <\"$err_file\" | tail -c 600)",
" rm -f \"$err_file\"",
" printf '%sUrl\\t%s\\n' \"$name\" \"$url\"",
" printf '%sExitCode\\t%s\\n' \"$name\" \"$rc\"",
" printf '%sHttpStatus\\t%s\\n' \"$name\" \"$http_status\"",
" printf '%sError\\t%s\\n' \"$name\" \"$error_text\"",
"}",
"probe web \"$web_url\"",
"probe apiHealth \"$api_url\"",
].join("\n");
const result = runTransHostScript(spec.nodeId, script, "", 35);
const fields = keyValueLinesFromText(result.stdout);
const web = targetHostPublicHttpProbeFromFields("web", fields, webUrl, result.exitCode === 0);
const apiHealth = targetHostPublicHttpProbeFromFields("apiHealth", fields, apiHealthUrl, result.exitCode === 0);
return {
node: spec.nodeId,
ready: web.ok === true && apiHealth.ok === true,
probeAvailable: result.exitCode === 0 && result.timedOut !== true,
web,
apiHealth,
result: compactRuntimeCommand(result),
};
}
function targetHostPublicHttpProbeFromFields(kind: string, fields: Record<string, string>, fallbackUrl: string, transportOk: boolean): Record<string, unknown> {
const exitCode = numericField(fields[`${kind}ExitCode`]);
const httpStatus = numericField(fields[`${kind}HttpStatus`]);
const error = fields[`${kind}Error`] ?? "";
return {
kind,
url: fields[`${kind}Url`] ?? fallbackUrl,
ok: transportOk && exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
httpStatus,
exitCode,
error: error.length > 0 ? error.slice(0, 600) : null,
};
}
function nodeRuntimePublicProbeDiagnostic(publicReady: boolean, targetHost: Record<string, unknown>): Record<string, unknown> {
const targetWeb = record(targetHost.web);
const targetApiHealth = record(targetHost.apiHealth);
const targetReady = targetHost.ready === true;
if (!publicReady) {
return {
kind: "public-entry-probe-failed",
affectsUserEntry: true,
targetHostReady: targetReady,
message: "control-plane public probe failed; treat this as a public endpoint readiness failure before using web-probe closeout evidence.",
nextAction: "run hwlab nodes web-probe run for the same node/lane after checking publicProbe.web and publicProbe.apiHealth",
};
}
if (targetHost.probeAvailable !== true) {
return {
kind: "target-host-public-probe-unavailable",
affectsUserEntry: false,
targetHostReady: false,
message: "control-plane public probe passed, but the target host diagnostic probe could not run; this does not invalidate user-entry evidence.",
nextAction: "use control-plane publicProbe and web-probe evidence for closeout; inspect target host SSH/trans health separately if host-side CLI must call the public URL",
};
}
if (!targetReady) {
return {
kind: "target-host-public-egress-mismatch",
affectsUserEntry: false,
targetHostReady: false,
failed: {
web: targetWeb.ok === true ? null : { httpStatus: targetWeb.httpStatus ?? null, exitCode: targetWeb.exitCode ?? null, error: targetWeb.error ?? null },
apiHealth: targetApiHealth.ok === true ? null : { httpStatus: targetApiHealth.httpStatus ?? null, exitCode: targetApiHealth.exitCode ?? null, error: targetApiHealth.error ?? null },
},
message: "control-plane public probe passed, but the target host cannot reach the same public URLs; classify this as target-host egress/hairpin diagnostics, not a public endpoint failure.",
nextAction: "use control-plane publicProbe and web-probe evidence for issue closeout; track host-side public URL access separately if hwlab-cli must run on that host",
};
}
return {
kind: "public-entry-and-target-host-ok",
affectsUserEntry: false,
targetHostReady: true,
message: "control-plane public probe and target-host public URL probe both passed.",
nextAction: null,
};
}
function joinUrlPath(baseUrl: string, suffix: string): string {
return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`;
}
@@ -3385,6 +3486,10 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length;
const webProbe = record(publicProbes.web);
const apiProbe = record(publicProbes.apiHealth);
const targetHostProbe = record(publicProbes.targetHost);
const targetHostWebProbe = record(targetHostProbe.web);
const targetHostApiProbe = record(targetHostProbe.apiHealth);
const publicProbeDiagnostic = record(publicProbes.diagnostic);
return {
ok: status.ok === true,
command: status.command,
@@ -3437,6 +3542,26 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
ready: publicProbes.ready === true,
web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
node: targetHostProbe.node ?? status.node ?? null,
ready: targetHostProbe.ready === true,
probeAvailable: targetHostProbe.probeAvailable === true,
web: {
url: targetHostWebProbe.url ?? null,
ok: targetHostWebProbe.ok === true,
httpStatus: targetHostWebProbe.httpStatus ?? null,
exitCode: targetHostWebProbe.exitCode ?? null,
error: targetHostWebProbe.error ?? null,
},
apiHealth: {
url: targetHostApiProbe.url ?? null,
ok: targetHostApiProbe.ok === true,
httpStatus: targetHostApiProbe.httpStatus ?? null,
exitCode: targetHostApiProbe.exitCode ?? null,
error: targetHostApiProbe.error ?? null,
},
},
diagnostic: Object.keys(publicProbeDiagnostic).length === 0 ? null : publicProbeDiagnostic,
},
gitMirror: {
ready: gitMirror.ready === true,
@@ -6212,6 +6337,7 @@ function runNodeWebProbeScript(
stderrTail: result.stderr.trim().slice(-2000),
valuesRedacted: true,
});
const issueEvidence = nullableRecord(report?.issueEvidence) ?? nullableRecord(effectiveSummary?.issueEvidence);
const compactResult = compactCommandResultRedacted(result, [material.password ?? ""]);
if (outputFailureKind !== null) {
compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]);
@@ -6230,6 +6356,7 @@ function runNodeWebProbeScript(
degradedReason,
failureKind,
summary: effectiveSummary,
issueEvidence,
probe: report,
reportLoad: stdoutReport !== null ? { source: "stdout", path: report?.reportPath ?? null, degradedReason: null } : recoveredReport === null ? null : {
source: recoveredReport.source,
@@ -1773,7 +1773,7 @@ function sanitize(value, depth = 0, seen = new WeakSet()) {
if (typeof value === "number" || typeof value === "boolean") return value;
if (typeof value === "bigint") return String(value);
if (typeof value === "function" || typeof value === "symbol") return "[" + typeof value + "]";
if (depth > 8) return "[max-depth]";
if (depth > 12) return "[max-depth]";
if (Array.isArray(value)) return value.slice(0, 200).map((item) => sanitize(item, depth + 1, seen));
if (typeof value === "object") {
if (seen.has(value)) return "[circular]";
@@ -1836,6 +1836,7 @@ async function emit(payload) {
}
function compactStdoutPayload(payload) {
const issueEvidence = compactIssueEvidenceForStdout(payload?.summary?.issueEvidence ?? issueEvidenceFromPayload(payload));
return {
ok: payload?.ok === true,
status: payload?.status ?? null,
@@ -1859,6 +1860,7 @@ function compactStdoutPayload(payload) {
lastScreenshot: compactArtifactForStdout(payload?.lastScreenshot),
readiness: compactJsonForStdout(payload?.readiness),
artifacts: compactArtifactsForStdout(payload?.artifacts),
issueEvidence,
summary: compactSummaryForStdout(payload?.summary),
safety: {
valuesRedacted: true,
@@ -1874,7 +1876,7 @@ function compactScriptForStdout(script) {
const steps = Array.isArray(value.steps) ? value.steps : [];
return {
ok: value.ok === true,
result: compactJsonForStdout(value.result),
result: compactJsonForEvidence(value.result),
stepCount: steps.length,
};
}
@@ -1932,6 +1934,7 @@ function compactSummaryForStdout(summary) {
apiMatrix: compactApiMatrixForStdout(value.apiMatrix),
stepCount: Number.isFinite(value.stepCount) ? value.stepCount : null,
lastStep: compactStepForStdout(value.lastStep),
issueEvidence: compactIssueEvidenceForStdout(value.issueEvidence),
valuesRedacted: true,
};
}
@@ -2000,6 +2003,98 @@ function compactJsonForStdout(value, depth = 0) {
return compactText(String(value), 180);
}
function issueEvidenceFromPayload(payload) {
const steps = publicSteps();
const artifacts = Array.isArray(payload?.artifacts?.items) ? payload.artifacts.items : artifactRecords;
const screenshots = artifacts
.filter((item) => item && typeof item === "object" && item.kind === "screenshot")
.slice(-5)
.map((item) => compactArtifactForStdout(item))
.filter(Boolean);
const ok = payload?.ok === true;
const degradedReason = ok ? null : payload?.error ?? payload?.failureKind ?? "web-probe-script-failed";
const failureKind = ok ? null : classifyIssueFailureKind(payload?.failureKind ?? payload?.error ?? degradedReason, payload?.errorMessage);
const failedCondition = ok ? null : payload?.errorMessage ?? payload?.error ?? payload?.failureKind ?? "script did not pass";
const script = payload?.script && typeof payload.script === "object" ? payload.script : {};
return {
ok,
status: payload?.status ?? null,
degradedReason,
failureKind,
failedCondition,
nextAction: ok ? null : issueNextAction(failureKind, payload),
baseUrl: payload?.baseUrl ?? null,
finalUrl: payload?.finalUrl ?? payload?.lastUrl ?? null,
lastUrl: payload?.lastUrl ?? payload?.finalUrl ?? null,
scriptSha256: payload?.scriptSha256 ?? null,
runDir,
reportPath: payload?.reportPath ?? null,
reportSha256: payload?.reportSha256 ?? null,
result: compactJsonForEvidence(script.result),
apiMatrix: compactApiMatrixForStdout(latestApiMatrixFromSteps(steps)),
lastStep: steps.length > 0 ? compactStepForEvidence(steps[steps.length - 1]) : null,
steps: steps.slice(-3).map((step) => compactStepForEvidence(step)),
lastScreenshot: compactArtifactForStdout(payload?.lastScreenshot),
screenshots,
valuesRedacted: true,
};
}
function compactIssueEvidenceForStdout(value) {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
return {
ok: value.ok === true,
status: value.status ?? null,
degradedReason: typeof value.degradedReason === "string" ? compactText(value.degradedReason, 600) : value.degradedReason ?? null,
failureKind: typeof value.failureKind === "string" ? compactText(value.failureKind, 300) : value.failureKind ?? null,
failedCondition: typeof value.failedCondition === "string" ? compactText(value.failedCondition, 1200) : value.failedCondition ?? null,
nextAction: typeof value.nextAction === "string" ? compactText(value.nextAction, 1200) : value.nextAction ?? null,
baseUrl: value.baseUrl ?? null,
finalUrl: value.finalUrl ?? null,
lastUrl: value.lastUrl ?? null,
scriptSha256: value.scriptSha256 ?? null,
runDir: value.runDir ?? runDir,
reportPath: value.reportPath ?? null,
reportSha256: value.reportSha256 ?? null,
result: compactJsonForEvidence(value.result),
apiMatrix: compactApiMatrixForStdout(value.apiMatrix),
lastStep: compactStepForEvidence(value.lastStep),
steps: Array.isArray(value.steps) ? value.steps.slice(-3).map((step) => compactStepForEvidence(step)) : [],
lastScreenshot: compactArtifactForStdout(value.lastScreenshot),
screenshots: Array.isArray(value.screenshots) ? value.screenshots.slice(-5).map((item) => compactArtifactForStdout(item)).filter(Boolean) : [],
valuesRedacted: true,
};
}
function compactStepForEvidence(step) {
if (!step || typeof step !== "object" || Array.isArray(step)) return null;
return {
index: Number.isFinite(step.index) ? step.index : null,
name: typeof step.name === "string" ? step.name : null,
ok: typeof step.ok === "boolean" ? step.ok : null,
atMs: Number.isFinite(step.atMs) ? step.atMs : null,
data: compactJsonForEvidence(step.data),
};
}
function compactJsonForEvidence(value, depth = 0) {
if (value === null || value === undefined) return value ?? null;
if (typeof value === "string") return compactText(value, 600);
if (typeof value === "number" || typeof value === "boolean") return value;
if (typeof value === "bigint") return String(value);
if (typeof value === "function" || typeof value === "symbol") return "[" + typeof value + "]";
if (depth >= 8) return "[max-depth]";
if (Array.isArray(value)) return value.slice(0, 16).map((item) => compactJsonForEvidence(item, depth + 1));
if (typeof value === "object") {
const out = {};
for (const [key, nested] of Object.entries(value).slice(0, 32)) {
out[key] = compactJsonForEvidence(nested, depth + 1);
}
return out;
}
return compactText(String(value), 600);
}
function compactText(value, maxChars) {
return redactString(String(value)).replace(/\s+/gu, " ").trim().slice(0, maxChars);
}
@@ -2037,6 +2132,7 @@ function scriptIssueSummary(payload) {
apiMatrix,
stepCount: stepRecords.length,
lastStep: stepRecords.length > 0 ? deepClonePlain(stepRecords[stepRecords.length - 1]) : null,
issueEvidence: issueEvidenceFromPayload(payload),
valuesRedacted: true,
};
}
+85 -1
View File
@@ -12,10 +12,12 @@ function nullableRecord(value: unknown): Record<string, unknown> | null {
export function compactWebProbeScriptResult(report: Record<string, unknown> | null): Record<string, unknown> | null {
if (report === null) return null;
const summary = compactIssueSummary(record(report.summary));
const issueEvidence = compactIssueEvidence(report.issueEvidence ?? summary.issueEvidence ?? fallbackIssueEvidence(report, summary));
return {
ok: report.ok === true,
status: typeof report.status === "string" ? report.status : null,
summary,
issueEvidence,
baseUrl: typeof report.baseUrl === "string" ? report.baseUrl : null,
finalUrl: typeof report.finalUrl === "string" ? report.finalUrl : null,
lastUrl: typeof report.lastUrl === "string" ? report.lastUrl : null,
@@ -41,7 +43,7 @@ function compactWebProbeScriptBlock(value: unknown): Record<string, unknown> {
const script = record(value);
return {
ok: script.ok === true,
result: compactJsonForIssue(script.result),
result: compactJsonForEvidence(script.result),
stepCount: Array.isArray(script.steps) ? script.steps.length : null,
};
}
@@ -80,6 +82,7 @@ function compactIssueSummary(value: Record<string, unknown>): Record<string, unk
apiMatrix: compactApiMatrixSummary(value.apiMatrix),
stepCount: typeof value.stepCount === "number" ? value.stepCount : null,
lastStep: compactStepForIssue(value.lastStep),
issueEvidence: compactIssueEvidence(value.issueEvidence),
valuesRedacted: value.valuesRedacted === true,
};
}
@@ -100,6 +103,87 @@ function compactJsonForIssue(value: unknown, depth = 0): unknown {
return String(value).slice(0, 240);
}
function fallbackIssueEvidence(report: Record<string, unknown>, summary: Record<string, unknown>): Record<string, unknown> {
const script = record(report.script);
return {
ok: report.ok === true,
status: typeof report.status === "string" ? report.status : summary.status ?? null,
degradedReason: summary.degradedReason ?? null,
failureKind: summary.failureKind ?? (typeof report.failureKind === "string" ? report.failureKind : null),
failedCondition: summary.failedCondition ?? (typeof report.errorMessage === "string" ? report.errorMessage : null),
nextAction: summary.nextAction ?? null,
baseUrl: typeof report.baseUrl === "string" ? report.baseUrl : null,
finalUrl: typeof report.finalUrl === "string" ? report.finalUrl : summary.finalUrl ?? null,
lastUrl: typeof report.lastUrl === "string" ? report.lastUrl : summary.lastUrl ?? null,
scriptSha256: typeof report.scriptSha256 === "string" ? report.scriptSha256 : summary.scriptSha256 ?? null,
runDir: typeof report.runDir === "string" ? report.runDir : summary.runDir ?? null,
reportPath: typeof report.reportPath === "string" ? report.reportPath : summary.reportPath ?? null,
reportSha256: typeof report.reportSha256 === "string" ? report.reportSha256 : summary.reportSha256 ?? null,
result: script.result,
apiMatrix: summary.apiMatrix ?? null,
lastStep: summary.lastStep ?? null,
steps: Array.isArray(report.steps) ? report.steps.slice(-3) : [],
lastScreenshot: report.lastScreenshot ?? summary.lastScreenshot ?? null,
screenshots: summary.screenshots ?? [],
valuesRedacted: true,
};
}
function compactIssueEvidence(value: unknown): Record<string, unknown> | null {
const evidence = nullableRecord(value);
if (evidence === null) return null;
return {
ok: evidence.ok === true,
status: typeof evidence.status === "string" ? evidence.status : null,
degradedReason: typeof evidence.degradedReason === "string" ? evidence.degradedReason : null,
failureKind: typeof evidence.failureKind === "string" ? evidence.failureKind : null,
failedCondition: typeof evidence.failedCondition === "string" ? evidence.failedCondition : null,
nextAction: typeof evidence.nextAction === "string" ? evidence.nextAction : null,
baseUrl: typeof evidence.baseUrl === "string" ? evidence.baseUrl : null,
finalUrl: typeof evidence.finalUrl === "string" ? evidence.finalUrl : null,
lastUrl: typeof evidence.lastUrl === "string" ? evidence.lastUrl : null,
scriptSha256: typeof evidence.scriptSha256 === "string" ? evidence.scriptSha256 : null,
runDir: typeof evidence.runDir === "string" ? evidence.runDir : null,
reportPath: typeof evidence.reportPath === "string" ? evidence.reportPath : null,
reportSha256: typeof evidence.reportSha256 === "string" ? evidence.reportSha256 : null,
result: compactJsonForEvidence(evidence.result),
apiMatrix: compactApiMatrixSummary(evidence.apiMatrix),
lastStep: compactStepForEvidence(evidence.lastStep),
steps: Array.isArray(evidence.steps) ? evidence.steps.slice(-3).map(compactStepForEvidence).filter((item): item is Record<string, unknown> => item !== null) : [],
lastScreenshot: nullableRecord(evidence.lastScreenshot),
screenshots: Array.isArray(evidence.screenshots) ? evidence.screenshots.slice(-5).map(nullableRecord).filter((item): item is Record<string, unknown> => item !== null) : [],
valuesRedacted: evidence.valuesRedacted === true,
};
}
function compactStepForEvidence(value: unknown): Record<string, unknown> | null {
const step = nullableRecord(value);
if (step === null) return null;
return {
index: typeof step.index === "number" ? step.index : null,
name: typeof step.name === "string" ? step.name : null,
ok: typeof step.ok === "boolean" ? step.ok : null,
atMs: typeof step.atMs === "number" ? step.atMs : null,
data: compactJsonForEvidence(step.data),
};
}
function compactJsonForEvidence(value: unknown, depth = 0): unknown {
if (value === null || value === undefined) return value ?? null;
if (typeof value === "string") return value.replace(/\s+/gu, " ").trim().slice(0, 600);
if (typeof value === "number" || typeof value === "boolean") return value;
if (depth >= 8) return "[max-depth]";
if (Array.isArray(value)) return value.slice(0, 16).map((item) => compactJsonForEvidence(item, depth + 1));
if (typeof value === "object") {
const out: Record<string, unknown> = {};
for (const [key, nested] of Object.entries(value as Record<string, unknown>).slice(0, 32)) {
out[key] = compactJsonForEvidence(nested, depth + 1);
}
return out;
}
return String(value).slice(0, 600);
}
function compactStepForIssue(value: unknown): Record<string, unknown> | null {
const step = nullableRecord(value);
if (step === null) return null;