fix: run long hwlab web probes asynchronously

2026-06-17 16:00:30 +00:00
parent 03ee13343a
commit 3d30ecaed3
1 changed files with 134 additions and 15 deletions
@@ -4920,21 +4920,8 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
    };
  }
  if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential);
-  const probeArgs = [
-    "node",
-    "scripts/web-live-dom-probe.mjs",
-    "run",
-    "--url", options.url,
-    "--timeout-ms", String(options.timeoutMs),
-    "--wait-after-submit-ms", String(options.waitAfterSubmitMs),
-    "--wait-messages-ms", String(options.waitMessagesMs),
-  ];
-  if (options.waitAgentTerminalMs > 0) probeArgs.push("--wait-agent-terminal-ms", String(options.waitAgentTerminalMs));
-  if (options.traceSampleCount > 0) probeArgs.push("--trace-sample-count", String(options.traceSampleCount), "--trace-sample-interval-ms", String(options.traceSampleIntervalMs));
-  if (options.freshSession) probeArgs.push("--fresh-session");
-  if (!options.cancelRunning) probeArgs.push("--no-cancel-running");
-  if (options.conversationId !== null) probeArgs.push("--conversation-id", options.conversationId);
-  if (options.message !== null) probeArgs.push("--message", options.message);
+  if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential);
+  const probeArgs = nodeWebProbeRunArgs(options, "run");
  const script = [
    "set -eu",
    `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password)} ${probeArgs.map(shellQuote).join(" ")}`,
@@ -4969,6 +4956,138 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
  };
 }

+function nodeWebProbeRunArgs(options: NodeWebProbeRunOptions, command: "run" | "start"): string[] {
+  const probeArgs = [
+    "node",
+    "scripts/web-live-dom-probe.mjs",
+    command,
+    "--url", options.url,
+    "--timeout-ms", String(options.timeoutMs),
+    "--wait-after-submit-ms", String(options.waitAfterSubmitMs),
+    "--wait-messages-ms", String(options.waitMessagesMs),
+  ];
+  if (options.waitAgentTerminalMs > 0) probeArgs.push("--wait-agent-terminal-ms", String(options.waitAgentTerminalMs));
+  if (options.traceSampleCount > 0) probeArgs.push("--trace-sample-count", String(options.traceSampleCount), "--trace-sample-interval-ms", String(options.traceSampleIntervalMs));
+  if (options.freshSession) probeArgs.push("--fresh-session");
+  if (!options.cancelRunning) probeArgs.push("--no-cancel-running");
+  if (options.conversationId !== null) probeArgs.push("--conversation-id", options.conversationId);
+  if (options.message !== null) probeArgs.push("--message", options.message);
+  return probeArgs;
+}
+
+function runNodeWebProbeAsync(
+  options: NodeWebProbeRunOptions,
+  spec: HwlabRuntimeLaneSpec,
+  secretSpec: RuntimeSecretSpec,
+  material: BootstrapAdminPasswordMaterial,
+  credential: Record<string, unknown>,
+): Record<string, unknown> {
+  const startArgs = nodeWebProbeRunArgs(options, "start");
+  const startScript = [
+    "set -eu",
+    `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password ?? "")} ${startArgs.map(shellQuote).join(" ")}`,
+  ].join("\n");
+  const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55);
+  const start = parseJsonObject(startResult.stdout);
+  const jobId = typeof start?.jobId === "string" ? start.jobId : null;
+  if (startResult.exitCode !== 0 || jobId === null) {
+    return {
+      ok: false,
+      status: "blocked",
+      command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
+      node: options.node,
+      lane: options.lane,
+      workspace: spec.workspace,
+      url: options.url,
+      credential,
+      mode: "async-start",
+      commandTimeout: webProbeCommandTimeoutSummary(options, false),
+      degradedReason: startResult.timedOut ? "web-probe-command-timeout" : "web-probe-async-start-failed",
+      start: start ?? null,
+      result: compactCommandResultRedacted(startResult, [material.password ?? ""]),
+      valuesRedacted: true,
+    };
+  }
+  const poll = pollNodeWebProbeJob(options, spec, jobId);
+  const report = record(record(poll.status).report);
+  const probe = compactWebProbeResult(Object.keys(report).length > 0 ? report : null);
+  const passed = probe?.status === "pass";
+  const degradedReason = poll.timedOut
+    ? "web-probe-command-timeout"
+    : typeof probe?.degradedReason === "string"
+      ? probe.degradedReason
+      : poll.status === null
+        ? "web-probe-async-status-failed"
+        : null;
+  return {
+    ok: passed,
+    status: passed ? "pass" : "blocked",
+    command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
+    node: options.node,
+    lane: options.lane,
+    workspace: spec.workspace,
+    url: options.url,
+    credential,
+    mode: "async",
+    commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut),
+    degradedReason,
+    job: {
+      jobId,
+      startedAt: start.startedAt ?? null,
+      polls: poll.polls,
+      elapsedMs: poll.elapsedMs,
+      statusCommand: start.statusCommand ?? `node scripts/web-live-dom-probe.mjs status ${jobId}`,
+    },
+    probe,
+    start: {
+      ok: start.ok === true,
+      status: start.status ?? null,
+      jobId,
+      traceSampling: start.traceSampling ?? null,
+      reportPath: start.reportPath ?? null,
+      screenshotPath: start.screenshotPath ?? null,
+    },
+    statusResult: poll.result === null ? null : compactCommandResult(poll.result),
+    valuesRedacted: true,
+  };
+}
+
+function pollNodeWebProbeJob(options: NodeWebProbeRunOptions, spec: HwlabRuntimeLaneSpec, jobId: string): {
+  status: Record<string, unknown> | null;
+  result: CommandResult | null;
+  polls: number;
+  elapsedMs: number;
+  timedOut: boolean;
+} {
+  const startedAt = Date.now();
+  const deadline = startedAt + options.commandTimeoutSeconds * 1000;
+  let lastStatus: Record<string, unknown> | null = null;
+  let lastResult: CommandResult | null = null;
+  let polls = 0;
+  while (Date.now() < deadline) {
+    polls += 1;
+    const script = `node scripts/web-live-dom-probe.mjs status ${shellQuote(jobId)}`;
+    const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55);
+    lastResult = result;
+    lastStatus = parseJsonObject(result.stdout);
+    const status = typeof lastStatus?.status === "string" ? lastStatus.status : null;
+    if (result.exitCode === 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false };
+    if (result.exitCode !== 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false };
+    sleepSync(Math.min(5000, Math.max(500, deadline - Date.now())));
+  }
+  return { status: lastStatus, result: lastResult, polls, elapsedMs: Date.now() - startedAt, timedOut: true };
+}
+
+function webProbeCommandTimeoutSummary(options: NodeWebProbeRunOptions, timedOut: boolean): Record<string, unknown> {
+  return {
+    seconds: options.commandTimeoutSeconds,
+    autoSeconds: options.commandTimeoutAutoSeconds,
+    userProvided: options.commandTimeoutUserProvided,
+    transportMode: options.commandTimeoutSeconds > 55 ? "async-start-status" : "direct",
+    timedOut,
+  };
+}
+
 function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAdminPasswordMaterial): Record<string, unknown> {
  return {
    username: secretSpec.bootstrapAdminUsername,