diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 1241f733..24d17ae1 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -4920,21 +4920,8 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record }; } if (options.action === "script") return runNodeWebProbeScript(options, spec, secretSpec, material, credential); - const probeArgs = [ - "node", - "scripts/web-live-dom-probe.mjs", - "run", - "--url", options.url, - "--timeout-ms", String(options.timeoutMs), - "--wait-after-submit-ms", String(options.waitAfterSubmitMs), - "--wait-messages-ms", String(options.waitMessagesMs), - ]; - if (options.waitAgentTerminalMs > 0) probeArgs.push("--wait-agent-terminal-ms", String(options.waitAgentTerminalMs)); - if (options.traceSampleCount > 0) probeArgs.push("--trace-sample-count", String(options.traceSampleCount), "--trace-sample-interval-ms", String(options.traceSampleIntervalMs)); - if (options.freshSession) probeArgs.push("--fresh-session"); - if (!options.cancelRunning) probeArgs.push("--no-cancel-running"); - if (options.conversationId !== null) probeArgs.push("--conversation-id", options.conversationId); - if (options.message !== null) probeArgs.push("--message", options.message); + if (options.commandTimeoutSeconds > 55) return runNodeWebProbeAsync(options, spec, secretSpec, material, credential); + const probeArgs = nodeWebProbeRunArgs(options, "run"); const script = [ "set -eu", `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password)} ${probeArgs.map(shellQuote).join(" ")}`, @@ -4969,6 +4956,138 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record }; } +function nodeWebProbeRunArgs(options: NodeWebProbeRunOptions, command: "run" | "start"): string[] { + const probeArgs = [ + "node", + "scripts/web-live-dom-probe.mjs", + command, + "--url", options.url, + "--timeout-ms", String(options.timeoutMs), + "--wait-after-submit-ms", String(options.waitAfterSubmitMs), + "--wait-messages-ms", String(options.waitMessagesMs), + ]; + if (options.waitAgentTerminalMs > 0) probeArgs.push("--wait-agent-terminal-ms", String(options.waitAgentTerminalMs)); + if (options.traceSampleCount > 0) probeArgs.push("--trace-sample-count", String(options.traceSampleCount), "--trace-sample-interval-ms", String(options.traceSampleIntervalMs)); + if (options.freshSession) probeArgs.push("--fresh-session"); + if (!options.cancelRunning) probeArgs.push("--no-cancel-running"); + if (options.conversationId !== null) probeArgs.push("--conversation-id", options.conversationId); + if (options.message !== null) probeArgs.push("--message", options.message); + return probeArgs; +} + +function runNodeWebProbeAsync( + options: NodeWebProbeRunOptions, + spec: HwlabRuntimeLaneSpec, + secretSpec: RuntimeSecretSpec, + material: BootstrapAdminPasswordMaterial, + credential: Record, +): Record { + const startArgs = nodeWebProbeRunArgs(options, "start"); + const startScript = [ + "set -eu", + `HWLAB_WEB_USER=${shellQuote(secretSpec.bootstrapAdminUsername)} HWLAB_WEB_PASS=${shellQuote(material.password ?? "")} ${startArgs.map(shellQuote).join(" ")}`, + ].join("\n"); + const startResult = runTransWorkspaceStdinScript(options.node, spec.workspace, startScript, 55); + const start = parseJsonObject(startResult.stdout); + const jobId = typeof start?.jobId === "string" ? start.jobId : null; + if (startResult.exitCode !== 0 || jobId === null) { + return { + ok: false, + status: "blocked", + command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + credential, + mode: "async-start", + commandTimeout: webProbeCommandTimeoutSummary(options, false), + degradedReason: startResult.timedOut ? "web-probe-command-timeout" : "web-probe-async-start-failed", + start: start ?? null, + result: compactCommandResultRedacted(startResult, [material.password ?? ""]), + valuesRedacted: true, + }; + } + const poll = pollNodeWebProbeJob(options, spec, jobId); + const report = record(record(poll.status).report); + const probe = compactWebProbeResult(Object.keys(report).length > 0 ? report : null); + const passed = probe?.status === "pass"; + const degradedReason = poll.timedOut + ? "web-probe-command-timeout" + : typeof probe?.degradedReason === "string" + ? probe.degradedReason + : poll.status === null + ? "web-probe-async-status-failed" + : null; + return { + ok: passed, + status: passed ? "pass" : "blocked", + command: `hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + workspace: spec.workspace, + url: options.url, + credential, + mode: "async", + commandTimeout: webProbeCommandTimeoutSummary(options, poll.timedOut), + degradedReason, + job: { + jobId, + startedAt: start.startedAt ?? null, + polls: poll.polls, + elapsedMs: poll.elapsedMs, + statusCommand: start.statusCommand ?? `node scripts/web-live-dom-probe.mjs status ${jobId}`, + }, + probe, + start: { + ok: start.ok === true, + status: start.status ?? null, + jobId, + traceSampling: start.traceSampling ?? null, + reportPath: start.reportPath ?? null, + screenshotPath: start.screenshotPath ?? null, + }, + statusResult: poll.result === null ? null : compactCommandResult(poll.result), + valuesRedacted: true, + }; +} + +function pollNodeWebProbeJob(options: NodeWebProbeRunOptions, spec: HwlabRuntimeLaneSpec, jobId: string): { + status: Record | null; + result: CommandResult | null; + polls: number; + elapsedMs: number; + timedOut: boolean; +} { + const startedAt = Date.now(); + const deadline = startedAt + options.commandTimeoutSeconds * 1000; + let lastStatus: Record | null = null; + let lastResult: CommandResult | null = null; + let polls = 0; + while (Date.now() < deadline) { + polls += 1; + const script = `node scripts/web-live-dom-probe.mjs status ${shellQuote(jobId)}`; + const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, 55); + lastResult = result; + lastStatus = parseJsonObject(result.stdout); + const status = typeof lastStatus?.status === "string" ? lastStatus.status : null; + if (result.exitCode === 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false }; + if (result.exitCode !== 0 && status !== "running") return { status: lastStatus, result, polls, elapsedMs: Date.now() - startedAt, timedOut: false }; + sleepSync(Math.min(5000, Math.max(500, deadline - Date.now()))); + } + return { status: lastStatus, result: lastResult, polls, elapsedMs: Date.now() - startedAt, timedOut: true }; +} + +function webProbeCommandTimeoutSummary(options: NodeWebProbeRunOptions, timedOut: boolean): Record { + return { + seconds: options.commandTimeoutSeconds, + autoSeconds: options.commandTimeoutAutoSeconds, + userProvided: options.commandTimeoutUserProvided, + transportMode: options.commandTimeoutSeconds > 55 ? "async-start-status" : "direct", + timedOut, + }; +} + function webProbeCredential(secretSpec: RuntimeSecretSpec, material: BootstrapAdminPasswordMaterial): Record { return { username: secretSpec.bootstrapAdminUsername,