mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(web-probe): recover observe startup after transport timeout

Browse Source
This commit is contained in:
Codex
2026-06-26 16:04:58 +00:00
parent 2d953cc911
commit 01f8da23ed
4 changed files with 173 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/web-probe-observe.ts
+47 -8
View File
@@ -14,7 +14,7 @@ import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeAuthLoginSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
@@ -1089,6 +1089,10 @@ export function nodeWebProbeProjectManagementConfig(spec: HwlabRuntimeLaneSpec):
return spec.webProbe?.projectManagement ?? null;
}
export function nodeWebProbeAuthLoginConfig(spec: HwlabRuntimeLaneSpec): HwlabRuntimeWebProbeAuthLoginSpec | null {
return spec.webProbe?.authLogin ?? null;
}
export interface NodeWebProbeHostProxyEnv {
readonly envAssignments: string[];
readonly summary: Record<string, unknown>;
@@ -1252,6 +1256,7 @@ export function runNodeWebProbeObserveStart(
const webProbeProxy = nodeWebProbeHostProxyEnv(spec, options.browserProxyMode);
const alertThresholds = nodeWebProbeAlertThresholds(spec);
const projectManagement = nodeWebProbeProjectManagementConfig(spec);
const authLogin = nodeWebProbeAuthLoginConfig(spec);
const runnerEnvAssignments = [
...webProbeProxy.envAssignments,
...webProbeAccountEnvAssignments(),
@@ -1269,6 +1274,14 @@ export function runNodeWebProbeObserveStart(
`UNIDESK_WEB_OBSERVE_BROWSER_PROXY_MODE=${shellQuote(options.browserProxyMode)}`,
`UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON=${shellQuote(JSON.stringify(alertThresholds))}`,
`UNIDESK_WEB_OBSERVE_PROJECT_MANAGEMENT_JSON=${shellQuote(JSON.stringify(projectManagement))}`,
...(authLogin === null
? []
: [
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_ATTEMPTS=${shellQuote(String(authLogin.maxAttempts))}`,
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_REQUEST_TIMEOUT_MS=${shellQuote(String(authLogin.requestTimeoutMs))}`,
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_INITIAL_DELAY_MS=${shellQuote(String(authLogin.initialDelayMs))}`,
`UNIDESK_WEB_OBSERVE_AUTH_LOGIN_MAX_DELAY_MS=${shellQuote(String(authLogin.maxDelayMs))}`,
]),
].join(" ");
const script = [
"set -eu",
@@ -1291,9 +1304,27 @@ export function runNodeWebProbeObserveStart(
].join("\n");
const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds);
const started = parseJsonObject(result.stdout);
const observerId = typeof started?.jobId === "string" ? started.jobId : jobId;
const index = result.exitCode === 0 && started?.ok === true
? upsertWebObserveIndexEntry({
const startOk = result.exitCode === 0 && started?.ok === true;
const recovery = startOk
? null
: readNodeWebProbeObserveRemoteStatus({ ...options, id: jobId, stateDir }, spec, 1, Math.min(options.commandTimeoutSeconds, 30));
const recoveredStatus = !startOk && recovery !== null && recovery.result.exitCode === 0 && recovery.status !== null && recovery.status.ok !== false
? recovery.status
: null;
const effectiveOk = startOk || recoveredStatus !== null;
const observer = startOk ? started : recoveredStatus;
const observerId = typeof started?.jobId === "string"
? started.jobId
: webObserveIdFromStatus(recoveredStatus, { ...options, id: jobId }) ?? jobId;
const degradedReason = !startOk && recoveredStatus !== null
? "web-probe-start-transport-timeout-recovered"
: result.timedOut
? "web-probe-command-timeout"
: result.exitCode !== 0
? "web-probe-observe-start-failed"
: null;
const index = effectiveOk
? upsertWebObserveIndexEntry(startOk ? {
id: observerId,
node: options.node,
lane: options.lane,
@@ -1305,11 +1336,11 @@ export function runNodeWebProbeObserveStart(
pid: typeof started.pid === "number" ? started.pid : null,
startedAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})
} : webObserveIndexEntryFromOptions({ ...options, id: observerId, stateDir }, spec, observerId, recoveredStatus ?? {}))
: null;
return renderWebObserveStartResult({
ok: result.exitCode === 0 && started?.ok === true,
status: result.exitCode === 0 && started?.ok === true ? "started" : "blocked",
ok: effectiveOk,
status: effectiveOk ? "started" : "blocked",
command: `web-probe observe start --node ${options.node} --lane ${options.lane}`,
node: options.node,
lane: options.lane,
@@ -1320,12 +1351,20 @@ export function runNodeWebProbeObserveStart(
projectManagement,
targetPath: options.targetPath,
id: observerId,
degradedReason,
credential,
observer: withWebObserveShortcuts(started, observerId),
observer: withWebObserveShortcuts(observer, observerId),
wrapper: buildWebObserveWrapperForObserveOptions("start", options, spec.workspace, { id: observerId, jobId: observerId, stateDir }),
index,
next: webObserveNextCommands(observerId),
result: compactCommandResultRedacted(result, [material.password ?? ""]),
recovery: recovery === null ? null : {
attempted: true,
ok: recoveredStatus !== null,
reason: degradedReason,
result: compactCommandResultWithStdoutTail(recovery.result),
valuesRedacted: true,
},
valuesRedacted: true,
});
}