mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix ssh tcp pool transient diagnostics

Browse Source
This commit is contained in:
Codex
2026-06-13 17:19:08 +00:00
parent 1ec60de8bb
commit 2cf767b635
7 changed files with 285 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node.ts
+41 -2
View File
@@ -4,6 +4,7 @@ import { join } from "node:path";
import { repoRoot, rootPath, type Config } from "./config";
import { runCommand, type CommandResult } from "./command";
import { startJob } from "./jobs";
import { classifySshTcpPoolFailure } from "./ssh";
import { runHwlabG14Command } from "./hwlab-g14";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes";
@@ -411,6 +412,25 @@ function compactRuntimeCommand(result: CommandResult): Record<string, unknown> {
};
}
function sshTcpPoolDiagnosticsFromCommand(spec: HwlabRuntimeLaneSpec, result: CommandResult): Record<string, unknown> | null {
if (isCommandSuccess(result)) return null;
const failureKind = classifySshTcpPoolFailure(`${result.stderr}\n${result.stdout}`);
if (failureKind === null) return null;
return {
classification: "ssh-tcp-pool-transient",
failureKind,
providerId: spec.nodeId,
route: spec.nodeKubeRoute,
message: "SSH tcp data pool failed while running the controlled command; treat this as transport/data-pool transient until provider labels and retry say otherwise.",
next: {
poolStatus: `bun scripts/cli.ts debug ssh-pool ${spec.nodeId}`,
retrySmoke: `trans ${spec.nodeId} argv true`,
retryApply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
},
};
}
function nodeRuntimeUnsupportedAction(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
return {
ok: false,
@@ -779,6 +799,7 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
const cleanup = render.location === "local"
? cleanupLocalNodeRuntimeRenderDir(spec, render)
: cleanupNodeRuntimeRenderDir(spec, render.renderDir);
const sshTcpPoolDiagnostics = sshTcpPoolDiagnosticsFromCommand(spec, apply);
return {
ok: isCommandSuccess(apply),
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
@@ -795,10 +816,14 @@ function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOpti
render: compactRuntimeCommand(render.result),
apply: compactRuntimeCommand(apply),
cleanupRenderDir: compactRuntimeCommand(cleanup),
diagnostics: sshTcpPoolDiagnostics,
degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed",
next: scoped.dryRun
? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` }
: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
: {
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
...(sshTcpPoolDiagnostics === null ? {} : { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` }),
},
};
}
@@ -1049,7 +1074,16 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
if (refresh.ok !== true) {
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "failed", sourceCommit, pipelineRun });
const diagnostics = record(refresh.diagnostics);
printNodeRuntimeTriggerProgress(spec, {
stage: "control-plane-refresh",
status: "failed",
sourceCommit,
pipelineRun,
...(diagnostics.classification === "ssh-tcp-pool-transient"
? { reason: "ssh-tcp-pool-transient", failureKind: diagnostics.failureKind ?? null }
: {}),
});
return {
ok: false,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
@@ -1059,7 +1093,12 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
sourceCommit,
pipelineRun,
refresh,
diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
next: {
retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
...(diagnostics.classification === "ssh-tcp-pool-transient" ? { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` } : {}),
},
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun });