refactor: split control-plane cli modules
This commit is contained in:
Codex
@@ -0,0 +1,529 @@
|
||||
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. git-mirror module for scripts/src/hwlab-node-impl.ts.
|
||||
|
||||
// Moved mechanically from scripts/src/hwlab-node-impl.ts:2800-3280 for #903.
|
||||
|
||||
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
||||
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { repoRoot, rootPath, type Config } from "../config";
|
||||
import { runCommand, type CommandResult } from "../command";
|
||||
import { startJob } from "../jobs";
|
||||
import { classifySshTcpPoolFailure } from "../ssh";
|
||||
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
||||
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
||||
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
||||
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
||||
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
||||
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
||||
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
||||
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
||||
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import { formatElapsedMs, isCommandSuccess, nodeRuntimePipelineRunName, nodeRuntimeRerunPipelineRunName, resolveNodeRuntimeLaneHead, shortValue } from "./cleanup";
|
||||
import { nodeRuntimeApply } from "./control-actions";
|
||||
import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS } from "./entry";
|
||||
import { parseNodeScopedDelegatedOptions } from "./plan";
|
||||
import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render";
|
||||
import { compactRuntimeCommand } from "./runtime-common";
|
||||
import { compactNodeRuntimeGitMirrorObservation, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
|
||||
import { record } from "./utils";
|
||||
import { webObserveTable } from "./web-observe-render";
|
||||
import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe";
|
||||
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
||||
|
||||
export function nodeRuntimeTriggerCurrentOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
||||
const result = nodeRuntimeTriggerCurrent(scoped);
|
||||
if (nodeScopedFullOutput(scoped)) return result;
|
||||
return withNodeRuntimeTriggerRendered(result, scoped);
|
||||
}
|
||||
|
||||
export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
||||
const spec = scoped.spec;
|
||||
const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
|
||||
const triggerStartedAt = Date.now();
|
||||
const triggerElapsedMs = () => Date.now() - triggerStartedAt;
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
|
||||
const head = resolveNodeRuntimeLaneHead(spec);
|
||||
const sourceCommit = head.sourceCommit;
|
||||
if (sourceCommit === null) {
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" });
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
phase: "source-head",
|
||||
degradedReason: "node-runtime-source-head-unresolved",
|
||||
headProbe: compactRuntimeCommand(head.result),
|
||||
};
|
||||
}
|
||||
const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
|
||||
const pipelineRun = scoped.rerun ? nodeRuntimeRerunPipelineRunName(spec, sourceCommit) : basePipelineRun;
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun });
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun });
|
||||
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null });
|
||||
if (scoped.dryRun) {
|
||||
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
|
||||
return {
|
||||
ok: true,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "dry-run",
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
rerunOf: scoped.rerun ? basePipelineRun : null,
|
||||
rerun: scoped.rerun,
|
||||
before,
|
||||
gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
|
||||
manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
|
||||
next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
|
||||
const pipelineWait = before.status === "Unknown"
|
||||
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
||||
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
||||
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
||||
})
|
||||
: { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 };
|
||||
const waitedPipelineRun = record(pipelineWait.pipelineRun);
|
||||
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
|
||||
const postFlush = waitedPipelineRun.status === "True"
|
||||
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
||||
: null;
|
||||
const pipelinePending = pipelineWait.status === "pending";
|
||||
const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true);
|
||||
const elapsedMs = triggerElapsedMs();
|
||||
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
|
||||
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "confirmed-trigger",
|
||||
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
||||
warning: pipelinePending ? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
|
||||
triggerElapsedMs: elapsedMs,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
before,
|
||||
pipelineWait,
|
||||
pipelineFailureSummary,
|
||||
postFlush,
|
||||
skipped: true,
|
||||
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
|
||||
skipPolicy: "source-commit-only",
|
||||
skipExplanation: "An existing PipelineRun for this HWLAB source commit was reused. If UniDesk node/lane YAML or other render inputs changed without a HWLAB source commit change, rerun the controlled publish with --rerun.",
|
||||
rerunAvailable: true,
|
||||
rerunCommand: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
|
||||
degradedReason: ok ? undefined : postFlush !== null && postFlush.ok !== true ? "node-runtime-git-mirror-post-flush-failed" : "node-runtime-existing-pipelinerun-not-succeeded",
|
||||
next: {
|
||||
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}`,
|
||||
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
|
||||
},
|
||||
};
|
||||
}
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun });
|
||||
const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit, pipelineRun);
|
||||
if (gitMirror.ok !== true) {
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "failed", sourceCommit, pipelineRun, reason: gitMirror.degradedReason ?? null });
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
phase: "git-mirror-pre-sync",
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
before,
|
||||
gitMirror,
|
||||
degradedReason: "node-runtime-git-mirror-pre-sync-failed",
|
||||
};
|
||||
}
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "succeeded", sourceCommit, pipelineRun });
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
|
||||
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
|
||||
if (refresh.ok !== true) {
|
||||
const diagnostics = record(refresh.diagnostics);
|
||||
printNodeRuntimeTriggerProgress(spec, {
|
||||
stage: "control-plane-refresh",
|
||||
status: "failed",
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
...(diagnostics.classification === "ssh-tcp-pool-transient"
|
||||
? { reason: "ssh-tcp-pool-transient", failureKind: diagnostics.failureKind ?? null }
|
||||
: {}),
|
||||
});
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
phase: "control-plane-apply",
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
refresh,
|
||||
diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
|
||||
degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
|
||||
next: {
|
||||
retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
||||
...(diagnostics.classification === "ssh-tcp-pool-transient" ? { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun });
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun });
|
||||
const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun);
|
||||
const after = getNodeRuntimePipelineRun(spec, pipelineRun);
|
||||
const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True");
|
||||
const createOk = isCommandSuccess(create) || createObserved;
|
||||
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
|
||||
const pipelineWait = createOk
|
||||
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
||||
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
||||
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
||||
})
|
||||
: null;
|
||||
const waitedPipelineRun = record(pipelineWait?.pipelineRun);
|
||||
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
|
||||
const postFlush = waitedPipelineRun.status === "True"
|
||||
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
||||
: null;
|
||||
const pipelineReady = pipelineWait !== null && pipelineWait.ok === true;
|
||||
const postFlushOk = postFlush === null || postFlush.ok === true;
|
||||
const pipelinePending = pipelineWait !== null && pipelineWait.status === "pending";
|
||||
const ok = createOk && (pipelineReady || pipelinePending) && postFlushOk;
|
||||
const elapsedMs = triggerElapsedMs();
|
||||
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
|
||||
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
|
||||
return {
|
||||
ok,
|
||||
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
||||
node: scoped.node,
|
||||
lane: scoped.lane,
|
||||
mode: "confirmed-trigger",
|
||||
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
||||
warning: pipelinePending ? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
|
||||
triggerElapsedMs: elapsedMs,
|
||||
mutation: createOk,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
rerunOf: scoped.rerun ? basePipelineRun : null,
|
||||
rerun: scoped.rerun,
|
||||
before,
|
||||
gitMirror,
|
||||
refresh,
|
||||
create: compactRuntimeCommand(create),
|
||||
after,
|
||||
pipelineWait,
|
||||
pipelineFailureSummary,
|
||||
postFlush,
|
||||
createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
|
||||
degradedReason: ok
|
||||
? undefined
|
||||
: !createOk
|
||||
? "node-runtime-pipelinerun-create-failed"
|
||||
: !pipelineReady
|
||||
? "node-runtime-pipelinerun-not-succeeded"
|
||||
: "node-runtime-git-mirror-post-flush-failed",
|
||||
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeRuntimeCicdWaitSeconds(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): number {
|
||||
return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS);
|
||||
}
|
||||
|
||||
export function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record<string, unknown> {
|
||||
const wait = record(pipelineWait);
|
||||
return {
|
||||
code: "node-runtime-cicd-wait-over-120s",
|
||||
message: `PipelineRun ${pipelineRun} is still non-terminal after ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; this is a severe timeout and requires further investigation of Tekton TaskRuns/Pods instead of treating it as normal wait time.`,
|
||||
waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
|
||||
elapsedMs: wait.elapsedMs ?? null,
|
||||
polls: wait.polls ?? null,
|
||||
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
||||
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeRuntimeTriggerElapsedWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, elapsedMs: number): Record<string, unknown> | null {
|
||||
if (elapsedMs < NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS) return null;
|
||||
return {
|
||||
code: "node-runtime-trigger-over-120s",
|
||||
message: `trigger-current total elapsed time exceeded ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; this is a severe timeout even if the PipelineRun eventually succeeded and requires follow-up investigation of slow control-plane, Tekton and git-mirror stages.`,
|
||||
thresholdMs: NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS,
|
||||
elapsedMs,
|
||||
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
||||
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function withNodeRuntimeTriggerRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
||||
const pipelineWait = record(result.pipelineWait);
|
||||
const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before);
|
||||
const warning = record(result.warning ?? pipelineWait.warning);
|
||||
const postFlush = record(result.postFlush);
|
||||
const gitMirror = record(result.gitMirror);
|
||||
const gitMirrorSummary = record(postFlush.afterSummary ?? postFlush.beforeSummary ?? gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
|
||||
const refresh = record(result.refresh);
|
||||
const create = record(result.create);
|
||||
const next = record(result.next);
|
||||
const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-";
|
||||
const completion = result.completion ?? (result.ok === true ? "completed" : "failed");
|
||||
const renderedText = [
|
||||
"hwlab nodes control-plane trigger-current",
|
||||
"",
|
||||
webObserveTable(
|
||||
["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"],
|
||||
[[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["STAGE", "STATUS", "DETAIL"],
|
||||
[
|
||||
["git-mirror", gitMirror.ok === true ? "ok" : gitMirror.ok === false ? "failed" : "-", webObserveText(gitMirror.mode ?? gitMirror.degradedReason)],
|
||||
["refresh", refresh.ok === true ? "ok" : refresh.ok === false ? "failed" : "-", webObserveText(refresh.degradedReason)],
|
||||
["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`],
|
||||
["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`],
|
||||
["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)],
|
||||
["total", result.triggerElapsedMs === undefined ? "-" : "elapsed", formatElapsedMs(result.triggerElapsedMs)],
|
||||
],
|
||||
),
|
||||
"",
|
||||
Object.keys(warning).length === 0
|
||||
? "WARNING\n-"
|
||||
: webObserveTable(
|
||||
["CODE", "MESSAGE"],
|
||||
[[webObserveText(warning.code ?? "warning"), webObserveText(warning.message ?? "PipelineRun is still active after the default wait window.")]],
|
||||
),
|
||||
"",
|
||||
Object.keys(warning).length === 0
|
||||
? "INSPECT\n-"
|
||||
: webObserveTable(
|
||||
["CHECK", "COMMAND"],
|
||||
[
|
||||
["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
|
||||
["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)],
|
||||
],
|
||||
),
|
||||
"",
|
||||
Object.keys(gitMirrorSummary).length === 0
|
||||
? "GIT_MIRROR\n-"
|
||||
: webObserveTable(
|
||||
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
||||
[[
|
||||
shortValue(gitMirrorSummary.localSource),
|
||||
shortValue(gitMirrorSummary.githubSource),
|
||||
shortValue(gitMirrorSummary.localGitops),
|
||||
shortValue(gitMirrorSummary.githubGitops),
|
||||
webObserveText(gitMirrorSummary.pendingFlush),
|
||||
webObserveText(gitMirrorSummary.githubInSync),
|
||||
]],
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun}`}`,
|
||||
"",
|
||||
"Disclosure:",
|
||||
" default view is a bounded CICD summary; use --full or --raw for the complete JSON payload.",
|
||||
` ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s is the severe timeout warning threshold for both PipelineRun wait and total trigger elapsed time.`,
|
||||
].join("\n");
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
|
||||
export function withNodeRuntimeControlPlanePlanRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
||||
const checks = record(result.checks);
|
||||
const next = record(result.next);
|
||||
const renderedText = [
|
||||
"hwlab nodes control-plane plan",
|
||||
"",
|
||||
webObserveTable(
|
||||
["NODE", "LANE", "STATUS", "MODE", "NAMESPACE", "EXT_PG", "PUBLIC"],
|
||||
[[
|
||||
scoped.node,
|
||||
scoped.lane,
|
||||
result.ok === true ? "ok" : "failed",
|
||||
webObserveText(result.mode),
|
||||
webObserveText(checks.runtimeNamespace),
|
||||
webObserveText(checks.externalPostgresDeclared),
|
||||
webObserveText(checks.publicExposureDeclared),
|
||||
]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["CHECK", "VALUE"],
|
||||
[
|
||||
["node-scoped-target", webObserveText(checks.nodeScopedTargetConfigured)],
|
||||
["local-postgres-absent", webObserveText(checks.localPostgresExpectedAbsent)],
|
||||
["secret-values-printed", webObserveText(checks.secretValuesPrinted)],
|
||||
],
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`}`,
|
||||
"",
|
||||
"Disclosure:",
|
||||
" default view is a bounded control-plane plan summary; use --full or --raw for the complete JSON payload.",
|
||||
].join("\n");
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
|
||||
export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
||||
const pipelineRun = record(result.pipelineRun);
|
||||
const pipelineRunDiagnostics = record(pipelineRun.diagnostics);
|
||||
const argo = record(result.argo);
|
||||
const runtime = record(result.runtime);
|
||||
const publicProbe = record(result.publicProbe);
|
||||
const gitMirror = record(result.gitMirror);
|
||||
const next = record(result.next);
|
||||
const diagnostic = record(publicProbe.diagnostic);
|
||||
const notReadyWorkloads = Array.isArray(runtime.notReadyWorkloads) ? runtime.notReadyWorkloads.map((item) => webObserveText(item)).filter(Boolean) : [];
|
||||
const runtimeDetail = [
|
||||
`workloads=${webObserveText(runtime.workloadReady)}`,
|
||||
`pg=${webObserveText(runtime.externalPostgresReady ?? runtime.localPostgresReady)}`,
|
||||
...(notReadyWorkloads.length > 0 ? [`notReady=${webObserveShort(notReadyWorkloads.join(","), 80)}`] : []),
|
||||
].join(" ");
|
||||
const failedTaskRuns = Array.isArray(pipelineRunDiagnostics.failedTaskRuns) ? pipelineRunDiagnostics.failedTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
|
||||
const pendingTaskRuns = Array.isArray(pipelineRunDiagnostics.pendingTaskRuns) ? pipelineRunDiagnostics.pendingTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
|
||||
const pipelineDetail = [
|
||||
webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 50),
|
||||
...(failedTaskRuns.length > 0 ? [`failed=${webObserveShort(failedTaskRuns.join(","), 80)}`] : []),
|
||||
...(pendingTaskRuns.length > 0 ? [`pending=${webObserveShort(pendingTaskRuns.join(","), 80)}`] : []),
|
||||
].filter(Boolean).join(" ");
|
||||
const status = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
||||
const renderedText = [
|
||||
"hwlab nodes control-plane status",
|
||||
"",
|
||||
webObserveTable(
|
||||
["NODE", "LANE", "STATUS", "REASON", "SOURCE", "PIPELINERUN"],
|
||||
[[
|
||||
scoped.node,
|
||||
scoped.lane,
|
||||
status,
|
||||
webObserveShort(webObserveText(result.degradedReason), 40),
|
||||
shortValue(result.sourceCommit),
|
||||
webObserveShort(webObserveText(pipelineRun.name), 36),
|
||||
]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["STAGE", "STATUS", "DETAIL"],
|
||||
[
|
||||
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", pipelineDetail],
|
||||
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
|
||||
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
|
||||
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(diagnostic.kind ?? diagnostic.message), 80)],
|
||||
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
||||
],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS"],
|
||||
[[
|
||||
shortValue(gitMirror.localSource),
|
||||
shortValue(gitMirror.githubSource),
|
||||
shortValue(gitMirror.localGitops),
|
||||
shortValue(gitMirror.githubGitops),
|
||||
]],
|
||||
),
|
||||
"",
|
||||
"NEXT",
|
||||
` ${result.nextAction ?? next.full ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --full`}`,
|
||||
"",
|
||||
"Disclosure:",
|
||||
" default view is a bounded control-plane status summary; use --full or --raw for the complete JSON payload.",
|
||||
].join("\n");
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
|
||||
export function withNodeRuntimeControlPlaneStatusFullRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
||||
const summary = record(result.summary);
|
||||
const pipelineRun = record(summary.pipelineRun);
|
||||
const argo = record(summary.argo);
|
||||
const runtimeSummary = record(summary.runtime);
|
||||
const publicProbe = record(summary.publicProbe);
|
||||
const gitMirror = record(summary.gitMirror);
|
||||
const runtime = record(result.runtime);
|
||||
const bridge = record(runtime.externalPostgresBridge);
|
||||
const secrets = record(runtime.externalPostgresSecrets);
|
||||
const next = record(summary.next);
|
||||
const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
|
||||
const workloadRows = workloadReadiness.length === 0
|
||||
? [["-", "-", "-", "-", "-"]]
|
||||
: workloadReadiness.slice(0, 18).map((item) => [
|
||||
webObserveText(item.ref),
|
||||
item.ready === true ? "ok" : "not-ready",
|
||||
webObserveText(item.readyReplicas),
|
||||
webObserveText(item.currentReplicas),
|
||||
webObserveText(item.desiredReplicas),
|
||||
]);
|
||||
const bridgeRows = bridge.required === false
|
||||
? [["not-required", "-", "-", "-", "-"]]
|
||||
: [[
|
||||
bridge.ready === true ? "ok" : "failed",
|
||||
webObserveText(bridge.serviceName),
|
||||
webObserveText(bridge.endpointAddress),
|
||||
webObserveText(bridge.port),
|
||||
`${webObserveText(bridge.expectedEndpointAddress)}:${webObserveText(bridge.expectedPort)}`,
|
||||
]];
|
||||
const secretRows = nodeRuntimeExternalPostgresSecretRows(secrets);
|
||||
const status = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
||||
const renderedText = [
|
||||
"hwlab nodes control-plane status --full",
|
||||
"",
|
||||
webObserveTable(
|
||||
["NODE", "LANE", "STATUS", "REASON", "SOURCE", "PIPELINERUN"],
|
||||
[[
|
||||
scoped.node,
|
||||
scoped.lane,
|
||||
status,
|
||||
webObserveShort(webObserveText(summary.degradedReason ?? result.degradedReason), 40),
|
||||
shortValue(summary.sourceCommit ?? result.sourceCommit),
|
||||
webObserveShort(webObserveText(pipelineRun.name ?? result.pipelineRun), 36),
|
||||
]],
|
||||
),
|
||||
"",
|
||||
webObserveTable(
|
||||
["STAGE", "STATUS", "DETAIL"],
|
||||
[
|
||||
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 80)],
|
||||
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
|
||||
["runtime", runtimeSummary.ready === true ? "ok" : "failed", `namespace=${webObserveText(runtimeSummary.namespace)} workloads=${webObserveText(runtimeSummary.workloadReady)} pg=${webObserveText(runtimeSummary.externalPostgresReady ?? runtimeSummary.localPostgresReady)}`],
|
||||
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(record(publicProbe.diagnostic).kind ?? record(publicProbe.diagnostic).message), 80)],
|
||||
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
||||
],
|
||||
),
|
||||
"",
|
||||
"RUNTIME_WORKLOADS",
|
||||
webObserveTable(["REF", "READY", "READY_REPLICAS", "CURRENT", "DESIRED"], workloadRows),
|
||||
...(workloadReadiness.length > workloadRows.length ? [` ... ${workloadReadiness.length - workloadRows.length} more workloads omitted; use --raw for complete JSON.`] : []),
|
||||
"",
|
||||
"EXTERNAL_POSTGRES_BRIDGE",
|
||||
webObserveTable(["STATUS", "SERVICE", "ENDPOINT", "PORT", "EXPECTED"], bridgeRows),
|
||||
"",
|
||||
"EXTERNAL_POSTGRES_SECRETS",
|
||||
webObserveTable(["ROLE", "SECRET", "KEY", "PRESENT", "VALUES_PRINTED"], secretRows),
|
||||
"",
|
||||
"NEXT",
|
||||
` ${summary.nextAction ?? next.full ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --full`}`,
|
||||
` bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --raw`,
|
||||
"",
|
||||
"Disclosure:",
|
||||
" --full status is a bounded runtime-workloads drill-down view; use --raw only when complete JSON is required.",
|
||||
" Secret evidence is limited to object metadata, key names, presence, and valuesPrinted=false.",
|
||||
].join("\n");
|
||||
return { ...result, renderedText, contentType: "text/plain" };
|
||||
}
|
||||
Reference in New Issue
Block a user