Lyon
532 lines
31 KiB
TypeScript
532 lines
31 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. git-mirror module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:2800-3280 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import { formatElapsedMs, isCommandSuccess, nodeRuntimePipelineRunName, nodeRuntimeRerunPipelineRunName, resolveNodeRuntimeLaneHead, shortValue } from "./cleanup";
|
|
import { nodeRuntimeApply } from "./control-actions";
|
|
import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS } from "./entry";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render";
|
|
import { compactRuntimeCommand } from "./runtime-common";
|
|
import { compactNodeRuntimeGitMirrorObservation, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
|
|
import { record } from "./utils";
|
|
import { webObserveTable } from "./web-observe-render";
|
|
import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe";
|
|
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
|
|
|
export function nodeRuntimeTriggerCurrentOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
|
|
const result = nodeRuntimeTriggerCurrent(scoped);
|
|
if (nodeScopedFullOutput(scoped)) return result;
|
|
return withNodeRuntimeTriggerRendered(result, scoped);
|
|
}
|
|
|
|
export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
|
|
const triggerStartedAt = Date.now();
|
|
const triggerElapsedMs = () => Date.now() - triggerStartedAt;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
|
|
const head = resolveNodeRuntimeLaneHead(spec);
|
|
const sourceCommit = head.sourceCommit;
|
|
if (sourceCommit === null) {
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "failed" });
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "source-head",
|
|
degradedReason: "node-runtime-source-head-unresolved",
|
|
headProbe: compactRuntimeCommand(head.result),
|
|
};
|
|
}
|
|
const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
|
|
const pipelineRun = scoped.rerun ? nodeRuntimeRerunPipelineRunName(spec, sourceCommit) : basePipelineRun;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun });
|
|
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "succeeded", sourceCommit, pipelineRun, existingStatus: before.status ?? null });
|
|
if (scoped.dryRun) {
|
|
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
|
|
return {
|
|
ok: true,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "dry-run",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
rerunOf: scoped.rerun ? basePipelineRun : null,
|
|
rerun: scoped.rerun,
|
|
before,
|
|
gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
|
|
manifest: nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun),
|
|
next: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm` },
|
|
};
|
|
}
|
|
if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
|
|
const pipelineWait = before.status === "Unknown"
|
|
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
|
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
|
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
|
})
|
|
: { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 };
|
|
const waitedPipelineRun = record(pipelineWait.pipelineRun);
|
|
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
|
|
const postFlush = waitedPipelineRun.status === "True"
|
|
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
|
: null;
|
|
const pipelinePending = pipelineWait.status === "pending";
|
|
const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true);
|
|
const elapsedMs = triggerElapsedMs();
|
|
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
|
|
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "confirmed-trigger",
|
|
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
|
warning: pipelinePending ? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
|
|
triggerElapsedMs: elapsedMs,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
pipelineWait,
|
|
pipelineFailureSummary,
|
|
postFlush,
|
|
skipped: true,
|
|
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
|
|
skipPolicy: "source-commit-only",
|
|
skipExplanation: "An existing PipelineRun for this HWLAB source commit was reused. If UniDesk node/lane YAML or other render inputs changed without a HWLAB source commit change, rerun the controlled publish with --rerun.",
|
|
rerunAvailable: true,
|
|
rerunCommand: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
|
|
degradedReason: ok ? undefined : postFlush !== null && postFlush.ok !== true ? "node-runtime-git-mirror-post-flush-failed" : "node-runtime-existing-pipelinerun-not-succeeded",
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}`,
|
|
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
|
|
},
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun });
|
|
const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit, pipelineRun);
|
|
if (gitMirror.ok !== true) {
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "failed", sourceCommit, pipelineRun, reason: gitMirror.degradedReason ?? null });
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "git-mirror-pre-sync",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirror,
|
|
degradedReason: "node-runtime-git-mirror-pre-sync-failed",
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "succeeded", sourceCommit, pipelineRun });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
|
|
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
|
|
if (refresh.ok !== true) {
|
|
const diagnostics = record(refresh.diagnostics);
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: "control-plane-refresh",
|
|
status: "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
...(diagnostics.classification === "ssh-tcp-pool-transient"
|
|
? { reason: "ssh-tcp-pool-transient", failureKind: diagnostics.failureKind ?? null }
|
|
: {}),
|
|
});
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
phase: "control-plane-apply",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
refresh,
|
|
diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : null,
|
|
degradedReason: "node-runtime-control-plane-apply-before-trigger-failed",
|
|
next: {
|
|
retryTriggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
...(diagnostics.classification === "ssh-tcp-pool-transient" ? { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` } : {}),
|
|
},
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun });
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun });
|
|
const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun);
|
|
const after = getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True");
|
|
const createOk = isCommandSuccess(create) || createObserved;
|
|
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
|
|
const pipelineWait = createOk
|
|
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
|
|
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
|
|
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
|
|
})
|
|
: null;
|
|
const waitedPipelineRun = record(pipelineWait?.pipelineRun);
|
|
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
|
|
const postFlush = waitedPipelineRun.status === "True"
|
|
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
|
|
: null;
|
|
const pipelineReady = pipelineWait !== null && pipelineWait.ok === true;
|
|
const postFlushOk = postFlush === null || postFlush.ok === true;
|
|
const pipelinePending = pipelineWait !== null && pipelineWait.status === "pending";
|
|
const ok = createOk && (pipelineReady || pipelinePending) && postFlushOk;
|
|
const elapsedMs = triggerElapsedMs();
|
|
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
|
|
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "confirmed-trigger",
|
|
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
|
|
warning: pipelinePending ? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
|
|
triggerElapsedMs: elapsedMs,
|
|
mutation: createOk,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
rerunOf: scoped.rerun ? basePipelineRun : null,
|
|
rerun: scoped.rerun,
|
|
before,
|
|
gitMirror,
|
|
refresh,
|
|
create: compactRuntimeCommand(create),
|
|
after,
|
|
pipelineWait,
|
|
pipelineFailureSummary,
|
|
postFlush,
|
|
createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
|
|
degradedReason: ok
|
|
? undefined
|
|
: !createOk
|
|
? "node-runtime-pipelinerun-create-failed"
|
|
: !pipelineReady
|
|
? "node-runtime-pipelinerun-not-succeeded"
|
|
: "node-runtime-git-mirror-post-flush-failed",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeCicdWaitSeconds(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): number {
|
|
return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS);
|
|
}
|
|
|
|
export function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record<string, unknown> {
|
|
const wait = record(pipelineWait);
|
|
return {
|
|
code: "node-runtime-cicd-wait-over-120s",
|
|
message: `PipelineRun ${pipelineRun} is still non-terminal after ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; this is a severe timeout and requires further investigation of Tekton TaskRuns/Pods instead of treating it as normal wait time.`,
|
|
waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
|
|
elapsedMs: wait.elapsedMs ?? null,
|
|
polls: wait.polls ?? null,
|
|
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeTriggerElapsedWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, elapsedMs: number): Record<string, unknown> | null {
|
|
if (elapsedMs < NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS) return null;
|
|
return {
|
|
code: "node-runtime-trigger-over-120s",
|
|
message: `trigger-current total elapsed time exceeded ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; this is a severe timeout even if the PipelineRun eventually succeeded and requires follow-up investigation of slow control-plane, Tekton and git-mirror stages.`,
|
|
thresholdMs: NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS,
|
|
elapsedMs,
|
|
inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
|
};
|
|
}
|
|
|
|
export function withNodeRuntimeTriggerRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const pipelineWait = record(result.pipelineWait);
|
|
const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before);
|
|
const warning = record(result.warning ?? pipelineWait.warning);
|
|
const postFlush = record(result.postFlush);
|
|
const gitMirror = record(result.gitMirror);
|
|
const gitMirrorSummary = record(postFlush.afterSummary ?? postFlush.beforeSummary ?? gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
|
|
const refresh = record(result.refresh);
|
|
const create = record(result.create);
|
|
const next = record(result.next);
|
|
const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-";
|
|
const completion = result.completion ?? (result.ok === true ? "completed" : "failed");
|
|
const renderedText = [
|
|
"hwlab nodes control-plane trigger-current",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"],
|
|
[[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["STAGE", "STATUS", "DETAIL"],
|
|
[
|
|
["git-mirror", gitMirror.ok === true ? "ok" : gitMirror.ok === false ? "failed" : "-", webObserveText(gitMirror.mode ?? gitMirror.degradedReason)],
|
|
["refresh", refresh.ok === true ? "ok" : refresh.ok === false ? "failed" : "-", webObserveText(refresh.degradedReason)],
|
|
["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`],
|
|
["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`],
|
|
["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)],
|
|
["total", result.triggerElapsedMs === undefined ? "-" : "elapsed", formatElapsedMs(result.triggerElapsedMs)],
|
|
],
|
|
),
|
|
"",
|
|
Object.keys(warning).length === 0
|
|
? "WARNING\n-"
|
|
: webObserveTable(
|
|
["CODE", "MESSAGE"],
|
|
[[webObserveText(warning.code ?? "warning"), webObserveText(warning.message ?? "PipelineRun is still active after the default wait window.")]],
|
|
),
|
|
"",
|
|
Object.keys(warning).length === 0
|
|
? "INSPECT\n-"
|
|
: webObserveTable(
|
|
["CHECK", "COMMAND"],
|
|
[
|
|
["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
|
|
["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)],
|
|
],
|
|
),
|
|
"",
|
|
Object.keys(gitMirrorSummary).length === 0
|
|
? "GIT_MIRROR\n-"
|
|
: webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
|
[[
|
|
shortValue(gitMirrorSummary.localSource),
|
|
shortValue(gitMirrorSummary.githubSource),
|
|
shortValue(gitMirrorSummary.localGitops),
|
|
shortValue(gitMirrorSummary.githubGitops),
|
|
webObserveText(gitMirrorSummary.pendingFlush),
|
|
webObserveText(gitMirrorSummary.githubInSync),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded CICD summary; use --full or --raw for the complete JSON payload.",
|
|
` ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s is the severe timeout warning threshold for both PipelineRun wait and total trigger elapsed time.`,
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
export function withNodeRuntimeControlPlanePlanRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const checks = record(result.checks);
|
|
const next = record(result.next);
|
|
const renderedText = [
|
|
"hwlab nodes control-plane plan",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "MODE", "NAMESPACE", "EXT_PG", "PUBLIC"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
result.ok === true ? "ok" : "failed",
|
|
webObserveText(result.mode),
|
|
webObserveText(checks.runtimeNamespace),
|
|
webObserveText(checks.externalPostgresActive),
|
|
webObserveText(checks.publicExposureDeclared),
|
|
]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["CHECK", "VALUE"],
|
|
[
|
|
["node-scoped-target", webObserveText(checks.nodeScopedTargetConfigured)],
|
|
["external-postgres-declared", webObserveText(checks.externalPostgresDeclared)],
|
|
["external-postgres-active", webObserveText(checks.externalPostgresActive)],
|
|
["local-postgres-absent", webObserveText(checks.localPostgresExpectedAbsent)],
|
|
["secret-values-printed", webObserveText(checks.secretValuesPrinted)],
|
|
],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded control-plane plan summary; use --full or --raw for the complete JSON payload.",
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const pipelineRun = record(result.pipelineRun);
|
|
const pipelineRunDiagnostics = record(pipelineRun.diagnostics);
|
|
const argo = record(result.argo);
|
|
const runtime = record(result.runtime);
|
|
const publicProbe = record(result.publicProbe);
|
|
const gitMirror = record(result.gitMirror);
|
|
const next = record(result.next);
|
|
const diagnostic = record(publicProbe.diagnostic);
|
|
const notReadyWorkloads = Array.isArray(runtime.notReadyWorkloads) ? runtime.notReadyWorkloads.map((item) => webObserveText(item)).filter(Boolean) : [];
|
|
const runtimeDetail = [
|
|
`workloads=${webObserveText(runtime.workloadReady)}`,
|
|
`pg=${webObserveText(runtime.externalPostgresReady ?? runtime.localPostgresReady)}`,
|
|
...(notReadyWorkloads.length > 0 ? [`notReady=${webObserveShort(notReadyWorkloads.join(","), 80)}`] : []),
|
|
].join(" ");
|
|
const failedTaskRuns = Array.isArray(pipelineRunDiagnostics.failedTaskRuns) ? pipelineRunDiagnostics.failedTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
|
|
const pendingTaskRuns = Array.isArray(pipelineRunDiagnostics.pendingTaskRuns) ? pipelineRunDiagnostics.pendingTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
|
|
const pipelineDetail = [
|
|
webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 50),
|
|
...(failedTaskRuns.length > 0 ? [`failed=${webObserveShort(failedTaskRuns.join(","), 80)}`] : []),
|
|
...(pendingTaskRuns.length > 0 ? [`pending=${webObserveShort(pendingTaskRuns.join(","), 80)}`] : []),
|
|
].filter(Boolean).join(" ");
|
|
const status = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
|
const renderedText = [
|
|
"hwlab nodes control-plane status",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "REASON", "SOURCE", "PIPELINERUN"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
status,
|
|
webObserveShort(webObserveText(result.degradedReason), 40),
|
|
shortValue(result.sourceCommit),
|
|
webObserveShort(webObserveText(pipelineRun.name), 36),
|
|
]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["STAGE", "STATUS", "DETAIL"],
|
|
[
|
|
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", pipelineDetail],
|
|
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
|
|
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
|
|
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(diagnostic.kind ?? diagnostic.message), 80)],
|
|
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
|
],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS"],
|
|
[[
|
|
shortValue(gitMirror.localSource),
|
|
shortValue(gitMirror.githubSource),
|
|
shortValue(gitMirror.localGitops),
|
|
shortValue(gitMirror.githubGitops),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${result.nextAction ?? next.full ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --full`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded control-plane status summary; use --full or --raw for the complete JSON payload.",
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|
|
|
|
export function withNodeRuntimeControlPlaneStatusFullRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const summary = record(result.summary);
|
|
const pipelineRun = record(summary.pipelineRun);
|
|
const argo = record(summary.argo);
|
|
const runtimeSummary = record(summary.runtime);
|
|
const publicProbe = record(summary.publicProbe);
|
|
const gitMirror = record(summary.gitMirror);
|
|
const runtime = record(result.runtime);
|
|
const bridge = record(runtime.externalPostgresBridge);
|
|
const secrets = record(runtime.externalPostgresSecrets);
|
|
const next = record(summary.next);
|
|
const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
|
|
const workloadRows = workloadReadiness.length === 0
|
|
? [["-", "-", "-", "-", "-"]]
|
|
: workloadReadiness.slice(0, 18).map((item) => [
|
|
webObserveText(item.ref),
|
|
item.ready === true ? "ok" : "not-ready",
|
|
webObserveText(item.readyReplicas),
|
|
webObserveText(item.currentReplicas),
|
|
webObserveText(item.desiredReplicas),
|
|
]);
|
|
const bridgeRows = bridge.required === false
|
|
? [["not-required", "-", "-", "-", "-"]]
|
|
: [[
|
|
bridge.ready === true ? "ok" : "failed",
|
|
webObserveText(bridge.serviceName),
|
|
webObserveText(bridge.endpointAddress),
|
|
webObserveText(bridge.port),
|
|
`${webObserveText(bridge.expectedEndpointAddress)}:${webObserveText(bridge.expectedPort)}`,
|
|
]];
|
|
const secretRows = nodeRuntimeExternalPostgresSecretRows(secrets);
|
|
const status = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
|
const renderedText = [
|
|
"hwlab nodes control-plane status --full",
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "STATUS", "REASON", "SOURCE", "PIPELINERUN"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
status,
|
|
webObserveShort(webObserveText(summary.degradedReason ?? result.degradedReason), 40),
|
|
shortValue(summary.sourceCommit ?? result.sourceCommit),
|
|
webObserveShort(webObserveText(pipelineRun.name ?? result.pipelineRun), 36),
|
|
]],
|
|
),
|
|
"",
|
|
webObserveTable(
|
|
["STAGE", "STATUS", "DETAIL"],
|
|
[
|
|
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 80)],
|
|
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
|
|
["runtime", runtimeSummary.ready === true ? "ok" : "failed", `namespace=${webObserveText(runtimeSummary.namespace)} workloads=${webObserveText(runtimeSummary.workloadReady)} pg=${webObserveText(runtimeSummary.externalPostgresReady ?? runtimeSummary.localPostgresReady)}`],
|
|
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(record(publicProbe.diagnostic).kind ?? record(publicProbe.diagnostic).message), 80)],
|
|
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
|
|
],
|
|
),
|
|
"",
|
|
"RUNTIME_WORKLOADS",
|
|
webObserveTable(["REF", "READY", "READY_REPLICAS", "CURRENT", "DESIRED"], workloadRows),
|
|
...(workloadReadiness.length > workloadRows.length ? [` ... ${workloadReadiness.length - workloadRows.length} more workloads omitted; use --raw for complete JSON.`] : []),
|
|
"",
|
|
"EXTERNAL_POSTGRES_BRIDGE",
|
|
webObserveTable(["STATUS", "SERVICE", "ENDPOINT", "PORT", "EXPECTED"], bridgeRows),
|
|
"",
|
|
"EXTERNAL_POSTGRES_SECRETS",
|
|
webObserveTable(["ROLE", "SECRET", "KEY", "PRESENT", "VALUES_PRINTED"], secretRows),
|
|
"",
|
|
"NEXT",
|
|
` ${summary.nextAction ?? next.full ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --full`}`,
|
|
` bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --raw`,
|
|
"",
|
|
"Disclosure:",
|
|
" --full status is a bounded runtime-workloads drill-down view; use --raw only when complete JSON is required.",
|
|
" Secret evidence is limited to object metadata, key names, presence, and valuesPrinted=false.",
|
|
].join("\n");
|
|
return { ...result, renderedText, contentType: "text/plain" };
|
|
}
|