|
|
|
@@ -15,6 +15,7 @@ import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from
|
|
|
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "./hwlab-node-web-probe-summary";
|
|
|
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "./hwlab-node-observability-promql";
|
|
|
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "./hwlab-node-transport";
|
|
|
|
|
import type { RenderedCliResult } from "./output";
|
|
|
|
|
|
|
|
|
|
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
|
|
|
|
|
type SecretPreset = "openfga" | "master-server-admin-api-key" | "bootstrap-admin" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
|
|
|
|
@@ -289,7 +290,7 @@ const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
|
|
|
|
|
const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
|
|
|
|
|
const HWLAB_CI_NAMESPACE = "hwlab-ci";
|
|
|
|
|
|
|
|
|
|
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown>> {
|
|
|
|
|
export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
|
|
|
|
if (args.length === 0) return hwlabNodeHelp();
|
|
|
|
|
const [domain] = args;
|
|
|
|
|
if (domain === "control-plane" && args[1] === "infra") {
|
|
|
|
@@ -7039,7 +7040,7 @@ function runNodeWebProbeObserveStart(
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
|
|
|
function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record<string, unknown> | RenderedCliResult {
|
|
|
|
|
const script = [
|
|
|
|
|
"set -eu",
|
|
|
|
|
nodeWebObserveResolveStateDirShell(options),
|
|
|
|
@@ -7062,7 +7063,7 @@ function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec:
|
|
|
|
|
const index = ok && observerId !== null && options.stateDir !== null
|
|
|
|
|
? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status))
|
|
|
|
|
: null;
|
|
|
|
|
return {
|
|
|
|
|
return withWebObserveStatusRendered({
|
|
|
|
|
ok,
|
|
|
|
|
status: ok ? "observed" : "blocked",
|
|
|
|
|
command: webObserveCommandLabel("status", options),
|
|
|
|
@@ -7076,9 +7077,108 @@ function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec:
|
|
|
|
|
next: observerId === null ? null : webObserveNextCommands(observerId),
|
|
|
|
|
result: compactCommandResult(result),
|
|
|
|
|
valuesRedacted: true,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function withWebObserveStatusRendered(value: Record<string, unknown>): RenderedCliResult {
|
|
|
|
|
return {
|
|
|
|
|
ok: value.ok !== false,
|
|
|
|
|
command: typeof value.command === "string" ? value.command : "hwlab nodes web-probe observe status",
|
|
|
|
|
contentType: "text/plain",
|
|
|
|
|
renderedText: renderWebObserveStatusTable(value),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function renderWebObserveStatusTable(value: Record<string, unknown>): string {
|
|
|
|
|
const observer = record(value.observer);
|
|
|
|
|
const manifest = record(observer?.manifest);
|
|
|
|
|
const heartbeat = record(observer?.heartbeat);
|
|
|
|
|
const tails = record(observer?.tails);
|
|
|
|
|
const samples = Array.isArray(tails?.samples) ? tails.samples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-5) : [];
|
|
|
|
|
const controls = Array.isArray(tails?.control) ? tails.control.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-5) : [];
|
|
|
|
|
const network = Array.isArray(tails?.network)
|
|
|
|
|
? tails.network.map(record).filter((item): item is Record<string, unknown> => item !== null).filter((item) => {
|
|
|
|
|
const status = typeof item.status === "number" ? item.status : null;
|
|
|
|
|
return status === null || status >= 400 || item.failure !== null && item.failure !== undefined;
|
|
|
|
|
}).slice(-5)
|
|
|
|
|
: [];
|
|
|
|
|
const next = record(value.next);
|
|
|
|
|
const lines = [
|
|
|
|
|
`hwlab nodes web-probe observe status (${webObserveText(value.status)})`,
|
|
|
|
|
"",
|
|
|
|
|
webObserveTable(["ID", "NODE", "LANE", "ALIVE", "PID", "SAMPLE", "UPDATED", "TARGET"], [[
|
|
|
|
|
webObserveText(value.id),
|
|
|
|
|
webObserveText(value.node),
|
|
|
|
|
webObserveText(value.lane),
|
|
|
|
|
webObserveText(observer?.processAlive),
|
|
|
|
|
webObserveText(observer?.pid),
|
|
|
|
|
webObserveText(heartbeat?.sampleSeq),
|
|
|
|
|
webObserveShort(webObserveText(heartbeat?.updatedAt ?? heartbeat?.lastSampleAt), 24),
|
|
|
|
|
webObserveShort(`${webObserveText(manifest?.baseUrl)}${webObserveText(manifest?.targetPath) === "-" ? "" : webObserveText(manifest?.targetPath)}`, 52),
|
|
|
|
|
]]),
|
|
|
|
|
"",
|
|
|
|
|
"Recent samples:",
|
|
|
|
|
webObserveTable(["SEQ", "TS", "PATH", "ROUTE_SESSION", "ACTIVE_SESSION", "MSG", "TRACE"], samples.length > 0 ? samples.map((sample) => [
|
|
|
|
|
webObserveText(sample.seq),
|
|
|
|
|
webObserveShort(webObserveText(sample.ts), 24),
|
|
|
|
|
webObserveShort(webObserveText(sample.path), 24),
|
|
|
|
|
webObserveShort(webObserveText(sample.routeSessionId), 20),
|
|
|
|
|
webObserveShort(webObserveText(sample.activeSessionId), 20),
|
|
|
|
|
webObserveText(sample.messageCount),
|
|
|
|
|
webObserveText(sample.traceRowCount),
|
|
|
|
|
]) : [["-", "-", "-", "-", "-", "-", "-"]]),
|
|
|
|
|
"",
|
|
|
|
|
"Recent control:",
|
|
|
|
|
webObserveTable(["SEQ", "TS", "PHASE", "TYPE", "COMMAND"], controls.length > 0 ? controls.map((control) => [
|
|
|
|
|
webObserveText(control.seq),
|
|
|
|
|
webObserveShort(webObserveText(control.ts), 24),
|
|
|
|
|
webObserveText(control.phase),
|
|
|
|
|
webObserveText(control.type),
|
|
|
|
|
webObserveShort(webObserveText(control.commandId), 28),
|
|
|
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
|
|
|
"",
|
|
|
|
|
"Network alerts:",
|
|
|
|
|
webObserveTable(["TS", "METHOD", "STATUS", "TYPE", "URL_OR_FAILURE"], network.length > 0 ? network.map((item) => [
|
|
|
|
|
webObserveShort(webObserveText(item.ts), 24),
|
|
|
|
|
webObserveText(item.method),
|
|
|
|
|
webObserveText(item.status),
|
|
|
|
|
webObserveText(item.type),
|
|
|
|
|
webObserveShort(webObserveText(item.failure ?? item.url), 80),
|
|
|
|
|
]) : [["-", "-", "-", "-", "-"]]),
|
|
|
|
|
"",
|
|
|
|
|
"Next:",
|
|
|
|
|
];
|
|
|
|
|
for (const key of ["status", "command", "stop", "analyze"] as const) {
|
|
|
|
|
const command = webObserveText(next?.[key]);
|
|
|
|
|
if (command !== "-") lines.push(` ${command}`);
|
|
|
|
|
}
|
|
|
|
|
lines.push("", "Disclosure:");
|
|
|
|
|
lines.push(" default view is a bounded table; use observe collect/analyze for artifact-level drill-down.");
|
|
|
|
|
return lines.join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function webObserveTable(headers: string[], rows: string[][]): string {
|
|
|
|
|
const normalizedRows = rows.map((row) => headers.map((_, index) => row[index] ?? ""));
|
|
|
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...normalizedRows.map((row) => row[index].length)));
|
|
|
|
|
return [
|
|
|
|
|
headers.map((header, index) => header.padEnd(widths[index])).join(" ").trimEnd(),
|
|
|
|
|
...normalizedRows.map((row) => row.map((cell, index) => cell.padEnd(widths[index])).join(" ").trimEnd()),
|
|
|
|
|
].join("\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function webObserveText(value: unknown): string {
|
|
|
|
|
if (value === null || value === undefined || value === "") return "-";
|
|
|
|
|
if (typeof value === "string") return value;
|
|
|
|
|
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
|
|
|
return JSON.stringify(value);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function webObserveShort(value: string, maxLength: number): string {
|
|
|
|
|
if (value.length <= maxLength) return value;
|
|
|
|
|
if (maxLength <= 1) return value.slice(0, maxLength);
|
|
|
|
|
return `${value.slice(0, maxLength - 1)}~`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record<string, unknown> {
|
|
|
|
|
const type = options.commandType ?? (stopCommand ? "stop" : null);
|
|
|
|
|
if (type === null) throw new Error("web-probe observe command requires --type");
|
|
|
|
|