feat: add HWLAB performance summary CLI
This commit is contained in:
Codex
@@ -58,7 +58,7 @@ interface NodeWebProbeScriptOptions {
|
||||
|
||||
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions;
|
||||
|
||||
type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary";
|
||||
type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary" | "performance-summary";
|
||||
|
||||
interface NodeObservabilityOptions {
|
||||
action: NodeObservabilityAction;
|
||||
@@ -253,8 +253,8 @@ function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions
|
||||
if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) {
|
||||
throw new Error("observability usage: observability ACTION --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
}
|
||||
if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary") {
|
||||
throw new Error(`observability action must be plan, apply, status, or workbench-summary; got ${actionRaw}`);
|
||||
if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary" && actionRaw !== "performance-summary") {
|
||||
throw new Error(`observability action must be plan, apply, status, workbench-summary, or performance-summary; got ${actionRaw}`);
|
||||
}
|
||||
assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--full", "--raw"]));
|
||||
const node = requiredOption(args, "--node");
|
||||
@@ -501,6 +501,7 @@ function runNodeObservability(options: NodeObservabilityOptions): Record<string,
|
||||
if (options.action === "plan") return nodeObservabilityPlan(options);
|
||||
if (options.action === "apply") return nodeObservabilityApply(options);
|
||||
if (options.action === "status") return nodeObservabilityStatus(options);
|
||||
if (options.action === "performance-summary") return nodeObservabilityPerformanceSummary(options);
|
||||
return nodeObservabilityWorkbenchSummary(options);
|
||||
}
|
||||
|
||||
@@ -522,6 +523,7 @@ function nodeObservabilityPlan(options: NodeObservabilityOptions): Record<string
|
||||
apply: `bun scripts/cli.ts hwlab nodes observability apply --node ${options.node} --lane ${options.lane} --confirm`,
|
||||
status: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -604,6 +606,7 @@ function nodeObservabilityStatus(options: NodeObservabilityOptions): Record<stri
|
||||
next: {
|
||||
plan: `bun scripts/cli.ts hwlab nodes observability plan --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
performanceSummary: `bun scripts/cli.ts hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
return options.full ? status : summarizeNodeObservabilityStatus(status);
|
||||
@@ -651,6 +654,330 @@ function nodeObservabilityWorkbenchSummary(options: NodeObservabilityOptions, ex
|
||||
};
|
||||
}
|
||||
|
||||
function nodeObservabilityPerformanceSummary(options: NodeObservabilityOptions): Record<string, unknown> {
|
||||
const workbench = options.spec.observability.workbench;
|
||||
const summaryPath = workbench?.summaryPath ?? "/v1/web-performance/summary";
|
||||
const endpoint = joinUrlPath(options.spec.publicWebUrl, summaryPath);
|
||||
const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane });
|
||||
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
||||
const credential = webProbeCredential(secretSpec, material);
|
||||
const command = `hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`;
|
||||
if (material.ok !== true || material.password === null) {
|
||||
return {
|
||||
ok: false,
|
||||
command,
|
||||
mode: "node-observability-performance-summary",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
target: {
|
||||
workspace: options.spec.workspace,
|
||||
webBaseUrl: options.spec.publicWebUrl,
|
||||
endpoint,
|
||||
},
|
||||
credential,
|
||||
degradedReason: material.error ?? "web-login-secret-unavailable",
|
||||
next: {
|
||||
secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name bootstrap-admin`,
|
||||
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
const timeoutSeconds = Math.max(10, Math.min(options.timeoutSeconds, 55));
|
||||
const script = nodePerformanceSummaryRemoteScript(options.spec.publicWebUrl, summaryPath, secretSpec.bootstrapAdminUsername, material.password);
|
||||
const result = runTransWorkspaceStdinScript(options.node, options.spec.workspace, script, timeoutSeconds);
|
||||
const report = parseJsonRecordFromText(result.stdout);
|
||||
const performance = record(report.performance);
|
||||
const rows = nodePerformanceRows(performance);
|
||||
const analysis = nodePerformanceAnalysis(performance, rows);
|
||||
const ok = result.exitCode === 0 && report.ok === true && performance.ok !== false;
|
||||
return {
|
||||
ok,
|
||||
command,
|
||||
mode: "node-observability-performance-summary",
|
||||
mutation: false,
|
||||
node: options.node,
|
||||
lane: options.lane,
|
||||
target: {
|
||||
workspace: options.spec.workspace,
|
||||
webBaseUrl: options.spec.publicWebUrl,
|
||||
endpoint,
|
||||
summaryPath,
|
||||
lowSampleThreshold: workbench?.lowSampleThreshold ?? null,
|
||||
},
|
||||
credential,
|
||||
auth: record(report.auth),
|
||||
httpStatus: report.httpStatus ?? null,
|
||||
fetchAttempts: report.fetchAttempts ?? null,
|
||||
summary: performance,
|
||||
analysis,
|
||||
probe: compactCommandResultRedacted(result, [material.password]),
|
||||
degradedReason: ok
|
||||
? undefined
|
||||
: typeof report.error === "string" && report.error.length > 0
|
||||
? report.error
|
||||
: result.exitCode === 0 ? "web-performance-summary-not-ready" : "web-performance-summary-probe-failed",
|
||||
next: {
|
||||
webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`,
|
||||
workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`,
|
||||
apiMetrics: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane} --full`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function nodePerformanceSummaryRemoteScript(baseUrl: string, summaryPath: string, username: string, password: string): string {
|
||||
const nodeScript = [
|
||||
"const baseUrl = process.env.HWLAB_WEB_BASE_URL;",
|
||||
"const summaryPath = process.env.HWLAB_PERFORMANCE_SUMMARY_PATH || '/v1/web-performance/summary';",
|
||||
"const username = process.env.HWLAB_WEB_USER;",
|
||||
"const password = process.env.HWLAB_WEB_PASS;",
|
||||
"function compactNumber(value) { const n = Number(value); return Number.isFinite(n) ? Math.round(n * 10) / 10 : null; }",
|
||||
"function compactSeconds(msValue, secondsValue) { const ms = Number(msValue); if (Number.isFinite(ms)) return Math.round((ms / 1000) * 10) / 10; return compactNumber(secondsValue); }",
|
||||
"function compactString(value) { return typeof value === 'string' && value.length > 0 ? value : null; }",
|
||||
"function compactRow(row) {",
|
||||
" if (!row || typeof row !== 'object' || Array.isArray(row)) return null;",
|
||||
" const route = compactString(row.route) || compactString(row.path) || compactString(row.name) || compactString(row.metric) || compactString(row.title);",
|
||||
" return {",
|
||||
" source: compactString(row.source) || compactString(row.kind) || compactString(row.category),",
|
||||
" metric: compactString(row.metric) || compactString(row.eventKind) || compactString(row.phase),",
|
||||
" route,",
|
||||
" problem: compactString(row.problem) || compactString(row.diagnosis) || compactString(row.reason),",
|
||||
" outcome: compactString(row.outcome) || compactString(row.result) || compactString(row.status),",
|
||||
" tone: compactString(row.tone) || compactString(row.severity),",
|
||||
" sampleState: compactString(row.sampleState) || compactString(row.sampleStatus),",
|
||||
" count: compactNumber(row.count ?? row.sampleCount ?? row.samples),",
|
||||
" durationUnit: 'seconds',",
|
||||
" p50Seconds: compactSeconds(row.p50Ms, row.p50Seconds ?? row.p50 ?? row.medianSeconds ?? row.median),",
|
||||
" p75Seconds: compactSeconds(row.p75Ms, row.p75Seconds ?? row.p75),",
|
||||
" p95Seconds: compactSeconds(row.p95Ms, row.p95Seconds ?? row.p95),",
|
||||
" maxSeconds: compactSeconds(row.maxMs, row.maxSeconds ?? row.max),",
|
||||
" };",
|
||||
"}",
|
||||
"function rowHasSignal(row) { return Boolean(row && (row.source || row.metric || row.route || row.problem || row.outcome)); }",
|
||||
"function compactRows(value, limit = 16) { return Array.isArray(value) ? value.map(compactRow).filter(rowHasSignal).slice(0, limit) : []; }",
|
||||
"function compactTopN(value) {",
|
||||
" if (!Array.isArray(value)) return [];",
|
||||
" return value.slice(0, 8).map((group) => {",
|
||||
" const rows = compactRows(group?.rows || group?.items || group?.data, 8);",
|
||||
" return { id: compactString(group?.id) || compactString(group?.key), title: compactString(group?.title) || compactString(group?.label), rows };",
|
||||
" }).filter((group) => group.rows.length > 0);",
|
||||
"}",
|
||||
"function compactCards(value) {",
|
||||
" if (!Array.isArray(value)) return [];",
|
||||
" return value.slice(0, 12).map((card) => ({ id: compactString(card?.id), title: compactString(card?.title), value: card?.value ?? null, tone: compactString(card?.tone) || compactString(card?.status), detail: compactString(card?.detail) || compactString(card?.hint) }));",
|
||||
"}",
|
||||
"function compactPerformance(body) {",
|
||||
" if (!body || typeof body !== 'object' || Array.isArray(body)) return { ok: false, degradedReason: 'non-json-summary' };",
|
||||
" const dashboard = body.dashboard && typeof body.dashboard === 'object' ? body.dashboard : {};",
|
||||
" const summary = body.summary && typeof body.summary === 'object' ? body.summary : {};",
|
||||
" const rows = compactRows(body.rows || body.tableRows || body.performanceRows, 24);",
|
||||
" const workbenchRows = compactRows(body.workbenchRows || body.workbench || body.workbenchTableRows, 12);",
|
||||
" const problems = compactRows(body.problems || body.problemRows || dashboard.problems, 24);",
|
||||
" const displayRows = rows.length > 0 ? rows.slice(0, 16) : problems.slice(0, 16);",
|
||||
" const displayProblems = rows.length > 0 ? [] : problems.slice(0, 16);",
|
||||
" const topN = compactTopN(dashboard.topN || body.topN);",
|
||||
" return {",
|
||||
" ok: true,",
|
||||
" schemaVersion: body.schemaVersion || dashboard.schemaVersion || null,",
|
||||
" source: body.source || dashboard.source || null,",
|
||||
" observedAt: body.observedAt || dashboard.observedAt || body.generatedAt || dashboard.generatedAt || null,",
|
||||
" window: body.window || body.sampleWindow || dashboard.window || null,",
|
||||
" freshness: body.freshness || dashboard.freshness || null,",
|
||||
" sampleCount: summary.sampleCount ?? body.sampleCount ?? dashboard.sampleCount ?? null,",
|
||||
" problemCount: summary.problemCount ?? body.problemCount ?? dashboard.problemCount ?? null,",
|
||||
" status: summary.status || body.status || dashboard.status || null,",
|
||||
" cards: compactCards(dashboard.cards || body.cards),",
|
||||
" rows: displayRows,",
|
||||
" workbenchRows,",
|
||||
" problems: displayProblems,",
|
||||
" topN,",
|
||||
" rowCount: rows.length,",
|
||||
" workbenchRowCount: workbenchRows.length,",
|
||||
" problemRowCount: problems.length,",
|
||||
" };",
|
||||
"}",
|
||||
"function splitSetCookie(raw) { return raw ? raw.split(/,(?=[^;,]+=)/g).map((item) => item.trim()).filter(Boolean) : []; }",
|
||||
"async function fetchWithTimeout(url, init, timeoutMs = 12000) {",
|
||||
" const controller = new AbortController();",
|
||||
" const timer = setTimeout(() => controller.abort(), timeoutMs);",
|
||||
" try {",
|
||||
" const response = await fetch(url, { ...init, signal: controller.signal });",
|
||||
" const text = await response.text();",
|
||||
" let body = null;",
|
||||
" try { body = JSON.parse(text); } catch {}",
|
||||
" return { response, text, body };",
|
||||
" } finally { clearTimeout(timer); }",
|
||||
"}",
|
||||
"function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }",
|
||||
"async function fetchWithRetry(url, init, attempts = 3) {",
|
||||
" let last = null;",
|
||||
" for (let attempt = 1; attempt <= attempts; attempt += 1) {",
|
||||
" try {",
|
||||
" const result = await fetchWithTimeout(url, init);",
|
||||
" result.attempt = attempt;",
|
||||
" last = result;",
|
||||
" if (![502, 503, 504].includes(result.response.status)) return result;",
|
||||
" } catch (error) {",
|
||||
" last = { attempt, error: error instanceof Error ? error.message : String(error), response: { status: 0, ok: false, headers: new Headers() }, body: null, text: '' };",
|
||||
" }",
|
||||
" if (attempt < attempts) await sleep(400 * attempt);",
|
||||
" }",
|
||||
" return last;",
|
||||
"}",
|
||||
"(async () => {",
|
||||
" try {",
|
||||
" const loginUrl = new URL('/auth/login', baseUrl).toString();",
|
||||
" const login = await fetchWithRetry(loginUrl, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ username, password }) });",
|
||||
" const setCookie = splitSetCookie(login.response.headers.get('set-cookie'));",
|
||||
" const cookie = setCookie.map((item) => item.split(';')[0].trim()).filter(Boolean).join('; ');",
|
||||
" const auth = { ok: login.response.ok && cookie.length > 0, httpStatus: login.response.status, cookiePresent: cookie.length > 0, attempts: login.attempt || 1 };",
|
||||
" if (!auth.ok) {",
|
||||
" console.log(JSON.stringify({ ok: false, auth, httpStatus: login.response.status, error: 'web-login-failed' }));",
|
||||
" process.exitCode = 1;",
|
||||
" return;",
|
||||
" }",
|
||||
" const summaryUrl = new URL(summaryPath, baseUrl).toString();",
|
||||
" const summary = await fetchWithRetry(summaryUrl, { method: 'GET', headers: { accept: 'application/json', cookie } });",
|
||||
" const performance = compactPerformance(summary.body);",
|
||||
" const ok = summary.response.ok && performance.ok === true;",
|
||||
" console.log(JSON.stringify({ ok, auth, httpStatus: summary.response.status, fetchAttempts: summary.attempt || 1, finalUrl: summaryUrl, performance, error: ok ? null : 'web-performance-summary-fetch-failed' }));",
|
||||
" process.exitCode = ok ? 0 : 1;",
|
||||
" } catch (error) {",
|
||||
" console.log(JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }));",
|
||||
" process.exitCode = 1;",
|
||||
" }",
|
||||
"})();",
|
||||
].join("\n");
|
||||
return [
|
||||
"set -eu",
|
||||
`HWLAB_WEB_BASE_URL=${shellQuote(baseUrl)} HWLAB_PERFORMANCE_SUMMARY_PATH=${shellQuote(summaryPath)} HWLAB_WEB_USER=${shellQuote(username)} HWLAB_WEB_PASS=${shellQuote(password)} node <<'NODE'`,
|
||||
nodeScript,
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function nodePerformanceRows(performance: Record<string, unknown>): Record<string, unknown>[] {
|
||||
const rows = [
|
||||
...nodePerformanceArray(performance.problems),
|
||||
...nodePerformanceArray(performance.rows),
|
||||
...nodePerformanceArray(performance.workbenchRows),
|
||||
];
|
||||
const topN = Array.isArray(performance.topN) ? performance.topN.map(record) : [];
|
||||
for (const group of topN) {
|
||||
for (const row of nodePerformanceArray(group.rows)) rows.push(row);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
return rows.filter((row) => {
|
||||
const key = [
|
||||
nodePerformanceString(row.source),
|
||||
nodePerformanceString(row.metric),
|
||||
nodePerformanceString(row.route),
|
||||
nodePerformanceString(row.outcome),
|
||||
nodePerformanceString(row.problem),
|
||||
].join("|");
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function nodePerformanceArray(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : [];
|
||||
}
|
||||
|
||||
function nodePerformanceAnalysis(performance: Record<string, unknown>, rows: Record<string, unknown>[]): Record<string, unknown> {
|
||||
const networkErrors = rows.filter((row) => nodePerformanceString(row.outcome) === "network_error" || nodePerformanceString(row.problem) === "network_error" || nodePerformanceString(row.problem) === "network error");
|
||||
const blockedRows = rows.filter((row) => nodePerformanceString(row.tone) === "blocked" || nodePerformanceString(row.status) === "blocked");
|
||||
const slowApiRows = rows.filter((row) => {
|
||||
const source = nodePerformanceString(row.source) ?? "";
|
||||
const metric = nodePerformanceString(row.metric) ?? "";
|
||||
const p95Seconds = nodePerformanceNumber(row.p95Seconds);
|
||||
return (source.includes("api") || metric.includes("api") || (nodePerformanceString(row.route) ?? "").startsWith("/v1") || (nodePerformanceString(row.route) ?? "").startsWith("/health")) && (p95Seconds === null || p95Seconds >= 5);
|
||||
});
|
||||
const lcpRows = rows.filter((row) => {
|
||||
const metric = `${nodePerformanceString(row.metric) ?? ""} ${nodePerformanceString(row.problem) ?? ""}`.toLowerCase();
|
||||
return metric.includes("lcp") || metric.includes("largest contentful") || metric.includes("最大内容");
|
||||
});
|
||||
const maxP95Seconds = Math.max(0, ...rows.map((row) => nodePerformanceNumber(row.p95Seconds) ?? 0));
|
||||
const headline = networkErrors.length > 0
|
||||
? "Public same-origin API has intermittent network_error samples; treat edge/proxy/API reachability as P0 before tuning page render."
|
||||
: lcpRows.length > 0
|
||||
? "Workbench page LCP is blocked, likely by first-load API waterfalls and late visible content."
|
||||
: slowApiRows.length > 0
|
||||
? "Same-origin API p95 is above the interactive budget; split edge wait, server work, and dependency latency."
|
||||
: "No blocking performance rows were returned by the current summary window.";
|
||||
const priorities = [
|
||||
networkErrors.length > 0
|
||||
? {
|
||||
priority: "P0",
|
||||
area: "public-edge-api-reachability",
|
||||
action: "Correlate network_error rows with Caddy/FRP/ingress, cloud-api pod restarts, response-header timeout, and browser Resource Timing failure stage.",
|
||||
evidence: nodePerformanceEvidence(networkErrors),
|
||||
}
|
||||
: null,
|
||||
slowApiRows.length > 0
|
||||
? {
|
||||
priority: "P1",
|
||||
area: "same-origin-api-latency",
|
||||
action: "Add server-side route histograms and dependency spans for slow /health, /v1/live-builds, /v1/workbench/sessions, trace events, hwpod specs, and hwpod-node-ops paths; then optimize the highest p95 route first.",
|
||||
evidence: nodePerformanceEvidence(slowApiRows),
|
||||
}
|
||||
: null,
|
||||
lcpRows.length > 0
|
||||
? {
|
||||
priority: "P1",
|
||||
area: "workbench-lcp",
|
||||
action: "Render a stable first screen before slow session/detail API calls complete, and split LCP by route plus blocking API waterfall.",
|
||||
evidence: nodePerformanceEvidence(lcpRows),
|
||||
}
|
||||
: null,
|
||||
blockedRows.length > 0
|
||||
? {
|
||||
priority: "P2",
|
||||
area: "alert-thresholds-and-sample-quality",
|
||||
action: "Keep low-sample rows visible but avoid closing incidents on count=1; require repeated windows or matching server metrics before labeling a regression fixed.",
|
||||
evidence: nodePerformanceEvidence(blockedRows),
|
||||
}
|
||||
: null,
|
||||
].filter(Boolean);
|
||||
return {
|
||||
headline,
|
||||
sampleCount: performance.sampleCount ?? null,
|
||||
problemCount: performance.problemCount ?? null,
|
||||
rowCount: rows.length,
|
||||
blockedRowCount: blockedRows.length,
|
||||
networkErrorRowCount: networkErrors.length,
|
||||
slowApiRowCount: slowApiRows.length,
|
||||
lcpRowCount: lcpRows.length,
|
||||
maxP95Seconds,
|
||||
priorities,
|
||||
};
|
||||
}
|
||||
|
||||
function nodePerformanceEvidence(rows: Record<string, unknown>[]): Record<string, unknown>[] {
|
||||
return rows.slice(0, 8).map((row) => ({
|
||||
source: nodePerformanceString(row.source),
|
||||
metric: nodePerformanceString(row.metric),
|
||||
route: nodePerformanceString(row.route),
|
||||
outcome: nodePerformanceString(row.outcome),
|
||||
problem: nodePerformanceString(row.problem),
|
||||
count: nodePerformanceNumber(row.count),
|
||||
durationUnit: nodePerformanceString(row.durationUnit) ?? "seconds",
|
||||
p50Seconds: nodePerformanceNumber(row.p50Seconds),
|
||||
p75Seconds: nodePerformanceNumber(row.p75Seconds),
|
||||
p95Seconds: nodePerformanceNumber(row.p95Seconds),
|
||||
}));
|
||||
}
|
||||
|
||||
function nodePerformanceString(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function nodePerformanceNumber(value: unknown): number | null {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function summarizeNodeObservabilityStatus(status: Record<string, unknown>): Record<string, unknown> {
|
||||
const service = record(status.service);
|
||||
const publicRawMetrics = record(status.publicRawMetrics);
|
||||
|
||||
Reference in New Issue
Block a user