From a0dbf3d084e6a1d0daabe14a8c1a956cf547d5d0 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 19 Jun 2026 13:30:21 +0000 Subject: [PATCH] feat: add HWLAB performance summary CLI --- scripts/src/help.ts | 5 +- scripts/src/hwlab-node-help.ts | 3 + scripts/src/hwlab-node-impl.ts | 333 ++++++++++++++++++++++++++++++++- 3 files changed, 336 insertions(+), 5 deletions(-) diff --git a/scripts/src/help.ts b/scripts/src/help.ts index d8a94068..da61e6f7 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -662,7 +662,7 @@ function platformInfraHelpSummary(): unknown { function hwlabNodeHelpSummary(): unknown { return { - command: "hwlab nodes control-plane|git-mirror|secret|test-accounts|web-probe --node --lane ", + command: "hwlab nodes control-plane|git-mirror|observability|secret|test-accounts|web-probe --node --lane ", output: "json", usage: [ "bun scripts/cli.ts hwlab nodes control-plane infra plan --node D601 --lane v03", @@ -671,12 +671,13 @@ function hwlabNodeHelpSummary(): unknown { "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", + "bun scripts/cli.ts hwlab nodes observability performance-summary --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name ", "bun scripts/cli.ts hwlab nodes test-accounts status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes test-accounts sync --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000", ], - description: "Operate HWLAB node/lane runtime prerequisites with node and lane passed as data. The infra subcommand manages YAML-controlled node-local CI/CD, git-mirror, public Dockerfile tools image, and declarative Argo CD prerequisites for D601 v03 while keeping cross-node work semi-automatic; test-accounts prepares UniDesk YAML-declared admin/test account API keys with redacted sourceRef/fingerprint output; web-probe runs the target workspace DOM probe with bootstrap Web credentials injected only as one-shot stdin/env.", + description: "Operate HWLAB node/lane runtime prerequisites with node and lane passed as data. The infra subcommand manages YAML-controlled node-local CI/CD, git-mirror, public Dockerfile tools image, and declarative Argo CD prerequisites for D601 v03 while keeping cross-node work semi-automatic; observability reads runtime metrics and authenticated Web Performance summaries; test-accounts prepares UniDesk YAML-declared admin/test account API keys with redacted sourceRef/fingerprint output; web-probe runs the target workspace DOM probe with bootstrap Web credentials injected only as one-shot stdin/env.", }; } diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 28c7470e..53ac94cb 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -53,6 +53,7 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability performance-summary --node D601 --lane v03", ], }; } @@ -95,12 +96,14 @@ export function hwlabNodeObservabilityHelp(): Record { "bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes observability workbench-summary --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes observability performance-summary --node D601 --lane v03", ], actions: { plan: "Render the YAML-declared collection mode, scrape target, boundary, and required Workbench metric series.", apply: "Apply the YAML-declared observability control plane. D601 pod-loopback mode validates the boundary and metrics without creating ad-hoc cluster objects.", status: "Check Kubernetes service/pod presence, public raw metrics denial, and Workbench metric readiness.", "workbench-summary": "Read the YAML-declared metrics endpoint via pod loopback and summarize required Workbench series.", + "performance-summary": "Fetch the authenticated same-origin Web Performance summary for the selected node/lane and return a compact ops interpretation.", }, }; } diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 22074031..35d8d6b8 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -58,7 +58,7 @@ interface NodeWebProbeScriptOptions { type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions; -type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary"; +type NodeObservabilityAction = "plan" | "apply" | "status" | "workbench-summary" | "performance-summary"; interface NodeObservabilityOptions { action: NodeObservabilityAction; @@ -253,8 +253,8 @@ function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) { throw new Error("observability usage: observability ACTION --node NODE --lane vNN [--dry-run|--confirm]"); } - if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary") { - throw new Error(`observability action must be plan, apply, status, or workbench-summary; got ${actionRaw}`); + if (actionRaw !== "plan" && actionRaw !== "apply" && actionRaw !== "status" && actionRaw !== "workbench-summary" && actionRaw !== "performance-summary") { + throw new Error(`observability action must be plan, apply, status, workbench-summary, or performance-summary; got ${actionRaw}`); } assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--full", "--raw"])); const node = requiredOption(args, "--node"); @@ -501,6 +501,7 @@ function runNodeObservability(options: NodeObservabilityOptions): Record { + const workbench = options.spec.observability.workbench; + const summaryPath = workbench?.summaryPath ?? "/v1/web-performance/summary"; + const endpoint = joinUrlPath(options.spec.publicWebUrl, summaryPath); + const secretSpec = runtimeSecretSpec({ node: options.node, lane: options.lane }); + const material = readBootstrapAdminPasswordMaterial(secretSpec); + const credential = webProbeCredential(secretSpec, material); + const command = `hwlab nodes observability performance-summary --node ${options.node} --lane ${options.lane}`; + if (material.ok !== true || material.password === null) { + return { + ok: false, + command, + mode: "node-observability-performance-summary", + mutation: false, + node: options.node, + lane: options.lane, + target: { + workspace: options.spec.workspace, + webBaseUrl: options.spec.publicWebUrl, + endpoint, + }, + credential, + degradedReason: material.error ?? "web-login-secret-unavailable", + next: { + secretStatus: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name bootstrap-admin`, + webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + }, + }; + } + const timeoutSeconds = Math.max(10, Math.min(options.timeoutSeconds, 55)); + const script = nodePerformanceSummaryRemoteScript(options.spec.publicWebUrl, summaryPath, secretSpec.bootstrapAdminUsername, material.password); + const result = runTransWorkspaceStdinScript(options.node, options.spec.workspace, script, timeoutSeconds); + const report = parseJsonRecordFromText(result.stdout); + const performance = record(report.performance); + const rows = nodePerformanceRows(performance); + const analysis = nodePerformanceAnalysis(performance, rows); + const ok = result.exitCode === 0 && report.ok === true && performance.ok !== false; + return { + ok, + command, + mode: "node-observability-performance-summary", + mutation: false, + node: options.node, + lane: options.lane, + target: { + workspace: options.spec.workspace, + webBaseUrl: options.spec.publicWebUrl, + endpoint, + summaryPath, + lowSampleThreshold: workbench?.lowSampleThreshold ?? null, + }, + credential, + auth: record(report.auth), + httpStatus: report.httpStatus ?? null, + fetchAttempts: report.fetchAttempts ?? null, + summary: performance, + analysis, + probe: compactCommandResultRedacted(result, [material.password]), + degradedReason: ok + ? undefined + : typeof report.error === "string" && report.error.length > 0 + ? report.error + : result.exitCode === 0 ? "web-performance-summary-not-ready" : "web-performance-summary-probe-failed", + next: { + webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${options.node} --lane ${options.lane}`, + workbenchSummary: `bun scripts/cli.ts hwlab nodes observability workbench-summary --node ${options.node} --lane ${options.lane}`, + apiMetrics: `bun scripts/cli.ts hwlab nodes observability status --node ${options.node} --lane ${options.lane} --full`, + }, + }; +} + +function nodePerformanceSummaryRemoteScript(baseUrl: string, summaryPath: string, username: string, password: string): string { + const nodeScript = [ + "const baseUrl = process.env.HWLAB_WEB_BASE_URL;", + "const summaryPath = process.env.HWLAB_PERFORMANCE_SUMMARY_PATH || '/v1/web-performance/summary';", + "const username = process.env.HWLAB_WEB_USER;", + "const password = process.env.HWLAB_WEB_PASS;", + "function compactNumber(value) { const n = Number(value); return Number.isFinite(n) ? Math.round(n * 10) / 10 : null; }", + "function compactSeconds(msValue, secondsValue) { const ms = Number(msValue); if (Number.isFinite(ms)) return Math.round((ms / 1000) * 10) / 10; return compactNumber(secondsValue); }", + "function compactString(value) { return typeof value === 'string' && value.length > 0 ? value : null; }", + "function compactRow(row) {", + " if (!row || typeof row !== 'object' || Array.isArray(row)) return null;", + " const route = compactString(row.route) || compactString(row.path) || compactString(row.name) || compactString(row.metric) || compactString(row.title);", + " return {", + " source: compactString(row.source) || compactString(row.kind) || compactString(row.category),", + " metric: compactString(row.metric) || compactString(row.eventKind) || compactString(row.phase),", + " route,", + " problem: compactString(row.problem) || compactString(row.diagnosis) || compactString(row.reason),", + " outcome: compactString(row.outcome) || compactString(row.result) || compactString(row.status),", + " tone: compactString(row.tone) || compactString(row.severity),", + " sampleState: compactString(row.sampleState) || compactString(row.sampleStatus),", + " count: compactNumber(row.count ?? row.sampleCount ?? row.samples),", + " durationUnit: 'seconds',", + " p50Seconds: compactSeconds(row.p50Ms, row.p50Seconds ?? row.p50 ?? row.medianSeconds ?? row.median),", + " p75Seconds: compactSeconds(row.p75Ms, row.p75Seconds ?? row.p75),", + " p95Seconds: compactSeconds(row.p95Ms, row.p95Seconds ?? row.p95),", + " maxSeconds: compactSeconds(row.maxMs, row.maxSeconds ?? row.max),", + " };", + "}", + "function rowHasSignal(row) { return Boolean(row && (row.source || row.metric || row.route || row.problem || row.outcome)); }", + "function compactRows(value, limit = 16) { return Array.isArray(value) ? value.map(compactRow).filter(rowHasSignal).slice(0, limit) : []; }", + "function compactTopN(value) {", + " if (!Array.isArray(value)) return [];", + " return value.slice(0, 8).map((group) => {", + " const rows = compactRows(group?.rows || group?.items || group?.data, 8);", + " return { id: compactString(group?.id) || compactString(group?.key), title: compactString(group?.title) || compactString(group?.label), rows };", + " }).filter((group) => group.rows.length > 0);", + "}", + "function compactCards(value) {", + " if (!Array.isArray(value)) return [];", + " return value.slice(0, 12).map((card) => ({ id: compactString(card?.id), title: compactString(card?.title), value: card?.value ?? null, tone: compactString(card?.tone) || compactString(card?.status), detail: compactString(card?.detail) || compactString(card?.hint) }));", + "}", + "function compactPerformance(body) {", + " if (!body || typeof body !== 'object' || Array.isArray(body)) return { ok: false, degradedReason: 'non-json-summary' };", + " const dashboard = body.dashboard && typeof body.dashboard === 'object' ? body.dashboard : {};", + " const summary = body.summary && typeof body.summary === 'object' ? body.summary : {};", + " const rows = compactRows(body.rows || body.tableRows || body.performanceRows, 24);", + " const workbenchRows = compactRows(body.workbenchRows || body.workbench || body.workbenchTableRows, 12);", + " const problems = compactRows(body.problems || body.problemRows || dashboard.problems, 24);", + " const displayRows = rows.length > 0 ? rows.slice(0, 16) : problems.slice(0, 16);", + " const displayProblems = rows.length > 0 ? [] : problems.slice(0, 16);", + " const topN = compactTopN(dashboard.topN || body.topN);", + " return {", + " ok: true,", + " schemaVersion: body.schemaVersion || dashboard.schemaVersion || null,", + " source: body.source || dashboard.source || null,", + " observedAt: body.observedAt || dashboard.observedAt || body.generatedAt || dashboard.generatedAt || null,", + " window: body.window || body.sampleWindow || dashboard.window || null,", + " freshness: body.freshness || dashboard.freshness || null,", + " sampleCount: summary.sampleCount ?? body.sampleCount ?? dashboard.sampleCount ?? null,", + " problemCount: summary.problemCount ?? body.problemCount ?? dashboard.problemCount ?? null,", + " status: summary.status || body.status || dashboard.status || null,", + " cards: compactCards(dashboard.cards || body.cards),", + " rows: displayRows,", + " workbenchRows,", + " problems: displayProblems,", + " topN,", + " rowCount: rows.length,", + " workbenchRowCount: workbenchRows.length,", + " problemRowCount: problems.length,", + " };", + "}", + "function splitSetCookie(raw) { return raw ? raw.split(/,(?=[^;,]+=)/g).map((item) => item.trim()).filter(Boolean) : []; }", + "async function fetchWithTimeout(url, init, timeoutMs = 12000) {", + " const controller = new AbortController();", + " const timer = setTimeout(() => controller.abort(), timeoutMs);", + " try {", + " const response = await fetch(url, { ...init, signal: controller.signal });", + " const text = await response.text();", + " let body = null;", + " try { body = JSON.parse(text); } catch {}", + " return { response, text, body };", + " } finally { clearTimeout(timer); }", + "}", + "function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }", + "async function fetchWithRetry(url, init, attempts = 3) {", + " let last = null;", + " for (let attempt = 1; attempt <= attempts; attempt += 1) {", + " try {", + " const result = await fetchWithTimeout(url, init);", + " result.attempt = attempt;", + " last = result;", + " if (![502, 503, 504].includes(result.response.status)) return result;", + " } catch (error) {", + " last = { attempt, error: error instanceof Error ? error.message : String(error), response: { status: 0, ok: false, headers: new Headers() }, body: null, text: '' };", + " }", + " if (attempt < attempts) await sleep(400 * attempt);", + " }", + " return last;", + "}", + "(async () => {", + " try {", + " const loginUrl = new URL('/auth/login', baseUrl).toString();", + " const login = await fetchWithRetry(loginUrl, { method: 'POST', headers: { accept: 'application/json', 'content-type': 'application/json' }, body: JSON.stringify({ username, password }) });", + " const setCookie = splitSetCookie(login.response.headers.get('set-cookie'));", + " const cookie = setCookie.map((item) => item.split(';')[0].trim()).filter(Boolean).join('; ');", + " const auth = { ok: login.response.ok && cookie.length > 0, httpStatus: login.response.status, cookiePresent: cookie.length > 0, attempts: login.attempt || 1 };", + " if (!auth.ok) {", + " console.log(JSON.stringify({ ok: false, auth, httpStatus: login.response.status, error: 'web-login-failed' }));", + " process.exitCode = 1;", + " return;", + " }", + " const summaryUrl = new URL(summaryPath, baseUrl).toString();", + " const summary = await fetchWithRetry(summaryUrl, { method: 'GET', headers: { accept: 'application/json', cookie } });", + " const performance = compactPerformance(summary.body);", + " const ok = summary.response.ok && performance.ok === true;", + " console.log(JSON.stringify({ ok, auth, httpStatus: summary.response.status, fetchAttempts: summary.attempt || 1, finalUrl: summaryUrl, performance, error: ok ? null : 'web-performance-summary-fetch-failed' }));", + " process.exitCode = ok ? 0 : 1;", + " } catch (error) {", + " console.log(JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }));", + " process.exitCode = 1;", + " }", + "})();", + ].join("\n"); + return [ + "set -eu", + `HWLAB_WEB_BASE_URL=${shellQuote(baseUrl)} HWLAB_PERFORMANCE_SUMMARY_PATH=${shellQuote(summaryPath)} HWLAB_WEB_USER=${shellQuote(username)} HWLAB_WEB_PASS=${shellQuote(password)} node <<'NODE'`, + nodeScript, + "NODE", + ].join("\n"); +} + +function nodePerformanceRows(performance: Record): Record[] { + const rows = [ + ...nodePerformanceArray(performance.problems), + ...nodePerformanceArray(performance.rows), + ...nodePerformanceArray(performance.workbenchRows), + ]; + const topN = Array.isArray(performance.topN) ? performance.topN.map(record) : []; + for (const group of topN) { + for (const row of nodePerformanceArray(group.rows)) rows.push(row); + } + const seen = new Set(); + return rows.filter((row) => { + const key = [ + nodePerformanceString(row.source), + nodePerformanceString(row.metric), + nodePerformanceString(row.route), + nodePerformanceString(row.outcome), + nodePerformanceString(row.problem), + ].join("|"); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function nodePerformanceArray(value: unknown): Record[] { + return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : []; +} + +function nodePerformanceAnalysis(performance: Record, rows: Record[]): Record { + const networkErrors = rows.filter((row) => nodePerformanceString(row.outcome) === "network_error" || nodePerformanceString(row.problem) === "network_error" || nodePerformanceString(row.problem) === "network error"); + const blockedRows = rows.filter((row) => nodePerformanceString(row.tone) === "blocked" || nodePerformanceString(row.status) === "blocked"); + const slowApiRows = rows.filter((row) => { + const source = nodePerformanceString(row.source) ?? ""; + const metric = nodePerformanceString(row.metric) ?? ""; + const p95Seconds = nodePerformanceNumber(row.p95Seconds); + return (source.includes("api") || metric.includes("api") || (nodePerformanceString(row.route) ?? "").startsWith("/v1") || (nodePerformanceString(row.route) ?? "").startsWith("/health")) && (p95Seconds === null || p95Seconds >= 5); + }); + const lcpRows = rows.filter((row) => { + const metric = `${nodePerformanceString(row.metric) ?? ""} ${nodePerformanceString(row.problem) ?? ""}`.toLowerCase(); + return metric.includes("lcp") || metric.includes("largest contentful") || metric.includes("最大内容"); + }); + const maxP95Seconds = Math.max(0, ...rows.map((row) => nodePerformanceNumber(row.p95Seconds) ?? 0)); + const headline = networkErrors.length > 0 + ? "Public same-origin API has intermittent network_error samples; treat edge/proxy/API reachability as P0 before tuning page render." + : lcpRows.length > 0 + ? "Workbench page LCP is blocked, likely by first-load API waterfalls and late visible content." + : slowApiRows.length > 0 + ? "Same-origin API p95 is above the interactive budget; split edge wait, server work, and dependency latency." + : "No blocking performance rows were returned by the current summary window."; + const priorities = [ + networkErrors.length > 0 + ? { + priority: "P0", + area: "public-edge-api-reachability", + action: "Correlate network_error rows with Caddy/FRP/ingress, cloud-api pod restarts, response-header timeout, and browser Resource Timing failure stage.", + evidence: nodePerformanceEvidence(networkErrors), + } + : null, + slowApiRows.length > 0 + ? { + priority: "P1", + area: "same-origin-api-latency", + action: "Add server-side route histograms and dependency spans for slow /health, /v1/live-builds, /v1/workbench/sessions, trace events, hwpod specs, and hwpod-node-ops paths; then optimize the highest p95 route first.", + evidence: nodePerformanceEvidence(slowApiRows), + } + : null, + lcpRows.length > 0 + ? { + priority: "P1", + area: "workbench-lcp", + action: "Render a stable first screen before slow session/detail API calls complete, and split LCP by route plus blocking API waterfall.", + evidence: nodePerformanceEvidence(lcpRows), + } + : null, + blockedRows.length > 0 + ? { + priority: "P2", + area: "alert-thresholds-and-sample-quality", + action: "Keep low-sample rows visible but avoid closing incidents on count=1; require repeated windows or matching server metrics before labeling a regression fixed.", + evidence: nodePerformanceEvidence(blockedRows), + } + : null, + ].filter(Boolean); + return { + headline, + sampleCount: performance.sampleCount ?? null, + problemCount: performance.problemCount ?? null, + rowCount: rows.length, + blockedRowCount: blockedRows.length, + networkErrorRowCount: networkErrors.length, + slowApiRowCount: slowApiRows.length, + lcpRowCount: lcpRows.length, + maxP95Seconds, + priorities, + }; +} + +function nodePerformanceEvidence(rows: Record[]): Record[] { + return rows.slice(0, 8).map((row) => ({ + source: nodePerformanceString(row.source), + metric: nodePerformanceString(row.metric), + route: nodePerformanceString(row.route), + outcome: nodePerformanceString(row.outcome), + problem: nodePerformanceString(row.problem), + count: nodePerformanceNumber(row.count), + durationUnit: nodePerformanceString(row.durationUnit) ?? "seconds", + p50Seconds: nodePerformanceNumber(row.p50Seconds), + p75Seconds: nodePerformanceNumber(row.p75Seconds), + p95Seconds: nodePerformanceNumber(row.p95Seconds), + })); +} + +function nodePerformanceString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function nodePerformanceNumber(value: unknown): number | null { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + function summarizeNodeObservabilityStatus(status: Record): Record { const service = record(status.service); const publicRawMetrics = record(status.publicRawMetrics);