From da3efa4c6ee946a45d25183ad5533c16cffccb98 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 04:19:32 +0000 Subject: [PATCH 1/6] fix: classify web-probe long-lived streams --- .../hwlab-node-web-observe-runner-source.ts | 64 +++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 2834cc16..ecebc644 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -1029,7 +1029,7 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } @@ -1088,6 +1088,8 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN if ((runtimeAlerts?.summary?.consoleAlertCount ?? 0) > 0) findings.push({ id: "runtime-console-alerts", severity: "amber", summary: "browser console warning/error entries were captured during observation", count: runtimeAlerts.summary.consoleAlertCount, groups: runtimeAlerts.consoleAlertsByPath.slice(0, 12) }); const slowApi = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0) : []; if (slowApi.length > 0) findings.push({ id: "page-performance-slow-same-origin-api", severity: "red", summary: "same-origin API resource timing exceeded 5s usability budget", count: slowApi.length, groups: slowApi.slice(0, 20) }); + const longLivedStreams = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream) : []; + if (longLivedStreams.length > 0) findings.push({ id: "page-performance-long-lived-streams", severity: "info", summary: "same-origin long-lived streams are reported separately; lifetime is not treated as API load latency", count: longLivedStreams.length, groups: longLivedStreams.slice(0, 20) }); if ((pageProvenance?.summary?.segmentCount ?? 0) > 1) findings.push({ id: "page-provenance-segments", severity: "info", summary: "observer crossed page asset provenance segments; interpret runtime findings by segment", segmentCount: pageProvenance.summary.segmentCount, segments: pageProvenance.segments.slice(0, 20) }); const naturalApi = network.filter((item) => item.observerInitiated === false && item.type === "response" && /\/v1\/|\/auth\//u.test(String(item.url || ""))); findings.push({ id: "natural-api-dom-lag-baseline", severity: "info", summary: "natural API responses and DOM samples are available for manual lag correlation", naturalApiResponses: naturalApi.length, sampleCount: samples.length }); @@ -1160,16 +1162,24 @@ function buildPagePerformanceReport(samples, manifest) { const parsed = parsePerformanceUrl(entry?.name, base); if (!parsed.sameOrigin || !isApiLikePath(parsed.path)) continue; const normalizedPath = normalizeApiPath(parsed.path); + const routeKind = classifyApiPerformanceRoute(normalizedPath, entry); + const isLongLivedStream = routeKind === "same-origin-api-stream"; + const streamOpenMs = streamOpenLatencyMs(entry); const dedupeKey = [parsed.path, entry.initiatorType || "", entry.startTime ?? "", Math.round(durationMs)].join("|"); if (seen.has(dedupeKey)) continue; seen.add(dedupeKey); const group = groups.get(normalizedPath) || { - routeKind: "same-origin-api", + routeKind, path: normalizedPath, + isLongLivedStream, + budgetMetric: isLongLivedStream ? "streamOpenMs" : "durationMs", rawPathSamples: [], sampleCount: 0, durationsMs: [], + streamOpenDurationsMs: [], overFiveSecondCount: 0, + streamLifetimeOverFiveSecondCount: 0, + streamOpenOverFiveSecondCount: 0, firstAt: sample.ts ?? null, lastAt: sample.ts ?? null, firstSeq: sample.seq ?? null, @@ -1180,7 +1190,18 @@ function buildPagePerformanceReport(samples, manifest) { }; group.sampleCount += 1; group.durationsMs.push(durationMs); - if (durationMs > 5000) group.overFiveSecondCount += 1; + if (isLongLivedStream) { + if (durationMs > 5000) group.streamLifetimeOverFiveSecondCount += 1; + if (streamOpenMs !== null) { + group.streamOpenDurationsMs.push(streamOpenMs); + if (streamOpenMs > 5000) { + group.streamOpenOverFiveSecondCount += 1; + group.overFiveSecondCount += 1; + } + } + } else if (durationMs > 5000) { + group.overFiveSecondCount += 1; + } group.lastAt = sample.ts ?? null; group.lastSeq = sample.seq ?? null; if (parsed.path && !group.rawPathSamples.includes(parsed.path)) group.rawPathSamples.push(parsed.path); @@ -1192,14 +1213,24 @@ function buildPagePerformanceReport(samples, manifest) { } const sameOriginApiByPath = Array.from(groups.values()).map((group) => { const durations = group.durationsMs.slice().sort((a, b) => a - b); + const streamOpenDurations = group.streamOpenDurationsMs.slice().sort((a, b) => a - b); return { routeKind: group.routeKind, path: group.path, + isLongLivedStream: group.isLongLivedStream === true, + budgetMetric: group.budgetMetric, sampleCount: group.sampleCount, p50Ms: percentile(durations, 50), p75Ms: percentile(durations, 75), p95Ms: percentile(durations, 95), maxMs: durations.length > 0 ? durations[durations.length - 1] : null, + streamOpenSampleCount: streamOpenDurations.length, + streamOpenP50Ms: percentile(streamOpenDurations, 50), + streamOpenP75Ms: percentile(streamOpenDurations, 75), + streamOpenP95Ms: percentile(streamOpenDurations, 95), + streamOpenMaxMs: streamOpenDurations.length > 0 ? streamOpenDurations[streamOpenDurations.length - 1] : null, + streamOpenOverFiveSecondCount: group.streamOpenOverFiveSecondCount, + streamLifetimeOverFiveSecondCount: group.streamLifetimeOverFiveSecondCount, overFiveSecondCount: group.overFiveSecondCount, overFiveSecondRatio: group.sampleCount > 0 ? Number((group.overFiveSecondCount / group.sampleCount).toFixed(3)) : 0, firstAt: group.firstAt, @@ -1213,14 +1244,20 @@ function buildPagePerformanceReport(samples, manifest) { }; }).sort((a, b) => (b.overFiveSecondCount - a.overFiveSecondCount) || (Number(b.p95Ms ?? 0) - Number(a.p95Ms ?? 0)) || a.path.localeCompare(b.path)); const slow = sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0); + const longLivedStreams = sameOriginApiByPath.filter((item) => item.isLongLivedStream); + const budgetP95Values = sameOriginApiByPath + .map((item) => Number(item.isLongLivedStream ? (item.streamOpenP95Ms ?? 0) : (item.p95Ms ?? 0))) + .filter((value) => Number.isFinite(value)); return { summary: { budgetMs: 5000, sameOriginApiPathCount: sameOriginApiByPath.length, sameOriginApiSampleCount: sameOriginApiByPath.reduce((sum, item) => sum + item.sampleCount, 0), + longLivedStreamPathCount: longLivedStreams.length, + longLivedStreamSampleCount: longLivedStreams.reduce((sum, item) => sum + item.sampleCount, 0), slowPathCount: slow.length, slowSampleCount: slow.reduce((sum, item) => sum + item.overFiveSecondCount, 0), - worstP95Ms: sameOriginApiByPath.length > 0 ? Math.max(...sameOriginApiByPath.map((item) => Number(item.p95Ms ?? 0))) : null, + worstP95Ms: budgetP95Values.length > 0 ? Math.max(...budgetP95Values) : null, valuesRedacted: true }, sameOriginApiByPath, @@ -1228,6 +1265,21 @@ function buildPagePerformanceReport(samples, manifest) { }; } +function classifyApiPerformanceRoute(normalizedPath, entry = {}) { + if (normalizedPath === "/v1/workbench/events") return "same-origin-api-stream"; + if (String(entry?.initiatorType ?? "").toLowerCase() === "eventsource") return "same-origin-api-stream"; + return "same-origin-api"; +} + +function streamOpenLatencyMs(entry = {}) { + const responseStart = Number(entry?.responseStart); + const startTime = Number(entry?.startTime); + if (!Number.isFinite(responseStart) || responseStart <= 0) return null; + if (!Number.isFinite(startTime) || startTime < 0) return Math.max(0, responseStart); + if (responseStart < startTime) return null; + return Math.max(0, responseStart - startTime); +} + function parsePerformanceUrl(value, base) { try { const url = new URL(String(value || ""), base); @@ -2348,7 +2400,7 @@ function renderMarkdown(report) { ? report.pageProvenance.segments.slice(0, 40).map((item) => "- fingerprint=" + (item.assetFingerprint || "-") + " samples=" + item.sampleCount + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " scripts=" + (item.scriptCount ?? "-") + " styles=" + (item.stylesheetCount ?? "-") + " urlPaths=" + (Array.isArray(item.urlPaths) ? item.urlPaths.slice(0, 4).join(",") : "-")).join("\n") : "- 无页面 provenance segment。"; const performanceLines = Array.isArray(report.pagePerformance?.sameOriginApiByPath) && report.pagePerformance.sameOriginApiByPath.length > 0 - ? report.pagePerformance.sameOriginApiByPath.slice(0, 80).map((item) => "- " + item.path + " samples=" + item.sampleCount + " p50=" + (item.p50Ms ?? "-") + "ms p75=" + (item.p75Ms ?? "-") + "ms p95=" + (item.p95Ms ?? "-") + "ms max=" + (item.maxMs ?? "-") + "ms >5s=" + (item.overFiveSecondCount ?? 0) + " window=" + (item.firstAt || "-") + ".." + (item.lastAt || "-")).join("\n") + ? report.pagePerformance.sameOriginApiByPath.slice(0, 80).map((item) => "- " + item.path + " kind=" + (item.routeKind || "same-origin-api") + " budgetMetric=" + (item.budgetMetric || "durationMs") + " samples=" + item.sampleCount + " p50=" + (item.p50Ms ?? "-") + "ms p75=" + (item.p75Ms ?? "-") + "ms p95=" + (item.p95Ms ?? "-") + "ms max=" + (item.maxMs ?? "-") + "ms >5s=" + (item.overFiveSecondCount ?? 0) + " streamOpenP95=" + (item.streamOpenP95Ms ?? "-") + "ms streamLifetime>5s=" + (item.streamLifetimeOverFiveSecondCount ?? 0) + " window=" + (item.firstAt || "-") + ".." + (item.lastAt || "-")).join("\n") : "- 无同源 API Resource Timing 样本。"; const metricLines = Array.isArray(report.sampleMetrics?.timeline) && report.sampleMetrics.timeline.length > 0 ? report.sampleMetrics.timeline.slice(0, 120).map((item) => "- #" + item.seq + " " + item.ts + " prompt=" + item.promptIndex + " totalElapsedSeconds=" + (item.totalElapsedSeconds ?? "-") + " recentUpdateSeconds=" + (item.recentUpdateSeconds ?? "-") + " terminal=" + item.terminalSeen + " finalText=" + item.finalResultTextSeen + " diagnostic=" + item.diagnosticSeen).join("\n") @@ -2388,6 +2440,8 @@ function renderMarkdown(report) { + "- budgetMs: " + (report.pagePerformance?.summary?.budgetMs ?? 5000) + "\n" + "- sameOriginApiPathCount: " + (report.pagePerformance?.summary?.sameOriginApiPathCount ?? 0) + "\n" + "- sameOriginApiSampleCount: " + (report.pagePerformance?.summary?.sameOriginApiSampleCount ?? 0) + "\n" + + "- longLivedStreamPathCount: " + (report.pagePerformance?.summary?.longLivedStreamPathCount ?? 0) + "\n" + + "- longLivedStreamSampleCount: " + (report.pagePerformance?.summary?.longLivedStreamSampleCount ?? 0) + "\n" + "- slowPathCount: " + (report.pagePerformance?.summary?.slowPathCount ?? 0) + "\n" + "- slowSampleCount: " + (report.pagePerformance?.summary?.slowSampleCount ?? 0) + "\n" + "- worstP95Ms: " + (report.pagePerformance?.summary?.worstP95Ms ?? "-") + "\n\n" From 06bffaf74a3bf231c68bd482c8d32ea398a5f228 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 16:46:17 +0000 Subject: [PATCH 2/6] fix cli output dumping and otel web-probe diagnostics --- AGENTS.md | 7 + config/cli-output.yaml | 6 + scripts/cli.ts | 10 + scripts/src/gh.ts | 198 ++++++++- scripts/src/hwlab-node-impl.ts | 247 +++++++++++- .../hwlab-node-web-observe-runner-source.ts | 2 +- scripts/src/output.ts | 170 +++++++- scripts/src/platform-infra-observability.ts | 378 +++++++++++++++++- scripts/src/ssh.ts | 17 +- 9 files changed, 959 insertions(+), 76 deletions(-) create mode 100644 config/cli-output.yaml diff --git a/AGENTS.md b/AGENTS.md index 0b342af6..eed241dc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,13 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - P0: `scripts/cli.ts`、`scripts/src/**`、`trans`、`tran` 和轻量 helper 的 CLI 改动，除非用户明确要求，否则不做单元测试、合同测试或新增测试脚本；默认最多只做语法检查和必要的帮助/命令形态人工确认。 - P0: 历史合同测试和 prompt lint 属于过度门禁，不再作为 CLI 变更的默认验证、派单前置或关闭依据；发现旧入口阻碍最新目标时直接删除，不做兼容保留。 +## P0 最高优先级：CLI 输出渐进披露规则 + +- P0: CLI 默认输出必须优先采用类似 Kubernetes CLI 的简洁表格、短摘要和可复制的 drill-down 命令；JSON 只作为 `--json`、`--raw`、`--full` 或明确机器消费模式输出，禁止把大段 JSON 当作面向人的默认阅读界面。 +- P0: 看到高频 CLI 默认输出仍是 JSON、长对象、长数组或容易超过 YAML 配置阈值的噪声输出时，必须见到一个改一个，优先把该命令改成表格优先、可再展开的渐进式披露形态，而不是反复依赖 dump 后再人工抽取。 +- P0: 超过 YAML 配置阈值的 CLI stdout 没有被自动 dump 是比单个命令输出 JSON 更严重的全局可见性事故；dump 兜底必须对所有 `emitJson`/`emitText` 全局生效，不能按命令白名单选择性启用，也不能让超长输出无缓冲直冲上下文。 +- P0: 自动 dump 只作为防止终端爆炸的兜底能力；一旦某个常用命令反复触发 dump，必须把 warning 视为 CLI 可用性缺陷并改进命令自身输出，不能把 dump 文件路径变成长期交互入口。 + ## P0 最高优先级：自有配置 YAML 优先规则 - P0: UniDesk 自有配置一律优先使用 YAML（`.yaml`/`.yml`），包括 `config/` 下的运行面、平台基础设施、节点/lane、部署参数和可调版本配置；除非外部工具硬性要求 JSON/TOML/ENV 等格式，禁止新增 JSON 作为 UniDesk 自有配置真相。 diff --git a/config/cli-output.yaml b/config/cli-output.yaml new file mode 100644 index 00000000..36737ac4 --- /dev/null +++ b/config/cli-output.yaml @@ -0,0 +1,6 @@ +output: + dump: + enabled: true + thresholdBytes: 10240 + previewChars: 2000 + dir: /tmp/unidesk-cli-output diff --git a/scripts/cli.ts b/scripts/cli.ts index 2351d367..296590ca 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -290,6 +290,11 @@ async function main(): Promise { if (top === "gh") { const result = await runGhCommand(args.slice(1)); const ok = (result as { ok?: unknown }).ok !== false; + if (isRenderedCliResult(result)) { + emitText(result.renderedText); + if (!ok) process.exitCode = 1; + return; + } emitJson(commandName, result, ok); if (!ok) process.exitCode = 1; return; @@ -316,6 +321,11 @@ async function main(): Promise { const { runHwlabNodeCommand } = await import("./src/hwlab-node"); const result = await runHwlabNodeCommand(readConfig(), args.slice(2)); const ok = (result as { ok?: unknown }).ok !== false; + if (isRenderedCliResult(result)) { + emitText(result.renderedText); + if (!ok) process.exitCode = 1; + return; + } emitJson(commandName, result, ok); if (!ok) process.exitCode = 1; return; diff --git a/scripts/src/gh.ts b/scripts/src/gh.ts index 4b83abb9..5ae7e5cf 100644 --- a/scripts/src/gh.ts +++ b/scripts/src/gh.ts @@ -4703,6 +4703,160 @@ function compactCommentSummary(comment: GitHubComment): Record }; } +function compactGhValue(value: unknown, maxLength = 96): string { + if (value === null || value === undefined) return "-"; + const text = typeof value === "string" ? value : String(value); + const singleLine = text.replace(/\s+/gu, " ").trim(); + if (singleLine.length <= maxLength) return singleLine.length === 0 ? "-" : singleLine; + return `${singleLine.slice(0, Math.max(1, maxLength - 1))}...`; +} + +function shortGhSha(value: unknown): string { + const text = typeof value === "string" ? value : ""; + if (text.length <= 12) return text || "-"; + return text.slice(0, 12); +} + +function renderGhTable(headers: string[], rows: unknown[][]): string { + const stringRows = rows.map((row) => row.map((value) => compactGhValue(value))); + const widths = headers.map((header, index) => Math.max( + header.length, + ...stringRows.map((row) => row[index]?.length ?? 0), + )); + const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); + return [renderRow(headers), ...stringRows.map(renderRow)].join("\n"); +} + +function withRenderedGhResult(result: GitHubCommandResult, renderedText: string): GitHubCommandResult { + return { + ...result, + renderedText, + contentType: "text/plain", + }; +} + +function renderIssueCreateResult(result: GitHubCommandResult): GitHubCommandResult { + const issue = isRecord(result.issue) ? result.issue : {}; + const bodySource = isRecord(result.bodySource) ? result.bodySource : {}; + const status = result.dryRun === true ? "planned" : "created"; + const renderedText = [ + renderGhTable( + ["COMMAND", "REPO", "STATUS", "ISSUE", "TITLE", "URL"], + [[result.command, result.repo, status, issue.number ?? "-", issue.title ?? result.title ?? "-", issue.url ?? "-"]], + ), + "", + renderGhTable( + ["BODY", "CHARS", "SHA", "SOURCE"], + [["body", issue.bodyChars ?? result.bodyChars ?? "-", shortGhSha(issue.bodySha ?? result.bodySha), bodySource.kind ?? result.source ?? "-"]], + ), + "", + "NEXT", + ` read: bun scripts/cli.ts gh issue view ${issue.number ?? ""} --repo ${result.repo}`, + ` full: bun scripts/cli.ts gh issue view ${issue.number ?? ""} --repo ${result.repo} --full`, + ].join("\n"); + return withRenderedGhResult(result, renderedText); +} + +function renderIssueCommentCreateResult(result: GitHubCommandResult): GitHubCommandResult { + const comment = isRecord(result.comment) ? result.comment : {}; + const status = result.dryRun === true ? "planned" : "created"; + const target = result.issueNumber === undefined ? "-" : `#${result.issueNumber}`; + const renderedText = [ + renderGhTable( + ["COMMAND", "REPO", "TARGET", "STATUS", "COMMENT_ID", "URL"], + [[result.command, result.repo, target, status, comment.id ?? "-", comment.url ?? "-"]], + ), + "", + renderGhTable( + ["BODY", "CHARS", "SHA", "SOURCE"], + [["body", result.bodyChars ?? comment.bodyChars ?? "-", shortGhSha(result.bodySha ?? comment.bodySha), result.source ?? "-"]], + ), + "", + "NEXT", + ` comments: bun scripts/cli.ts gh issue view ${result.issueNumber ?? ""} --repo ${result.repo} --json comments`, + ].join("\n"); + return withRenderedGhResult(result, renderedText); +} + +function renderPrCreateResult(result: GitHubCommandResult): GitHubCommandResult { + const pr = isRecord(result.pr) ? result.pr : {}; + const validation = isRecord(result.validation) ? result.validation : {}; + const compare = isRecord(validation.compare) ? validation.compare : {}; + const status = result.dryRun === true ? "planned" : "created"; + const renderedText = [ + renderGhTable( + ["COMMAND", "REPO", "STATUS", "PR", "TITLE", "BASE", "HEAD", "URL"], + [[result.command, result.repo, status, pr.number ?? "-", pr.title ?? result.title ?? "-", pr.baseRefName ?? result.base ?? "-", pr.headRefName ?? result.head ?? "-", pr.url ?? "-"]], + ), + "", + renderGhTable( + ["CHECK", "VALUE"], + [ + ["aheadBy", compare.aheadBy ?? "-"], + ["draft", result.draft ?? pr.draft ?? "-"], + ["bodySource", isRecord(validation.bodySource) ? validation.bodySource.kind ?? "-" : "-"], + ], + ), + "", + "NEXT", + ` view: bun scripts/cli.ts gh pr view ${pr.number ?? ""} --repo ${result.repo}`, + ` preflight: bun scripts/cli.ts gh pr preflight ${pr.number ?? ""} --repo ${result.repo}`, + ].join("\n"); + return withRenderedGhResult(result, renderedText); +} + +function renderPrCommentCreateResult(result: GitHubCommandResult): GitHubCommandResult { + const comment = isRecord(result.comment) ? result.comment : {}; + const pr = isRecord(result.pr) ? result.pr : {}; + const status = result.dryRun === true ? "planned" : "created"; + const target = result.issueNumber === undefined ? "-" : `#${result.issueNumber}`; + const renderedText = [ + renderGhTable( + ["COMMAND", "REPO", "PR", "STATUS", "COMMENT_ID", "URL"], + [[result.command, result.repo, target, status, comment.id ?? "-", comment.url ?? "-"]], + ), + "", + renderGhTable( + ["BODY", "CHARS", "SHA", "SOURCE"], + [["body", result.bodyChars ?? "-", shortGhSha(result.bodySha), result.source ?? "-"]], + ), + "", + "NEXT", + ` view: bun scripts/cli.ts gh pr view ${pr.number ?? result.issueNumber ?? ""} --repo ${result.repo}`, + ].join("\n"); + return withRenderedGhResult(result, renderedText); +} + +function renderPrMergeResult(result: GitHubCommandResult): GitHubCommandResult { + const pr = isRecord(result.pullRequest) ? result.pullRequest : {}; + const mergeability = isRecord(result.mergeability) ? result.mergeability : {}; + const branchDeletion = isRecord(result.branchDeletion) ? result.branchDeletion : {}; + const status = result.alreadyMerged === true + ? "already-merged" + : result.dryRun === true + ? "planned" + : "merged"; + const renderedText = [ + renderGhTable( + ["COMMAND", "REPO", "PR", "STATUS", "METHOD", "TITLE", "URL"], + [[result.command, result.repo, result.number ?? pr.number ?? "-", status, result.method ?? "-", pr.title ?? "-", pr.url ?? "-"]], + ), + "", + renderGhTable( + ["CHECK", "VALUE"], + [ + ["mergeability", mergeability.conclusion ?? (result.alreadyMerged === true ? "already-merged" : "-")], + ["deleteBranch", branchDeletion.attempted === true ? "attempted" : branchDeletion.skippedReason ?? result.deleteBranch ?? "-"], + ["mergedAt", pr.mergedAt ?? "-"], + ], + ), + "", + "NEXT", + ` view: bun scripts/cli.ts gh pr view ${result.number ?? pr.number ?? ""} --repo ${result.repo}`, + ].join("\n"); + return withRenderedGhResult(result, renderedText); +} + function prStateDetail(pr: GitHubPullRequest): "open" | "closed" | "merged" { if (pr.merged === true || pr.merged_at !== null && pr.merged_at !== undefined) return "merged"; return pr.state === "closed" ? "closed" : "open"; @@ -5131,7 +5285,7 @@ async function prMerge(repo: string, token: string, number: number, options: Git if (isGitHubError(pr)) return commandError("pr merge", repo, pr, { number, phase: "fetch-pr" }); const summary = prSummary(pr); if (summary.merged === true) { - return { + return renderPrMergeResult({ ok: true, command: "pr merge", repo, @@ -5141,7 +5295,7 @@ async function prMerge(repo: string, token: string, number: number, options: Git pullRequest: summary, branchDeletion: { attempted: false, skippedReason: "already-merged" }, rest: true, - }; + }); } const metadata = await prGraphqlMetadata(repo, token, number); if (isGitHubError(metadata)) return commandError("pr merge", repo, metadata, { number, phase: "fetch-pr-closeout-metadata", pullRequest: summary }); @@ -5157,7 +5311,7 @@ async function prMerge(repo: string, token: string, number: number, options: Git }); } if (options.dryRun) { - return { + return renderPrMergeResult({ ok: true, command: "pr merge", repo, @@ -5169,7 +5323,7 @@ async function prMerge(repo: string, token: string, number: number, options: Git pullRequest: preflightPullRequestSummary(summary), mergeability, statusChecks, - }; + }); } const merged = await githubRequest>(token, "PUT", `/repos/${owner}/${name}/pulls/${number}/merge`, { merge_method: options.mergeMethod, @@ -5178,7 +5332,7 @@ async function prMerge(repo: string, token: string, number: number, options: Git const after = await githubRequest(token, "GET", `/repos/${owner}/${name}/pulls/${number}`); if (isGitHubError(after)) return commandError("pr merge", repo, after, { number, phase: "fetch-after-merge", mergeResult: merged }); const branchDeletion = options.deleteBranch ? await deleteHeadBranchAfterMerge(repo, token, after) : { attempted: false, skippedReason: "delete-branch-not-requested" }; - return { + return renderPrMergeResult({ ok: true, command: "pr merge", repo, @@ -5188,7 +5342,7 @@ async function prMerge(repo: string, token: string, number: number, options: Git pullRequest: prSummary(after), branchDeletion, rest: true, - }; + }); } async function prPreflight(repo: string, number: number, commandName: "preflight" | "pr preflight" | "pr closeout", includeRaw: boolean): Promise { @@ -5407,7 +5561,7 @@ async function prCreate(repo: string, token: string, options: GitHubOptions): Pr const body = bodySource.body; const planned = prCreatePlannedOperation(repo, options, body, bodySource.bodySource); if (options.dryRun) { - return { + return renderPrCreateResult({ ok: true, command: "pr create", repo, @@ -5415,7 +5569,7 @@ async function prCreate(repo: string, token: string, options: GitHubOptions): Pr planned: true, draft: options.draft, ...planned, - }; + }); } const repoResult = await repoInfo(token, repo); @@ -5445,7 +5599,7 @@ async function prCreate(repo: string, token: string, options: GitHubOptions): Pr }; const pr = await githubRequest(token, "POST", `/repos/${owner}/${name}/pulls`, payload); if (isGitHubError(pr)) return commandError("pr create", repo, pr, { base: options.base, head: options.head, planned }); - return { + return renderPrCreateResult({ ok: true, command: "pr create", repo, @@ -5467,7 +5621,7 @@ async function prCreate(repo: string, token: string, options: GitHubOptions): Pr bodyChars: body.length, }, rest: true, - }; + }); } async function prComment(repo: string, token: string, issueNumber: number, options: GitHubOptions): Promise { @@ -5480,7 +5634,7 @@ async function prComment(repo: string, token: string, issueNumber: number, optio const body = bodySource.body; const planned = prCommentPlannedOperation(repo, issueNumber, body, bodySource.bodySource); if (options.dryRun) { - return { + return renderPrCommentCreateResult({ ok: true, command: "pr comment", repo, @@ -5488,7 +5642,7 @@ async function prComment(repo: string, token: string, issueNumber: number, optio planned: true, issueNumber, ...planned, - }; + }); } const repoResult = await repoInfo(token, repo); @@ -5498,7 +5652,7 @@ async function prComment(repo: string, token: string, issueNumber: number, optio if (isGitHubError(prResult)) return commandError("pr comment", repo, prResult, { issueNumber, planned }); const comment = await githubRequest(token, "POST", `/repos/${owner}/${name}/issues/${issueNumber}/comments`, { body }); if (isGitHubError(comment)) return commandError("pr comment", repo, comment, { issueNumber, planned }); - return { + return renderPrCommentCreateResult({ ok: true, command: "pr comment create", repo, @@ -5521,7 +5675,7 @@ async function prComment(repo: string, token: string, issueNumber: number, optio bodySource: bodySource.bodySource, }, rest: true, - }; + }); } async function prUpdate(repo: string, token: string, number: number, options: GitHubOptions, commandName = "pr update"): Promise { @@ -6375,7 +6529,7 @@ async function issueCreate(repo: string, token: string, options: GitHubOptions): }, "issue create"); const labels = options.labels; if (options.dryRun) { - return { + return renderIssueCreateResult({ ok: true, command: "issue create", repo, @@ -6384,7 +6538,7 @@ async function issueCreate(repo: string, token: string, options: GitHubOptions): title: options.title, labels, ...writeBodyPlan("issue create", repo, body, bodySource, { title: options.title, labels }), - }; + }); } const { owner, name } = repoParts(repo); const payload: Record = { title: options.title, body }; @@ -6401,7 +6555,7 @@ async function issueCreate(repo: string, token: string, options: GitHubOptions): missingLabels, }); } - return { ok: true, command: "issue create", repo, issue: issueSummary(issue), labels, bodySource, rest: true }; + return renderIssueCreateResult({ ok: true, command: "issue create", repo, issue: issueSummary(issue), labels, bodySource, rest: true }); } async function issuePatch(repo: string, token: string, issueNumber: number, options: GitHubOptions): Promise { @@ -6712,7 +6866,7 @@ async function issueComment(repo: string, token: string, issueNumber: number, op } const { body, bodySource } = bodyInput; if (options.dryRun) { - return { + return renderIssueCommentCreateResult({ ok: true, command: "issue comment create", repo, @@ -6721,12 +6875,12 @@ async function issueComment(repo: string, token: string, issueNumber: number, op issueNumber, readCommands: issueCommentReadCommands(repo, issueNumber), ...writeBodyPlan("issue comment create", repo, body, bodySource, { issueNumber }), - }; + }); } const { owner, name } = repoParts(repo); const comment = await githubRequest(token, "POST", `/repos/${owner}/${name}/issues/${issueNumber}/comments`, { body }); if (isGitHubError(comment)) return commandError("issue comment", repo, comment, { issueNumber }); - return { + return renderIssueCommentCreateResult({ ok: true, command: "issue comment create", repo, @@ -6739,7 +6893,7 @@ async function issueComment(repo: string, token: string, issueNumber: number, op source: String(bodySource.kind ?? "unknown"), readCommands: issueCommentReadCommands(repo, issueNumber), rest: true, - }; + }); } async function commentPatch(repo: string, token: string, ownerKind: "issue" | "pr", commentId: number, options: GitHubOptions, commandName?: string): Promise { @@ -7741,7 +7895,7 @@ export function ghHelp(): unknown { "issue view is the canonical GitHub CLI-compatible read path; read remains a UniDesk compatibility alias. View/read accept positional numbers, GitHub issue URLs, and owner/repo#number shorthand, deriving --repo unless an explicit conflicting --repo is supplied. --number is accepted on single issue/comment numeric target commands for low-friction compatibility and returns a standard syntax hint; list/create/scan-escape/cleanup-plan/board-audit/board-row list do not accept it. Comment delete treats --number as commentId, not an issue number. View supports lifecycle fields closed/closedAt plus legacy --json field selection; full body is included only when requested with --json body, --full, or --raw, and unsupported fields fail structurally.", "issue attachment list/download scan issue body and comments for GitHub user attachment URLs (`https://github.com/user-attachments/assets/...`). list is read-only and returns bounded attachment metadata. download writes the selected attachment to --output or /tmp/unidesk-gh-attachments, returns bytes/SHA-256/content-type/path, redacts redirected signed URL query parameters, and never prints binary bytes.", "--raw and --full are explicit full-disclosure aliases for gh issue list/read/view/update/edit/patch and gh pr list/read/view. For issue writes, default success output omits full issue.body and returns bodyChars/bodySha/bodyPreview plus readCommands; --full|--raw includes the full returned issue body only on commands that explicitly support full disclosure.", - "GitHub CLI output larger than 20 KiB is automatically written to /tmp/unidesk-cli-output/*.json; stdout stays bounded JSON with outputTruncated=true, the dump path, total bytes/lines, and head/tail previews.", + "All CLI stdout larger than the YAML-configured dump threshold is automatically written to /tmp/unidesk-cli-output/*; stdout stays bounded with outputTruncated=true or UNIDESK_CLI_OUTPUT_TRUNCATED, the dump path, total bytes/lines, and head/tail commands.", "issue create accepts --body-stdin or --body-file plus repeatable --label values and comma-separated labels; inline --body is intentionally unsupported for issue creation. Dry-run prints the parsed labels and non-dry-run sends them in the GitHub REST create-issue payload.", "--body-stdin is the first-class heredoc/stdin source for Markdown bodies. Use quoted heredoc syntax such as bun scripts/cli.ts gh issue comment create 1 --body-stdin <<'EOF' so real newlines, backticks, and tables are read as stdin bytes instead of shell arguments.", "update defaults to --mode replace; --mode append reads the current body and appends file bytes so real newlines, backticks, and Markdown tables are preserved.", diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index aec3c82d..93ffab52 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -6610,12 +6610,19 @@ function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const status = parseJsonObject(result.stdout); const observerId = webObserveIdFromStatus(status, options); - const index = status?.ok !== false && observerId !== null && options.stateDir !== null + const statusOk = status !== null && status.ok !== false; + const commandResult = compactCommandResult(result); + if (status === null) { + commandResult.failureKind = "web-observe-status-parse-failed"; + commandResult.stdoutTail = result.stdout.slice(-2000); + commandResult.stderrTail = result.stderr.slice(-2000); + } + const index = statusOk && observerId !== null && options.stateDir !== null ? upsertWebObserveIndexEntry(webObserveIndexEntryFromOptions(options, spec, observerId, status)) : null; - return { - ok: result.exitCode === 0 && status?.ok !== false, - status: result.exitCode === 0 ? "observed" : "blocked", + return renderWebObserveStatusResult({ + ok: result.exitCode === 0 && statusOk, + status: result.exitCode === 0 && statusOk ? "observed" : "blocked", command: webObserveCommandLabel("status", options), id: observerId, node: options.node, @@ -6624,9 +6631,9 @@ function runNodeWebProbeObserveStatus(options: NodeWebProbeObserveOptions, spec: observer: withWebObserveShortcuts(status, observerId), index, next: observerId === null ? null : webObserveNextCommands(observerId), - result: compactCommandResult(result), + result: commandResult, valuesRedacted: true, - }; + }); } function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec, stopCommand: boolean): Record { @@ -6661,7 +6668,7 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec "if [ -f \"$state_dir/pid\" ]; then kill \"$(cat \"$state_dir/pid\")\" >/dev/null 2>&1 || true; fi", "printf '{\"ok\":true,\"forced\":true,\"stateDir\":\"%s\"}\\n' \"$state_dir\"", ].join("\n"), 55); - return { + return renderWebObserveCommandResult({ ok: killResult.exitCode === 0, status: killResult.exitCode === 0 ? "forced-stop-requested" : "blocked", command: webObserveCommandLabel("stop", options), @@ -6674,9 +6681,9 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec gracefulResult: compactCommandResult(result), forceResult: compactCommandResult(killResult), valuesRedacted: true, - }; + }); } - return { + return renderWebObserveCommandResult({ ok: result.exitCode === 0 && commandResult?.ok !== false, status: result.exitCode === 0 ? (options.waitMs > 0 ? "completed-or-queued" : "queued") : "blocked", command: webObserveCommandLabel(stopCommand ? "stop" : "command", options), @@ -6689,7 +6696,7 @@ function runNodeWebProbeObserveCommand(options: NodeWebProbeObserveOptions, spec observer: commandResult, result: compactCommandResult(result), valuesRedacted: true, - }; + }); } function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record { @@ -6726,7 +6733,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec ].join("\n"); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const analysis = parseJsonObject(result.stdout); - return { + return renderWebObserveAnalyzeResult({ ok: result.exitCode === 0 && analysis?.ok === true, status: result.exitCode === 0 && analysis?.ok === true ? "analyzed" : "blocked", command: webObserveCommandLabel("analyze", options), @@ -6737,7 +6744,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec analysis, result: compactCommandResult(result), valuesRedacted: true, - }; + }); } function nodeWebObserveResolveStateDirShell(options: Pick): string { @@ -6979,6 +6986,222 @@ function webObserveNextCommands(id: string): Record { }; } +function webObserveCell(value: unknown, maxLength = 96): string { + if (value === null || value === undefined) return "-"; + const text = typeof value === "string" ? value : String(value); + const compact = text.replace(/\s+/gu, " ").trim(); + if (compact.length === 0) return "-"; + if (compact.length <= maxLength) return compact; + return `${compact.slice(0, Math.max(1, maxLength - 1))}...`; +} + +function webObserveTable(headers: string[], rows: unknown[][]): string { + const stringRows = rows.map((row) => row.map((value) => webObserveCell(value))); + const widths = headers.map((header, index) => Math.max(header.length, ...stringRows.map((row) => row[index]?.length ?? 0))); + const renderRow = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); + return [renderRow(headers), ...stringRows.map(renderRow)].join("\n"); +} + +function webObserveArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function withWebObserveRendered(result: Record, renderedText: string): Record { + return { + ...result, + renderedText, + contentType: "text/plain", + }; +} + +function renderWebObserveStatusResult(result: Record): Record { + const observer = record(result.observer); + const commandResult = record(result.result); + const manifest = record(observer.manifest); + const heartbeat = record(observer.heartbeat); + const tails = record(observer.tails); + const samples = webObserveArray(tails.samples); + const network = webObserveArray(tails.network); + const control = webObserveArray(tails.control); + const artifacts = webObserveArray(tails.artifacts); + const lastSample = record(samples[samples.length - 1]); + const failedNetwork = network + .map((item) => record(item)) + .filter((item) => item.type === "requestfailed" || typeof item.status === "number" && item.status >= 500); + const id = result.id ?? observer.id ?? manifest.jobId ?? heartbeat.jobId ?? "-"; + const resultSection = result.ok === true ? [] : [ + "", + webObserveTable( + ["RESULT", "VALUE"], + [ + ["exitCode", commandResult.exitCode ?? "-"], + ["timedOut", commandResult.timedOut ?? "-"], + ["stdoutTail", commandResult.stdoutTail ?? commandResult.stdout ?? "-"], + ["stderrTail", commandResult.stderrTail ?? commandResult.stderr ?? "-"], + ], + ), + ]; + const renderedText = [ + webObserveTable( + ["OBSERVER", "NODE", "LANE", "STATUS", "ALIVE", "PID", "SAMPLE", "COMMAND", "UPDATED"], + [[id, result.node, result.lane, manifest.status ?? heartbeat.status ?? result.status, observer.processAlive, observer.pid, heartbeat.sampleSeq, heartbeat.commandSeq, heartbeat.updatedAt]], + ), + "", + webObserveTable( + ["URL", "ROUTE_SESSION", "ACTIVE_SESSION", "MESSAGES", "TRACE_ROWS"], + [[heartbeat.currentUrl, lastSample.routeSessionId, lastSample.activeSessionId, lastSample.messageCount, lastSample.traceRowCount]], + ), + "", + webObserveTable( + ["TAIL", "COUNT", "DETAIL"], + [ + ["control", control.length, record(control[control.length - 1]).type ?? "-"], + ["samples", samples.length, record(samples[samples.length - 1]).ts ?? "-"], + ["network", network.length, failedNetwork.length === 0 ? "no recent failed/5xx tail" : `${failedNetwork.length} recent failed/5xx`], + ["artifacts", artifacts.length, record(artifacts[artifacts.length - 1]).sha256 ?? "-"], + ], + ), + ...resultSection, + "", + "NEXT", + ` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`, + ` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`, + ` command: bun scripts/cli.ts hwlab nodes web-probe observe command ${id} --type mark --label checkpoint`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function renderWebObserveCommandResult(result: Record): Record { + const observerCommand = record(result.observerCommand); + const observer = record(result.observer); + const id = result.id ?? "-"; + const renderedText = [ + webObserveTable( + ["OBSERVER", "NODE", "LANE", "COMMAND_ID", "TYPE", "STATUS", "DETAIL"], + [[id, result.node, result.lane, result.commandId, observerCommand.type, result.status, observer.queued === true ? "queued" : observer.phase ?? observer.status ?? "-"]], + ), + "", + webObserveTable( + ["INPUT", "VALUE"], + [ + ["label", observerCommand.label ?? "-"], + ["path", observerCommand.path ?? "-"], + ["provider", observerCommand.provider ?? "-"], + ["sessionId", observerCommand.sessionId ?? "-"], + ["text", observerCommand.textHash === null || observerCommand.textHash === undefined ? "-" : `${observerCommand.textBytes ?? "-"}B ${observerCommand.textHash}`], + ], + ), + "", + "NEXT", + ` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`, + ` analyze: bun scripts/cli.ts hwlab nodes web-probe observe analyze ${id}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function renderWebObserveAnalyzeResult(result: Record): Record { + const analysis = record(result.analysis); + const counts = record(analysis.counts); + const sampleMetrics = record(analysis.sampleMetrics); + const runtimeAlerts = record(analysis.runtimeAlerts); + const pagePerformance = record(analysis.pagePerformance); + const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item)); + const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item)); + const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); + const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); + const id = result.id ?? "-"; + const renderedText = [ + webObserveTable( + ["OBSERVER", "NODE", "LANE", "STATUS", "REPORT_JSON", "REPORT_MD"], + [[id, result.node, result.lane, result.status, analysis.reportJsonSha256, analysis.reportMdSha256]], + ), + "", + webObserveTable( + ["SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS", "ROUNDS", "ROWS"], + [[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]], + ), + "", + webObserveTable( + ["TURN_TIMING", "VALUE"], + [ + ["nonMonotonic", sampleMetrics.turnTimingNonMonotonicCount], + ["elapsedDecrease", sampleMetrics.turnTimingTotalElapsedDecreaseCount], + ["recentJump", sampleMetrics.turnTimingRecentUpdateJumpCount], + ["maxRecentStepSec", sampleMetrics.turnTimingRecentUpdateMaxIncreaseSeconds], + ], + ), + "", + rounds.length === 0 + ? "ROUNDS\n-" + : webObserveTable( + ["ROUND", "COMMAND", "SAMPLES", "TOTAL_MAX", "TOTAL_LAST", "RECENT_MAX", "RECENT_LAST", "DIAG", "TERM", "NONMONO", "JUMP"], + rounds.map((item) => [ + item.promptIndex, + item.promptCommandId, + item.sampleCount, + item.maxTotalElapsedSeconds, + item.lastTotalElapsedSeconds, + item.maxRecentUpdateSeconds, + item.lastRecentUpdateSeconds, + item.diagnosticSamples, + item.terminalSamples, + item.turnTimingNonMonotonicCount, + item.turnTimingRecentUpdateJumpCount, + ]), + ), + "", + turnColumns.length === 0 + ? "TURN_COLUMNS\n-" + : webObserveTable( + ["TURN", "PROMPT", "TRACE", "MESSAGE", "FIRST_SEQ", "LAST_SEQ", "LAST_TS"], + turnColumns.map((item) => [item.label ?? item.id, item.lastPromptIndex ?? item.promptIndex, item.traceId, item.messageId, item.firstSeq, item.lastSeq, item.lastTs]), + ), + "", + webObserveTable( + ["ALERT", "COUNT"], + [ + ["httpError", runtimeAlerts.httpErrorCount], + ["requestFailed", runtimeAlerts.requestFailedCount], + ["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount], + ["consoleAlerts", runtimeAlerts.consoleAlertCount], + ["executionErrors", runtimeAlerts.executionErrorCount], + ], + ), + "", + webObserveTable( + ["PERF", "VALUE"], + [ + ["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount], + ["slowPathCount", pagePerformance.slowPathCount], + ["slowSampleCount", pagePerformance.slowSampleCount], + ["worstP95Ms", pagePerformance.worstP95Ms], + ], + ), + "", + slowApis.length === 0 + ? "SLOW_API\n-" + : webObserveTable( + ["SLOW_API", "P50", "P75", "P95", ">5S", "COUNT"], + slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overFiveSecondCount, item.sampleCount]), + ), + "", + findings.length === 0 + ? "FINDINGS\n-" + : webObserveTable( + ["FINDING", "SEVERITY", "COUNT", "SUMMARY"], + findings.map((item) => [item.id, item.severity, item.count, item.summary]), + ), + "", + "REPORTS", + ` json: ${analysis.reportJsonPath ?? "-"}`, + ` md: ${analysis.reportMdPath ?? "-"}`, + "", + "NEXT", + ` status: bun scripts/cli.ts hwlab nodes web-probe observe status ${id}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + function withWebObserveShortcuts(value: Record | null, id: string | null): Record | null { if (value === null || id === null) return value; return { diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index ecebc644..9fec0dd0 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -1029,7 +1029,7 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: sampleMetrics.summary, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } diff --git a/scripts/src/output.ts b/scripts/src/output.ts index 4d906187..36f08b91 100644 --- a/scripts/src/output.ts +++ b/scripts/src/output.ts @@ -1,7 +1,8 @@ import { randomBytes } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; export interface JsonEnvelope { ok: boolean; @@ -17,9 +18,21 @@ export interface RenderedCliResult { contentType: "text/plain" | "application/json" | "application/yaml"; } -const GH_OUTPUT_DUMP_THRESHOLD_BYTES = 20 * 1024; -const OUTPUT_DUMP_PREVIEW_CHARS = 2000; -const OUTPUT_DUMP_DIR = join(tmpdir(), "unidesk-cli-output"); +export interface CliOutputDumpConfig { + enabled: boolean; + thresholdBytes: number; + previewChars: number; + dir: string; + source: string; + warning: string | null; +} + +const FALLBACK_OUTPUT_DUMP_THRESHOLD_BYTES = 10 * 1024; +const FALLBACK_OUTPUT_DUMP_PREVIEW_CHARS = 2000; +const FALLBACK_OUTPUT_DUMP_DIR = join(tmpdir(), "unidesk-cli-output"); +const moduleDir = dirname(fileURLToPath(import.meta.url)); +const CLI_OUTPUT_CONFIG_PATH = join(moduleDir, "..", "..", "config", "cli-output.yaml"); +let cachedOutputDumpConfig: CliOutputDumpConfig | null = null; function isEpipe(error: unknown): boolean { return typeof error === "object" && error !== null && "code" in error && (error as { code?: unknown }).code === "EPIPE"; @@ -49,7 +62,8 @@ export function isRenderedCliResult(value: unknown): value is RenderedCliResult } export function emitText(text: string): void { - safeStdoutWrite(text.endsWith("\n") ? text : `${text}\n`); + const output = text.endsWith("\n") ? text : `${text}\n`; + safeStdoutWrite(renderPlainTextOutput("text", output)); } export function emitError(command: string, error: unknown): void { @@ -130,10 +144,11 @@ function renderEnvelope(command: string, envelope: JsonEnvelope): string { const fullText = `${JSON.stringify(envelope, null, 2)}\n`; if (!shouldDumpLargeOutput(command, fullText, envelope)) return fullText; - const dump = dumpLargeOutput(command, fullText); + const dump = dumpLargeOutput(command, fullText, "json"); const compactPayload = { outputTruncated: true, reason: "stdout-json-exceeded-threshold", + warning: "CLI output exceeded the global dump threshold. Fix this command's default output instead of repeatedly relying on dump.", message: "Full JSON output was written to a temporary file; stdout contains only bounded head/tail previews.", dump, summary: summarizeEnvelope(envelope), @@ -144,48 +159,132 @@ function renderEnvelope(command: string, envelope: JsonEnvelope): string { return `${JSON.stringify(compactEnvelope, null, 2)}\n`; } +function renderPlainTextOutput(command: string, text: string): string { + if (!shouldDumpLargeText(command, text)) return text; + const dump = dumpLargeOutput(command, text, "txt"); + return [ + "UNIDESK_CLI_OUTPUT_TRUNCATED", + "warning: CLI text output exceeded the global dump threshold. Fix this command's default output instead of repeatedly relying on dump.", + `dumpPath: ${dump.path}`, + `bytes: ${dump.bytes}`, + `lines: ${dump.lines}`, + `thresholdBytes: ${dump.thresholdBytes}`, + `headCommand: ${recordString(record(dump.readCommands), "head") ?? "-"}`, + `tailCommand: ${recordString(record(dump.readCommands), "tail") ?? "-"}`, + "", + ].join("\n"); +} + function shouldDumpLargeOutput(command: string, text: string, envelope: JsonEnvelope): boolean { - if (!isLargeOutputDumpCommand(command)) return false; if (process.env.UNIDESK_CLI_GH_OUTPUT_DUMP_DISABLED === "1" || process.env.UNIDESK_CLI_OUTPUT_DUMP_DISABLED === "1") return false; if (typeof envelope.data === "object" && envelope.data !== null && (envelope.data as { noDump?: unknown }).noDump === true) return false; - const threshold = configuredDumpThresholdBytes(); + const config = outputDumpConfig(); + if (!config.enabled) return false; + const threshold = configuredDumpThresholdBytes(config); return Buffer.byteLength(text, "utf8") > threshold; } -function isLargeOutputDumpCommand(command: string): boolean { - return command === "gh" || command.startsWith("gh ") || command === "agentrun" || command.startsWith("agentrun "); +function shouldDumpLargeText(command: string, text: string): boolean { + if (process.env.UNIDESK_CLI_OUTPUT_DUMP_DISABLED === "1") return false; + const config = outputDumpConfig(); + if (!config.enabled) return false; + const threshold = configuredDumpThresholdBytes(config); + return Buffer.byteLength(text, "utf8") > threshold; } -function configuredDumpThresholdBytes(): number { +function configuredDumpThresholdBytes(config = outputDumpConfig()): number { const genericRaw = process.env.UNIDESK_CLI_OUTPUT_DUMP_THRESHOLD_BYTES; if (genericRaw !== undefined && genericRaw.trim().length > 0) { const genericValue = Number(genericRaw); if (Number.isInteger(genericValue) && genericValue > 0) return genericValue; } const raw = process.env.UNIDESK_CLI_GH_OUTPUT_DUMP_THRESHOLD_BYTES; - if (raw === undefined || raw.trim().length === 0) return GH_OUTPUT_DUMP_THRESHOLD_BYTES; + if (raw === undefined || raw.trim().length === 0) return config.thresholdBytes; const value = Number(raw); - if (!Number.isInteger(value) || value <= 0) return GH_OUTPUT_DUMP_THRESHOLD_BYTES; + if (!Number.isInteger(value) || value <= 0) return config.thresholdBytes; return value; } -function dumpLargeOutput(command: string, text: string): Record { - mkdirSync(OUTPUT_DUMP_DIR, { recursive: true, mode: 0o700 }); +function configuredDumpPreviewChars(config = outputDumpConfig()): number { + const threshold = configuredDumpThresholdBytes(config); + return Math.max(1, Math.min(config.previewChars, Math.floor(threshold / 4))); +} + +function configuredDumpDir(config = outputDumpConfig()): string { + return config.dir; +} + +export function cliOutputDumpRuntimeConfig(): CliOutputDumpConfig { + const config = outputDumpConfig(); + return { + ...config, + thresholdBytes: configuredDumpThresholdBytes(config), + previewChars: configuredDumpPreviewChars(config), + dir: configuredDumpDir(config), + }; +} + +function outputDumpConfig(): CliOutputDumpConfig { + if (cachedOutputDumpConfig !== null) return cachedOutputDumpConfig; + cachedOutputDumpConfig = readOutputDumpConfig(); + return cachedOutputDumpConfig; +} + +function readOutputDumpConfig(): CliOutputDumpConfig { + const fallback: CliOutputDumpConfig = { + enabled: true, + thresholdBytes: FALLBACK_OUTPUT_DUMP_THRESHOLD_BYTES, + previewChars: FALLBACK_OUTPUT_DUMP_PREVIEW_CHARS, + dir: FALLBACK_OUTPUT_DUMP_DIR, + source: "fallback", + warning: null, + }; + if (!existsSync(CLI_OUTPUT_CONFIG_PATH)) { + return { ...fallback, warning: `config missing: ${CLI_OUTPUT_CONFIG_PATH}` }; + } + try { + const parsed = Bun.YAML.parse(readFileSync(CLI_OUTPUT_CONFIG_PATH, "utf8")) as unknown; + const root = record(parsed); + const output = record(root.output); + const dump = record(output.dump); + return { + enabled: booleanField(dump, "enabled", `${CLI_OUTPUT_CONFIG_PATH}.output.dump`), + thresholdBytes: positiveIntegerField(dump, "thresholdBytes", `${CLI_OUTPUT_CONFIG_PATH}.output.dump`), + previewChars: positiveIntegerField(dump, "previewChars", `${CLI_OUTPUT_CONFIG_PATH}.output.dump`), + dir: nonEmptyStringField(dump, "dir", `${CLI_OUTPUT_CONFIG_PATH}.output.dump`), + source: CLI_OUTPUT_CONFIG_PATH, + warning: null, + }; + } catch (error) { + return { + ...fallback, + warning: `config invalid: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + +function dumpLargeOutput(command: string, text: string, extension: "json" | "txt"): Record { + const config = outputDumpConfig(); + const outputDir = configuredDumpDir(config); + mkdirSync(outputDir, { recursive: true, mode: 0o700 }); const timestamp = new Date().toISOString().replace(/[:.]/gu, "-"); const suffix = randomBytes(4).toString("hex"); const slug = command.replace(/[^A-Za-z0-9._-]+/gu, "-").replace(/^-+|-+$/gu, "").slice(0, 80) || "command"; - const path = join(OUTPUT_DUMP_DIR, `${timestamp}-${process.pid}-${suffix}-${slug}.json`); + const path = join(outputDir, `${timestamp}-${process.pid}-${suffix}-${slug}.${extension}`); writeFileSync(path, text, { encoding: "utf8", mode: 0o600 }); + const previewChars = configuredDumpPreviewChars(config); return { path, - thresholdBytes: configuredDumpThresholdBytes(), + thresholdBytes: configuredDumpThresholdBytes(config), + configSource: config.source, + configWarning: config.warning, bytes: Buffer.byteLength(text, "utf8"), chars: text.length, lines: countLines(text), - headChars: OUTPUT_DUMP_PREVIEW_CHARS, - tailChars: OUTPUT_DUMP_PREVIEW_CHARS, - head: text.slice(0, OUTPUT_DUMP_PREVIEW_CHARS), - tail: text.slice(Math.max(0, text.length - OUTPUT_DUMP_PREVIEW_CHARS)), + headChars: previewChars, + tailChars: previewChars, + head: text.slice(0, previewChars), + tail: text.slice(Math.max(0, text.length - previewChars)), readCommands: { full: `cat ${JSON.stringify(path)}`, head: `sed -n '1,80p' ${JSON.stringify(path)}`, @@ -240,6 +339,33 @@ function summarizeEnvelope(envelope: JsonEnvelope): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function recordString(source: Record, key: string): string | null { + const value = source[key]; + return typeof value === "string" ? value : null; +} + +function booleanField(source: Record, key: string, path: string): boolean { + const value = source[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +function positiveIntegerField(source: Record, key: string, path: string): number { + const value = source[key]; + if (!Number.isInteger(value) || Number(value) <= 0) throw new Error(`${path}.${key} must be a positive integer`); + return Number(value); +} + +function nonEmptyStringField(source: Record, key: string, path: string): string { + const value = source[key]; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value; +} + function pickSummary(source: Record, keys: string[]): Record { const result: Record = {}; for (const key of keys) { diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index 97e1da13..f769bd69 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -4,6 +4,7 @@ import { Buffer } from "node:buffer"; import { readFileSync } from "node:fs"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; +import type { RenderedCliResult } from "./output"; import { compactCapture, createYamlFieldReader, @@ -269,9 +270,9 @@ function parseSearchOptions(args: string[]): SearchOptions { let lookbackMinutes = 360; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; - if (arg === "--grep") { + if (arg === "--grep" || arg === "--path") { const value = args[index + 1]; - if (value === undefined || value.startsWith("--")) throw new Error("--grep requires a value"); + if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`); grep = value; index += 1; } else if (arg === "--query" || arg === "--q") { @@ -620,14 +621,14 @@ async function validate(config: UniDeskConfig, options: CommonOptions): Promise< }; } -async function trace(config: UniDeskConfig, options: TraceOptions): Promise> { +async function trace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { if (options.traceId === null) throw new Error("observability trace requires --trace-id "); const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const tracePath = observability.probes.traceQueryPathTemplate.replaceAll("{{traceId}}", encodeURIComponent(options.traceId)); const result = await capture(config, target.route, ["sh"], traceScript(observability, target, tracePath, options)); const parsed = parseJsonOutput(result.stdout); - return { + const response = { ok: result.exitCode === 0 && parsed?.ok === true, action: "platform-infra-observability-trace", mutation: false, @@ -640,9 +641,11 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise> { +async function search(config: UniDeskConfig, options: SearchOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const endSeconds = Math.floor(Date.now() / 1000); @@ -656,7 +659,7 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise> { +async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAgentOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); const endSeconds = Math.floor(Date.now() / 1000); @@ -692,7 +697,7 @@ async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAge } const result = await capture(config, target.route, ["sh"], diagnoseCodeAgentScript(observability, target, searchPath, options)); const parsed = parseJsonOutput(result.stdout); - return { + const response = { ok: result.exitCode === 0 && parsed?.ok === true, action: "platform-infra-observability-diagnose-code-agent", mutation: false, @@ -709,6 +714,220 @@ async function diagnoseCodeAgent(config: UniDeskConfig, options: DiagnoseCodeAge }, result: redactSensitiveUnknown(parsed) ?? compactCapture(result, { full: true }), }; + if (options.raw || options.full) return response; + return renderDiagnoseCodeAgentResult(response); +} + +function renderDiagnoseCodeAgentResult(response: Record): RenderedCliResult { + const target = recordValue(response.target); + const query = recordValue(response.query); + const result = recordValue(response.result); + const mapping = recordValue(result.mapping); + const summary = recordValue(result.summary); + const identity = recordValue(result.identity); + const agentrun = recordValue(result.agentrun); + const http = recordValue(result.http); + const servicePath = recordValue(result.servicePath); + const selection = recordValue(mapping.selection); + const candidates = arrayValue(mapping.candidateScores); + const rootCandidates = arrayValue(summary.rootCauseCandidates); + const problemCounts = arrayValue(http.problemCounts); + + const lines: string[] = []; + lines.push("COMMAND TARGET OK BUSINESS_TRACE OTEL_TRACE"); + lines.push(`${pad("platform-infra observability diagnose", 44)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(textValue(query.businessTraceId) || "-", 27)} ${textValue(mapping.otelTraceId) || "-"}`); + lines.push(""); + lines.push("TRACE_SELECTION"); + lines.push(`mode: ${textValue(selection.mode) || textValue(mapping.mode) || "-"}`); + lines.push(`candidateTraceCount: ${textValue(mapping.candidateTraceCount) || "-"}`); + lines.push(`selectedScore: ${textValue(selection.score) || "-"}`); + lines.push(`selectedReason: ${arrayValue(selection.reasons).map((item) => textValue(item)).filter(Boolean).join(", ") || "-"}`); + if (textValue(mapping.selectionWarning)) lines.push(`warning: ${textValue(mapping.selectionWarning)}`); + lines.push(""); + lines.push("SERVICE_PATH"); + for (const service of ["hwlab-cloud-api", "agentrun-manager", "agentrun-runner"]) { + lines.push(`${pad(service, 22)} ${textValue(servicePath[service]) || "missing"}`); + } + lines.push(""); + lines.push("AGENTRUN"); + lines.push(`terminalStatus: ${textValue(agentrun.terminalStatus) || "-"}`); + lines.push(`terminalEventType: ${textValue(agentrun.terminalEventType) || "-"}`); + lines.push(`runnerProviderClassification: ${textValue(agentrun.runnerProviderClassification) || "-"}`); + lines.push(""); + lines.push("IDENTITY"); + for (const key of ["runId", "commandId", "runnerJobId", "runnerId", "sessionId", "turnId", "backendProfile", "sourceCommit"]) { + const value = textValue(identity[key]); + if (value) lines.push(`${pad(key, 16)} ${value}`); + } + if (lines[lines.length - 1] === "IDENTITY") lines.push("-"); + lines.push(""); + lines.push("ROOT_CAUSE"); + lines.push(textValue(summary.rootCause) || "-"); + for (const item of rootCandidates.slice(0, 5)) { + const row = recordValue(item); + lines.push(`- ${textValue(row.code) || textValue(row.label) || "candidate"} confidence=${textValue(row.confidence) || "-"} ${textValue(row.summary) || ""}`.trim()); + } + lines.push(""); + lines.push("HTTP_PROBLEMS"); + if (problemCounts.length === 0) { + lines.push("-"); + } else { + lines.push("METHOD ROUTE STATUS COUNT"); + for (const item of problemCounts.slice(0, 8)) { + const row = recordValue(item); + lines.push(`${pad(textValue(row.method) || "-", 7)} ${pad(truncate(textValue(row.route) || "-", 29), 29)} ${pad(textValue(row.status) || "-", 7)} ${textValue(row.count) || "-"}`); + } + } + if (candidates.length > 0) { + lines.push(""); + lines.push("CANDIDATE_TRACE_SCORES"); + lines.push("SCORE SPANS SERVICES TRACE"); + for (const item of candidates.slice(0, 8)) { + const row = recordValue(item); + lines.push(`${pad(textValue(row.score) || "0", 6)} ${pad(textValue(row.spanCount) || "-", 6)} ${pad(truncate(arrayValue(row.services).map((value) => textValue(value)).filter(Boolean).join(","), 32), 32)} ${textValue(row.traceId) || "-"}`); + } + } + lines.push(""); + lines.push("Next:"); + if (textValue(mapping.otelTraceId)) { + lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || textValue(query.target) || "D601"} --trace-id ${textValue(mapping.otelTraceId)} --limit 40`); + lines.push(` bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${textValue(target.id) || "D601"} --business-trace-id ${textValue(query.businessTraceId) || ""} --full`); + } else { + lines.push(` bun scripts/cli.ts platform-infra observability diagnose-code-agent --target ${textValue(target.id) || "D601"} --business-trace-id ${textValue(query.businessTraceId) || ""} --lookback-minutes 10080 --candidate-limit 100`); + } + return { + ok: response.ok === true, + command: "platform-infra observability diagnose-code-agent", + contentType: "text/plain", + renderedText: `${lines.join("\n")}\n`, + }; +} + +function renderSearchResult(response: Record): RenderedCliResult { + const target = recordValue(response.target); + const query = recordValue(response.query); + const result = recordValue(response.result); + const traces = arrayValue(result.traces); + const lines: string[] = []; + lines.push("COMMAND TARGET OK GREP_OR_QUERY MATCHED SCANNED CANDIDATES"); + lines.push(`${pad("platform-infra observability search", 38)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(truncate(textValue(query.grep) || textValue(query.tempoQuery) || "-", 35), 35)} ${pad(textValue(result.matchedTraceCount) || "-", 8)} ${pad(textValue(result.scannedTraceCount) || "-", 8)} ${textValue(result.candidateTraceCount) || "-"}`); + if (textValue(result.scanStopped)) lines.push(`warning: scanStopped=${textValue(result.scanStopped)}`); + const truncated = recordValue(result.truncated); + if (truncated.candidateTraces === true || truncated.matchedTraces === true) { + lines.push(`warning: truncated candidateTraces=${textValue(truncated.candidateTraces)} matchedTraces=${textValue(truncated.matchedTraces)}`); + } + lines.push(""); + lines.push("TRACES"); + if (traces.length === 0) { + lines.push("-"); + } else { + lines.push("TRACE SPANS ERRORS MATCH SERVICES ROOT"); + for (const item of traces.slice(0, 20)) { + const row = recordValue(item); + const meta = recordValue(row.meta); + const services = arrayValue(row.services).map((value) => textValue(value)).filter(Boolean).join(","); + lines.push(`${pad(textValue(row.traceId) || "-", 34)} ${pad(textValue(row.spanCount) || "-", 6)} ${pad(textValue(row.errorSpanCount) || "-", 7)} ${pad(textValue(row.matchedSpanCount) || "-", 6)} ${pad(truncate(services || textValue(meta.rootServiceName) || "-", 32), 32)} ${truncate(textValue(meta.rootTraceName) || "-", 48)}`); + } + } + lines.push(""); + lines.push("Next:"); + lines.push(` bun scripts/cli.ts platform-infra observability search --target ${textValue(target.id) || "D601"} --grep --lookback-minutes ${textValue(query.lookbackMinutes) || "360"} --candidate-limit ${textValue(query.candidateLimit) || "80"} --limit ${textValue(query.limit) || "20"} --full`); + lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || "D601"} --trace-id --grep --limit 80`); + return { + ok: response.ok === true, + command: "platform-infra observability search", + contentType: "text/plain", + renderedText: `${lines.join("\n")}\n`, + }; +} + +function renderTraceResult(response: Record): RenderedCliResult { + const target = recordValue(response.target); + const traceId = textValue(response.traceId); + const result = recordValue(response.result); + const services = arrayValue(result.services).map((value) => textValue(value)).filter(Boolean); + const businessTraceIds = arrayValue(result.businessTraceIds).map((value) => textValue(value)).filter(Boolean); + const spanNameCounts = arrayValue(result.spanNameCounts); + const errorSpans = arrayValue(result.errorSpans); + const matchedSpans = arrayValue(result.spans).length > 0 ? arrayValue(result.spans) : arrayValue(result.matchedSpans); + const lines: string[] = []; + lines.push("COMMAND TARGET OK TRACE SPANS ERRORS MATCH"); + lines.push(`${pad("platform-infra observability trace", 37)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(traceId || "-", 34)} ${pad(textValue(result.spanCount) || "-", 6)} ${pad(textValue(result.errorSpanCount) || "-", 7)} ${textValue(result.matchedSpanCount) || "-"}`); + lines.push(""); + lines.push(`services: ${services.join(", ") || "-"}`); + lines.push(`businessTraceIds: ${businessTraceIds.join(", ") || "-"}`); + if (textValue(result.grep)) lines.push(`grep: ${textValue(result.grep)}`); + const truncated = recordValue(result.truncated); + if (truncated.errorSpans === true || truncated.spans === true) { + lines.push(`warning: truncated errorSpans=${textValue(truncated.errorSpans)} spans=${textValue(truncated.spans)}`); + } + lines.push(""); + lines.push("SPAN_NAME_COUNTS"); + if (spanNameCounts.length === 0) { + lines.push("-"); + } else { + lines.push("COUNT NAME"); + for (const item of spanNameCounts.slice(0, 12)) { + const row = recordValue(item); + lines.push(`${pad(textValue(row.count) || "-", 6)} ${truncate(textValue(row.name) || "-", 80)}`); + } + } + lines.push(""); + lines.push("ERROR_SPANS"); + if (errorSpans.length === 0) { + lines.push("-"); + } else { + lines.push("SERVICE DURATION_MS NAME"); + for (const item of errorSpans.slice(0, 8)) { + const row = recordValue(item); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${truncate(textValue(row.name) || "-", 80)}`); + } + } + if (matchedSpans.length > 0) { + lines.push(""); + lines.push("MATCHED_SPANS"); + lines.push("SERVICE DURATION_MS NAME"); + for (const item of matchedSpans.slice(0, 8)) { + const row = recordValue(item); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${truncate(textValue(row.name) || "-", 80)}`); + } + } + lines.push(""); + lines.push("Next:"); + lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || "D601"} --trace-id ${traceId || ""} --grep --limit 80`); + lines.push(` bun scripts/cli.ts platform-infra observability trace --target ${textValue(target.id) || "D601"} --trace-id ${traceId || ""} --full --limit 200`); + return { + ok: response.ok === true, + command: "platform-infra observability trace", + contentType: "text/plain", + renderedText: `${lines.join("\n")}\n`, + }; +} + +function recordValue(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function arrayValue(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function textValue(value: unknown): string { + if (value === null || value === undefined) return ""; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return ""; +} + +function pad(value: string, width: number): string { + return value.length >= width ? value : `${value}${" ".repeat(width - value.length)}`; +} + +function truncate(value: string, width: number): string { + if (value.length <= width) return value; + if (width <= 1) return value.slice(0, width); + if (width <= 3) return value.slice(0, width); + return `${value.slice(0, width - 3)}...`; } function renderManifest(observability: ObservabilityConfig, target: ObservabilityTarget): string { @@ -2416,6 +2635,101 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True) return candidates +def score_trace_candidate(trace_id, meta, index): + trace_path = TRACE_PATH_TEMPLATE.replace("{{traceId}}", trace_id) + trace_proxy_path = TRACE_PROXY_PREFIX + trace_path + trace_rc, trace_body, trace_err = run_kubectl(trace_proxy_path, timeout=5) + trace_parsed = parse_json(trace_body) + score = 0 + reasons = [] + services = [] + span_count = 0 + error_span_count = 0 + business_trace_ids = [] + root_name = None + fetch_ok = trace_rc == 0 and isinstance(trace_parsed, dict) + if not fetch_ok: + return { + "traceId": trace_id, + "index": index, + "score": -100, + "reasons": ["trace_fetch_failed"], + "services": services, + "spanCount": span_count, + "errorSpanCount": error_span_count, + "businessTraceIds": business_trace_ids, + "rootName": root_name, + "fetchOk": False, + "selectedMeta": compact_meta(meta), + "stderrTail": (trace_err or "")[-1000:], + } + flat = flatten_trace(trace_parsed) + spans = flat.get("spans", []) + services = flat.get("services", []) + business_trace_ids = flat.get("businessTraceIds", []) + error_span_count = len(flat.get("errorSpans", [])) + span_count = len(spans) + if spans: + root_name = spans[0].get("name") + service_set = set(str(service) for service in services) + span_names = [str(item.get("name") or "") for item in spans] + attr_fragments = [] + for item in spans: + attrs = item.get("attributes", {}) if isinstance(item.get("attributes"), dict) else {} + for key in ("runId", "commandId", "runnerJobId", "terminalStatus", "eventType", "failureKind", "traceId"): + value = attrs.get(key) + if value not in (None, ""): + attr_fragments.append("%s=%s" % (key, value)) + haystack = " ".join(span_names + attr_fragments).lower() + if BUSINESS_TRACE_ID and BUSINESS_TRACE_ID in business_trace_ids: + score += 20 + reasons.append("business_trace_id_matched") + if "hwlab-cloud-api" in service_set: + score += 20 + reasons.append("hwlab_cloud_api_reached") + if "agentrun-manager" in service_set: + score += 80 + reasons.append("agentrun_manager_reached") + if "agentrun-runner" in service_set: + score += 120 + reasons.append("agentrun_runner_reached") + if "codex_stdio" in haystack: + score += 80 + reasons.append("codex_stdio_spans_present") + if "runner_terminal" in haystack or "terminalstatus=" in haystack: + score += 60 + reasons.append("runner_terminal_present") + if "runid=" in haystack: + score += 30 + reasons.append("run_id_present") + if "commandid=" in haystack: + score += 30 + reasons.append("command_id_present") + if error_span_count > 0: + score += min(30, error_span_count * 5) + reasons.append("error_spans_present") + if span_count > 1: + score += min(40, span_count) + if service_set == {"hwlab-cloud-api"} and span_count <= 2 and any(name.startswith("GET /v1/workbench/events") for name in span_names): + score -= 120 + reasons.append("penalized_workbench_events_sse_only") + if any(name.startswith("GET /v1/workbench/events") for name in span_names): + score -= 20 + reasons.append("workbench_events_candidate") + return { + "traceId": trace_id, + "index": index, + "score": score, + "reasons": reasons, + "services": services, + "spanCount": span_count, + "errorSpanCount": error_span_count, + "businessTraceIds": business_trace_ids, + "rootName": root_name, + "fetchOk": True, + "selectedMeta": compact_meta(meta), + } + def resolve_trace(): if TRACE_ID is not None: return TRACE_ID, { @@ -2433,14 +2747,36 @@ def resolve_trace(): search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15) search_parsed = parse_json(search_body) metas = extract_traces(search_parsed) - selected = None - selected_trace_id = None - for meta in metas: + raw_candidates = [] + for index, meta in enumerate(metas): trace_id = trace_id_from_meta(meta) if trace_id: - selected = meta - selected_trace_id = trace_id + raw_candidates.append((index, trace_id, meta)) + candidate_probe_limit = 60 + probe_deadline_seconds = 45 + probe_deadline = time.time() + probe_deadline_seconds + candidate_probe_stopped = None + candidate_scores = [] + for index, trace_id, meta in raw_candidates[:candidate_probe_limit]: + if time.time() > probe_deadline: + candidate_probe_stopped = "deadline" break + scored = score_trace_candidate(trace_id, meta, index) + candidate_scores.append(scored) + if scored.get("score", 0) >= 260 and len(candidate_scores) >= 12: + candidate_probe_stopped = "strong-candidate-found" + break + candidate_scores.sort(key=lambda item: (item.get("score", -9999), item.get("spanCount", 0), -item.get("index", 0)), reverse=True) + selected_score = candidate_scores[0] if candidate_scores else None + selected_trace_id = selected_score.get("traceId") if isinstance(selected_score, dict) else None + selected = selected_score.get("selectedMeta") if isinstance(selected_score, dict) else None + selection_warning = None + if selected_score is not None: + selected_services = set(str(service) for service in selected_score.get("services", [])) + if "agentrun-runner" not in selected_services: + selection_warning = "no candidate contained agentrun-runner; selected best available trace" + if selected_score.get("score", 0) < 0: + selection_warning = "candidate fetch/scoring did not find a strong trace; selected best available trace" mapping = { "mode": "business-trace-id", "businessTraceId": BUSINESS_TRACE_ID, @@ -2450,7 +2786,21 @@ def resolve_trace(): "searchOk": search_rc == 0, "searchParseOk": isinstance(search_parsed, dict), "candidateTraceCount": len(metas), - "selectedMeta": compact_meta(selected), + "candidateProbeLimit": candidate_probe_limit, + "candidateProbeDeadlineSeconds": probe_deadline_seconds, + "candidateProbeStopped": candidate_probe_stopped, + "probedCandidateCount": len(candidate_scores), + "selectedMeta": selected, + "selection": { + "mode": "scored-candidates", + "score": selected_score.get("score") if isinstance(selected_score, dict) else None, + "reasons": selected_score.get("reasons") if isinstance(selected_score, dict) else [], + "rootName": selected_score.get("rootName") if isinstance(selected_score, dict) else None, + "services": selected_score.get("services") if isinstance(selected_score, dict) else [], + "spanCount": selected_score.get("spanCount") if isinstance(selected_score, dict) else None, + }, + "selectionWarning": selection_warning, + "candidateScores": candidate_scores[:12], "searchStderrTail": (search_err or "")[-2000:], } if RAW: diff --git a/scripts/src/ssh.ts b/scripts/src/ssh.ts index 25b8f7c9..10448a02 100644 --- a/scripts/src/ssh.ts +++ b/scripts/src/ssh.ts @@ -4,6 +4,7 @@ import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { type UniDeskConfig, repoRoot } from "./config"; +import { cliOutputDumpRuntimeConfig } from "./output"; import { decodeApplyPatchV2BulkRead, formatApplyPatchV2BulkReplacementPayload, @@ -196,7 +197,7 @@ const windowsCmdExeNativePath = "C:\\Windows\\System32\\cmd.exe"; const defaultSshSlowWarningMs = 10_000; const defaultSshRuntimeTimeoutMs = 60_000; const maxSshRuntimeTimeoutMs = 60_000; -const defaultSshStdoutStreamMaxBytes = 256 * 1024; +const defaultSshStdoutStreamMaxBytes = 10 * 1024; const minSshStdoutStreamMaxBytes = 4 * 1024; const maxSshStdoutStreamMaxBytes = 16 * 1024 * 1024; const sshStdoutDumpDir = join(tmpdir(), "unidesk-cli-output"); @@ -2513,9 +2514,15 @@ export function formatSshRuntimeTimeoutHint(hint: SshRuntimeTimeoutHint): string } export function sshStdoutStreamMaxBytes(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES ?? env.UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES; + const raw = env.UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES + ?? env.UNIDESK_TRAN_STDOUT_STREAM_MAX_BYTES + ?? env.UNIDESK_CLI_OUTPUT_DUMP_THRESHOLD_BYTES; const parsed = raw === undefined ? NaN : Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return defaultSshStdoutStreamMaxBytes; + if (!Number.isFinite(parsed) || parsed <= 0) { + const config = cliOutputDumpRuntimeConfig(); + if (config.enabled) return Math.min(maxSshStdoutStreamMaxBytes, Math.max(minSshStdoutStreamMaxBytes, Math.trunc(config.thresholdBytes))); + return defaultSshStdoutStreamMaxBytes; + } return Math.min(maxSshStdoutStreamMaxBytes, Math.max(minSshStdoutStreamMaxBytes, Math.trunc(parsed))); } @@ -2538,8 +2545,8 @@ export function sshStdoutTruncationHint(options: { observedBytesAtTruncation: options.observedBytesAtTruncation, dumpPath: options.dumpPath, dumpError: options.dumpError ?? null, - message: `ssh stdout exceeded ${options.thresholdBytes} bytes; stdout is bounded and the complete stream is written to a local dump when possible.`, - action: "Inspect the dump path, or rerun a narrower remote command with tail/paging instead of emitting full logs or huge JSON.", + message: `ssh/trans stdout exceeded the global dump threshold (${options.thresholdBytes} bytes); stdout is bounded and the complete stream is written to a local dump when possible.`, + action: "Fix the called CLI default output to be concise/table-like; use the dump only as a fallback, or rerun a narrower remote command with tail/paging.", note: "This hint is written to stderr and intentionally does not echo the original remote command.", }; } From a08dfea9dace7bcf8198358d8a47f0361b688d9a Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 16:51:50 +0000 Subject: [PATCH 3/6] improve otel trace details and internal trans capture --- scripts/src/hwlab-node-impl.ts | 13 ++++-- scripts/src/platform-infra-observability.ts | 44 +++++++++++++++++++-- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 93ffab52..72b9d14e 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -8401,15 +8401,22 @@ ${tlsLines} @api path /health* /auth* /v1* /json-rpc* /openapi* /docs* /swagg } function runTransScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { - return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); + return runCommand([transPath(), `${node}:k3s`, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000, env: internalTransCaptureEnv() }); } function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { - return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); + return runCommand([transPath(), node, "sh", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000, env: internalTransCaptureEnv() }); } function runTransWorkspaceStdinScript(node: string, workspace: string, script: string, timeoutSeconds: number): CommandResult { - return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000 }); + return runCommand([transPath(), `${node}:${workspace}`, "sh"], repoRoot, { input: script, timeoutMs: timeoutSeconds * 1000, env: internalTransCaptureEnv() }); +} + +function internalTransCaptureEnv(): NodeJS.ProcessEnv { + return { + ...process.env, + UNIDESK_SSH_STDOUT_STREAM_MAX_BYTES: "16777216", + }; } function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index f769bd69..b8fbcb21 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -877,19 +877,19 @@ function renderTraceResult(response: Record): RenderedCliResult if (errorSpans.length === 0) { lines.push("-"); } else { - lines.push("SERVICE DURATION_MS NAME"); + lines.push("SERVICE DURATION_MS NAME DETAIL"); for (const item of errorSpans.slice(0, 8)) { const row = recordValue(item); - lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${truncate(textValue(row.name) || "-", 80)}`); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`); } } if (matchedSpans.length > 0) { lines.push(""); lines.push("MATCHED_SPANS"); - lines.push("SERVICE DURATION_MS NAME"); + lines.push("SERVICE DURATION_MS NAME DETAIL"); for (const item of matchedSpans.slice(0, 8)) { const row = recordValue(item); - lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${truncate(textValue(row.name) || "-", 80)}`); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`); } } lines.push(""); @@ -904,6 +904,36 @@ function renderTraceResult(response: Record): RenderedCliResult }; } +function traceSpanDetail(span: Record): string { + const attrs = recordValue(span.attributes); + const parts: string[] = []; + for (const key of [ + "failureKind", + "message", + "eventType", + "toolName", + "exitCode", + "durationMs", + "waitingFor", + "lastEventLabel", + "idleMs", + "idleSeconds", + "idleThresholdMs", + "lastEventAgeMs", + "terminalStatus", + "http.method", + "http.route", + "http.status_code", + "runId", + "commandId", + "runnerJobId", + ]) { + const value = textValue(attrs[key]); + if (value) parts.push(`${key}=${value}`); + } + return parts.join(" ") || "-"; +} + function recordValue(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } @@ -1547,6 +1577,8 @@ IMPORTANT_ATTRS = [ "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount", "maxSeq", "traceLastSeq", "endSeq", "commandFiltered", + "waitingFor", "lastEventLabel", "idleMs", "idleSeconds", "idleThresholdMs", + "lastEventAgeMs", "toolName", "exitCode", "durationMs", "command", "error", "error.message", "exception.type", "exception.message", "valuesPrinted", ] @@ -1789,6 +1821,8 @@ IMPORTANT_ATTRS = [ "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount", "maxSeq", "traceLastSeq", "endSeq", "commandFiltered", + "waitingFor", "lastEventLabel", "idleMs", "idleSeconds", "idleThresholdMs", + "lastEventAgeMs", "toolName", "exitCode", "durationMs", "command", "error", "error.message", "exception.type", "exception.message", "valuesPrinted", ] @@ -2102,6 +2136,8 @@ IMPORTANT_ATTRS = [ "maxSeq", "traceLastSeq", "endSeq", "commandFiltered", "seq", "eventSeq", "sourceSeq", "sourceLatestSeq", "latestSeq", "lastSeq", "sourceEventCount", "projectedSeq", "lastProjectedSeq", + "waitingFor", "lastEventLabel", "idleMs", "idleSeconds", "idleThresholdMs", + "lastEventAgeMs", "toolName", "exitCode", "durationMs", "command", "status", "turnStatus", "error", "error.message", "exception.type", "exception.message", "valuesPrinted", ] From 1789700f23388060c2ce2c8b708f5362129657d7 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 21 Jun 2026 17:56:40 +0000 Subject: [PATCH 4/6] fix: expose web probe and agentrun timeout diagnostics --- config/agentrun.yaml | 2 + scripts/src/agentrun-lanes.ts | 2 + scripts/src/agentrun-manifests.ts | 1 + scripts/src/hwlab-node-impl.ts | 148 +++++++++++++++++- .../hwlab-node-web-observe-runner-source.ts | 77 ++++++++- scripts/src/platform-infra-observability.ts | 51 ++++-- 6 files changed, 262 insertions(+), 19 deletions(-) diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 45bf6681..eccc1ddd 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -151,6 +151,7 @@ controlPlane: serviceAccount: agentrun-v01-runner jobNamePrefix: agentrun-v01-runner idleTimeoutMs: 600000 + missingTerminalAfterToolTimeoutMs: 60000 apiKeySecretRef: name: agentrun-v01-api-key key: HWLAB_API_KEY @@ -324,6 +325,7 @@ controlPlane: serviceAccount: agentrun-v02-runner jobNamePrefix: agentrun-v02-runner idleTimeoutMs: 172800000 + missingTerminalAfterToolTimeoutMs: 30000 apiKeySecretRef: name: agentrun-v02-api-key key: HWLAB_API_KEY diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 17c2a250..9751c04b 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -104,6 +104,7 @@ export interface AgentRunLaneSpec { readonly serviceAccount: string; readonly jobNamePrefix: string; readonly idleTimeoutMs: number; + readonly missingTerminalAfterToolTimeoutMs: number; readonly apiKeySecretRef: { readonly name: string; readonly key: string }; readonly egressProxyUrl: string | null; readonly noProxyExtra: readonly string[]; @@ -541,6 +542,7 @@ function parseDeployment(input: Record, path: string): AgentRun serviceAccount: stringField(runner, "serviceAccount", `${path}.runner`), jobNamePrefix: stringField(runner, "jobNamePrefix", `${path}.runner`), idleTimeoutMs: positiveIntegerField(runner, "idleTimeoutMs", `${path}.runner`), + missingTerminalAfterToolTimeoutMs: positiveIntegerField(runner, "missingTerminalAfterToolTimeoutMs", `${path}.runner`), apiKeySecretRef: parseSecretRef(recordField(runner, "apiKeySecretRef", `${path}.runner`), `${path}.runner.apiKeySecretRef`), egressProxyUrl: optionalStringField(runner, "egressProxyUrl", `${path}.runner`) ?? null, noProxyExtra: optionalStringArrayField(runner, "noProxyExtra", `${path}.runner`), diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts index 087d3ef5..e464a6ac 100644 --- a/scripts/src/agentrun-manifests.ts +++ b/scripts/src/agentrun-manifests.ts @@ -439,6 +439,7 @@ function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: stri { name: "AGENTRUN_RUNNER_SERVICE_ACCOUNT", value: spec.deployment.runner.serviceAccount }, { name: "AGENTRUN_RUNNER_JOB_NAME_PREFIX", value: spec.deployment.runner.jobNamePrefix }, { name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(spec.deployment.runner.idleTimeoutMs) }, + { name: "AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(spec.deployment.runner.missingTerminalAfterToolTimeoutMs) }, { name: "AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", value: String(spec.deployment.runner.retention.maxRunners) }, { name: "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER", value: spec.deployment.runner.retention.cleanupOrder }, { name: "AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", value: String(spec.deployment.runner.retention.activeHeartbeatMaxAgeMs) }, diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 72b9d14e..87d70e65 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -462,6 +462,7 @@ function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record): Record< const pagePerformance = record(analysis.pagePerformance); const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item)); const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item)); + const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item)); + const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item)); + const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item)); const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); const id = result.id ?? "-"; @@ -7121,6 +7132,13 @@ function renderWebObserveAnalyzeResult(result: Record): Record< [[counts.samples, counts.control, counts.network, counts.console, counts.artifacts, rounds.length || sampleMetrics.rounds, sampleMetrics.turnTimingRows]], ), "", + jsonlReadIssues.length === 0 + ? "JSONL_READ_ISSUES\n-" + : webObserveTable( + ["FILE", "CODE", "MESSAGE"], + jsonlReadIssues.map((item) => [item.file, item.code, item.message]), + ), + "", webObserveTable( ["TURN_TIMING", "VALUE"], [ @@ -7163,11 +7181,47 @@ function renderWebObserveAnalyzeResult(result: Record): Record< ["httpError", runtimeAlerts.httpErrorCount], ["requestFailed", runtimeAlerts.requestFailedCount], ["domDiagnosticSamples", runtimeAlerts.domDiagnosticSampleCount], + ["domDiagnosticGroups", runtimeAlerts.domDiagnosticGroupCount], ["consoleAlerts", runtimeAlerts.consoleAlertCount], ["executionErrors", runtimeAlerts.executionErrorCount], ], ), "", + domDiagnosticGroups.length === 0 + ? "DOM_DIAGNOSTIC_GROUPS\n-" + : webObserveTable( + ["CODE", "COUNT", "FIRST_SEQ", "LAST_SEQ", "FIRST", "LAST", "PROMPTS", "TRACE", "PREVIEW"], + domDiagnosticGroups.map((item) => [ + item.diagnosticCode, + item.count, + item.firstSeq, + item.lastSeq, + item.firstAt, + item.lastAt, + webObserveArray(item.promptIndexes).join(",") || "-", + item.traceId, + String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 140), + ]), + ), + "", + domDiagnostics.length === 0 + ? "DOM_DIAGNOSTICS\n-" + : webObserveTable( + ["SEQ", "TS", "PROMPT", "CODE", "TRACE", "HTTP", "IDLE", "WAITING_FOR", "LAST_EVENT", "PREVIEW"], + domDiagnostics.map((item) => [ + item.seq, + item.ts, + item.promptIndex, + item.diagnosticCode, + item.traceId, + item.httpStatus, + item.idleSeconds, + item.waitingFor, + item.lastEventLabel, + String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 180), + ]), + ), + "", webObserveTable( ["PERF", "VALUE"], [ @@ -7202,6 +7256,94 @@ function renderWebObserveAnalyzeResult(result: Record): Record< return withWebObserveRendered(result, renderedText); } +function renderWebProbeScriptResult(result: Record): Record { + const summary = record(result.summary); + const issueEvidence = record(result.issueEvidence); + const probe = record(result.probe); + const script = record(probe.script); + const scriptResult = record(script.result); + const reportLoad = record(result.reportLoad); + const commandResult = record(result.result); + const recoveredArtifacts = record(result.recoveredArtifacts); + const recoveredArtifactSummary = record(recoveredArtifacts.artifacts); + const recoveredItems = webObserveArray(recoveredArtifactSummary.items).slice(-8).map((item) => record(item)); + const steps = webObserveArray(probe.steps).slice(-5).map((item) => record(item)); + const resultRows = webProbeScriptRecordRows(scriptResult, 12); + const evidenceRows = webProbeScriptRecordRows(issueEvidence, 12); + const summaryRows = [ + ["ok", result.ok], + ["status", result.status], + ["degradedReason", result.degradedReason ?? "-"], + ["failureKind", result.failureKind ?? "-"], + ["scriptSource", result.scriptSource ?? "-"], + ["reportSource", reportLoad.source ?? summary.recoveredFrom ?? "-"], + ["reportPath", reportLoad.path ?? probe.reportPath ?? "-"], + ["reportSha256", probe.reportSha256 ?? "-"], + ["scriptSha256", probe.scriptSha256 ?? "-"], + ["transportTimedOut", summary.transportTimedOut ?? commandResult.timedOut ?? "-"], + ["exitCode", commandResult.exitCode ?? "-"], + ]; + const renderedText = [ + webObserveTable( + ["WEB_PROBE_SCRIPT", "NODE", "LANE", "STATUS", "OK", "URL"], + [[result.command ?? "web-probe script", result.node, result.lane, result.status, result.ok, result.url]], + ), + "", + webObserveTable(["FIELD", "VALUE"], summaryRows), + "", + resultRows.length === 0 + ? "SCRIPT_RESULT\n-" + : webObserveTable(["KEY", "VALUE"], resultRows), + "", + evidenceRows.length === 0 + ? "ISSUE_EVIDENCE\n-" + : webObserveTable(["KEY", "VALUE"], evidenceRows), + "", + steps.length === 0 + ? "STEPS\n-" + : webObserveTable( + ["STEP", "OK", "DETAIL"], + steps.map((item) => [ + item.name ?? item.step ?? item.label ?? "-", + item.ok ?? item.status ?? "-", + webProbeScriptPreview(item.result ?? item.data ?? item.error ?? item.message ?? item), + ]), + ), + "", + recoveredItems.length === 0 + ? "ARTIFACTS\n-" + : webObserveTable( + ["KIND", "BYTES", "PATH"], + recoveredItems.map((item) => [item.kind, item.byteCount, item.path]), + ), + "", + "NEXT", + ` report: ${reportLoad.path ?? probe.reportPath ?? "-"}`, + ` rerun: ${result.command ?? `hwlab nodes web-probe script --node ${result.node ?? "-"} --lane ${result.lane ?? "-"}`}`, + ].join("\n"); + return withWebObserveRendered(result, renderedText); +} + +function webProbeScriptRecordRows(value: Record, limit: number): unknown[][] { + return Object.entries(value) + .slice(0, limit) + .map(([key, nested]) => [key, webProbeScriptPreview(nested)]); +} + +function webProbeScriptPreview(value: unknown): string { + if (value === null || value === undefined) return "-"; + if (typeof value === "string") return value.replace(/\s+/gu, " ").trim(); + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (Array.isArray(value)) { + const preview = value.slice(0, 3).map((item) => webProbeScriptPreview(item)).join(", "); + return `[${value.length}] ${preview}`; + } + if (typeof value === "object") { + return JSON.stringify(value).replace(/\s+/gu, " ").slice(0, 240); + } + return String(value); +} + function withWebObserveShortcuts(value: Record | null, id: string | null): Record | null { if (value === null || id === null) return value; return { @@ -7374,7 +7516,7 @@ function runNodeWebProbeScript( compactResult.stdoutTail = redactKnownSecrets(result.stdout.slice(-2000), [material.password ?? ""]); compactResult.stderrTail = redactKnownSecrets(result.stderr.slice(-2000), [material.password ?? ""]); } - return { + return renderWebProbeScriptResult({ ok: passed, status: passed ? "pass" : "blocked", command: `hwlab nodes web-probe script --node ${options.node} --lane ${options.lane}`, @@ -7403,7 +7545,7 @@ function runNodeWebProbeScript( }, result: compactResult, valuesRedacted: true, - }; + }); } function nodeWebProbeScriptRemoteShell(options: NodeWebProbeScriptOptions, secretSpec: RuntimeSecretSpec, password: string): string { diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index 9fec0dd0..ac266e12 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -982,13 +982,16 @@ function sleep(ms) { export function nodeWebObserveAnalyzerSource(): string { return String.raw`#!/usr/bin/env node import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; +import { createInterface } from "node:readline"; const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual"); const analysisDir = path.join(stateDir, "analysis"); const reportJsonPath = path.join(analysisDir, "report.json"); const reportMdPath = path.join(analysisDir, "report.md"); +const jsonlReadIssues = []; const samples = await readJsonl(path.join(stateDir, "samples.jsonl")); const control = await readJsonl(path.join(stateDir, "control.jsonl")); const network = await readJsonl(path.join(stateDir, "network.jsonl")); @@ -1015,6 +1018,7 @@ const report = { manifest: compactManifest(manifest), heartbeat: compactHeartbeat(heartbeat), counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length }, + jsonlReadIssues, commandTimeline, transitions, sampleMetrics, @@ -1029,19 +1033,36 @@ const report = { await writeFile(reportJsonPath, JSON.stringify(report, null, 2) + "\n", { mode: 0o600 }); await writeFile(reportMdPath, renderMarkdown(report), { mode: 0o600 }); const [jsonMeta, mdMeta] = await Promise.all([fileMeta(reportJsonPath), fileMeta(reportMdPath)]); -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: runtimeAlerts.summary, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +const compactRuntimeAlerts = { + ...runtimeAlerts.summary, + domDiagnostics: runtimeAlerts.domDiagnostics.slice(-80), + domDiagnosticsByFingerprint: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 80), +}; +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } } async function readJsonl(file) { + const rows = []; try { - const text = await readFile(file, "utf8"); - return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { - try { return JSON.parse(line); } catch { return { parseError: true, rawHash: sha256(line) }; } - }); - } catch { return []; } + const input = createReadStream(file, { encoding: "utf8" }); + const lines = createInterface({ input, crlfDelay: Infinity }); + for await (const line of lines) { + const trimmed = String(line || "").trim(); + if (!trimmed) continue; + try { + rows.push(JSON.parse(trimmed)); + } catch { + rows.push({ parseError: true, rawHash: sha256(trimmed) }); + } + } + } catch (error) { + if (error?.code === "ENOENT") return rows; + jsonlReadIssues.push({ file: path.basename(file), code: error?.code ?? error?.name ?? "jsonl_read_failed", message: String(error?.message || "JSONL read failed").slice(0, 240), valuesPrinted: false }); + } + return rows; } function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance) { @@ -1414,6 +1435,10 @@ function parseDomDiagnosticSummary(text) { const lastEventLabelMatch = value.match(/\blastEventLabel=([^\s；;，,)]+)/iu); const diagnosticCode = httpStatusMatch ? "http-" + httpStatusMatch[1] + : /projection-resume:sync-failed/iu.test(value) + ? "projection-resume-sync-failed" + : /AgentRun\s+GET\b[\s\S]*\/result\b[\s\S]*timed out after\s+\d+ms/iu.test(value) + ? "agentrun-result-timeout" : /turn\s*超过|无新活动/iu.test(value) ? "turn-idle-no-activity" : /Failed to fetch/iu.test(value) @@ -1564,13 +1589,15 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) { pageErrorCount: pageErrors.length, networkErrorGroupCount: groupNetworkAlerts(httpErrors).length, requestFailedGroupCount: groupNetworkAlerts(requestFailed).length, + domDiagnosticGroupCount: groupDomDiagnostics(domDiagnostics).length, executionErrorGroupCount: groupExecutionErrors(executionErrors).length, baselineExecutionErrorGroupCount: groupExecutionErrors(baselineExecutionErrors).length, consoleAlertGroupCount: groupConsoleAlerts(consoleAlerts).length }, networkHttpErrorsByPath: groupNetworkAlerts(httpErrors), networkRequestFailedByPath: groupNetworkAlerts(requestFailed), - domDiagnostics: domDiagnostics.slice(0, 80), + domDiagnostics: domDiagnostics.slice(-80), + domDiagnosticsByFingerprint: groupDomDiagnostics(domDiagnostics).slice(0, 80), runtimeExecutionErrors: executionErrors.slice(0, 120), runtimeExecutionErrorsByCode: groupExecutionErrors(executionErrors), baselineRuntimeExecutionErrors: baselineExecutionErrors.slice(0, 80), @@ -1676,6 +1703,42 @@ function groupExecutionErrors(events) { return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(a.code).localeCompare(String(b.code))); } +function groupDomDiagnostics(events) { + const groups = new Map(); + for (const event of events) { + const key = [event.diagnosticCode || "-", event.traceId || "-", event.textHash || "-"].join(" "); + const group = groups.get(key) || { + diagnosticCode: event.diagnosticCode ?? null, + traceId: event.traceId ?? null, + textHash: event.textHash ?? null, + count: 0, + firstSeq: event.seq ?? null, + lastSeq: event.seq ?? null, + firstAt: event.ts ?? null, + lastAt: event.ts ?? null, + promptIndexes: [], + sources: [], + httpStatus: event.httpStatus ?? null, + idleSeconds: event.idleSeconds ?? null, + waitingFor: event.waitingFor ?? null, + lastEventLabel: event.lastEventLabel ?? null, + preview: event.preview ?? null, + valuesRedacted: true + }; + group.count += 1; + group.lastSeq = event.seq ?? group.lastSeq; + group.lastAt = event.ts ?? group.lastAt; + if (event.promptIndex && !group.promptIndexes.includes(event.promptIndex)) group.promptIndexes.push(event.promptIndex); + if (event.source && !group.sources.includes(event.source)) group.sources.push(event.source); + if (event.httpStatus && !group.httpStatus) group.httpStatus = event.httpStatus; + if (event.idleSeconds && !group.idleSeconds) group.idleSeconds = event.idleSeconds; + if (event.waitingFor && !group.waitingFor) group.waitingFor = event.waitingFor; + if (event.lastEventLabel && !group.lastEventLabel) group.lastEventLabel = event.lastEventLabel; + groups.set(key, group); + } + return Array.from(groups.values()).sort((a, b) => b.count - a.count || String(b.lastAt ?? "").localeCompare(String(a.lastAt ?? ""))); +} + function consoleAlertEvent(item, promptTimes) { const text = String(item?.text || ""); const statusMatch = text.match(/\bstatus\s+of\s+([1-5][0-9]{2})\b/iu) || text.match(/\bHTTP\s+([1-5][0-9]{2})\b/iu); diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index b8fbcb21..d3170f5a 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -648,16 +648,24 @@ async function trace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); + const businessTraceId = options.query === null ? businessTraceIdFromSearchText(options.grep) : null; + const effectiveLookbackMinutes = businessTraceId === null ? options.lookbackMinutes : Math.max(options.lookbackMinutes, 10080); + const effectiveTempoQuery = options.query ?? (businessTraceId === null ? null : `{ .traceId = "${businessTraceId}" }`); + const effectiveOptions: SearchOptions = { + ...options, + query: effectiveTempoQuery, + lookbackMinutes: effectiveLookbackMinutes, + }; const endSeconds = Math.floor(Date.now() / 1000); - const startSeconds = endSeconds - (options.lookbackMinutes * 60); + const startSeconds = endSeconds - (effectiveLookbackMinutes * 60); const params = new URLSearchParams({ start: String(startSeconds), end: String(endSeconds), limit: String(options.candidateLimit), }); - if (options.query !== null) params.set("q", options.query); + if (effectiveTempoQuery !== null) params.set("q", effectiveTempoQuery); const searchPath = `/api/search?${params.toString()}`; - const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, options)); + const result = await capture(config, target.route, ["sh"], searchScript(observability, target, searchPath, effectiveOptions)); const parsed = parseJsonOutput(result.stdout); const response = { ok: result.exitCode === 0 && parsed?.ok === true, @@ -669,8 +677,11 @@ async function search(config: UniDeskConfig, options: SearchOptions): Promise | RenderedCliResult> { const observability = readObservabilityConfig(); const target = resolveTarget(observability, options.targetId); @@ -811,6 +827,9 @@ function renderSearchResult(response: Record): RenderedCliResul const lines: string[] = []; lines.push("COMMAND TARGET OK GREP_OR_QUERY MATCHED SCANNED CANDIDATES"); lines.push(`${pad("platform-infra observability search", 38)} ${pad(textValue(target.id) || "-", 7)} ${pad(response.ok === true ? "true" : "false", 6)} ${pad(truncate(textValue(query.grep) || textValue(query.tempoQuery) || "-", 35), 35)} ${pad(textValue(result.matchedTraceCount) || "-", 8)} ${pad(textValue(result.scannedTraceCount) || "-", 8)} ${textValue(result.candidateTraceCount) || "-"}`); + if (textValue(query.mode) === "business-trace-exact") { + lines.push(`info: businessTraceId=${textValue(query.businessTraceId) || "-"} uses exact TraceQL and lookbackMinutes=${textValue(query.lookbackMinutes) || "-"} (requested=${textValue(query.requestedLookbackMinutes) || "-"})`); + } if (textValue(result.scanStopped)) lines.push(`warning: scanStopped=${textValue(result.scanStopped)}`); const truncated = recordValue(result.truncated); if (truncated.candidateTraces === true || truncated.matchedTraces === true) { @@ -880,7 +899,7 @@ function renderTraceResult(response: Record): RenderedCliResult lines.push("SERVICE DURATION_MS NAME DETAIL"); for (const item of errorSpans.slice(0, 8)) { const row = recordValue(item); - lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 160)}`); } } if (matchedSpans.length > 0) { @@ -889,7 +908,7 @@ function renderTraceResult(response: Record): RenderedCliResult lines.push("SERVICE DURATION_MS NAME DETAIL"); for (const item of matchedSpans.slice(0, 8)) { const row = recordValue(item); - lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 96)}`); + lines.push(`${pad(truncate(textValue(row.service) || "-", 23), 23)} ${pad(textValue(row.durationMs) || "-", 12)} ${pad(truncate(textValue(row.name) || "-", 40), 40)} ${truncate(traceSpanDetail(row), 160)}`); } } lines.push(""); @@ -914,6 +933,7 @@ function traceSpanDetail(span: Record): string { "toolName", "exitCode", "durationMs", + "command", "waitingFor", "lastEventLabel", "idleMs", @@ -1799,7 +1819,7 @@ function searchScript(observability: ObservabilityConfig, target: ObservabilityT return ` set -u python3 - <<'PY' -import collections, json, subprocess, time +import collections, json, re, subprocess, time SEARCH_PROXY_PATH = ${JSON.stringify(searchProxyPath)} TRACE_PROXY_PREFIX = ${JSON.stringify(proxyPrefix)} @@ -1811,6 +1831,7 @@ QUERY = ${queryLiteral} LIMIT = ${options.limit} CANDIDATE_LIMIT = ${options.candidateLimit} DEADLINE = time.time() + 50 +BUSINESS_TRACE_GREP = bool(GREP is not None and re.search(r"\\btrc_[A-Za-z0-9_-]+\\b", GREP)) IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", @@ -2037,6 +2058,15 @@ def trace_summary(trace_id, meta, body, rc, stderr): "stderrTail": (stderr or "")[-1000:], } +def rich_trace_rank(summary): + services = summary.get("services", []) + if not isinstance(services, list): + services = [] + span_count = summary.get("spanCount") if isinstance(summary.get("spanCount"), int) else 0 + error_count = summary.get("errorSpanCount") if isinstance(summary.get("errorSpanCount"), int) else 0 + matched_count = summary.get("matchedSpanCount") if isinstance(summary.get("matchedSpanCount"), int) else 0 + return (len(services), span_count, error_count, matched_count) + search_rc, search_body, search_err = run_kubectl(SEARCH_PROXY_PATH, timeout=15) search_parsed = parse_json(search_body) search_parse_ok = isinstance(search_parsed, dict) @@ -2065,8 +2095,10 @@ for trace_id, meta in candidate_trace_ids: scanned.append(summary) if GREP is None or summary.get("rawMatched") is True or (summary.get("matchedSpanCount") or 0) > 0: matched.append(summary) - if len(matched) >= LIMIT and GREP is not None: + if len(matched) >= LIMIT and GREP is not None and not BUSINESS_TRACE_GREP: break +if BUSINESS_TRACE_GREP: + matched.sort(key=rich_trace_rank, reverse=True) payload = { "ok": search_rc == 0 and search_parse_ok, @@ -2074,6 +2106,7 @@ payload = { "searchProxyPath": SEARCH_PROXY_PATH, "grep": GREP, "tempoQuery": QUERY, + "businessTraceSearch": BUSINESS_TRACE_GREP, "limit": LIMIT, "candidateLimit": CANDIDATE_LIMIT, "searchParseOk": search_parse_ok, From 0817e3d22a1e9d4e44956ba728921d82a5549892 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 23 Jun 2026 04:31:37 +0000 Subject: [PATCH 5/6] fix: make web-probe observe thresholds yaml-driven --- config/hwlab-node-lanes.yaml | 26 ++ scripts/src/hwlab-node-impl.ts | 43 ++- scripts/src/hwlab-node-lanes.ts | 56 +++ .../hwlab-node-web-observe-runner-source.ts | 335 ++++++++++++++++-- 4 files changed, 431 insertions(+), 29 deletions(-) diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 2bb9b970..eebcc4c4 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -104,6 +104,19 @@ lanes: - hwlab-agent-skills observability: prometheusOperator: true + webProbe: + observe: + analysisThresholds: + sameOriginApiBudgetMs: 5000 + longLivedStreamOpenBudgetMs: 5000 + longLivedStreamLifetimeBudgetMs: 5000 + recentUpdateSawtoothMinAllowedIncreaseSeconds: 3 + recentUpdateSawtoothSampleSlackSeconds: 2 + uncommandedStateChangeCommandWindowMs: 10000 + scrollJumpCommandWindowMs: 8000 + scrollJumpFromY: 250 + scrollJumpToY: 40 + loadingVisibleBudgetMs: 10000 public: webUrl: http://74.48.78.17:20666 apiUrl: http://74.48.78.17:20667 @@ -321,6 +334,19 @@ lanes: thresholdSeconds: 10 minSamples: 1 for: 10m + webProbe: + observe: + analysisThresholds: + sameOriginApiBudgetMs: 5000 + longLivedStreamOpenBudgetMs: 5000 + longLivedStreamLifetimeBudgetMs: 5000 + recentUpdateSawtoothMinAllowedIncreaseSeconds: 3 + recentUpdateSawtoothSampleSlackSeconds: 2 + uncommandedStateChangeCommandWindowMs: 10000 + scrollJumpCommandWindowMs: 8000 + scrollJumpFromY: 250 + scrollJumpToY: 40 + loadingVisibleBudgetMs: 10000 runtimeImageRewrites: - source: fatedier/frpc:v0.68.1 target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 87d70e65..19007a54 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -6730,6 +6730,21 @@ function runNodeWebProbeObserveCollect(options: NodeWebProbeObserveOptions, spec } function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec: HwlabRuntimeLaneSpec): Record { + const analysisThresholds = spec.observability.webProbe?.observe?.analysisThresholds ?? null; + if (analysisThresholds === null) { + return { + ok: false, + status: "blocked", + command: webObserveCommandLabel("analyze", options), + id: webObserveIdFromOptions(options), + node: options.node, + lane: options.lane, + workspace: spec.workspace, + degradedReason: "web-probe-analysis-thresholds-not-declared", + configPath: "observability.webProbe.observe.analysisThresholds", + valuesRedacted: true, + }; + } const analyzerB64 = Buffer.from(nodeWebObserveAnalyzerSource(), "utf8").toString("base64"); const script = [ "set -eu", @@ -6737,7 +6752,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec "analyzer=\"$state_dir/observer-analyzer.mjs\"", `node -e "require('fs').writeFileSync(process.argv[1], Buffer.from(process.argv[2], 'base64'))" "$analyzer" ${shellQuote(analyzerB64)}`, "chmod 700 \"$analyzer\"", - "UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" node \"$analyzer\"", + `UNIDESK_WEB_OBSERVE_ANALYSIS_THRESHOLDS_JSON=${shellQuote(JSON.stringify(analysisThresholds))} UNIDESK_WEB_OBSERVE_STATE_DIR="$state_dir" node "$analyzer"`, ].join("\n"); const result = runTransWorkspaceStdinScript(options.node, spec.workspace, script, options.commandTimeoutSeconds); const analysis = parseJsonObject(result.stdout); @@ -7113,12 +7128,15 @@ function renderWebObserveAnalyzeResult(result: Record): Record< const sampleMetrics = record(analysis.sampleMetrics); const runtimeAlerts = record(analysis.runtimeAlerts); const pagePerformance = record(analysis.pagePerformance); + const loading = record(analysis.loading); + const thresholds = record(analysis.analysisThresholds); const rounds = webObserveArray(sampleMetrics.rounds).slice(-10).map((item) => record(item)); const turnColumns = webObserveArray(sampleMetrics.turnColumns).slice(-12).map((item) => record(item)); const domDiagnosticGroups = webObserveArray(runtimeAlerts.domDiagnosticsByFingerprint).slice(0, 10).map((item) => record(item)); const domDiagnostics = webObserveArray(runtimeAlerts.domDiagnostics).slice(-12).map((item) => record(item)); const jsonlReadIssues = webObserveArray(analysis.jsonlReadIssues).slice(0, 8).map((item) => record(item)); const slowApis = webObserveArray(analysis.pagePerformanceSlowApi).slice(0, 8).map((item) => record(item)); + const loadingSegments = webObserveArray(loading.overBudgetSegments).slice(0, 8).map((item) => record(item)); const findings = webObserveArray(analysis.findings).slice(0, 8).map((item) => record(item)); const id = result.id ?? "-"; const renderedText = [ @@ -7225,6 +7243,7 @@ function renderWebObserveAnalyzeResult(result: Record): Record< webObserveTable( ["PERF", "VALUE"], [ + ["sameOriginApiBudgetMs", thresholds.sameOriginApiBudgetMs], ["sameOriginApiPaths", pagePerformance.sameOriginApiPathCount], ["slowPathCount", pagePerformance.slowPathCount], ["slowSampleCount", pagePerformance.slowSampleCount], @@ -7235,8 +7254,26 @@ function renderWebObserveAnalyzeResult(result: Record): Record< slowApis.length === 0 ? "SLOW_API\n-" : webObserveTable( - ["SLOW_API", "P50", "P75", "P95", ">5S", "COUNT"], - slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overFiveSecondCount, item.sampleCount]), + ["SLOW_API", "P50", "P75", "P95", "OVER_BUDGET", "COUNT"], + slowApis.map((item) => [item.path, item.p50Ms, item.p75Ms, item.p95Ms, item.overBudgetCount ?? item.overFiveSecondCount, item.sampleCount]), + ), + "", + webObserveTable( + ["LOADING", "VALUE"], + [ + ["visibleBudgetMs", thresholds.loadingVisibleBudgetMs], + ["samplesWithLoading", loading.samplesWithLoading], + ["maxVisibleCount", loading.maxVisibleCount], + ["overBudgetSegments", loading.overBudgetSegmentCount], + ["longestDurationMs", loading.longestDurationMs], + ], + ), + "", + loadingSegments.length === 0 + ? "LOADING_OVER_BUDGET\n-" + : webObserveTable( + ["OWNER", "TEXT", "SAMPLES", "DURATION_MS", "FIRST", "LAST"], + loadingSegments.map((item) => [item.owner, String(item.preview ?? "").replace(/\s+/g, " ").slice(0, 120), item.sampleCount, item.durationMs, item.firstAt, item.lastAt]), ), "", findings.length === 0 diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index f3c81f99..ded8c836 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -100,6 +100,7 @@ export interface HwlabRuntimeObservabilitySpec { readonly traceExplorerUrlTemplate?: string; readonly metricsEndpoint?: HwlabRuntimeObservabilityMetricsEndpointSpec; readonly workbench?: HwlabRuntimeObservabilityWorkbenchSpec; + readonly webProbe?: HwlabRuntimeObservabilityWebProbeSpec; readonly recordingRules: readonly HwlabRuntimeObservabilityRecordingRuleSpec[]; readonly warningAlerts: readonly HwlabRuntimeObservabilityWarningAlertSpec[]; } @@ -124,6 +125,27 @@ export interface HwlabRuntimeObservabilityWorkbenchSpec { readonly maxUnknownEventLines: number; } +export interface HwlabRuntimeObservabilityWebProbeSpec { + readonly observe?: HwlabRuntimeObservabilityWebProbeObserveSpec; +} + +export interface HwlabRuntimeObservabilityWebProbeObserveSpec { + readonly analysisThresholds?: HwlabRuntimeObservabilityWebProbeAnalysisThresholdsSpec; +} + +export interface HwlabRuntimeObservabilityWebProbeAnalysisThresholdsSpec { + readonly sameOriginApiBudgetMs: number; + readonly longLivedStreamOpenBudgetMs: number; + readonly longLivedStreamLifetimeBudgetMs: number; + readonly recentUpdateSawtoothMinAllowedIncreaseSeconds: number; + readonly recentUpdateSawtoothSampleSlackSeconds: number; + readonly uncommandedStateChangeCommandWindowMs: number; + readonly scrollJumpCommandWindowMs: number; + readonly scrollJumpFromY: number; + readonly scrollJumpToY: number; + readonly loadingVisibleBudgetMs: number; +} + export interface HwlabRuntimeObservabilityRecordingRuleSpec { readonly id: string; readonly metric: string; @@ -671,6 +693,7 @@ function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservab traceExplorerUrlTemplate, metricsEndpoint: observabilityMetricsEndpointConfig(raw.metricsEndpoint, `${path}.metricsEndpoint`), workbench: observabilityWorkbenchConfig(raw.workbench, `${path}.workbench`), + webProbe: observabilityWebProbeConfig(raw.webProbe, `${path}.webProbe`), recordingRules, warningAlerts, }; @@ -726,6 +749,39 @@ function observabilityWorkbenchConfig(value: unknown, path: string): HwlabRuntim }; } +function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntimeObservabilityWebProbeSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + observe: observabilityWebProbeObserveConfig(raw.observe, `${path}.observe`), + }; +} + +function observabilityWebProbeObserveConfig(value: unknown, path: string): HwlabRuntimeObservabilityWebProbeObserveSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + analysisThresholds: observabilityWebProbeAnalysisThresholdsConfig(raw.analysisThresholds, `${path}.analysisThresholds`), + }; +} + +function observabilityWebProbeAnalysisThresholdsConfig(value: unknown, path: string): HwlabRuntimeObservabilityWebProbeAnalysisThresholdsSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + sameOriginApiBudgetMs: numberField(raw, "sameOriginApiBudgetMs", path), + longLivedStreamOpenBudgetMs: numberField(raw, "longLivedStreamOpenBudgetMs", path), + longLivedStreamLifetimeBudgetMs: numberField(raw, "longLivedStreamLifetimeBudgetMs", path), + recentUpdateSawtoothMinAllowedIncreaseSeconds: positiveNumberField(raw, "recentUpdateSawtoothMinAllowedIncreaseSeconds", path), + recentUpdateSawtoothSampleSlackSeconds: positiveNumberField(raw, "recentUpdateSawtoothSampleSlackSeconds", path), + uncommandedStateChangeCommandWindowMs: numberField(raw, "uncommandedStateChangeCommandWindowMs", path), + scrollJumpCommandWindowMs: numberField(raw, "scrollJumpCommandWindowMs", path), + scrollJumpFromY: numberField(raw, "scrollJumpFromY", path), + scrollJumpToY: nonNegativeIntegerField(raw, "scrollJumpToY", path), + loadingVisibleBudgetMs: numberField(raw, "loadingVisibleBudgetMs", path), + }; +} + function observabilityRecordingRulesConfig(value: unknown, path: string): HwlabRuntimeObservabilityRecordingRuleSpec[] { if (value === undefined) return []; if (!Array.isArray(value)) throw new Error(`${path} must be an array`); diff --git a/scripts/src/hwlab-node-web-observe-runner-source.ts b/scripts/src/hwlab-node-web-observe-runner-source.ts index ac266e12..b6caa682 100644 --- a/scripts/src/hwlab-node-web-observe-runner-source.ts +++ b/scripts/src/hwlab-node-web-observe-runner-source.ts @@ -713,6 +713,19 @@ async function samplePage(reason) { const messageSelector = '[data-testid*="message" i], [class*="message" i], article, [role="article"]'; const traceSelector = '[data-testid*="trace" i], [class*="trace" i], [data-trace-id], [data-testid*="event" i]'; const diagnosticSelector = '.api-error-diagnostic, .api-error-diagnostic.message-diagnostic, .api-error-diagnostic.projection-diagnostic, [data-testid="api-error-diagnostic" i], [data-testid="error-diagnostic" i], [role="alert"], .alert, .warning, .error'; + const loadingSelector = '[aria-busy="true"], [role="progressbar"], [data-testid*="loading" i], [data-testid*="spinner" i], [class*="loading" i], [class*="spinner" i]'; + const loadingTextPattern = /(?:^|[\s:：])(?:加载中|正在加载|加载数据|Loading|loading|Please wait|请稍候|处理中|等待中)(?:[\s.。:：…]|$)/u; + const ownerSummary = (element) => { + const owner = element.closest('article.message-card, .message-card, [data-session-id], [data-testid], main, aside, section, article') || element.parentElement || element; + return { + tag: owner.tagName.toLowerCase(), + testId: owner.getAttribute("data-testid"), + role: owner.getAttribute("role"), + sessionId: owner.getAttribute("data-session-id"), + messageId: owner.getAttribute("data-message-id") || owner.getAttribute("id"), + className: String(owner.className || "").slice(0, 160), + }; + }; const messages = summarize(messageSelector, 20); const traceRows = summarize(traceSelector, 30); const diagnostics = Array.from(document.querySelectorAll(diagnosticSelector)).filter(visible).slice(-40).map((element, index) => { @@ -744,6 +757,37 @@ async function samplePage(reason) { rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, }; }).filter(Boolean); + const loading = []; + const loadingSeen = new Set(); + const addLoadingCandidate = (element, reason) => { + if (!element || !visible(element)) return; + const text = textHashInput(element); + if (reason === "text" && !loadingTextPattern.test(text)) return; + const rect = element.getBoundingClientRect(); + const key = [element.tagName, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height), text.slice(0, 80)].join("|"); + if (loadingSeen.has(key)) return; + loadingSeen.add(key); + loading.push({ + index: loading.length, + reason, + tag: element.tagName.toLowerCase(), + testId: element.getAttribute("data-testid"), + role: element.getAttribute("role"), + ariaBusy: element.getAttribute("aria-busy"), + className: String(element.className || "").slice(0, 160), + owner: ownerSummary(element), + text, + rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, + }); + }; + for (const element of Array.from(document.querySelectorAll(loadingSelector)).slice(0, 120)) addLoadingCandidate(element, "selector"); + for (const element of Array.from(document.querySelectorAll("body *")).slice(0, 1200)) { + if (!visible(element)) continue; + const text = textHashInput(element); + if (!loadingTextPattern.test(text)) continue; + const childHasLoading = Array.from(element.children).some((child) => visible(child) && loadingTextPattern.test(textHashInput(child))); + if (!childHasLoading) addLoadingCandidate(element, "text"); + } const turns = Array.from(document.querySelectorAll('article.message-card[data-role="agent"], .message-card[data-role="agent"], article[data-role="agent"]')).filter(visible).map((element, index) => { const rect = element.getBoundingClientRect(); const text = textHashInput(element); @@ -776,6 +820,7 @@ async function samplePage(reason) { messages, traceRows, diagnostics, + loading: loading.slice(0, 80), turns, pageProvenance: { url: location.href, @@ -832,10 +877,11 @@ function digestDom(dom) { const messages = Array.isArray(dom.messages) ? dom.messages.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : []; const traceRows = Array.isArray(dom.traceRows) ? dom.traceRows.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : []; const diagnostics = Array.isArray(dom.diagnostics) ? dom.diagnostics.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 260), textBytes: Buffer.byteLength(item.text || "") })) : []; + const loading = Array.isArray(dom.loading) ? dom.loading.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 160), textBytes: Buffer.byteLength(item.text || "") })) : []; const turns = Array.isArray(dom.turns) ? dom.turns.map((item) => ({ ...item, textHash: sha256Text(item.text || ""), textPreview: truncate(item.text || "", 200), textBytes: Buffer.byteLength(item.text || "") })) : []; const pageProvenance = normalizePageProvenance(dom.pageProvenance, { reason: "sample", pageLoadSeq: currentPageProvenance?.pageLoadSeq ?? pageLoadSeq }); currentPageProvenance = pageProvenance; - return { ...dom, messages, traceRows, diagnostics, turns, pageProvenance: compactPageProvenance(pageProvenance) }; + return { ...dom, messages, traceRows, diagnostics, loading, turns, pageProvenance: compactPageProvenance(pageProvenance) }; } async function captureScreenshot(reason, imageType = "png") { @@ -1000,15 +1046,17 @@ const errors = await readJsonl(path.join(stateDir, "errors.jsonl")); const artifacts = await readJsonl(path.join(stateDir, "artifacts.jsonl")); const manifest = await readJson(path.join(stateDir, "manifest.json")); const heartbeat = await readJson(path.join(stateDir, "heartbeat.json")); +const analysisThresholds = parseAnalysisThresholds(process.env.UNIDESK_WEB_OBSERVE_ANALYSIS_THRESHOLDS_JSON); await mkdir(analysisDir, { recursive: true, mode: 0o700 }); const transitions = buildTransitions(samples); -const sampleMetrics = buildSampleMetrics(samples, control); +const sampleMetrics = buildSampleMetrics(samples, control, analysisThresholds); const pageProvenance = buildPageProvenanceReport(samples, control, manifest); -const pagePerformance = buildPagePerformanceReport(samples, manifest); +const pagePerformance = buildPagePerformanceReport(samples, manifest, analysisThresholds); +const loading = buildLoadingReport(samples, analysisThresholds); const promptNetwork = buildPromptNetworkReport(control, network); const runtimeAlerts = buildRuntimeAlerts(samples, control, network, consoleEvents, errors); -const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance); +const findings = buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, loading, analysisThresholds); const commandTimeline = control.filter((item) => item.phase === "completed" || item.phase === "failed").map((item) => ({ ts: item.ts, phase: item.phase, commandId: item.commandId, type: item.type, input: item.input, afterUrl: item.afterUrl })); const report = { ok: findings.filter((item) => item.severity === "red").length === 0, @@ -1020,10 +1068,12 @@ const report = { counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length }, jsonlReadIssues, commandTimeline, + analysisThresholds, transitions, sampleMetrics, pageProvenance, pagePerformance, + loading, promptNetwork, runtimeAlerts, findings, @@ -1038,7 +1088,7 @@ const compactRuntimeAlerts = { domDiagnostics: runtimeAlerts.domDiagnostics.slice(-80), domDiagnosticsByFingerprint: runtimeAlerts.domDiagnosticsByFingerprint.slice(0, 80), }; -console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); +console.log(JSON.stringify({ ok: true, command: "web-probe-observe analyze", stateDir, reportJsonPath, reportJsonSha256: jsonMeta.sha256, reportMdPath, reportMdSha256: mdMeta.sha256, counts: report.counts, jsonlReadIssues: jsonlReadIssues.slice(0, 20), analysisThresholds, sampleMetrics: { ...sampleMetrics.summary, rounds: sampleMetrics.rounds.slice(-20), turnColumns: sampleMetrics.turnColumns.slice(-50) }, pageProvenance: pageProvenance.summary, pagePerformance: pagePerformance.summary, loading: { ...loading.summary, overBudgetSegments: loading.overBudgetSegments.slice(0, 20), ownerGroups: loading.ownerGroups.slice(0, 20) }, promptNetwork: promptNetwork.summary, runtimeAlerts: compactRuntimeAlerts, turnTimingRecentUpdateSawtoothJumps: sampleMetrics.turnTimingRecentUpdateSawtoothJumps.slice(0, 20), turnTimingRecentUpdateLargestSteps: sampleMetrics.turnTimingRecentUpdateLargestSteps.slice(0, 20), pagePerformanceSlowApi: pagePerformance.sameOriginApiByPath.filter((item) => item.overBudgetCount > 0).slice(0, 20), pagePerformanceLongLivedStreams: pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream).slice(0, 20), findings: findings.slice(0, 20), valuesRedacted: true }, null, 2)); async function readJson(file) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } @@ -1065,7 +1115,45 @@ async function readJsonl(file) { return rows; } -function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance) { +function parseAnalysisThresholds(rawJson) { + if (!rawJson) throw new Error("missing UNIDESK_WEB_OBSERVE_ANALYSIS_THRESHOLDS_JSON; configure observability.webProbe.observe.analysisThresholds in YAML"); + let raw; + try { + raw = JSON.parse(rawJson); + } catch (error) { + throw new Error("invalid UNIDESK_WEB_OBSERVE_ANALYSIS_THRESHOLDS_JSON: " + (error?.message || String(error))); + } + const positiveInteger = (key) => { + const value = Number(raw?.[key]); + if (!Number.isInteger(value) || value <= 0) throw new Error("analysis threshold " + key + " must be a positive integer"); + return value; + }; + const nonNegativeInteger = (key) => { + const value = Number(raw?.[key]); + if (!Number.isInteger(value) || value < 0) throw new Error("analysis threshold " + key + " must be a non-negative integer"); + return value; + }; + const positiveNumber = (key) => { + const value = Number(raw?.[key]); + if (!Number.isFinite(value) || value <= 0) throw new Error("analysis threshold " + key + " must be a positive number"); + return value; + }; + return { + sameOriginApiBudgetMs: positiveInteger("sameOriginApiBudgetMs"), + longLivedStreamOpenBudgetMs: positiveInteger("longLivedStreamOpenBudgetMs"), + longLivedStreamLifetimeBudgetMs: positiveInteger("longLivedStreamLifetimeBudgetMs"), + recentUpdateSawtoothMinAllowedIncreaseSeconds: positiveNumber("recentUpdateSawtoothMinAllowedIncreaseSeconds"), + recentUpdateSawtoothSampleSlackSeconds: positiveNumber("recentUpdateSawtoothSampleSlackSeconds"), + uncommandedStateChangeCommandWindowMs: positiveInteger("uncommandedStateChangeCommandWindowMs"), + scrollJumpCommandWindowMs: positiveInteger("scrollJumpCommandWindowMs"), + scrollJumpFromY: positiveInteger("scrollJumpFromY"), + scrollJumpToY: nonNegativeInteger("scrollJumpToY"), + loadingVisibleBudgetMs: positiveInteger("loadingVisibleBudgetMs"), + valuesRedacted: true + }; +} + +function buildFindings(samples, control, network, errors, sampleMetrics, promptNetwork, runtimeAlerts, pagePerformance, pageProvenance, loading, thresholds) { const findings = []; const commandTimes = control.filter((item) => item.phase === "completed" || item.phase === "started").map((item) => Date.parse(item.ts)).filter(Number.isFinite); const routeSessions = new Set(samples.map((item) => item.routeSessionId).filter(Boolean)); @@ -1079,7 +1167,7 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN for (let i = 1; i < samples.length; i += 1) { const prev = digestSample(samples[i - 1]); const next = digestSample(samples[i]); - if (prev !== next && !commandedPromptSeqs.has(samples[i]?.seq) && !nearCommand(samples[i], commandTimes, 10000)) uncommandedChanges.push(ref(samples[i])); + if (prev !== next && !commandedPromptSeqs.has(samples[i]?.seq) && !nearCommand(samples[i], commandTimes, thresholds.uncommandedStateChangeCommandWindowMs)) uncommandedChanges.push(ref(samples[i])); } if (uncommandedChanges.length > 0) findings.push({ id: "uncommanded-visible-state-change", severity: "amber", summary: "visible message/trace digest changed without a nearby command", count: uncommandedChanges.length, samples: uncommandedChanges.slice(0, 20) }); const finalFlicker = detectFinalFlicker(samples); @@ -1088,7 +1176,7 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN for (let i = 1; i < samples.length; i += 1) { const prevY = Number(samples[i - 1]?.scroll?.y ?? 0); const nextY = Number(samples[i]?.scroll?.y ?? 0); - if (prevY > 250 && nextY < 40 && !nearCommand(samples[i], commandTimes, 8000)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) }); + if (prevY > thresholds.scrollJumpFromY && nextY < thresholds.scrollJumpToY && !nearCommand(samples[i], commandTimes, thresholds.scrollJumpCommandWindowMs)) scrollJumps.push({ from: ref(samples[i - 1]), to: ref(samples[i]) }); } if (scrollJumps.length > 0) findings.push({ id: "scroll-jump-top", severity: "amber", summary: "scroll position jumped near top without nearby command", count: scrollJumps.length, samples: scrollJumps.slice(0, 10) }); const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => isTerminalTraceText((row.status || "") + " " + (row.textPreview || "")))); @@ -1107,8 +1195,10 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN if ((runtimeAlerts?.summary?.domDiagnosticSampleCount ?? 0) > 0) findings.push({ id: "runtime-dom-diagnostics", severity: "amber", summary: "diagnostic/error/warning-like text was visible in sampled DOM", count: runtimeAlerts.summary.domDiagnosticSampleCount, samples: runtimeAlerts.domDiagnostics.slice(0, 12) }); if ((runtimeAlerts?.summary?.executionErrorCount ?? 0) > 0) findings.push({ id: "runtime-execution-errors", severity: "red", summary: "Workbench rendered execution failure/error rows during observation", count: runtimeAlerts.summary.executionErrorCount, groups: runtimeAlerts.runtimeExecutionErrorsByCode.slice(0, 12) }); if ((runtimeAlerts?.summary?.consoleAlertCount ?? 0) > 0) findings.push({ id: "runtime-console-alerts", severity: "amber", summary: "browser console warning/error entries were captured during observation", count: runtimeAlerts.summary.consoleAlertCount, groups: runtimeAlerts.consoleAlertsByPath.slice(0, 12) }); - const slowApi = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0) : []; - if (slowApi.length > 0) findings.push({ id: "page-performance-slow-same-origin-api", severity: "red", summary: "same-origin API resource timing exceeded 5s usability budget", count: slowApi.length, groups: slowApi.slice(0, 20) }); + const slowApi = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath.filter((item) => item.overBudgetCount > 0) : []; + if (slowApi.length > 0) findings.push({ id: "page-performance-slow-same-origin-api", severity: "red", summary: "same-origin API resource timing exceeded YAML-configured usability budget", budgetMs: thresholds.sameOriginApiBudgetMs, count: slowApi.length, groups: slowApi.slice(0, 20) }); + const slowLoading = Array.isArray(loading?.overBudgetSegments) ? loading.overBudgetSegments : []; + if (slowLoading.length > 0) findings.push({ id: "page-loading-visible-too-long", severity: "red", summary: "loading indicators stayed visible longer than YAML-configured budget", budgetMs: thresholds.loadingVisibleBudgetMs, count: slowLoading.length, groups: slowLoading.slice(0, 20) }); const longLivedStreams = Array.isArray(pagePerformance?.sameOriginApiByPath) ? pagePerformance.sameOriginApiByPath.filter((item) => item.isLongLivedStream) : []; if (longLivedStreams.length > 0) findings.push({ id: "page-performance-long-lived-streams", severity: "info", summary: "same-origin long-lived streams are reported separately; lifetime is not treated as API load latency", count: longLivedStreams.length, groups: longLivedStreams.slice(0, 20) }); if ((pageProvenance?.summary?.segmentCount ?? 0) > 1) findings.push({ id: "page-provenance-segments", severity: "info", summary: "observer crossed page asset provenance segments; interpret runtime findings by segment", segmentCount: pageProvenance.summary.segmentCount, segments: pageProvenance.segments.slice(0, 20) }); @@ -1171,7 +1261,7 @@ function buildPageProvenanceReport(samples, control, manifest) { }; } -function buildPagePerformanceReport(samples, manifest) { +function buildPagePerformanceReport(samples, manifest, thresholds) { const base = manifest?.baseUrl || "http://invalid.local"; const seen = new Set(); const groups = new Map(); @@ -1212,15 +1302,15 @@ function buildPagePerformanceReport(samples, manifest) { group.sampleCount += 1; group.durationsMs.push(durationMs); if (isLongLivedStream) { - if (durationMs > 5000) group.streamLifetimeOverFiveSecondCount += 1; + if (durationMs > thresholds.longLivedStreamLifetimeBudgetMs) group.streamLifetimeOverFiveSecondCount += 1; if (streamOpenMs !== null) { group.streamOpenDurationsMs.push(streamOpenMs); - if (streamOpenMs > 5000) { + if (streamOpenMs > thresholds.longLivedStreamOpenBudgetMs) { group.streamOpenOverFiveSecondCount += 1; group.overFiveSecondCount += 1; } } - } else if (durationMs > 5000) { + } else if (durationMs > thresholds.sameOriginApiBudgetMs) { group.overFiveSecondCount += 1; } group.lastAt = sample.ts ?? null; @@ -1250,9 +1340,13 @@ function buildPagePerformanceReport(samples, manifest) { streamOpenP75Ms: percentile(streamOpenDurations, 75), streamOpenP95Ms: percentile(streamOpenDurations, 95), streamOpenMaxMs: streamOpenDurations.length > 0 ? streamOpenDurations[streamOpenDurations.length - 1] : null, + streamOpenOverBudgetCount: group.streamOpenOverFiveSecondCount, streamOpenOverFiveSecondCount: group.streamOpenOverFiveSecondCount, + streamLifetimeOverBudgetCount: group.streamLifetimeOverFiveSecondCount, streamLifetimeOverFiveSecondCount: group.streamLifetimeOverFiveSecondCount, + overBudgetCount: group.overFiveSecondCount, overFiveSecondCount: group.overFiveSecondCount, + overBudgetRatio: group.sampleCount > 0 ? Number((group.overFiveSecondCount / group.sampleCount).toFixed(3)) : 0, overFiveSecondRatio: group.sampleCount > 0 ? Number((group.overFiveSecondCount / group.sampleCount).toFixed(3)) : 0, firstAt: group.firstAt, lastAt: group.lastAt, @@ -1264,20 +1358,23 @@ function buildPagePerformanceReport(samples, manifest) { valuesRedacted: true }; }).sort((a, b) => (b.overFiveSecondCount - a.overFiveSecondCount) || (Number(b.p95Ms ?? 0) - Number(a.p95Ms ?? 0)) || a.path.localeCompare(b.path)); - const slow = sameOriginApiByPath.filter((item) => item.overFiveSecondCount > 0); + const slow = sameOriginApiByPath.filter((item) => item.overBudgetCount > 0); const longLivedStreams = sameOriginApiByPath.filter((item) => item.isLongLivedStream); const budgetP95Values = sameOriginApiByPath .map((item) => Number(item.isLongLivedStream ? (item.streamOpenP95Ms ?? 0) : (item.p95Ms ?? 0))) .filter((value) => Number.isFinite(value)); return { summary: { - budgetMs: 5000, + budgetMs: thresholds.sameOriginApiBudgetMs, + sameOriginApiBudgetMs: thresholds.sameOriginApiBudgetMs, + longLivedStreamOpenBudgetMs: thresholds.longLivedStreamOpenBudgetMs, + longLivedStreamLifetimeBudgetMs: thresholds.longLivedStreamLifetimeBudgetMs, sameOriginApiPathCount: sameOriginApiByPath.length, sameOriginApiSampleCount: sameOriginApiByPath.reduce((sum, item) => sum + item.sampleCount, 0), longLivedStreamPathCount: longLivedStreams.length, longLivedStreamSampleCount: longLivedStreams.reduce((sum, item) => sum + item.sampleCount, 0), slowPathCount: slow.length, - slowSampleCount: slow.reduce((sum, item) => sum + item.overFiveSecondCount, 0), + slowSampleCount: slow.reduce((sum, item) => sum + item.overBudgetCount, 0), worstP95Ms: budgetP95Values.length > 0 ? Math.max(...budgetP95Values) : null, valuesRedacted: true }, @@ -1286,6 +1383,165 @@ function buildPagePerformanceReport(samples, manifest) { }; } +function buildLoadingReport(samples, thresholds) { + const timeline = []; + const segments = []; + const active = new Map(); + for (const sample of samples) { + const tsMs = Date.parse(String(sample?.ts ?? "")); + const items = Array.isArray(sample?.loading) ? sample.loading : []; + const current = new Map(); + for (const item of items) { + const key = loadingItemKey(item); + if (!current.has(key)) current.set(key, item); + } + for (const [key, segment] of Array.from(active.entries())) { + if (!current.has(key)) { + segments.push(closeLoadingSegment(segment, thresholds)); + active.delete(key); + } + } + for (const [key, item] of current.entries()) { + const owner = loadingOwnerLabel(item); + const preview = limitText(item?.textPreview || item?.text || "", 160); + const segment = active.get(key); + if (!segment) { + active.set(key, { + keyHash: sha256(key), + owner, + preview, + reason: item?.reason ?? null, + firstSeq: sample?.seq ?? null, + lastSeq: sample?.seq ?? null, + firstAt: sample?.ts ?? null, + lastAt: sample?.ts ?? null, + firstTsMs: Number.isFinite(tsMs) ? tsMs : null, + lastTsMs: Number.isFinite(tsMs) ? tsMs : null, + sampleCount: 1, + routeSessionId: sample?.routeSessionId ?? null, + activeSessionId: sample?.activeSessionId ?? null, + valuesRedacted: true + }); + } else { + segment.lastSeq = sample?.seq ?? null; + segment.lastAt = sample?.ts ?? null; + segment.lastTsMs = Number.isFinite(tsMs) ? tsMs : segment.lastTsMs; + segment.sampleCount += 1; + } + } + timeline.push({ + seq: sample?.seq ?? null, + ts: sample?.ts ?? null, + routeSessionId: sample?.routeSessionId ?? null, + activeSessionId: sample?.activeSessionId ?? null, + count: current.size, + owners: Array.from(current.values()).map(loadingOwnerLabel).slice(0, 20) + }); + } + for (const segment of active.values()) segments.push(closeLoadingSegment(segment, thresholds)); + const sortedSegments = segments.sort((a, b) => Number(b.durationMs ?? 0) - Number(a.durationMs ?? 0)); + const overBudgetSegments = sortedSegments.filter((item) => item.overBudget); + const ownerGroups = groupLoadingOwners(sortedSegments); + return { + summary: { + visibleBudgetMs: thresholds.loadingVisibleBudgetMs, + sampleCount: samples.length, + samplesWithLoading: timeline.filter((item) => item.count > 0).length, + maxVisibleCount: timeline.reduce((max, item) => Math.max(max, item.count), 0), + segmentCount: sortedSegments.length, + overBudgetSegmentCount: overBudgetSegments.length, + longestDurationMs: sortedSegments.length > 0 ? sortedSegments[0].durationMs : 0, + ownerGroupCount: ownerGroups.length, + valuesRedacted: true + }, + timeline: timeline.slice(0, 500), + segments: sortedSegments.slice(0, 200), + overBudgetSegments: overBudgetSegments.slice(0, 80), + ownerGroups, + valuesRedacted: true + }; +} + +function closeLoadingSegment(segment, thresholds) { + const durationMs = segment.firstTsMs !== null && segment.lastTsMs !== null ? Math.max(0, segment.lastTsMs - segment.firstTsMs) : null; + return { + keyHash: segment.keyHash, + owner: segment.owner, + preview: segment.preview, + reason: segment.reason, + firstSeq: segment.firstSeq, + lastSeq: segment.lastSeq, + firstAt: segment.firstAt, + lastAt: segment.lastAt, + sampleCount: segment.sampleCount, + durationMs, + budgetMs: thresholds.loadingVisibleBudgetMs, + overBudget: durationMs !== null && durationMs > thresholds.loadingVisibleBudgetMs, + routeSessionId: segment.routeSessionId, + activeSessionId: segment.activeSessionId, + valuesRedacted: true + }; +} + +function loadingItemKey(item) { + const owner = loadingOwnerLabel(item); + const rect = item?.rect || {}; + return [ + owner, + item?.tag ?? "", + item?.testId ?? "", + item?.role ?? "", + item?.ariaBusy ?? "", + item?.textHash ?? sha256(String(item?.textPreview || item?.text || "")), + Math.round(Number(rect.x ?? 0) / 20), + Math.round(Number(rect.y ?? 0) / 20), + Math.round(Number(rect.width ?? 0) / 20), + Math.round(Number(rect.height ?? 0) / 20) + ].join("|"); +} + +function loadingOwnerLabel(item) { + const owner = item?.owner || {}; + const parts = [ + owner.testId ? "testId=" + owner.testId : "", + owner.sessionId ? "session=" + owner.sessionId : "", + owner.messageId ? "message=" + owner.messageId : "", + owner.role ? "role=" + owner.role : "", + owner.tag ? "tag=" + owner.tag : "", + owner.className ? "class=" + String(owner.className).replace(/\s+/g, ".").slice(0, 80) : "" + ].filter(Boolean); + return parts.length > 0 ? parts.join(" ") : "unknown"; +} + +function groupLoadingOwners(segments) { + const groups = new Map(); + for (const segment of segments) { + const key = segment.owner || "unknown"; + const group = groups.get(key) || { + owner: key, + segmentCount: 0, + overBudgetSegmentCount: 0, + sampleCount: 0, + longestDurationMs: 0, + previews: [], + firstAt: segment.firstAt, + lastAt: segment.lastAt, + valuesRedacted: true + }; + group.segmentCount += 1; + if (segment.overBudget) group.overBudgetSegmentCount += 1; + group.sampleCount += Number(segment.sampleCount ?? 0); + group.longestDurationMs = Math.max(group.longestDurationMs, Number(segment.durationMs ?? 0)); + if (segment.preview && !group.previews.includes(segment.preview)) group.previews.push(segment.preview); + group.lastAt = segment.lastAt; + groups.set(key, group); + } + return Array.from(groups.values()) + .map((item) => ({ ...item, previews: item.previews.slice(0, 5) })) + .sort((a, b) => (b.overBudgetSegmentCount - a.overBudgetSegmentCount) || (b.longestDurationMs - a.longestDurationMs) || a.owner.localeCompare(b.owner)) + .slice(0, 80); +} + function classifyApiPerformanceRoute(normalizedPath, entry = {}) { if (normalizedPath === "/v1/workbench/events") return "same-origin-api-stream"; if (String(entry?.initiatorType ?? "").toLowerCase() === "eventsource") return "same-origin-api-stream"; @@ -1835,7 +2091,7 @@ function isFinalResultText(text) { return /已完成第\d+轮|final response|sealed final response|最终结果|已完成[:：]/iu.test(String(text || "")); } -function buildSampleMetrics(samples, control) { +function buildSampleMetrics(samples, control, thresholds) { const promptCommands = control .filter((item) => item.type === "sendPrompt" && item.phase === "completed") .map((item) => ({ ts: item.ts, tsMs: Date.parse(item.ts), commandId: item.commandId ?? null, textHash: item.input?.textHash ?? null, textBytes: item.input?.textBytes ?? null })) @@ -1868,7 +2124,7 @@ function buildSampleMetrics(samples, control) { textDigest: digestSample(sample) }; }); - const turnTiming = buildTurnTimingTable(samples, timeline); + const turnTiming = buildTurnTimingTable(samples, timeline, thresholds); const turnCells = turnTiming.rows.flatMap((row) => Object.values(row.cells || {})); const turnTimingNonMonotonic = Array.isArray(turnTiming.nonMonotonic) ? turnTiming.nonMonotonic : []; const turnTimingRecentUpdateSawtoothJumps = turnTimingNonMonotonic.filter((item) => item.metric === "recentUpdateSeconds" && item.anomaly === "jump"); @@ -1930,7 +2186,7 @@ function buildSampleMetrics(samples, control) { }; } -function buildTurnTimingTable(samples, timeline) { +function buildTurnTimingTable(samples, timeline, thresholds) { const columns = []; const registry = new Map(); const rows = []; @@ -1985,7 +2241,7 @@ function buildTurnTimingTable(samples, timeline) { cells }); } - const timingEvents = detectTurnTimingNonMonotonic(columns, rows); + const timingEvents = detectTurnTimingNonMonotonic(columns, rows, thresholds); return { columns, rows, @@ -1995,7 +2251,7 @@ function buildTurnTimingTable(samples, timeline) { }; } -function detectTurnTimingNonMonotonic(columns, rows) { +function detectTurnTimingNonMonotonic(columns, rows, thresholds) { const anomalies = []; const recentUpdateResets = []; const recentUpdateSteps = []; @@ -2033,7 +2289,9 @@ function detectTurnTimingNonMonotonic(columns, rows) { const elapsedMs = Date.parse(String(row.ts ?? "")) - Date.parse(String(previous.ts ?? "")); const elapsedSeconds = Number.isFinite(elapsedMs) && elapsedMs >= 0 ? elapsedMs / 1000 : null; const increase = current - previous.value; - const allowedIncrease = elapsedSeconds === null ? 3 : Math.max(3, elapsedSeconds + 2); + const allowedIncrease = elapsedSeconds === null + ? thresholds.recentUpdateSawtoothMinAllowedIncreaseSeconds + : Math.max(thresholds.recentUpdateSawtoothMinAllowedIncreaseSeconds, elapsedSeconds + thresholds.recentUpdateSawtoothSampleSlackSeconds); const excessiveIncrease = increase > allowedIncrease ? increase - allowedIncrease : 0; recentUpdateSteps.push({ columnId: column.id, @@ -2336,7 +2594,8 @@ function digestSample(sample) { const messages = Array.isArray(sample.messages) ? sample.messages.map((item) => item.textHash || item.textPreview || "").join("|") : ""; const trace = Array.isArray(sample.traceRows) ? sample.traceRows.map((item) => (item.status || "") + ":" + (item.textHash || item.textPreview || "")).join("|") : ""; const diagnostics = Array.isArray(sample.diagnostics) ? sample.diagnostics.map((item) => (item.className || "") + ":" + (item.textHash || item.textPreview || "")).join("|") : ""; - return sha256((sample.routeSessionId || "") + "|" + (sample.activeSessionId || "") + "|" + messages + "|" + trace + "|" + diagnostics); + const loading = Array.isArray(sample.loading) ? sample.loading.map((item) => (item.owner ? loadingOwnerLabel(item) : "") + ":" + (item.textHash || item.textPreview || "")).join("|") : ""; + return sha256((sample.routeSessionId || "") + "|" + (sample.activeSessionId || "") + "|" + messages + "|" + trace + "|" + diagnostics + "|" + loading); } function nearCommand(sample, commandTimes, windowMs) { @@ -2436,8 +2695,14 @@ function renderMarkdown(report) { const findingLines = report.findings.length === 0 ? "- 无红灯项。" : report.findings.map((item) => "- " + item.severity + ": " + item.id + " - " + item.summary).join("\n"); const commandLines = report.commandTimeline.length === 0 ? "- 无控制命令。" : report.commandTimeline.map((item) => "- " + item.ts + " " + item.phase + " " + item.type + " " + item.commandId + " " + (item.afterUrl || "")).join("\n"); const transitionLines = report.transitions.length === 0 ? "- 无状态变化。" : report.transitions.slice(0, 80).map((item) => "- #" + item.seq + " " + item.ts + " messages=" + item.messageCount + " traceRows=" + item.traceRowCount + " route=" + (item.routeSessionId || "-") + " active=" + (item.activeSessionId || "-")).join("\n"); + const thresholds = report.analysisThresholds || {}; const metricSummary = report.sampleMetrics?.summary || {}; const alertSummary = report.runtimeAlerts?.summary || {}; + const loadingSummary = report.loading?.summary || {}; + const thresholdLines = Object.entries(thresholds) + .filter(([key]) => key !== "valuesRedacted") + .map(([key, value]) => "- " + key + ": " + value) + .join("\n") || "- 未记录阈值。"; const httpAlertLines = Array.isArray(report.runtimeAlerts?.networkHttpErrorsByPath) && report.runtimeAlerts.networkHttpErrorsByPath.length > 0 ? report.runtimeAlerts.networkHttpErrorsByPath.slice(0, 40).map((item) => "- HTTP " + (item.status ?? "-") + " " + item.method + " " + item.urlPath + " count=" + item.count + " prompts=" + (item.promptIndexes?.join(",") || "-") + " first=" + (item.firstAt || "-") + " last=" + (item.lastAt || "-")).join("\n") : "- 无 HTTP 错误。"; @@ -2463,8 +2728,14 @@ function renderMarkdown(report) { ? report.pageProvenance.segments.slice(0, 40).map((item) => "- fingerprint=" + (item.assetFingerprint || "-") + " samples=" + item.sampleCount + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " scripts=" + (item.scriptCount ?? "-") + " styles=" + (item.stylesheetCount ?? "-") + " urlPaths=" + (Array.isArray(item.urlPaths) ? item.urlPaths.slice(0, 4).join(",") : "-")).join("\n") : "- 无页面 provenance segment。"; const performanceLines = Array.isArray(report.pagePerformance?.sameOriginApiByPath) && report.pagePerformance.sameOriginApiByPath.length > 0 - ? report.pagePerformance.sameOriginApiByPath.slice(0, 80).map((item) => "- " + item.path + " kind=" + (item.routeKind || "same-origin-api") + " budgetMetric=" + (item.budgetMetric || "durationMs") + " samples=" + item.sampleCount + " p50=" + (item.p50Ms ?? "-") + "ms p75=" + (item.p75Ms ?? "-") + "ms p95=" + (item.p95Ms ?? "-") + "ms max=" + (item.maxMs ?? "-") + "ms >5s=" + (item.overFiveSecondCount ?? 0) + " streamOpenP95=" + (item.streamOpenP95Ms ?? "-") + "ms streamLifetime>5s=" + (item.streamLifetimeOverFiveSecondCount ?? 0) + " window=" + (item.firstAt || "-") + ".." + (item.lastAt || "-")).join("\n") + ? report.pagePerformance.sameOriginApiByPath.slice(0, 80).map((item) => "- " + item.path + " kind=" + (item.routeKind || "same-origin-api") + " budgetMetric=" + (item.budgetMetric || "durationMs") + " samples=" + item.sampleCount + " p50=" + (item.p50Ms ?? "-") + "ms p75=" + (item.p75Ms ?? "-") + "ms p95=" + (item.p95Ms ?? "-") + "ms max=" + (item.maxMs ?? "-") + "ms overBudget=" + (item.overBudgetCount ?? item.overFiveSecondCount ?? 0) + " streamOpenP95=" + (item.streamOpenP95Ms ?? "-") + "ms streamLifetimeOverBudget=" + (item.streamLifetimeOverFiveSecondCount ?? 0) + " window=" + (item.firstAt || "-") + ".." + (item.lastAt || "-")).join("\n") : "- 无同源 API Resource Timing 样本。"; + const loadingLines = Array.isArray(report.loading?.segments) && report.loading.segments.length > 0 + ? report.loading.segments.slice(0, 80).map((item) => "- owner=" + escapeMarkdownCell(item.owner || "-") + " samples=" + item.sampleCount + " durationMs=" + (item.durationMs ?? "-") + " overBudget=" + String(item.overBudget) + " seq=" + (item.firstSeq ?? "-") + ".." + (item.lastSeq ?? "-") + " ts=" + (item.firstAt || "-") + ".." + (item.lastAt || "-") + " preview=" + escapeMarkdownCell(item.preview || "")).join("\n") + : "- 无 loading 可见段。"; + const loadingGroupLines = Array.isArray(report.loading?.ownerGroups) && report.loading.ownerGroups.length > 0 + ? report.loading.ownerGroups.slice(0, 80).map((item) => "- owner=" + escapeMarkdownCell(item.owner || "-") + " segments=" + item.segmentCount + " overBudget=" + item.overBudgetSegmentCount + " samples=" + item.sampleCount + " longestMs=" + item.longestDurationMs + " previews=" + escapeMarkdownCell(Array.isArray(item.previews) ? item.previews.join(" / ") : "")).join("\n") + : "- 无 loading owner 分组。"; const metricLines = Array.isArray(report.sampleMetrics?.timeline) && report.sampleMetrics.timeline.length > 0 ? report.sampleMetrics.timeline.slice(0, 120).map((item) => "- #" + item.seq + " " + item.ts + " prompt=" + item.promptIndex + " totalElapsedSeconds=" + (item.totalElapsedSeconds ?? "-") + " recentUpdateSeconds=" + (item.recentUpdateSeconds ?? "-") + " terminal=" + item.terminalSeen + " finalText=" + item.finalResultTextSeen + " diagnostic=" + item.diagnosticSeen).join("\n") : "- 无采样指标。"; @@ -2477,6 +2748,7 @@ function renderMarkdown(report) { + "- network: " + report.counts.network + "\n" + "- console: " + (report.counts.console ?? 0) + "\n" + "- errors: " + report.counts.errors + "\n\n" + + "## Analysis thresholds\n\n" + thresholdLines + "\n\n" + "## Findings\n\n" + findingLines + "\n\n" + "## Sample metrics\n\n" + "- sampleCount: " + (metricSummary.sampleCount ?? 0) + "\n" @@ -2500,7 +2772,9 @@ function renderMarkdown(report) { + "- controlSegmentCount: " + (report.pageProvenance?.summary?.controlSegmentCount ?? 0) + "\n\n" + provenanceLines + "\n\n" + "### Page performance: same-origin API Resource Timing\n\n" - + "- budgetMs: " + (report.pagePerformance?.summary?.budgetMs ?? 5000) + "\n" + + "- budgetMs: " + (report.pagePerformance?.summary?.budgetMs ?? thresholds.sameOriginApiBudgetMs ?? "-") + "\n" + + "- longLivedStreamOpenBudgetMs: " + (report.pagePerformance?.summary?.longLivedStreamOpenBudgetMs ?? thresholds.longLivedStreamOpenBudgetMs ?? "-") + "\n" + + "- longLivedStreamLifetimeBudgetMs: " + (report.pagePerformance?.summary?.longLivedStreamLifetimeBudgetMs ?? thresholds.longLivedStreamLifetimeBudgetMs ?? "-") + "\n" + "- sameOriginApiPathCount: " + (report.pagePerformance?.summary?.sameOriginApiPathCount ?? 0) + "\n" + "- sameOriginApiSampleCount: " + (report.pagePerformance?.summary?.sameOriginApiSampleCount ?? 0) + "\n" + "- longLivedStreamPathCount: " + (report.pagePerformance?.summary?.longLivedStreamPathCount ?? 0) + "\n" @@ -2509,6 +2783,15 @@ function renderMarkdown(report) { + "- slowSampleCount: " + (report.pagePerformance?.summary?.slowSampleCount ?? 0) + "\n" + "- worstP95Ms: " + (report.pagePerformance?.summary?.worstP95Ms ?? "-") + "\n\n" + performanceLines + "\n\n" + + "### Loading indicators\n\n" + + "- visibleBudgetMs: " + (loadingSummary.visibleBudgetMs ?? thresholds.loadingVisibleBudgetMs ?? "-") + "\n" + + "- samplesWithLoading: " + (loadingSummary.samplesWithLoading ?? 0) + "\n" + + "- maxVisibleCount: " + (loadingSummary.maxVisibleCount ?? 0) + "\n" + + "- segmentCount: " + (loadingSummary.segmentCount ?? 0) + "\n" + + "- overBudgetSegmentCount: " + (loadingSummary.overBudgetSegmentCount ?? 0) + "\n" + + "- longestDurationMs: " + (loadingSummary.longestDurationMs ?? 0) + "\n\n" + + "#### Loading owner groups\n\n" + loadingGroupLines + "\n\n" + + "#### Loading segments\n\n" + loadingLines + "\n\n" + "### Prompt network\n\n" + promptNetworkLines + "\n\n" + "### Runtime alerts\n\n" + "- httpErrorCount: " + (alertSummary.httpErrorCount ?? 0) + "\n" From 17087adcbd49fee9c67fc71a6834c47bdf0089d7 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Jun 2026 06:14:09 +0000 Subject: [PATCH 6/6] docs: record issue 973 p1 recovery --- AGENTS.md | 1 + docs/issue/issue-973-p1-reflog-recovery.md | 39 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 docs/issue/issue-973-p1-reflog-recovery.md diff --git a/AGENTS.md b/AGENTS.md index 8ba7cb43..59d424c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台。本文 ## P0: 工作区、语义合并与并行修改 - P0: 本机可能有多个并行开发任务；发现未预期本地修改时默认保留，不得自动 `git reset`、`git checkout --`、删除或回滚他人修改。 +- P0: 当前目录的主工作区（非 `.worktree/` 路径）必须固定停留在 `master` 分支；禁止把主工作区切到其他分支，非 `master` 开发必须使用独立 `.worktree/`。 - P0: 固定主 repo 通常只做 source-truth 预检、fetch、status 和 worktree anchor；源码、配置、部署脚本、issue closeout 和高风险 dad-dev 工作必须使用独立 `.worktree/`。文档/skill/长期参考轻量修改可直接在当前主 worktree 处理。细则见 `docs/reference/devops-hygiene.md`。 - P0: 清理任务 worktree 前必须按语义确认已合入 `master`：工作区 clean，相关提交是 `origin/master` 祖先或被 `--cherry-pick` 判定为等价吸收，必要时检查关键文件 diff/PR merge commit；不得仅因分支落后就删除。仍有未合入语义、未提交文件或不确定归属时，先合并/提交/推送到 `master` 或记录阻塞，不清理 worktree。 - P0: UniDesk 工具链自身修复必须先合并到 `master`，再把实际执行工具命令的 checkout 更新到包含该 merge 的最新 `master` 后继续使用；若当前 checkout 有并行脏改或非 master 分支导致不能安全 fast-forward，禁止 reset/stash/checkout 清理，改用干净 `origin/master` worktree 运行工具并说明原因。 diff --git a/docs/issue/issue-973-p1-reflog-recovery.md b/docs/issue/issue-973-p1-reflog-recovery.md new file mode 100644 index 00000000..6a7ecba4 --- /dev/null +++ b/docs/issue/issue-973-p1-reflog-recovery.md @@ -0,0 +1,39 @@ +# Issue 973 P1 Reflog 恢复证据 + +目标合并分支: UniDesk master + +## 范围 + +本文记录 [#973](https://github.com/pikasTech/unidesk/issues/973) 的 P1 恢复证据。恢复对象是本地分支 `fix/hwlab-tools-image-bun-yaml-2026`，该分支的 reflog-only 提交在恢复前仍会被以下命令列出： + +```bash +git cherry -v HEAD fix/hwlab-tools-image-bun-yaml-2026 +``` + +本恢复 PR 之前，该命令仍会显示未处理的 `+` 提交。 + +## 恢复决策 + +本 PR 使用 `-s ours` 合并旧分支以保留 ancestry，但不恢复过时的单体实现。直接 cherry-pick `06bffaf` 时会与当前拆分模块冲突（`scripts/src/gh/*`、`scripts/src/hwlab-node/*`、`scripts/src/platform-infra-observability/*`），继续套用会把已经被替代的单文件实现重新引入仓库。 + +当前 `master` 已在现有模块边界中保留这些行为： + +| 旧提交 | 恢复状态 | 当前语义落点 | +|---|---|---| +| `0817e3d` | 语义已保留 | `config/hwlab-node-lanes.yaml` 已包含 `webProbe.alertThresholds`；`scripts/src/hwlab-node-lanes.ts` 解析 typed thresholds；`scripts/src/hwlab-node/web-probe-observe.ts` 注入 `UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON`；analyzer/runner 消费 YAML 阈值。 | +| `1789700` | 语义已保留 | `scripts/src/hwlab-node/web-probe-observe.ts` 已暴露 `webProbeCommandTimeoutSummary`、async start/status fallback、`degradedReason`、report-file 读取失败和 analyzer timeout 恢复元数据。 | +| `a08dfea` | 语义已保留 | `scripts/src/platform-infra-observability/manifest.ts` 和 `render.ts` 已输出 trace identity、key spans、resource timing 列、business trace id 和 drill-down 命令。 | +| `06bffaf` | 语义已保留 | `config/unidesk-cli.yaml` 与 `scripts/src/output.ts` 已提供全局 JSON/text dump 保护；GitHub CLI 已拆到 `scripts/src/gh/*`，默认输出为 bounded table 和渐进披露。 | +| `da3efa4` | patch-id 已等价 | long-lived stream 已与慢 API lifetime 分开分类，同时保留 stream-open budget finding。 | + +## 验证 + +本分支完成 ancestry merge 后，reflog 分支不再报告未处理的 `+` 提交： + +```bash +git cherry -v HEAD fix/hwlab-tools-image-bun-yaml-2026 +``` + +输出为空。 + +`AGENTS.md` 同步补充了“主 worktree 必须固定停留在 `master`”的 P0 规则，用于满足 [#973](https://github.com/pikasTech/unidesk/issues/973) 验收 A2。