fix: rotate web-probe jsonl on start (#702)

Co-authored-by: Codex <codex@noreply.local>
2026-06-22 23:22:11 +08:00
parent 8aca736294
commit e49764796e
3 changed files with 80 additions and 10 deletions
@@ -97,7 +97,7 @@ bun scripts/cli.ts hwlab nodes web-probe observe analyze webobs-xxxx
 - `web-probe observe start` 默认是被动观测：记录 DOM 摘要、自然页面 request/response/requestfailed、截图和 performance 样本，不主动 fetch Workbench API、不切换 control session、不拦截路由、不调用 repair helper。长程 Workbench 观测必须保留 control/observer 双页面模型：control 页面执行显式 command，observer 页面只同步到同一 session URL 后被动采样，并按默认 180000ms 周期整页刷新同一 session 来模拟用户往返；周期刷新只作用于 observer，不得改变 control active session 或作为通过条件。两页的 `pageRole`、`pageId`、`sampleGroupSeq` 必须进入样本和 analyzer 报表。任何 `newSession`、`selectProvider`、`sendPrompt`、`goto`、`screenshot`、`mark`、`stop` 都必须通过 `observe command` 显式下发，并进入 `control.jsonl`。
 - `web-probe observe` 的 issue evidence 优先记录 observer id、stateDir、report JSON/Markdown SHA、samples/control/network/artifact 计数、routeSessionId、activeSessionId、prompt hash/textBytes、traceId、AgentRun runId/commandId、最终 status 和必要摘要；不要把 prompt 原文、assistant 大段正文、完整 stdout/stderr 或 provider payload 粘贴到 issue。
 - 多轮 Workbench 采样必须证明同一个 `sessionId` 连续承载所有轮次；每轮至少记录 prompt hash、traceId、终态、最终回答摘要和性能/产物表。若 Web UI 投影卡住但 Code Agent/AgentRun result 已 terminal，应同时登记“执行终态”和“Workbench 投影未收敛”，不得用 `goto`、reload、切 session 或 result polling 把 UI 失败伪装成通过。
- `observe analyze` 是离线分析，只读取 artifact JSONL 并写 `analysis/report.md` 与 `analysis/report.json`，不访问 Workbench API、不驱动浏览器。报告必须输出采样点 vs 每个 turn 的总耗时/最近更新时间表、可见“加载中”的数量/归属/并发 owner/连续出现区间、DOM diagnostic/HTTP/console/requestfailed/runtime execution error 分组、page asset provenance segment、同源 API Resource Timing 分位表和超过 5s 的慢路径 finding；页面/API 加载超过 5s 视为不可用级性能红线，可见“加载中”持续超过 5s 也必须作为真实慢加载证据登记到上游问题。修复必须降低真实请求、投影、渲染或后端路径耗时，禁止为了减少“加载中”出现时间而提前展示未加载完的内容，也不能靠下游 retry/reload/fallback 掩盖。报告里的 `final-response-flicker`、`uncommanded-visible-state-change`、session changed、network 503 等 finding 是排障线索；用于 closeout 时必须结合原始 session/trace/DOM 证据解释，避免把采样噪声直接当作业务结论。
+- `observe analyze` 是离线分析，只读取 artifact JSONL 并写 `analysis/report.md` 与 `analysis/report.json`，不访问 Workbench API、不驱动浏览器。`observe start` 每次启动必须先把同一 stateDir 中已有的根目录 JSONL 轮转到带时间戳的 `archive/` 文件；`observe analyze` 默认只分析当前根目录 JSONL，不扫描历史 archive，只有显式指定 archive prefix 时才分析历史轮转窗口。报告必须输出采样点 vs 每个 turn 的总耗时/最近更新时间表、可见“加载中”的数量/归属/并发 owner/连续出现区间、DOM diagnostic/HTTP/console/requestfailed/runtime execution error 分组、page asset provenance segment、同源 API Resource Timing 分位表和超过 5s 的慢路径 finding；页面/API 加载超过 5s 视为不可用级性能红线，可见“加载中”持续超过 5s 也必须作为真实慢加载证据登记到上游问题。修复必须降低真实请求、投影、渲染或后端路径耗时，禁止为了减少“加载中”出现时间而提前展示未加载完的内容，也不能靠下游 retry/reload/fallback 掩盖。报告里的 `final-response-flicker`、`uncommanded-visible-state-change`、session changed、network 503 等 finding 是排障线索；用于 closeout 时必须结合原始 session/trace/DOM 证据解释，避免把采样噪声直接当作业务结论。
 - 自定义 `web-probe script` 仍运行在 UniDesk `trans` 60s 最外层短连接约束内；能在一轮内完成的 P4 验收优先把 `--command-timeout-seconds` 控制在 55 秒以内，并减少无界 selector/network 等待。确需等待更久时，改用 `web-probe run` 的异步 job/status 语义，或把动作拆成“提交/采样/截图/状态读取”多次短 probe。若输出出现 `UNIDESK_SSH_RUNTIME_TIMEOUT` 但同时恢复了 `reportPath`、`reportSha256`、screenshots 或 DOM steps，先按远端报告判断脚本/页面实际状态；最终关闭证据仍优先用一次未触发短连接超时的 bounded rerun。
 - issue closeout 优先引用 `web-probe script` 输出的顶层 `issueEvidence` 或 `summary.issueEvidence`；只有需要展开调查时才粘贴 `probe.script.result`、`probe.steps` 或完整 `reportPath`，避免 stdout、summary 和 report 多层重复同一证据。
 - stdin heredoc 与 `--script-file` 都按 ES module 加载，脚本必须导出 `export default async ({ page, gotoStable, recordStep, ... }) => { ... }`；不要在模块顶层直接写 `return`。失败为 `Illegal return statement`、`does not provide an export named default` 或 finalUrl 仍是 `about:blank` 且 stepCount=0 时，先按 probe 脚本入口误用处理，不要归因成 Cloud Web 行为失败。
@@ -78,6 +78,7 @@ interface NodeWebProbeObserveOptions {
  tailLines: number;
  maxFiles: number;
  collectFile: string | null;
+  analyzeArchivePrefix: string | null;
  full: boolean;
  stateDir: string | null;
  jobId: string | null;
@@ -6905,6 +6906,7 @@ function parseNodeWebProbeObserveOptions(
    "--tail-lines",
    "--max-files",
    "--file",
+    "--archive-prefix",
    "--state-dir",
    "--job-id",
    "--type",
@@ -6922,6 +6924,8 @@ function parseNodeWebProbeObserveOptions(
  if (jobId !== null && !isSafeWebObserveJobId(jobId)) throw new Error(`unsafe web-probe observe --job-id: ${jobId}`);
  const collectFile = optionValue(args, "--file") ?? null;
  if (collectFile !== null && !isSafeWebObserveCollectFile(collectFile)) throw new Error(`unsafe web-probe observe --file: ${collectFile}`);
+  const analyzeArchivePrefix = optionValue(args, "--archive-prefix") ?? null;
+  if (analyzeArchivePrefix !== null && !isSafeWebObserveArchivePrefix(analyzeArchivePrefix)) throw new Error(`unsafe web-probe observe --archive-prefix: ${analyzeArchivePrefix}`);
  if (observeActionRaw !== "start" && stateDir === null && jobId === null) {
    throw new Error("web-probe observe status|command|stop|collect|analyze requires --state-dir or --job-id");
  }
@@ -6942,6 +6946,7 @@ function parseNodeWebProbeObserveOptions(
    tailLines: positiveIntegerOption(args, "--tail-lines", 5, 200),
    maxFiles: positiveIntegerOption(args, "--max-files", 80, 5000),
    collectFile,
+    analyzeArchivePrefix,
    full: args.includes("--full"),
    stateDir,
    jobId,
@@ -7396,7 +7401,8 @@ function runNodeWebProbeObserveStart(
  const jobId = `webobs-${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
  const timestamp = new Date().toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
  const day = timestamp.slice(0, 8);
-  const stateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
+  const defaultStateDir = `.state/web-observe/${safeWebObserveSegment(options.node)}/${safeWebObserveSegment(options.lane)}/${day.slice(0, 4)}/${day.slice(4, 6)}/${day.slice(6, 8)}/${timestamp}_${safeWebObserveTargetSegment(options.targetPath)}_${jobId}`;
+  const stateDir = options.stateDir ?? defaultStateDir;
  const runnerB64 = Buffer.from(nodeWebObserveRunnerSource(), "utf8").toString("base64");
  const webProbeProxy = nodeWebProbeHostProxyEnv(spec);
  const script = [
@@ -7943,7 +7949,8 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
    "analysis_stdout=\"$state_dir/analysis/analyzer-stdout.json\"",
    "analysis_stderr=\"$state_dir/analysis/analyzer-stderr.log\"",
    "set +e",
-    "UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
+    `UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=${shellQuote(options.analyzeArchivePrefix ?? "")}`,
+    "UNIDESK_WEB_OBSERVE_STATE_DIR=\"$state_dir\" UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX=\"$UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX\" node \"$analyzer\" >\"$analysis_stdout\" 2>\"$analysis_stderr\"",
    "analyzer_exit=$?",
    "set -e",
    "report_json=\"$state_dir/analysis/report.json\"",
@@ -8030,6 +8037,7 @@ function runNodeWebProbeObserveAnalyze(options: NodeWebProbeObserveOptions, spec
    "const compact = source ? {",
    "  ok: analyzerExit === 0,",
    "  counts: source.counts ?? null,",
+    "  jsonlScope: objectOrNull(source.jsonlScope),",
    "  analysisWindow: objectOrNull(source.analysisWindow),",
    "  archiveSummary: objectOrNull(source.archiveSummary),",
    "  sampleMetrics: metrics,",
@@ -8203,6 +8211,7 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
  const analyzer = record(analysis?.analyzer);
  const result = record(value.result);
  const counts = record(analysis?.counts);
+  const jsonlScope = record(analysis?.jsonlScope);
  const analysisWindow = record(analysis?.analysisWindow);
  const archiveSummary = record(analysis?.archiveSummary);
  const sampleMetrics = record(analysis?.sampleMetrics);
@@ -8335,10 +8344,11 @@ function renderWebObserveAnalyzeTable(value: Record<string, unknown>): string {
      ]]),
      "",
    ] : []),
-    webObserveTable(["ID", "NODE", "LANE", "SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS"], [[
+    webObserveTable(["ID", "NODE", "LANE", "SCOPE", "SAMPLES", "CONTROL", "NETWORK", "CONSOLE", "ARTIFACTS"], [[
      webObserveText(value.id),
      webObserveText(value.node),
      webObserveText(value.lane),
+      webObserveShort([webObserveText(jsonlScope?.mode ?? "current"), jsonlScope?.archivePrefix ? webObserveText(jsonlScope.archivePrefix) : ""].filter((part) => part && part !== "-").join(":"), 36),
      webObserveText(counts?.samples),
      webObserveText(counts?.control),
      webObserveText(counts?.network),
@@ -9356,6 +9366,14 @@ function isSafeWebObserveCollectFile(value: string): boolean {
    && /^[A-Za-z0-9_.\/-]+$/u.test(value);
 }

+function isSafeWebObserveArchivePrefix(value: string): boolean {
+  return value.length > 0
+    && value.length <= 120
+    && !value.includes("\0")
+    && !value.includes("..")
+    && /^[A-Za-z0-9_.-]+$/u.test(value);
+}
+
 function isSafeWebObserveJobId(value: string): boolean {
  return /^webobs-[A-Za-z0-9_.-]+$/u.test(value);
 }
@@ -33,6 +33,7 @@ const dirs = {
  commandsFailed: path.join(stateDir, "commands", "failed"),
  screenshots: path.join(stateDir, "screenshots"),
  analysis: path.join(stateDir, "analysis"),
+  archive: path.join(stateDir, "archive"),
 };
 const files = {
  manifest: path.join(stateDir, "manifest.json"),
@@ -60,12 +61,15 @@ let lastScreenshotAtMs = 0;
 let auth = null;
 let pageLoadSeq = 0;
 let currentPageProvenance = null;
+const jsonlRotation = { stamp: compactFileTimestamp(startedAt), files: [] };

 try {
  if (!password) throw new Error("missing HWLAB_WEB_PASS");
  await prepareDirs();
+  await rotateExistingJsonlArtifacts();
  await writeManifest({ status: "starting" });
  await writeHeartbeat({ status: "starting" });
+  if (jsonlRotation.files.length > 0) await appendJsonl(files.control, eventRecord("jsonl-rotated", { stamp: jsonlRotation.stamp, archiveDir: path.relative(stateDir, dirs.archive), files: jsonlRotation.files, valuesRedacted: true }));
  const launcher = await import(pathToFileURL(path.resolve("scripts/src/browser-launcher.mjs")).href);
  const { chromium } = await launcher.importPlaywright();
  browser = await launcher.launchChromium(chromium, chromiumLaunchOptions);
@@ -114,6 +118,41 @@ async function prepareDirs() {
  await Promise.all(Object.values(dirs).map((dir) => mkdir(dir, { recursive: true, mode: 0o700 })));
 }

+async function rotateExistingJsonlArtifacts() {
+  for (const [key, file] of Object.entries(files)) {
+    if (!file.endsWith(".jsonl")) continue;
+    let meta = null;
+    try {
+      meta = await stat(file);
+    } catch (error) {
+      if (error?.code === "ENOENT") continue;
+      throw error;
+    }
+    if (!meta?.isFile()) continue;
+    const archiveFile = await uniqueArchiveFile(jsonlRotation.stamp + "-" + path.basename(file));
+    await rename(file, archiveFile);
+    jsonlRotation.files.push({ key, from: path.relative(stateDir, file), to: path.relative(stateDir, archiveFile), byteCount: meta.size });
+  }
+}
+
+async function uniqueArchiveFile(name) {
+  let candidate = path.join(dirs.archive, name);
+  const parsed = path.parse(name);
+  for (let index = 1; ; index += 1) {
+    try {
+      await stat(candidate);
+      candidate = path.join(dirs.archive, parsed.name + "-" + index + parsed.ext);
+    } catch (error) {
+      if (error?.code === "ENOENT") return candidate;
+      throw error;
+    }
+  }
+}
+
+function compactFileTimestamp(value) {
+  return new Date(value).toISOString().replace(/[-:]/gu, "").replace(/[.]\d{3}Z$/u, "Z");
+}
+
 async function writeManifest(extra = {}) {
  const manifest = {
    ok: extra.status !== "failed",
@@ -128,6 +167,7 @@ async function writeManifest(extra = {}) {
    pageAuthority: { browser: "chromium", context: "shared-auth", pageMode: "dual-control-observer", controlPageId: pageId, observerPageId, continuityBreaksRecorded: true },
    pageProvenance: compactPageProvenance(currentPageProvenance),
    sampling: { mode: "passive", sampleIntervalMs, screenshotIntervalMs, maxSamples, observerRefreshIntervalMs, observerInitiatedDefault: false, responseBodyReadDefault: false },
+    jsonlRotation,
    commandDirs: dirs,
    artifacts: files,
    safety: { pureClient: true, inboundApi: false, database: false, queueConsumer: false, k8sWorkload: false, valuesRedacted: true, secretValuesPrinted: false },
@@ -1676,16 +1716,19 @@ import path from "node:path";
 import { createInterface } from "node:readline";

 const stateDir = path.resolve(process.env.UNIDESK_WEB_OBSERVE_STATE_DIR || process.argv[2] || ".state/web-observe/manual");
+const archivePrefix = safeArchivePrefix(process.env.UNIDESK_WEB_OBSERVE_ANALYZE_ARCHIVE_PREFIX || "");
+const dataDir = archivePrefix ? path.join(stateDir, "archive") : stateDir;
+const dataFile = (name) => path.join(dataDir, archivePrefix ? archivePrefix + "-" + name : name);
 const analysisDir = path.join(stateDir, "analysis");
 const reportJsonPath = path.join(analysisDir, "report.json");
 const reportMdPath = path.join(analysisDir, "report.md");
 const jsonlReadIssues = [];
-const samples = await readJsonl(path.join(stateDir, "samples.jsonl"), { compact: compactSampleForAnalysis });
-const control = await readJsonl(path.join(stateDir, "control.jsonl"));
-const network = await readJsonl(path.join(stateDir, "network.jsonl"));
-const consoleEvents = await readJsonl(path.join(stateDir, "console.jsonl"));
-const errors = await readJsonl(path.join(stateDir, "errors.jsonl"));
-const artifacts = await readJsonl(path.join(stateDir, "artifacts.jsonl"));
+const samples = await readJsonl(dataFile("samples.jsonl"), { compact: compactSampleForAnalysis });
+const control = await readJsonl(dataFile("control.jsonl"));
+const network = await readJsonl(dataFile("network.jsonl"));
+const consoleEvents = await readJsonl(dataFile("console.jsonl"));
+const errors = await readJsonl(dataFile("errors.jsonl"));
+const artifacts = await readJsonl(dataFile("artifacts.jsonl"));
 const manifest = await readJson(path.join(stateDir, "manifest.json"));
 const heartbeat = await readJson(path.join(stateDir, "heartbeat.json"));

@@ -1745,6 +1788,7 @@ const report = {
  command: "web-probe-observe analyze",
  generatedAt: new Date().toISOString(),
  stateDir,
+  jsonlScope: { mode: archivePrefix ? "archive" : "current", archivePrefix: archivePrefix || null, dataDir, valuesRedacted: true },
  manifest: compactManifest(manifest),
  heartbeat: compactHeartbeat(heartbeat),
  counts: { samples: samples.length, control: control.length, network: network.length, console: consoleEvents.length, errors: errors.length, artifacts: artifacts.length },
@@ -1769,6 +1813,7 @@ console.log(JSON.stringify({
  ok: true,
  command: "web-probe-observe analyze",
  stateDir,
+  jsonlScope: report.jsonlScope,
  reportJsonPath,
  reportJsonSha256: jsonMeta.sha256,
  reportMdPath,
@@ -1909,6 +1954,13 @@ async function readJson(file) {
  try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; }
 }

+function safeArchivePrefix(value) {
+  const text = String(value || "").trim();
+  if (!text) return "";
+  if (!/^[A-Za-z0-9_.-]+$/u.test(text) || text.includes("..")) throw new Error("unsafe archive prefix: " + text);
+  return text;
+}
+
 async function readJsonl(file, options = {}) {
  const rows = [];
  try {