fix: support v03 control-plane retry visibility

2026-06-08 15:48:41 +00:00
parent 8f51bbca91
commit 47dd562697
4 changed files with 161 additions and 14 deletions
@@ -58,7 +58,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P
  `--dry-run` 只报告是否会 pre-sync，不创建 Job；confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径，避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞；`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件，覆盖 `control-plane-refresh`、`git-mirror-pre-sync` 和 `create-pipelinerun`，避免异步 job 长时间只有启动命令而无法判断卡点；默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要，保留长度与 hash，最终 trigger 结果只返回阶段摘要和关键 tail，完整内容通过 job stdout/stderr 文件渐进披露；只有现场同步调试才显式加 `--wait`；旧 `rerun-current` 只作为输入别名保留。PipelineRun `Completed`、Argo `Synced/Healthy` 和 `webAssets.ok=true` 只证明 G14 runtime 已更新；交付收口还必须用 `hwlab g14 git-mirror status` 查看 `cache.summary.pendingFlush`，若为 true，继续执行受控 `hwlab g14 git-mirror flush --confirm` 并用 job status 轮询到 `pendingFlush=false`。
 - `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行；不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`。
 - `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]` 和 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-openfga|hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的标准入口：OpenFGA preset 确保 preshared token、datastore URI 和 Postgres password 存在；v02 继续确保独立 `hwlab_openfga` 数据库/角色存在，v03+ 默认从 lane spec 派生 namespace、Postgres Secret 和主 DB 用户，避免复制 v02 专用硬编码。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`，并同步到对应 lane 的 `*-master-server-admin-api-key/api-key`。`status` 只返回 key 是否存在、解码后字节数、key prefix 和 DB 对象存在性，永远不读取、不打印、不回传 secret 明文。`hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret，默认 dry-run，拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret；共享 device-pod API key 已退出当前授权路径，不再提供 ensure/bootstrap 入口。
- `hwlab g14 control-plane cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口；真实清理只删除已完成 PipelineRun，让 Tekton/local-path 回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。
+- `hwlab g14 control-plane cleanup-runs --lane v02|v03|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口；真实清理只删除已完成 PipelineRun，让 Tekton/local-path 回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。`hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run <name>` 是 v0.3 failed run 受控重试前的清理入口。
 - `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口；只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。
 - `hwlab g14 git-mirror status|apply|sync|flush [--dry-run|--confirm]` 是 `devops-infra` git mirror/relay 的受控维护入口：`apply` 渲染并 server-side apply `devops-infra/git-mirror.yaml`，同时删除遗留 `git-mirror-hwlab-sync` CronJob；`sync` 创建一次性 manual Job，把 GitHub allowlist refs 拉入本地 mirror；`flush` 创建一次性 manual Job，把本地 `v0.2-gitops` 快进推回 GitHub。
  `status` 返回 read/write URL、last sync/write/flush、本地 ref、GitHub staging ref 和 pending flush 状态，并在 `cache.summary` 给出 `localV02`、`localGitops`、`githubGitops`、`pendingFlush`、`flushNeeded`、`githubInSync` 和下一条受控 `flushCommand`。confirmed `sync` 和 `flush` 默认创建 `.state/jobs/` 异步 job 并立刻返回可查询状态，只有现场同步调试才显式加 `--wait`；mirror 不设置 CronJob。
@@ -334,6 +334,7 @@ assertCondition(gitMirrorSummary.v03SourceInSync === true && gitMirrorSummary.v0
 const renderScript = v02ControlPlaneRenderScript("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
 const renderScriptHash = v02ControlPlaneRefreshScriptHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
 const sourceText = await Bun.file(new URL("./src/hwlab-g14.ts", import.meta.url)).text();
+const jobsSourceText = await Bun.file(new URL("./src/jobs.ts", import.meta.url)).text();
 assertCondition(
  renderScript.includes("git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"")
    && renderScript.includes("git -C \"$worktree_dir\" checkout --detach \"$source_commit\"")
@@ -475,6 +476,17 @@ assertCondition(
    && sourceText.includes("protected-latest-pipelinerun"),
  "control-plane cleanup-runs must protect the latest PipelineRun per lane by default",
 );
+assertCondition(
+  hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run"))
+    && hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit"))
+    && sourceText.includes("control-plane cleanup-runs requires --lane v02|v03|g14|all")
+    && sourceText.includes("printRuntimeLaneTriggerProgress")
+    && sourceText.includes("hwlab.runtime-lane.trigger.progress")
+    && jobsSourceText.includes("hwlab-runtime-lane-trigger")
+    && jobsSourceText.includes("hwlab.runtime-lane.trigger.progress")
+    && jobsSourceText.includes("hwlab nodes control-plane status --node"),
+  "v0.3 runtime lane retry and trigger visibility must be represented by controlled cleanup/help/progress paths",
+);

 const staleSuccessAlignment = v02CommitAlignment({
  expectedSourceHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
@@ -361,12 +361,12 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
    actionRaw !== "cleanup-released-pvs" &&
    actionRaw !== "runtime-migration"
  ) {
-    throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
+    throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|v03|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
  }
  const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
  let lane: G14ControlPlaneOptions["lane"];
  if (actionRaw === "cleanup-runs") {
-    if (laneRaw !== "v02" && laneRaw !== "g14" && laneRaw !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
+    if (laneRaw !== "g14" && laneRaw !== "all" && !isHwlabRuntimeLane(laneRaw)) throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all");
    lane = laneRaw;
  } else if (actionRaw === "cleanup-released-pvs") {
    if (laneRaw !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
@@ -918,6 +918,14 @@ function printV02PrMonitorProgress(data: Record<string, unknown> = {}): void {
  printProgressEvent("hwlab.v02.pr-monitor.progress", data);
 }

+function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record<string, unknown> = {}): void {
+  printProgressEvent("hwlab.runtime-lane.trigger.progress", {
+    lane: spec.lane,
+    node: spec.nodeId,
+    ...data,
+  });
+}
+
 function shortSha(sha: string): string {
  return sha.slice(0, 12);
 }
@@ -2560,10 +2568,10 @@ function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, co
  return parsePipelineRunRows(text, limit, nowMs);
 }

-function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] {
-  if (lane === "v02") return ["hwlab-v02-ci-poll-"];
+function pipelinePrefixesForLane(lane: G14ControlPlaneOptions["lane"]): string[] {
+  if (isHwlabRuntimeLane(lane)) return [`${hwlabRuntimeLaneSpec(lane).pipelineRunPrefix}-`];
  if (lane === "g14") return ["hwlab-g14-ci-poll-"];
-  return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"];
+  return [...hwlabRuntimeLaneIds().map((runtimeLane) => `${hwlabRuntimeLaneSpec(runtimeLane).pipelineRunPrefix}-`), "hwlab-g14-ci-poll-"];
 }

 function commandErrorSummary(result: CommandJsonResult): string {
@@ -2579,8 +2587,8 @@ function tailText(value: unknown, maxBytes = 2000): string {
 }

 function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
-  if (options.lane !== "v02" && options.lane !== "g14" && options.lane !== "all") {
-    throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
+  if (options.lane !== "g14" && options.lane !== "all" && !isHwlabRuntimeLane(options.lane)) {
+    throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all");
  }
  const result = g14K3s([
    "kubectl",
@@ -2594,7 +2602,11 @@ function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string
  if (!isCommandSuccess(result)) {
    throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
  }
-  const targetPipelineRun = options.pipelineRun ?? (options.sourceCommit === undefined ? undefined : v02PipelineRunName(options.sourceCommit));
+  const targetPipelineRun = options.pipelineRun ?? (options.sourceCommit === undefined
+    ? undefined
+    : isHwlabRuntimeLane(options.lane)
+      ? runtimeLanePipelineRunName(hwlabRuntimeLaneSpec(options.lane), options.sourceCommit)
+      : v02PipelineRunName(options.sourceCommit));
  const prefixes = pipelinePrefixesForLane(options.lane);
  const now = Date.now();
  const terminalRuns = statusText(result)
@@ -2831,6 +2843,7 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string,
    .filter((item) => item.selected !== false)
    .map((item) => String(item.name));
  const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
+  const followUpStatusLane = isHwlabRuntimeLane(options.lane) ? options.lane : "v02";
  if (options.dryRun) {
    return {
      ok: true,
@@ -2875,7 +2888,7 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string,
    ownedPvcCountBefore: ownedPvcs.length,
    deletion,
    followUp: {
-      status: "bun scripts/cli.ts hwlab g14 control-plane status --lane v02",
+      status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${followUpStatusLane}`,
      diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{.spec.taints}{\"\\n\"}{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
    },
  };
@@ -3859,13 +3872,14 @@ function refreshRuntimeLaneArgoApplication(spec: HwlabRuntimeLaneSpec, options:
 }

 function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record<string, unknown> {
-  if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-runs" || options.action === "cleanup-released-pvs") {
+  if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
+  if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-released-pvs") {
    return {
      ok: false,
      command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`,
      lane: spec.lane,
      degradedReason: "unsupported-runtime-lane-action",
-      message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh`,
+      message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh/cleanup-runs`,
    };
  }
  if (options.action === "status" && options.pipelineRun !== undefined) {
@@ -4000,6 +4014,12 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
      next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
    };
  }
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "git-mirror-sync",
+    status: "started",
+    sourceCommit,
+    pipelineRun,
+  });
  const gitMirrorSync = runGitMirrorSync({
    action: "sync",
    lane: spec.lane,
@@ -4008,6 +4028,14 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
    wait: true,
    timeoutSeconds: options.timeoutSeconds,
  });
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "git-mirror-sync",
+    status: gitMirrorSync.ok === true ? "succeeded" : "failed",
+    sourceCommit,
+    pipelineRun,
+    jobName: gitMirrorSync.jobName ?? null,
+    elapsedMs: gitMirrorSync.elapsedMs ?? null,
+  });
  if (gitMirrorSync.ok !== true) {
    return {
      ok: false,
@@ -4021,7 +4049,21 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
      degradedReason: "git-mirror-sync-before-trigger-failed",
    };
  }
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "source-render",
+    status: "started",
+    sourceCommit,
+    pipelineRun,
+  });
  const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "source-render",
+    status: isCommandSuccess(render.result) ? "succeeded" : "failed",
+    sourceCommit,
+    pipelineRun,
+    renderDir: render.renderDir,
+    exitCode: render.result.exitCode,
+  });
  if (!isCommandSuccess(render.result)) {
    return {
      ok: false,
@@ -4038,8 +4080,22 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
      degradedReason: "control-plane-render-failed",
    };
  }
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "control-plane-apply",
+    status: "started",
+    sourceCommit,
+    pipelineRun,
+    renderDir: render.renderDir,
+  });
  const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, false, options.timeoutSeconds);
  const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "control-plane-apply",
+    status: isCommandSuccess(apply) ? "succeeded" : "failed",
+    sourceCommit,
+    pipelineRun,
+    exitCode: apply.exitCode,
+  });
  if (!isCommandSuccess(apply)) {
    return {
      ok: false,
@@ -4058,7 +4114,20 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont
      degradedReason: "control-plane-apply-before-trigger-failed",
    };
  }
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "create-pipelinerun",
+    status: "started",
+    sourceCommit,
+    pipelineRun,
+  });
  const create = createRuntimeLanePipelineRun(spec, sourceCommit, options.timeoutSeconds);
+  printRuntimeLaneTriggerProgress(spec, {
+    stage: "create-pipelinerun",
+    status: isCommandSuccess(create) ? "succeeded" : "failed",
+    sourceCommit,
+    pipelineRun,
+    exitCode: create.exitCode,
+  });
  return {
    ok: isCommandSuccess(create),
    command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
@@ -8682,6 +8751,8 @@ export function hwlabG14Help(): Record<string, unknown> {
      "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
      "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll-<short-sha> --dry-run",
      "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --source-commit <full-sha> --confirm",
+      "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run hwlab-v03-ci-poll-<short-sha> --dry-run",
+      "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit <full-sha> --confirm",
      "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
      "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
      "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run",
@@ -25,7 +25,7 @@ export interface JobRecord {
 }

 export interface JobProgressSummary {
-  kind: "hwlab-v02-trigger" | "git-mirror" | "generic";
+  kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "git-mirror" | "generic";
  stage: string | null;
  stageStatus: string | null;
  sourceCommit: string | null;
@@ -225,13 +225,15 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
 function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
  const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
  const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
+  const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
  const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
-  if (!knownWorkflow && !v02PrMonitorWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
+  if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
  const nowMs = Date.now();
  const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
  const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
  const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
  if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs);
+  if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
  const events = v02PrMonitorWorkflow
    ? [
      ...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"),
@@ -355,6 +357,68 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder
  };
 }

+function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
+  const events = parseJsonLineEvents(stderrTail, "hwlab.runtime-lane.trigger.progress")
+    .sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
+  const lastEvent = events.at(-1) ?? {};
+  const stage = stringField(lastEvent.stage);
+  const stageStatus = stringField(lastEvent.status);
+  const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
+  const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
+  const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail)
+    ? true
+    : stage === "create-pipelinerun" && stageStatus === "failed"
+      ? false
+      : null;
+  const lastEventAt = stringField(lastEvent.at);
+  const elapsedSeconds = jobElapsedSeconds(job, nowMs);
+  const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
+  const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
+  const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`);
+  const warnings = jobProgressWarnings({
+    job,
+    eventsObserved: events.length,
+    elapsedSeconds,
+    stage,
+    stageStatus,
+    stageElapsedSeconds,
+    lastEventAgeSeconds,
+  });
+  const slow = warnings.length > 0;
+  return {
+    kind: "hwlab-runtime-lane-trigger",
+    stage,
+    stageStatus,
+    sourceCommit,
+    pipelineRun,
+    pipelineCreated,
+    elapsedSeconds,
+    stageElapsedSeconds,
+    lastEventAt,
+    lastEventAgeSeconds,
+    eventsObserved: events.length,
+    slow,
+    warnings,
+    timings,
+    summary: [
+      job.status,
+      stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
+      sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
+      pipelineRun ? `pipelineRun=${pipelineRun}` : null,
+      pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
+      elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
+      stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
+      lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
+      slow ? "visibility-warning" : null,
+    ].filter(Boolean).join(" "),
+    nextCommand: pipelineRun
+      ? `bun scripts/cli.ts hwlab nodes control-plane status --node ${stringField(lastEvent.node) ?? "G14"} --lane ${stringField(lastEvent.lane) ?? "v03"} --pipeline-run ${pipelineRun}`
+      : job.status === "running"
+        ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
+        : null,
+  };
+}
+
 function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary {
  const nowMs = Date.now();
  const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);