fix: improve cli visibility for hwlab rollout

scripts/src/gh.ts

@@ -29,7 +29,13 @@ const ISSUE_LIST_STATES = ["open", "closed", "all"] as const;
 const BODY_UPDATE_MODES = ["replace", "append"] as const;
 const BOARD_MUTATION_SECTIONS = ["open", "closed"] as const;
 const BOARD_GITHUB_STATUSES = ["OPEN", "CLOSED"] as const;
-const GH_VALUE_OPTIONS = new Set(["--repo", "--limit", "--board-issue", "--known-meta-issue", "--ignore-issue", "--title", "--body-file", "--body", "--base", "--head", "--json", "--state", "--mode", "--expect-updated-at", "--expect-body-sha", "--body-profile", "--label", "--field", "--value", "--section", "--to", "--status", "--row-file", "--category", "--branch", "--tasks", "--summary", "--focus", "--validation", "--progress", "--number"]);
+const GH_VALUE_OPTIONS = new Set([
+  "--repo", "--limit", "--board-issue", "--known-meta-issue", "--ignore-issue", "--title",
+  "--body-file", "--body", "--base", "--head", "--json", "--state", "--mode",
+  "--expect-updated-at", "--expect-body-sha", "--body-profile", "--label", "--field",
+  "--value", "--section", "--to", "--status", "--row-file", "--category", "--branch",
+  "--tasks", "--summary", "--focus", "--validation", "--progress", "--number", "--pr",
+]);
 const GH_FLAG_OPTIONS = new Set(["--dry-run", "--draft", "--notify-claudeqq-brief-diff", "--allow-short-body", "--raw", "--full", "--stat", "--merge", "--squash", "--rebase", "--delete-branch"]);
 const MIN_SAFE_BODY_SCAN_CHARS = MIN_SAFE_ISSUE_BODY_CHARS;
 const ISSUE_SCAN_MAX_FINDINGS = 60;
@@ -816,15 +822,27 @@ function parsePositionalNumberForCommand(repo: string, args: string[], startInde
 
 function resolvePositionalPrReference(args: string[], startIndex: number, label: string, options: GitHubOptions): GitHubResolvedNumberReference | GitHubCommandResult {
   const targets = positionalArgs(args.slice(startIndex));
-  if (targets.length !== 1) {
-    return validationError(label, options.repo, `${label} requires exactly one positive integer or owner/repo#number positional argument`, {
+  const prOption = optionValue(args, "--pr");
+  if (targets.length > 0 && prOption !== undefined) {
+    return validationError(label, options.repo, `${label} accepts either a positional PR target or --pr, not both`, {
       supportedCommands: [
         `bun scripts/cli.ts gh ${label} <number> --repo ${options.repo}`,
+        `bun scripts/cli.ts gh ${label} --pr <number> --repo ${options.repo}`,
         `bun scripts/cli.ts gh ${label} ${options.repo}#<number>`,
       ],
     });
   }
-  const shorthand = parseOwnerRepoNumberShorthand(targets[0]);
+  const effectiveTargets = prOption !== undefined ? [prOption] : targets;
+  if (effectiveTargets.length !== 1) {
+    return validationError(label, options.repo, `${label} requires exactly one positive integer or owner/repo#number positional argument`, {
+      supportedCommands: [
+        `bun scripts/cli.ts gh ${label} <number> --repo ${options.repo}`,
+        `bun scripts/cli.ts gh ${label} --pr <number> --repo ${options.repo}`,
+        `bun scripts/cli.ts gh ${label} ${options.repo}#<number>`,
+      ],
+    });
+  }
+  const shorthand = parseOwnerRepoNumberShorthand(effectiveTargets[0]);
   if (shorthand !== null) {
     const explicitRepo = optionValue(args, "--repo");
     if (explicitRepo !== undefined && explicitRepo !== shorthand.repo) {
@@ -835,7 +853,7 @@ function resolvePositionalPrReference(args: string[], startIndex: number, label:
     }
     return { repo: shorthand.repo, number: shorthand.number, shorthand };
   }
-  const number = parseNumberForCommand(options.repo, targets[0], label);
+  const number = parseNumberForCommand(options.repo, effectiveTargets[0], label);
   if (typeof number !== "number") return number;
   return { repo: options.repo, number };
 }

scripts/src/hwlab-g14.ts

@@ -796,7 +796,15 @@ function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun:
   };
 }
 
-function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
+function v02WebAssetsFromText(
+  text: string,
+  commandOk: boolean,
+  sourceCommit: string | null,
+  argoSyncRevision: string | null,
+  exitCode: number | null,
+  stderr: string,
+  activePipelineRuns: unknown[] = [],
+): Record<string, unknown> {
   const fields: Record<string, string> = {};
   for (const line of text.split(/\r?\n/u)) {
     const [key = "", ...rest] = line.split("\t");
@@ -846,14 +854,25 @@ function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: st
       apiHealth: numericField(fields.apiHealthOk),
     },
     apiRevision,
-    note: apiRevision && sourceCommit && apiRevision !== sourceCommit
-      ? "cloud-api image revision can differ when a change only republishes Cloud Web static assets; use webAssets.checks for 19666 frontend asset readiness."
-      : null,
+    note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns),
     exitCode,
     stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
   };
 }
 
+function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
+  if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
+  const activeItems = activePipelineRuns.map((item) => record(item));
+  const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit)));
+  if (sourceIsActive) {
+    return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`;
+  }
+  if (activeItems.length > 0) {
+    return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`;
+  }
+  return `runtime revision lag: cloud-api apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; inspect recentPipelineRuns, argo, and runtime-ready state`;
+}
+
 function numericField(value: string | undefined): number | null {
   if (value === undefined || value.trim().length === 0) return null;
   const parsed = Number(value);
@@ -1396,6 +1415,7 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
       syncRevision,
       webAssetsSection?.exitCode ?? null,
       bundle.stderr,
+      activePipelineRuns,
     ),
     activePipelineRuns,
     recentPipelineRuns,
@@ -1497,9 +1517,27 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
       degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed",
     };
   }
-  printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
+  printProgressEvent("hwlab.v02.trigger.progress", {
+    stage: "git-mirror-pre-sync",
+    status: "started",
+    sourceCommit,
+    pipelineRun: v02PipelineRunName(sourceCommit),
+    reason: "ensure local git mirror has source commit before creating PipelineRun",
+  });
   const gitMirrorPreSync = preSyncV02GitMirror(sourceCommit, options);
-  printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: gitMirrorPreSync.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), mode: record(gitMirrorPreSync).mode ?? null, degradedReason: record(gitMirrorPreSync).degradedReason ?? null });
+  printProgressEvent("hwlab.v02.trigger.progress", {
+    stage: "git-mirror-pre-sync",
+    status: gitMirrorPreSync.ok === true ? "succeeded" : "failed",
+    sourceCommit,
+    pipelineRun: v02PipelineRunName(sourceCommit),
+    mode: record(gitMirrorPreSync).mode ?? null,
+    required: nested(gitMirrorPreSync, ["before", "required"]) ?? null,
+    localV02: nested(gitMirrorPreSync, ["before", "localV02"]) ?? null,
+    pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null,
+    reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null,
+    syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null,
+    degradedReason: record(gitMirrorPreSync).degradedReason ?? null,
+  });
   if (gitMirrorPreSync.ok !== true) {
     return {
       ok: false,
@@ -1706,6 +1744,7 @@ function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unk
     mode: sync.mode ?? null,
     namespace: sync.namespace ?? GIT_MIRROR_NAMESPACE,
     jobName: sync.jobName ?? null,
+    elapsedMs: sync.elapsedMs ?? null,
     exitCode: nested(sync, ["result", "exitCode"]) ?? null,
     stdoutTail: tailText(nested(sync, ["result", "stdout"]), 1600),
     stderrTail: tailText(nested(sync, ["result", "stderr"]), 1200),
@@ -1715,8 +1754,20 @@ function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unk
 }
 
 function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
+  const statusStartMs = Date.now();
+  printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
   const before = runGitMirrorStatus();
   const beforeSummary = compactGitMirrorStatus(before, sourceCommit);
+  printProgressEvent("hwlab.v02.trigger.progress", {
+    stage: "git-mirror-pre-sync-status",
+    status: beforeSummary.ok === true ? "succeeded" : "failed",
+    sourceCommit,
+    durationMs: Date.now() - statusStartMs,
+    required: beforeSummary.required,
+    localV02: beforeSummary.localV02,
+    pendingFlush: beforeSummary.pendingFlush,
+    reason: beforeSummary.reason,
+  });
   if (options.dryRun) {
     return {
       ok: true,
@@ -1735,13 +1786,29 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlane
       before: beforeSummary,
     };
   }
+  printProgressEvent("hwlab.v02.trigger.progress", {
+    stage: "git-mirror-pre-sync-sync",
+    status: "started",
+    sourceCommit,
+    reason: beforeSummary.reason,
+    localV02: beforeSummary.localV02,
+    pendingFlush: beforeSummary.pendingFlush,
+  });
   const sync = runGitMirrorSync({
     action: "sync",
     confirm: true,
     dryRun: false,
     timeoutSeconds: options.timeoutSeconds,
   });
-  const after = runGitMirrorStatus();
+  printProgressEvent("hwlab.v02.trigger.progress", {
+    stage: "git-mirror-pre-sync-sync",
+    status: sync.ok === true ? "succeeded" : "failed",
+    sourceCommit,
+    durationMs: sync.elapsedMs ?? null,
+    jobName: sync.jobName ?? null,
+  });
+  const syncStatus = record(sync.status);
+  const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
   const afterSummary = compactGitMirrorStatus(after, sourceCommit);
   const ok = sync.ok === true && afterSummary.required === false;
   return {
@@ -1893,6 +1960,7 @@ function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown
 }
 
 function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown> {
+  const startedAtMs = Date.now();
   const jobName = gitMirrorSyncJobName();
   const manifest = gitMirrorSyncJobManifest(jobName);
   const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
@@ -1934,6 +2002,7 @@ function runGitMirrorSync(options: G14GitMirrorOptions): Record<string, unknown>
     mode: options.dryRun ? "dry-run" : "confirmed-sync",
     namespace: GIT_MIRROR_NAMESPACE,
     jobName,
+    elapsedMs: Date.now() - startedAtMs,
     manifest: options.dryRun ? manifest : undefined,
     result: compactCommandResult(result),
     status: options.dryRun ? undefined : runGitMirrorStatus(),

scripts/src/jobs.ts

@@ -31,8 +31,14 @@ export interface JobProgressSummary {
   sourceCommit: string | null;
   pipelineRun: string | null;
   pipelineCreated: boolean | null;
+  elapsedSeconds: number | null;
+  stageElapsedSeconds: number | null;
   lastEventAt: string | null;
+  lastEventAgeSeconds: number | null;
   eventsObserved: number;
+  slow: boolean;
+  warnings: string[];
+  timings: Record<string, number>;
   summary: string;
   nextCommand: string | null;
 }
@@ -191,6 +197,7 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
 function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary {
   const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
   if (!knownWorkflow && tails === undefined) return genericJobProgress(job);
+  const nowMs = Date.now();
   const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
   const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
   const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
@@ -207,6 +214,20 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
       : null;
   const lastEventAt = stringField(lastEvent.at);
   const kind = events.length > 0 || knownWorkflow ? "hwlab-v02-trigger" : "generic";
+  const elapsedSeconds = jobElapsedSeconds(job, nowMs);
+  const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs);
+  const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
+  const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`);
+  const warnings = jobProgressWarnings({
+    job,
+    eventsObserved: events.length,
+    elapsedSeconds,
+    stage,
+    stageStatus,
+    stageElapsedSeconds,
+    lastEventAgeSeconds,
+  });
+  const slow = warnings.length > 0;
   const nextCommand = pipelineRun
     ? `bun scripts/cli.ts hwlab g14 control-plane status --lane v02`
     : job.status === "running"
@@ -219,6 +240,10 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
         sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
         pipelineRun ? `pipelineRun=${pipelineRun}` : null,
         pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null,
+        elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
+        stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
+        lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
+        slow ? "visibility-warning" : null,
       ].filter(Boolean).join(" ")
     : `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`;
   return {
@@ -228,14 +253,31 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
     sourceCommit,
     pipelineRun,
     pipelineCreated,
+    elapsedSeconds,
+    stageElapsedSeconds,
     lastEventAt,
+    lastEventAgeSeconds,
     eventsObserved: events.length,
+    slow,
+    warnings,
+    timings,
     summary,
     nextCommand,
   };
 }
 
 function genericJobProgress(job: JobRecord): JobProgressSummary {
+  const nowMs = Date.now();
+  const elapsedSeconds = jobElapsedSeconds(job, nowMs);
+  const warnings = jobProgressWarnings({
+    job,
+    eventsObserved: 0,
+    elapsedSeconds,
+    stage: null,
+    stageStatus: null,
+    stageElapsedSeconds: null,
+    lastEventAgeSeconds: null,
+  });
   return {
     kind: "generic",
     stage: null,
@@ -243,13 +285,79 @@ function genericJobProgress(job: JobRecord): JobProgressSummary {
     sourceCommit: null,
     pipelineRun: null,
     pipelineCreated: null,
+    elapsedSeconds,
+    stageElapsedSeconds: null,
     lastEventAt: null,
+    lastEventAgeSeconds: null,
     eventsObserved: 0,
+    slow: warnings.length > 0,
+    warnings,
+    timings: {},
     summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`,
     nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null,
   };
 }
 
+function jobElapsedSeconds(job: JobRecord, nowMs = Date.now()): number | null {
+  const startMs = timestampMs(job.startedAt ?? job.createdAt);
+  if (startMs === null) return null;
+  const endMs = timestampMs(job.finishedAt) ?? nowMs;
+  if (endMs < startMs) return null;
+  return Math.round((endMs - startMs) / 1000);
+}
+
+function secondsSince(value: string, end: string | number = Date.now()): number | null {
+  const startMs = timestampMs(value);
+  const endMs = typeof end === "number" ? end : timestampMs(end);
+  if (startMs === null || endMs === null || endMs < startMs) return null;
+  return Math.round((endMs - startMs) / 1000);
+}
+
+function timestampMs(value: unknown): number | null {
+  if (typeof value !== "string" || value.trim().length === 0) return null;
+  const parsed = Date.parse(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function currentStageElapsedSeconds(events: Record<string, unknown>[], stage: string | null, stageStatus: string | null, job: JobRecord, nowMs = Date.now()): number | null {
+  if (stage === null) return null;
+  const stageEvents = events.filter((event) => stringField(event.stage) === stage);
+  const startEvent = stageEvents.findLast((event) => stringField(event.status) === "started");
+  const startAt = stringField(startEvent?.at);
+  if (startAt === null) return null;
+  const endEvent = stageStatus === "started"
+    ? null
+    : stageEvents.findLast((event) => {
+      const status = stringField(event.status);
+      return status !== null && status !== "started";
+    });
+  const endAt = stringField(endEvent?.at) ?? job.finishedAt ?? (job.status === "running" ? new Date(nowMs).toISOString() : null);
+  return endAt === null ? null : secondsSince(startAt, endAt);
+}
+
+function jobProgressWarnings(input: {
+  job: JobRecord;
+  eventsObserved: number;
+  elapsedSeconds: number | null;
+  stage: string | null;
+  stageStatus: string | null;
+  stageElapsedSeconds: number | null;
+  lastEventAgeSeconds: number | null;
+}): string[] {
+  if (input.job.status !== "running") return [];
+  const warnings: string[] = [];
+  if (input.eventsObserved === 0 && (input.elapsedSeconds ?? 0) >= 30) {
+    warnings.push(`running for ${input.elapsedSeconds}s without progress events; inspect stdoutTail/stderrTail and fullLogPaths`);
+  }
+  if ((input.lastEventAgeSeconds ?? 0) >= 30) {
+    warnings.push(`no progress event for ${input.lastEventAgeSeconds}s; current stage may be blocked or waiting on remote I/O`);
+  }
+  if (input.stage !== null && input.stageStatus !== null && input.stageStatus !== "succeeded" && input.stageStatus !== "failed" && (input.stageElapsedSeconds ?? 0) >= 30) {
+    warnings.push(`stage ${input.stage}:${input.stageStatus} has been active for ${input.stageElapsedSeconds}s`);
+  }
+  return warnings;
+}
+
 function tailTextByBytes(text: string, maxBytes: number): string {
   const safeMaxBytes = Math.max(0, Math.floor(maxBytes));
   if (safeMaxBytes === 0) return "";
@@ -275,6 +383,51 @@ function parseJsonLineEvents(text: string, eventName: string): Record<string, un
   return events;
 }
 
+function extractGitMirrorTimings(text: string): Record<string, number> {
+  const timings: Record<string, number> = {};
+  for (const normalizedText of [text, text.replaceAll('\\"', '"').replaceAll("\\n", "\n")]) {
+    extractGitMirrorTimingJsonLines(normalizedText, timings);
+    extractGitMirrorTimingRegex(normalizedText, timings);
+    extractSshRuntimeTimingRegex(normalizedText, timings);
+  }
+  return timings;
+}
+
+function extractGitMirrorTimingJsonLines(text: string, timings: Record<string, number>): void {
+  for (const line of text.split(/\r?\n/u)) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith("{")) continue;
+    try {
+      const parsed = JSON.parse(trimmed) as unknown;
+      if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) continue;
+      const event = parsed as Record<string, unknown>;
+      if (event.event !== "git-mirror-sync-timing") continue;
+      const phase = stringField(event.phase);
+      const durationMs = typeof event.durationMs === "number" && Number.isFinite(event.durationMs) ? event.durationMs : null;
+      if (phase !== null && durationMs !== null) timings[`gitMirror.${phase}Ms`] = Math.round(durationMs);
+    } catch {
+      // Ignore non-JSON lines; raw logs remain available through job status tails.
+    }
+  }
+}
+
+function extractGitMirrorTimingRegex(text: string, timings: Record<string, number>): void {
+  const pattern = /"event"\s*:\s*"git-mirror-sync-timing"[\s\S]{0,240}?"phase"\s*:\s*"([^"]+)"[\s\S]{0,240}?"durationMs"\s*:\s*(\d+)/gu;
+  for (const match of text.matchAll(pattern)) {
+    const phase = match[1];
+    const durationMs = Number(match[2]);
+    if (phase !== undefined && Number.isFinite(durationMs)) timings[`gitMirror.${phase}Ms`] = Math.round(durationMs);
+  }
+}
+
+function extractSshRuntimeTimingRegex(text: string, timings: Record<string, number>): void {
+  const pattern = /"code"\s*:\s*"ssh-runtime-timing"[\s\S]{0,320}?"elapsedMs"\s*:\s*(\d+)/gu;
+  const values = [...text.matchAll(pattern)]
+    .map((match) => Number(match[1]))
+    .filter((value) => Number.isFinite(value));
+  if (values.length > 0) timings.sshRuntimeMaxMs = Math.max(...values);
+}
+
 function stringField(value: unknown): string | null {
   return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
 }