From a8b43716652e1c71686fb526a1a9b1f8153a4ba4 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 31 May 2026 10:40:34 +0000 Subject: [PATCH] fix: improve cli visibility for hwlab rollout --- scripts/src/gh.ts | 28 +++++-- scripts/src/hwlab-g14.ts | 83 +++++++++++++++++++-- scripts/src/jobs.ts | 153 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 12 deletions(-) diff --git a/scripts/src/gh.ts b/scripts/src/gh.ts index de7c7a58..033b59d7 100644 --- a/scripts/src/gh.ts +++ b/scripts/src/gh.ts @@ -29,7 +29,13 @@ const ISSUE_LIST_STATES = ["open", "closed", "all"] as const; const BODY_UPDATE_MODES = ["replace", "append"] as const; const BOARD_MUTATION_SECTIONS = ["open", "closed"] as const; const BOARD_GITHUB_STATUSES = ["OPEN", "CLOSED"] as const; -const GH_VALUE_OPTIONS = new Set(["--repo", "--limit", "--board-issue", "--known-meta-issue", "--ignore-issue", "--title", "--body-file", "--body", "--base", "--head", "--json", "--state", "--mode", "--expect-updated-at", "--expect-body-sha", "--body-profile", "--label", "--field", "--value", "--section", "--to", "--status", "--row-file", "--category", "--branch", "--tasks", "--summary", "--focus", "--validation", "--progress", "--number"]); +const GH_VALUE_OPTIONS = new Set([ + "--repo", "--limit", "--board-issue", "--known-meta-issue", "--ignore-issue", "--title", + "--body-file", "--body", "--base", "--head", "--json", "--state", "--mode", + "--expect-updated-at", "--expect-body-sha", "--body-profile", "--label", "--field", + "--value", "--section", "--to", "--status", "--row-file", "--category", "--branch", + "--tasks", "--summary", "--focus", "--validation", "--progress", "--number", "--pr", +]); const GH_FLAG_OPTIONS = new Set(["--dry-run", "--draft", "--notify-claudeqq-brief-diff", "--allow-short-body", "--raw", "--full", "--stat", "--merge", "--squash", "--rebase", "--delete-branch"]); const MIN_SAFE_BODY_SCAN_CHARS = MIN_SAFE_ISSUE_BODY_CHARS; const ISSUE_SCAN_MAX_FINDINGS = 60; @@ -816,15 +822,27 @@ function parsePositionalNumberForCommand(repo: string, args: string[], startInde function resolvePositionalPrReference(args: string[], startIndex: number, label: string, options: GitHubOptions): GitHubResolvedNumberReference | GitHubCommandResult { const targets = positionalArgs(args.slice(startIndex)); - if (targets.length !== 1) { - return validationError(label, options.repo, `${label} requires exactly one positive integer or owner/repo#number positional argument`, { + const prOption = optionValue(args, "--pr"); + if (targets.length > 0 && prOption !== undefined) { + return validationError(label, options.repo, `${label} accepts either a positional PR target or --pr, not both`, { supportedCommands: [ `bun scripts/cli.ts gh ${label} --repo ${options.repo}`, + `bun scripts/cli.ts gh ${label} --pr --repo ${options.repo}`, `bun scripts/cli.ts gh ${label} ${options.repo}#`, ], }); } - const shorthand = parseOwnerRepoNumberShorthand(targets[0]); + const effectiveTargets = prOption !== undefined ? [prOption] : targets; + if (effectiveTargets.length !== 1) { + return validationError(label, options.repo, `${label} requires exactly one positive integer or owner/repo#number positional argument`, { + supportedCommands: [ + `bun scripts/cli.ts gh ${label} --repo ${options.repo}`, + `bun scripts/cli.ts gh ${label} --pr --repo ${options.repo}`, + `bun scripts/cli.ts gh ${label} ${options.repo}#`, + ], + }); + } + const shorthand = parseOwnerRepoNumberShorthand(effectiveTargets[0]); if (shorthand !== null) { const explicitRepo = optionValue(args, "--repo"); if (explicitRepo !== undefined && explicitRepo !== shorthand.repo) { @@ -835,7 +853,7 @@ function resolvePositionalPrReference(args: string[], startIndex: number, label: } return { repo: shorthand.repo, number: shorthand.number, shorthand }; } - const number = parseNumberForCommand(options.repo, targets[0], label); + const number = parseNumberForCommand(options.repo, effectiveTargets[0], label); if (typeof number !== "number") return number; return { repo: options.repo, number }; } diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 07d6004f..328d0780 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -796,7 +796,15 @@ function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: }; } -function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string): Record { +function v02WebAssetsFromText( + text: string, + commandOk: boolean, + sourceCommit: string | null, + argoSyncRevision: string | null, + exitCode: number | null, + stderr: string, + activePipelineRuns: unknown[] = [], +): Record { const fields: Record = {}; for (const line of text.split(/\r?\n/u)) { const [key = "", ...rest] = line.split("\t"); @@ -846,14 +854,25 @@ function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: st apiHealth: numericField(fields.apiHealthOk), }, apiRevision, - note: apiRevision && sourceCommit && apiRevision !== sourceCommit - ? "cloud-api image revision can differ when a change only republishes Cloud Web static assets; use webAssets.checks for 19666 frontend asset readiness." - : null, + note: webAssetsRevisionNote(apiRevision, sourceCommit, activePipelineRuns), exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), }; } +function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null { + if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null; + const activeItems = activePipelineRuns.map((item) => record(item)); + const sourceIsActive = activeItems.some((item) => item.sourceCommit === sourceCommit || String(item.name ?? "").includes(shortSha(sourceCommit))); + if (sourceIsActive) { + return `runtime rollout still in progress; cloud-api apiRevision=${apiRevision} has not reached sourceCommit=${sourceCommit} yet`; + } + if (activeItems.length > 0) { + return `runtime apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; active v02 PipelineRun observed, wait for rollout before judging runtime revision`; + } + return `runtime revision lag: cloud-api apiRevision=${apiRevision} differs from sourceCommit=${sourceCommit}; inspect recentPipelineRuns, argo, and runtime-ready state`; +} + function numericField(value: string | undefined): number | null { if (value === undefined || value.trim().length === 0) return null; const parsed = Number(value); @@ -1396,6 +1415,7 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record): Record): Record): Record { + const statusStartMs = Date.now(); + printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit }); const before = runGitMirrorStatus(); const beforeSummary = compactGitMirrorStatus(before, sourceCommit); + printProgressEvent("hwlab.v02.trigger.progress", { + stage: "git-mirror-pre-sync-status", + status: beforeSummary.ok === true ? "succeeded" : "failed", + sourceCommit, + durationMs: Date.now() - statusStartMs, + required: beforeSummary.required, + localV02: beforeSummary.localV02, + pendingFlush: beforeSummary.pendingFlush, + reason: beforeSummary.reason, + }); if (options.dryRun) { return { ok: true, @@ -1735,13 +1786,29 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick { + const startedAtMs = Date.now(); const jobName = gitMirrorSyncJobName(); const manifest = gitMirrorSyncJobManifest(jobName); const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); @@ -1934,6 +2002,7 @@ function runGitMirrorSync(options: G14GitMirrorOptions): Record mode: options.dryRun ? "dry-run" : "confirmed-sync", namespace: GIT_MIRROR_NAMESPACE, jobName, + elapsedMs: Date.now() - startedAtMs, manifest: options.dryRun ? manifest : undefined, result: compactCommandResult(result), status: options.dryRun ? undefined : runGitMirrorStatus(), diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index a47f01d8..3b0004f0 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -31,8 +31,14 @@ export interface JobProgressSummary { sourceCommit: string | null; pipelineRun: string | null; pipelineCreated: boolean | null; + elapsedSeconds: number | null; + stageElapsedSeconds: number | null; lastEventAt: string | null; + lastEventAgeSeconds: number | null; eventsObserved: number; + slow: boolean; + warnings: string[]; + timings: Record; summary: string; nextCommand: string | null; } @@ -191,6 +197,7 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary { const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current"; if (!knownWorkflow && tails === undefined) return genericJobProgress(job); + const nowMs = Date.now(); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); @@ -207,6 +214,20 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou : null; const lastEventAt = stringField(lastEvent.at); const kind = events.length > 0 || knownWorkflow ? "hwlab-v02-trigger" : "generic"; + const elapsedSeconds = jobElapsedSeconds(job, nowMs); + const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); + const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); + const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); + const warnings = jobProgressWarnings({ + job, + eventsObserved: events.length, + elapsedSeconds, + stage, + stageStatus, + stageElapsedSeconds, + lastEventAgeSeconds, + }); + const slow = warnings.length > 0; const nextCommand = pipelineRun ? `bun scripts/cli.ts hwlab g14 control-plane status --lane v02` : job.status === "running" @@ -219,6 +240,10 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, pipelineRun ? `pipelineRun=${pipelineRun}` : null, pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null, + elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, + stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, + lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, + slow ? "visibility-warning" : null, ].filter(Boolean).join(" ") : `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`; return { @@ -228,14 +253,31 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou sourceCommit, pipelineRun, pipelineCreated, + elapsedSeconds, + stageElapsedSeconds, lastEventAt, + lastEventAgeSeconds, eventsObserved: events.length, + slow, + warnings, + timings, summary, nextCommand, }; } function genericJobProgress(job: JobRecord): JobProgressSummary { + const nowMs = Date.now(); + const elapsedSeconds = jobElapsedSeconds(job, nowMs); + const warnings = jobProgressWarnings({ + job, + eventsObserved: 0, + elapsedSeconds, + stage: null, + stageStatus: null, + stageElapsedSeconds: null, + lastEventAgeSeconds: null, + }); return { kind: "generic", stage: null, @@ -243,13 +285,79 @@ function genericJobProgress(job: JobRecord): JobProgressSummary { sourceCommit: null, pipelineRun: null, pipelineCreated: null, + elapsedSeconds, + stageElapsedSeconds: null, lastEventAt: null, + lastEventAgeSeconds: null, eventsObserved: 0, + slow: warnings.length > 0, + warnings, + timings: {}, summary: `${job.status}${job.exitCode === null ? "" : ` exit=${job.exitCode}`}`, nextCommand: job.status === "running" ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` : null, }; } +function jobElapsedSeconds(job: JobRecord, nowMs = Date.now()): number | null { + const startMs = timestampMs(job.startedAt ?? job.createdAt); + if (startMs === null) return null; + const endMs = timestampMs(job.finishedAt) ?? nowMs; + if (endMs < startMs) return null; + return Math.round((endMs - startMs) / 1000); +} + +function secondsSince(value: string, end: string | number = Date.now()): number | null { + const startMs = timestampMs(value); + const endMs = typeof end === "number" ? end : timestampMs(end); + if (startMs === null || endMs === null || endMs < startMs) return null; + return Math.round((endMs - startMs) / 1000); +} + +function timestampMs(value: unknown): number | null { + if (typeof value !== "string" || value.trim().length === 0) return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function currentStageElapsedSeconds(events: Record[], stage: string | null, stageStatus: string | null, job: JobRecord, nowMs = Date.now()): number | null { + if (stage === null) return null; + const stageEvents = events.filter((event) => stringField(event.stage) === stage); + const startEvent = stageEvents.findLast((event) => stringField(event.status) === "started"); + const startAt = stringField(startEvent?.at); + if (startAt === null) return null; + const endEvent = stageStatus === "started" + ? null + : stageEvents.findLast((event) => { + const status = stringField(event.status); + return status !== null && status !== "started"; + }); + const endAt = stringField(endEvent?.at) ?? job.finishedAt ?? (job.status === "running" ? new Date(nowMs).toISOString() : null); + return endAt === null ? null : secondsSince(startAt, endAt); +} + +function jobProgressWarnings(input: { + job: JobRecord; + eventsObserved: number; + elapsedSeconds: number | null; + stage: string | null; + stageStatus: string | null; + stageElapsedSeconds: number | null; + lastEventAgeSeconds: number | null; +}): string[] { + if (input.job.status !== "running") return []; + const warnings: string[] = []; + if (input.eventsObserved === 0 && (input.elapsedSeconds ?? 0) >= 30) { + warnings.push(`running for ${input.elapsedSeconds}s without progress events; inspect stdoutTail/stderrTail and fullLogPaths`); + } + if ((input.lastEventAgeSeconds ?? 0) >= 30) { + warnings.push(`no progress event for ${input.lastEventAgeSeconds}s; current stage may be blocked or waiting on remote I/O`); + } + if (input.stage !== null && input.stageStatus !== null && input.stageStatus !== "succeeded" && input.stageStatus !== "failed" && (input.stageElapsedSeconds ?? 0) >= 30) { + warnings.push(`stage ${input.stage}:${input.stageStatus} has been active for ${input.stageElapsedSeconds}s`); + } + return warnings; +} + function tailTextByBytes(text: string, maxBytes: number): string { const safeMaxBytes = Math.max(0, Math.floor(maxBytes)); if (safeMaxBytes === 0) return ""; @@ -275,6 +383,51 @@ function parseJsonLineEvents(text: string, eventName: string): Record { + const timings: Record = {}; + for (const normalizedText of [text, text.replaceAll('\\"', '"').replaceAll("\\n", "\n")]) { + extractGitMirrorTimingJsonLines(normalizedText, timings); + extractGitMirrorTimingRegex(normalizedText, timings); + extractSshRuntimeTimingRegex(normalizedText, timings); + } + return timings; +} + +function extractGitMirrorTimingJsonLines(text: string, timings: Record): void { + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) continue; + try { + const parsed = JSON.parse(trimmed) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) continue; + const event = parsed as Record; + if (event.event !== "git-mirror-sync-timing") continue; + const phase = stringField(event.phase); + const durationMs = typeof event.durationMs === "number" && Number.isFinite(event.durationMs) ? event.durationMs : null; + if (phase !== null && durationMs !== null) timings[`gitMirror.${phase}Ms`] = Math.round(durationMs); + } catch { + // Ignore non-JSON lines; raw logs remain available through job status tails. + } + } +} + +function extractGitMirrorTimingRegex(text: string, timings: Record): void { + const pattern = /"event"\s*:\s*"git-mirror-sync-timing"[\s\S]{0,240}?"phase"\s*:\s*"([^"]+)"[\s\S]{0,240}?"durationMs"\s*:\s*(\d+)/gu; + for (const match of text.matchAll(pattern)) { + const phase = match[1]; + const durationMs = Number(match[2]); + if (phase !== undefined && Number.isFinite(durationMs)) timings[`gitMirror.${phase}Ms`] = Math.round(durationMs); + } +} + +function extractSshRuntimeTimingRegex(text: string, timings: Record): void { + const pattern = /"code"\s*:\s*"ssh-runtime-timing"[\s\S]{0,320}?"elapsedMs"\s*:\s*(\d+)/gu; + const values = [...text.matchAll(pattern)] + .map((match) => Number(match[1])) + .filter((value) => Number.isFinite(value)); + if (values.length > 0) timings.sshRuntimeMaxMs = Math.max(...values); +} + function stringField(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; }