From 995b75dd2a83dabc7bdfb1a3f9cd8c753fcbb1a3 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 31 May 2026 08:19:46 +0000 Subject: [PATCH] fix: bound hwlab g14 cli output --- docs/reference/cli.md | 4 +- scripts/src/hwlab-g14.ts | 420 +++++++++++++++++++++++++++++++++------ 2 files changed, 366 insertions(+), 58 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 4ba48a6a..fcaf2b54 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -43,10 +43,10 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P - `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`，由 systemd + Docker Compose 管理，位于 native k3s 故障域外；`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s，并做 live commit 验证，不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名，标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`。 - `commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr` 是 host Codex 指挥官直管微服务 skeleton 入口。当前命令返回 `phase=source-contract`、service/API/state/bridge/prompt/trace/#20/#46/ClaudeQQ 审批边界、.state/commander/ 状态模型、dev 无 daemon smoke contract、dry-run 计划和 GPT-5.5 PR prompt 边界辅助 lint，不接 live bridge、不注入 prompt、不发送 ClaudeQQ。`approval request --dry-run` 会生成 200 字以内中文纯文本 ClaudeQQ 审批草案、`notification-path-unavailable` blocker 和授权后唯一可用的 `bun scripts/cli.ts microservice proxy claudeqq /api/push/text --method POST --body-json '' --raw` 命令；不得提示使用本机 ClaudeQQ skill、powershell 或本地 server。`prompt-lint` 支持 `--prompt-file` 与 `--stdin`，输出 `ok`、`missingClauses`、`riskLevel`、`suggestedPatchSnippet` 且不回显完整 prompt；它是 commander 辅助检查，不是业务 PR 门禁，也不改变 `codex submit` 默认行为。`plan`、`smoke` 与 `approval request` 必须带 `--dry-run`；缺少时返回 `error=dry-run-required`。长期规则见 `docs/reference/host-codex-commander.md`。 - `hwlab g14 monitor-prs [--once] [--dry-run] [--interval-seconds N] [--max-cycles N] [--timeout-seconds N]` 是当前 HWLAB G14 PR -> CI/CD -> DEV rollout 的一行式入口。普通调用创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand` 和 stdout/stderr 路径；后台 worker 每轮通过 UniDesk `gh pr list/preflight/merge` 监控 `pikasTech/HWLAB` base=`G14` 的 open PR，ready 时合并，然后通过 UniDesk `ssh G14:k3s` 观察 `hwlab-g14-ci-poll-`、Argo `hwlab-g14-dev` 和 DEV `/health/live`，直到 DEV `Synced/Healthy` 且 Deployment/StatefulSet ready；历史 `Completed` smoke/debug pod 不作为 rollout blocker。每次成功 DEV rollout 后，worker 会定位或创建 #7“指挥简报索引”中的北京日期每日简报 issue，并追加 CI/CD 耗时、CI/CD 关键指标、语义化上线 changelog、自动 diff 摘要、PipelineRun、GitOps revision 和 DEV 验证摘要；关键指标来自 G14 Tekton TaskRun results，固定包含 `lazy build reused: x/y`、reused services、rebuild services 和每个 service 的独立耗时/状态/backend，用于观察 lazy build 机制效果。语义化 changelog 优先从 PR body 的 `## 修改`/`## 变更`/`## Changelog` 等段落提取，diff 摘要只作为文件和统计证据保留，不替代 changelog。也可用 `hwlab g14 record-rollout --pr --source-commit ` 手动补记，手动补记同样会按 PipelineRun 采集 TaskRun 指标。状态指针按用途分离：长期监控只写 `.state/hwlab-g14/latest-monitor-job.json`，`--once` 写 `latest-once-job.json`，`--dry-run` 写 `latest-dry-run-job.json`，`--once --dry-run` 写 `latest-once-dry-run-job.json`，避免一次性收口覆盖持续监控入口。`--once --dry-run` 只做单轮监控和 merge plan，不写 GitHub、不等待 rollout。该命令禁止使用原生 `gh` 或手拼 GitHub 请求；如果 UniDesk `gh` 子命令字段或行为不够，必须先改进 `scripts/src/gh.ts` 后再使用。 -- `hwlab g14 control-plane status|apply --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口，只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`；`status` 只读汇总 pipeline、RBAC/ServiceAccount、Argo、当前 commit PipelineRun 和遗留 v02 CronJob 清理状态；`apply` 先在 G14 workspace 快进并执行 render check，再经 `G14:k3s` server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`，confirmed apply 会删除遗留 v02 CronJob，但不会应用 runtime-v02 workload、Secret 或数据迁移。 +- `hwlab g14 control-plane status|apply --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口，只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`；`status` 只读汇总 pipeline、RBAC/ServiceAccount、Argo、当前 commit PipelineRun、最近 PipelineRun 摘要、活跃 PipelineRun 和遗留 v02 CronJob 清理状态，默认只读取必要字段，禁止把完整 PipelineRun spec、Tekton 内联脚本或历史大对象展开到默认输出；`apply` 先在 G14 workspace 快进并执行 render check，再经 `G14:k3s` server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`，confirmed apply 会删除遗留 v02 CronJob，但不会应用 runtime-v02 workload、Secret 或数据迁移。 - `hwlab g14 control-plane trigger-current --lane v02 [--dry-run|--confirm]` 是 v02 标准手动触发入口：解析当前 `origin/v0.2` full SHA，创建 commit-pinned `hwlab-v02-ci-poll-` PipelineRun；读 Git 走 `git-mirror-http.devops-infra.svc.cluster.local`，GitOps promotion 写 `git-mirror-write.devops-infra.svc.cluster.local`；confirmed trigger 在删除/创建 PipelineRun 前会先按当前 source commit render 并 server-side apply v02 Tekton RBAC、Pipeline 与 Argo Application，避免 CI/CD 脚本或 runtime-ready 逻辑已合并但集群仍执行旧 Pipeline 定义；同名 PipelineRun 成功或运行中时拒绝重复触发，失败或不存在时才删除旧对象并重新创建。 创建 PipelineRun 前会读取 `devops-infra` mirror refs，若 `localV02` 未等于当前 source commit，则自动执行一次受控 manual `git-mirror sync` Job 并复核 ref，复核失败时停止触发，避免 Tekton `prepare-source` 已知失败；services 参数只包含 v02 runtime service matrix，`hwlab-cli` 是固定 repo 短连接源码工具，不进入 PipelineRun service build。 - `--dry-run` 只报告是否会 pre-sync，不创建 Job；confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径，避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞；`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件，覆盖 `control-plane-refresh`、`git-mirror-pre-sync`、`delete-existing-pipelinerun` 和 `create-pipelinerun`，避免异步 job 长时间只有启动命令而无法判断卡点；只有现场同步调试才显式加 `--wait`；旧 `rerun-current` 只作为输入别名保留。 + `--dry-run` 只报告是否会 pre-sync，不创建 Job；confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径，避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞；`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件，覆盖 `control-plane-refresh`、`git-mirror-pre-sync`、`delete-existing-pipelinerun` 和 `create-pipelinerun`，避免异步 job 长时间只有启动命令而无法判断卡点；默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要，保留长度与 hash，完整内容通过 job stdout/stderr 文件渐进披露；只有现场同步调试才显式加 `--wait`；旧 `rerun-current` 只作为输入别名保留。 - `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行；不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`。 - `hwlab g14 control-plane cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口；真实清理只删除已完成 PipelineRun，让 Tekton/local-path 回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。 - `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口；只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。 diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index cedcf8ce..d3bbcc95 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -1,5 +1,6 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { createHash } from "node:crypto"; import { repoRoot, rootPath, type Config } from "./config"; import { runCommand } from "./command"; import { startJob } from "./jobs"; @@ -114,6 +115,11 @@ interface CommandJsonResult { parsed: unknown | null; } +interface ShellSection { + stdout: string; + exitCode: number | null; +} + interface OpenPullRequest { number: number; title?: string; @@ -344,7 +350,68 @@ function expandedParsedRoot(result: CommandJsonResult): Record } function commandData(result: CommandJsonResult): Record { - return record(expandedParsedRoot(result).data); + return record(sanitizeCliOutput(record(expandedParsedRoot(result).data))); +} + +function textHash(value: string): string { + return createHash("sha256").update(value).digest("hex").slice(0, 12); +} + +function redactLargePayloads(value: string): string { + return value + .replace(/manifest_b64='([^']{128,})'/gu, (_match, payload: string) => { + return `manifest_b64=''`; + }) + .replace(/manifest_b64="([^"]{128,})"/gu, (_match, payload: string) => { + return `manifest_b64=""`; + }) + .replace(/manifest_b64=([A-Za-z0-9+/=]{128,})/gu, (_match, payload: string) => { + return `manifest_b64=`; + }) + .replace(/[A-Za-z0-9+/=]{1024,}/gu, (payload: string) => { + return ``; + }); +} + +function compactInlineText(value: string, maxChars = 4000): string { + const redacted = redactLargePayloads(value); + if (redacted.length <= maxChars) return redacted; + const headChars = Math.floor(maxChars * 0.35); + const tailChars = maxChars - headChars; + const omittedChars = redacted.length - headChars - tailChars; + return [ + redacted.slice(0, headChars), + ``, + redacted.slice(-tailChars), + ].join("\n"); +} + +function sanitizeCliOutput(value: unknown): unknown { + if (typeof value === "string") return compactInlineText(value); + if (Array.isArray(value)) return value.map((item) => sanitizeCliOutput(item)); + if (typeof value !== "object" || value === null) return value; + const output: Record = {}; + for (const [key, item] of Object.entries(value)) output[key] = sanitizeCliOutput(item); + return output; +} + +function redactedCommand(command: string[]): string[] { + return command.map((arg) => compactInlineText(arg, 1200)); +} + +function compactCommandResult(result: CommandJsonResult): Record { + const data = record(expandedParsedRoot(result).data); + const stdout = String(data.stdout ?? result.stdout ?? ""); + const stderr = String(data.stderr ?? result.stderr ?? ""); + return { + ok: isCommandSuccess(result), + exitCode: result.exitCode, + command: redactedCommand(result.command), + stdout: compactInlineText(stdout.trim(), 4000), + stderr: compactInlineText(stderr.trim(), 4000), + stdoutBytes: Buffer.byteLength(stdout), + stderrBytes: Buffer.byteLength(stderr), + }; } function isCommandSuccess(result: CommandJsonResult): boolean { @@ -410,6 +477,14 @@ function getV02Head(): string | null { return match?.[0] ?? null; } +function getV02CachedHead(): string | null { + const result = v02WorkspaceScript("git rev-parse origin/v0.2", 30_000); + if (!isCommandSuccess(result)) return null; + const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim(); + const match = /[0-9a-f]{40}/iu.exec(output); + return match?.[0] ?? null; +} + function v02PipelineRunName(sourceCommit: string): string { return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`; } @@ -425,22 +500,240 @@ function getPipelineRunCompact(name: string): Record { "-o", "jsonpath={.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}{.status.conditions[0].message}{\"\\n\"}", ], 60_000); - const text = statusText(result); + return pipelineRunCompactFromText(name, statusText(result), isCommandSuccess(result), result.command, result.exitCode, result.stderr); +} + +function pipelineRunCompactFromText( + name: string, + text: string, + commandOk: boolean, + command: string[] | string, + exitCode: number | null, + stderr: string, +): Record { const [status = "", reason = "", message = ""] = text.split(/\r?\n/u); - const notFound = !isCommandSuccess(result) && /not found/iu.test(`${result.stdout}\n${result.stderr}`); + const notFound = !commandOk && /not found/iu.test(`${text}\n${stderr}`); return { - ok: isCommandSuccess(result), - exists: isCommandSuccess(result) || !notFound, + ok: commandOk, + exists: commandOk || !notFound, pipelineRun: name, status: status || null, reason: reason || null, message: message || null, - command: result.command, - exitCode: result.exitCode, - stderr: result.stderr.trim().slice(0, 2000), + command: Array.isArray(command) ? redactedCommand(command) : command, + exitCode, + stderr: stderr.trim().slice(0, 2000), }; } +function timestampMs(value: unknown): number | null { + if (typeof value !== "string" || value.trim().length === 0) return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function secondsBetween(start: unknown, end: unknown): number | null { + const startMs = timestampMs(start); + const endMs = timestampMs(end); + if (startMs === null || endMs === null || endMs < startMs) return null; + return Math.round((endMs - startMs) / 1000); +} + +function pipelineRunStatusRow(item: unknown, nowMs: number): Record { + const root = record(item); + const metadata = record(root.metadata); + const status = record(root.status); + const labels = record(metadata.labels); + const conditions = Array.isArray(status.conditions) ? status.conditions : []; + const condition = record(conditions[0]); + const startTime = status.startTime ?? null; + const completionTime = status.completionTime ?? null; + const startMs = timestampMs(startTime); + const conditionStatus = typeof condition.status === "string" ? condition.status : null; + return { + name: metadata.name ?? null, + sourceCommit: labels["hwlab.pikastech.local/source-commit"] ?? null, + status: conditionStatus, + reason: condition.reason ?? null, + createdAt: metadata.creationTimestamp ?? null, + startTime, + completionTime, + durationSeconds: secondsBetween(startTime, completionTime), + elapsedSeconds: startMs === null || completionTime !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)), + active: completionTime === null && conditionStatus !== "True" && conditionStatus !== "False", + }; +} + +function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record | null { + const [ + name = "", + sourceCommit = "", + createdAt = "", + startTime = "", + completionTime = "", + status = "", + reason = "", + ] = line.split("\t"); + if (!name.startsWith(V02_PIPELINERUN_PREFIX)) return null; + const startMs = timestampMs(startTime); + const completion = completionTime.trim().length > 0 ? completionTime : null; + const conditionStatus = status.trim().length > 0 ? status : null; + return { + name, + sourceCommit: sourceCommit || null, + status: conditionStatus, + reason: reason || null, + createdAt: createdAt || null, + startTime: startTime || null, + completionTime: completion, + durationSeconds: secondsBetween(startTime, completion), + elapsedSeconds: startMs === null || completion !== null ? null : Math.max(0, Math.round((nowMs - startMs) / 1000)), + active: completion === null && conditionStatus !== "True" && conditionStatus !== "False", + }; +} + +function parsePipelineRunRows(text: string, limit: number): Record { + const nowMs = Date.now(); + const rows = text + .split(/\r?\n/u) + .map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs)) + .filter((item): item is Record => item !== null) + .sort((left, right) => { + return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0); + }); + const activeItems = rows.filter((item) => item.active === true); + return { + ok: true, + count: rows.length, + activeCount: activeItems.length, + items: rows.slice(0, limit), + activeItems: activeItems.slice(0, limit), + disclosure: rows.length > limit ? `showing newest ${limit} of ${rows.length}` : "complete", + }; +} + +function pipelineRunRowsJsonPath(): string { + return [ + "jsonpath={range .items[*]}", + "{.metadata.name}", + "{\"\\t\"}", + "{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}", + "{\"\\t\"}", + "{.metadata.creationTimestamp}", + "{\"\\t\"}", + "{.status.startTime}", + "{\"\\t\"}", + "{.status.completionTime}", + "{\"\\t\"}", + "{.status.conditions[0].status}", + "{\"\\t\"}", + "{.status.conditions[0].reason}", + "{\"\\n\"}", + "{end}", + ].join(""); +} + +function listV02PipelineRunsCompact(limit = 8): Record { + const result = g14K3s([ + "kubectl", + "get", + "pipelinerun", + "-n", + CI_NAMESPACE, + "-l", + "hwlab.pikastech.local/gitops-target=v02", + "-o", + pipelineRunRowsJsonPath(), + ], 60_000); + if (!isCommandSuccess(result)) { + return { + ok: false, + command: redactedCommand(result.command), + exitCode: result.exitCode, + stderr: commandErrorSummary(result), + items: [], + activeItems: [], + }; + } + return parsePipelineRunRows(statusText(result), limit); +} + +function parseShellSections(output: string): Record { + const sections: Record = {}; + let currentName: string | null = null; + let currentLines: string[] = []; + let currentExitCode: number | null = null; + const flush = (): void => { + if (currentName === null) return; + sections[currentName] = { + stdout: currentLines.join("\n").trim(), + exitCode: currentExitCode, + }; + }; + for (const line of output.split(/\r?\n/u)) { + const begin = /^__UNIDESK_SECTION_BEGIN__ ([A-Za-z0-9_-]+)$/u.exec(line); + if (begin !== null) { + flush(); + currentName = begin[1] ?? null; + currentLines = []; + currentExitCode = null; + continue; + } + const end = /^__UNIDESK_SECTION_END__ ([A-Za-z0-9_-]+) exit=([0-9]+)$/u.exec(line); + if (end !== null && currentName === end[1]) { + currentExitCode = Number(end[2]); + flush(); + currentName = null; + currentLines = []; + currentExitCode = null; + continue; + } + if (currentName !== null) currentLines.push(line); + } + flush(); + return sections; +} + +function shellSectionOk(section: ShellSection | undefined): boolean { + return section?.exitCode === 0; +} + +function v02ControlPlaneStatusBundle(pipelineRun: string | null): CommandJsonResult { + const script = [ + "set +e", + "section() {", + " name=\"$1\"", + " shift", + " printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"", + " \"$@\"", + " code=$?", + " printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"", + "}", + `section controlPlane kubectl get pipeline,role,rolebinding,serviceaccount -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o name`, + `section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`, + `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`, + pipelineRun === null + ? "section pipelineRun sh -c 'true'" + : `section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`, + `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`, + ].join("\n"); + return g14K3s(["script", "--", script], 60_000); +} + +function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8): Record { + if (!commandOk) { + return { + ok: false, + command: Array.isArray(command) ? redactedCommand(command) : command, + exitCode, + stderr: stderr.trim().slice(0, 4000), + items: [], + activeItems: [], + }; + } + return parsePipelineRunRows(text, limit); +} + function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] { if (lane === "v02") return ["hwlab-v02-ci-poll-"]; if (lane === "g14") return ["hwlab-g14-ci-poll-"]; @@ -774,8 +1067,8 @@ function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSecond sourceCommit, renderDir: v02RenderDir(sourceCommit), render: commandData(render), - apply, - cleanupObsoleteCronJobs, + apply: compactCommandResult(apply), + cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs), degradedReason: !isCommandSuccess(apply) ? "control-plane-apply-failed" : cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs) @@ -884,36 +1177,29 @@ function deleteV02PipelineRun(pipelineRun: string): CommandJsonResult { return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, pipelineRun, "--ignore-not-found=true"], 60_000); } -function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Record { +function v02ControlPlaneStatus(sourceCommit: string | null = getV02CachedHead()): Record { const pipelineRun = sourceCommit === null ? null : v02PipelineRunName(sourceCommit); - const controlPlane = g14K3s([ - "kubectl", - "get", - "pipeline,role,rolebinding,serviceaccount", - "-n", - CI_NAMESPACE, - "-l", - "hwlab.pikastech.local/gitops-target=v02", - "-o", - "name", - ], 60_000); - const obsoleteCronJobs = getV02ObsoleteCronJobs(); - const argo = g14K3s([ - "kubectl", - "get", - "application", - "-n", - ARGO_NAMESPACE, - V02_APP, - "-o", - "jsonpath={.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}", - ], 60_000); - const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = statusText(argo).split(/\r?\n/u); + const bundle = v02ControlPlaneStatusBundle(pipelineRun); + const sections = parseShellSections(statusText(bundle)); + const controlPlane = sections.controlPlane; + const obsoleteCronJobs = sections.obsoleteCronJobs; + const argo = sections.argo; + const pipelineRunSection = sections.pipelineRun; + const recentPipelineRuns = listV02PipelineRunsCompactFromText( + sections.recentPipelineRuns?.stdout ?? "", + shellSectionOk(sections.recentPipelineRuns), + "kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=", + sections.recentPipelineRuns?.exitCode ?? null, + bundle.stderr, + ); + const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : []; + const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u); return { - ok: sourceCommit !== null && isCommandSuccess(controlPlane) && isCommandSuccess(argo), + ok: sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo), command: "hwlab g14 control-plane status --lane v02", lane: "v02", sourceCommit, + sourceCommitSource: "cached origin/v0.2; write operations fetch before acting", expected: { workspace: V02_WORKSPACE, branch: V02_SOURCE_BRANCH, @@ -924,29 +1210,49 @@ function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Reco argoApplication: V02_APP, }, controlPlane: { - ok: isCommandSuccess(controlPlane), - names: statusText(controlPlane).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), - stderr: controlPlane.stderr.trim().slice(0, 2000), + ok: shellSectionOk(controlPlane), + names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), + exitCode: controlPlane?.exitCode ?? null, }, obsoleteCronJobs: { - ok: isCommandSuccess(obsoleteCronJobs), - names: statusText(obsoleteCronJobs).split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), + ok: shellSectionOk(obsoleteCronJobs), + names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), + exitCode: obsoleteCronJobs?.exitCode ?? null, cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", }, argo: { - ok: isCommandSuccess(argo), - raw: statusText(argo), + ok: shellSectionOk(argo), + raw: argo?.stdout ?? "", fields: { targetRevision, path, syncRevision, syncStatus, health }, - stderr: argo.stderr.trim().slice(0, 2000), + exitCode: argo?.exitCode ?? null, }, - pipelineRun: pipelineRun === null ? null : getPipelineRunCompact(pipelineRun), + pipelineRun: pipelineRun === null + ? null + : pipelineRunCompactFromText( + pipelineRun, + pipelineRunSection?.stdout ?? "", + shellSectionOk(pipelineRunSection), + `kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`, + pipelineRunSection?.exitCode ?? null, + bundle.stderr, + ), + activePipelineRuns, + recentPipelineRuns, + query: { + ok: isCommandSuccess(bundle), + exitCode: bundle.exitCode, + stderr: bundle.stderr.trim().slice(0, 2000), + }, + visibilityHint: activePipelineRuns.length > 0 + ? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger" + : "no active v02 PipelineRun observed", }; } function runV02ControlPlane(options: G14ControlPlaneOptions): Record { if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options); - const sourceCommit = getV02Head(); + const sourceCommit = options.action === "status" ? getV02CachedHead() : getV02Head(); if (sourceCommit === null) { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE }; } @@ -960,7 +1266,7 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record): Record namespace: GIT_MIRROR_NAMESPACE, jobName, manifest: options.dryRun ? manifest : undefined, - result, + result: compactCommandResult(result), status: options.dryRun ? undefined : runGitMirrorStatus(), next: options.dryRun ? { sync: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }, }; @@ -1512,7 +1820,7 @@ function runGitMirrorFlush(options: G14GitMirrorOptions): Record function startAsyncHwlabG14Job(name: string, command: string[], note: string): Record { const job = startJob(name, command, note); - const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`; + const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`; return { ok: true, mode: "async-job",