diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d0a42700..ee27bbf1 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -58,7 +58,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P `--dry-run` 只报告是否会 pre-sync，不创建 Job；confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径，避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞；`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件，覆盖 `control-plane-refresh`、`git-mirror-pre-sync` 和 `create-pipelinerun`，避免异步 job 长时间只有启动命令而无法判断卡点；默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要，保留长度与 hash，最终 trigger 结果只返回阶段摘要和关键 tail，完整内容通过 job stdout/stderr 文件渐进披露；只有现场同步调试才显式加 `--wait`；旧 `rerun-current` 只作为输入别名保留。PipelineRun `Completed`、Argo `Synced/Healthy` 和 `webAssets.ok=true` 只证明 G14 runtime 已更新；交付收口还必须用 `hwlab g14 git-mirror status` 查看 `cache.summary.pendingFlush`，若为 true，继续执行受控 `hwlab g14 git-mirror flush --confirm` 并用 job status 轮询到 `pendingFlush=false`。 - `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行；不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`。 - `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]` 和 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-openfga|hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的标准入口：OpenFGA preset 确保 preshared token、datastore URI 和 Postgres password 存在；v02 继续确保独立 `hwlab_openfga` 数据库/角色存在，v03+ 默认从 lane spec 派生 namespace、Postgres Secret 和主 DB 用户，避免复制 v02 专用硬编码。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`，并同步到对应 lane 的 `*-master-server-admin-api-key/api-key`。`status` 只返回 key 是否存在、解码后字节数、key prefix 和 DB 对象存在性，永远不读取、不打印、不回传 secret 明文。`hwlab g14 secret delete --lane v02 --name [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret，默认 dry-run，拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret；共享 device-pod API key 已退出当前授权路径，不再提供 ensure/bootstrap 入口。 -- `hwlab g14 control-plane cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口；真实清理只删除已完成 PipelineRun，让 Tekton/local-path 回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。 +- `hwlab g14 control-plane cleanup-runs --lane v02|v03|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口；真实清理只删除已完成 PipelineRun，让 Tekton/local-path 回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。`hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run ` 是 v0.3 failed run 受控重试前的清理入口。 - `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口；只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。 - `hwlab g14 git-mirror status|apply|sync|flush [--dry-run|--confirm]` 是 `devops-infra` git mirror/relay 的受控维护入口：`apply` 渲染并 server-side apply `devops-infra/git-mirror.yaml`，同时删除遗留 `git-mirror-hwlab-sync` CronJob；`sync` 创建一次性 manual Job，把 GitHub allowlist refs 拉入本地 mirror；`flush` 创建一次性 manual Job，把本地 `v0.2-gitops` 快进推回 GitHub。 `status` 返回 read/write URL、last sync/write/flush、本地 ref、GitHub staging ref 和 pending flush 状态，并在 `cache.summary` 给出 `localV02`、`localGitops`、`githubGitops`、`pendingFlush`、`flushNeeded`、`githubInSync` 和下一条受控 `flushCommand`。confirmed `sync` 和 `flush` 默认创建 `.state/jobs/` 异步 job 并立刻返回可查询状态，只有现场同步调试才显式加 `--wait`；mirror 不设置 CronJob。 diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index d92b9ac8..48148c78 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -334,6 +334,7 @@ assertCondition(gitMirrorSummary.v03SourceInSync === true && gitMirrorSummary.v0 const renderScript = v02ControlPlaneRenderScript("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); const renderScriptHash = v02ControlPlaneRefreshScriptHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); const sourceText = await Bun.file(new URL("./src/hwlab-g14.ts", import.meta.url)).text(); +const jobsSourceText = await Bun.file(new URL("./src/jobs.ts", import.meta.url)).text(); assertCondition( renderScript.includes("git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"") && renderScript.includes("git -C \"$worktree_dir\" checkout --detach \"$source_commit\"") @@ -475,6 +476,17 @@ assertCondition( && sourceText.includes("protected-latest-pipelinerun"), "control-plane cleanup-runs must protect the latest PipelineRun per lane by default", ); +assertCondition( + hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run")) + && hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit")) + && sourceText.includes("control-plane cleanup-runs requires --lane v02|v03|g14|all") + && sourceText.includes("printRuntimeLaneTriggerProgress") + && sourceText.includes("hwlab.runtime-lane.trigger.progress") + && jobsSourceText.includes("hwlab-runtime-lane-trigger") + && jobsSourceText.includes("hwlab.runtime-lane.trigger.progress") + && jobsSourceText.includes("hwlab nodes control-plane status --node"), + "v0.3 runtime lane retry and trigger visibility must be represented by controlled cleanup/help/progress paths", +); const staleSuccessAlignment = v02CommitAlignment({ expectedSourceHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index c09d13de..d021b3ba 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -361,12 +361,12 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { actionRaw !== "cleanup-released-pvs" && actionRaw !== "runtime-migration" ) { - throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); + throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|v03|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); } const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02"); let lane: G14ControlPlaneOptions["lane"]; if (actionRaw === "cleanup-runs") { - if (laneRaw !== "v02" && laneRaw !== "g14" && laneRaw !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all"); + if (laneRaw !== "g14" && laneRaw !== "all" && !isHwlabRuntimeLane(laneRaw)) throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all"); lane = laneRaw; } else if (actionRaw === "cleanup-released-pvs") { if (laneRaw !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane"); @@ -918,6 +918,14 @@ function printV02PrMonitorProgress(data: Record = {}): void { printProgressEvent("hwlab.v02.pr-monitor.progress", data); } +function printRuntimeLaneTriggerProgress(spec: HwlabRuntimeLaneSpec, data: Record = {}): void { + printProgressEvent("hwlab.runtime-lane.trigger.progress", { + lane: spec.lane, + node: spec.nodeId, + ...data, + }); +} + function shortSha(sha: string): string { return sha.slice(0, 12); } @@ -2560,10 +2568,10 @@ function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, co return parsePipelineRunRows(text, limit, nowMs); } -function pipelinePrefixesForLane(lane: "v02" | "g14" | "all"): string[] { - if (lane === "v02") return ["hwlab-v02-ci-poll-"]; +function pipelinePrefixesForLane(lane: G14ControlPlaneOptions["lane"]): string[] { + if (isHwlabRuntimeLane(lane)) return [`${hwlabRuntimeLaneSpec(lane).pipelineRunPrefix}-`]; if (lane === "g14") return ["hwlab-g14-ci-poll-"]; - return ["hwlab-v02-ci-poll-", "hwlab-g14-ci-poll-"]; + return [...hwlabRuntimeLaneIds().map((runtimeLane) => `${hwlabRuntimeLaneSpec(runtimeLane).pipelineRunPrefix}-`), "hwlab-g14-ci-poll-"]; } function commandErrorSummary(result: CommandJsonResult): string { @@ -2579,8 +2587,8 @@ function tailText(value: unknown, maxBytes = 2000): string { } function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record[] { - if (options.lane !== "v02" && options.lane !== "g14" && options.lane !== "all") { - throw new Error("control-plane cleanup-runs requires --lane v02|g14|all"); + if (options.lane !== "g14" && options.lane !== "all" && !isHwlabRuntimeLane(options.lane)) { + throw new Error("control-plane cleanup-runs requires --lane v02|v03|g14|all"); } const result = g14K3s([ "kubectl", @@ -2594,7 +2602,11 @@ function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record item.selected !== false) .map((item) => String(item.name)); const ownedPvcs = listOwnedWorkspacePvcs(candidateNames); + const followUpStatusLane = isHwlabRuntimeLane(options.lane) ? options.lane : "v02"; if (options.dryRun) { return { ok: true, @@ -2875,7 +2888,7 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record { - if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-runs" || options.action === "cleanup-released-pvs") { + if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); + if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-released-pvs") { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`, lane: spec.lane, degradedReason: "unsupported-runtime-lane-action", - message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh`, + message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh/cleanup-runs`, }; } if (options.action === "status" && options.pipelineRun !== undefined) { @@ -4000,6 +4014,12 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --pipeline-run ${pipelineRun}` }, }; } + printRuntimeLaneTriggerProgress(spec, { + stage: "git-mirror-sync", + status: "started", + sourceCommit, + pipelineRun, + }); const gitMirrorSync = runGitMirrorSync({ action: "sync", lane: spec.lane, @@ -4008,6 +4028,14 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont wait: true, timeoutSeconds: options.timeoutSeconds, }); + printRuntimeLaneTriggerProgress(spec, { + stage: "git-mirror-sync", + status: gitMirrorSync.ok === true ? "succeeded" : "failed", + sourceCommit, + pipelineRun, + jobName: gitMirrorSync.jobName ?? null, + elapsedMs: gitMirrorSync.elapsedMs ?? null, + }); if (gitMirrorSync.ok !== true) { return { ok: false, @@ -4021,7 +4049,21 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont degradedReason: "git-mirror-sync-before-trigger-failed", }; } + printRuntimeLaneTriggerProgress(spec, { + stage: "source-render", + status: "started", + sourceCommit, + pipelineRun, + }); const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); + printRuntimeLaneTriggerProgress(spec, { + stage: "source-render", + status: isCommandSuccess(render.result) ? "succeeded" : "failed", + sourceCommit, + pipelineRun, + renderDir: render.renderDir, + exitCode: render.result.exitCode, + }); if (!isCommandSuccess(render.result)) { return { ok: false, @@ -4038,8 +4080,22 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont degradedReason: "control-plane-render-failed", }; } + printRuntimeLaneTriggerProgress(spec, { + stage: "control-plane-apply", + status: "started", + sourceCommit, + pipelineRun, + renderDir: render.renderDir, + }); const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, false, options.timeoutSeconds); const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null; + printRuntimeLaneTriggerProgress(spec, { + stage: "control-plane-apply", + status: isCommandSuccess(apply) ? "succeeded" : "failed", + sourceCommit, + pipelineRun, + exitCode: apply.exitCode, + }); if (!isCommandSuccess(apply)) { return { ok: false, @@ -4058,7 +4114,20 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont degradedReason: "control-plane-apply-before-trigger-failed", }; } + printRuntimeLaneTriggerProgress(spec, { + stage: "create-pipelinerun", + status: "started", + sourceCommit, + pipelineRun, + }); const create = createRuntimeLanePipelineRun(spec, sourceCommit, options.timeoutSeconds); + printRuntimeLaneTriggerProgress(spec, { + stage: "create-pipelinerun", + status: isCommandSuccess(create) ? "succeeded" : "failed", + sourceCommit, + pipelineRun, + exitCode: create.exitCode, + }); return { ok: isCommandSuccess(create), command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, @@ -8682,6 +8751,8 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll- --dry-run", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --source-commit --confirm", + "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run hwlab-v03-ci-poll- --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --source-commit --confirm", "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm", "bun scripts/cli.ts hwlab g14 control-plane runtime-migration --lane v02 --dry-run", diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index 38a6d4df..f6a2071b 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -25,7 +25,7 @@ export interface JobRecord { } export interface JobProgressSummary { - kind: "hwlab-v02-trigger" | "git-mirror" | "generic"; + kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "git-mirror" | "generic"; stage: string | null; stageStatus: string | null; sourceCommit: string | null; @@ -225,13 +225,15 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdoutTail: string; stderrTail: string }): JobProgressSummary { const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current"; const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor"; + const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name); const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush"; - if (!knownWorkflow && !v02PrMonitorWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail); + if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail); const nowMs = Date.now(); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs); + if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs); const events = v02PrMonitorWorkflow ? [ ...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"), @@ -355,6 +357,68 @@ function summarizeGitMirrorJobProgress(job: JobRecord, stdoutTail: string, stder }; } +function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { + const events = parseJsonLineEvents(stderrTail, "hwlab.runtime-lane.trigger.progress") + .sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? ""))); + const lastEvent = events.at(-1) ?? {}; + const stage = stringField(lastEvent.stage); + const stageStatus = stringField(lastEvent.status); + const sourceCommit = stringField(lastEvent.sourceCommit) ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); + const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); + const pipelineCreated = /pipelinerun\.tekton\.dev\/[^ \n]+ created/u.test(stdoutTail) + ? true + : stage === "create-pipelinerun" && stageStatus === "failed" + ? false + : null; + const lastEventAt = stringField(lastEvent.at); + const elapsedSeconds = jobElapsedSeconds(job, nowMs); + const stageElapsedSeconds = currentStageElapsedSeconds(events, stage, stageStatus, job, nowMs); + const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); + const timings = extractGitMirrorTimings(`${stderrTail}\n${stdoutTail}`); + const warnings = jobProgressWarnings({ + job, + eventsObserved: events.length, + elapsedSeconds, + stage, + stageStatus, + stageElapsedSeconds, + lastEventAgeSeconds, + }); + const slow = warnings.length > 0; + return { + kind: "hwlab-runtime-lane-trigger", + stage, + stageStatus, + sourceCommit, + pipelineRun, + pipelineCreated, + elapsedSeconds, + stageElapsedSeconds, + lastEventAt, + lastEventAgeSeconds, + eventsObserved: events.length, + slow, + warnings, + timings, + summary: [ + job.status, + stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", + sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, + pipelineRun ? `pipelineRun=${pipelineRun}` : null, + pipelineCreated === true ? "created" : pipelineCreated === false ? "create-failed" : null, + elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, + stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, + lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, + slow ? "visibility-warning" : null, + ].filter(Boolean).join(" "), + nextCommand: pipelineRun + ? `bun scripts/cli.ts hwlab nodes control-plane status --node ${stringField(lastEvent.node) ?? "G14"} --lane ${stringField(lastEvent.lane) ?? "v03"} --pipeline-run ${pipelineRun}` + : job.status === "running" + ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` + : null, + }; +} + function genericJobProgress(job: JobRecord, stderrTailOverride?: string): JobProgressSummary { const nowMs = Date.now(); const stderrTail = stderrTailOverride ?? tailFile(job.stderrFile, 96_000);