From e6e15057b9df676e3d6819564650e6b06ead5928 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:11:32 +0800 Subject: [PATCH] fix(cicd): bound node trigger wait output (#646) Co-authored-by: Codex --- .agents/skills/unidesk-cicd/SKILL.md | 3 + scripts/src/hwlab-node-help.ts | 5 +- scripts/src/hwlab-node-impl.ts | 150 ++++++++++++++++++++++++--- 3 files changed, 141 insertions(+), 17 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 670cdd1f..d26c12d8 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -92,12 +92,15 @@ bun scripts/cli.ts hwlab nodes control-plane infra runtime-image logs --node D60 bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run |--source-commit ] [--full|--raw] +bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait ``` 从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。 `hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。 +`hwlab nodes control-plane trigger-current --node --lane --confirm --wait` 是 node/lane CI/CD 一键入口：按 YAML 解析 source head，执行 git-mirror pre-sync/pre-flush，刷新 control-plane，创建或复用 commit-pinned PipelineRun，等待 PipelineRun 终态，并在终态成功后执行 post-flush。默认输出必须是低噪声 CICD 表格摘要；完整 JSON 只能通过 `--full` 或 `--raw` 展开。`--wait` 默认最多等待 120 秒；超过 120 秒时 CLI 返回 `pending` warning，并直接给出 env-reuse 检查命令 `control-plane status --full` 和 git-mirror 检查命令 `git-mirror status`，不继续长时间阻塞，也不把仍在运行误报为构建失败。 + ### G14 v0.3 runtime base image ```bash diff --git a/scripts/src/hwlab-node-help.ts b/scripts/src/hwlab-node-help.ts index 6dc3fde9..5196bf12 100644 --- a/scripts/src/hwlab-node-help.ts +++ b/scripts/src/hwlab-node-help.ts @@ -30,7 +30,8 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", - "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --rerun", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait --rerun", "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm", @@ -58,6 +59,8 @@ export function hwlabNodeHelp(): Record { ], notes: [ "`trigger-current` skips an already succeeded PipelineRun for the same HWLAB source commit by default.", + "`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.", + "`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.", "Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not." ], }; diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 3d38fb15..b4e66c54 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -289,6 +289,7 @@ const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key"; const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02"; const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider"; const HWLAB_CI_NAMESPACE = "hwlab-ci"; +const NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS = 120; export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise | RenderedCliResult> { if (args.length === 0) return hwlabNodeHelp(); @@ -352,7 +353,7 @@ function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions }; } -async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { +async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise | RenderedCliResult> { const scoped = parseNodeScopedDelegatedOptions(domain, args); const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane); if (domain === "control-plane" && scoped.action === "public-exposure") { @@ -439,7 +440,7 @@ function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: stri wait: args.includes("--wait"), rerun: args.includes("--rerun"), allowLiveDbRead: args.includes("--allow-live-db-read"), - timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600), + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, 3600), originalArgs: [...args], spec, }; @@ -1825,6 +1826,21 @@ function shortSha(value: string): string { return value.slice(0, 12).toLowerCase(); } +function shortValue(value: unknown): string { + if (typeof value !== "string" || value.length === 0) return "-"; + if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value); + return value.length > 30 ? `${value.slice(0, 27)}~` : value; +} + +function formatElapsedMs(value: unknown): string { + const ms = Number(value); + if (!Number.isFinite(ms) || ms < 0) return "-"; + const seconds = Math.round(ms / 1000); + const minutes = Math.floor(seconds / 60); + const rest = seconds % 60; + return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`; +} + function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; } @@ -1891,11 +1907,11 @@ function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { ].join("\n"); } -function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record { +function nodeRuntimeControlPlaneRun(scoped: ReturnType): Record | RenderedCliResult { if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped); if (scoped.action === "sync") return nodeRuntimeSync(scoped); if (scoped.action === "apply") return nodeRuntimeApply(scoped); - if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped); + if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped); if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped); if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped); return nodeRuntimeUnsupportedAction(scoped); @@ -2635,8 +2651,15 @@ function nodeRuntimeSync(scoped: ReturnType): Record | RenderedCliResult { + const result = nodeRuntimeTriggerCurrent(scoped); + if (nodeScopedFullOutput(scoped)) return result; + return withNodeRuntimeTriggerRendered(result, scoped); +} + function nodeRuntimeTriggerCurrent(scoped: ReturnType): Record { const spec = scoped.spec; + const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped); printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" }); const head = resolveNodeRuntimeLaneHead(spec); const sourceCommit = head.sourceCommit; @@ -2678,7 +2701,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun), opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun), }) @@ -2688,6 +2711,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun), opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun), }) @@ -2782,13 +2808,16 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType): number { + return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS); +} + +function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record { + const wait = record(pipelineWait); + return { + code: "node-runtime-cicd-wait-over-120s", + message: `PipelineRun ${pipelineRun} is still active after ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; CICD command stopped waiting and left the deployment running.`, + waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, + elapsedMs: wait.elapsedMs ?? null, + polls: wait.polls ?? null, + inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, + inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, + }; +} + +function withNodeRuntimeTriggerRendered(result: Record, scoped: ReturnType): RenderedCliResult { + const pipelineWait = record(result.pipelineWait); + const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before); + const warning = record(result.warning ?? pipelineWait.warning); + const postFlush = record(result.postFlush); + const gitMirror = record(result.gitMirror); + const gitMirrorSummary = record(gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary); + const refresh = record(result.refresh); + const create = record(result.create); + const next = record(result.next); + const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-"; + const completion = result.completion ?? (result.ok === true ? "completed" : "failed"); + const renderedText = [ + "hwlab nodes control-plane trigger-current", + "", + webObserveTable( + ["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"], + [[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]], + ), + "", + webObserveTable( + ["STAGE", "STATUS", "DETAIL"], + [ + ["git-mirror", gitMirror.ok === true ? "ok" : gitMirror.ok === false ? "failed" : "-", webObserveText(gitMirror.mode ?? gitMirror.degradedReason)], + ["refresh", refresh.ok === true ? "ok" : refresh.ok === false ? "failed" : "-", webObserveText(refresh.degradedReason)], + ["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`], + ["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`], + ["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)], + ], + ), + "", + Object.keys(warning).length === 0 + ? "WARNING\n-" + : webObserveTable( + ["CODE", "MESSAGE"], + [[webObserveText(warning.code ?? "warning"), webObserveText(warning.message ?? "PipelineRun is still active after the default wait window.")]], + ), + "", + Object.keys(warning).length === 0 + ? "INSPECT\n-" + : webObserveTable( + ["CHECK", "COMMAND"], + [ + ["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)], + ["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)], + ], + ), + "", + Object.keys(gitMirrorSummary).length === 0 + ? "GIT_MIRROR\n-" + : webObserveTable( + ["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"], + [[ + shortValue(gitMirrorSummary.localSource), + shortValue(gitMirrorSummary.githubSource), + shortValue(gitMirrorSummary.localGitops), + shortValue(gitMirrorSummary.githubGitops), + webObserveText(gitMirrorSummary.pendingFlush), + webObserveText(gitMirrorSummary.githubInSync), + ]], + ), + "", + "NEXT", + ` ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun}`}`, + "", + "Disclosure:", + " default view is a bounded CICD summary; use --full or --raw for the complete JSON payload.", + ` default wait is ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; if exceeded, inspect env-reuse and git-mirror before rerun.`, + ].join("\n"); + return { ...result, renderedText, contentType: "text/plain" }; +} + function nodeRuntimeGitMirrorStatus(scoped: ReturnType): Record { const spec = scoped.spec; const mirror = nodeRuntimeGitMirrorTarget(spec); @@ -5802,20 +5920,20 @@ function waitForNodeRuntimePipelineRunTerminal( } sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now()))); } - printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt }); - const diagnostics = nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); - const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics); + const elapsedMs = Date.now() - startedAt; + const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs }); + printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds }); return { - ok: false, - status: "timeout", + ok: true, + status: "pending", pipelineRun: last, - diagnostics, - failureSummary, polls, - elapsedMs: Date.now() - startedAt, + elapsedMs, + waitLimitSeconds: timeoutSeconds, + warning, opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined, opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined, - degradedReason: "node-runtime-pipelinerun-wait-timeout", + next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` }, }; }