fix(cicd): bound node trigger wait output (#646)

Co-authored-by: Codex <codex@noreply.local>
2026-06-22 11:11:32 +08:00
parent 77b467a72b
commit e6e15057b9
3 changed files with 141 additions and 17 deletions
@@ -92,12 +92,15 @@ bun scripts/cli.ts hwlab nodes control-plane infra runtime-image logs --node D60
 bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03
 bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm
 bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run <name>|--source-commit <sha>] [--full|--raw]
+bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait
 ```

 从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。

 `hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。

+`hwlab nodes control-plane trigger-current --node <node> --lane <lane> --confirm --wait` 是 node/lane CI/CD 一键入口：按 YAML 解析 source head，执行 git-mirror pre-sync/pre-flush，刷新 control-plane，创建或复用 commit-pinned PipelineRun，等待 PipelineRun 终态，并在终态成功后执行 post-flush。默认输出必须是低噪声 CICD 表格摘要；完整 JSON 只能通过 `--full` 或 `--raw` 展开。`--wait` 默认最多等待 120 秒；超过 120 秒时 CLI 返回 `pending` warning，并直接给出 env-reuse 检查命令 `control-plane status --full` 和 git-mirror 检查命令 `git-mirror status`，不继续长时间阻塞，也不把仍在运行误报为构建失败。
+
 ### G14 v0.3 runtime base image

 ```bash
@@ -30,7 +30,8 @@ export function hwlabNodeHelp(): Record<string, unknown> {
      "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --dry-run",
      "bun scripts/cli.ts hwlab nodes control-plane public-exposure --node D601 --lane v03 --confirm",
      "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm",
-      "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --rerun",
+      "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait",
+      "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait --rerun",
      "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --dry-run",
      "bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node D601 --lane v03 --confirm",
      "bun scripts/cli.ts hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 --confirm",
@@ -58,6 +59,8 @@ export function hwlabNodeHelp(): Record<string, unknown> {
    ],
    notes: [
      "`trigger-current` skips an already succeeded PipelineRun for the same HWLAB source commit by default.",
+      "`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.",
+      "`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.",
      "Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not."
    ],
  };
@@ -289,6 +289,7 @@ const CODE_AGENT_PROVIDER_OPENCODE_KEY = "opencode-api-key";
 const CODE_AGENT_PROVIDER_SOURCE_NAMESPACE = "hwlab-v02";
 const CODE_AGENT_PROVIDER_SOURCE_SECRET = "hwlab-v02-code-agent-provider";
 const HWLAB_CI_NAMESPACE = "hwlab-ci";
+const NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS = 120;

 export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
  if (args.length === 0) return hwlabNodeHelp();
@@ -352,7 +353,7 @@ function parseNodeObservabilityOptions(args: string[]): NodeObservabilityOptions
  };
 }

-async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown>> {
+async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
  const scoped = parseNodeScopedDelegatedOptions(domain, args);
  const defaultSpec = hwlabRuntimeLaneSpec(scoped.lane);
  if (domain === "control-plane" && scoped.action === "public-exposure") {
@@ -439,7 +440,7 @@ function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: stri
    wait: args.includes("--wait"),
    rerun: args.includes("--rerun"),
    allowLiveDbRead: args.includes("--allow-live-db-read"),
-    timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600),
+    timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, 3600),
    originalArgs: [...args],
    spec,
  };
@@ -1825,6 +1826,21 @@ function shortSha(value: string): string {
  return value.slice(0, 12).toLowerCase();
 }

+function shortValue(value: unknown): string {
+  if (typeof value !== "string" || value.length === 0) return "-";
+  if (/^[0-9a-f]{40}$/iu.test(value)) return shortSha(value);
+  return value.length > 30 ? `${value.slice(0, 27)}~` : value;
+}
+
+function formatElapsedMs(value: unknown): string {
+  const ms = Number(value);
+  if (!Number.isFinite(ms) || ms < 0) return "-";
+  const seconds = Math.round(ms / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const rest = seconds % 60;
+  return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`;
+}
+
 function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
  return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
 }
@@ -1891,11 +1907,11 @@ function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string {
  ].join("\n");
 }

-function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
+function nodeRuntimeControlPlaneRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
  if (scoped.action === "refresh") return nodeRuntimeRefresh(scoped);
  if (scoped.action === "sync") return nodeRuntimeSync(scoped);
  if (scoped.action === "apply") return nodeRuntimeApply(scoped);
-  if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrent(scoped);
+  if (scoped.action === "trigger-current") return nodeRuntimeTriggerCurrentOutput(scoped);
  if (scoped.action === "runtime-migration") return nodeRuntimeMigration(scoped);
  if (scoped.action === "cleanup-runs") return nodeRuntimeCleanupRuns(scoped);
  return nodeRuntimeUnsupportedAction(scoped);
@@ -2635,8 +2651,15 @@ function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptio
  };
 }

+function nodeRuntimeTriggerCurrentOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> | RenderedCliResult {
+  const result = nodeRuntimeTriggerCurrent(scoped);
+  if (nodeScopedFullOutput(scoped)) return result;
+  return withNodeRuntimeTriggerRendered(result, scoped);
+}
+
 function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const spec = scoped.spec;
+  const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
  printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "started" });
  const head = resolveNodeRuntimeLaneHead(spec);
  const sourceCommit = head.sourceCommit;
@@ -2678,7 +2701,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
  }
  if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
    const pipelineWait = before.status === "Unknown"
-      ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds, {
+      ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
        opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
        opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
      })
@@ -2688,6 +2711,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
    const postFlush = waitedPipelineRun.status === "True"
      ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
      : null;
+    const pipelinePending = pipelineWait.status === "pending";
    const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true);
    return {
      ok,
@@ -2695,6 +2719,8 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
      node: scoped.node,
      lane: scoped.lane,
      mode: "confirmed-trigger",
+      completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
+      warning: pipelinePending ? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : undefined,
      sourceCommit,
      pipelineRun,
      before,
@@ -2770,7 +2796,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
  const createOk = isCommandSuccess(create) || createObserved;
  printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
  const pipelineWait = createOk
-    ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds, {
+    ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
      opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
      opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
    })
@@ -2782,13 +2808,16 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
    : null;
  const pipelineReady = pipelineWait !== null && pipelineWait.ok === true;
  const postFlushOk = postFlush === null || postFlush.ok === true;
-  const ok = createOk && pipelineReady && postFlushOk;
+  const pipelinePending = pipelineWait !== null && pipelineWait.status === "pending";
+  const ok = createOk && (pipelineReady || pipelinePending) && postFlushOk;
  return {
    ok,
    command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
    node: scoped.node,
    lane: scoped.lane,
    mode: "confirmed-trigger",
+    completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
+    warning: pipelinePending ? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : undefined,
    mutation: createOk,
    sourceCommit,
    pipelineRun,
@@ -2814,6 +2843,95 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
  };
 }

+function nodeRuntimeCicdWaitSeconds(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): number {
+  return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS);
+}
+
+function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record<string, unknown> {
+  const wait = record(pipelineWait);
+  return {
+    code: "node-runtime-cicd-wait-over-120s",
+    message: `PipelineRun ${pipelineRun} is still active after ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; CICD command stopped waiting and left the deployment running.`,
+    waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
+    elapsedMs: wait.elapsedMs ?? null,
+    polls: wait.polls ?? null,
+    inspectEnvReuse: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
+    inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
+  };
+}
+
+function withNodeRuntimeTriggerRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
+  const pipelineWait = record(result.pipelineWait);
+  const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before);
+  const warning = record(result.warning ?? pipelineWait.warning);
+  const postFlush = record(result.postFlush);
+  const gitMirror = record(result.gitMirror);
+  const gitMirrorSummary = record(gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
+  const refresh = record(result.refresh);
+  const create = record(result.create);
+  const next = record(result.next);
+  const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-";
+  const completion = result.completion ?? (result.ok === true ? "completed" : "failed");
+  const renderedText = [
+    "hwlab nodes control-plane trigger-current",
+    "",
+    webObserveTable(
+      ["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"],
+      [[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
+    ),
+    "",
+    webObserveTable(
+      ["STAGE", "STATUS", "DETAIL"],
+      [
+        ["git-mirror", gitMirror.ok === true ? "ok" : gitMirror.ok === false ? "failed" : "-", webObserveText(gitMirror.mode ?? gitMirror.degradedReason)],
+        ["refresh", refresh.ok === true ? "ok" : refresh.ok === false ? "failed" : "-", webObserveText(refresh.degradedReason)],
+        ["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`],
+        ["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`],
+        ["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)],
+      ],
+    ),
+    "",
+    Object.keys(warning).length === 0
+      ? "WARNING\n-"
+      : webObserveTable(
+        ["CODE", "MESSAGE"],
+        [[webObserveText(warning.code ?? "warning"), webObserveText(warning.message ?? "PipelineRun is still active after the default wait window.")]],
+      ),
+    "",
+    Object.keys(warning).length === 0
+      ? "INSPECT\n-"
+      : webObserveTable(
+        ["CHECK", "COMMAND"],
+        [
+          ["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
+          ["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)],
+        ],
+      ),
+    "",
+    Object.keys(gitMirrorSummary).length === 0
+      ? "GIT_MIRROR\n-"
+      : webObserveTable(
+        ["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
+        [[
+          shortValue(gitMirrorSummary.localSource),
+          shortValue(gitMirrorSummary.githubSource),
+          shortValue(gitMirrorSummary.localGitops),
+          shortValue(gitMirrorSummary.githubGitops),
+          webObserveText(gitMirrorSummary.pendingFlush),
+          webObserveText(gitMirrorSummary.githubInSync),
+        ]],
+      ),
+    "",
+    "NEXT",
+    `  ${next.status ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun}`}`,
+    "",
+    "Disclosure:",
+    "  default view is a bounded CICD summary; use --full or --raw for the complete JSON payload.",
+    `  default wait is ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s; if exceeded, inspect env-reuse and git-mirror before rerun.`,
+  ].join("\n");
+  return { ...result, renderedText, contentType: "text/plain" };
+}
+
 function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
  const spec = scoped.spec;
  const mirror = nodeRuntimeGitMirrorTarget(spec);
@@ -5802,20 +5920,20 @@ function waitForNodeRuntimePipelineRunTerminal(
    }
    sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
  }
-  printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt });
-  const diagnostics = nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
-  const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
+  const elapsedMs = Date.now() - startedAt;
+  const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
+  printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
  return {
-    ok: false,
-    status: "timeout",
+    ok: true,
+    status: "pending",
    pipelineRun: last,
-    diagnostics,
-    failureSummary,
    polls,
-    elapsedMs: Date.now() - startedAt,
+    elapsedMs,
+    waitLimitSeconds: timeoutSeconds,
+    warning,
    opportunisticPostSyncs: opportunisticPostSyncs.length > 0 ? opportunisticPostSyncs : undefined,
    opportunisticPostFlushes: opportunisticPostFlushes.length > 0 ? opportunisticPostFlushes : undefined,
-    degradedReason: "node-runtime-pipelinerun-wait-timeout",
+    next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
  };
 }