fix: compact hwlab node control plane status

2026-06-17 10:33:41 +00:00
parent 59b20254d4
commit 0836f0dfaf
3 changed files with 188 additions and 6 deletions
@@ -91,10 +91,13 @@ bun scripts/cli.ts hwlab nodes control-plane infra runtime-image preload --node
 bun scripts/cli.ts hwlab nodes control-plane infra runtime-image logs --node D601 --lane v03
 bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03
 bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm
+bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run <name>|--source-commit <sha>] [--full|--raw]
 ```

 从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。

+`hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。
+
 ### G14 v0.3 runtime base image

 ```bash
@@ -218,6 +218,7 @@ export function hwlabNodeHelp(): Record<string, unknown> {
      "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm",
      "bun scripts/cli.ts hwlab nodes control-plane plan --node D601 --lane v03",
      "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03",
+      "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 --full",
      "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03",
      "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run",
      "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm",
@@ -1582,14 +1583,21 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
    ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
    : null;
  const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
+  const workloadReadinessProbe = namespaceExists
+    ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60)
+    : null;
+  const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
  const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
  const secrets = externalPostgresSecretStatus(spec, namespaceExists);
+  const publicProbes = nodeRuntimePublicProbeStatus(spec);
  const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
-  const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready));
+  const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready);
+  const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && workloadsReady && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready));
  const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
  const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
-  return {
-    ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady,
+  const publicReady = publicProbes.ready === true;
+  const fullStatus = {
+    ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady,
    command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
    mode: "node-scoped-runtime-status",
    mutation: false,
@@ -1620,16 +1628,21 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
    },
    pipelineRun: pipelineRunProbe,
    runtime: {
+      ready: runtimeReady,
      namespace: spec.runtimeNamespace,
      namespaceExists,
      localPostgresObjects,
      localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0,
      workloadNames,
      workloadCount: workloadNames.length,
+      workloadReadiness,
+      workloadsReady,
+      workloadReadinessResult: workloadReadinessProbe === null ? null : compactRuntimeCommand(workloadReadinessProbe),
      workloadResult: workloads === null ? null : compactRuntimeCommand(workloads),
      externalPostgresBridge: bridge,
      externalPostgresSecrets: secrets,
    },
+    publicProbes,
    probes: {
      namespace: compactRuntimeCommand(namespace),
      postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects),
@@ -1637,7 +1650,9 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
    degradedReason: controlPlaneReady
      ? runtimeReady
        ? argoReady
-          ? pipelineRunReady ? undefined : "pipelinerun-not-succeeded"
+          ? pipelineRunReady
+            ? publicReady ? undefined : "public-probe-not-ready"
+            : "pipelinerun-not-succeeded"
          : "argo-not-synced-healthy"
        : namespaceExists ? "runtime-not-ready" : "runtime-namespace-missing"
      : "control-plane-not-ready",
@@ -1647,6 +1662,150 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
      triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
    },
  };
+  const summary = summarizeNodeRuntimeControlPlaneStatus(fullStatus, scoped);
+  if (scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw")) return { ...fullStatus, summary };
+  return summary;
+}
+
+function parseNodeRuntimeWorkloadReadiness(text: string): Array<Record<string, unknown>> {
+  return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
+    const [ref = "", counts = ""] = line.split(/\t/u);
+    const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/");
+    const readyReplicas = nullableInteger(readyRaw);
+    const currentReplicas = nullableInteger(currentRaw);
+    const desiredReplicas = nullableInteger(desiredRaw);
+    const ready = desiredReplicas !== null
+      ? (readyReplicas ?? 0) >= desiredReplicas
+      : readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas;
+    return {
+      ref,
+      readyReplicas,
+      currentReplicas,
+      desiredReplicas,
+      ready,
+    };
+  });
+}
+
+function nullableInteger(value: string): number | null {
+  if (!/^[0-9]+$/u.test(value)) return null;
+  return Number(value);
+}
+
+function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
+  const web = publicHttpProbe("web", spec.publicWebUrl);
+  const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
+  return {
+    ready: web.ok === true && apiHealth.ok === true,
+    web,
+    apiHealth,
+  };
+}
+
+function publicHttpProbe(kind: string, url: string): Record<string, unknown> {
+  const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
+  const httpStatus = nullableInteger(result.stdout.trim().slice(-3));
+  return {
+    kind,
+    url,
+    ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
+    httpStatus,
+    result: compactRuntimeCommand(result),
+  };
+}
+
+function joinUrlPath(baseUrl: string, suffix: string): string {
+  return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`;
+}
+
+function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
+  const pipelineRun = record(status.pipelineRun);
+  const argo = record(status.argo);
+  const runtime = record(status.runtime);
+  const publicProbes = record(status.publicProbes);
+  const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
+  const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length;
+  const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length;
+  const webProbe = record(publicProbes.web);
+  const apiProbe = record(publicProbes.apiHealth);
+  return {
+    ok: status.ok === true,
+    command: status.command,
+    mode: "node-scoped-runtime-status-summary",
+    mutation: false,
+    node: status.node,
+    lane: status.lane,
+    sourceCommit: status.sourceCommit,
+    degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
+    pipelineRun: {
+      name: pipelineRun.name ?? null,
+      exists: pipelineRun.exists === true,
+      status: pipelineRun.status ?? null,
+      reason: pipelineRun.reason ?? null,
+      message: pipelineRun.message ?? null,
+      createdAt: pipelineRun.createdAt ?? null,
+      ready: pipelineRun.status === "True",
+    },
+    argo: {
+      application: argo.application ?? null,
+      ready: argo.ready === true,
+      syncRevision: argo.syncRevision ?? null,
+      syncStatus: argo.syncStatus ?? null,
+      health: argo.health ?? null,
+    },
+    runtime: {
+      namespace: runtime.namespace ?? null,
+      namespaceExists: runtime.namespaceExists === true,
+      ready: runtime.ready === true,
+      workloadCount,
+      workloadReady: `${readyWorkloads}/${workloadReadiness.length}`,
+      localPostgresAbsent: runtime.localPostgresAbsent === true,
+      externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined
+        ? null
+        : record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true,
+    },
+    publicProbe: {
+      ready: publicProbes.ready === true,
+      web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
+      apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
+    },
+    nextAction: nodeRuntimeStatusNextAction(status, scoped),
+    next: {
+      full: `${nodeRuntimeStatusCommand(scoped)} --full`,
+      plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
+      triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
+      webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`,
+    },
+  };
+}
+
+function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
+  const reason = typeof status.degradedReason === "string" ? status.degradedReason : null;
+  if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`;
+  if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") {
+    return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`;
+  }
+  if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`;
+  if (reason === "argo-not-synced-healthy") {
+    return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`;
+  }
+  if (reason === "pipelinerun-not-succeeded") {
+    return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
+  }
+  if (reason === "public-probe-not-ready") {
+    return `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
+  }
+  return `${nodeRuntimeStatusCommand(scoped)} --full`;
+}
+
+function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
+  const sourceCommit = optionValue(scoped.originalArgs, "--source-commit");
+  const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run");
+  return [
+    `bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`,
+    sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`,
+    pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`,
+  ].filter(Boolean).join(" ");
 }

 function nodeRuntimeRenderToken(): string {
@@ -208,18 +208,22 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & {
  const progressTailBytes = Math.max(maxBytes, 96_000);
  const stdoutProgressTail = tailFile(job.stdoutFile, progressTailBytes);
  const stderrProgressTail = tailFile(job.stderrFile, progressTailBytes);
+  const progress = summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail });
+  const rawStdoutTail = tailTextByBytes(stdoutProgressTail, maxBytes);
+  const stdoutTail = compactJobStdoutTail(job, progress, rawStdoutTail, maxBytes);
  return {
    ...job,
-    progress: summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail }),
+    progress,
    tailPolicy: {
      requestedTailBytes: maxBytes,
      stdoutBytes,
      stderrBytes,
      stdoutTruncated: stdoutBytes > maxBytes,
      stderrTruncated: stderrBytes > maxBytes,
+      stdoutCompacted: stdoutTail !== rawStdoutTail,
      fullLogPaths: { stdoutFile: job.stdoutFile, stderrFile: job.stderrFile },
    },
-    stdoutTail: tailTextByBytes(stdoutProgressTail, maxBytes),
+    stdoutTail,
    stderrTail: tailTextByBytes(stderrProgressTail, maxBytes),
  };
 }
@@ -791,6 +795,22 @@ function tailTextByBytes(text: string, maxBytes: number): string {
  return buffer.subarray(buffer.length - safeMaxBytes).toString("utf8");
 }

+function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawTail: string, requestedTailBytes: number): string {
+  if (progress.kind !== "hwlab-runtime-lane-trigger") return rawTail;
+  if (Buffer.byteLength(rawTail, "utf8") <= 4096 && !rawTail.includes("\"expected\"") && !rawTail.includes("stdoutTail")) return rawTail;
+  const interestingLines = rawTail.split(/\r?\n/u)
+    .map((line) => line.trimEnd())
+    .filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line))
+    .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line))
+    .slice(-80);
+  const header = [
+    `[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`,
+    `full stdout: ${job.stdoutFile}`,
+  ].join("\n");
+  if (interestingLines.length === 0) return `${header}\n${tailTextByBytes(rawTail, Math.min(requestedTailBytes, 3000))}`;
+  return `${header}\n${interestingLines.join("\n")}\n`;
+}
+
 function parseJsonLineEvents(text: string, eventName: string): Record<string, unknown>[] {
  const events: Record<string, unknown>[] = [];
  for (const line of text.split(/\r?\n/u)) {