fix: improve hwlab v02 cli observability

2026-05-31 09:36:50 +00:00
parent ab2369c909
commit a37d2b1374
5 changed files with 283 additions and 8 deletions
@@ -43,6 +43,8 @@ const V02_SERVICE_IDS = [
  "hwlab-edge-proxy",
  "hwlab-agent-skills",
 ];
+const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666";
+const V02_CLOUD_API_URL = "http://74.48.78.17:19667";

 export function v02PipelineServiceIds(): string[] {
  return [...V02_SERVICE_IDS];
@@ -712,11 +714,152 @@ function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): C
    `section obsoleteCronJobs kubectl get cronjob -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(V02_POLLER)} ${shellQuote(V02_RECONCILER)} --ignore-not-found -o name`,
    `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
    `if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
+    `if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
    `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
+    `section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
  ].join("\n");
  return g14K3s(["script", "--", script], 60_000);
 }

+function v02WebAssetsProbeScript(): string {
+  return [
+    "set +e",
+    `base=${shellQuote(V02_CLOUD_WEB_URL)}`,
+    `api=${shellQuote(V02_CLOUD_API_URL)}`,
+    "fetch_url() {",
+    "  if command -v curl >/dev/null 2>&1; then",
+    "    curl -fsS --connect-timeout 2 --max-time 5 \"$1\"",
+    "  elif command -v wget >/dev/null 2>&1; then",
+    "    wget -q -T 5 -O - \"$1\"",
+    "  else",
+    "    return 127",
+    "  fi",
+    "}",
+    "printf 'baseUrl\\t%s\\n' \"$base\"",
+    "printf 'apiUrl\\t%s\\n' \"$api\"",
+    "html=$(fetch_url \"$base/\" 2>/dev/null)",
+    "html_code=$?",
+    "printf 'htmlOk\\t%s\\n' \"$html_code\"",
+    "printf 'readonlyNote\\t%s\\n' \"$(printf '%s' \"$html\" | grep -Eiq 'readonly-rpc|复核入口'; printf '%s' \"$?\")\"",
+    "css=$(fetch_url \"$base/styles.css\" 2>/dev/null)",
+    "css_code=$?",
+    "printf 'cssOk\\t%s\\n' \"$css_code\"",
+    "printf 'sidebarFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
+    "printf 'workspaceFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(0,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
+    "printf 'eventPanelFitCss\\t%s\\n' \"$(printf '%s' \"$css\" | grep -Eq 'grid-template-rows:[[:space:]]*auto[[:space:]]+minmax\\(132px,[[:space:]]*1fr\\)'; printf '%s' \"$?\")\"",
+    "health=$(fetch_url \"$api/health/live\" 2>/dev/null)",
+    "health_code=$?",
+    "printf 'apiHealthOk\\t%s\\n' \"$health_code\"",
+    "printf 'apiRevision\\t%s\\n' \"$(printf '%s' \"$health\" | sed -n 's/.*\"revision\"[[:space:]]*:[[:space:]]*\"\\([0-9A-Za-z._-]*\\)\".*/\\1/p' | head -1)\"",
+  ].join("\n");
+}
+
+function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
+  if (!commandOk) {
+    return {
+      ok: false,
+      pipelineRun,
+      exitCode,
+      stderr: stderr.trim().slice(0, 2000),
+      counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
+      items: [],
+    };
+  }
+  const items = text
+    .split(/\r?\n/u)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const [name = "", status = "", reason = "", startTime = "", completionTime = ""] = line.split("\t");
+      return {
+        name,
+        status: status || null,
+        reason: reason || null,
+        startTime: startTime || null,
+        completionTime: completionTime || null,
+        durationSeconds: secondsBetween(startTime, completionTime),
+      };
+    });
+  const counts = {
+    succeeded: items.filter((item) => item.status === "True").length,
+    failed: items.filter((item) => item.status === "False").length,
+    running: items.filter((item) => item.status === "Unknown").length,
+    unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
+  };
+  return {
+    ok: true,
+    pipelineRun,
+    counts,
+    items,
+    summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}`,
+    disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
+  };
+}
+
+function v02WebAssetsFromText(text: string, commandOk: boolean, sourceCommit: string | null, argoSyncRevision: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
+  const fields: Record<string, string> = {};
+  for (const line of text.split(/\r?\n/u)) {
+    const [key = "", ...rest] = line.split("\t");
+    if (key.length > 0) fields[key] = rest.join("\t");
+  }
+  const htmlOk = fields.htmlOk === "0";
+  const cssOk = fields.cssOk === "0";
+  const apiHealthOk = fields.apiHealthOk === "0";
+  const readonlyNoteAbsent = fields.readonlyNote === "1";
+  const sidebarFitCss = fields.sidebarFitCss === "0";
+  const workspaceFitCss = fields.workspaceFitCss === "0";
+  const eventPanelFitCss = fields.eventPanelFitCss === "0";
+  const apiRevision = fields.apiRevision || null;
+  const webChecksPass = htmlOk && cssOk && readonlyNoteAbsent && sidebarFitCss && workspaceFitCss && eventPanelFitCss && apiHealthOk;
+  const failedChecks = Object.entries({
+    htmlOk,
+    cssOk,
+    readonlyNoteAbsent,
+    sidebarFitCss,
+    workspaceFitCss,
+    eventPanelFitCss,
+    apiHealthOk,
+  }).filter(([, ok]) => !ok).map(([name]) => name);
+  return {
+    ok: commandOk && webChecksPass,
+    summary: commandOk && webChecksPass ? "19666/19667 probes passed" : `19666/19667 probe issues: ${failedChecks.join(", ") || "command failed"}`,
+    baseUrl: fields.baseUrl || V02_CLOUD_WEB_URL,
+    apiUrl: fields.apiUrl || V02_CLOUD_API_URL,
+    sourceCommit,
+    argoSyncRevision: argoSyncRevision || null,
+    checks: {
+      htmlOk,
+      cssOk,
+      readonlyNoteAbsent,
+      sidebarFitCss,
+      workspaceFitCss,
+      eventPanelFitCss,
+      apiHealthOk,
+    },
+    probeExitCodes: {
+      html: numericField(fields.htmlOk),
+      css: numericField(fields.cssOk),
+      readonlyNoteGrep: numericField(fields.readonlyNote),
+      sidebarFitCssGrep: numericField(fields.sidebarFitCss),
+      workspaceFitCssGrep: numericField(fields.workspaceFitCss),
+      eventPanelFitCssGrep: numericField(fields.eventPanelFitCss),
+      apiHealth: numericField(fields.apiHealthOk),
+    },
+    apiRevision,
+    note: apiRevision && sourceCommit && apiRevision !== sourceCommit
+      ? "cloud-api image revision can differ when a change only republishes Cloud Web static assets; use webAssets.checks for 19666 frontend asset readiness."
+      : null,
+    exitCode,
+    stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
+  };
+}
+
+function numericField(value: string | undefined): number | null {
+  if (value === undefined || value.trim().length === 0) return null;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
 function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
  if (!commandOk) {
    return {
@@ -1184,6 +1327,8 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
  const obsoleteCronJobs = sections.obsoleteCronJobs;
  const argo = sections.argo;
  const pipelineRunSection = sections.pipelineRun;
+  const taskRunsSection = sections.taskRuns;
+  const webAssetsSection = sections.webAssets;
  const recentPipelineRuns = listV02PipelineRunsCompactFromText(
    sections.recentPipelineRuns?.stdout ?? "",
    shellSectionOk(sections.recentPipelineRuns),
@@ -1237,6 +1382,21 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
        pipelineRunSection?.exitCode ?? null,
        bundle.stderr,
      ),
+    taskRuns: taskRunsCompactFromText(
+      taskRunsSection?.stdout ?? "",
+      shellSectionOk(taskRunsSection),
+      pipelineRun,
+      taskRunsSection?.exitCode ?? null,
+      bundle.stderr,
+    ),
+    webAssets: v02WebAssetsFromText(
+      webAssetsSection?.stdout ?? "",
+      shellSectionOk(webAssetsSection),
+      sourceCommit,
+      syncRevision,
+      webAssetsSection?.exitCode ?? null,
+      bundle.stderr,
+    ),
    activePipelineRuns,
    recentPipelineRuns,
    query: {