fix: improve AgentRun status visibility

2026-06-16 05:50:52 +00:00
parent 131698ee82
commit 5a2c72c87b
5 changed files with 300 additions and 51 deletions
@@ -2117,52 +2117,74 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
    ...(secrets.ready === true ? [] : ["lane-secret-missing"]),
    ...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []),
  ];
-  return {
+  const aligned = blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected;
+  const runtimeAlignment = {
+    pipelineSucceeded,
+    argoRevision: stringOrNull(argo.revision),
+    argoSyncedToGitops,
+    managerSourceCommit: stringOrNull(manager.sourceCommit),
+    managerSourceMatchesExpected,
+  };
+  const summary = {
+    aligned,
+    blockers,
+    sourceCommit,
+    expectedPipelineRun: pipelineRunName,
+    expectedGitopsRevision,
+    runtimeAlignment,
+    source: {
+      workspaceExists: sourcePayload.workspaceExists ?? false,
+      workspaceClean: sourcePayload.workspaceClean ?? null,
+      localHead: sourcePayload.localHead ?? null,
+      remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
+      remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
+    },
+    gitMirror: {
+      readReady: mirrorPayload.readReady ?? false,
+      writeReady: mirrorPayload.writeReady ?? false,
+      cachePvcExists: mirrorPayload.cachePvcExists ?? false,
+      repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null,
+      sourceCommit: stringOrNull(mirrorPayload.sourceCommit),
+      gitopsCommit: expectedGitopsRevision,
+    },
+    ci: {
+      namespaceExists: runtimePayload.ciNamespaceExists ?? false,
+      serviceAccountExists: runtimePayload.serviceAccountExists ?? false,
+      pipeline,
+      pipelineRun: pipelineRunStatus,
+    },
+    argo,
+    runtime: {
+      namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
+      manager: {
+        deploymentExists: manager.deploymentExists ?? false,
+        serviceExists: manager.serviceExists ?? false,
+        deployment: manager.deployment ?? null,
+        service: manager.service ?? null,
+        image: manager.image ?? null,
+        sourceCommit: stringOrNull(manager.sourceCommit),
+        sourceMatchesExpected: managerSourceMatchesExpected,
+      },
+      database,
+      secrets: compactLaneSecretsStatus(secrets),
+      localPostgres,
+    },
+  };
+  const statusFullCommand = agentRunControlPlaneStatusCommand(spec, options, true);
+  const result: Record<string, unknown> = {
    ok: sourceProbe.value.exitCode === 0 && runtimeProbe.value.exitCode === 0 && mirrorProbe.value.exitCode === 0 && blockers.length === 0,
    command: "agentrun control-plane status",
    mode: "yaml-declared-node-lane",
    configPath: target.configPath,
-    target: agentRunLaneSummary(spec),
-    summary: {
-      aligned: blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected,
+    target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec),
+    summary,
+    alignment: {
+      aligned,
      blockers,
      sourceCommit,
      expectedPipelineRun: pipelineRunName,
      expectedGitopsRevision,
-      runtimeAlignment: {
-        pipelineSucceeded,
-        argoRevision: stringOrNull(argo.revision),
-        argoSyncedToGitops,
-        managerSourceCommit: stringOrNull(manager.sourceCommit),
-        managerSourceMatchesExpected,
-      },
-      source: {
-        workspaceExists: sourcePayload.workspaceExists ?? false,
-        workspaceClean: sourcePayload.workspaceClean ?? null,
-        localHead: sourcePayload.localHead ?? null,
-        remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
-        remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
-      },
-      gitMirror: {
-        readReady: mirrorPayload.readReady ?? false,
-        writeReady: mirrorPayload.writeReady ?? false,
-        cachePvcExists: mirrorPayload.cachePvcExists ?? false,
-        repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null,
-      },
-      ci: {
-        namespaceExists: runtimePayload.ciNamespaceExists ?? false,
-        serviceAccountExists: runtimePayload.serviceAccountExists ?? false,
-        pipeline,
-        pipelineRun: pipelineRunStatus,
-      },
-      argo,
-      runtime: {
-        namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
-        manager,
-        database,
-        secrets,
-        localPostgres,
-      },
+      runtimeAlignment,
    },
    timings: {
      sourceMs: sourceProbe.elapsedMs,
@@ -2170,24 +2192,118 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
      gitMirrorMs: mirrorProbe.elapsedMs,
      totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs),
    },
-    source: sourcePayload,
-    runtime: runtimePayload,
-    gitMirror: mirrorPayload,
    captures: {
      source: compactCapture(sourceProbe.value, { full: options.full || options.raw || sourceProbe.value.exitCode !== 0 }),
      runtime: compactCapture(runtimeProbe.value, { full: options.full || options.raw || runtimeProbe.value.exitCode !== 0 }),
      gitMirror: compactCapture(mirrorProbe.value, { full: options.full || options.raw || mirrorProbe.value.exitCode !== 0 }),
    },
+    disclosure: {
+      output: options.full || options.raw ? "full" : "compact-summary",
+      full: options.full,
+      raw: options.raw,
+      omittedWhenCompact: options.full || options.raw ? [] : ["full target YAML summary", "source", "runtime", "gitMirror", "capture stdout/stderr tails for successful probes"],
+      fullCommand: statusFullCommand,
+    },
    next: {
      plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`,
      postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null,
      postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null,
      secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
      restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
-      statusFull: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --full`,
+      statusFull: statusFullCommand,
    },
    valuesPrinted: false,
  };
+  if (options.full || options.raw) {
+    result.source = sourcePayload;
+    result.runtime = runtimePayload;
+    result.gitMirror = mirrorPayload;
+  }
+  return result;
+}
+
+function agentRunControlPlaneStatusCommand(spec: AgentRunLaneSpec, options: Pick<StatusOptions, "sourceCommit" | "pipelineRun">, full: boolean): string {
+  return [
+    "bun scripts/cli.ts agentrun control-plane status",
+    `--node ${spec.nodeId}`,
+    `--lane ${spec.lane}`,
+    options.pipelineRun === null ? null : `--pipeline-run ${options.pipelineRun}`,
+    options.sourceCommit === null ? null : `--source-commit ${options.sourceCommit}`,
+    full ? "--full" : null,
+  ].filter(Boolean).join(" ");
+}
+
+function compactLaneSecretsStatus(secrets: Record<string, unknown>): Record<string, unknown> {
+  const items = Array.isArray(secrets.items) ? secrets.items.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
+  const missing = items
+    .filter((item) => item.present !== true || item.keyPresent !== true)
+    .map((item) => ({
+      namespace: item.namespace ?? null,
+      name: item.name ?? null,
+      key: item.key ?? null,
+      present: item.present ?? false,
+      keyPresent: item.keyPresent ?? false,
+    }));
+  return {
+    ready: secrets.ready ?? false,
+    count: secrets.count ?? items.length,
+    missingCount: missing.length,
+    missing,
+    valuesPrinted: false,
+  };
+}
+
+function compactAgentRunLaneStatusTarget(spec: AgentRunLaneSpec): Record<string, unknown> {
+  return {
+    node: {
+      id: spec.nodeId,
+      route: spec.nodeRoute,
+      kubeRoute: spec.nodeKubeRoute,
+    },
+    lane: spec.lane,
+    version: spec.version,
+    source: {
+      repository: spec.source.repository,
+      branch: spec.source.branch,
+      workspace: spec.source.workspace,
+    },
+    runtime: {
+      namespace: spec.runtime.namespace,
+      managerDeployment: spec.runtime.managerDeployment,
+      managerService: spec.runtime.managerService,
+      internalBaseUrl: spec.runtime.internalBaseUrl,
+    },
+    ci: {
+      namespace: spec.ci.namespace,
+      pipeline: spec.ci.pipeline,
+      pipelineRunPrefix: spec.ci.pipelineRunPrefix,
+      registryPrefix: spec.ci.registryPrefix,
+    },
+    gitops: {
+      branch: spec.gitops.branch,
+      path: spec.gitops.path,
+      argoNamespace: spec.gitops.argoNamespace,
+      argoApplication: spec.gitops.argoApplication,
+    },
+    gitMirror: {
+      namespace: spec.gitMirror.namespace,
+      readService: spec.gitMirror.readService,
+      writeService: spec.gitMirror.writeService,
+      repositoryCount: spec.gitMirror.repositories.length,
+    },
+    database: {
+      mode: spec.database.mode,
+      provider: spec.database.provider,
+      configRef: spec.database.configRef,
+      database: spec.database.database,
+      user: spec.database.user,
+      secretRef: spec.database.secretRef,
+      localPostgresExpectedAbsent: spec.database.localPostgresExpectedAbsent,
+      valuesPrinted: false,
+    },
+    secretCount: spec.secrets.length,
+    valuesPrinted: false,
+  };
 }

 async function restartYamlLane(config: UniDeskConfig, options: LaneConfirmOptions): Promise<Record<string, unknown>> {
@@ -26,7 +26,7 @@ export interface JobRecord {
 }

 export interface JobProgressSummary {
-  kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "git-mirror" | "generic";
+  kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic";
  stage: string | null;
  stageStatus: string | null;
  sourceCommit: string | null;
@@ -229,15 +229,17 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
  const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
  const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
  const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
+  const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name);
  const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
  const ciInstallWorkflow = job.name === "ci_install";
  if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs);
-  if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
+  if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
  const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
  const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
  const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
  if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs);
  if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
+  if (agentRunYamlLaneTriggerWorkflow) return summarizeAgentRunYamlLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
  const events = v02PrMonitorWorkflow
    ? [
      ...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"),
@@ -479,6 +481,123 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
  };
 }

+const agentRunYamlLaneProgressEvents = [
+  { event: "agentrun.yaml-lane.source-bootstrap.progress", stage: "source-bootstrap" },
+  { event: "agentrun.yaml-lane.image-build.progress", stage: "image-build" },
+  { event: "agentrun.yaml-lane.gitops-publish.progress", stage: "gitops-publish" },
+  { event: "agentrun.yaml-lane.git-mirror.progress", stage: "git-mirror" },
+] as const;
+
+function summarizeAgentRunYamlLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
+  const events = agentRunYamlLaneProgressEvents
+    .flatMap(({ event, stage }) => parseJsonLineEvents(stderrTail, event).map((item) => ({ ...item, stage })))
+    .sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
+  const lastEvent = events.at(-1) ?? {};
+  const stage = stringField(lastEvent.stage);
+  const stageStatus = agentRunYamlLaneStageStatus(lastEvent);
+  const sourceCommit = lastStringField(events, "sourceCommit") ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
+  const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
+  const pipelineCreatedValue = firstMatch(stdoutTail, /"created"\s*:\s*\{[\s\S]{0,400}?"created"\s*:\s*(true|false)/u);
+  const pipelineCreated = pipelineCreatedValue === "true" ? true : pipelineCreatedValue === "false" ? false : null;
+  const lastEventAt = stringField(lastEvent.at);
+  const elapsedSeconds = jobElapsedSeconds(job, nowMs);
+  const eventElapsedMs = numberField(lastEvent.elapsedMs);
+  const stageElapsedSeconds = eventElapsedMs === null ? null : Math.round(eventElapsedMs / 1000);
+  const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
+  const timings = agentRunYamlLaneTimings(events);
+  const warnings = jobProgressWarnings({
+    job,
+    eventsObserved: events.length,
+    elapsedSeconds,
+    stage,
+    stageStatus,
+    stageElapsedSeconds,
+    lastEventAgeSeconds,
+  });
+  const slow = warnings.length > 0;
+  const eventNode = stringField(lastEvent.node);
+  const eventLane = stringField(lastEvent.lane) ?? firstMatch(job.name, /^agentrun_(v[0-9]{2})_trigger_current$/u);
+  const nextCommand = pipelineRun
+    ? [
+        "bun scripts/cli.ts agentrun control-plane status",
+        eventNode ? `--node ${eventNode}` : null,
+        eventLane ? `--lane ${eventLane}` : null,
+        `--pipeline-run ${pipelineRun}`,
+      ].filter(Boolean).join(" ")
+    : job.status === "running"
+      ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
+      : null;
+  return {
+    kind: "agentrun-yaml-lane-trigger",
+    stage,
+    stageStatus,
+    sourceCommit,
+    pipelineRun,
+    pipelineCreated,
+    elapsedSeconds,
+    stageElapsedSeconds,
+    lastEventAt,
+    lastEventAgeSeconds,
+    eventsObserved: events.length,
+    slow,
+    warnings,
+    diagnostics: {
+      stages: agentRunYamlLaneStageDiagnostics(events),
+    },
+    timings,
+    summary: [
+      job.status,
+      stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
+      sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
+      pipelineRun ? `pipelineRun=${pipelineRun}` : null,
+      pipelineCreated === true ? "created" : pipelineCreated === false ? "create-reused" : null,
+      elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
+      stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
+      lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
+      `events=${events.length}`,
+      slow ? "visibility-warning" : null,
+    ].filter(Boolean).join(" "),
+    nextCommand,
+  };
+}
+
+function agentRunYamlLaneStageStatus(event: Record<string, unknown>): string | null {
+  const explicitStatus = stringField(event.status);
+  if (explicitStatus !== null) return explicitStatus;
+  if (event.succeeded === true) return "succeeded";
+  if (event.failed === true) return "failed";
+  if (event.stage !== undefined) return "running";
+  return null;
+}
+
+function agentRunYamlLaneTimings(events: Record<string, unknown>[]): Record<string, number> {
+  const timings: Record<string, number> = {};
+  for (const event of events) {
+    const stage = stringField(event.stage);
+    const elapsedMs = numberField(event.elapsedMs);
+    if (stage !== null && elapsedMs !== null) timings[`agentrunYamlLane.${stage}.elapsedMs`] = Math.round(elapsedMs);
+  }
+  return timings;
+}
+
+function agentRunYamlLaneStageDiagnostics(events: Record<string, unknown>[]): Array<Record<string, unknown>> {
+  return agentRunYamlLaneProgressEvents
+    .map(({ stage }) => {
+      const stageEvents = events.filter((event) => stringField(event.stage) === stage);
+      const lastEvent = stageEvents.at(-1) ?? {};
+      return {
+        stage,
+        eventsObserved: stageEvents.length,
+        status: agentRunYamlLaneStageStatus(lastEvent),
+        jobId: stringField(lastEvent.jobId),
+        jobName: stringField(lastEvent.jobName),
+        polls: numberField(lastEvent.polls),
+        elapsedMs: numberField(lastEvent.elapsedMs),
+      };
+    })
+    .filter((item) => item.eventsObserved > 0);
+}
+
 function hwlabRuntimeLaneStatusNextCommand(pipelineRun: string | null, lastEvent: Record<string, unknown>): { command: string | null; warning: string | null } {
  if (pipelineRun === null) return { command: null, warning: null };
  const eventNode = stringField(lastEvent.node);
@@ -738,6 +857,20 @@ function stringField(value: unknown): string | null {
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
 }

+function numberField(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "string" && value.trim().length > 0) {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return null;
+}
+
+function lastStringField(events: Record<string, unknown>[], field: string): string | null {
+  const event = events.findLast((item) => stringField(item[field]) !== null);
+  return stringField(event?.[field]);
+}
+
 function firstMatch(text: string, pattern: RegExp): string | null {
  const match = pattern.exec(text);
  return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null;