fix: expose agentrun lane observability

2026-06-20 09:40:45 +00:00
parent 7e574a1d4c
commit 2a82e2972c
2 changed files with 222 additions and 25 deletions
@@ -32,6 +32,7 @@ export function agentRunHelp(): unknown {
    output: "human by default; use -o json|yaml or --raw for machine/debug output",
    usage: [
      "bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
+      "bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
      "bun scripts/cli.ts agentrun get tasks -o wide",
      "bun scripts/cli.ts agentrun get tasks -o json",
      "bun scripts/cli.ts agentrun describe task/<taskId>",
@@ -77,7 +78,7 @@ export function agentRunHelp(): unknown {
      "bun scripts/cli.ts agentrun git-mirror flush --confirm",
    ],
    resources: ["task/qt", "run", "command/cmd", "runnerjob/rjob", "session/ses", "aipodspec/aps"],
-    description: "Operate AgentRun v0.1 through Kubernetes-style resource verbs. Human output is compact by default; -o json|yaml returns the UniDesk render-only client schema, and --raw exposes the direct AgentRun REST envelope.",
+    description: "Operate AgentRun through Kubernetes-style resource verbs. Human output is compact by default; -o json|yaml returns the UniDesk render-only client schema, --raw exposes the REST envelope, and --node/--lane targets a YAML-declared runtime lane.",
    legacyCompatibility: "queue/runs/commands/runner/sessions/aipod-specs remain as compatibility groups backed by direct HTTP; new commander work should use get/describe/events/logs/result/ack/cancel/dispatch/create/apply/send. sessions turn/steer are removed; use send only.",
  };
 }
@@ -189,11 +190,12 @@ function agentRunHelpText(args: string[]): string {
  }
  if (verb === "describe") {
    return [
-      "Usage: bun scripts/cli.ts agentrun describe <kind/name> [--full] [-o json|yaml] [--raw]",
+      "Usage: bun scripts/cli.ts agentrun describe <kind/name> [--node <node> --lane <lane>] [--full] [-o json|yaml] [--raw]",
      "",
      "Kinds: task|run|command|runnerjob|session|aipodspec",
      "Examples:",
      "  bun scripts/cli.ts agentrun describe task/qt_...",
+      "  bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
      "  bun scripts/cli.ts agentrun describe command/cmd_... --run <runId>",
      "  bun scripts/cli.ts agentrun describe session/<sessionId> --full",
    ].join("\n");
@@ -259,6 +261,7 @@ function agentRunHelpText(args: string[]): string {
      "Use resource primitives for daily commander work:",
      "  bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
      "  bun scripts/cli.ts agentrun describe task/<taskId>",
+      "  bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
      "  bun scripts/cli.ts agentrun events run/<runId> --after-seq 0 --limit 100",
      "  bun scripts/cli.ts agentrun logs session/<sessionId> --tail 100",
      "",
@@ -274,6 +277,7 @@ function agentRunHelpText(args: string[]): string {
    "Common:",
    "  bun scripts/cli.ts agentrun get tasks --queue commander --limit 20",
    "  bun scripts/cli.ts agentrun describe task/<taskId>",
+    "  bun scripts/cli.ts agentrun describe run/<runId> --node D601 --lane v02",
    "  bun scripts/cli.ts agentrun events run/<runId> --after-seq 0 --limit 100",
    "  bun scripts/cli.ts agentrun logs session/<sessionId> --tail 100",
    "  bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin",
@@ -281,6 +285,7 @@ function agentRunHelpText(args: string[]): string {
    "Machine/debug output:",
    "  -o json|yaml emits a stable UniDesk resource object without the global JSON envelope.",
    "  --raw emits the direct AgentRun REST envelope.",
+    "  --node/--lane targets a YAML-declared AgentRun runtime lane; default remains the configured public manager.",
  ].join("\n");
 }

@@ -303,17 +308,20 @@ async function runAgentRunResourceCommand(config: UniDeskConfig | null, verb: Ag
  const bridgeActionArgs = stripAgentRunResourceWrapperArgs(actionArgs);
  const command = `agentrun ${canonicalArgs.join(" ")}`.trim();
  try {
-    if (verb === "get") return await resourceGet(config, command, action, bridgeActionArgs, options);
-    if (verb === "describe") return await resourceDescribe(config, command, action, bridgeActionArgs, options);
-    if (verb === "events") return await resourceEvents(config, command, action, bridgeActionArgs, options);
-    if (verb === "logs") return await resourceLogs(config, command, action, bridgeActionArgs, options);
-    if (verb === "result") return await resourceResult(config, command, action, bridgeActionArgs, options);
-    if (verb === "ack") return await resourceAck(config, command, action, bridgeActionArgs, options);
-    if (verb === "cancel") return await resourceCancel(config, command, action, bridgeActionArgs, options);
-    if (verb === "dispatch") return await resourceDispatch(config, command, action, bridgeActionArgs, options);
-    if (verb === "create") return await resourceCreate(config, command, action, bridgeActionArgs, options);
-    if (verb === "apply") return await resourceApply(config, command, bridgeActionArgs, options);
-    if (verb === "send") return await resourceSessionPromptCommand(config, command, action, bridgeActionArgs, options);
+    return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, options), async () => {
+      if (verb === "get") return await resourceGet(config, command, action, bridgeActionArgs, options);
+      if (verb === "describe") return await resourceDescribe(config, command, action, bridgeActionArgs, options);
+      if (verb === "events") return await resourceEvents(config, command, action, bridgeActionArgs, options);
+      if (verb === "logs") return await resourceLogs(config, command, action, bridgeActionArgs, options);
+      if (verb === "result") return await resourceResult(config, command, action, bridgeActionArgs, options);
+      if (verb === "ack") return await resourceAck(config, command, action, bridgeActionArgs, options);
+      if (verb === "cancel") return await resourceCancel(config, command, action, bridgeActionArgs, options);
+      if (verb === "dispatch") return await resourceDispatch(config, command, action, bridgeActionArgs, options);
+      if (verb === "create") return await resourceCreate(config, command, action, bridgeActionArgs, options);
+      if (verb === "apply") return await resourceApply(config, command, bridgeActionArgs, options);
+      if (verb === "send") return await resourceSessionPromptCommand(config, command, action, bridgeActionArgs, options);
+      return renderedCliResult(false, command, `Unsupported AgentRun resource command. Try: bun scripts/cli.ts agentrun --help`);
+    });
  } catch (error) {
    if (error instanceof AgentRunRestError) return renderAgentRunRestError(command, error, options);
    return renderedCliResult(false, command, `Error: ${error instanceof Error ? error.message : String(error)}`);
@@ -329,7 +337,12 @@ export function stripAgentRunResourceWrapperArgs(args: string[]): string[] {
      index += 1;
      continue;
    }
+    if (arg === "--node" || arg === "--lane") {
+      index += 1;
+      continue;
+    }
    if (arg.startsWith("--output=") || arg.startsWith("-o=")) continue;
+    if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue;
    if (arg === "--full" || arg === "--raw" || arg === "--debug" || arg === "--full-text") continue;
    result.push(arg);
  }
@@ -360,9 +373,11 @@ function parseResourceOptions(args: string[]): AgentRunResourceOptions {
    aipod: null,
    idempotencyKey: null,
    promptStdin: false,
+    node: null,
+    lane: null,
    passthroughArgs: [],
  };
-  const valueFlags = new Set(["-o", "--output", "--limit", "--queue", "--state", "--reader-id", "--task", "--task-id", "--run", "--run-id", "--command", "--command-id", "--session", "--session-id", "--after-seq", "--tail", "--reason", "-f", "--file", "--filename", "--aipod", "--idempotency-key"]);
+  const valueFlags = new Set(["-o", "--output", "--limit", "--queue", "--state", "--reader-id", "--task", "--task-id", "--run", "--run-id", "--command", "--command-id", "--session", "--session-id", "--after-seq", "--tail", "--reason", "-f", "--file", "--filename", "--aipod", "--idempotency-key", "--node", "--lane"]);
  const booleanFlags = new Set(["--full", "--raw", "--debug", "--unread", "--dry-run", "--full-text", "--prompt-stdin", "--stdin"]);
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index] ?? "";
@@ -414,6 +429,8 @@ function applyResourceOption(options: AgentRunResourceOptions, flag: string, val
  else if (flag === "-f" || flag === "--file" || flag === "--filename") options.file = requiredValue(value, flag);
  else if (flag === "--aipod") options.aipod = requiredValue(value, flag);
  else if (flag === "--idempotency-key") options.idempotencyKey = requiredValue(value, flag);
+  else if (flag === "--node") options.node = requiredValue(value, flag);
+  else if (flag === "--lane") options.lane = requiredValue(value, flag);
 }

 function parseNonNegativeInt(raw: string | null, flag: string, defaultValue: number, maxValue: number): number {
@@ -5345,7 +5362,9 @@ function startAsyncAgentRunJob(name: string, command: string[], note: string): R

 async function runAgentRunRestCompatCommand(config: UniDeskConfig | null, group: AgentRunRestCompatGroup, args: string[], canonicalArgs: string[]): Promise<Record<string, unknown>> {
  try {
-    return await runAgentRunRestCommand(config, group, args);
+    return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, parseAgentRunRestTargetOptions(args)), async () => {
+      return await runAgentRunRestCommand(config, group, stripAgentRunLaneTargetArgs(args));
+    });
  } catch (error) {
    if (error instanceof AgentRunRestError) return error.toPayload(`agentrun ${canonicalArgs.join(" ")}`.trim());
    throw error;
@@ -5365,6 +5384,27 @@ async function runAgentRunRestCommand(config: UniDeskConfig | null, group: Agent
  throw new AgentRunRestError("unsupported-version", `unsupported AgentRun REST compatibility group: ${group}`);
 }

+function parseAgentRunRestTargetOptions(args: string[]): AgentRunRestTargetOptions {
+  return {
+    node: agentRunOption(args, "node"),
+    lane: agentRunOption(args, "lane"),
+  };
+}
+
+function stripAgentRunLaneTargetArgs(args: string[]): string[] {
+  const result: string[] = [];
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index] ?? "";
+    if (arg === "--node" || arg === "--lane") {
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue;
+    result.push(arg);
+  }
+  return result;
+}
+
 async function runAgentRunQueueRest(action: string | undefined, id: string | undefined, args: string[]): Promise<Record<string, unknown>> {
  if (action === "list") return await agentRunRestRequest("agentrun queue list", "GET", `/api/v1/queue/tasks${agentRunQuery(args, ["queue", "state", "cursor", "limit", "updated-after"])}`);
  if (action === "commander") return await agentRunRestRequest("agentrun queue commander", "GET", `/api/v1/queue/commander${agentRunQuery(args, ["queue", "reader-id"])}`);
@@ -5619,6 +5659,7 @@ async function sessionRunnerJobBody(args: string[], defaults: Record<string, unk

 async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body?: unknown): Promise<Record<string, unknown>> {
  const clientConfig = readAgentRunClientConfig();
+  if (activeAgentRunRestTarget !== null) return await agentRunLaneRestRequest(command, method, pathValue, body, clientConfig, activeAgentRunRestTarget);
  const auth = resolveAgentRunAuth(clientConfig);
  const bridgeBase = agentRunRestBridgeMetadata(clientConfig, auth, method, pathValue);
  const startedAt = Date.now();
@@ -5652,7 +5693,7 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
  if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) });
  if (!response.ok) {
    const details = response.status === 404
-      ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue)
+      ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, null)
      : safeAgentRunEnvelope(envelope);
    throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details });
  }
@@ -5668,6 +5709,120 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod,
  };
 }

+async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise<Record<string, unknown>> {
+  const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue);
+  const startedAt = Date.now();
+  const request = {
+    method,
+    path: pathValue,
+    body: body ?? null,
+    baseUrl: target.spec.runtime.internalBaseUrl,
+    timeoutMs: clientConfig.manager.timeoutMs,
+    authEnv: clientConfig.auth.env,
+    header: clientConfig.auth.header,
+    scheme: clientConfig.auth.scheme,
+    valuesPrinted: false,
+  };
+  const captureResult = await capture(target.config, target.spec.nodeKubeRoute, ["sh", "--", agentRunLaneRestProxyScript(target.spec, request)]);
+  const captureSummary = compactCapture(captureResult, { full: false, stdoutTailChars: 1200, stderrTailChars: 2000 });
+  const bridge = { ...bridgeBase, elapsedMs: Date.now() - startedAt, capture: captureSummary };
+  if (captureResult.exitCode !== 0) {
+    throw new AgentRunRestError("agentrun-unreachable", `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed for ${method} ${pathValue}`, { bridge });
+  }
+  const proxy = captureJsonPayload(captureResult);
+  if (proxy.ok !== true) {
+    throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) });
+  }
+  const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0;
+  const text = stringOrNull(proxy.text) ?? "";
+  const responseBridge = { ...bridge, httpStatus, proxyElapsedMs: nonNegativeIntegerOrNull(proxy.elapsedMs) };
+  if (text.trim().length === 0) throw new AgentRunRestError("schema-mismatch", `AgentRun server returned an empty response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus });
+  let envelope: Record<string, unknown>;
+  try {
+    envelope = record(JSON.parse(text) as unknown);
+  } catch {
+    throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus });
+  }
+  if (httpStatus === 401 || httpStatus === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) });
+  if (httpStatus < 200 || httpStatus >= 300) {
+    const details = httpStatus === 404
+      ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, target)
+      : safeAgentRunEnvelope(envelope);
+    throw new AgentRunRestError(httpStatus === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${httpStatus}`, { bridge: responseBridge, httpStatus, details });
+  }
+  if (envelope.ok !== true) {
+    const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), httpStatus);
+    throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) });
+  }
+  return {
+    ok: true,
+    command,
+    data: envelope.data ?? null,
+    bridge: responseBridge,
+  };
+}
+
+function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record<string, unknown>): string {
+  const requestB64 = Buffer.from(JSON.stringify(request), "utf8").toString("base64");
+  const evalScript = String.raw`
+const startedAt = Date.now();
+try {
+  const input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8"));
+  const headers = {};
+  const authEnv = String(input.authEnv || "HWLAB_API_KEY");
+  const apiKey = process.env[authEnv] || process.env.AGENTRUN_API_KEY || process.env.HWLAB_API_KEY || "";
+  if (apiKey && input.header && input.scheme) headers[String(input.header)] = String(input.scheme) + " " + apiKey;
+  let body = undefined;
+  if (input.body !== null && input.body !== undefined) {
+    headers["content-type"] = "application/json";
+    body = JSON.stringify(input.body);
+  }
+  const controller = new AbortController();
+  const timeoutMs = Number(input.timeoutMs || 15000);
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  let response;
+  try {
+    response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), {
+      method: String(input.method || "GET"),
+      headers,
+      body,
+      signal: controller.signal,
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+  const text = await response.text();
+  console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
+} catch (error) {
+  console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false }));
+}
+`;
+  return [
+    "set -eu",
+    `namespace=${shQuote(spec.runtime.namespace)}`,
+    `deployment=${shQuote(spec.runtime.managerDeployment)}`,
+    `request_b64=${shQuote(requestB64)}`,
+    `kubectl -n "$namespace" exec deploy/"$deployment" -- env AGENTRUN_LANE_REST_REQUEST_B64="$request_b64" bun --eval ${shQuote(evalScript)}`,
+  ].join("\n");
+}
+
+function resolveAgentRunRestTarget(config: UniDeskConfig | null, options: AgentRunRestTargetOptions): AgentRunRestTarget | null {
+  if (options.node === null && options.lane === null) return null;
+  if (config === null) throw new AgentRunRestError("validation-failed", "--node/--lane resource queries require UniDesk config");
+  const { configPath, spec } = resolveAgentRunLaneTarget(options);
+  return { config, configPath, spec };
+}
+
+async function withAgentRunRestTarget<T>(target: AgentRunRestTarget | null, action: () => Promise<T>): Promise<T> {
+  const previous = activeAgentRunRestTarget;
+  activeAgentRunRestTarget = target;
+  try {
+    return await action();
+  } finally {
+    activeAgentRunRestTarget = previous;
+  }
+}
+
 function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult {
  const payload = error.toPayload(command);
  if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false);
@@ -5717,12 +5872,13 @@ class AgentRunRestError extends Error {
  }

  toPayload(command: string): Record<string, unknown> {
+    const transport = stringOrNull(this.bridge?.mode) ?? "direct-http";
    return {
      ok: false,
      command,
      failureKind: this.failureKind,
      message: this.message,
-      transport: "direct-http",
+      transport,
      clientRole: "render-only",
      ...(this.httpStatus === null ? {} : { httpStatus: this.httpStatus }),
      ...(this.bridge === null ? {} : { bridge: this.bridge }),
@@ -5923,7 +6079,25 @@ function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRun
  };
 }

-function addAgentRunNotFoundLookupHint(details: Record<string, unknown>, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string): Record<string, unknown> {
+function agentRunLaneRestBridgeMetadata(config: AgentRunClientConfig, target: AgentRunRestTarget, method: AgentRunHttpMethod, pathValue: string): Record<string, unknown> {
+  return {
+    mode: "lane-k8s-service-proxy",
+    clientRole: "render-only",
+    configPath: config.sourcePath,
+    laneConfigPath: target.configPath,
+    node: target.spec.nodeId,
+    lane: target.spec.lane,
+    kubeRoute: target.spec.nodeKubeRoute,
+    namespace: target.spec.runtime.namespace,
+    managerDeployment: target.spec.runtime.managerDeployment,
+    baseUrl: target.spec.runtime.internalBaseUrl,
+    request: { method, path: pathValue, timeoutMs: config.manager.timeoutMs },
+    auth: { source: "manager-pod-env", env: config.auth.env, fallbackEnv: "AGENTRUN_API_KEY", header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false },
+    valuesPrinted: false,
+  };
+}
+
+function addAgentRunNotFoundLookupHint(details: Record<string, unknown>, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string, target: AgentRunRestTarget | null): Record<string, unknown> {
  const resource = agentRunResourceFromPath(pathValue);
  const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath);
  const candidateLanes = laneConfig?.lanes ?? [];
@@ -5944,15 +6118,19 @@ function addAgentRunNotFoundLookupHint(details: Record<string, unknown>, config:
        ...(resource === null ? {} : { resource }),
      },
      currentEndpoint: {
-        transport: "direct-http",
-        baseUrl: config.manager.baseUrl,
+        transport: target === null ? "direct-http" : "lane-k8s-service-proxy",
+        baseUrl: target === null ? config.manager.baseUrl : target.spec.runtime.internalBaseUrl,
        configPath: config.sourcePath,
        defaultNode: laneConfig?.defaultNode ?? null,
        defaultLane: laneConfig?.defaultLane ?? null,
+        selectedNode: target?.spec.nodeId ?? null,
+        selectedLane: target?.spec.lane ?? null,
      },
      candidateLanes,
      nextCommands,
-      note: "Resource commands use manager.baseUrl from the client config; --node/--lane currently belongs to control-plane inspection commands.",
+      note: target === null
+        ? "Pass --node <node> --lane <lane> to query a YAML-declared non-default AgentRun lane."
+        : "The resource was not found on the selected AgentRun lane; verify the run/session id and the HWLAB runtime lane that created it.",
      valuesPrinted: false,
    },
  };
@@ -6357,9 +6535,24 @@ interface AgentRunResourceOptions {
  aipod: string | null;
  idempotencyKey: string | null;
  promptStdin: boolean;
+  node: string | null;
+  lane: string | null;
  passthroughArgs: string[];
 }

+interface AgentRunRestTargetOptions {
+  node: string | null;
+  lane: string | null;
+}
+
+interface AgentRunRestTarget {
+  config: UniDeskConfig;
+  configPath: string;
+  spec: AgentRunLaneSpec;
+}
+
+let activeAgentRunRestTarget: AgentRunRestTarget | null = null;
+
 type AgentRunBridgeCaptureBackend = "local-backend-core-broker" | "remote-frontend-websocket";

 interface LocalBackendCoreStatus {