From 2a82e2972cef02c7af1d82b4f3df962bf83c20fd Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 20 Jun 2026 09:40:45 +0000 Subject: [PATCH] fix: expose agentrun lane observability --- scripts/src/agentrun.ts | 235 ++++++++++++++++++-- scripts/src/platform-infra-observability.ts | 12 +- 2 files changed, 222 insertions(+), 25 deletions(-) diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 7a842e67..8c28d1a6 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -32,6 +32,7 @@ export function agentRunHelp(): unknown { output: "human by default; use -o json|yaml or --raw for machine/debug output", usage: [ "bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", + "bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", "bun scripts/cli.ts agentrun get tasks -o wide", "bun scripts/cli.ts agentrun get tasks -o json", "bun scripts/cli.ts agentrun describe task/", @@ -77,7 +78,7 @@ export function agentRunHelp(): unknown { "bun scripts/cli.ts agentrun git-mirror flush --confirm", ], resources: ["task/qt", "run", "command/cmd", "runnerjob/rjob", "session/ses", "aipodspec/aps"], - description: "Operate AgentRun v0.1 through Kubernetes-style resource verbs. Human output is compact by default; -o json|yaml returns the UniDesk render-only client schema, and --raw exposes the direct AgentRun REST envelope.", + description: "Operate AgentRun through Kubernetes-style resource verbs. Human output is compact by default; -o json|yaml returns the UniDesk render-only client schema, --raw exposes the REST envelope, and --node/--lane targets a YAML-declared runtime lane.", legacyCompatibility: "queue/runs/commands/runner/sessions/aipod-specs remain as compatibility groups backed by direct HTTP; new commander work should use get/describe/events/logs/result/ack/cancel/dispatch/create/apply/send. sessions turn/steer are removed; use send only.", }; } @@ -189,11 +190,12 @@ function agentRunHelpText(args: string[]): string { } if (verb === "describe") { return [ - "Usage: bun scripts/cli.ts agentrun describe [--full] [-o json|yaml] [--raw]", + "Usage: bun scripts/cli.ts agentrun describe [--node --lane ] [--full] [-o json|yaml] [--raw]", "", "Kinds: task|run|command|runnerjob|session|aipodspec", "Examples:", " bun scripts/cli.ts agentrun describe task/qt_...", + " bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", " bun scripts/cli.ts agentrun describe command/cmd_... --run ", " bun scripts/cli.ts agentrun describe session/ --full", ].join("\n"); @@ -259,6 +261,7 @@ function agentRunHelpText(args: string[]): string { "Use resource primitives for daily commander work:", " bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", " bun scripts/cli.ts agentrun describe task/", + " bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", " bun scripts/cli.ts agentrun events run/ --after-seq 0 --limit 100", " bun scripts/cli.ts agentrun logs session/ --tail 100", "", @@ -274,6 +277,7 @@ function agentRunHelpText(args: string[]): string { "Common:", " bun scripts/cli.ts agentrun get tasks --queue commander --limit 20", " bun scripts/cli.ts agentrun describe task/", + " bun scripts/cli.ts agentrun describe run/ --node D601 --lane v02", " bun scripts/cli.ts agentrun events run/ --after-seq 0 --limit 100", " bun scripts/cli.ts agentrun logs session/ --tail 100", " bun scripts/cli.ts agentrun create task --aipod Artificer --prompt-stdin", @@ -281,6 +285,7 @@ function agentRunHelpText(args: string[]): string { "Machine/debug output:", " -o json|yaml emits a stable UniDesk resource object without the global JSON envelope.", " --raw emits the direct AgentRun REST envelope.", + " --node/--lane targets a YAML-declared AgentRun runtime lane; default remains the configured public manager.", ].join("\n"); } @@ -303,17 +308,20 @@ async function runAgentRunResourceCommand(config: UniDeskConfig | null, verb: Ag const bridgeActionArgs = stripAgentRunResourceWrapperArgs(actionArgs); const command = `agentrun ${canonicalArgs.join(" ")}`.trim(); try { - if (verb === "get") return await resourceGet(config, command, action, bridgeActionArgs, options); - if (verb === "describe") return await resourceDescribe(config, command, action, bridgeActionArgs, options); - if (verb === "events") return await resourceEvents(config, command, action, bridgeActionArgs, options); - if (verb === "logs") return await resourceLogs(config, command, action, bridgeActionArgs, options); - if (verb === "result") return await resourceResult(config, command, action, bridgeActionArgs, options); - if (verb === "ack") return await resourceAck(config, command, action, bridgeActionArgs, options); - if (verb === "cancel") return await resourceCancel(config, command, action, bridgeActionArgs, options); - if (verb === "dispatch") return await resourceDispatch(config, command, action, bridgeActionArgs, options); - if (verb === "create") return await resourceCreate(config, command, action, bridgeActionArgs, options); - if (verb === "apply") return await resourceApply(config, command, bridgeActionArgs, options); - if (verb === "send") return await resourceSessionPromptCommand(config, command, action, bridgeActionArgs, options); + return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, options), async () => { + if (verb === "get") return await resourceGet(config, command, action, bridgeActionArgs, options); + if (verb === "describe") return await resourceDescribe(config, command, action, bridgeActionArgs, options); + if (verb === "events") return await resourceEvents(config, command, action, bridgeActionArgs, options); + if (verb === "logs") return await resourceLogs(config, command, action, bridgeActionArgs, options); + if (verb === "result") return await resourceResult(config, command, action, bridgeActionArgs, options); + if (verb === "ack") return await resourceAck(config, command, action, bridgeActionArgs, options); + if (verb === "cancel") return await resourceCancel(config, command, action, bridgeActionArgs, options); + if (verb === "dispatch") return await resourceDispatch(config, command, action, bridgeActionArgs, options); + if (verb === "create") return await resourceCreate(config, command, action, bridgeActionArgs, options); + if (verb === "apply") return await resourceApply(config, command, bridgeActionArgs, options); + if (verb === "send") return await resourceSessionPromptCommand(config, command, action, bridgeActionArgs, options); + return renderedCliResult(false, command, `Unsupported AgentRun resource command. Try: bun scripts/cli.ts agentrun --help`); + }); } catch (error) { if (error instanceof AgentRunRestError) return renderAgentRunRestError(command, error, options); return renderedCliResult(false, command, `Error: ${error instanceof Error ? error.message : String(error)}`); @@ -329,7 +337,12 @@ export function stripAgentRunResourceWrapperArgs(args: string[]): string[] { index += 1; continue; } + if (arg === "--node" || arg === "--lane") { + index += 1; + continue; + } if (arg.startsWith("--output=") || arg.startsWith("-o=")) continue; + if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue; if (arg === "--full" || arg === "--raw" || arg === "--debug" || arg === "--full-text") continue; result.push(arg); } @@ -360,9 +373,11 @@ function parseResourceOptions(args: string[]): AgentRunResourceOptions { aipod: null, idempotencyKey: null, promptStdin: false, + node: null, + lane: null, passthroughArgs: [], }; - const valueFlags = new Set(["-o", "--output", "--limit", "--queue", "--state", "--reader-id", "--task", "--task-id", "--run", "--run-id", "--command", "--command-id", "--session", "--session-id", "--after-seq", "--tail", "--reason", "-f", "--file", "--filename", "--aipod", "--idempotency-key"]); + const valueFlags = new Set(["-o", "--output", "--limit", "--queue", "--state", "--reader-id", "--task", "--task-id", "--run", "--run-id", "--command", "--command-id", "--session", "--session-id", "--after-seq", "--tail", "--reason", "-f", "--file", "--filename", "--aipod", "--idempotency-key", "--node", "--lane"]); const booleanFlags = new Set(["--full", "--raw", "--debug", "--unread", "--dry-run", "--full-text", "--prompt-stdin", "--stdin"]); for (let index = 0; index < args.length; index += 1) { const arg = args[index] ?? ""; @@ -414,6 +429,8 @@ function applyResourceOption(options: AgentRunResourceOptions, flag: string, val else if (flag === "-f" || flag === "--file" || flag === "--filename") options.file = requiredValue(value, flag); else if (flag === "--aipod") options.aipod = requiredValue(value, flag); else if (flag === "--idempotency-key") options.idempotencyKey = requiredValue(value, flag); + else if (flag === "--node") options.node = requiredValue(value, flag); + else if (flag === "--lane") options.lane = requiredValue(value, flag); } function parseNonNegativeInt(raw: string | null, flag: string, defaultValue: number, maxValue: number): number { @@ -5345,7 +5362,9 @@ function startAsyncAgentRunJob(name: string, command: string[], note: string): R async function runAgentRunRestCompatCommand(config: UniDeskConfig | null, group: AgentRunRestCompatGroup, args: string[], canonicalArgs: string[]): Promise> { try { - return await runAgentRunRestCommand(config, group, args); + return await withAgentRunRestTarget(resolveAgentRunRestTarget(config, parseAgentRunRestTargetOptions(args)), async () => { + return await runAgentRunRestCommand(config, group, stripAgentRunLaneTargetArgs(args)); + }); } catch (error) { if (error instanceof AgentRunRestError) return error.toPayload(`agentrun ${canonicalArgs.join(" ")}`.trim()); throw error; @@ -5365,6 +5384,27 @@ async function runAgentRunRestCommand(config: UniDeskConfig | null, group: Agent throw new AgentRunRestError("unsupported-version", `unsupported AgentRun REST compatibility group: ${group}`); } +function parseAgentRunRestTargetOptions(args: string[]): AgentRunRestTargetOptions { + return { + node: agentRunOption(args, "node"), + lane: agentRunOption(args, "lane"), + }; +} + +function stripAgentRunLaneTargetArgs(args: string[]): string[] { + const result: string[] = []; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + if (arg === "--node" || arg === "--lane") { + index += 1; + continue; + } + if (arg.startsWith("--node=") || arg.startsWith("--lane=")) continue; + result.push(arg); + } + return result; +} + async function runAgentRunQueueRest(action: string | undefined, id: string | undefined, args: string[]): Promise> { if (action === "list") return await agentRunRestRequest("agentrun queue list", "GET", `/api/v1/queue/tasks${agentRunQuery(args, ["queue", "state", "cursor", "limit", "updated-after"])}`); if (action === "commander") return await agentRunRestRequest("agentrun queue commander", "GET", `/api/v1/queue/commander${agentRunQuery(args, ["queue", "reader-id"])}`); @@ -5619,6 +5659,7 @@ async function sessionRunnerJobBody(args: string[], defaults: Record> { const clientConfig = readAgentRunClientConfig(); + if (activeAgentRunRestTarget !== null) return await agentRunLaneRestRequest(command, method, pathValue, body, clientConfig, activeAgentRunRestTarget); const auth = resolveAgentRunAuth(clientConfig); const bridgeBase = agentRunRestBridgeMetadata(clientConfig, auth, method, pathValue); const startedAt = Date.now(); @@ -5652,7 +5693,7 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, if (response.status === 401 || response.status === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge, httpStatus: response.status, details: safeAgentRunEnvelope(envelope) }); if (!response.ok) { const details = response.status === 404 - ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue) + ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, null) : safeAgentRunEnvelope(envelope); throw new AgentRunRestError(response.status === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${response.status}`, { bridge, httpStatus: response.status, details }); } @@ -5668,6 +5709,120 @@ async function agentRunRestRequest(command: string, method: AgentRunHttpMethod, }; } +async function agentRunLaneRestRequest(command: string, method: AgentRunHttpMethod, pathValue: string, body: unknown, clientConfig: AgentRunClientConfig, target: AgentRunRestTarget): Promise> { + const bridgeBase = agentRunLaneRestBridgeMetadata(clientConfig, target, method, pathValue); + const startedAt = Date.now(); + const request = { + method, + path: pathValue, + body: body ?? null, + baseUrl: target.spec.runtime.internalBaseUrl, + timeoutMs: clientConfig.manager.timeoutMs, + authEnv: clientConfig.auth.env, + header: clientConfig.auth.header, + scheme: clientConfig.auth.scheme, + valuesPrinted: false, + }; + const captureResult = await capture(target.config, target.spec.nodeKubeRoute, ["sh", "--", agentRunLaneRestProxyScript(target.spec, request)]); + const captureSummary = compactCapture(captureResult, { full: false, stdoutTailChars: 1200, stderrTailChars: 2000 }); + const bridge = { ...bridgeBase, elapsedMs: Date.now() - startedAt, capture: captureSummary }; + if (captureResult.exitCode !== 0) { + throw new AgentRunRestError("agentrun-unreachable", `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed for ${method} ${pathValue}`, { bridge }); + } + const proxy = captureJsonPayload(captureResult); + if (proxy.ok !== true) { + throw new AgentRunRestError("agentrun-unreachable", stringOrNull(proxy.message) ?? `AgentRun lane ${target.spec.nodeId}/${target.spec.lane} manager proxy failed`, { bridge: { ...bridge, proxy }, details: pickCompact(proxy, ["ok", "failureKind", "message", "elapsedMs", "valuesPrinted"]) }); + } + const httpStatus = nonNegativeIntegerOrNull(proxy.httpStatus) ?? 0; + const text = stringOrNull(proxy.text) ?? ""; + const responseBridge = { ...bridge, httpStatus, proxyElapsedMs: nonNegativeIntegerOrNull(proxy.elapsedMs) }; + if (text.trim().length === 0) throw new AgentRunRestError("schema-mismatch", `AgentRun server returned an empty response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus }); + let envelope: Record; + try { + envelope = record(JSON.parse(text) as unknown); + } catch { + throw new AgentRunRestError("schema-mismatch", `AgentRun server returned non-JSON response for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus }); + } + if (httpStatus === 401 || httpStatus === 403) throw new AgentRunRestError("auth-failed", stringOrNull(envelope.message) ?? "AgentRun API key was rejected", { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) }); + if (httpStatus < 200 || httpStatus >= 300) { + const details = httpStatus === 404 + ? addAgentRunNotFoundLookupHint(safeAgentRunEnvelope(envelope), clientConfig, method, pathValue, target) + : safeAgentRunEnvelope(envelope); + throw new AgentRunRestError(httpStatus === 404 ? "not-found" : "validation-failed", stringOrNull(envelope.message) ?? `AgentRun request failed with HTTP ${httpStatus}`, { bridge: responseBridge, httpStatus, details }); + } + if (envelope.ok !== true) { + const failureKind = normalizeAgentRunFailureKind(stringOrNull(envelope.failureKind), httpStatus); + throw new AgentRunRestError(failureKind, stringOrNull(envelope.message) ?? `AgentRun request failed for ${method} ${pathValue}`, { bridge: responseBridge, httpStatus, details: safeAgentRunEnvelope(envelope) }); + } + return { + ok: true, + command, + data: envelope.data ?? null, + bridge: responseBridge, + }; +} + +function agentRunLaneRestProxyScript(spec: AgentRunLaneSpec, request: Record): string { + const requestB64 = Buffer.from(JSON.stringify(request), "utf8").toString("base64"); + const evalScript = String.raw` +const startedAt = Date.now(); +try { + const input = JSON.parse(Buffer.from(process.env.AGENTRUN_LANE_REST_REQUEST_B64 || "", "base64").toString("utf8")); + const headers = {}; + const authEnv = String(input.authEnv || "HWLAB_API_KEY"); + const apiKey = process.env[authEnv] || process.env.AGENTRUN_API_KEY || process.env.HWLAB_API_KEY || ""; + if (apiKey && input.header && input.scheme) headers[String(input.header)] = String(input.scheme) + " " + apiKey; + let body = undefined; + if (input.body !== null && input.body !== undefined) { + headers["content-type"] = "application/json"; + body = JSON.stringify(input.body); + } + const controller = new AbortController(); + const timeoutMs = Number(input.timeoutMs || 15000); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let response; + try { + response = await fetch(new URL(String(input.path || "/"), String(input.baseUrl || "http://127.0.0.1:8080")).toString(), { + method: String(input.method || "GET"), + headers, + body, + signal: controller.signal, + }); + } finally { + clearTimeout(timeout); + } + const text = await response.text(); + console.log(JSON.stringify({ ok: true, httpStatus: response.status, statusText: response.statusText, text, elapsedMs: Date.now() - startedAt, valuesPrinted: false })); +} catch (error) { + console.log(JSON.stringify({ ok: false, failureKind: "manager-pod-fetch-failed", message: error instanceof Error ? error.message : String(error), elapsedMs: Date.now() - startedAt, valuesPrinted: false })); +} +`; + return [ + "set -eu", + `namespace=${shQuote(spec.runtime.namespace)}`, + `deployment=${shQuote(spec.runtime.managerDeployment)}`, + `request_b64=${shQuote(requestB64)}`, + `kubectl -n "$namespace" exec deploy/"$deployment" -- env AGENTRUN_LANE_REST_REQUEST_B64="$request_b64" bun --eval ${shQuote(evalScript)}`, + ].join("\n"); +} + +function resolveAgentRunRestTarget(config: UniDeskConfig | null, options: AgentRunRestTargetOptions): AgentRunRestTarget | null { + if (options.node === null && options.lane === null) return null; + if (config === null) throw new AgentRunRestError("validation-failed", "--node/--lane resource queries require UniDesk config"); + const { configPath, spec } = resolveAgentRunLaneTarget(options); + return { config, configPath, spec }; +} + +async function withAgentRunRestTarget(target: AgentRunRestTarget | null, action: () => Promise): Promise { + const previous = activeAgentRunRestTarget; + activeAgentRunRestTarget = target; + try { + return await action(); + } finally { + activeAgentRunRestTarget = previous; + } +} + function renderAgentRunRestError(command: string, error: AgentRunRestError, options: AgentRunResourceOptions): RenderedCliResult { const payload = error.toPayload(command); if (options.raw || options.output === "json" || options.output === "yaml") return renderMachine(command, payload, options.output === "yaml" ? "yaml" : "json", false); @@ -5717,12 +5872,13 @@ class AgentRunRestError extends Error { } toPayload(command: string): Record { + const transport = stringOrNull(this.bridge?.mode) ?? "direct-http"; return { ok: false, command, failureKind: this.failureKind, message: this.message, - transport: "direct-http", + transport, clientRole: "render-only", ...(this.httpStatus === null ? {} : { httpStatus: this.httpStatus }), ...(this.bridge === null ? {} : { bridge: this.bridge }), @@ -5923,7 +6079,25 @@ function agentRunRestBridgeMetadata(config: AgentRunClientConfig, auth: AgentRun }; } -function addAgentRunNotFoundLookupHint(details: Record, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string): Record { +function agentRunLaneRestBridgeMetadata(config: AgentRunClientConfig, target: AgentRunRestTarget, method: AgentRunHttpMethod, pathValue: string): Record { + return { + mode: "lane-k8s-service-proxy", + clientRole: "render-only", + configPath: config.sourcePath, + laneConfigPath: target.configPath, + node: target.spec.nodeId, + lane: target.spec.lane, + kubeRoute: target.spec.nodeKubeRoute, + namespace: target.spec.runtime.namespace, + managerDeployment: target.spec.runtime.managerDeployment, + baseUrl: target.spec.runtime.internalBaseUrl, + request: { method, path: pathValue, timeoutMs: config.manager.timeoutMs }, + auth: { source: "manager-pod-env", env: config.auth.env, fallbackEnv: "AGENTRUN_API_KEY", header: config.auth.header, scheme: config.auth.scheme, valuesPrinted: false }, + valuesPrinted: false, + }; +} + +function addAgentRunNotFoundLookupHint(details: Record, config: AgentRunClientConfig, method: AgentRunHttpMethod, pathValue: string, target: AgentRunRestTarget | null): Record { const resource = agentRunResourceFromPath(pathValue); const laneConfig = readAgentRunLaneLookupConfig(config.sourcePath); const candidateLanes = laneConfig?.lanes ?? []; @@ -5944,15 +6118,19 @@ function addAgentRunNotFoundLookupHint(details: Record, config: ...(resource === null ? {} : { resource }), }, currentEndpoint: { - transport: "direct-http", - baseUrl: config.manager.baseUrl, + transport: target === null ? "direct-http" : "lane-k8s-service-proxy", + baseUrl: target === null ? config.manager.baseUrl : target.spec.runtime.internalBaseUrl, configPath: config.sourcePath, defaultNode: laneConfig?.defaultNode ?? null, defaultLane: laneConfig?.defaultLane ?? null, + selectedNode: target?.spec.nodeId ?? null, + selectedLane: target?.spec.lane ?? null, }, candidateLanes, nextCommands, - note: "Resource commands use manager.baseUrl from the client config; --node/--lane currently belongs to control-plane inspection commands.", + note: target === null + ? "Pass --node --lane to query a YAML-declared non-default AgentRun lane." + : "The resource was not found on the selected AgentRun lane; verify the run/session id and the HWLAB runtime lane that created it.", valuesPrinted: false, }, }; @@ -6357,9 +6535,24 @@ interface AgentRunResourceOptions { aipod: string | null; idempotencyKey: string | null; promptStdin: boolean; + node: string | null; + lane: string | null; passthroughArgs: string[]; } +interface AgentRunRestTargetOptions { + node: string | null; + lane: string | null; +} + +interface AgentRunRestTarget { + config: UniDeskConfig; + configPath: string; + spec: AgentRunLaneSpec; +} + +let activeAgentRunRestTarget: AgentRunRestTarget | null = null; + type AgentRunBridgeCaptureBackend = "local-backend-core-broker" | "remote-frontend-websocket"; interface LocalBackendCoreStatus { diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index 7e3fa295..cf0763ec 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -1220,14 +1220,16 @@ GREP = ${grepLiteral} LIMIT = ${options.limit} IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", - "sessionId", "turnId", "threadId", "failureKind", "willRetry", + "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", "terminalStatus", "phase", "message", "http.route", "http.status_code", - "http.method", "eventType", "runnerId", "attemptId", "storageKind", + "http.method", "eventType", "runnerId", "attemptId", "backendProfile", + "sourceCommit", "jobName", "podName", "logPath", "storageKind", "storagePvcName", "storagePvcPhase", "resumeMode", "rolloutId", "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount", "maxSeq", "traceLastSeq", "endSeq", "commandFiltered", "error", "error.message", "exception.type", "exception.message", + "valuesPrinted", ] IMPORTANT_NAMES = { "durable_admission", "billing_preflight", "agentrun_dispatch", @@ -1460,14 +1462,16 @@ CANDIDATE_LIMIT = ${options.candidateLimit} DEADLINE = time.time() + 50 IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", - "sessionId", "turnId", "threadId", "failureKind", "willRetry", + "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", "terminalStatus", "phase", "message", "http.route", "http.status_code", - "http.method", "eventType", "runnerId", "attemptId", "storageKind", + "http.method", "eventType", "runnerId", "attemptId", "backendProfile", + "sourceCommit", "jobName", "podName", "logPath", "storageKind", "storagePvcName", "storagePvcPhase", "resumeMode", "rolloutId", "returnedEvents", "sinceSeq", "afterSeq", "limit", "fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded", "rawEventCount", "maxSeq", "traceLastSeq", "endSeq", "commandFiltered", "error", "error.message", "exception.type", "exception.message", + "valuesPrinted", ] def run_kubectl(proxy_path, timeout=10):