fix: expose Code Agent idle warning diagnostics (#588)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
@@ -1350,6 +1350,9 @@ IMPORTANT_ATTRS = [
|
||||
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
|
||||
"sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
|
||||
"retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
|
||||
"idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
|
||||
"waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
|
||||
"lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
|
||||
"upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
|
||||
"terminalStatus", "phase", "message", "http.route", "http.status_code",
|
||||
"http.method", "eventType", "runnerId", "attemptId", "backendProfile",
|
||||
@@ -1602,6 +1605,9 @@ IMPORTANT_ATTRS = [
|
||||
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
|
||||
"sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
|
||||
"retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
|
||||
"idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
|
||||
"waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
|
||||
"lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
|
||||
"upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
|
||||
"terminalStatus", "phase", "message", "http.route", "http.status_code",
|
||||
"http.response.status_code", "http.method", "http.request.method",
|
||||
@@ -1966,6 +1972,9 @@ IMPORTANT_ATTRS = [
|
||||
"traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId",
|
||||
"sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry",
|
||||
"retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
|
||||
"idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
|
||||
"waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
|
||||
"lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
|
||||
"upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
|
||||
"terminalStatus", "phase", "message", "http.route", "http.status_code",
|
||||
"http.method", "eventType", "runnerId", "attemptId", "backendProfile",
|
||||
@@ -2174,6 +2183,9 @@ def tiny_span(item):
|
||||
"fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded",
|
||||
"afterSeq", "rawEventCount", "failureKind", "willRetry",
|
||||
"retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs",
|
||||
"idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds",
|
||||
"waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs",
|
||||
"lastNotificationAt", "lastNotificationMs", "lastToolCallAt",
|
||||
"upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary",
|
||||
"toolName", "type", "itemType", "itemId", "status", "exitCode",
|
||||
"durationMs", "cwd", "processId", "command", "commandFingerprint",
|
||||
@@ -2460,7 +2472,7 @@ def projection_lag_summary(agentrun, read_model):
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error_spans):
|
||||
def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error_spans, idle_warning_spans):
|
||||
candidates = []
|
||||
if http_summary.get("actorForbidden"):
|
||||
candidates.append({
|
||||
@@ -2514,6 +2526,14 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error
|
||||
"sampleNames": sorted(set(str(item.get("name") or "") for item in error_spans))[:5],
|
||||
},
|
||||
})
|
||||
if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""):
|
||||
candidates.append({
|
||||
"code": "agentrun_runner_idle_warning_active",
|
||||
"label": "AgentRun runner idle warning active",
|
||||
"confidence": 0.72,
|
||||
"summary": "AgentRun runner is still emitting idle warnings and no runner terminal span is present; inspect waitingFor/lastEventLabel before treating the Workbench UI as terminal.",
|
||||
"evidence": [tiny_span(item) for item in idle_warning_spans[-3:]],
|
||||
})
|
||||
candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True)
|
||||
return candidates
|
||||
|
||||
@@ -2727,12 +2747,13 @@ spans = flat["spans"]
|
||||
services = flat["services"]
|
||||
business_trace_ids = flat["businessTraceIds"]
|
||||
error_spans = flat["errorSpans"]
|
||||
idle_warning_spans = [item for item in spans if str(item.get("name") or "") == "codex_stdio.idle_warning"]
|
||||
http_summary = http_status_summary(spans)
|
||||
agentrun = agentrun_summary(spans)
|
||||
read_model = hwlab_read_model_summary(spans)
|
||||
lag = projection_lag_summary(agentrun, read_model)
|
||||
identity = identity_from_spans(spans)
|
||||
candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans)
|
||||
candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans, idle_warning_spans)
|
||||
expected_services = ["hwlab-cloud-api", "agentrun-manager", "agentrun-runner"]
|
||||
service_path = {
|
||||
service: ("reached" if service in services else "missing")
|
||||
@@ -2746,6 +2767,8 @@ if agentrun.get("terminalStatus") == "completed":
|
||||
facts.append("AgentRun completed")
|
||||
if lag.get("status") in ("confirmed", "suspected"):
|
||||
facts.append("HWLAB projection stale")
|
||||
if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""):
|
||||
facts.append("AgentRun runner idle warnings active")
|
||||
if not facts:
|
||||
facts.append("no dominant Code Agent root cause classified")
|
||||
summary = {
|
||||
@@ -2769,6 +2792,8 @@ evidence = {
|
||||
"projectionSpanTail": [tiny_span(item) for item in read_model.get("projectionSpans", [])[-3:]],
|
||||
"errorSpanCount": len(error_spans),
|
||||
"errorSpanSampleNames": sorted(set(str(item.get("name") or "") for item in error_spans))[:5],
|
||||
"idleWarningSpanCount": len(idle_warning_spans),
|
||||
"idleWarningSpanTail": [tiny_span(item) for item in idle_warning_spans[-3:]],
|
||||
}
|
||||
payload = {
|
||||
"ok": trace_rc == 0 and len(spans) > 0,
|
||||
|
||||
Reference in New Issue
Block a user