diff --git a/scripts/src/platform-infra-observability.ts b/scripts/src/platform-infra-observability.ts index d128f950..d1681766 100644 --- a/scripts/src/platform-infra-observability.ts +++ b/scripts/src/platform-infra-observability.ts @@ -1350,6 +1350,9 @@ IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs", + "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds", + "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs", + "lastNotificationAt", "lastNotificationMs", "lastToolCallAt", "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary", "terminalStatus", "phase", "message", "http.route", "http.status_code", "http.method", "eventType", "runnerId", "attemptId", "backendProfile", @@ -1602,6 +1605,9 @@ IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs", + "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds", + "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs", + "lastNotificationAt", "lastNotificationMs", "lastToolCallAt", "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary", "terminalStatus", "phase", "message", "http.route", "http.status_code", "http.response.status_code", "http.method", "http.request.method", @@ -1966,6 +1972,9 @@ IMPORTANT_ATTRS = [ "traceId", "otel.trace_id", "agentrun.stage", "runId", "commandId", "sessionId", "turnId", "threadId", "runnerJobId", "failureKind", "willRetry", "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs", + "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds", + "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs", + "lastNotificationAt", "lastNotificationMs", "lastToolCallAt", "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary", "terminalStatus", "phase", "message", "http.route", "http.status_code", "http.method", "eventType", "runnerId", "attemptId", "backendProfile", @@ -2174,6 +2183,9 @@ def tiny_span(item): "fromSeq", "toSeq", "totalEvents", "hasMore", "fullTraceLoaded", "afterSeq", "rawEventCount", "failureKind", "willRetry", "retryAttempt", "retryMax", "retryExhausted", "retryBackoffMs", + "idleMs", "idleSeconds", "idleThresholdMs", "idleThresholdSeconds", + "waitingFor", "lastEventLabel", "lastActivityAt", "lastActivityMs", + "lastNotificationAt", "lastNotificationMs", "lastToolCallAt", "upstreamHttpStatus", "upstreamHost", "providerErrorClass", "errorSummary", "toolName", "type", "itemType", "itemId", "status", "exitCode", "durationMs", "cwd", "processId", "command", "commandFingerprint", @@ -2460,7 +2472,7 @@ def projection_lag_summary(agentrun, read_model): "reasons": reasons, } -def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error_spans): +def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error_spans, idle_warning_spans): candidates = [] if http_summary.get("actorForbidden"): candidates.append({ @@ -2514,6 +2526,14 @@ def root_cause_candidates(http_summary, agentrun, read_model, lag_summary, error "sampleNames": sorted(set(str(item.get("name") or "") for item in error_spans))[:5], }, }) + if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""): + candidates.append({ + "code": "agentrun_runner_idle_warning_active", + "label": "AgentRun runner idle warning active", + "confidence": 0.72, + "summary": "AgentRun runner is still emitting idle warnings and no runner terminal span is present; inspect waitingFor/lastEventLabel before treating the Workbench UI as terminal.", + "evidence": [tiny_span(item) for item in idle_warning_spans[-3:]], + }) candidates.sort(key=lambda item: item.get("confidence", 0), reverse=True) return candidates @@ -2727,12 +2747,13 @@ spans = flat["spans"] services = flat["services"] business_trace_ids = flat["businessTraceIds"] error_spans = flat["errorSpans"] +idle_warning_spans = [item for item in spans if str(item.get("name") or "") == "codex_stdio.idle_warning"] http_summary = http_status_summary(spans) agentrun = agentrun_summary(spans) read_model = hwlab_read_model_summary(spans) lag = projection_lag_summary(agentrun, read_model) identity = identity_from_spans(spans) -candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans) +candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans, idle_warning_spans) expected_services = ["hwlab-cloud-api", "agentrun-manager", "agentrun-runner"] service_path = { service: ("reached" if service in services else "missing") @@ -2746,6 +2767,8 @@ if agentrun.get("terminalStatus") == "completed": facts.append("AgentRun completed") if lag.get("status") in ("confirmed", "suspected"): facts.append("HWLAB projection stale") +if idle_warning_spans and agentrun.get("terminalStatus") in (None, ""): + facts.append("AgentRun runner idle warnings active") if not facts: facts.append("no dominant Code Agent root cause classified") summary = { @@ -2769,6 +2792,8 @@ evidence = { "projectionSpanTail": [tiny_span(item) for item in read_model.get("projectionSpans", [])[-3:]], "errorSpanCount": len(error_spans), "errorSpanSampleNames": sorted(set(str(item.get("name") or "") for item in error_spans))[:5], + "idleWarningSpanCount": len(idle_warning_spans), + "idleWarningSpanTail": [tiny_span(item) for item in idle_warning_spans[-3:]], } payload = { "ok": trace_rc == 0 and len(spans) > 0,