fix: 修正 warm session 终态诊断

This commit is contained in:
Codex
2026-07-11 22:25:29 +02:00
parent df73e5a265
commit 1e3a805840
4 changed files with 312 additions and 17 deletions
@@ -1057,16 +1057,19 @@ def unwrap_payload(value):
return result
return value
def guess_agentrun_namespace(identity):
def guess_agentrun_namespace(identity, provider_decision):
runner_namespace = provider_decision.get("runnerNamespace") if isinstance(provider_decision, dict) else None
if isinstance(runner_namespace, str) and re.fullmatch(r"agentrun-v[0-9]+", runner_namespace.strip()):
return runner_namespace.strip(), "provider-decision"
if isinstance(identity, dict):
for key in ("jobName", "podName", "runnerJobId"):
text = str(identity.get(key) or "")
match = re.search(r"\\bagentrun-v[0-9]+\\b", text)
if match:
return match.group(0)
return match.group(0), "workload-identity"
if str(TARGET_ID).upper() == "D601":
return "agentrun-v02"
return "agentrun-v01"
return "agentrun-v02", "target-default"
return "agentrun-v01", "target-default"
def agentrun_service_proxy_paths(namespace, api_path):
return [
@@ -1131,9 +1134,22 @@ def event_items(value):
return [item for item in items if isinstance(item, dict)]
return []
def terminal_from_events(events):
def event_matches_command(item, command_id):
if not isinstance(item, dict) or command_id in (None, ""):
return False
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
return str(first_present(
payload.get("commandId"),
payload.get("targetCommandId"),
item.get("commandId"),
item.get("targetCommandId"),
) or "") == str(command_id)
def terminal_from_events(events, command_id):
latest = None
for item in events:
if not event_matches_command(item, command_id):
continue
payload = item.get("payload") if isinstance(item.get("payload"), dict) else {}
event_type = item.get("type") or item.get("eventType") or payload.get("type")
status = first_present(payload.get("terminalStatus"), payload.get("status"), item.get("terminalStatus"), item.get("status"))
@@ -1146,14 +1162,15 @@ def terminal_from_events(events):
}
return latest
def classify_terminal_status(status, spans):
def classify_terminal_status(status, authority):
if status in ("completed", "succeeded", "success"):
return "completed"
if status in ("failed", "error", "timeout", "cancelled", "canceled"):
return "failed"
if status == "blocked":
return "blocked"
if any(str(item.get("service") or "") == "agentrun-runner" for item in spans):
command_state = authority.get("commandState") if isinstance(authority, dict) and authority.get("ok") else None
if command_state in ("pending", "claimed", "running", "processing", "in_progress", "in-progress"):
return "running"
return "unknown"
@@ -1179,7 +1196,7 @@ def preferred_failure_kind(values):
return kind
return kinds[-1]
def agentrun_authority_summary(identity):
def agentrun_authority_summary(identity, provider_decision):
run_id = identity.get("runId") if isinstance(identity, dict) else None
command_id = identity.get("commandId") if isinstance(identity, dict) else None
if not run_id or not command_id:
@@ -1187,7 +1204,7 @@ def agentrun_authority_summary(identity):
"queried": False,
"reason": "missing-run-or-command-id",
}
namespace = guess_agentrun_namespace(identity)
namespace, namespace_source = guess_agentrun_namespace(identity, provider_decision)
run_q = urllib.parse.quote(str(run_id), safe="")
command_q = urllib.parse.quote(str(command_id), safe="")
command_response, command_attempts = get_agentrun_json(namespace, "/api/v1/runs/%s/commands/%s" % (run_q, command_q), timeout=8)
@@ -1196,7 +1213,8 @@ def agentrun_authority_summary(identity):
command_payload = unwrap_payload(command_response.get("json"))
result_payload = unwrap_payload(result_response.get("json"))
events = event_items(events_response.get("json"))
terminal_event = terminal_from_events(events)
scoped_events = [item for item in events if event_matches_command(item, command_id)]
terminal_event = terminal_from_events(scoped_events, command_id)
command_state = first_present(command_payload.get("state"), command_payload.get("status"))
result_status = first_present(
result_payload.get("terminalStatus"),
@@ -1205,13 +1223,15 @@ def agentrun_authority_summary(identity):
result_payload.get("status"),
)
terminal_status = first_present(
terminal_event.get("status") if isinstance(terminal_event, dict) else None,
result_status,
terminal_event.get("status") if isinstance(terminal_event, dict) else None,
command_state if command_state in ("completed", "failed", "blocked", "cancelled", "canceled") else None,
)
return {
"queried": True,
"namespace": namespace,
"namespaceSource": namespace_source,
"providerDecisionNamespace": provider_decision.get("runnerNamespace") if isinstance(provider_decision, dict) else None,
"ok": bool(command_response.get("ok") or result_response.get("ok") or events_response.get("ok")),
"commandOk": command_response.get("ok"),
"resultOk": result_response.get("ok"),
@@ -1230,6 +1250,7 @@ def agentrun_authority_summary(identity):
"terminalEventType": terminal_event.get("type") if isinstance(terminal_event, dict) else None,
"terminalEventSeq": terminal_event.get("seq") if isinstance(terminal_event, dict) else None,
"eventCount": len(events),
"scopedEventCount": len(scoped_events),
"attempts": {
"command": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in command_attempts],
"result": [{"ok": item.get("ok"), "rc": item.get("rc"), "method": item.get("method"), "proxyPath": item.get("proxyPath"), "stderrTail": item.get("stderrTail")} for item in result_attempts],
@@ -1762,7 +1783,7 @@ def agentrun_summary(spans, authority=None):
failure_kinds.append(authority.get("failureKind"))
terminal_source = "agentrun-authority"
failure_kind = preferred_failure_kind(failure_kinds)
runner_classification = classify_terminal_status(terminal_status, spans)
runner_classification = classify_terminal_status(terminal_status, authority)
return {
"terminalStatus": terminal_status,
"failureKind": failure_kind,
@@ -2266,7 +2287,7 @@ http_summary = http_status_summary(spans)
read_model = hwlab_read_model_summary(spans)
identity = identity_from_spans(spans)
provider_decision = provider_decision_summary(spans)
agentrun_authority = agentrun_authority_summary(identity)
agentrun_authority = agentrun_authority_summary(identity, provider_decision)
agentrun = agentrun_summary(spans, agentrun_authority)
lag = projection_lag_summary(agentrun, read_model)
candidates = root_cause_candidates(http_summary, agentrun, read_model, lag, error_spans, idle_warning_spans)
@@ -2277,12 +2298,37 @@ service_path = {
}
missing_services = [service for service in expected_services if service not in services]
service_path["complete"] = len(missing_services) == 0
terminal_status = agentrun.get("terminalStatus")
terminal_span_count = len(agentrun.get("terminalSpans", []))
authority_queried = bool(agentrun_authority.get("queried")) if isinstance(agentrun_authority, dict) else False
authority_ok = bool(agentrun_authority.get("ok")) if isinstance(agentrun_authority, dict) else False
authority_query_failed = authority_queried and not authority_ok
if missing_services:
observability_gap_status = "missing-service-spans"
elif terminal_status not in (None, "") and terminal_span_count == 0:
observability_gap_status = "missing-command-terminal-span"
elif terminal_span_count == 0 and authority_query_failed:
observability_gap_status = "authority-query-failed"
elif terminal_span_count == 0 and agentrun.get("runnerProviderClassification") == "unknown":
observability_gap_status = "unknown-command-terminal-state"
else:
observability_gap_status = "complete"
observability_gap = {
"status": "complete" if len(missing_services) == 0 else "missing-service-spans",
"status": observability_gap_status,
"expectedServices": expected_services,
"seenServices": sorted(services),
"missingServices": missing_services,
"complete": len(missing_services) == 0,
"serviceCoverageComplete": len(missing_services) == 0,
"terminalSpanSeen": terminal_span_count > 0,
"terminalSpanCount": terminal_span_count,
"authorityQuery": {
"queried": authority_queried,
"ok": authority_ok,
"namespace": agentrun_authority.get("namespace") if isinstance(agentrun_authority, dict) else None,
"namespaceSource": agentrun_authority.get("namespaceSource") if isinstance(agentrun_authority, dict) else None,
"commandState": agentrun_authority.get("commandState") if isinstance(agentrun_authority, dict) else None,
},
"complete": observability_gap_status == "complete",
}
if missing_services and ("hwlab-cloud-api" in services or identity.get("runId") not in (None, "") or identity.get("commandId") not in (None, "")):
candidates.insert(0, {
@@ -2292,14 +2338,43 @@ if missing_services and ("hwlab-cloud-api" in services or identity.get("runId")
"summary": "The business trace is correlated to Code Agent context but is missing expected service spans; do not interpret the missing manager/runner spans as proof that those services were not involved.",
"evidence": observability_gap,
})
elif observability_gap_status == "missing-command-terminal-span":
candidates.insert(0, {
"code": "observability_gap_missing_command_terminal_span",
"label": "command terminal span missing",
"confidence": 0.92,
"summary": "The command-specific AgentRun authority is terminal, but the scoped business trace has no command terminal span; this is an instrumentation gap, not an active runner.",
"evidence": observability_gap,
})
elif observability_gap_status == "authority-query-failed":
candidates.insert(0, {
"code": "observability_gap_authority_query_failed",
"label": "AgentRun authority query failed",
"confidence": 0.9,
"summary": "The scoped business trace has no command terminal span and the AgentRun authority query failed; terminal state is unknown and must not be reported as running.",
"evidence": observability_gap,
})
elif observability_gap_status == "unknown-command-terminal-state":
candidates.insert(0, {
"code": "observability_gap_unknown_command_terminal_state",
"label": "command terminal state unknown",
"confidence": 0.82,
"summary": "The scoped business trace has no command terminal span and command authority has no active or terminal state; classify it as unknown rather than running.",
"evidence": observability_gap,
})
facts = []
if missing_services:
if observability_gap_status == "missing-service-spans":
facts.append("observability gap: missing service spans " + ",".join(missing_services))
elif observability_gap_status == "missing-command-terminal-span":
facts.append("observability gap: missing command terminal span")
elif observability_gap_status == "authority-query-failed":
facts.append("observability gap: AgentRun authority query failed")
elif observability_gap_status == "unknown-command-terminal-state":
facts.append("observability gap: command terminal state unknown")
if business_scope.get("mode") == "business-trace-scoped" and business_scope.get("crossBusinessTraceIds"):
facts.append("cross-business-trace spans excluded")
if http_summary.get("actorForbidden"):
facts.append("actor forbidden")
terminal_status = agentrun.get("terminalStatus")
read_model_turn_status = read_model.get("turnStatus")
agentrun_read_model_status_conflict = terminal_status_conflicts(terminal_status, read_model_turn_status)
if agentrun_read_model_status_conflict: