feat: add in-cluster Kafka replay diagnostics
This commit is contained in:
Codex
@@ -0,0 +1,165 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def run(argv, timeout):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
completed = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
|
||||
return {
|
||||
"exitCode": completed.returncode,
|
||||
"stdout": completed.stdout,
|
||||
"stderr": completed.stderr,
|
||||
"elapsedMs": round((time.monotonic() - started) * 1000),
|
||||
"timedOut": False,
|
||||
}
|
||||
except subprocess.TimeoutExpired as error:
|
||||
stdout = error.stdout.decode("utf-8", errors="replace") if isinstance(error.stdout, bytes) else (error.stdout or "")
|
||||
stderr = error.stderr.decode("utf-8", errors="replace") if isinstance(error.stderr, bytes) else (error.stderr or "")
|
||||
return {
|
||||
"exitCode": 124,
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"elapsedMs": round((time.monotonic() - started) * 1000),
|
||||
"timedOut": True,
|
||||
}
|
||||
|
||||
|
||||
def json_object(text):
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip().startswith("{")]
|
||||
for line in reversed(lines):
|
||||
try:
|
||||
value = json.loads(line)
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def offsets(namespace, cluster, bootstrap, topic, timeout):
|
||||
pod_result = run([
|
||||
"kubectl", "-n", namespace, "get", "pod",
|
||||
"-l", "strimzi.io/cluster=" + cluster + ",strimzi.io/kind=Kafka",
|
||||
"-o", "jsonpath={.items[0].metadata.name}",
|
||||
], timeout)
|
||||
pod = pod_result["stdout"].strip()
|
||||
if pod_result["exitCode"] != 0 or not pod:
|
||||
return {"ok": False, "partitions": [], "errorClass": "broker-pod-unavailable"}
|
||||
result = run([
|
||||
"kubectl", "-n", namespace, "exec", pod, "--",
|
||||
"bin/kafka-get-offsets.sh", "--bootstrap-server", bootstrap,
|
||||
"--topic", topic, "--time", "-2",
|
||||
], timeout)
|
||||
end_result = run([
|
||||
"kubectl", "-n", namespace, "exec", pod, "--",
|
||||
"bin/kafka-get-offsets.sh", "--bootstrap-server", bootstrap,
|
||||
"--topic", topic, "--time", "-1",
|
||||
], timeout)
|
||||
earliest = {}
|
||||
end = {}
|
||||
for output, target in ((result["stdout"], earliest), (end_result["stdout"], end)):
|
||||
for line in output.splitlines():
|
||||
parts = line.rsplit(":", 2)
|
||||
if len(parts) == 3 and parts[1].isdigit() and parts[2].isdigit():
|
||||
target[int(parts[1])] = int(parts[2])
|
||||
partitions = [
|
||||
{"partition": partition, "earliestOffset": earliest.get(partition), "endOffset": end.get(partition), "retainedRecords": max(0, end.get(partition, 0) - earliest.get(partition, 0))}
|
||||
for partition in sorted(set(earliest) | set(end))
|
||||
]
|
||||
return {
|
||||
"ok": result["exitCode"] == 0 and end_result["exitCode"] == 0,
|
||||
"partitions": partitions,
|
||||
"errorClass": None if result["exitCode"] == 0 and end_result["exitCode"] == 0 else "offset-query-failed",
|
||||
}
|
||||
|
||||
|
||||
def classify(replay, window, execution):
|
||||
data = replay.get("data") if isinstance(replay, dict) and isinstance(replay.get("data"), dict) else {}
|
||||
source = data.get("source") if isinstance(data.get("source"), dict) else {}
|
||||
output = data.get("output") if isinstance(data.get("output"), dict) else {}
|
||||
retained = sum(item.get("retainedRecords", 0) for item in window.get("partitions", []))
|
||||
scanned = source.get("scanned", 0)
|
||||
selected = data.get("selectedFrameCount", 0)
|
||||
invalid = source.get("invalidJson", 0)
|
||||
ignored = data.get("ignoredFrames") if isinstance(data.get("ignoredFrames"), dict) else {}
|
||||
if execution.get("timedOut") or window.get("errorClass") or execution.get("exitCode") in (126, 127):
|
||||
return "network-or-runtime-error"
|
||||
if isinstance(replay, dict) and replay.get("ok") is True and output.get("recordCount", 0) > 0:
|
||||
return "matched"
|
||||
if retained == 0 and sum(item.get("endOffset", 0) for item in window.get("partitions", [])) == 0:
|
||||
return "producer-not-written"
|
||||
if retained == 0:
|
||||
return "retention-offset-empty-window"
|
||||
error = replay.get("error") if isinstance(replay.get("error"), dict) else {}
|
||||
details = error.get("details") if isinstance(error.get("details"), dict) else {}
|
||||
if invalid and not selected:
|
||||
return "decode-schema-rejection"
|
||||
if scanned or ignored or details.get("inputRecordCount") == 0:
|
||||
return "filter-mismatch"
|
||||
return "replay-rejected"
|
||||
|
||||
|
||||
namespace = os.environ["KAFKA_NAMESPACE"]
|
||||
cluster = os.environ["KAFKA_CLUSTER"]
|
||||
bootstrap = os.environ["KAFKA_BOOTSTRAP"]
|
||||
topic = os.environ["KAFKA_INPUT_TOPIC"]
|
||||
manager_namespace = os.environ["AGENTRUN_NAMESPACE"]
|
||||
manager_deployment = os.environ["AGENTRUN_MANAGER_DEPLOYMENT"]
|
||||
timeout_ms = int(os.environ["KAFKA_REPLAY_TIMEOUT_MS"])
|
||||
command_grace_ms = int(os.environ["KAFKA_REPLAY_COMMAND_GRACE_MS"])
|
||||
timeout_seconds = max(1, int((timeout_ms + command_grace_ms) / 1000))
|
||||
window = offsets(namespace, cluster, bootstrap, topic, timeout_seconds)
|
||||
command = [
|
||||
"kubectl", "-n", manager_namespace, "exec", "deployment/" + manager_deployment, "--",
|
||||
"sh", "-c",
|
||||
"cd \"${AGENTRUN_APP_ROOT:-/workspace/agentrun}\" && exec ./scripts/agentrun \"$@\"",
|
||||
"agentrun", "kafka", "regenerate", "agentrun",
|
||||
"--session-id", os.environ["KAFKA_REPLAY_SESSION_ID"],
|
||||
"--input-topic", topic,
|
||||
"--limit", os.environ["KAFKA_REPLAY_LIMIT"],
|
||||
"--timeout-ms", str(timeout_ms),
|
||||
"--no-publish", "--format", "json",
|
||||
]
|
||||
trace_id = os.environ.get("KAFKA_REPLAY_TRACE_ID", "")
|
||||
if trace_id:
|
||||
command.extend(["--trace-id", trace_id])
|
||||
execution = run(command, timeout_seconds)
|
||||
replay = json_object(execution["stdout"]) or json_object(execution["stderr"]) or {}
|
||||
reason = classify(replay, window, execution)
|
||||
data = replay.get("data") if isinstance(replay.get("data"), dict) else {}
|
||||
source = data.get("source") if isinstance(data.get("source"), dict) else {}
|
||||
output = data.get("output") if isinstance(data.get("output"), dict) else {}
|
||||
payload = {
|
||||
"ok": reason == "matched",
|
||||
"networkPlane": "k3s-application-pod",
|
||||
"application": "agentrun",
|
||||
"namespace": manager_namespace,
|
||||
"workload": "deployment/" + manager_deployment,
|
||||
"topic": topic,
|
||||
"partitionOffsetWindow": window,
|
||||
"scanned": source.get("scanned", 0),
|
||||
"accepted": data.get("selectedFrameCount", 0),
|
||||
"rejected": {
|
||||
"invalidJson": source.get("invalidJson", 0),
|
||||
"byReason": data.get("ignoredFrames", {}),
|
||||
"classification": reason,
|
||||
},
|
||||
"produced": output.get("recordCount", 0),
|
||||
"firstMismatchIdentity": None if reason == "matched" else {"sessionId": os.environ["KAFKA_REPLAY_SESSION_ID"], "traceId": trace_id or None, "reason": reason},
|
||||
"mutation": False,
|
||||
"topicAppended": False,
|
||||
"valuesPrinted": False,
|
||||
"execution": {
|
||||
"exitCode": execution["exitCode"],
|
||||
"elapsedMs": execution["elapsedMs"],
|
||||
"timedOut": execution["timedOut"],
|
||||
"stderrTail": execution["stderr"][-800:],
|
||||
},
|
||||
"replay": data,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
sys.exit(0)
|
||||
Reference in New Issue
Block a user