mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(code-queue): add issue 3 gateway diagnostics

Browse Source
This commit is contained in:
Codex
2026-05-19 03:32:16 +00:00
parent c00c695197
commit 9d46ca2531
18 changed files with 1494 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/backend-core/src/microservice_proxy.rs
+304 -12
View File
@@ -149,6 +149,168 @@ fn code_queue_k3s_service_id_for_request(method: &Method, target_path: &str) ->
}
}
fn json_bool(value: &Value, path: &[&str]) -> Option<bool> {
let mut current = value;
for key in path {
current = current.get(*key)?;
}
current.as_bool()
}
fn json_i64(value: &Value, path: &[&str]) -> Option<i64> {
let mut current = value;
for key in path {
current = current.get(*key)?;
}
current.as_i64()
}
fn code_queue_route_postgres_check(scheduler_body: &Value) -> Value {
let storage = scheduler_body
.get("queue")
.and_then(|queue| queue.get("storage"))
.unwrap_or(&Value::Null);
let postgres_ready = storage
.get("postgresReady")
.and_then(Value::as_bool)
.unwrap_or(false);
let last_error = storage.get("lastError").cloned().unwrap_or(Value::Null);
json!({
"ok": postgres_ready && last_error.is_null(),
"route": "d601-tcp-egress-gateway.unidesk.svc.cluster.local:15432",
"postgresReady": postgres_ready,
"lastError": last_error,
"source": "scheduler.health.queue.storage"
})
}
fn code_queue_reconcile_check(scheduler_body: &Value) -> Value {
let queue = scheduler_body.get("queue").unwrap_or(&Value::Null);
let reconcile = queue.get("reconcile").unwrap_or(&Value::Null);
let orphaned_active_count = queue
.get("orphanedActiveTaskCount")
.and_then(Value::as_i64)
.unwrap_or(0);
let active_run_slot_waiters = queue
.get("activeRunSlotWaiters")
.and_then(Value::as_array)
.map(|items| items.len() as i64)
.unwrap_or(0);
let recoverable_orphaned_active_count = reconcile
.get("recoverableOrphanedActiveTaskCount")
.and_then(Value::as_i64)
.unwrap_or(orphaned_active_count);
let retry_wait_count = reconcile
.get("retryWaitTaskCount")
.and_then(Value::as_i64)
.or_else(|| {
queue
.get("counts")
.and_then(|counts| counts.get("retry_wait"))
.and_then(Value::as_i64)
})
.unwrap_or(0);
let active_without_local_run = recoverable_orphaned_active_count > 0;
json!({
"ok": !active_without_local_run,
"status": if active_without_local_run { "recovery-required" } else { "idle-or-reconciled" },
"orphanedActiveTaskCount": orphaned_active_count,
"recoverableOrphanedActiveTaskCount": recoverable_orphaned_active_count,
"retryWaitTaskCount": retry_wait_count,
"activeRunSlotWaiterCount": active_run_slot_waiters,
"lastRunAt": reconcile.get("lastRunAt").cloned().unwrap_or(Value::Null),
"lastReason": reconcile.get("lastReason").cloned().unwrap_or(Value::Null),
"lastRecovered": reconcile.get("lastRecovered").cloned().unwrap_or(Value::Null),
"source": "scheduler.health.queue.reconcile"
})
}
fn code_queue_dependency_check(service_id: &str, diagnostics: &Value) -> Value {
let checks = diagnostics.get("checks").unwrap_or(&Value::Null);
let deployment_available = json_bool(checks, &["deployment", "available"]).unwrap_or(false);
let endpoint_non_empty = json_i64(checks, &["endpoint", "readyAddressCount"])
.map(|count| count > 0)
.unwrap_or_else(|| json_bool(checks, &["endpoint", "ok"]).unwrap_or(false));
let target_ok = json_bool(checks, &["targetService", "ok"])
.or_else(|| json_bool(checks, &["kubernetesApiServiceProxy", "ok"]))
.or_else(|| json_bool(checks, &["managedService", "ok"]))
.unwrap_or(false);
let ok = diagnostics
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& deployment_available
&& endpoint_non_empty
&& target_ok;
json!({
"ok": ok,
"serviceId": service_id,
"deploymentAvailable": deployment_available,
"endpointNonEmpty": endpoint_non_empty,
"targetServiceOk": target_ok,
"deployment": compact_json(checks.get("deployment").unwrap_or(&json!({ "ok": false, "skipped": true }))),
"endpoint": compact_json(checks.get("endpoint").unwrap_or(&json!({ "ok": false, "skipped": true }))),
"targetService": compact_json(checks.get("targetService").unwrap_or(&json!({ "ok": false, "skipped": true }))),
"managedService": compact_json(checks.get("managedService").unwrap_or(&json!({ "ok": false, "skipped": true }))),
"kubernetesApiServiceProxy": compact_json(checks.get("kubernetesApiServiceProxy").unwrap_or(&json!({ "ok": false, "skipped": true })))
})
}
async fn code_queue_adapter_diagnostics(
state: &Arc<AppState>,
service: &MicroserviceConfig,
k3s_service_id: &str,
) -> Value {
let adapter_service_id = service
.deployment
.adapter_service_id
.as_deref()
.unwrap_or("k3sctl-adapter");
let Some(adapter) = service_by_id(state, adapter_service_id) else {
return json!({
"ok": false,
"serviceId": k3s_service_id,
"error": format!("k3sctl adapter microservice not found: {adapter_service_id}")
});
};
let adapter_path = format!(
"/api/services/{}/diagnostics",
urlencoding_like(k3s_service_id)
);
let response = fetch_microservice_upstream_response(
state,
&adapter,
&Method::GET,
&adapter_path,
&ProxyOptions {
query: String::new(),
json_array_limits: json!({}),
},
&json!({ "accept": "application/json" }),
String::new(),
)
.await;
let status = response.status().as_u16();
let (status2, content_type, body_text) = response_to_text(response).await.unwrap_or((
status,
"application/json".to_string(),
"{}".to_string(),
));
let body: Value = serde_json::from_str(&body_text)
.unwrap_or_else(|_| json!({ "text": truncate_text(&body_text, 4000) }));
if (200..300).contains(&status2) && body.get("ok").and_then(Value::as_bool) == Some(true) {
body
} else {
json!({
"ok": false,
"serviceId": k3s_service_id,
"status": status2,
"contentType": content_type,
"body": body
})
}
}
fn code_queue_scheduler_only_path(method: &Method, target_path: &str) -> bool {
code_queue_k3s_service_id_for_request(method, target_path) == "code-queue-scheduler"
&& !(target_path == "/" || target_path == "/health")
@@ -191,7 +353,10 @@ fn code_queue_fallback_workdirs_response(
Some(response_with_body(
200,
"application/json; charset=utf-8",
Body::from(serde_json::to_string(&fallback).unwrap_or_else(|_| "{\"ok\":true,\"workdirs\":[]}".to_string())),
Body::from(
serde_json::to_string(&fallback)
.unwrap_or_else(|_| "{\"ok\":true,\"workdirs\":[]}".to_string()),
),
))
}
@@ -977,12 +1142,18 @@ async fn fetch_microservice_upstream_response(
)
.await;
if method == Method::GET && target_path == "/api/workdirs" {
let (status, content_type, response_body) = response_to_text(response).await.unwrap_or((
502,
"text/plain".to_string(),
"failed to read code-queue workdirs response".to_string(),
));
if let Some(fallback) = code_queue_fallback_workdirs_response(service, status, &content_type, &response_body) {
let (status, content_type, response_body) =
response_to_text(response).await.unwrap_or((
502,
"text/plain".to_string(),
"failed to read code-queue workdirs response".to_string(),
));
if let Some(fallback) = code_queue_fallback_workdirs_response(
service,
status,
&content_type,
&response_body,
) {
return fallback;
}
return response_with_body(status, &content_type, Body::from(response_body));
@@ -1522,15 +1693,63 @@ async fn code_queue_health_response(
.unwrap_or_else(|_| json!({ "text": truncate_text(&scheduler_text, 4000) }));
let scheduler_healthy = (200..300).contains(&scheduler_status)
&& scheduler_body.get("ok").and_then(Value::as_bool) != Some(false);
let status = if mgr_healthy { 200 } else { 503 };
let scheduler_diagnostics =
code_queue_adapter_diagnostics(state, service, "code-queue-scheduler").await;
let provider_egress_diagnostics =
code_queue_adapter_diagnostics(state, service, "d601-provider-egress-proxy").await;
let tcp_egress_diagnostics =
code_queue_adapter_diagnostics(state, service, "d601-tcp-egress-gateway").await;
let scheduler_dependency =
code_queue_dependency_check("code-queue-scheduler", &scheduler_diagnostics);
let provider_egress =
code_queue_dependency_check("d601-provider-egress-proxy", &provider_egress_diagnostics);
let tcp_egress =
code_queue_dependency_check("d601-tcp-egress-gateway", &tcp_egress_diagnostics);
let postgres_route = code_queue_route_postgres_check(&scheduler_body);
let stale_reconcile = code_queue_reconcile_check(&scheduler_body);
let dependencies_ok = scheduler_dependency
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& provider_egress
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& tcp_egress
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& postgres_route
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& stale_reconcile
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false);
let ok = mgr_healthy && scheduler_healthy && dependencies_ok;
let status = if ok { 200 } else { 503 };
let body = json!({
"ok": mgr_healthy,
"ok": ok,
"service": "code-queue",
"role": "hybrid-master-control-plane",
"status": if ok { "healthy" } else { "degraded" },
"checkedAt": crate::json_util::now_iso(),
"manager": { "ok": mgr_healthy, "status": mgr_status, "body": mgr_body, "timedOut": false },
"trace": { "ok": mgr_healthy, "source": "code-queue-mgr", "readonlyUserConfigured": trace_body.get("readonlyUserConfigured").cloned().unwrap_or(Value::Null), "body": trace_body },
"scheduler": { "ok": scheduler_healthy, "status": scheduler_status, "body": scheduler_body, "timedOut": false, "requiredFor": ["running task steer", "running task interrupt", "scheduler claim/runner execution", "dev-container control"] },
"checks": {
"scheduler": scheduler_dependency,
"d601ProviderEgressProxy": provider_egress,
"d601TcpEgressGateway": tcp_egress,
"postgresRoute": postgres_route,
"staleActiveReconcile": stale_reconcile
},
"diagnostics": {
"scheduler": scheduler_diagnostics,
"d601ProviderEgressProxy": provider_egress_diagnostics,
"d601TcpEgressGateway": tcp_egress_diagnostics
},
});
if head_only {
response_with_body(status, "application/json; charset=utf-8", Body::empty())
@@ -1698,8 +1917,73 @@ async fn k3sctl_managed_diagnostics_response(
.get("checks")
.cloned()
.unwrap_or_else(|| json!({}));
let ok =
(200..300).contains(&status2) && provider_online && proxy_mode == "provider-ws-http-tunnel";
let mut code_queue_checks = Value::Null;
let mut code_queue_dependencies = Value::Null;
let mut code_queue_adapter = Value::Null;
let mut code_queue_ok = true;
if service.id == "code-queue" {
let scheduler_body = adapter_body
.get("checks")
.and_then(|checks| checks.get("targetService"))
.and_then(|target| target.get("body"))
.cloned()
.unwrap_or(Value::Null);
let scheduler_diagnostics =
code_queue_adapter_diagnostics(state, service, "code-queue-scheduler").await;
let provider_egress_diagnostics =
code_queue_adapter_diagnostics(state, service, "d601-provider-egress-proxy").await;
let tcp_egress_diagnostics =
code_queue_adapter_diagnostics(state, service, "d601-tcp-egress-gateway").await;
let scheduler_dependency =
code_queue_dependency_check("code-queue-scheduler", &scheduler_diagnostics);
let provider_egress =
code_queue_dependency_check("d601-provider-egress-proxy", &provider_egress_diagnostics);
let tcp_egress =
code_queue_dependency_check("d601-tcp-egress-gateway", &tcp_egress_diagnostics);
let postgres_route = code_queue_route_postgres_check(&scheduler_body);
let stale_reconcile = code_queue_reconcile_check(&scheduler_body);
code_queue_ok = scheduler_dependency
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& provider_egress
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& tcp_egress
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& postgres_route
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false)
&& stale_reconcile
.get("ok")
.and_then(Value::as_bool)
.unwrap_or(false);
code_queue_checks = json!({
"scheduler": scheduler_dependency,
"d601ProviderEgressProxy": provider_egress,
"d601TcpEgressGateway": tcp_egress,
"postgresRoute": postgres_route,
"staleActiveReconcile": stale_reconcile
});
code_queue_dependencies = json!({
"postgresRoute": "d601-tcp-egress-gateway.unidesk.svc.cluster.local:15432",
"requiredDeployments": ["d601-tcp-egress-gateway", "d601-provider-egress-proxy", "code-queue"],
"requiredEndpoints": ["d601-tcp-egress-gateway", "d601-provider-egress-proxy", "code-queue-scheduler"]
});
code_queue_adapter = json!({
"scheduler": scheduler_diagnostics,
"d601ProviderEgressProxy": provider_egress_diagnostics,
"d601TcpEgressGateway": tcp_egress_diagnostics
});
}
let ok = (200..300).contains(&status2)
&& provider_online
&& proxy_mode == "provider-ws-http-tunnel"
&& code_queue_ok;
json_response(
json!({
"ok": ok,
@@ -1715,10 +1999,18 @@ async fn k3sctl_managed_diagnostics_response(
"kubernetesApiServiceProxy": compact_json(adapter_checks.get("kubernetesApiServiceProxy").unwrap_or(&json!({ "ok": false, "skipped": true }))),
"targetService": compact_json(adapter_checks.get("targetService").or_else(|| adapter_checks.get("managedService")).unwrap_or(&json!({ "ok": false, "skipped": true }))),
},
"codeQueue": {
"ok": code_queue_ok,
"checks": code_queue_checks,
"dependencies": code_queue_dependencies,
"adapter": code_queue_adapter
},
"adapter": adapter_body,
}),
if (200..300).contains(&status2) {
if ok {
200
} else if (200..300).contains(&status2) {
503
} else {
status2
},