fix: restore sub2api accounts from sentinel runtime state
This commit is contained in:
Codex
@@ -2382,6 +2382,8 @@ function renderSentinelReport(
|
||||
`ok=${parsed.ok === true ? "true" : "false"}`,
|
||||
`accounts=${summary.accountCount ?? accounts.length}`,
|
||||
`quarantined=${summary.quarantinedCount ?? "?"}`,
|
||||
`schedulable=${summary.runtimeSchedulableCount ?? "?"}`,
|
||||
`unschedulable=${summary.runtimeUnschedulableCount ?? "?"}`,
|
||||
`history=${summary.historyCount ?? runs.length}`,
|
||||
`window=${formatWindow(summary.historyFrom, summary.historyTo)}`,
|
||||
].join(" "));
|
||||
@@ -2396,11 +2398,12 @@ function renderSentinelReport(
|
||||
lines.push("");
|
||||
lines.push("ACCOUNTS");
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "STATE", "Q", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
||||
["ACCOUNT", "STATE", "Q", "SCH", "T", "PROT", "P_FAIL", "F_MIN", "S_MIN", "S_MAX", "PROBES", "LAST", "HTTP", "M", "KIND", "ACTION", "NEXT", "OBS_MIN"],
|
||||
...accounts.map((account) => [
|
||||
stringValue(account.account) ?? "-",
|
||||
stringValue(account.status) ?? "-",
|
||||
account.quarantineActive === true ? "Y" : "-",
|
||||
account.runtimeSchedulable === true ? "Y" : account.runtimeSchedulable === false ? "N" : "-",
|
||||
account.trustUpstream === true ? "Y" : account.trustUpstream === false ? "N" : "-",
|
||||
account.sentinelProtectEnabled === true ? textValue(account.sentinelProtectThreshold) : "-",
|
||||
account.sentinelProtectDecision ? `${textValue(account.sentinelProtectFailureCount)}/${textValue(account.sentinelProtectThreshold)}` : "-",
|
||||
@@ -2437,7 +2440,7 @@ function renderSentinelReport(
|
||||
]));
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("LEGEND Q=quarantined T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
||||
lines.push("LEGEND Q=quarantined SCH=Sub2API runtime schedulable T=trusted upstream PROT=sentinel protect consecutive-failure threshold P_FAIL=last protect failures/threshold M=marker matched F_MIN=freeze interval S_MIN=success interval S_MAX=success max interval OBS_MIN=last event to next probe minutes TF=transport failures GF=gateway failures GACT=gateway freeze actions");
|
||||
lines.push("Raw: bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --raw");
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -3665,6 +3668,14 @@ def report():
|
||||
"quarantineApplied": quarantine.get("applied") if isinstance(quarantine, dict) else None,
|
||||
"freezeIntervalMin": quarantine.get("intervalMinutes") if isinstance(quarantine, dict) else None,
|
||||
"freezeUntil": quarantine.get("until") if isinstance(quarantine, dict) else None,
|
||||
"runtimeAccountId": account_state.get("runtimeAccountId"),
|
||||
"runtimeStatus": account_state.get("runtimeStatus"),
|
||||
"runtimeSchedulable": account_state.get("runtimeSchedulable"),
|
||||
"runtimeMissing": account_state.get("runtimeMissing"),
|
||||
"runtimeTempUnschedulableSet": account_state.get("runtimeTempUnschedulableSet"),
|
||||
"runtimeTempUnschedulableUntil": account_state.get("runtimeTempUnschedulableUntil"),
|
||||
"runtimeSyncedAt": account_state.get("runtimeSyncedAt"),
|
||||
"runtimeSyncError": account_state.get("runtimeSyncError"),
|
||||
"trustUpstream": account_state.get("trustUpstream") if account_state.get("trustUpstream") is not None else probe.get("trustUpstream"),
|
||||
"successStreak": account_state.get("successStreak") or 0,
|
||||
"successIntervalMin": account_state.get("successIntervalMinutes") or 0,
|
||||
@@ -3710,6 +3721,8 @@ def report():
|
||||
"reasserts": len(item.get("reconcile") or []),
|
||||
})
|
||||
quarantined = [item for item in account_rows if item.get("quarantineActive") is True]
|
||||
runtime_schedulable = [item for item in account_rows if item.get("runtimeSchedulable") is True]
|
||||
runtime_unschedulable = [item for item in account_rows if item.get("runtimeSchedulable") is False]
|
||||
cron_spec = cronjob.get("spec") if isinstance(cronjob, dict) else {}
|
||||
cron_status = cronjob.get("status") if isinstance(cronjob, dict) else {}
|
||||
global_ledger = day_ledgers(state)
|
||||
@@ -3733,10 +3746,13 @@ def report():
|
||||
"summary": {
|
||||
"accountCount": len(account_rows),
|
||||
"quarantinedCount": len(quarantined),
|
||||
"runtimeSchedulableCount": len(runtime_schedulable),
|
||||
"runtimeUnschedulableCount": len(runtime_unschedulable),
|
||||
"historyCount": len(history),
|
||||
"historyFrom": run_rows[0].get("at") if run_rows else None,
|
||||
"historyTo": run_rows[-1].get("at") if run_rows else None,
|
||||
"lastRun": state.get("lastRun"),
|
||||
"runtimeSchedulable": state.get("runtimeSchedulable"),
|
||||
},
|
||||
"globalLedger": global_ledger,
|
||||
"accounts": account_rows,
|
||||
@@ -4368,13 +4384,6 @@ def account_payload(profile, group_id):
|
||||
"confirm_mixed_channel_risk": True,
|
||||
}
|
||||
|
||||
def ensure_account_schedulable(token, account_id, account_name, schedulable=True):
|
||||
data = ensure_success(
|
||||
curl_api("POST", f"/api/v1/admin/accounts/{account_id}/schedulable", bearer=token, payload={"schedulable": bool(schedulable)}),
|
||||
f"set account {account_name} schedulable={bool(schedulable)}",
|
||||
)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
def planned_sentinel_account_results(profiles, existing_accounts):
|
||||
existing = {item.get("name"): item for item in existing_accounts if isinstance(item, dict)}
|
||||
results = []
|
||||
@@ -4422,10 +4431,6 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_fr
|
||||
else:
|
||||
data = ensure_success(curl_api("POST", "/api/v1/admin/accounts", bearer=token, payload=payload), f"create account {profile['accountName']}")
|
||||
action = "created"
|
||||
if isinstance(data, dict) and data.get("id") is not None:
|
||||
schedulable_data = ensure_account_schedulable(token, data["id"], profile["accountName"], not keep_frozen)
|
||||
if schedulable_data:
|
||||
data = schedulable_data
|
||||
results.append({
|
||||
"profile": profile["profile"],
|
||||
"accountName": profile["accountName"],
|
||||
@@ -4445,6 +4450,7 @@ def ensure_accounts(token, profiles, group_id, prune_removed=False, protected_fr
|
||||
"sentinelProbePending": quality_gate_required,
|
||||
"sentinelDefaultFrozen": False,
|
||||
"sentinelFreezeProtected": keep_frozen,
|
||||
"schedulableControl": "sentinel-marker-restore",
|
||||
"openaiResponsesWebSocketsV2Mode": profile.get("openaiResponsesWebSocketsV2Mode"),
|
||||
"trustUpstream": profile.get("trustUpstream") is True,
|
||||
"priority": int(profile.get("priority", POOL_DEFAULT_ACCOUNT_PRIORITY) or POOL_DEFAULT_ACCOUNT_PRIORITY),
|
||||
@@ -5936,7 +5942,7 @@ def run_sync():
|
||||
"pruneMode": "explicit" if prune_removed else "disabled-by-default",
|
||||
"items": account_results,
|
||||
"prunedItems": pruned_account_results,
|
||||
"processControl": {"schedulableRestore": "POST /api/v1/admin/accounts/:id/schedulable for non-quarantined accounts only", "durableConfig": False},
|
||||
"processControl": {"schedulableRestore": "sentinel marker probe only; sync does not restore schedulable for existing accounts", "durableConfig": False},
|
||||
"valuesPrinted": False,
|
||||
},
|
||||
"capacity": capacity_status,
|
||||
|
||||
Reference in New Issue
Block a user