feat: add marker-only sub2api sentinel reporting
This commit is contained in:
Codex
+4
-1
@@ -58,7 +58,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "hwlab nodes control-plane|git-mirror|secret --node G14 --lane v03", description: "Manage HWLAB node/lane runtime prerequisites for v0.3+ with the node identity passed as data instead of a command family." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." },
|
||||
{ command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|steer|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; legacy bridge groups remain available for raw compatibility." },
|
||||
{ command: "platform-infra sub2api plan|apply|status|validate|codex-pool", description: "Deploy Sub2API in G14 platform-infra, manage the YAML-controlled Codex upstream pool, expose the unified API through FRP when needed, and configure master ~/.codex without printing API keys." },
|
||||
{ command: "platform-infra sub2api plan|apply|status|validate|codex-pool", description: "Deploy Sub2API in G14 platform-infra, manage the YAML-controlled Codex upstream pool, expose the unified API, and inspect marker sentinel state with low-noise reports without printing API keys." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
@@ -622,6 +622,9 @@ function platformInfraHelpSummary(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api status [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report",
|
||||
],
|
||||
description: "Operate G14 platform-infra services such as Sub2API and the YAML-controlled Codex pool.",
|
||||
};
|
||||
|
||||
@@ -6,8 +6,6 @@ export interface CodexPoolSentinelConfig {
|
||||
};
|
||||
actions: {
|
||||
enabled: boolean;
|
||||
freezeOnMarkerMismatch: boolean;
|
||||
freezeOnTransportError: boolean;
|
||||
};
|
||||
schedule: string;
|
||||
image: string;
|
||||
@@ -24,9 +22,12 @@ export interface CodexPoolSentinelConfig {
|
||||
};
|
||||
probe: {
|
||||
timeoutSeconds: number;
|
||||
maxResponseBytes: number;
|
||||
maxOutputTokens: number;
|
||||
transportRetryMinutes: number;
|
||||
userAgent: string;
|
||||
};
|
||||
sdk: {
|
||||
openaiPythonVersion: string;
|
||||
};
|
||||
cadence: {
|
||||
successInitialIntervalMinutes: number;
|
||||
@@ -47,6 +48,13 @@ export interface CodexPoolSentinelConfig {
|
||||
historyLimit: number;
|
||||
}
|
||||
|
||||
export interface CodexPoolSentinelImageTarget {
|
||||
baseImage: string;
|
||||
runtimeImage: string;
|
||||
repository: string;
|
||||
tag: string;
|
||||
}
|
||||
|
||||
export interface CodexPoolSentinelProfileSecret {
|
||||
accountName: string;
|
||||
profile: string;
|
||||
@@ -69,8 +77,6 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
},
|
||||
actions: {
|
||||
enabled: false,
|
||||
freezeOnMarkerMismatch: true,
|
||||
freezeOnTransportError: false,
|
||||
},
|
||||
schedule: "*/1 * * * *",
|
||||
image: "python:3.12-alpine",
|
||||
@@ -87,9 +93,12 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
},
|
||||
probe: {
|
||||
timeoutSeconds: 30,
|
||||
maxResponseBytes: 8192,
|
||||
maxOutputTokens: 12,
|
||||
maxOutputTokens: 16,
|
||||
transportRetryMinutes: 5,
|
||||
userAgent: "Go-http-client/1.1",
|
||||
},
|
||||
sdk: {
|
||||
openaiPythonVersion: "2.41.1",
|
||||
},
|
||||
cadence: {
|
||||
successInitialIntervalMinutes: 1,
|
||||
@@ -98,7 +107,7 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
jitterPercent: 10,
|
||||
},
|
||||
freeze: {
|
||||
initialTtlMinutes: 10,
|
||||
initialTtlMinutes: 2,
|
||||
maxTtlMinutes: 120,
|
||||
backoffMultiplier: 2,
|
||||
jitterPercent: 10,
|
||||
@@ -111,12 +120,28 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
};
|
||||
}
|
||||
|
||||
export function codexPoolSentinelRuntimeImage(config: CodexPoolSentinelConfig): CodexPoolSentinelImageTarget {
|
||||
const baseTag = config.image
|
||||
.replace(/[^A-Za-z0-9_.-]+/gu, "-")
|
||||
.replace(/^-+|-+$/gu, "")
|
||||
.slice(0, 80) || "python";
|
||||
const tag = `${baseTag}-openai-${config.sdk.openaiPythonVersion}`;
|
||||
const repository = "127.0.0.1:5000/platform-infra/sub2api-account-sentinel";
|
||||
return {
|
||||
baseImage: config.image,
|
||||
runtimeImage: `${repository}:${tag}`,
|
||||
repository,
|
||||
tag,
|
||||
};
|
||||
}
|
||||
|
||||
export function readCodexPoolSentinelConfig(value: unknown, defaults: CodexPoolSentinelConfig, sourcePath: string): CodexPoolSentinelConfig {
|
||||
if (!isRecord(value)) return defaults;
|
||||
const monitor = isRecord(value.monitor) ? value.monitor : {};
|
||||
const actions = isRecord(value.actions) ? value.actions : {};
|
||||
const marker = isRecord(value.marker) ? value.marker : {};
|
||||
const probe = isRecord(value.probe) ? value.probe : {};
|
||||
const sdk = isRecord(value.sdk) ? value.sdk : {};
|
||||
const cadence = isRecord(value.cadence) ? value.cadence : {};
|
||||
const freeze = isRecord(value.freeze) ? value.freeze : {};
|
||||
const pricing = isRecord(value.pricing) ? value.pricing : {};
|
||||
@@ -126,8 +151,6 @@ export function readCodexPoolSentinelConfig(value: unknown, defaults: CodexPoolS
|
||||
},
|
||||
actions: {
|
||||
enabled: readBoolean(valueAt(actions, "enabled"), `${sourcePath}.sentinel.actions.enabled`, defaults.actions.enabled),
|
||||
freezeOnMarkerMismatch: readBoolean(valueAt(actions, "freezeOnMarkerMismatch"), `${sourcePath}.sentinel.actions.freezeOnMarkerMismatch`, defaults.actions.freezeOnMarkerMismatch),
|
||||
freezeOnTransportError: readBoolean(valueAt(actions, "freezeOnTransportError"), `${sourcePath}.sentinel.actions.freezeOnTransportError`, defaults.actions.freezeOnTransportError),
|
||||
},
|
||||
schedule: readString(valueAt(value, "schedule"), `${sourcePath}.sentinel.schedule`, defaults.schedule),
|
||||
image: readString(valueAt(value, "image"), `${sourcePath}.sentinel.image`, defaults.image),
|
||||
@@ -144,9 +167,12 @@ export function readCodexPoolSentinelConfig(value: unknown, defaults: CodexPoolS
|
||||
},
|
||||
probe: {
|
||||
timeoutSeconds: readInt(valueAt(probe, "timeoutSeconds"), `${sourcePath}.sentinel.probe.timeoutSeconds`, defaults.probe.timeoutSeconds, 3, 300),
|
||||
maxResponseBytes: readInt(valueAt(probe, "maxResponseBytes"), `${sourcePath}.sentinel.probe.maxResponseBytes`, defaults.probe.maxResponseBytes, 1024, 1048576),
|
||||
maxOutputTokens: readInt(valueAt(probe, "maxOutputTokens"), `${sourcePath}.sentinel.probe.maxOutputTokens`, defaults.probe.maxOutputTokens, 1, 128),
|
||||
transportRetryMinutes: readInt(valueAt(probe, "transportRetryMinutes"), `${sourcePath}.sentinel.probe.transportRetryMinutes`, defaults.probe.transportRetryMinutes, 1, 120),
|
||||
userAgent: readUserAgent(valueAt(probe, "userAgent"), `${sourcePath}.sentinel.probe.userAgent`, defaults.probe.userAgent),
|
||||
},
|
||||
sdk: {
|
||||
openaiPythonVersion: readOpenAiPythonVersion(valueAt(sdk, "openaiPythonVersion"), `${sourcePath}.sentinel.sdk.openaiPythonVersion`, defaults.sdk.openaiPythonVersion),
|
||||
},
|
||||
cadence: {
|
||||
successInitialIntervalMinutes: readInt(valueAt(cadence, "successInitialIntervalMinutes"), `${sourcePath}.sentinel.cadence.successInitialIntervalMinutes`, defaults.cadence.successInitialIntervalMinutes, 1, 1440),
|
||||
@@ -197,6 +223,7 @@ export function codexPoolSentinelSummary(config: CodexPoolSentinelConfig): Recor
|
||||
model: config.model,
|
||||
endpoint: config.endpoint,
|
||||
probe: config.probe,
|
||||
sdk: config.sdk,
|
||||
cadence: config.cadence,
|
||||
freeze: config.freeze,
|
||||
accounting: {
|
||||
@@ -228,6 +255,7 @@ export function renderCodexPoolSentinelManifest(
|
||||
endpoint: config.endpoint,
|
||||
marker: config.marker,
|
||||
probe: config.probe,
|
||||
sdk: config.sdk,
|
||||
cadence: config.cadence,
|
||||
freeze: config.freeze,
|
||||
pricing: config.pricing,
|
||||
@@ -237,7 +265,9 @@ export function renderCodexPoolSentinelManifest(
|
||||
},
|
||||
};
|
||||
const suspend = config.monitor.enabled ? "false" : "true";
|
||||
const activeDeadlineSeconds = Math.max(120, Math.min(3600, config.probe.timeoutSeconds + 120));
|
||||
const activeDeadlineSeconds = Math.max(300, Math.min(3600, config.probe.timeoutSeconds + 240));
|
||||
const command = sentinelContainerShellCommand(config);
|
||||
const runtimeImage = codexPoolSentinelRuntimeImage(config).runtimeImage;
|
||||
return `apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
@@ -339,9 +369,11 @@ spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: sentinel
|
||||
image: ${config.image}
|
||||
image: ${runtimeImage}
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["python3", "/opt/sentinel/sentinel.py"]
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- ${JSON.stringify(command)}
|
||||
env:
|
||||
- name: ADMIN_EMAIL
|
||||
valueFrom:
|
||||
@@ -377,6 +409,23 @@ spec:
|
||||
`;
|
||||
}
|
||||
|
||||
export function sentinelContainerShellCommand(config: CodexPoolSentinelConfig): string {
|
||||
return [
|
||||
"set -eu",
|
||||
"python3 - <<'PY'",
|
||||
"import importlib.metadata",
|
||||
`expected = ${JSON.stringify(config.sdk.openaiPythonVersion)}`,
|
||||
"try:",
|
||||
" current = importlib.metadata.version('openai')",
|
||||
"except importlib.metadata.PackageNotFoundError:",
|
||||
" current = None",
|
||||
"if current != expected:",
|
||||
" raise SystemExit(f'openai-python-version-mismatch expected={expected} current={current}')",
|
||||
"PY",
|
||||
"exec python3 /opt/sentinel/sentinel.py",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function sentinelRunnerPython(): string {
|
||||
return String.raw`#!/usr/bin/env python3
|
||||
import base64
|
||||
@@ -390,7 +439,8 @@ import time
|
||||
import traceback
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib import error, request
|
||||
from urllib import error, parse, request
|
||||
from openai import APIConnectionError, APIStatusError, APITimeoutError, OpenAI
|
||||
|
||||
CONFIG_PATH = "/opt/sentinel/config.json"
|
||||
PROFILES_PATH = "/opt/sentinel-secrets/profiles.json"
|
||||
@@ -547,8 +597,9 @@ def http_json(method, url, headers=None, payload=None, timeout=30, max_bytes=655
|
||||
parsed = json.loads(text) if text.strip() else None
|
||||
except Exception:
|
||||
parsed = None
|
||||
app_success = not (isinstance(parsed, dict) and parsed.get("code") not in (None, 0))
|
||||
return {
|
||||
"ok": 200 <= resp.status < 300 and not too_large,
|
||||
"ok": 200 <= resp.status < 300 and not too_large and app_success,
|
||||
"status": resp.status,
|
||||
"json": parsed,
|
||||
"text": text,
|
||||
@@ -599,6 +650,9 @@ def find_token(value):
|
||||
return found
|
||||
return None
|
||||
|
||||
def url_quote(value):
|
||||
return parse.quote(str(value), safe="")
|
||||
|
||||
class Sub2ApiAdmin:
|
||||
def __init__(self, config):
|
||||
self.base = config["service"]["baseUrl"].rstrip("/")
|
||||
@@ -647,15 +701,32 @@ class Sub2ApiAdmin:
|
||||
self.accounts_by_name = {item.get("name"): item for item in items if isinstance(item, dict) and isinstance(item.get("name"), str)}
|
||||
return self.accounts_by_name
|
||||
|
||||
def account(self, account_name):
|
||||
if self.accounts_by_name is not None and account_name in self.accounts_by_name:
|
||||
return self.accounts_by_name[account_name]
|
||||
data = self.request("GET", "/api/v1/admin/accounts?page=1&page_size=20&platform=openai&type=apikey&search=" + url_quote(account_name))
|
||||
items = data if isinstance(data, list) else []
|
||||
if isinstance(data, dict):
|
||||
for key in ("items", "accounts"):
|
||||
if isinstance(data.get(key), list):
|
||||
items = data[key]
|
||||
break
|
||||
for item in items:
|
||||
if isinstance(item, dict) and item.get("name") == account_name:
|
||||
if self.accounts_by_name is not None:
|
||||
self.accounts_by_name[account_name] = item
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_schedulable(self, account_name, schedulable):
|
||||
account = self.accounts().get(account_name)
|
||||
account = self.account(account_name)
|
||||
if not account or account.get("id") is None:
|
||||
raise RuntimeError(f"account {account_name} not found")
|
||||
self.request("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", {"schedulable": bool(schedulable)})
|
||||
return {"accountId": account.get("id"), "schedulable": bool(schedulable)}
|
||||
|
||||
def recover_state(self, account_name):
|
||||
account = self.accounts().get(account_name)
|
||||
account = self.account(account_name)
|
||||
if not account or account.get("id") is None:
|
||||
return {"skipped": True, "reason": "account-not-found"}
|
||||
try:
|
||||
@@ -664,11 +735,9 @@ class Sub2ApiAdmin:
|
||||
except Exception as exc:
|
||||
return {"ok": False, "accountId": account.get("id"), "error": str(exc)}
|
||||
|
||||
def upstream_responses_url(base_url):
|
||||
def upstream_base_url(base_url):
|
||||
base = str(base_url).rstrip("/")
|
||||
if base.endswith("/v1"):
|
||||
return base + "/responses"
|
||||
return base + "/v1/responses"
|
||||
return base if base.endswith("/v1") else base + "/v1"
|
||||
|
||||
def output_text(parsed):
|
||||
if isinstance(parsed, dict) and isinstance(parsed.get("output_text"), str):
|
||||
@@ -687,6 +756,230 @@ def output_text(parsed):
|
||||
parts.append(block["text"])
|
||||
return "\n".join(parts)
|
||||
|
||||
def model_dump(value):
|
||||
if hasattr(value, "model_dump"):
|
||||
return value.model_dump()
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {}
|
||||
|
||||
def body_text(value):
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
def redact_diagnostic(value):
|
||||
if isinstance(value, dict):
|
||||
redacted = {}
|
||||
for key, item in value.items():
|
||||
key_text = str(key)
|
||||
if any(token in key_text.lower() for token in ("key", "token", "secret", "password", "credential", "authorization")):
|
||||
redacted[key_text] = "[redacted]"
|
||||
else:
|
||||
redacted[key_text] = redact_diagnostic(item)
|
||||
return redacted
|
||||
if isinstance(value, list):
|
||||
return [redact_diagnostic(item) for item in value[:20]]
|
||||
if isinstance(value, str):
|
||||
return value if len(value) <= 2000 else value[:2000] + "...[truncated]"
|
||||
if isinstance(value, (int, float, bool)) or value is None:
|
||||
return value
|
||||
return str(value)
|
||||
|
||||
def selected_headers(headers):
|
||||
if headers is None:
|
||||
return {}
|
||||
selected = {}
|
||||
for key in (
|
||||
"content-type",
|
||||
"x-request-id",
|
||||
"x-ratelimit-limit-requests",
|
||||
"x-ratelimit-remaining-requests",
|
||||
"x-ratelimit-reset-requests",
|
||||
"cf-ray",
|
||||
"server",
|
||||
):
|
||||
try:
|
||||
value = headers.get(key)
|
||||
except Exception:
|
||||
value = None
|
||||
if value:
|
||||
selected[key] = str(value)
|
||||
return selected
|
||||
|
||||
def openai_error_fields(body):
|
||||
if not isinstance(body, dict):
|
||||
return {}
|
||||
error_obj = body.get("error")
|
||||
if isinstance(error_obj, dict):
|
||||
return {
|
||||
"message": error_obj.get("message"),
|
||||
"type": error_obj.get("type"),
|
||||
"param": error_obj.get("param"),
|
||||
"code": error_obj.get("code"),
|
||||
}
|
||||
return {
|
||||
"message": body.get("message"),
|
||||
"type": body.get("type"),
|
||||
"param": body.get("param"),
|
||||
"code": body.get("code"),
|
||||
}
|
||||
|
||||
def error_details(kind, status, body=None, message=None, headers=None):
|
||||
text = body_text(body)
|
||||
return {
|
||||
"kind": kind,
|
||||
"statusCode": status,
|
||||
"message": str(message) if message else None,
|
||||
"openaiError": openai_error_fields(body),
|
||||
"body": redact_diagnostic(body) if isinstance(body, (dict, list)) else None,
|
||||
"bodyHash": sha(text),
|
||||
"bodyPreview": preview(text, 1000),
|
||||
"headers": selected_headers(headers),
|
||||
}
|
||||
|
||||
def sub2api_style_input(prompt):
|
||||
return [{
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": prompt,
|
||||
}],
|
||||
}]
|
||||
|
||||
def sub2api_style_instructions():
|
||||
return (
|
||||
"You are Codex, based on GPT-5. You are running as a coding agent in the Codex CLI on a user's computer."
|
||||
)
|
||||
|
||||
def event_type(event):
|
||||
if isinstance(event, dict):
|
||||
return event.get("type")
|
||||
return getattr(event, "type", None)
|
||||
|
||||
def event_delta(event):
|
||||
if isinstance(event, dict):
|
||||
value = event.get("delta")
|
||||
return value if isinstance(value, str) else ""
|
||||
value = getattr(event, "delta", "")
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
def event_error_message(event):
|
||||
data = model_dump(event)
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data.get("error"), dict):
|
||||
message = data["error"].get("message")
|
||||
if isinstance(message, str) and message:
|
||||
return message
|
||||
if isinstance(data.get("response"), dict) and isinstance(data["response"].get("error"), dict):
|
||||
message = data["response"]["error"].get("message")
|
||||
if isinstance(message, str) and message:
|
||||
return message
|
||||
return None
|
||||
|
||||
def openai_responses_create(profile, config, marker, prompt):
|
||||
headers = {
|
||||
"User-Agent": profile.get("upstreamUserAgent") or config["probe"].get("userAgent") or "Go-http-client/1.1",
|
||||
"X-Request-ID": "unidesk-account-sentinel-" + hashlib.sha256(marker.encode()).hexdigest()[:16],
|
||||
}
|
||||
client = OpenAI(
|
||||
api_key=profile["apiKey"],
|
||||
base_url=upstream_base_url(profile["baseUrl"]),
|
||||
timeout=float(config["probe"]["timeoutSeconds"]),
|
||||
max_retries=0,
|
||||
)
|
||||
started = time.time()
|
||||
try:
|
||||
stream = client.responses.create(
|
||||
model=config["model"],
|
||||
input=sub2api_style_input(prompt),
|
||||
instructions=sub2api_style_instructions(),
|
||||
stream=True,
|
||||
extra_headers=headers,
|
||||
)
|
||||
deltas = []
|
||||
events = []
|
||||
seen_completed = False
|
||||
max_chars = max(32, int(config["probe"]["maxOutputTokens"]) * 12)
|
||||
for event in stream:
|
||||
event_data = model_dump(event)
|
||||
etype = event_type(event)
|
||||
events.append({"type": etype, "preview": preview(body_text(event_data), 240)})
|
||||
if etype == "response.output_text.delta":
|
||||
delta = event_delta(event)
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
if len("".join(deltas)) > max_chars:
|
||||
break
|
||||
elif etype in ("response.completed", "response.done"):
|
||||
seen_completed = True
|
||||
break
|
||||
elif etype in ("response.failed", "error"):
|
||||
message = event_error_message(event) or "OpenAI response failed"
|
||||
raise RuntimeError(message)
|
||||
out = "".join(deltas)
|
||||
parsed = {"stream": True, "completed": seen_completed, "events": events[-20:], "output_text": out}
|
||||
if not seen_completed:
|
||||
parsed["streamError"] = "stream ended before response.completed"
|
||||
return {
|
||||
"ok": seen_completed,
|
||||
"status": 200 if seen_completed else 0,
|
||||
"json": parsed,
|
||||
"outputText": out,
|
||||
"text": body_text(parsed),
|
||||
"tooLarge": not seen_completed and len(out) > max_chars,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
except APIStatusError as exc:
|
||||
status = getattr(exc, "status_code", 0) or 0
|
||||
body = getattr(exc, "body", None)
|
||||
response = getattr(exc, "response", None)
|
||||
return {
|
||||
"ok": False,
|
||||
"status": status,
|
||||
"json": body if isinstance(body, dict) else None,
|
||||
"text": body_text(body or response or ""),
|
||||
"tooLarge": False,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"error": str(exc),
|
||||
"errorDetails": error_details("APIStatusError", status, body, str(exc), getattr(response, "headers", None)),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
except (APITimeoutError, APIConnectionError) as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"status": 0,
|
||||
"json": None,
|
||||
"text": "",
|
||||
"tooLarge": False,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"error": str(exc),
|
||||
"errorDetails": error_details(exc.__class__.__name__, 0, None, str(exc), None),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"status": 0,
|
||||
"json": None,
|
||||
"text": "",
|
||||
"tooLarge": False,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"error": str(exc),
|
||||
"errorDetails": error_details(exc.__class__.__name__, 0, None, str(exc), None),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
|
||||
def usage_from(parsed, prompt, out, config):
|
||||
usage = parsed.get("usage") if isinstance(parsed, dict) and isinstance(parsed.get("usage"), dict) else {}
|
||||
input_tokens = usage.get("input_tokens")
|
||||
@@ -717,36 +1010,25 @@ def usage_from(parsed, prompt, out, config):
|
||||
def probe_account(profile, config, purpose):
|
||||
marker = config["marker"]["prefix"] + "_" + hashlib.sha256((profile["accountName"] + str(time.time()) + str(random.random())).encode()).hexdigest()[:10]
|
||||
prompt = "Return exactly this marker and no other text: " + marker
|
||||
payload = {
|
||||
"model": config["model"],
|
||||
"input": prompt,
|
||||
"stream": False,
|
||||
"store": False,
|
||||
"max_output_tokens": int(config["probe"]["maxOutputTokens"]),
|
||||
}
|
||||
headers = {
|
||||
"Authorization": "Bearer " + profile["apiKey"],
|
||||
"Content-Type": "application/json",
|
||||
"X-Request-ID": "unidesk-account-sentinel-" + hashlib.sha256(marker.encode()).hexdigest()[:16],
|
||||
}
|
||||
if profile.get("upstreamUserAgent"):
|
||||
headers["User-Agent"] = profile["upstreamUserAgent"]
|
||||
resp = http_json(
|
||||
"POST",
|
||||
upstream_responses_url(profile["baseUrl"]),
|
||||
headers,
|
||||
payload,
|
||||
timeout=int(config["probe"]["timeoutSeconds"]),
|
||||
max_bytes=int(config["probe"]["maxResponseBytes"]),
|
||||
)
|
||||
resp = openai_responses_create(profile, config, marker, prompt)
|
||||
parsed = resp.get("json")
|
||||
out = output_text(parsed)
|
||||
out = resp.get("outputText") if isinstance(resp.get("outputText"), str) else output_text(parsed)
|
||||
trimmed = out.strip()
|
||||
marker_matched = trimmed == marker if config["marker"].get("exact", True) else marker in trimmed
|
||||
usage = usage_from(parsed if isinstance(parsed, dict) else {}, prompt, out or resp.get("text", ""), config)
|
||||
http_success = isinstance(resp.get("status"), int) and 200 <= resp.get("status") < 300
|
||||
ok = resp["ok"] and marker_matched
|
||||
mismatch = http_success and not marker_matched
|
||||
ok = marker_matched
|
||||
mismatch = not marker_matched
|
||||
if marker_matched:
|
||||
failure_kind = "none"
|
||||
elif resp.get("tooLarge"):
|
||||
failure_kind = "response-too-large"
|
||||
elif not resp["ok"]:
|
||||
failure_kind = "transport-or-http-failure"
|
||||
elif http_success:
|
||||
failure_kind = "success-body-mismatch"
|
||||
else:
|
||||
failure_kind = "unknown-marker-mismatch"
|
||||
return {
|
||||
"accountName": profile["accountName"],
|
||||
"profile": profile.get("profile"),
|
||||
@@ -760,11 +1042,16 @@ def probe_account(profile, config, purpose):
|
||||
"durationMs": resp.get("durationMs"),
|
||||
"outputHash": sha(out),
|
||||
"outputPreview": "" if marker_matched else preview(out or resp.get("text", ""), 160),
|
||||
"bodyPreviewHash": sha(resp.get("text", "")),
|
||||
"responseBodyHash": sha(resp.get("text", "")),
|
||||
"responseBodyPreview": "" if marker_matched else preview(resp.get("text", ""), 1000),
|
||||
"error": resp.get("error"),
|
||||
"errorDetails": resp.get("errorDetails"),
|
||||
"usage": usage,
|
||||
"mismatch": mismatch,
|
||||
"transportFailure": not resp["ok"],
|
||||
"failureKind": failure_kind,
|
||||
"sdk": resp.get("sdk"),
|
||||
"requestShape": resp.get("requestShape"),
|
||||
}
|
||||
|
||||
def ledger_for(state, now):
|
||||
@@ -806,6 +1093,28 @@ def choose_due_profiles(profiles, state, config, now):
|
||||
due.sort(key=lambda item: item["dueAt"] or "")
|
||||
return due, {"selected": len(due), "due": len(due), "limit": "all-due", "budgetMode": "record-only", "ledger": ledger}
|
||||
|
||||
def forced_account_names():
|
||||
raw = os.environ.get("SENTINEL_ACCOUNT_NAMES") or ""
|
||||
names = [item.strip() for item in raw.split(",") if item.strip()]
|
||||
return set(names)
|
||||
|
||||
def choose_forced_profiles(profiles, state, config, now, names):
|
||||
accounts = state.setdefault("accounts", {})
|
||||
found = []
|
||||
missing = sorted(names)
|
||||
due = []
|
||||
for profile in profiles:
|
||||
name = profile["accountName"]
|
||||
if name not in names:
|
||||
continue
|
||||
account_state = accounts.setdefault(name, {})
|
||||
quarantine = account_state.get("quarantine")
|
||||
purpose = "manual-recovery" if isinstance(quarantine, dict) and quarantine.get("active") is True else "manual-health"
|
||||
due.append({"profile": profile, "purpose": purpose, "dueAt": "forced"})
|
||||
found.append(name)
|
||||
missing = sorted(name for name in names if name not in set(found))
|
||||
return due, {"selected": len(due), "due": len(due), "limit": "forced-accounts", "budgetMode": "record-only", "ledger": ledger_for(state, now)[1], "requestedAccounts": sorted(names), "missingAccounts": missing}
|
||||
|
||||
def next_success_interval(account_state, config):
|
||||
streak = int(account_state.get("successStreak") or 0)
|
||||
previous = int(account_state.get("successIntervalMinutes") or 0)
|
||||
@@ -849,9 +1158,7 @@ def apply_result(result, state, config, now, admin):
|
||||
account_state["lastOkAt"] = iso(now)
|
||||
account_state["lastStatus"] = "ok"
|
||||
else:
|
||||
should_freeze = bool(result.get("mismatch")) and bool(config["actions"]["freezeOnMarkerMismatch"])
|
||||
if result.get("transportFailure") and bool(config["actions"].get("freezeOnTransportError")):
|
||||
should_freeze = True
|
||||
should_freeze = result.get("markerMatched") is not True
|
||||
if should_freeze:
|
||||
interval = next_freeze_interval(account_state, config, was_recovery)
|
||||
until = add_minutes(now, interval, int(config["freeze"]["jitterPercent"]))
|
||||
@@ -869,9 +1176,12 @@ def apply_result(result, state, config, now, admin):
|
||||
"applied": applied,
|
||||
"until": iso(until),
|
||||
"intervalMinutes": interval,
|
||||
"reason": "marker-mismatch" if result.get("mismatch") else "transport-failure",
|
||||
"reason": "marker-not-matched",
|
||||
"failureKind": result.get("failureKind"),
|
||||
"markerHash": result.get("markerHash"),
|
||||
"outputHash": result.get("outputHash"),
|
||||
"responseBodyHash": result.get("responseBodyHash"),
|
||||
"errorDetails": result.get("errorDetails"),
|
||||
"lastBadAt": iso(now),
|
||||
}
|
||||
account_state["nextProbeAfter"] = iso(until)
|
||||
@@ -881,7 +1191,7 @@ def apply_result(result, state, config, now, admin):
|
||||
else:
|
||||
retry = int(config["probe"]["transportRetryMinutes"])
|
||||
account_state["nextProbeAfter"] = iso(add_minutes(now, retry, int(config["cadence"]["jitterPercent"])))
|
||||
account_state["lastStatus"] = "transport-failed-no-freeze"
|
||||
account_state["lastStatus"] = "marker-not-matched-no-freeze"
|
||||
account_state["lastFailureAt"] = iso(now)
|
||||
account_state["lastProbeAt"] = iso(now)
|
||||
account_state["lastProbe"] = {
|
||||
@@ -890,9 +1200,17 @@ def apply_result(result, state, config, now, admin):
|
||||
"httpStatus": result.get("httpStatus"),
|
||||
"durationMs": result.get("durationMs"),
|
||||
"markerMatched": result.get("markerMatched"),
|
||||
"transportOk": result.get("transportOk"),
|
||||
"outputHash": result.get("outputHash"),
|
||||
"outputPreview": result.get("outputPreview"),
|
||||
"responseBodyHash": result.get("responseBodyHash"),
|
||||
"responseBodyPreview": result.get("responseBodyPreview"),
|
||||
"error": result.get("error"),
|
||||
"errorDetails": result.get("errorDetails"),
|
||||
"usage": result.get("usage"),
|
||||
"failureKind": result.get("failureKind"),
|
||||
"sdk": result.get("sdk"),
|
||||
"requestShape": result.get("requestShape"),
|
||||
"action": action,
|
||||
}
|
||||
return action
|
||||
@@ -909,7 +1227,13 @@ def reconcile_active_quarantines(state, config, now, admin):
|
||||
if until is not None and until <= now:
|
||||
continue
|
||||
if quarantine.get("applied") is not True:
|
||||
actions.append({"accountName": name, "type": "virtual-freeze-not-applied", "ok": True})
|
||||
try:
|
||||
admin.set_schedulable(name, False)
|
||||
quarantine["applied"] = True
|
||||
quarantine["appliedAt"] = iso(now)
|
||||
actions.append({"accountName": name, "type": "apply-pending-freeze", "ok": True})
|
||||
except Exception as exc:
|
||||
actions.append({"accountName": name, "type": "apply-pending-freeze", "ok": False, "error": str(exc)})
|
||||
continue
|
||||
try:
|
||||
admin.set_schedulable(name, False)
|
||||
@@ -927,10 +1251,14 @@ def main():
|
||||
state_obj, state = load_state(kube, config)
|
||||
admin = Sub2ApiAdmin(config)
|
||||
reconcile = reconcile_active_quarantines(state, config, now, admin)
|
||||
due, selection = choose_due_profiles(profiles, state, config, now)
|
||||
forced_names = forced_account_names()
|
||||
if forced_names:
|
||||
due, selection = choose_forced_profiles(profiles, state, config, now, forced_names)
|
||||
else:
|
||||
due, selection = choose_due_profiles(profiles, state, config, now)
|
||||
results = []
|
||||
actions = []
|
||||
if config["monitor"]["enabled"] and due:
|
||||
if (config["monitor"]["enabled"] or forced_names) and due:
|
||||
with ThreadPoolExecutor(max_workers=max(1, len(due))) as executor:
|
||||
futures = [executor.submit(probe_account, item["profile"], config, item["purpose"]) for item in due]
|
||||
for future in as_completed(futures):
|
||||
@@ -945,7 +1273,8 @@ def main():
|
||||
"profileCount": len(profiles),
|
||||
"selected": len(due),
|
||||
"okCount": sum(1 for item in results if item.get("ok") is True),
|
||||
"mismatchCount": sum(1 for item in results if item.get("mismatch") is True),
|
||||
"mismatchCount": sum(1 for item in results if item.get("markerMatched") is not True),
|
||||
"markerMismatchCount": sum(1 for item in results if item.get("markerMatched") is not True),
|
||||
"transportFailureCount": sum(1 for item in results if item.get("transportFailure") is True),
|
||||
"actionsTaken": sum(1 for item in actions if item.get("taken") is True),
|
||||
"selection": selection,
|
||||
@@ -968,7 +1297,13 @@ def main():
|
||||
"usage": item.get("usage"),
|
||||
"outputHash": item.get("outputHash"),
|
||||
"outputPreview": item.get("outputPreview"),
|
||||
"responseBodyHash": item.get("responseBodyHash"),
|
||||
"responseBodyPreview": item.get("responseBodyPreview"),
|
||||
"error": item.get("error"),
|
||||
"errorDetails": item.get("errorDetails"),
|
||||
"failureKind": item.get("failureKind"),
|
||||
"sdk": item.get("sdk"),
|
||||
"requestShape": item.get("requestShape"),
|
||||
} for item in results],
|
||||
"actions": actions,
|
||||
"valuesPrinted": False,
|
||||
@@ -1030,6 +1365,19 @@ function readMarkerPrefix(value: unknown, key: string, fallback: string): string
|
||||
return text;
|
||||
}
|
||||
|
||||
function readUserAgent(value: unknown, key: string, fallback: string): string {
|
||||
const text = readString(value, key, fallback);
|
||||
if (/[\r\n]/u.test(text)) throw new Error(`${key} must not contain newlines`);
|
||||
if (Buffer.byteLength(text, "utf8") > 200) throw new Error(`${key} must be at most 200 bytes`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function readOpenAiPythonVersion(value: unknown, key: string, fallback: string): string {
|
||||
const text = readString(value, key, fallback);
|
||||
if (!/^[0-9]+[.][0-9]+[.][0-9]+$/u.test(text)) throw new Error(`${key} must be a pinned semver version like 2.41.1`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function readInt(value: unknown, key: string, fallback: number, min: number, max: number): number {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
const parsed = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@ import { readFileSync } from "node:fs";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { rootPath } from "./config";
|
||||
import { startJob } from "./jobs";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
||||
|
||||
const g14K3sRoute = "G14:k3s";
|
||||
@@ -43,6 +44,8 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
||||
],
|
||||
description: "Operate the G14 k3s internal-only Sub2API deployment in the shared platform-infra namespace. This entry creates no Ingress, NodePort, LoadBalancer, hostPort, hostNetwork, ResourceQuota, LimitRange, or CPU/memory resource requests/limits.",
|
||||
target: {
|
||||
@@ -59,13 +62,14 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
],
|
||||
module: "scripts/src/platform-infra-sub2api-codex.ts",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function runPlatformInfraCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
||||
export async function runPlatformInfraCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const [target, action] = args;
|
||||
if (target !== "sub2api") return unsupported(args);
|
||||
if (action === "plan" || action === undefined) return plan();
|
||||
|
||||
Reference in New Issue
Block a user