feat: add marker-only sub2api sentinel reporting
This commit is contained in:
Codex
@@ -333,6 +333,11 @@ async function main(): Promise<void> {
|
||||
const { runPlatformInfraCommand } = await import("./src/platform-infra");
|
||||
const result = await runPlatformInfraCommand(readConfig(), args.slice(1));
|
||||
const ok = (result as { ok?: unknown }).ok !== false;
|
||||
if (isRenderedCliResult(result)) {
|
||||
emitText(result.renderedText);
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
|
||||
@@ -3,10 +3,13 @@ import { rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { rootPath } from "./src/config";
|
||||
import { codexPoolHelp, defaultCodexTempUnschedulablePolicy } from "./src/platform-infra-sub2api-codex";
|
||||
import {
|
||||
codexPoolSentinelRuntimeImage,
|
||||
defaultCodexPoolSentinelConfig,
|
||||
readCodexPoolSentinelConfig,
|
||||
renderCodexPoolSentinelManifest,
|
||||
sentinelContainerShellCommand,
|
||||
sentinelRunnerPython,
|
||||
} from "./src/platform-infra-sub2api-codex-sentinel";
|
||||
|
||||
@@ -15,21 +18,33 @@ function assertCondition(condition: unknown, message: string, detail: unknown =
|
||||
}
|
||||
|
||||
const configPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml");
|
||||
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as { sentinel?: unknown };
|
||||
const sentinelDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile");
|
||||
const parsed = Bun.YAML.parse(readFileSync(configPath, "utf8")) as {
|
||||
sentinel?: unknown;
|
||||
pool?: { defaultTempUnschedulable?: { rules?: Array<{ statusCode?: unknown }> } };
|
||||
};
|
||||
const sentinel = readCodexPoolSentinelConfig(parsed.sentinel, defaultCodexPoolSentinelConfig(), configPath);
|
||||
const sentinelRuntimeImage = codexPoolSentinelRuntimeImage(sentinel);
|
||||
const sentinelDockerfile = readFileSync(sentinelDockerfilePath, "utf8");
|
||||
const yamlTempUnschedulableRules = parsed.pool?.defaultTempUnschedulable?.rules ?? [];
|
||||
|
||||
assertCondition(sentinel.monitor.enabled === true, "sentinel monitor must be enabled for observation-first rollout", sentinel);
|
||||
assertCondition(sentinel.actions.enabled === false, "sentinel actions must default off until monitoring quality is reviewed", sentinel);
|
||||
assertCondition(sentinel.actions.freezeOnMarkerMismatch === true, "marker mismatch must be configured as freeze-worthy when actions are enabled", sentinel.actions);
|
||||
assertCondition(sentinel.actions.freezeOnTransportError === false, "transport errors must not freeze by default", sentinel.actions);
|
||||
assertCondition(sentinel.monitor.enabled === true, "sentinel monitor must be enabled for marker-only guard rollout", sentinel);
|
||||
assertCondition(sentinel.actions.enabled === true, "sentinel actions must be enabled so marker-only guard can freeze and recover accounts", sentinel);
|
||||
assertCondition(!yamlTempUnschedulableRules.some((rule) => rule.statusCode === 200), "native Sub2API temp-unschedulable policy must not classify HTTP 200 bodies; marker-only sentinel owns 200 semantic failures", yamlTempUnschedulableRules);
|
||||
assertCondition(!defaultCodexTempUnschedulablePolicy().rules.some((rule) => rule.statusCode === 200), "default temp-unschedulable policy must not reintroduce HTTP 200 body classifiers", defaultCodexTempUnschedulablePolicy());
|
||||
assertCondition(!("freezeOnMarkerMismatch" in sentinel.actions), "sentinel must not keep a marker-specific freeze branch; marker match is the only health standard", sentinel.actions);
|
||||
assertCondition(!("freezeOnTransportError" in sentinel.actions), "sentinel must not keep a transport-specific freeze branch; non-marker results all use the same freeze state machine", sentinel.actions);
|
||||
assertCondition(sentinel.endpoint === "responses", "v1 sentinel must target OpenAI Responses only", sentinel);
|
||||
assertCondition(sentinel.model === "gpt-5.5", "v1 sentinel must use GPT-5.5", sentinel);
|
||||
assertCondition(sentinel.probe.maxOutputTokens > 0 && sentinel.probe.maxOutputTokens <= 16, "sentinel maxOutputTokens must be tightly capped", sentinel.probe);
|
||||
assertCondition(sentinel.probe.maxOutputTokens > 0 && sentinel.probe.maxOutputTokens <= 16, "sentinel local stream capture limit must be tightly capped", sentinel.probe);
|
||||
assertCondition(!("maxResponseBytes" in sentinel.probe), "sentinel must not use hand-rolled response byte parsing for OpenAI model probes", sentinel.probe);
|
||||
assertCondition(sentinel.probe.userAgent === "Go-http-client/1.1", "sentinel default User-Agent must match Sub2API net/http account test shape", sentinel.probe);
|
||||
assertCondition(sentinel.sdk.openaiPythonVersion === "2.41.1", "sentinel must pin the OpenAI Python SDK version in YAML", sentinel.sdk);
|
||||
assertCondition(!("concurrency" in sentinel.probe), "sentinel must not cap probe concurrency; all due accounts are probed concurrently", sentinel.probe);
|
||||
assertCondition(!("maxAccountsPerRun" in sentinel.probe), "sentinel must not cap accounts per run; all due accounts are eligible", sentinel.probe);
|
||||
assertCondition(sentinel.cadence.successInitialIntervalMinutes === 1, "success trust backoff must start at 1 minute", sentinel.cadence);
|
||||
assertCondition(sentinel.cadence.successMaxIntervalMinutes === 20, "success trust backoff must cap at 20 minutes", sentinel.cadence);
|
||||
assertCondition(sentinel.freeze.initialTtlMinutes === 10, "freeze backoff must start at 10 minutes", sentinel.freeze);
|
||||
assertCondition(sentinel.freeze.initialTtlMinutes === 2, "freeze backoff must start at 2 minutes", sentinel.freeze);
|
||||
assertCondition(sentinel.freeze.maxTtlMinutes === 120, "freeze backoff must cap at 2 hours", sentinel.freeze);
|
||||
assertCondition(!("budget" in sentinel), "sentinel must not use token budgets as a probe gate; usage is recorded only", sentinel);
|
||||
|
||||
@@ -53,11 +68,40 @@ assertCondition(manifest.includes("concurrencyPolicy: Forbid"), "sentinel CronJo
|
||||
assertCondition(manifest.includes("suspend: false"), "monitor.enabled=true must unsuspend the CronJob", manifest);
|
||||
assertCondition(manifest.includes("kind: ServiceAccount") && manifest.includes("kind: Role") && manifest.includes("kind: RoleBinding"), "sentinel manifest must include minimal RBAC", manifest);
|
||||
assertCondition(manifest.includes("sub2api-account-sentinel-state"), "sentinel manifest must reference the state ConfigMap", manifest);
|
||||
assertCondition(manifest.includes("\"enabled\": false"), "sentinel manifest must preserve actions.enabled=false in config.json", manifest);
|
||||
assertCondition(manifest.includes("\"enabled\": true"), "sentinel manifest must preserve actions.enabled=true in config.json", manifest);
|
||||
assertCondition(!manifest.includes("sk-test-secret"), "sentinel manifest must not expose upstream credentials as plaintext", manifest);
|
||||
assertCondition(manifest.includes("profiles.json:"), "sentinel credentials Secret must include the profiles payload as Secret data", manifest);
|
||||
assertCondition(manifest.includes("\"budgetMode\": \"record-only\""), "sentinel runner must expose record-only budget/accounting mode", manifest);
|
||||
assertCondition(manifest.includes("max_workers=max(1, len(due))"), "sentinel runner must probe all due accounts concurrently", manifest);
|
||||
assertCondition(manifest.includes(`image: ${sentinelRuntimeImage.runtimeImage}`), "sentinel manifest must use the reusable prebuilt runtime image", { image: sentinelRuntimeImage.runtimeImage, manifest });
|
||||
assertCondition(!manifest.includes("transport-failed-no-freeze"), "sentinel runner must not exempt transport failures from marker-based freezing", manifest);
|
||||
const command = sentinelContainerShellCommand(sentinel);
|
||||
assertCondition(command.includes("openai-python-version-mismatch"), "sentinel command must fail fast when the image SDK version does not match YAML", command);
|
||||
assertCondition(!command.includes("pip install") && !command.includes("subprocess.check_call"), "sentinel command must not install Python packages at runtime", command);
|
||||
assertCondition(sentinelDockerfile.includes("ARG OPENAI_PYTHON_VERSION=2.41.1"), "sentinel Dockerfile must make the OpenAI SDK version a build arg with the current default", sentinelDockerfile);
|
||||
assertCondition(sentinelDockerfile.includes('"openai==${OPENAI_PYTHON_VERSION}"'), "sentinel Dockerfile must preinstall the pinned OpenAI SDK", sentinelDockerfile);
|
||||
|
||||
const help = codexPoolHelp() as { usage?: unknown };
|
||||
assertCondition(Array.isArray(help.usage) && help.usage.some((item) => typeof item === "string" && item.includes("sentinel-probe --account")), "codex-pool help must expose manual sentinel-probe by account", help);
|
||||
assertCondition(Array.isArray(help.usage) && help.usage.some((item) => typeof item === "string" && item.includes("sentinel-image build")), "codex-pool help must expose reusable sentinel image build", help);
|
||||
assertCondition(Array.isArray(help.usage) && help.usage.some((item) => typeof item === "string" && item.includes("sentinel-report")), "codex-pool help must expose low-noise sentinel-report", help);
|
||||
assertCondition(typeof (help as { output?: unknown }).output === "string" && String((help as { output?: unknown }).output).includes("ps-like text table"), "codex-pool help must document sentinel-report text table output", help);
|
||||
const runner = sentinelRunnerPython();
|
||||
assertCondition(runner.includes("from openai import APIConnectionError, APIStatusError, APITimeoutError, OpenAI"), "sentinel runner must use the standard OpenAI Python SDK", runner);
|
||||
assertCondition(runner.includes("client.responses.create(") && runner.includes("stream=True"), "sentinel runner must use the SDK Responses streaming create method", runner);
|
||||
assertCondition(runner.includes("sub2api_style_input(prompt)") && runner.includes("sub2api_style_instructions()"), "sentinel runner must mirror Sub2API WebUI default account test request shape", runner);
|
||||
assertCondition(runner.includes("extra_headers=headers"), "sentinel runner must pass configured User-Agent through SDK extra_headers", runner);
|
||||
assertCondition(!runner.includes("store=False"), "sentinel runner must not add store=false to API-key account probes", runner);
|
||||
assertCondition(!runner.includes("max_output_tokens="), "sentinel runner must not send max_output_tokens upstream for WebUI-compatible probes", runner);
|
||||
assertCondition(!runner.includes("Originator") && !runner.includes("Session_ID") && !runner.includes("OpenAI-Beta"), "sentinel runner must not add Codex/compact headers to default account probes", runner);
|
||||
assertCondition(!runner.includes("upstream_responses_url"), "sentinel runner must not hand-roll /v1/responses URLs for model probes", runner);
|
||||
assertCondition(runner.includes("def error_details("), "sentinel runner must emit structured error diagnostics for failed probes", runner);
|
||||
assertCondition(runner.includes('"openaiError": openai_error_fields(body)'), "sentinel diagnostics must expose OpenAI error type/code/message fields", runner);
|
||||
assertCondition(runner.includes('"responseBodyHash": result.get("responseBodyHash")'), "sentinel state must keep response body hashes for diagnostics", runner);
|
||||
assertCondition(runner.includes('"responseBodyPreview": item.get("responseBodyPreview")'), "sentinel CLI output must include bounded response body previews for diagnostics", runner);
|
||||
assertCondition(runner.includes("SENTINEL_ACCOUNT_NAMES"), "sentinel runner must support forced account probes for CLI manual measurement", runner);
|
||||
assertCondition(runner.includes('parsed.get("code") not in (None, 0)'), "sentinel admin client must treat Sub2API {code:0,message:success,data} envelopes as successful", runner);
|
||||
assertCondition(runner.includes("page_size=20&platform=openai&type=apikey&search="), "sentinel admin client must query one target account instead of fetching all accounts into the 64KiB admin response cap", runner);
|
||||
|
||||
const disabledMonitor = {
|
||||
...sentinel,
|
||||
@@ -92,13 +136,20 @@ console.log(JSON.stringify({
|
||||
ok: true,
|
||||
checks: [
|
||||
"sentinel has independent monitor/actions YAML switches",
|
||||
"observation-first rollout keeps actions disabled",
|
||||
"marker-only guard actions are enabled",
|
||||
"v1 scope is OpenAI Responses + GPT-5.5",
|
||||
"probe max_output_tokens is tightly capped",
|
||||
"probe local stream capture limit is tightly capped",
|
||||
"probe uses the standard OpenAI Python SDK streaming Responses API",
|
||||
"probe mirrors Sub2API WebUI default account test request shape",
|
||||
"probe passes configured User-Agent through SDK extra_headers",
|
||||
"OpenAI Python SDK version is YAML-pinned",
|
||||
"OpenAI Python SDK is preinstalled in a reusable sentinel image",
|
||||
"manual account probe CLI is exposed",
|
||||
"probe concurrency is not artificially capped",
|
||||
"marker match is the only health standard",
|
||||
"budget is record-only and does not gate probes",
|
||||
"success trust backoff is 1m to 20m",
|
||||
"freeze backoff is 10m to 120m",
|
||||
"freeze backoff is 2m to 120m",
|
||||
"CronJob is k8s-native with Forbid concurrency and minimal RBAC",
|
||||
"monitor switch controls CronJob suspend state",
|
||||
"rendered Secret avoids plaintext upstream credentials",
|
||||
|
||||
+4
-1
@@ -58,7 +58,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "hwlab nodes control-plane|git-mirror|secret --node G14 --lane v03", description: "Manage HWLAB node/lane runtime prerequisites for v0.3+ with the node identity passed as data instead of a command family." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." },
|
||||
{ command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|steer|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; legacy bridge groups remain available for raw compatibility." },
|
||||
{ command: "platform-infra sub2api plan|apply|status|validate|codex-pool", description: "Deploy Sub2API in G14 platform-infra, manage the YAML-controlled Codex upstream pool, expose the unified API through FRP when needed, and configure master ~/.codex without printing API keys." },
|
||||
{ command: "platform-infra sub2api plan|apply|status|validate|codex-pool", description: "Deploy Sub2API in G14 platform-infra, manage the YAML-controlled Codex upstream pool, expose the unified API, and inspect marker sentinel state with low-noise reports without printing API keys." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
@@ -622,6 +622,9 @@ function platformInfraHelpSummary(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api status [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report",
|
||||
],
|
||||
description: "Operate G14 platform-infra services such as Sub2API and the YAML-controlled Codex pool.",
|
||||
};
|
||||
|
||||
@@ -6,8 +6,6 @@ export interface CodexPoolSentinelConfig {
|
||||
};
|
||||
actions: {
|
||||
enabled: boolean;
|
||||
freezeOnMarkerMismatch: boolean;
|
||||
freezeOnTransportError: boolean;
|
||||
};
|
||||
schedule: string;
|
||||
image: string;
|
||||
@@ -24,9 +22,12 @@ export interface CodexPoolSentinelConfig {
|
||||
};
|
||||
probe: {
|
||||
timeoutSeconds: number;
|
||||
maxResponseBytes: number;
|
||||
maxOutputTokens: number;
|
||||
transportRetryMinutes: number;
|
||||
userAgent: string;
|
||||
};
|
||||
sdk: {
|
||||
openaiPythonVersion: string;
|
||||
};
|
||||
cadence: {
|
||||
successInitialIntervalMinutes: number;
|
||||
@@ -47,6 +48,13 @@ export interface CodexPoolSentinelConfig {
|
||||
historyLimit: number;
|
||||
}
|
||||
|
||||
export interface CodexPoolSentinelImageTarget {
|
||||
baseImage: string;
|
||||
runtimeImage: string;
|
||||
repository: string;
|
||||
tag: string;
|
||||
}
|
||||
|
||||
export interface CodexPoolSentinelProfileSecret {
|
||||
accountName: string;
|
||||
profile: string;
|
||||
@@ -69,8 +77,6 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
},
|
||||
actions: {
|
||||
enabled: false,
|
||||
freezeOnMarkerMismatch: true,
|
||||
freezeOnTransportError: false,
|
||||
},
|
||||
schedule: "*/1 * * * *",
|
||||
image: "python:3.12-alpine",
|
||||
@@ -87,9 +93,12 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
},
|
||||
probe: {
|
||||
timeoutSeconds: 30,
|
||||
maxResponseBytes: 8192,
|
||||
maxOutputTokens: 12,
|
||||
maxOutputTokens: 16,
|
||||
transportRetryMinutes: 5,
|
||||
userAgent: "Go-http-client/1.1",
|
||||
},
|
||||
sdk: {
|
||||
openaiPythonVersion: "2.41.1",
|
||||
},
|
||||
cadence: {
|
||||
successInitialIntervalMinutes: 1,
|
||||
@@ -98,7 +107,7 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
jitterPercent: 10,
|
||||
},
|
||||
freeze: {
|
||||
initialTtlMinutes: 10,
|
||||
initialTtlMinutes: 2,
|
||||
maxTtlMinutes: 120,
|
||||
backoffMultiplier: 2,
|
||||
jitterPercent: 10,
|
||||
@@ -111,12 +120,28 @@ export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig {
|
||||
};
|
||||
}
|
||||
|
||||
export function codexPoolSentinelRuntimeImage(config: CodexPoolSentinelConfig): CodexPoolSentinelImageTarget {
|
||||
const baseTag = config.image
|
||||
.replace(/[^A-Za-z0-9_.-]+/gu, "-")
|
||||
.replace(/^-+|-+$/gu, "")
|
||||
.slice(0, 80) || "python";
|
||||
const tag = `${baseTag}-openai-${config.sdk.openaiPythonVersion}`;
|
||||
const repository = "127.0.0.1:5000/platform-infra/sub2api-account-sentinel";
|
||||
return {
|
||||
baseImage: config.image,
|
||||
runtimeImage: `${repository}:${tag}`,
|
||||
repository,
|
||||
tag,
|
||||
};
|
||||
}
|
||||
|
||||
export function readCodexPoolSentinelConfig(value: unknown, defaults: CodexPoolSentinelConfig, sourcePath: string): CodexPoolSentinelConfig {
|
||||
if (!isRecord(value)) return defaults;
|
||||
const monitor = isRecord(value.monitor) ? value.monitor : {};
|
||||
const actions = isRecord(value.actions) ? value.actions : {};
|
||||
const marker = isRecord(value.marker) ? value.marker : {};
|
||||
const probe = isRecord(value.probe) ? value.probe : {};
|
||||
const sdk = isRecord(value.sdk) ? value.sdk : {};
|
||||
const cadence = isRecord(value.cadence) ? value.cadence : {};
|
||||
const freeze = isRecord(value.freeze) ? value.freeze : {};
|
||||
const pricing = isRecord(value.pricing) ? value.pricing : {};
|
||||
@@ -126,8 +151,6 @@ export function readCodexPoolSentinelConfig(value: unknown, defaults: CodexPoolS
|
||||
},
|
||||
actions: {
|
||||
enabled: readBoolean(valueAt(actions, "enabled"), `${sourcePath}.sentinel.actions.enabled`, defaults.actions.enabled),
|
||||
freezeOnMarkerMismatch: readBoolean(valueAt(actions, "freezeOnMarkerMismatch"), `${sourcePath}.sentinel.actions.freezeOnMarkerMismatch`, defaults.actions.freezeOnMarkerMismatch),
|
||||
freezeOnTransportError: readBoolean(valueAt(actions, "freezeOnTransportError"), `${sourcePath}.sentinel.actions.freezeOnTransportError`, defaults.actions.freezeOnTransportError),
|
||||
},
|
||||
schedule: readString(valueAt(value, "schedule"), `${sourcePath}.sentinel.schedule`, defaults.schedule),
|
||||
image: readString(valueAt(value, "image"), `${sourcePath}.sentinel.image`, defaults.image),
|
||||
@@ -144,9 +167,12 @@ export function readCodexPoolSentinelConfig(value: unknown, defaults: CodexPoolS
|
||||
},
|
||||
probe: {
|
||||
timeoutSeconds: readInt(valueAt(probe, "timeoutSeconds"), `${sourcePath}.sentinel.probe.timeoutSeconds`, defaults.probe.timeoutSeconds, 3, 300),
|
||||
maxResponseBytes: readInt(valueAt(probe, "maxResponseBytes"), `${sourcePath}.sentinel.probe.maxResponseBytes`, defaults.probe.maxResponseBytes, 1024, 1048576),
|
||||
maxOutputTokens: readInt(valueAt(probe, "maxOutputTokens"), `${sourcePath}.sentinel.probe.maxOutputTokens`, defaults.probe.maxOutputTokens, 1, 128),
|
||||
transportRetryMinutes: readInt(valueAt(probe, "transportRetryMinutes"), `${sourcePath}.sentinel.probe.transportRetryMinutes`, defaults.probe.transportRetryMinutes, 1, 120),
|
||||
userAgent: readUserAgent(valueAt(probe, "userAgent"), `${sourcePath}.sentinel.probe.userAgent`, defaults.probe.userAgent),
|
||||
},
|
||||
sdk: {
|
||||
openaiPythonVersion: readOpenAiPythonVersion(valueAt(sdk, "openaiPythonVersion"), `${sourcePath}.sentinel.sdk.openaiPythonVersion`, defaults.sdk.openaiPythonVersion),
|
||||
},
|
||||
cadence: {
|
||||
successInitialIntervalMinutes: readInt(valueAt(cadence, "successInitialIntervalMinutes"), `${sourcePath}.sentinel.cadence.successInitialIntervalMinutes`, defaults.cadence.successInitialIntervalMinutes, 1, 1440),
|
||||
@@ -197,6 +223,7 @@ export function codexPoolSentinelSummary(config: CodexPoolSentinelConfig): Recor
|
||||
model: config.model,
|
||||
endpoint: config.endpoint,
|
||||
probe: config.probe,
|
||||
sdk: config.sdk,
|
||||
cadence: config.cadence,
|
||||
freeze: config.freeze,
|
||||
accounting: {
|
||||
@@ -228,6 +255,7 @@ export function renderCodexPoolSentinelManifest(
|
||||
endpoint: config.endpoint,
|
||||
marker: config.marker,
|
||||
probe: config.probe,
|
||||
sdk: config.sdk,
|
||||
cadence: config.cadence,
|
||||
freeze: config.freeze,
|
||||
pricing: config.pricing,
|
||||
@@ -237,7 +265,9 @@ export function renderCodexPoolSentinelManifest(
|
||||
},
|
||||
};
|
||||
const suspend = config.monitor.enabled ? "false" : "true";
|
||||
const activeDeadlineSeconds = Math.max(120, Math.min(3600, config.probe.timeoutSeconds + 120));
|
||||
const activeDeadlineSeconds = Math.max(300, Math.min(3600, config.probe.timeoutSeconds + 240));
|
||||
const command = sentinelContainerShellCommand(config);
|
||||
const runtimeImage = codexPoolSentinelRuntimeImage(config).runtimeImage;
|
||||
return `apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
@@ -339,9 +369,11 @@ spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: sentinel
|
||||
image: ${config.image}
|
||||
image: ${runtimeImage}
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["python3", "/opt/sentinel/sentinel.py"]
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- ${JSON.stringify(command)}
|
||||
env:
|
||||
- name: ADMIN_EMAIL
|
||||
valueFrom:
|
||||
@@ -377,6 +409,23 @@ spec:
|
||||
`;
|
||||
}
|
||||
|
||||
export function sentinelContainerShellCommand(config: CodexPoolSentinelConfig): string {
|
||||
return [
|
||||
"set -eu",
|
||||
"python3 - <<'PY'",
|
||||
"import importlib.metadata",
|
||||
`expected = ${JSON.stringify(config.sdk.openaiPythonVersion)}`,
|
||||
"try:",
|
||||
" current = importlib.metadata.version('openai')",
|
||||
"except importlib.metadata.PackageNotFoundError:",
|
||||
" current = None",
|
||||
"if current != expected:",
|
||||
" raise SystemExit(f'openai-python-version-mismatch expected={expected} current={current}')",
|
||||
"PY",
|
||||
"exec python3 /opt/sentinel/sentinel.py",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function sentinelRunnerPython(): string {
|
||||
return String.raw`#!/usr/bin/env python3
|
||||
import base64
|
||||
@@ -390,7 +439,8 @@ import time
|
||||
import traceback
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib import error, request
|
||||
from urllib import error, parse, request
|
||||
from openai import APIConnectionError, APIStatusError, APITimeoutError, OpenAI
|
||||
|
||||
CONFIG_PATH = "/opt/sentinel/config.json"
|
||||
PROFILES_PATH = "/opt/sentinel-secrets/profiles.json"
|
||||
@@ -547,8 +597,9 @@ def http_json(method, url, headers=None, payload=None, timeout=30, max_bytes=655
|
||||
parsed = json.loads(text) if text.strip() else None
|
||||
except Exception:
|
||||
parsed = None
|
||||
app_success = not (isinstance(parsed, dict) and parsed.get("code") not in (None, 0))
|
||||
return {
|
||||
"ok": 200 <= resp.status < 300 and not too_large,
|
||||
"ok": 200 <= resp.status < 300 and not too_large and app_success,
|
||||
"status": resp.status,
|
||||
"json": parsed,
|
||||
"text": text,
|
||||
@@ -599,6 +650,9 @@ def find_token(value):
|
||||
return found
|
||||
return None
|
||||
|
||||
def url_quote(value):
|
||||
return parse.quote(str(value), safe="")
|
||||
|
||||
class Sub2ApiAdmin:
|
||||
def __init__(self, config):
|
||||
self.base = config["service"]["baseUrl"].rstrip("/")
|
||||
@@ -647,15 +701,32 @@ class Sub2ApiAdmin:
|
||||
self.accounts_by_name = {item.get("name"): item for item in items if isinstance(item, dict) and isinstance(item.get("name"), str)}
|
||||
return self.accounts_by_name
|
||||
|
||||
def account(self, account_name):
|
||||
if self.accounts_by_name is not None and account_name in self.accounts_by_name:
|
||||
return self.accounts_by_name[account_name]
|
||||
data = self.request("GET", "/api/v1/admin/accounts?page=1&page_size=20&platform=openai&type=apikey&search=" + url_quote(account_name))
|
||||
items = data if isinstance(data, list) else []
|
||||
if isinstance(data, dict):
|
||||
for key in ("items", "accounts"):
|
||||
if isinstance(data.get(key), list):
|
||||
items = data[key]
|
||||
break
|
||||
for item in items:
|
||||
if isinstance(item, dict) and item.get("name") == account_name:
|
||||
if self.accounts_by_name is not None:
|
||||
self.accounts_by_name[account_name] = item
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_schedulable(self, account_name, schedulable):
|
||||
account = self.accounts().get(account_name)
|
||||
account = self.account(account_name)
|
||||
if not account or account.get("id") is None:
|
||||
raise RuntimeError(f"account {account_name} not found")
|
||||
self.request("POST", f"/api/v1/admin/accounts/{account['id']}/schedulable", {"schedulable": bool(schedulable)})
|
||||
return {"accountId": account.get("id"), "schedulable": bool(schedulable)}
|
||||
|
||||
def recover_state(self, account_name):
|
||||
account = self.accounts().get(account_name)
|
||||
account = self.account(account_name)
|
||||
if not account or account.get("id") is None:
|
||||
return {"skipped": True, "reason": "account-not-found"}
|
||||
try:
|
||||
@@ -664,11 +735,9 @@ class Sub2ApiAdmin:
|
||||
except Exception as exc:
|
||||
return {"ok": False, "accountId": account.get("id"), "error": str(exc)}
|
||||
|
||||
def upstream_responses_url(base_url):
|
||||
def upstream_base_url(base_url):
|
||||
base = str(base_url).rstrip("/")
|
||||
if base.endswith("/v1"):
|
||||
return base + "/responses"
|
||||
return base + "/v1/responses"
|
||||
return base if base.endswith("/v1") else base + "/v1"
|
||||
|
||||
def output_text(parsed):
|
||||
if isinstance(parsed, dict) and isinstance(parsed.get("output_text"), str):
|
||||
@@ -687,6 +756,230 @@ def output_text(parsed):
|
||||
parts.append(block["text"])
|
||||
return "\n".join(parts)
|
||||
|
||||
def model_dump(value):
|
||||
if hasattr(value, "model_dump"):
|
||||
return value.model_dump()
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {}
|
||||
|
||||
def body_text(value):
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
except Exception:
|
||||
return str(value)
|
||||
|
||||
def redact_diagnostic(value):
|
||||
if isinstance(value, dict):
|
||||
redacted = {}
|
||||
for key, item in value.items():
|
||||
key_text = str(key)
|
||||
if any(token in key_text.lower() for token in ("key", "token", "secret", "password", "credential", "authorization")):
|
||||
redacted[key_text] = "[redacted]"
|
||||
else:
|
||||
redacted[key_text] = redact_diagnostic(item)
|
||||
return redacted
|
||||
if isinstance(value, list):
|
||||
return [redact_diagnostic(item) for item in value[:20]]
|
||||
if isinstance(value, str):
|
||||
return value if len(value) <= 2000 else value[:2000] + "...[truncated]"
|
||||
if isinstance(value, (int, float, bool)) or value is None:
|
||||
return value
|
||||
return str(value)
|
||||
|
||||
def selected_headers(headers):
|
||||
if headers is None:
|
||||
return {}
|
||||
selected = {}
|
||||
for key in (
|
||||
"content-type",
|
||||
"x-request-id",
|
||||
"x-ratelimit-limit-requests",
|
||||
"x-ratelimit-remaining-requests",
|
||||
"x-ratelimit-reset-requests",
|
||||
"cf-ray",
|
||||
"server",
|
||||
):
|
||||
try:
|
||||
value = headers.get(key)
|
||||
except Exception:
|
||||
value = None
|
||||
if value:
|
||||
selected[key] = str(value)
|
||||
return selected
|
||||
|
||||
def openai_error_fields(body):
|
||||
if not isinstance(body, dict):
|
||||
return {}
|
||||
error_obj = body.get("error")
|
||||
if isinstance(error_obj, dict):
|
||||
return {
|
||||
"message": error_obj.get("message"),
|
||||
"type": error_obj.get("type"),
|
||||
"param": error_obj.get("param"),
|
||||
"code": error_obj.get("code"),
|
||||
}
|
||||
return {
|
||||
"message": body.get("message"),
|
||||
"type": body.get("type"),
|
||||
"param": body.get("param"),
|
||||
"code": body.get("code"),
|
||||
}
|
||||
|
||||
def error_details(kind, status, body=None, message=None, headers=None):
|
||||
text = body_text(body)
|
||||
return {
|
||||
"kind": kind,
|
||||
"statusCode": status,
|
||||
"message": str(message) if message else None,
|
||||
"openaiError": openai_error_fields(body),
|
||||
"body": redact_diagnostic(body) if isinstance(body, (dict, list)) else None,
|
||||
"bodyHash": sha(text),
|
||||
"bodyPreview": preview(text, 1000),
|
||||
"headers": selected_headers(headers),
|
||||
}
|
||||
|
||||
def sub2api_style_input(prompt):
|
||||
return [{
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": prompt,
|
||||
}],
|
||||
}]
|
||||
|
||||
def sub2api_style_instructions():
|
||||
return (
|
||||
"You are Codex, based on GPT-5. You are running as a coding agent in the Codex CLI on a user's computer."
|
||||
)
|
||||
|
||||
def event_type(event):
|
||||
if isinstance(event, dict):
|
||||
return event.get("type")
|
||||
return getattr(event, "type", None)
|
||||
|
||||
def event_delta(event):
|
||||
if isinstance(event, dict):
|
||||
value = event.get("delta")
|
||||
return value if isinstance(value, str) else ""
|
||||
value = getattr(event, "delta", "")
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
def event_error_message(event):
|
||||
data = model_dump(event)
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data.get("error"), dict):
|
||||
message = data["error"].get("message")
|
||||
if isinstance(message, str) and message:
|
||||
return message
|
||||
if isinstance(data.get("response"), dict) and isinstance(data["response"].get("error"), dict):
|
||||
message = data["response"]["error"].get("message")
|
||||
if isinstance(message, str) and message:
|
||||
return message
|
||||
return None
|
||||
|
||||
def openai_responses_create(profile, config, marker, prompt):
|
||||
headers = {
|
||||
"User-Agent": profile.get("upstreamUserAgent") or config["probe"].get("userAgent") or "Go-http-client/1.1",
|
||||
"X-Request-ID": "unidesk-account-sentinel-" + hashlib.sha256(marker.encode()).hexdigest()[:16],
|
||||
}
|
||||
client = OpenAI(
|
||||
api_key=profile["apiKey"],
|
||||
base_url=upstream_base_url(profile["baseUrl"]),
|
||||
timeout=float(config["probe"]["timeoutSeconds"]),
|
||||
max_retries=0,
|
||||
)
|
||||
started = time.time()
|
||||
try:
|
||||
stream = client.responses.create(
|
||||
model=config["model"],
|
||||
input=sub2api_style_input(prompt),
|
||||
instructions=sub2api_style_instructions(),
|
||||
stream=True,
|
||||
extra_headers=headers,
|
||||
)
|
||||
deltas = []
|
||||
events = []
|
||||
seen_completed = False
|
||||
max_chars = max(32, int(config["probe"]["maxOutputTokens"]) * 12)
|
||||
for event in stream:
|
||||
event_data = model_dump(event)
|
||||
etype = event_type(event)
|
||||
events.append({"type": etype, "preview": preview(body_text(event_data), 240)})
|
||||
if etype == "response.output_text.delta":
|
||||
delta = event_delta(event)
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
if len("".join(deltas)) > max_chars:
|
||||
break
|
||||
elif etype in ("response.completed", "response.done"):
|
||||
seen_completed = True
|
||||
break
|
||||
elif etype in ("response.failed", "error"):
|
||||
message = event_error_message(event) or "OpenAI response failed"
|
||||
raise RuntimeError(message)
|
||||
out = "".join(deltas)
|
||||
parsed = {"stream": True, "completed": seen_completed, "events": events[-20:], "output_text": out}
|
||||
if not seen_completed:
|
||||
parsed["streamError"] = "stream ended before response.completed"
|
||||
return {
|
||||
"ok": seen_completed,
|
||||
"status": 200 if seen_completed else 0,
|
||||
"json": parsed,
|
||||
"outputText": out,
|
||||
"text": body_text(parsed),
|
||||
"tooLarge": not seen_completed and len(out) > max_chars,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
except APIStatusError as exc:
|
||||
status = getattr(exc, "status_code", 0) or 0
|
||||
body = getattr(exc, "body", None)
|
||||
response = getattr(exc, "response", None)
|
||||
return {
|
||||
"ok": False,
|
||||
"status": status,
|
||||
"json": body if isinstance(body, dict) else None,
|
||||
"text": body_text(body or response or ""),
|
||||
"tooLarge": False,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"error": str(exc),
|
||||
"errorDetails": error_details("APIStatusError", status, body, str(exc), getattr(response, "headers", None)),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
except (APITimeoutError, APIConnectionError) as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"status": 0,
|
||||
"json": None,
|
||||
"text": "",
|
||||
"tooLarge": False,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"error": str(exc),
|
||||
"errorDetails": error_details(exc.__class__.__name__, 0, None, str(exc), None),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"status": 0,
|
||||
"json": None,
|
||||
"text": "",
|
||||
"tooLarge": False,
|
||||
"durationMs": int((time.time() - started) * 1000),
|
||||
"error": str(exc),
|
||||
"errorDetails": error_details(exc.__class__.__name__, 0, None, str(exc), None),
|
||||
"sdk": "openai-python",
|
||||
"requestShape": "sub2api-account-test-streaming-responses",
|
||||
}
|
||||
|
||||
def usage_from(parsed, prompt, out, config):
|
||||
usage = parsed.get("usage") if isinstance(parsed, dict) and isinstance(parsed.get("usage"), dict) else {}
|
||||
input_tokens = usage.get("input_tokens")
|
||||
@@ -717,36 +1010,25 @@ def usage_from(parsed, prompt, out, config):
|
||||
def probe_account(profile, config, purpose):
|
||||
marker = config["marker"]["prefix"] + "_" + hashlib.sha256((profile["accountName"] + str(time.time()) + str(random.random())).encode()).hexdigest()[:10]
|
||||
prompt = "Return exactly this marker and no other text: " + marker
|
||||
payload = {
|
||||
"model": config["model"],
|
||||
"input": prompt,
|
||||
"stream": False,
|
||||
"store": False,
|
||||
"max_output_tokens": int(config["probe"]["maxOutputTokens"]),
|
||||
}
|
||||
headers = {
|
||||
"Authorization": "Bearer " + profile["apiKey"],
|
||||
"Content-Type": "application/json",
|
||||
"X-Request-ID": "unidesk-account-sentinel-" + hashlib.sha256(marker.encode()).hexdigest()[:16],
|
||||
}
|
||||
if profile.get("upstreamUserAgent"):
|
||||
headers["User-Agent"] = profile["upstreamUserAgent"]
|
||||
resp = http_json(
|
||||
"POST",
|
||||
upstream_responses_url(profile["baseUrl"]),
|
||||
headers,
|
||||
payload,
|
||||
timeout=int(config["probe"]["timeoutSeconds"]),
|
||||
max_bytes=int(config["probe"]["maxResponseBytes"]),
|
||||
)
|
||||
resp = openai_responses_create(profile, config, marker, prompt)
|
||||
parsed = resp.get("json")
|
||||
out = output_text(parsed)
|
||||
out = resp.get("outputText") if isinstance(resp.get("outputText"), str) else output_text(parsed)
|
||||
trimmed = out.strip()
|
||||
marker_matched = trimmed == marker if config["marker"].get("exact", True) else marker in trimmed
|
||||
usage = usage_from(parsed if isinstance(parsed, dict) else {}, prompt, out or resp.get("text", ""), config)
|
||||
http_success = isinstance(resp.get("status"), int) and 200 <= resp.get("status") < 300
|
||||
ok = resp["ok"] and marker_matched
|
||||
mismatch = http_success and not marker_matched
|
||||
ok = marker_matched
|
||||
mismatch = not marker_matched
|
||||
if marker_matched:
|
||||
failure_kind = "none"
|
||||
elif resp.get("tooLarge"):
|
||||
failure_kind = "response-too-large"
|
||||
elif not resp["ok"]:
|
||||
failure_kind = "transport-or-http-failure"
|
||||
elif http_success:
|
||||
failure_kind = "success-body-mismatch"
|
||||
else:
|
||||
failure_kind = "unknown-marker-mismatch"
|
||||
return {
|
||||
"accountName": profile["accountName"],
|
||||
"profile": profile.get("profile"),
|
||||
@@ -760,11 +1042,16 @@ def probe_account(profile, config, purpose):
|
||||
"durationMs": resp.get("durationMs"),
|
||||
"outputHash": sha(out),
|
||||
"outputPreview": "" if marker_matched else preview(out or resp.get("text", ""), 160),
|
||||
"bodyPreviewHash": sha(resp.get("text", "")),
|
||||
"responseBodyHash": sha(resp.get("text", "")),
|
||||
"responseBodyPreview": "" if marker_matched else preview(resp.get("text", ""), 1000),
|
||||
"error": resp.get("error"),
|
||||
"errorDetails": resp.get("errorDetails"),
|
||||
"usage": usage,
|
||||
"mismatch": mismatch,
|
||||
"transportFailure": not resp["ok"],
|
||||
"failureKind": failure_kind,
|
||||
"sdk": resp.get("sdk"),
|
||||
"requestShape": resp.get("requestShape"),
|
||||
}
|
||||
|
||||
def ledger_for(state, now):
|
||||
@@ -806,6 +1093,28 @@ def choose_due_profiles(profiles, state, config, now):
|
||||
due.sort(key=lambda item: item["dueAt"] or "")
|
||||
return due, {"selected": len(due), "due": len(due), "limit": "all-due", "budgetMode": "record-only", "ledger": ledger}
|
||||
|
||||
def forced_account_names():
|
||||
raw = os.environ.get("SENTINEL_ACCOUNT_NAMES") or ""
|
||||
names = [item.strip() for item in raw.split(",") if item.strip()]
|
||||
return set(names)
|
||||
|
||||
def choose_forced_profiles(profiles, state, config, now, names):
|
||||
accounts = state.setdefault("accounts", {})
|
||||
found = []
|
||||
missing = sorted(names)
|
||||
due = []
|
||||
for profile in profiles:
|
||||
name = profile["accountName"]
|
||||
if name not in names:
|
||||
continue
|
||||
account_state = accounts.setdefault(name, {})
|
||||
quarantine = account_state.get("quarantine")
|
||||
purpose = "manual-recovery" if isinstance(quarantine, dict) and quarantine.get("active") is True else "manual-health"
|
||||
due.append({"profile": profile, "purpose": purpose, "dueAt": "forced"})
|
||||
found.append(name)
|
||||
missing = sorted(name for name in names if name not in set(found))
|
||||
return due, {"selected": len(due), "due": len(due), "limit": "forced-accounts", "budgetMode": "record-only", "ledger": ledger_for(state, now)[1], "requestedAccounts": sorted(names), "missingAccounts": missing}
|
||||
|
||||
def next_success_interval(account_state, config):
|
||||
streak = int(account_state.get("successStreak") or 0)
|
||||
previous = int(account_state.get("successIntervalMinutes") or 0)
|
||||
@@ -849,9 +1158,7 @@ def apply_result(result, state, config, now, admin):
|
||||
account_state["lastOkAt"] = iso(now)
|
||||
account_state["lastStatus"] = "ok"
|
||||
else:
|
||||
should_freeze = bool(result.get("mismatch")) and bool(config["actions"]["freezeOnMarkerMismatch"])
|
||||
if result.get("transportFailure") and bool(config["actions"].get("freezeOnTransportError")):
|
||||
should_freeze = True
|
||||
should_freeze = result.get("markerMatched") is not True
|
||||
if should_freeze:
|
||||
interval = next_freeze_interval(account_state, config, was_recovery)
|
||||
until = add_minutes(now, interval, int(config["freeze"]["jitterPercent"]))
|
||||
@@ -869,9 +1176,12 @@ def apply_result(result, state, config, now, admin):
|
||||
"applied": applied,
|
||||
"until": iso(until),
|
||||
"intervalMinutes": interval,
|
||||
"reason": "marker-mismatch" if result.get("mismatch") else "transport-failure",
|
||||
"reason": "marker-not-matched",
|
||||
"failureKind": result.get("failureKind"),
|
||||
"markerHash": result.get("markerHash"),
|
||||
"outputHash": result.get("outputHash"),
|
||||
"responseBodyHash": result.get("responseBodyHash"),
|
||||
"errorDetails": result.get("errorDetails"),
|
||||
"lastBadAt": iso(now),
|
||||
}
|
||||
account_state["nextProbeAfter"] = iso(until)
|
||||
@@ -881,7 +1191,7 @@ def apply_result(result, state, config, now, admin):
|
||||
else:
|
||||
retry = int(config["probe"]["transportRetryMinutes"])
|
||||
account_state["nextProbeAfter"] = iso(add_minutes(now, retry, int(config["cadence"]["jitterPercent"])))
|
||||
account_state["lastStatus"] = "transport-failed-no-freeze"
|
||||
account_state["lastStatus"] = "marker-not-matched-no-freeze"
|
||||
account_state["lastFailureAt"] = iso(now)
|
||||
account_state["lastProbeAt"] = iso(now)
|
||||
account_state["lastProbe"] = {
|
||||
@@ -890,9 +1200,17 @@ def apply_result(result, state, config, now, admin):
|
||||
"httpStatus": result.get("httpStatus"),
|
||||
"durationMs": result.get("durationMs"),
|
||||
"markerMatched": result.get("markerMatched"),
|
||||
"transportOk": result.get("transportOk"),
|
||||
"outputHash": result.get("outputHash"),
|
||||
"outputPreview": result.get("outputPreview"),
|
||||
"responseBodyHash": result.get("responseBodyHash"),
|
||||
"responseBodyPreview": result.get("responseBodyPreview"),
|
||||
"error": result.get("error"),
|
||||
"errorDetails": result.get("errorDetails"),
|
||||
"usage": result.get("usage"),
|
||||
"failureKind": result.get("failureKind"),
|
||||
"sdk": result.get("sdk"),
|
||||
"requestShape": result.get("requestShape"),
|
||||
"action": action,
|
||||
}
|
||||
return action
|
||||
@@ -909,7 +1227,13 @@ def reconcile_active_quarantines(state, config, now, admin):
|
||||
if until is not None and until <= now:
|
||||
continue
|
||||
if quarantine.get("applied") is not True:
|
||||
actions.append({"accountName": name, "type": "virtual-freeze-not-applied", "ok": True})
|
||||
try:
|
||||
admin.set_schedulable(name, False)
|
||||
quarantine["applied"] = True
|
||||
quarantine["appliedAt"] = iso(now)
|
||||
actions.append({"accountName": name, "type": "apply-pending-freeze", "ok": True})
|
||||
except Exception as exc:
|
||||
actions.append({"accountName": name, "type": "apply-pending-freeze", "ok": False, "error": str(exc)})
|
||||
continue
|
||||
try:
|
||||
admin.set_schedulable(name, False)
|
||||
@@ -927,10 +1251,14 @@ def main():
|
||||
state_obj, state = load_state(kube, config)
|
||||
admin = Sub2ApiAdmin(config)
|
||||
reconcile = reconcile_active_quarantines(state, config, now, admin)
|
||||
due, selection = choose_due_profiles(profiles, state, config, now)
|
||||
forced_names = forced_account_names()
|
||||
if forced_names:
|
||||
due, selection = choose_forced_profiles(profiles, state, config, now, forced_names)
|
||||
else:
|
||||
due, selection = choose_due_profiles(profiles, state, config, now)
|
||||
results = []
|
||||
actions = []
|
||||
if config["monitor"]["enabled"] and due:
|
||||
if (config["monitor"]["enabled"] or forced_names) and due:
|
||||
with ThreadPoolExecutor(max_workers=max(1, len(due))) as executor:
|
||||
futures = [executor.submit(probe_account, item["profile"], config, item["purpose"]) for item in due]
|
||||
for future in as_completed(futures):
|
||||
@@ -945,7 +1273,8 @@ def main():
|
||||
"profileCount": len(profiles),
|
||||
"selected": len(due),
|
||||
"okCount": sum(1 for item in results if item.get("ok") is True),
|
||||
"mismatchCount": sum(1 for item in results if item.get("mismatch") is True),
|
||||
"mismatchCount": sum(1 for item in results if item.get("markerMatched") is not True),
|
||||
"markerMismatchCount": sum(1 for item in results if item.get("markerMatched") is not True),
|
||||
"transportFailureCount": sum(1 for item in results if item.get("transportFailure") is True),
|
||||
"actionsTaken": sum(1 for item in actions if item.get("taken") is True),
|
||||
"selection": selection,
|
||||
@@ -968,7 +1297,13 @@ def main():
|
||||
"usage": item.get("usage"),
|
||||
"outputHash": item.get("outputHash"),
|
||||
"outputPreview": item.get("outputPreview"),
|
||||
"responseBodyHash": item.get("responseBodyHash"),
|
||||
"responseBodyPreview": item.get("responseBodyPreview"),
|
||||
"error": item.get("error"),
|
||||
"errorDetails": item.get("errorDetails"),
|
||||
"failureKind": item.get("failureKind"),
|
||||
"sdk": item.get("sdk"),
|
||||
"requestShape": item.get("requestShape"),
|
||||
} for item in results],
|
||||
"actions": actions,
|
||||
"valuesPrinted": False,
|
||||
@@ -1030,6 +1365,19 @@ function readMarkerPrefix(value: unknown, key: string, fallback: string): string
|
||||
return text;
|
||||
}
|
||||
|
||||
function readUserAgent(value: unknown, key: string, fallback: string): string {
|
||||
const text = readString(value, key, fallback);
|
||||
if (/[\r\n]/u.test(text)) throw new Error(`${key} must not contain newlines`);
|
||||
if (Buffer.byteLength(text, "utf8") > 200) throw new Error(`${key} must be at most 200 bytes`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function readOpenAiPythonVersion(value: unknown, key: string, fallback: string): string {
|
||||
const text = readString(value, key, fallback);
|
||||
if (!/^[0-9]+[.][0-9]+[.][0-9]+$/u.test(text)) throw new Error(`${key} must be a pinned semver version like 2.41.1`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function readInt(value: unknown, key: string, fallback: number, min: number, max: number): number {
|
||||
if (value === undefined || value === null) return fallback;
|
||||
const parsed = typeof value === "number" ? value : typeof value === "string" && value.trim() ? Number(value) : Number.NaN;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@ import { readFileSync } from "node:fs";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { rootPath } from "./config";
|
||||
import { startJob } from "./jobs";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { runSshCommandCapture, type SshCaptureResult } from "./ssh";
|
||||
|
||||
const g14K3sRoute = "G14:k3s";
|
||||
@@ -43,6 +44,8 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
||||
],
|
||||
description: "Operate the G14 k3s internal-only Sub2API deployment in the shared platform-infra namespace. This entry creates no Ingress, NodePort, LoadBalancer, hostPort, hostNetwork, ResourceQuota, LimitRange, or CPU/memory resource requests/limits.",
|
||||
target: {
|
||||
@@ -59,13 +62,14 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
],
|
||||
module: "scripts/src/platform-infra-sub2api-codex.ts",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function runPlatformInfraCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
||||
export async function runPlatformInfraCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const [target, action] = args;
|
||||
if (target !== "sub2api") return unsupported(args);
|
||||
if (action === "plan" || action === undefined) return plan();
|
||||
|
||||
Reference in New Issue
Block a user