Merge remote-tracking branch 'origin/master' into feat/sub2api-runtime-batch-reconcile
This commit is contained in:
Codex
@@ -100,14 +100,14 @@ export function ghHelp(): unknown {
|
||||
"Commander brief ClaudeQQ defaults to private target 645275593 through backend-core /api/microservices/claudeqq/proxy; UNIDESK_COMMANDER_BRIEF_CLAUDEQQ_* env vars can override target, base URL, timeout, and enabled state.",
|
||||
"comment update/edit PATCHes /repos/{owner}/{repo}/issues/comments/{comment_id} and preserves the comment id/timeline; comment delete is supported because GitHub supports deleting issue comments, but routine wording fixes should use update/edit. issue/pr hard delete is unsupported and close is the lifecycle alternative.",
|
||||
"PR files is the canonical compact changed-file/stat summary. It uses GitHub REST, returns bounded file rows, additions/deletions/changes when available, truncation metadata, and a next command for full details. Raw diff patches are not emitted by default; gh pr diff <number> --stat is a compatibility alias for the same JSON summary.",
|
||||
"PR review-plan is the review-first bounded patch index. It uses GitHub REST PR files, prints changed files with additions/deletions/hunk counts/patch-line counts/default truncation flags, and emits one per-file gh pr diff --file drill-down command without creating a local worktree.",
|
||||
"PR review-plan is the review-first bounded patch index. It uses GitHub REST PR files, prints changed files with additions/deletions/hunk counts/patch-line counts/default truncation flags, and emits per-file gh pr diff --file drill-down commands without creating a local worktree. Use only the drill-downs needed for review; do not mechanically expand every file.",
|
||||
"PR diff --file reads one changed file patch from GitHub REST and prints only a bounded excerpt by default. Add --hunk N to inspect one hunk, --limit N to change displayed patch lines, or --full/--raw for explicit structured full-patch disclosure. --stat remains the no-patch file/stat compatibility path.",
|
||||
"PR edit/update PATCHes /repos/{owner}/{repo}/pulls/{number} through REST only, never GitHub Projects Classic GraphQL/projectCards, and returns low-noise JSON with repo, PR number, changedFields, url, and body size/SHA metadata instead of echoing the full body.",
|
||||
"PR view is the canonical bounded metadata summary; read remains a UniDesk compatibility alias. Human full-body reading uses `trans gh:/owner/repo/pr/<number> cat`, and targeted lookup uses the same route with `rg <pattern>`. `--json body`, `--full`, and `--raw` are explicit structured machine disclosure only. View/read retain positional numbers, GitHub URLs, owner/repo#number shorthand, compatible --number, REST closeout fields, and on-demand GraphQL closeout metadata.",
|
||||
"PR preflight/closeout accept the same owner/repo#number shorthand as PR view/read so merge readiness checks do not require repeating --repo after a PR URL has already been normalized.",
|
||||
"PR list does not fetch mergeability or statusCheckRollup; request those closeout fields with gh pr view <number> --json headRefName,baseRefName,mergeable,mergeStateStatus,statusCheckRollup.",
|
||||
"PR preflight is a low-noise read-only closeout helper for explicit diagnosis only. It combines redacted auth capability, PR branch/state metadata, mergeability, mergeStateStatus, compact status check counts, and an explicit read-only policy. It is not a required step before gh pr merge. Use --full or --raw to include all fetched status contexts.",
|
||||
"PR merge is the one-command guarded write path: it reads closeout metadata itself, retries GitHub UNKNOWN/null mergeability with YAML-configured exponential backoff, refuses non-open/draft/conflicting/non-clean/failed/pending PRs, then uses GitHub REST merge. It defaults to deleting the merged same-repo head branch, cleaning the matching local .worktree when clean, and fast-forwarding the local main worktree on the PR base branch; --sync-node NODE additionally runs mapped node source-workspace sync. Use --dry-run to see the exact merge and closeout plan without writing.",
|
||||
"PR merge is the one-command guarded write path: it reads closeout metadata itself, retries GitHub UNKNOWN/null mergeability with YAML-configured exponential backoff, refuses non-open/draft/conflicting/non-clean/failed/pending PRs, then uses GitHub REST merge. Its success summary includes mergeCommit and mergedAt, so a routine follow-up pr view is unnecessary. It defaults to deleting the merged same-repo head branch, cleaning the matching local .worktree when clean, and fast-forwarding the local main worktree on the PR base branch; --sync-node NODE additionally runs mapped node source-workspace sync. Use --dry-run to see the exact merge and closeout plan without writing.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -169,6 +169,12 @@ export function renderPrMergeTable(result: GitHubCommandResult, fallbackDetails?
|
||||
: {};
|
||||
const mergeMethod = result.method ?? result.mergeMethod ?? details.method;
|
||||
const deleteBranch = result.deleteBranch ?? details.deleteBranch;
|
||||
const mergeResult = isRecord(result.mergeResult) ? result.mergeResult : {};
|
||||
const mergeCommitValue = pullRequest.mergeCommit;
|
||||
const mergeCommit = isRecord(mergeCommitValue)
|
||||
? mergeCommitValue.oid
|
||||
: mergeCommitValue ?? mergeResult.sha;
|
||||
const mergedAt = pullRequest.mergedAt;
|
||||
const status = result.ok === true
|
||||
? result.alreadyMerged === true
|
||||
? "already-merged"
|
||||
@@ -198,13 +204,14 @@ export function renderPrMergeTable(result: GitHubCommandResult, fallbackDetails?
|
||||
` repo=${ghText(result.repo)} title=${ghShort(ghText(pullRequest.title), 96)}`,
|
||||
` blockers=${blockers.length === 0 ? "-" : blockers.join(",")} pending=${pending.length === 0 ? "-" : pending.join(",")}`,
|
||||
` retry=${retry.attempts !== undefined && retry.maxAttempts !== undefined ? `${ghText(retry.attempts)}/${ghText(retry.maxAttempts)} exhausted=${ghText(retry.exhausted)}` : "-"}`,
|
||||
` mergeCommit=${ghText(mergeCommit)} mergedAt=${ghText(mergedAt)}`,
|
||||
` branchDeletion=${ghText(branchDeletion.ok ?? branchDeletion.skippedReason ?? branchDeletion.attempted)}`,
|
||||
` closeout localWorktree=${closeoutCell(localWorktree)} mainWorktree=${closeoutCell(mainWorktree)} nodeSyncs=${nodeSyncs.length === 0 ? "-" : nodeSyncs.map(closeoutCell).join(",")}`,
|
||||
"",
|
||||
"Next:",
|
||||
];
|
||||
if (result.ok === true && result.alreadyMerged !== true && result.dryRun !== true) {
|
||||
lines.push(` bun scripts/cli.ts gh pr view ${ghText(result.number ?? details.number)} --repo ${ghText(result.repo)}`);
|
||||
if (result.ok === true && result.dryRun !== true) {
|
||||
lines.push(" no follow-up required; merge facts are included in Summary.");
|
||||
} else {
|
||||
lines.push(` ${prMergeRetryCommand(ghText(result.repo), result.number ?? details.number, mergeMethod, deleteBranch)}`);
|
||||
lines.push(` bun scripts/cli.ts gh pr preflight ${ghText(result.number ?? details.number)} --repo ${ghText(result.repo)} --full`);
|
||||
|
||||
@@ -871,6 +871,8 @@ function platformInfraHelpSummary(): unknown {
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api status [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api ops diagnosis [--platform <name>] [--group <id-or-name>] [--id <diagnosis-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api ops channels [--platform <name>] [--channel <id-or-name>] [--window 7d|15d|30d] [--page-token <record-id>|--record <record-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
|
||||
+413
-10
@@ -12,6 +12,7 @@ import { compactText, fingerprintEnvValues as fingerprintValues, parseEnvFile, p
|
||||
const defaultConfigPath = "config/platform-db/postgres-pk01.yaml";
|
||||
const managedHbaStart = "# BEGIN unidesk managed pk01-platform-postgres";
|
||||
const managedHbaEnd = "# END unidesk managed pk01-platform-postgres";
|
||||
const compactActionablePreviewLimit = 4;
|
||||
|
||||
interface PlatformDbOptions {
|
||||
configPath: string;
|
||||
@@ -302,10 +303,10 @@ export function platformDbHelp(): unknown {
|
||||
command: "platform-db postgres plan|status|apply|export-secrets",
|
||||
output: "json",
|
||||
usage: [
|
||||
`bun scripts/cli.ts platform-db postgres plan --config ${defaultConfigPath}`,
|
||||
`bun scripts/cli.ts platform-db postgres plan --config ${defaultConfigPath} [--full|--raw]`,
|
||||
`bun scripts/cli.ts platform-db postgres status --config ${defaultConfigPath} [--full|--raw]`,
|
||||
`bun scripts/cli.ts platform-db postgres export-secrets --config ${defaultConfigPath} --dry-run`,
|
||||
`bun scripts/cli.ts platform-db postgres export-secrets --config ${defaultConfigPath} --confirm`,
|
||||
`bun scripts/cli.ts platform-db postgres export-secrets --config ${defaultConfigPath} --dry-run [--full|--raw]`,
|
||||
`bun scripts/cli.ts platform-db postgres export-secrets --config ${defaultConfigPath} --confirm [--full|--raw]`,
|
||||
`bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --dry-run`,
|
||||
`bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --confirm`,
|
||||
`bun scripts/cli.ts platform-db postgres apply --config ${defaultConfigPath} --confirm --wait`,
|
||||
@@ -318,6 +319,7 @@ export function platformDbHelp(): unknown {
|
||||
configTruth: defaultConfigPath,
|
||||
defaultMutation: false,
|
||||
exportSecrets: "export-secrets only materializes YAML-declared local Secret source/export files; it does not touch the remote PostgreSQL service.",
|
||||
disclosure: "plan, status and export-secrets use bounded summaries by default; --full and --raw explicitly disclose the complete structured result without printing Secret values.",
|
||||
apply: "apply --confirm returns an async UniDesk job; the job uses short trans calls and a remote PK01 job instead of keeping one long SSH session open.",
|
||||
monitorReset: "monitor reset is restricted by owning YAML to the dedicated web_probe_monitor database and requires --confirm; it cannot accept a database name or arbitrary SQL.",
|
||||
redaction: "Passwords and full DATABASE_URL values are never printed; output is limited to key names, presence, fingerprints and state.",
|
||||
@@ -439,14 +441,15 @@ async function plan(config: UniDeskConfig, options: PlatformDbOptions): Promise<
|
||||
const remote = await remoteFacts(config, pg, null);
|
||||
const facts = remote.parsed;
|
||||
const checks = facts === null ? [] : desiredChecks(pg, facts, secrets);
|
||||
return {
|
||||
const desired = desiredSummary(pg, secrets, facts);
|
||||
const result = {
|
||||
ok: remote.capture.exitCode === 0 && facts !== null && secrets.ok,
|
||||
action: "platform-db-postgres-plan",
|
||||
mutation: false,
|
||||
config: configSummary(pg),
|
||||
secrets: secretSummary(secrets),
|
||||
remoteFacts: facts,
|
||||
desired: desiredSummary(pg, secrets, facts),
|
||||
desired,
|
||||
checks,
|
||||
next: {
|
||||
apply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configPath} --confirm`,
|
||||
@@ -456,6 +459,8 @@ async function plan(config: UniDeskConfig, options: PlatformDbOptions): Promise<
|
||||
remote: compactCapture(remote.capture, { full: options.full || remote.capture.exitCode !== 0 }),
|
||||
...(options.raw ? { raw: remote.capture } : {}),
|
||||
};
|
||||
if (options.full) return result;
|
||||
return compactPlan(pg, secrets, facts, checks, remote.capture, result.ok);
|
||||
}
|
||||
|
||||
async function status(config: UniDeskConfig, options: PlatformDbOptions): Promise<Record<string, unknown>> {
|
||||
@@ -485,7 +490,8 @@ async function status(config: UniDeskConfig, options: PlatformDbOptions): Promis
|
||||
...(secrets.ok ? [] : ["secrets-unhealthy"]),
|
||||
...(endpointHealthy ? [] : [controllerConnection.attempted ? "connection-host-probe-failed" : "connection-host-probe-unavailable"]),
|
||||
];
|
||||
return {
|
||||
const exportTargets = inspectExportTargets(pg);
|
||||
const result = {
|
||||
ok: remote.capture.exitCode === 0 && facts !== null && deploymentHealthy && secrets.ok,
|
||||
action: "platform-db-postgres-status",
|
||||
mutation: false,
|
||||
@@ -522,9 +528,12 @@ async function status(config: UniDeskConfig, options: PlatformDbOptions): Promis
|
||||
},
|
||||
remoteFacts: facts,
|
||||
secrets: secretSummary(secrets),
|
||||
exports: exportTargets,
|
||||
remote: compactCapture(remote.capture, { full: options.full || remote.capture.exitCode !== 0 }),
|
||||
...(options.raw ? { raw: remote.capture } : {}),
|
||||
};
|
||||
if (options.full) return result;
|
||||
return compactStatus(pg, secrets, exportTargets, facts, controllerConnection, remote.capture, result.summary, result.ok);
|
||||
}
|
||||
|
||||
async function apply(config: UniDeskConfig, options: PlatformDbOptions): Promise<Record<string, unknown>> {
|
||||
@@ -600,7 +609,7 @@ async function exportSecrets(options: PlatformDbOptions): Promise<Record<string,
|
||||
const pg = readPostgresHostConfig(options.configPath);
|
||||
if (!options.confirm || options.dryRun) {
|
||||
const localState = buildLocalSecretState(pg, false);
|
||||
return {
|
||||
const result = {
|
||||
ok: localState.inspection.ok,
|
||||
action: "platform-db-postgres-export-secrets",
|
||||
mode: "dry-run",
|
||||
@@ -613,9 +622,11 @@ async function exportSecrets(options: PlatformDbOptions): Promise<Record<string,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
if (options.full) return result;
|
||||
return compactExportSecrets(pg, localState, "dry-run", false, result.ok);
|
||||
}
|
||||
const localState = ensureLocalSecretState(pg);
|
||||
return {
|
||||
const result = {
|
||||
ok: true,
|
||||
action: "platform-db-postgres-export-secrets",
|
||||
mode: "confirmed",
|
||||
@@ -628,6 +639,8 @@ async function exportSecrets(options: PlatformDbOptions): Promise<Record<string,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
if (options.full) return result;
|
||||
return compactExportSecrets(pg, localState, "confirmed", true, result.ok);
|
||||
}
|
||||
|
||||
function readPostgresHostConfig(pathArg: string): PostgresHostConfig {
|
||||
@@ -1071,6 +1084,360 @@ function configSummary(pg: PostgresHostConfig): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function compactPlan(
|
||||
pg: PostgresHostConfig,
|
||||
secrets: SecretInspection,
|
||||
facts: RemoteFacts | null,
|
||||
checks: Array<Record<string, unknown>>,
|
||||
capture: SshCaptureResult,
|
||||
ok: boolean,
|
||||
): Record<string, unknown> {
|
||||
const observedRoles = facts?.postgres.roles ?? [];
|
||||
const observedDatabases = facts?.postgres.databases ?? [];
|
||||
return {
|
||||
ok,
|
||||
action: "platform-db-postgres-plan",
|
||||
mutation: false,
|
||||
target: {
|
||||
node: pg.node.id,
|
||||
route: pg.node.route,
|
||||
mode: pg.node.mode,
|
||||
},
|
||||
config: {
|
||||
path: pg.configPath,
|
||||
id: pg.metadata.id,
|
||||
pgVersion: pg.postgres.package.version,
|
||||
connectionHost: pg.postgres.network.connectionHost,
|
||||
publicDns: pg.postgres.network.publicDns,
|
||||
port: pg.postgres.network.port,
|
||||
transport: pg.postgres.network.transport,
|
||||
sslmode: pg.postgres.network.sslmode,
|
||||
},
|
||||
roles: pg.objects.roles.map((role) => ({
|
||||
name: role.name,
|
||||
exists: observedRoles.find((item) => item.name === role.name)?.exists ?? null,
|
||||
action: observedRoles.some((item) => item.name === role.name && item.exists) ? "none" : "create-or-update",
|
||||
})),
|
||||
databases: pg.objects.databases.map((database) => ({
|
||||
name: database.name,
|
||||
owner: database.owner,
|
||||
exists: observedDatabases.find((item) => item.name === database.name)?.exists ?? null,
|
||||
action: observedDatabases.some((item) => item.name === database.name && item.exists) ? "none" : "create",
|
||||
})),
|
||||
secrets: {
|
||||
ok: secrets.ok,
|
||||
root: secrets.root,
|
||||
entries: secrets.entries.map((entry) => ({
|
||||
sourceRef: entry.sourceRef,
|
||||
missingKeys: entry.missingKeys,
|
||||
action: entry.action,
|
||||
})),
|
||||
valuesPrinted: false,
|
||||
},
|
||||
exports: pg.exports.connectionStrings.map((item) => ({
|
||||
name: item.name,
|
||||
sourceRef: item.sourceSecretRef,
|
||||
targetRef: item.writeToSecretSource.sourceRef,
|
||||
key: item.writeToSecretSource.key,
|
||||
consumerScopes: item.consumers.map((consumer) => consumer.scope),
|
||||
})),
|
||||
checks: {
|
||||
ok: checks.every((check) => check.ok === true),
|
||||
passed: checks.filter((check) => check.ok === true).length,
|
||||
total: checks.length,
|
||||
items: checks.map((check) => ({
|
||||
name: check.name,
|
||||
ok: check.ok,
|
||||
...(check.ok === true ? {} : { detail: compactText(JSON.stringify(check)).slice(0, 480) }),
|
||||
})),
|
||||
},
|
||||
remoteFacts: compactRemoteFacts(facts),
|
||||
remote: compactPlanCapture(capture),
|
||||
next: {
|
||||
apply: `bun scripts/cli.ts platform-db postgres apply --config ${pg.configPath} --confirm`,
|
||||
status: `bun scripts/cli.ts platform-db postgres status --config ${pg.configPath}`,
|
||||
full: `bun scripts/cli.ts platform-db postgres plan --config ${pg.configPath} --full`,
|
||||
raw: `bun scripts/cli.ts platform-db postgres plan --config ${pg.configPath} --raw`,
|
||||
issues: pg.metadata.relatedIssues.map((issue) => `https://github.com/pikasTech/unidesk/issues/${issue}`),
|
||||
},
|
||||
disclosure: {
|
||||
compact: true,
|
||||
omitted: ["package", "host policy", "TLS paths", "backup", "full remote facts"],
|
||||
fullAvailable: true,
|
||||
rawAvailable: true,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactStatus(
|
||||
pg: PostgresHostConfig,
|
||||
secrets: SecretInspection,
|
||||
exports: Array<Record<string, unknown>>,
|
||||
facts: RemoteFacts | null,
|
||||
controllerConnection: ControllerConnectionProbe,
|
||||
capture: SshCaptureResult,
|
||||
summary: Record<string, unknown> | null,
|
||||
ok: boolean,
|
||||
): Record<string, unknown> {
|
||||
const blockers = summary === null
|
||||
? ["remote-facts-unavailable"]
|
||||
: Array.isArray(summary.cutoverBlockers)
|
||||
? summary.cutoverBlockers
|
||||
: [];
|
||||
const roles = pg.objects.roles.map((role) => ({
|
||||
name: role.name,
|
||||
exists: facts?.postgres.roles.find((item) => item.name === role.name)?.exists ?? null,
|
||||
}));
|
||||
const databases = pg.objects.databases.map((database) => ({
|
||||
name: database.name,
|
||||
owner: database.owner,
|
||||
exists: facts?.postgres.databases.find((item) => item.name === database.name)?.exists ?? null,
|
||||
}));
|
||||
const appConnections = facts?.postgres.appConnections ?? [];
|
||||
return {
|
||||
ok,
|
||||
action: "platform-db-postgres-status",
|
||||
mutation: false,
|
||||
failure: ok ? null : {
|
||||
type: capture.exitCode !== 0 || facts === null
|
||||
? "remote-facts-unavailable"
|
||||
: secrets.ok
|
||||
? "deployment-unhealthy"
|
||||
: "secrets-unhealthy",
|
||||
blockers,
|
||||
},
|
||||
target: {
|
||||
node: pg.node.id,
|
||||
route: pg.node.route,
|
||||
mode: pg.node.mode,
|
||||
},
|
||||
config: {
|
||||
path: pg.configPath,
|
||||
id: pg.metadata.id,
|
||||
pgVersion: pg.postgres.package.version,
|
||||
connectionHost: pg.postgres.network.connectionHost,
|
||||
port: pg.postgres.network.port,
|
||||
sslmode: pg.postgres.network.sslmode,
|
||||
},
|
||||
summary: summary === null ? null : {
|
||||
healthy: summary.healthy,
|
||||
deploymentHealthy: summary.deploymentHealthy,
|
||||
cutoverReady: summary.cutoverReady,
|
||||
cutoverBlockers: summary.cutoverBlockers,
|
||||
packageInstalled: summary.packageInstalled,
|
||||
serviceActive: summary.serviceActive,
|
||||
serviceEnabled: summary.serviceEnabled,
|
||||
sslOn: summary.sslOn,
|
||||
port5432Listening: summary.port5432Listening,
|
||||
dnsResolves: summary.dnsResolves,
|
||||
dnsDisposition: summary.dnsDisposition,
|
||||
secretsOk: summary.secretsOk,
|
||||
connectionHostProbe: {
|
||||
attempted: controllerConnection.attempted,
|
||||
ok: controllerConnection.ok,
|
||||
ssl: controllerConnection.ssl,
|
||||
user: controllerConnection.user,
|
||||
database: controllerConnection.database,
|
||||
host: controllerConnection.host,
|
||||
port: controllerConnection.port,
|
||||
error: controllerConnection.error,
|
||||
},
|
||||
},
|
||||
roles: compactActionableSummary(roles, {
|
||||
passed: (item) => item.exists === true,
|
||||
missing: (item) => item.exists === false,
|
||||
actionable: (item) => item.exists !== true,
|
||||
project: (item) => item,
|
||||
}),
|
||||
databases: compactActionableSummary(databases, {
|
||||
passed: (item) => item.exists === true,
|
||||
missing: (item) => item.exists === false,
|
||||
actionable: (item) => item.exists !== true,
|
||||
project: (item) => item,
|
||||
}),
|
||||
appConnections: compactActionableSummary(appConnections, {
|
||||
passed: (item) => item.ok === true && item.ssl === true,
|
||||
missing: (item) => item.ok !== true || item.ssl !== true,
|
||||
actionable: (item) => item.ok !== true || item.ssl !== true,
|
||||
project: (item) => ({ user: item.user, database: item.database, ok: item.ok, ssl: item.ssl, error: item.error }),
|
||||
}),
|
||||
secrets: compactSecretInspection(secrets),
|
||||
exports: compactActionableSummary(exports, {
|
||||
passed: (item) => item.present === true,
|
||||
missing: (item) => item.present !== true,
|
||||
actionable: (item) => item.present !== true,
|
||||
project: (item) => ({
|
||||
targetRef: item.targetRef,
|
||||
key: item.key,
|
||||
exists: item.exists,
|
||||
present: item.present,
|
||||
fingerprint: item.fingerprint,
|
||||
}),
|
||||
}),
|
||||
remote: compactPlanCapture(capture),
|
||||
next: {
|
||||
plan: `bun scripts/cli.ts platform-db postgres plan --config ${pg.configPath}`,
|
||||
full: `bun scripts/cli.ts platform-db postgres status --config ${pg.configPath} --full`,
|
||||
raw: `bun scripts/cli.ts platform-db postgres status --config ${pg.configPath} --raw`,
|
||||
},
|
||||
disclosure: {
|
||||
compact: true,
|
||||
omitted: ["host facts", "package repository", "TLS paths", "raw remote capture"],
|
||||
fullAvailable: true,
|
||||
rawAvailable: true,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactExportSecrets(
|
||||
pg: PostgresHostConfig,
|
||||
localState: { inspection: SecretInspection; summary: Record<string, unknown>; exports: Array<Record<string, unknown>> },
|
||||
mode: "dry-run" | "confirmed",
|
||||
mutation: boolean,
|
||||
ok: boolean,
|
||||
): Record<string, unknown> {
|
||||
const suffix = mode === "dry-run" ? " --dry-run" : " --confirm";
|
||||
const exports = localState.exports.map((item) => {
|
||||
const before = asRecord(item.before, "export before");
|
||||
const after = asRecord(item.after, "export after");
|
||||
return {
|
||||
name: item.name,
|
||||
sourceRef: item.sourceRef,
|
||||
targetRef: item.targetRef,
|
||||
key: item.key,
|
||||
before,
|
||||
after,
|
||||
action: item.action,
|
||||
desiredFingerprint: item.desiredFingerprint,
|
||||
};
|
||||
});
|
||||
return {
|
||||
ok,
|
||||
action: "platform-db-postgres-export-secrets",
|
||||
mode,
|
||||
mutation,
|
||||
failure: ok ? null : {
|
||||
type: "secrets-unhealthy",
|
||||
blockers: localState.inspection.entries.filter((entry) => entry.missingKeys.length > 0).map((entry) => entry.sourceRef),
|
||||
},
|
||||
config: {
|
||||
path: pg.configPath,
|
||||
id: pg.metadata.id,
|
||||
secretRoot: localState.inspection.root,
|
||||
},
|
||||
secrets: compactSecretInspection(localState.inspection),
|
||||
exports: compactActionableSummary(exports, {
|
||||
passed: (item) => item.action === "none" && item.after.present === true,
|
||||
missing: (item) => item.after.present !== true,
|
||||
actionable: (item) => item.action !== "none" || item.after.present !== true,
|
||||
project: (item) => item,
|
||||
}),
|
||||
next: {
|
||||
...(mode === "dry-run" ? { confirm: `bun scripts/cli.ts platform-db postgres export-secrets --config ${pg.configPath} --confirm` } : {}),
|
||||
full: `bun scripts/cli.ts platform-db postgres export-secrets --config ${pg.configPath}${suffix} --full`,
|
||||
raw: `bun scripts/cli.ts platform-db postgres export-secrets --config ${pg.configPath}${suffix} --raw`,
|
||||
syncConsumers: "run the consumer-specific secret sync command after confirmed export",
|
||||
},
|
||||
disclosure: {
|
||||
compact: true,
|
||||
omitted: ["source paths", "required key lists", "consumer Secret details"],
|
||||
fullAvailable: true,
|
||||
rawAvailable: true,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactSecretInspection(secrets: SecretInspection): Record<string, unknown> {
|
||||
const entries = secrets.entries.map((entry) => ({
|
||||
sourceRef: entry.sourceRef,
|
||||
exists: entry.exists,
|
||||
presentKeys: entry.presentKeys,
|
||||
missingKeys: entry.missingKeys,
|
||||
action: entry.action,
|
||||
fingerprint: entry.fingerprint,
|
||||
}));
|
||||
return {
|
||||
ok: secrets.ok,
|
||||
root: secrets.root,
|
||||
...compactActionableSummary(entries, {
|
||||
passed: (item) => item.exists && item.missingKeys.length === 0 && item.action === "none",
|
||||
missing: (item) => !item.exists || item.missingKeys.length > 0,
|
||||
actionable: (item) => item.action !== "none" || !item.exists || item.missingKeys.length > 0,
|
||||
project: (item) => item,
|
||||
}),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactActionableSummary<T>(
|
||||
items: T[],
|
||||
selectors: {
|
||||
passed: (item: T) => boolean;
|
||||
missing: (item: T) => boolean;
|
||||
actionable: (item: T) => boolean;
|
||||
project: (item: T) => unknown;
|
||||
},
|
||||
): Record<string, unknown> {
|
||||
const actionableItems = items.filter(selectors.actionable);
|
||||
const visibleItems = actionableItems.slice(0, compactActionablePreviewLimit);
|
||||
return {
|
||||
total: items.length,
|
||||
passed: items.filter(selectors.passed).length,
|
||||
missing: items.filter(selectors.missing).length,
|
||||
actionable: actionableItems.length,
|
||||
listed: visibleItems.length,
|
||||
omitted: items.length - visibleItems.length,
|
||||
actionableOmitted: actionableItems.length - visibleItems.length,
|
||||
items: visibleItems.map(selectors.project),
|
||||
};
|
||||
}
|
||||
|
||||
function compactRemoteFacts(facts: RemoteFacts | null): Record<string, unknown> | null {
|
||||
if (facts === null) return null;
|
||||
return {
|
||||
ok: facts.ok,
|
||||
observedAt: facts.observedAt,
|
||||
host: {
|
||||
hostname: facts.host.hostname,
|
||||
cpuCount: facts.host.cpuCount,
|
||||
memoryTotalMiB: facts.host.memoryTotalMiB,
|
||||
swapTotalMiB: facts.host.swapTotalMiB,
|
||||
rootAvailGiB: facts.host.rootAvailGiB,
|
||||
targetDataAvailGiB: facts.host.targetDataAvailGiB,
|
||||
},
|
||||
network: {
|
||||
port5432Listening: facts.network.port5432Listening,
|
||||
dns: facts.network.dns,
|
||||
},
|
||||
repo: {
|
||||
reachable: facts.repo.reachable,
|
||||
releaseUrl: facts.repo.releaseUrl,
|
||||
},
|
||||
postgres: {
|
||||
packageInstalled: facts.postgres.packageInstalled,
|
||||
serviceActive: facts.postgres.serviceActive,
|
||||
serviceEnabled: facts.postgres.serviceEnabled,
|
||||
roleCount: facts.postgres.roles.length,
|
||||
databaseCount: facts.postgres.databases.length,
|
||||
serverVersion: facts.postgres.serverVersion,
|
||||
sslOn: facts.postgres.sslOn,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function compactPlanCapture(result: SshCaptureResult): Record<string, unknown> {
|
||||
return {
|
||||
exitCode: result.exitCode,
|
||||
stdoutBytes: Buffer.byteLength(result.stdout, "utf8"),
|
||||
stderrBytes: Buffer.byteLength(result.stderr, "utf8"),
|
||||
stdoutTail: result.exitCode === 0 ? "" : result.stdout.slice(-1200),
|
||||
stderrTail: result.exitCode === 0 ? "" : result.stderr.slice(-1200),
|
||||
};
|
||||
}
|
||||
|
||||
function requireMonitorResetConfig(pg: PostgresHostConfig): MonitorResetConfig {
|
||||
const target = pg.managedOperations?.monitorReset;
|
||||
if (target === undefined) throw new Error(`${pg.configPath}.managedOperations.monitorReset is required`);
|
||||
@@ -1252,8 +1619,12 @@ function connectionStringExportState(pg: PostgresHostConfig, inspection: SecretI
|
||||
const rendered = renderConnectionString(item, source.values);
|
||||
const root = secretRoot(pg);
|
||||
const targetPath = join(root, item.writeToSecretSource.sourceRef);
|
||||
const existing = existsSync(targetPath) ? parseEnvFile(readFileSync(targetPath, "utf8")) : {};
|
||||
const exists = existsSync(targetPath);
|
||||
const existing = exists ? parseEnvFile(readFileSync(targetPath, "utf8")) : {};
|
||||
const before = existing[item.writeToSecretSource.key];
|
||||
const beforePresent = before !== undefined && before.length > 0;
|
||||
const beforeFingerprint = beforePresent ? fingerprintValues({ [item.writeToSecretSource.key]: before }, [item.writeToSecretSource.key]) : null;
|
||||
const desiredFingerprint = fingerprintValues({ [item.writeToSecretSource.key]: rendered }, [item.writeToSecretSource.key]);
|
||||
const next = { ...existing, [item.writeToSecretSource.key]: rendered };
|
||||
if (materialize) writeEnvFile(targetPath, next);
|
||||
return {
|
||||
@@ -1261,13 +1632,45 @@ function connectionStringExportState(pg: PostgresHostConfig, inspection: SecretI
|
||||
sourceRef: item.sourceSecretRef,
|
||||
targetRef: item.writeToSecretSource.sourceRef,
|
||||
key: item.writeToSecretSource.key,
|
||||
before: {
|
||||
exists,
|
||||
present: beforePresent,
|
||||
fingerprint: beforeFingerprint,
|
||||
},
|
||||
after: {
|
||||
exists: materialize ? true : exists,
|
||||
present: materialize ? true : beforePresent,
|
||||
fingerprint: materialize ? desiredFingerprint : beforeFingerprint,
|
||||
},
|
||||
action: before === rendered ? "none" : before === undefined ? "create" : "update",
|
||||
fingerprint: fingerprintValues({ [item.writeToSecretSource.key]: rendered }, [item.writeToSecretSource.key]),
|
||||
desiredFingerprint,
|
||||
consumers: item.consumers,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function inspectExportTargets(pg: PostgresHostConfig): Array<Record<string, unknown>> {
|
||||
const root = secretRoot(pg);
|
||||
return pg.exports.connectionStrings.map((item) => {
|
||||
const targetPath = join(root, item.writeToSecretSource.sourceRef);
|
||||
const exists = existsSync(targetPath);
|
||||
const values = exists ? parseEnvFile(readFileSync(targetPath, "utf8")) : {};
|
||||
const value = values[item.writeToSecretSource.key];
|
||||
const present = value !== undefined && value.length > 0;
|
||||
return {
|
||||
name: item.name,
|
||||
sourceRef: item.sourceSecretRef,
|
||||
targetRef: item.writeToSecretSource.sourceRef,
|
||||
key: item.writeToSecretSource.key,
|
||||
exists,
|
||||
present,
|
||||
fingerprint: present ? fingerprintValues({ [item.writeToSecretSource.key]: value }, [item.writeToSecretSource.key]) : null,
|
||||
consumerScopes: item.consumers.map((consumer) => consumer.scope),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function renderConnectionString(item: ConnectionStringExportConfig, values: Record<string, string>): string {
|
||||
let output = item.render.format;
|
||||
const merged = { ...values, ...(item.render.variables ?? {}) };
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
readGiteaConfig,
|
||||
resolveTarget,
|
||||
} from "./platform-infra-gitea-config";
|
||||
import { buildMirrorWebhookCredentialBlockedResult, renderManifest } from "./platform-infra-gitea";
|
||||
import { buildGiteaMirrorBootstrapCredentialPreflight, buildMirrorWebhookCredentialBlockedResult, renderManifest } from "./platform-infra-gitea";
|
||||
import { renderMirrorWebhookCredentialBlocked } from "./platform-infra-gitea-render";
|
||||
|
||||
function syntheticMissingSelfmediaCredential(): Record<string, unknown> {
|
||||
@@ -65,6 +65,28 @@ describe("platform-infra Gitea repository GitHub credentials", () => {
|
||||
expect(nc01Overrides.map((credential) => credential.sourceRef)).toContain("pikainc-selfmedia-gh-token.txt");
|
||||
});
|
||||
|
||||
test("bootstrap preflight only requires the selected repository effective credential", () => {
|
||||
const config = readGiteaConfig();
|
||||
const repository = config.sourceAuthority.repositories.find((repo) => repo.key === "selfmedia-nc01");
|
||||
expect(repository).toBeDefined();
|
||||
const result = buildGiteaMirrorBootstrapCredentialPreflight(config, [repository!], (credential) => (
|
||||
credential.sourceRef === "pikainc-selfmedia-gh-token.txt"
|
||||
? { [credential.sourceKey]: "fixture-token" }
|
||||
: null
|
||||
));
|
||||
|
||||
expect(result.blockers).toEqual([]);
|
||||
expect(result.credentials).toHaveLength(1);
|
||||
expect(result.credentials[0]).toMatchObject({
|
||||
id: "github-upstream:selfmedia-nc01",
|
||||
sourceRef: "pikainc-selfmedia-gh-token.txt",
|
||||
present: true,
|
||||
requiredKeysPresent: true,
|
||||
});
|
||||
expect(JSON.stringify(result)).not.toContain(config.sourceAuthority.credentials.admin.sourceRef);
|
||||
expect(JSON.stringify(result)).not.toContain(config.sourceAuthority.credentials.github.sourceRef);
|
||||
});
|
||||
|
||||
test("does not inject the NC01 selfmedia token into JD01 bridge containers", () => {
|
||||
const config = readGiteaConfig();
|
||||
const selfmediaTokenEnv = githubTokenEnvNameForSecretKey("github-token-pikainc-selfmedia");
|
||||
|
||||
@@ -221,11 +221,45 @@ payload = {
|
||||
},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if payload["ok"] else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
run_mirror_preflight() {
|
||||
python3 - "$tmp/repos.json" <<'PY'
|
||||
import base64, json, os, subprocess, sys
|
||||
repos = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
rows = []
|
||||
timeout = int(os.environ.get("UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS", "20"))
|
||||
for repo in repos:
|
||||
credential = repo.get("githubCredential") or {}
|
||||
token_env = credential.get("tokenEnv") or os.environ.get("UNIDESK_GITEA_GITHUB_DEFAULT_TOKEN_ENV", "")
|
||||
token = os.environ.get(token_env, "")
|
||||
clone_url = repo["upstream"]["cloneUrl"]
|
||||
branch = repo["upstream"]["branch"]
|
||||
if not token:
|
||||
rows.append({"key": repo["key"], "repository": repo["upstream"]["repository"], "branch": branch, "ok": False, "code": "github-credential-unavailable", "valuesPrinted": False})
|
||||
continue
|
||||
basic = base64.b64encode(("x-access-token:" + token).encode()).decode()
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
["git", "-c", "http.extraHeader=Authorization: Basic " + basic, "ls-remote", "--exit-code", clone_url, "refs/heads/" + branch],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
error = " ".join(completed.stderr.split())[-800:]
|
||||
rows.append({"key": repo["key"], "repository": repo["upstream"]["repository"], "branch": branch, "ok": completed.returncode == 0, "code": "github-upstream-available" if completed.returncode == 0 else "github-repository-not-found-or-no-access", "exitCode": completed.returncode, "errorTail": error, "valuesPrinted": False})
|
||||
except subprocess.TimeoutExpired:
|
||||
rows.append({"key": repo["key"], "repository": repo["upstream"]["repository"], "branch": branch, "ok": False, "code": "github-upstream-timeout", "valuesPrinted": False})
|
||||
payload = {"ok": bool(rows) and all(row["ok"] for row in rows), "mutation": False, "repositories": rows, "valuesPrinted": False}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
sys.exit(0 if payload["ok"] else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
run_status() {
|
||||
selector="app.kubernetes.io/name=$UNIDESK_GITEA_APP_NAME,app.kubernetes.io/component=gitea"
|
||||
capture_json namespace kubectl get namespace "$UNIDESK_GITEA_NAMESPACE"
|
||||
@@ -1125,6 +1159,7 @@ case "$UNIDESK_GITEA_ACTION" in
|
||||
apply) run_apply ;;
|
||||
status) run_status ;;
|
||||
validate) run_validate ;;
|
||||
mirror-preflight) run_mirror_preflight ;;
|
||||
mirror-bootstrap) run_mirror_bootstrap ;;
|
||||
mirror-sync) run_mirror_sync ;;
|
||||
mirror-status) run_mirror_status ;;
|
||||
|
||||
@@ -123,6 +123,69 @@ export async function runPlatformInfraGiteaCommand(config: UniDeskConfig, args:
|
||||
};
|
||||
}
|
||||
|
||||
export async function runGiteaMirrorBootstrapPreflight(
|
||||
config: UniDeskConfig,
|
||||
targetId: string,
|
||||
repoKey: string,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const gitea = readGiteaConfig();
|
||||
const target = resolveCommandTarget(gitea, targetId);
|
||||
const repos = selectedRepositories(gitea, target, repoKey);
|
||||
const { credentials, blockers } = buildGiteaMirrorBootstrapCredentialPreflight(gitea, repos);
|
||||
if (blockers.length > 0) {
|
||||
return { ok: false, mutation: false, code: "github-credential-unavailable", blockers, repositories: repos.map((repo) => repositorySummary(gitea, repo)), valuesPrinted: false };
|
||||
}
|
||||
const secrets = mirrorPreflightSecrets(gitea, repos);
|
||||
const result = await capture(config, target.route, ["sh"], remoteScript("mirror-preflight", gitea, target, "", { targetId, repoKey, confirm: false, dryRun: true, wait: false, full: false, raw: false }, { repos, secrets }).script);
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
return parsed ?? { ok: false, mutation: false, code: "github-upstream-preflight-failed", remote: compactCapture(result, { full: true }), valuesPrinted: false };
|
||||
}
|
||||
|
||||
export function buildGiteaMirrorBootstrapCredentialPreflight(
|
||||
gitea: GiteaConfig,
|
||||
repositories: GiteaMirrorRepository[],
|
||||
readCredential: (credential: GiteaGithubCredential) => Record<string, string> | null = (credential) => readGithubCredential(gitea, credential),
|
||||
): { credentials: Array<Record<string, unknown>>; blockers: string[] } {
|
||||
const credentials = repositories.map((repo) => {
|
||||
const github = githubCredentialForRepo(gitea, repo);
|
||||
const values = readCredential(github);
|
||||
const present = values !== null;
|
||||
const requiredKeysPresent = Boolean(values?.[github.sourceKey]);
|
||||
return {
|
||||
id: `github-upstream:${repo.key}`,
|
||||
repository: repo.upstream.repository,
|
||||
sourceRef: github.sourceRef,
|
||||
present,
|
||||
requiredKeysPresent,
|
||||
fingerprint: present ? fingerprintKeys(values, [github.sourceKey]) : null,
|
||||
permissionResult: present && requiredKeysPresent
|
||||
? { status: "not-probed", permissions: github.permissions, error: null }
|
||||
: { status: "blocked-missing-credential", permissions: github.permissions, error: "credential material is absent" },
|
||||
valuesPrinted: false,
|
||||
};
|
||||
});
|
||||
return { credentials, blockers: credentialBlockers(credentials) };
|
||||
}
|
||||
|
||||
function readGithubCredential(gitea: GiteaConfig, github: GiteaGithubCredential): Record<string, string> | null {
|
||||
const sourcePath = credentialPath(gitea, github.sourceRef);
|
||||
return existsSync(sourcePath) ? parseGithubCredentialFile(sourcePath, github) : null;
|
||||
}
|
||||
|
||||
function mirrorPreflightSecrets(gitea: GiteaConfig, repositories: GiteaMirrorRepository[]): MirrorSecrets {
|
||||
const githubTokens: Record<string, string> = {};
|
||||
for (const repo of repositories) {
|
||||
const github = githubCredentialForRepo(gitea, repo);
|
||||
const githubPath = credentialPath(gitea, github.sourceRef);
|
||||
if (!existsSync(githubPath)) throw new Error(`${github.sourceRef} is missing; cannot probe GitHub upstream`);
|
||||
const githubEnv = parseGithubCredentialFile(githubPath, github);
|
||||
const githubToken = githubEnv[github.sourceKey];
|
||||
if (!githubToken) throw new Error(`${github.sourceRef} must contain ${github.sourceKey}`);
|
||||
githubTokens[github.gitFetchCredential.secretRef.key] = githubToken;
|
||||
}
|
||||
return { adminUsername: "", adminPassword: "", githubTokens, webhookSecret: "" };
|
||||
}
|
||||
|
||||
export function giteaHelp(scope?: string): Record<string, unknown> {
|
||||
if (scope === "platform-bootstrap") {
|
||||
return {
|
||||
@@ -1116,7 +1179,7 @@ function envVars(gitea: GiteaConfig, target: GiteaTarget): string {
|
||||
value: ${yamlQuote(value)}`).join("\n");
|
||||
}
|
||||
|
||||
function remoteScript(action: "apply" | "status" | "validate" | "mirror-bootstrap" | "mirror-sync" | "mirror-status" | "mirror-webhook-apply" | "mirror-webhook-status", gitea: GiteaConfig, target: GiteaTarget, manifest: string, options: ApplyOptions | MirrorOptions, params: MirrorRemoteParams = {}): { script: string; payloadSummary: Record<string, unknown> } {
|
||||
function remoteScript(action: "apply" | "status" | "validate" | "mirror-preflight" | "mirror-bootstrap" | "mirror-sync" | "mirror-status" | "mirror-webhook-apply" | "mirror-webhook-status", gitea: GiteaConfig, target: GiteaTarget, manifest: string, options: ApplyOptions | MirrorOptions, params: MirrorRemoteParams = {}): { script: string; payloadSummary: Record<string, unknown> } {
|
||||
const frpcExposure = targetFrpcExposure(gitea, target);
|
||||
const sync = targetWebhookSync(gitea, target);
|
||||
const env: Record<string, string> = {
|
||||
|
||||
@@ -3,7 +3,12 @@ import { spawnSync } from "node:child_process";
|
||||
import { rootPath } from "./config";
|
||||
import { isRenderedCliResult } from "./output";
|
||||
import { runPlatformObservabilityCommand } from "./platform-infra-observability";
|
||||
import { resolveTarget } from "./platform-infra-observability/actions";
|
||||
import { isKubernetesLabelValue, isKubernetesQualifiedName, readObservabilityConfig } from "./platform-infra-observability/config";
|
||||
import { renderObservabilityPlanFailure } from "./platform-infra-observability/plan-render";
|
||||
import { prometheusMetaLabel, re2Literal } from "./platform-infra-observability/prometheus";
|
||||
import { statusScript } from "./platform-infra-observability/search-script";
|
||||
import { renderManifest } from "./platform-infra-observability/trace-script";
|
||||
|
||||
describe("platform-infra observability progressive disclosure", () => {
|
||||
test("default plan is a bounded table while full and raw preserve the complete plan", async () => {
|
||||
@@ -13,6 +18,8 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
expect(Buffer.byteLength(compact.renderedText, "utf8")).toBeLessThan(10_240);
|
||||
expect(compact.renderedText).toContain("serviceConnections=8");
|
||||
expect(compact.renderedText).toContain("configRefs=6/6 missing=0");
|
||||
expect(compact.renderedText).toContain("prometheus=enabled service=prometheus targets=NC01");
|
||||
expect(compact.renderedText).toContain("scrapeSelector=unidesk.ai/metrics=true");
|
||||
expect(compact.renderedText).toContain("--full");
|
||||
expect(compact.renderedText).toContain("--raw");
|
||||
|
||||
@@ -25,6 +32,7 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
renderPlan: { target: { id: "NC01", route: "NC01:k3s", namespace: "platform-infra" } },
|
||||
});
|
||||
expect((expanded as Record<string, any>).renderPlan.instrumentation).toHaveLength(8);
|
||||
expect((expanded as Record<string, any>).config.metricsBackend).toMatchObject({ type: "prometheus", enabled: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -38,8 +46,66 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
expect(result.stdout).not.toContain("/tmp/unidesk-cli-output");
|
||||
});
|
||||
|
||||
test("search, trace and diagnose-code-agent expose only scoped help", () => {
|
||||
for (const operation of ["search", "trace", "diagnose-code-agent"]) {
|
||||
test("Prometheus manifests and runtime access only apply to YAML-selected targets", async () => {
|
||||
const observability = readObservabilityConfig();
|
||||
const nc01 = resolveTarget(observability, "NC01");
|
||||
const d518 = resolveTarget(observability, "D518");
|
||||
const jd01 = resolveTarget(observability, "JD01");
|
||||
const nc01Manifest = renderManifest(observability, nc01);
|
||||
for (const target of [d518, jd01]) {
|
||||
const manifest = renderManifest(observability, target);
|
||||
expect(manifest).not.toContain("kind: ClusterRole\nmetadata:\n name: platform-infra-prometheus");
|
||||
expect(manifest).not.toContain("kind: PersistentVolumeClaim\nmetadata:\n name: prometheus-data");
|
||||
expect(manifest).not.toContain("app.kubernetes.io/component: metrics-backend");
|
||||
const plan = await runPlatformObservabilityCommand({} as never, ["plan", "--target", target.id]);
|
||||
expect(isRenderedCliResult(plan)).toBe(true);
|
||||
if (!isRenderedCliResult(plan)) throw new Error("expected rendered plan");
|
||||
expect(plan.renderedText).toContain("prometheus=disabled-for-target");
|
||||
expect(plan.renderedText).toContain("objects=8");
|
||||
const script = statusScript(observability, target, false);
|
||||
expect(script).not.toContain("capture_json metrics_deployments");
|
||||
expect(script).not.toContain("prometheus-ready");
|
||||
const query = await runPlatformObservabilityCommand({} as never, ["metrics-query", "--target", target.id, "--query", "up", "--full"]);
|
||||
expect(query).toMatchObject({ metrics: { enabled: false, disposition: "disabled-for-target", targetIds: ["NC01"] } });
|
||||
}
|
||||
expect(nc01Manifest).toContain("kind: ClusterRole\nmetadata:\n name: platform-infra-prometheus");
|
||||
expect(nc01Manifest).toContain("kind: PersistentVolumeClaim\nmetadata:\n name: prometheus-data");
|
||||
});
|
||||
|
||||
test("Prometheus relabel rules prefer an explicit path and only default when absent", () => {
|
||||
const observability = readObservabilityConfig();
|
||||
const manifest = renderManifest(observability, resolveTarget(observability, "NC01"));
|
||||
const explicitPath = "source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: (/.+)";
|
||||
const defaultPath = "source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]\n action: replace\n target_label: __metrics_path__\n regex: ^$\n replacement: /metrics";
|
||||
expect(manifest).toContain("source_labels: [__meta_kubernetes_pod_label_unidesk_ai_metrics]");
|
||||
expect(manifest).toContain("regex: '^true$'");
|
||||
expect(manifest).toContain(explicitPath);
|
||||
expect(manifest).toContain(defaultPath);
|
||||
expect(manifest.indexOf(explicitPath)).toBeLessThan(manifest.indexOf(defaultPath));
|
||||
});
|
||||
|
||||
test("Prometheus selector rendering uses qualified names, stable meta labels and literal RE2", () => {
|
||||
const observability = readObservabilityConfig();
|
||||
observability.metricsBackend.discovery.labelSelector = {
|
||||
key: "metrics.example.com/release-track",
|
||||
value: "v1.2-beta",
|
||||
};
|
||||
observability.metricsBackend.discovery.annotationKeys.scrape = "prometheus.io/scrape-enabled";
|
||||
const manifest = renderManifest(observability, resolveTarget(observability, "NC01"));
|
||||
expect(manifest).toContain("__meta_kubernetes_pod_label_metrics_example_com_release_track");
|
||||
expect(manifest).toContain("__meta_kubernetes_pod_annotation_prometheus_io_scrape_enabled");
|
||||
expect(manifest).toContain("regex: '^v1\\.2-beta$'");
|
||||
expect(prometheusMetaLabel("a.b/c-d_e")).toBe("a_b_c_d_e");
|
||||
expect(re2Literal("v1.2-beta")).toBe("^v1\\.2-beta$");
|
||||
expect(isKubernetesQualifiedName("metrics.example.com/release-track")).toBe(true);
|
||||
expect(isKubernetesQualifiedName("metrics.example.com/release/track")).toBe(false);
|
||||
expect(isKubernetesQualifiedName("Metrics.example.com/release-track")).toBe(false);
|
||||
expect(isKubernetesQualifiedName(`${"a".repeat(64)}.example/release-track`)).toBe(false);
|
||||
expect(isKubernetesLabelValue("v1.2-beta")).toBe(true);
|
||||
});
|
||||
|
||||
test("query commands expose only scoped help", () => {
|
||||
for (const operation of ["search", "trace", "diagnose-code-agent", "metrics-query"]) {
|
||||
const result = cli(["platform-infra", "observability", operation, "--help"]);
|
||||
expect(result.status).toBe(0);
|
||||
const payload = JSON.parse(result.stdout) as Record<string, any>;
|
||||
@@ -57,7 +123,7 @@ describe("platform-infra observability progressive disclosure", () => {
|
||||
expect(result.status).toBe(0);
|
||||
const payload = JSON.parse(result.stdout) as Record<string, any>;
|
||||
expect(payload.data.operations.map((item: Record<string, unknown>) => item.name)).toEqual([
|
||||
"plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent",
|
||||
"plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent", "metrics-query",
|
||||
]);
|
||||
expect(payload.data.scopedHelp).toContain("<operation> --help");
|
||||
expect(payload.data.usage).toBeUndefined();
|
||||
|
||||
@@ -27,6 +27,7 @@ import { compactStatus, configSummary, manifestObjectSummary, policyChecks, stat
|
||||
import { renderManifest } from "./trace-script";
|
||||
import { formatTable, shortenEnd, textValue } from "./manifest";
|
||||
import { apiPathField, configLabel, kubernetesNameField, stringField } from "./types";
|
||||
import { metricsEnabledForTarget, metricsTargetDisposition } from "./metrics-target";
|
||||
|
||||
export function parseStatusEndpoint(record: Record<string, unknown>, index: number): StatusEndpoint {
|
||||
const path = `probes.statusEndpoints[${index}]`;
|
||||
@@ -57,6 +58,7 @@ export function plan(options: CommonOptions): Record<string, unknown> {
|
||||
const target = resolveTarget(observability, options.targetId);
|
||||
const yaml = renderManifest(observability, target);
|
||||
const policy = policyChecks(yaml, target);
|
||||
const metricsDisposition = metricsTargetDisposition(observability, target);
|
||||
return {
|
||||
ok: policy.every((check) => check.ok),
|
||||
action: "platform-infra-observability-plan",
|
||||
@@ -65,10 +67,16 @@ export function plan(options: CommonOptions): Record<string, unknown> {
|
||||
renderPlan: {
|
||||
target: targetSummary(target),
|
||||
objects: manifestObjectSummary(yaml),
|
||||
metrics: {
|
||||
disposition: metricsDisposition,
|
||||
targetIds: observability.metricsBackend.targetIds,
|
||||
},
|
||||
otlp: {
|
||||
collectorGrpcEndpoint: `${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.grpcPort}`,
|
||||
collectorHttpEndpoint: `http://${observability.collector.serviceName}.${target.namespace}.svc.cluster.local:${observability.collector.otlp.httpPort}`,
|
||||
backendGrpcEndpoint: `${observability.traceBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.traceBackend.otlp.grpcPort}`,
|
||||
prometheusHttpEndpoint: metricsEnabledForTarget(observability, target) ? `http://${observability.metricsBackend.serviceName}.${target.namespace}.svc.cluster.local:${observability.metricsBackend.httpPort}` : null,
|
||||
scrapeDiscovery: observability.metricsBackend.discovery,
|
||||
},
|
||||
instrumentation: observability.instrumentation.serviceConnections,
|
||||
resourceAttributes: observability.resourceAttributes,
|
||||
@@ -80,6 +88,7 @@ export function plan(options: CommonOptions): Record<string, unknown> {
|
||||
status: `bun scripts/cli.ts platform-infra observability status --target ${target.id}`,
|
||||
validate: `bun scripts/cli.ts platform-infra observability validate --target ${target.id}`,
|
||||
trace: `bun scripts/cli.ts platform-infra observability trace --target ${target.id} --trace-id <traceId>`,
|
||||
metricsQuery: `bun scripts/cli.ts platform-infra observability metrics-query --target ${target.id} --query <promql>`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -39,6 +39,12 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
const traceBackend = objectField(root, "traceBackend", "");
|
||||
const traceBackendOtlp = objectField(traceBackend, "otlp", "traceBackend");
|
||||
const traceBackendStorage = objectField(traceBackend, "storage", "traceBackend");
|
||||
const metricsBackend = objectField(root, "metricsBackend", "");
|
||||
const metricsStorage = objectField(metricsBackend, "storage", "metricsBackend");
|
||||
const metricsDiscovery = objectField(metricsBackend, "discovery", "metricsBackend");
|
||||
const metricsLabelSelector = objectField(metricsDiscovery, "labelSelector", "metricsBackend.discovery");
|
||||
const metricsAnnotations = objectField(metricsDiscovery, "annotationKeys", "metricsBackend.discovery");
|
||||
const metricsQuery = objectField(metricsBackend, "query", "metricsBackend");
|
||||
const sampling = objectField(root, "sampling", "");
|
||||
const instrumentation = objectField(root, "instrumentation", "");
|
||||
const resourceAttributes = objectField(root, "resourceAttributes", "");
|
||||
@@ -56,6 +62,7 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
images: {
|
||||
collector: imageSpec(objectField(images, "collector", "images"), "images.collector"),
|
||||
tempo: imageSpec(objectField(images, "tempo", "images"), "images.tempo"),
|
||||
prometheus: imageSpec(objectField(images, "prometheus", "images"), "images.prometheus"),
|
||||
},
|
||||
targets: arrayOfRecords(root.targets, "targets").map(parseTarget),
|
||||
collector: {
|
||||
@@ -79,6 +86,46 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
retention: stringField(traceBackendStorage, "retention", "traceBackend.storage"),
|
||||
},
|
||||
},
|
||||
metricsBackend: {
|
||||
type: enumField(metricsBackend, "type", "metricsBackend", ["prometheus"] as const),
|
||||
enabled: booleanField(metricsBackend, "enabled", "metricsBackend"),
|
||||
targetIds: stringArrayField(metricsBackend, "targetIds", "metricsBackend"),
|
||||
deploymentName: kubernetesNameField(metricsBackend, "deploymentName", "metricsBackend"),
|
||||
serviceName: kubernetesNameField(metricsBackend, "serviceName", "metricsBackend"),
|
||||
configMapName: kubernetesNameField(metricsBackend, "configMapName", "metricsBackend"),
|
||||
serviceAccountName: kubernetesNameField(metricsBackend, "serviceAccountName", "metricsBackend"),
|
||||
clusterRoleName: kubernetesNameField(metricsBackend, "clusterRoleName", "metricsBackend"),
|
||||
clusterRoleBindingName: kubernetesNameField(metricsBackend, "clusterRoleBindingName", "metricsBackend"),
|
||||
persistentVolumeClaimName: kubernetesNameField(metricsBackend, "persistentVolumeClaimName", "metricsBackend"),
|
||||
replicas: integerField(metricsBackend, "replicas", "metricsBackend"),
|
||||
httpPort: portField(metricsBackend, "httpPort", "metricsBackend"),
|
||||
storage: {
|
||||
mode: enumField(metricsStorage, "mode", "metricsBackend.storage", ["persistentVolumeClaim"] as const),
|
||||
size: stringField(metricsStorage, "size", "metricsBackend.storage"),
|
||||
retention: stringField(metricsStorage, "retention", "metricsBackend.storage"),
|
||||
},
|
||||
discovery: {
|
||||
namespaces: stringArrayField(metricsDiscovery, "namespaces", "metricsBackend.discovery"),
|
||||
labelSelector: {
|
||||
key: stringField(metricsLabelSelector, "key", "metricsBackend.discovery.labelSelector"),
|
||||
value: stringField(metricsLabelSelector, "value", "metricsBackend.discovery.labelSelector"),
|
||||
},
|
||||
annotationKeys: {
|
||||
scrape: stringField(metricsAnnotations, "scrape", "metricsBackend.discovery.annotationKeys"),
|
||||
path: stringField(metricsAnnotations, "path", "metricsBackend.discovery.annotationKeys"),
|
||||
port: stringField(metricsAnnotations, "port", "metricsBackend.discovery.annotationKeys"),
|
||||
},
|
||||
defaultPath: apiPathField(metricsDiscovery, "defaultPath", "metricsBackend.discovery"),
|
||||
scrapeInterval: stringField(metricsDiscovery, "scrapeInterval", "metricsBackend.discovery"),
|
||||
scrapeTimeout: stringField(metricsDiscovery, "scrapeTimeout", "metricsBackend.discovery"),
|
||||
},
|
||||
query: {
|
||||
path: apiPathField(metricsQuery, "path", "metricsBackend.query"),
|
||||
timeoutSeconds: integerField(metricsQuery, "timeoutSeconds", "metricsBackend.query"),
|
||||
maxSeries: integerField(metricsQuery, "maxSeries", "metricsBackend.query"),
|
||||
maxResponseBytes: integerField(metricsQuery, "maxResponseBytes", "metricsBackend.query"),
|
||||
},
|
||||
},
|
||||
sampling: {
|
||||
mode: enumField(sampling, "mode", "sampling", ["parentbased_traceidratio"] as const),
|
||||
ratio: numberField(sampling, "ratio", "sampling"),
|
||||
@@ -99,12 +146,37 @@ export function readObservabilityConfig(): ObservabilityConfig {
|
||||
};
|
||||
if (config.targets.length === 0) throw new Error(`${configLabel}.targets must not be empty`);
|
||||
assertKnownEnabledTarget(config.targets, config.defaults.targetId, "defaults.targetId");
|
||||
if (config.collector.replicas < 0 || config.traceBackend.replicas < 0) throw new Error(`${configLabel} replicas must be >= 0`);
|
||||
for (const targetId of config.metricsBackend.targetIds) assertKnownEnabledTarget(config.targets, targetId, "metricsBackend.targetIds");
|
||||
if (config.metricsBackend.enabled && config.metricsBackend.targetIds.length === 0) throw new Error(`${configLabel}.metricsBackend.targetIds must not be empty when metricsBackend.enabled is true`);
|
||||
if (new Set(config.metricsBackend.targetIds.map((targetId) => targetId.toLowerCase())).size !== config.metricsBackend.targetIds.length) throw new Error(`${configLabel}.metricsBackend.targetIds must not contain duplicates`);
|
||||
if (!isKubernetesQualifiedName(config.metricsBackend.discovery.labelSelector.key)) throw new Error(`${configLabel}.metricsBackend.discovery.labelSelector.key must be a Kubernetes qualified name`);
|
||||
if (!isKubernetesLabelValue(config.metricsBackend.discovery.labelSelector.value)) throw new Error(`${configLabel}.metricsBackend.discovery.labelSelector.value must be a Kubernetes label value`);
|
||||
if (config.collector.replicas < 0 || config.traceBackend.replicas < 0 || config.metricsBackend.replicas < 0) throw new Error(`${configLabel} replicas must be >= 0`);
|
||||
if (config.metricsBackend.query.timeoutSeconds < 1 || config.metricsBackend.query.maxSeries < 1 || config.metricsBackend.query.maxResponseBytes < 1024) throw new Error(`${configLabel}.metricsBackend.query budgets must be positive and maxResponseBytes must be >= 1024`);
|
||||
if (config.sampling.ratio < 0 || config.sampling.ratio > 1) throw new Error(`${configLabel}.sampling.ratio must be between 0 and 1`);
|
||||
if (!config.probes.traceQueryPathTemplate.includes("{{traceId}}")) throw new Error(`${configLabel}.probes.traceQueryPathTemplate must include {{traceId}}`);
|
||||
return config;
|
||||
}
|
||||
|
||||
export function isKubernetesQualifiedName(value: string): boolean {
|
||||
const slash = value.indexOf("/");
|
||||
if (slash !== value.lastIndexOf("/")) return false;
|
||||
const prefix = slash < 0 ? null : value.slice(0, slash);
|
||||
const name = slash < 0 ? value : value.slice(slash + 1);
|
||||
if (!isKubernetesNamePart(name)) return false;
|
||||
if (prefix === null) return true;
|
||||
if (prefix.length === 0 || prefix.length > 253) return false;
|
||||
return prefix.split(".").every((segment) => segment.length <= 63 && /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(segment));
|
||||
}
|
||||
|
||||
export function isKubernetesLabelValue(value: string): boolean {
|
||||
return value.length <= 63 && (value === "" || isKubernetesNamePart(value));
|
||||
}
|
||||
|
||||
function isKubernetesNamePart(value: string): boolean {
|
||||
return value.length > 0 && value.length <= 63 && /^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(value);
|
||||
}
|
||||
|
||||
export function imageSpec(record: Record<string, unknown>, path: string): ImageSpec {
|
||||
const image = {
|
||||
repository: stringField(record, "repository", path),
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { ObservabilityConfig, ObservabilityTarget } from "./types";
|
||||
|
||||
export type MetricsTargetDisposition = "enabled" | "disabled" | "disabled-for-target";
|
||||
|
||||
export function metricsTargetDisposition(observability: ObservabilityConfig, target: ObservabilityTarget): MetricsTargetDisposition {
|
||||
if (!observability.metricsBackend.enabled) return "disabled";
|
||||
return observability.metricsBackend.targetIds.some((targetId) => targetId.toLowerCase() === target.id.toLowerCase()) ? "enabled" : "disabled-for-target";
|
||||
}
|
||||
|
||||
export function metricsEnabledForTarget(observability: ObservabilityConfig, target: ObservabilityTarget): boolean {
|
||||
return metricsTargetDisposition(observability, target) === "enabled";
|
||||
}
|
||||
@@ -19,18 +19,19 @@ import {
|
||||
capture,
|
||||
} from "../platform-infra-ops-library";
|
||||
|
||||
import type { ApplyOptions, CommonOptions, DiagnoseCodeAgentOptions, SearchOptions, TraceOptions } from "./types";
|
||||
import type { ApplyOptions, CommonOptions, DiagnoseCodeAgentOptions, MetricsQueryOptions, SearchOptions, TraceOptions } from "./types";
|
||||
import { apply, plan, status, validate } from "./actions";
|
||||
import { diagnoseCodeAgent, search, trace } from "./render";
|
||||
import { renderObservabilityPlan, renderObservabilityPlanFailure } from "./plan-render";
|
||||
import { metricsQuery } from "./prometheus";
|
||||
|
||||
const observabilityOperations = ["plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent"] as const;
|
||||
const observabilityOperations = ["plan", "apply", "status", "validate", "trace", "search", "diagnose-code-agent", "metrics-query"] as const;
|
||||
|
||||
export function observabilityHelp(scope: string | null = null): Record<string, unknown> {
|
||||
const scoped = scope === null ? null : observabilityOperationHelp(scope);
|
||||
if (scoped !== null) return scoped;
|
||||
return {
|
||||
command: "platform-infra observability plan|apply|status|validate|trace|search|diagnose-code-agent",
|
||||
command: "platform-infra observability plan|apply|status|validate|trace|search|diagnose-code-agent|metrics-query",
|
||||
output: "默认输出为有界摘要；--full/--raw 显式披露完整结构",
|
||||
configTruth: "config/platform-infra/observability.yaml",
|
||||
spec: "PJ2026-01060501 OTel追踪 draft-2026-06-19-p0",
|
||||
@@ -39,7 +40,7 @@ export function observabilityHelp(scope: string | null = null): Record<string, u
|
||||
help: `bun scripts/cli.ts platform-infra observability ${name} --help`,
|
||||
})),
|
||||
scopedHelp: "bun scripts/cli.ts platform-infra observability <operation> --help",
|
||||
boundary: "Prometheus 仍是指标来源；本命令只负责 platform-infra OTel Collector、Tempo readiness 与 trace 查询。",
|
||||
boundary: "Collector + Tempo 是 trace authority；Prometheus 仅承担 metrics，故障和漂移保持非阻塞 warning。",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -62,6 +63,7 @@ export async function runPlatformObservabilityCommand(config: UniDeskConfig, arg
|
||||
if (action === "trace") return await trace(config, parseTraceOptions(args.slice(1)));
|
||||
if (action === "search") return await search(config, parseSearchOptions(args.slice(1)));
|
||||
if (action === "diagnose-code-agent") return await diagnoseCodeAgent(config, parseDiagnoseCodeAgentOptions(args.slice(1)));
|
||||
if (action === "metrics-query") return await metricsQuery(config, parseMetricsQueryOptions(args.slice(1)));
|
||||
return { ok: false, error: "unsupported-platform-infra-observability-command", args, help: observabilityHelp() };
|
||||
}
|
||||
|
||||
@@ -105,6 +107,12 @@ function observabilityOperationHelp(scope: string): Record<string, unknown> | nu
|
||||
],
|
||||
options: ["--trace-id <32-hex>", "--target <NODE>", "--grep <text>", "--limit <1..500>", "--full", "--raw"],
|
||||
};
|
||||
if (scope === "metrics-query") return {
|
||||
...common,
|
||||
command: "platform-infra observability metrics-query",
|
||||
usage: ["bun scripts/cli.ts platform-infra observability metrics-query --target <NODE> --query <promql> [--full|--raw]"],
|
||||
options: ["--target <NODE>", "--query <promql>", "--full", "--raw"],
|
||||
};
|
||||
if (scope === "search") return {
|
||||
...common,
|
||||
command: "platform-infra observability search",
|
||||
@@ -126,6 +134,27 @@ function observabilityOperationHelp(scope: string): Record<string, unknown> | nu
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseMetricsQueryOptions(args: string[]): MetricsQueryOptions {
|
||||
const commonArgs: string[] = [];
|
||||
let query: string | null = null;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--query") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error("--query requires a value");
|
||||
query = value;
|
||||
index += 1;
|
||||
} else {
|
||||
commonArgs.push(arg);
|
||||
if (arg === "--target") {
|
||||
commonArgs.push(args[index + 1] ?? "");
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ...parseCommonOptions(commonArgs), query };
|
||||
}
|
||||
|
||||
export function parseCommonOptions(args: string[]): CommonOptions {
|
||||
let targetId: string | null = null;
|
||||
let full = false;
|
||||
|
||||
@@ -8,7 +8,9 @@ export function renderObservabilityPlan(result: Record<string, unknown>): Render
|
||||
const images = record(config.images);
|
||||
const collector = record(config.collector);
|
||||
const traceBackend = record(config.traceBackend);
|
||||
const metricsBackend = record(config.metricsBackend);
|
||||
const renderPlan = record(result.renderPlan);
|
||||
const metricsPlan = record(renderPlan.metrics);
|
||||
const otlp = record(renderPlan.otlp);
|
||||
const connections = records(config.serviceConnections);
|
||||
const objects = records(renderPlan.objects);
|
||||
@@ -39,6 +41,8 @@ export function renderObservabilityPlan(result: Record<string, unknown>): Render
|
||||
`target=${textValue(target.id)} route=${textValue(target.route)} namespace=${textValue(target.namespace)} role=${textValue(target.role)}`,
|
||||
`collector=${textValue(collector.serviceName)} replicas=${textValue(collector.replicas)} image=${textValue(images.collector)}`,
|
||||
`tempo=${textValue(traceBackend.serviceName)} replicas=${textValue(traceBackend.replicas)} retention=${textValue(record(traceBackend.storage).retention)} image=${textValue(images.traceBackend)}`,
|
||||
`prometheus=${textValue(metricsPlan.disposition)} service=${textValue(metricsBackend.serviceName)} targets=${values(metricsPlan.targetIds).join(",")} replicas=${textValue(metricsBackend.replicas)} retention=${textValue(record(metricsBackend.storage).retention)} image=${textValue(images.metricsBackend)}`,
|
||||
`scrapeSelector=${textValue(record(record(metricsBackend.discovery).labelSelector).key)}=${textValue(record(record(metricsBackend.discovery).labelSelector).value)} namespaces=${values(record(metricsBackend.discovery).namespaces).length === 0 ? "all" : values(record(metricsBackend.discovery).namespaces).join(",")} queryBudget=${textValue(record(metricsBackend.query).timeoutSeconds)}s/${textValue(record(metricsBackend.query).maxSeries)}series/${textValue(record(metricsBackend.query).maxResponseBytes)}bytes`,
|
||||
`serviceConnections=${connections.length} services=${serviceNames.size} nodes=${nodes.size} configRefs=${presentConfigRefs.length}/${configRefs.length} missing=${missingConfigRefs.length} objects=${objects.length}`,
|
||||
"",
|
||||
"Service connections:",
|
||||
@@ -77,6 +81,7 @@ export function renderObservabilityPlan(result: Record<string, unknown>): Render
|
||||
` collector-grpc: ${textValue(otlp.collectorGrpcEndpoint)}`,
|
||||
` collector-http: ${textValue(otlp.collectorHttpEndpoint)}`,
|
||||
` tempo-grpc: ${textValue(otlp.backendGrpcEndpoint)}`,
|
||||
` prometheus-http: ${textValue(otlp.prometheusHttpEndpoint)}`,
|
||||
"",
|
||||
"Next:",
|
||||
` bun scripts/cli.ts platform-infra observability plan --target ${textValue(target.id)} --full`,
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
import { capture, compactCapture, parseJsonOutput, redactSensitiveUnknown, shQuote } from "../platform-infra-ops-library";
|
||||
import { resolveTarget } from "./actions";
|
||||
import { readObservabilityConfig } from "./config";
|
||||
import { formatTable, shortenEnd, textValue } from "./manifest";
|
||||
import { imageReference, indent, targetSummary } from "./summary";
|
||||
import type { MetricsQueryOptions, ObservabilityConfig, ObservabilityTarget } from "./types";
|
||||
import { metricsEnabledForTarget, metricsTargetDisposition } from "./metrics-target";
|
||||
|
||||
export function prometheusManifests(observability: ObservabilityConfig, target: ObservabilityTarget): string[] {
|
||||
const prometheus = observability.metricsBackend;
|
||||
if (!metricsEnabledForTarget(observability, target)) return [];
|
||||
const namespaceNames = prometheus.discovery.namespaces.length > 0
|
||||
? `\n namespaces:\n names:\n${prometheus.discovery.namespaces.map((name) => ` - ${name}`).join("\n")}`
|
||||
: "";
|
||||
const config = `global:\n scrape_interval: ${prometheus.discovery.scrapeInterval}\n scrape_timeout: ${prometheus.discovery.scrapeTimeout}\nscrape_configs:\n - job_name: kubernetes-pods\n kubernetes_sd_configs:\n - role: pod${namespaceNames}\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_${prometheusMetaLabel(prometheus.discovery.annotationKeys.scrape)}]\n action: keep\n regex: \"true\"\n - source_labels: [__meta_kubernetes_pod_label_${prometheusMetaLabel(prometheus.discovery.labelSelector.key)}]\n action: keep\n regex: '${re2Literal(prometheus.discovery.labelSelector.value)}'\n - source_labels: [__meta_kubernetes_pod_annotation_${prometheusMetaLabel(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: (/.+)\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_${prometheusMetaLabel(prometheus.discovery.annotationKeys.port)}]\n action: replace\n regex: ([^:]+)(?::\\d+)?;(\\d+)\n replacement: \$1:\$2\n target_label: __address__\n - source_labels: [__meta_kubernetes_pod_annotation_${prometheusMetaLabel(prometheus.discovery.annotationKeys.path)}]\n action: replace\n target_label: __metrics_path__\n regex: ^$\n replacement: ${prometheus.discovery.defaultPath}\n`;
|
||||
const labels = ` app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n app.kubernetes.io/part-of: platform-infra\n app.kubernetes.io/managed-by: unidesk`;
|
||||
return [
|
||||
`apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: ${prometheus.serviceAccountName}\n namespace: ${target.namespace}\n labels:\n${labels}\n`,
|
||||
`apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\n name: ${prometheus.clusterRoleName}\nrules:\n - apiGroups: [\"\"]\n resources: [\"nodes\", \"nodes/proxy\", \"services\", \"endpoints\", \"pods\"]\n verbs: [\"get\", \"list\", \"watch\"]\n`,
|
||||
`apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\n name: ${prometheus.clusterRoleBindingName}\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: ${prometheus.clusterRoleName}\nsubjects:\n - kind: ServiceAccount\n name: ${prometheus.serviceAccountName}\n namespace: ${target.namespace}\n`,
|
||||
`apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ${prometheus.configMapName}\n namespace: ${target.namespace}\n labels:\n${labels}\ndata:\n prometheus.yml: |\n${indent(config, 4)}\n`,
|
||||
`apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: ${prometheus.persistentVolumeClaimName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n accessModes: [ReadWriteOnce]\n resources:\n requests:\n storage: ${prometheus.storage.size}\n`,
|
||||
`apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: ${prometheus.deploymentName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n replicas: ${prometheus.replicas}\n selector:\n matchLabels:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n template:\n metadata:\n labels:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n app.kubernetes.io/part-of: platform-infra\n spec:\n serviceAccountName: ${prometheus.serviceAccountName}\n containers:\n - name: prometheus\n image: ${imageReference(observability.images.prometheus)}\n imagePullPolicy: ${observability.images.prometheus.pullPolicy}\n args:\n - --config.file=/etc/prometheus/prometheus.yml\n - --storage.tsdb.path=/prometheus\n - --storage.tsdb.retention.time=${prometheus.storage.retention}\n ports:\n - name: http\n containerPort: ${prometheus.httpPort}\n readinessProbe:\n httpGet:\n path: /-/ready\n port: http\n volumeMounts:\n - name: config\n mountPath: /etc/prometheus/prometheus.yml\n subPath: prometheus.yml\n readOnly: true\n - name: data\n mountPath: /prometheus\n volumes:\n - name: config\n configMap:\n name: ${prometheus.configMapName}\n - name: data\n persistentVolumeClaim:\n claimName: ${prometheus.persistentVolumeClaimName}\n`,
|
||||
`apiVersion: v1\nkind: Service\nmetadata:\n name: ${prometheus.serviceName}\n namespace: ${target.namespace}\n labels:\n${labels}\nspec:\n type: ClusterIP\n selector:\n app.kubernetes.io/name: ${prometheus.deploymentName}\n app.kubernetes.io/component: metrics-backend\n ports:\n - name: http\n port: ${prometheus.httpPort}\n targetPort: http\n`,
|
||||
];
|
||||
}
|
||||
|
||||
export async function metricsQuery(config: UniDeskConfig, options: MetricsQueryOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (options.query === null || options.query.trim() === "") throw new Error("observability metrics-query requires --query <promql>");
|
||||
const observability = readObservabilityConfig();
|
||||
const target = resolveTarget(observability, options.targetId);
|
||||
const prometheus = observability.metricsBackend;
|
||||
const disposition = metricsTargetDisposition(observability, target);
|
||||
if (disposition !== "enabled") return { ok: false, action: "platform-infra-observability-metrics-query", mutation: false, target: targetSummary(target), metrics: { enabled: false, disposition, targetIds: prometheus.targetIds }, warnings: [`Prometheus is ${disposition} in config/platform-infra/observability.yaml`] };
|
||||
const path = `${prometheus.query.path}?query=${encodeURIComponent(options.query)}`;
|
||||
const script = `set -u\npython3 - <<'PY'\nimport json, subprocess\npath = ${JSON.stringify(`/api/v1/namespaces/${target.namespace}/services/http:${prometheus.serviceName}:http/proxy${path}`)}\nproc = subprocess.run([\"kubectl\", \"get\", \"--raw\", path], text=True, capture_output=True, timeout=${prometheus.query.timeoutSeconds})\nbody = proc.stdout[:${prometheus.query.maxResponseBytes}]\ntry:\n parsed = json.loads(body) if body else None\nexcept Exception:\n parsed = None\nresult = parsed.get(\"data\", {}).get(\"result\", []) if isinstance(parsed, dict) else []\nprint(json.dumps({\"ok\": proc.returncode == 0 and isinstance(parsed, dict) and parsed.get(\"status\") == \"success\", \"exitCode\": proc.returncode, \"truncated\": len(proc.stdout) > ${prometheus.query.maxResponseBytes} or len(result) > ${prometheus.query.maxSeries}, \"resultType\": parsed.get(\"data\", {}).get(\"resultType\") if isinstance(parsed, dict) else None, \"result\": result[:${prometheus.query.maxSeries}], \"stderrTail\": proc.stderr[-2000:]}, ensure_ascii=False))\nPY`;
|
||||
const result = await capture(config, target.route, ["sh"], script);
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && parsed?.ok === true;
|
||||
if (options.full || options.raw) return { ok, action: "platform-infra-observability-metrics-query", mutation: false, target: targetSummary(target), query: options.query, budgets: prometheus.query, result: parsed === null ? compactCapture(result, { full: true }) : redactSensitiveUnknown(parsed) };
|
||||
const rows = Array.isArray(parsed?.result) ? parsed.result.slice(0, 20).map((item: unknown) => [shortenEnd(JSON.stringify(item), 160)]) : [];
|
||||
return { ok, command: "platform-infra observability metrics-query", contentType: "text/plain", renderedText: [`platform-infra observability metrics-query (${ok ? "ok" : "not-ok"})`, "", `target=${target.id} prometheus=${prometheus.serviceName}.${target.namespace}.svc.cluster.local:${prometheus.httpPort}`, `query=${options.query}`, `budget=timeout:${prometheus.query.timeoutSeconds}s series:${prometheus.query.maxSeries} bytes:${prometheus.query.maxResponseBytes} truncated=${textValue(parsed?.truncated)}`, "", formatTable(["RESULT"], rows.length > 0 ? rows : [["-"]]), "", `Next: bun scripts/cli.ts platform-infra observability metrics-query --target ${target.id} --query ${shQuote(options.query)} --full`].join("\n") };
|
||||
}
|
||||
|
||||
export function prometheusMetaLabel(value: string): string {
|
||||
return value.replace(/[^A-Za-z0-9_]/gu, "_");
|
||||
}
|
||||
|
||||
export function re2Literal(value: string): string {
|
||||
return `^${value.replace(/[\\.^$|?*+()[\]{}]/gu, "\\$&")}$`;
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
|
||||
import type { ObservabilityConfig, ObservabilityTarget } from "./types";
|
||||
import { fieldManager } from "./types";
|
||||
import { metricsEnabledForTarget, metricsTargetDisposition } from "./metrics-target";
|
||||
|
||||
export function tempoService(observability: ObservabilityConfig, target: ObservabilityTarget): string {
|
||||
return `apiVersion: v1
|
||||
@@ -102,7 +103,15 @@ PY
|
||||
}
|
||||
|
||||
export function statusScript(observability: ObservabilityConfig, target: ObservabilityTarget, full: boolean, generateValidationTrace = false): string {
|
||||
const endpointsJson = JSON.stringify(observability.probes.statusEndpoints);
|
||||
const metricsEnabled = metricsEnabledForTarget(observability, target);
|
||||
const metricsDisposition = metricsTargetDisposition(observability, target);
|
||||
const enabledEndpoints = metricsEnabled
|
||||
? observability.probes.statusEndpoints
|
||||
: observability.probes.statusEndpoints.filter((endpoint) => endpoint.service !== observability.metricsBackend.serviceName);
|
||||
const endpointsJson = JSON.stringify(enabledEndpoints);
|
||||
const metricsCaptures = metricsEnabled
|
||||
? `capture_json metrics_deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.metricsBackend.deploymentName)} -o json\ncapture_json metrics_services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.metricsBackend.serviceName)} -o json\ncapture_json metrics_pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name=${observability.metricsBackend.deploymentName}`)} -o json`
|
||||
: "";
|
||||
return `
|
||||
set -u
|
||||
tmp="$(mktemp -d)"
|
||||
@@ -123,6 +132,7 @@ capture_json namespace kubectl get namespace ${shQuote(target.namespace)} -o jso
|
||||
capture_json deployments kubectl -n ${shQuote(target.namespace)} get deployment ${shQuote(observability.collector.deploymentName)} ${shQuote(observability.traceBackend.deploymentName)} -o json
|
||||
capture_json services kubectl -n ${shQuote(target.namespace)} get service ${shQuote(observability.collector.serviceName)} ${shQuote(observability.traceBackend.serviceName)} -o json
|
||||
capture_json pods kubectl -n ${shQuote(target.namespace)} get pods -l ${shQuote(`app.kubernetes.io/name in (${observability.collector.deploymentName},${observability.traceBackend.deploymentName})`)} -o json
|
||||
${metricsCaptures}
|
||||
capture_json events kubectl -n ${shQuote(target.namespace)} get events -o json
|
||||
python3 - "$tmp" '${endpointsJson.replaceAll("'", "'\"'\"'")}' <<'PY'
|
||||
import json, random, socket, subprocess, sys, time, urllib.error, urllib.request
|
||||
@@ -304,7 +314,9 @@ def generate_test_trace():
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=3)
|
||||
runtime_ready = rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in probe_results)
|
||||
blocking_probes = [item for item in probe_results if item.get("service") != "${observability.metricsBackend.serviceName}"]
|
||||
metrics_probes = [item for item in probe_results if item.get("service") == "${observability.metricsBackend.serviceName}"]
|
||||
runtime_ready = rc("namespace") == 0 and rc("deployments") == 0 and rc("services") == 0 and all(item["ok"] for item in blocking_probes)
|
||||
validation_trace = generate_test_trace() if (${generateValidationTrace ? "True" : "False"} and runtime_ready) else None
|
||||
payload = {
|
||||
"ok": runtime_ready and (validation_trace is None or validation_trace.get("ok") is True),
|
||||
@@ -316,8 +328,12 @@ payload = {
|
||||
"services": compact_section("services"),
|
||||
"pods": compact_section("pods"),
|
||||
"events": compact_section("events"),
|
||||
"metricsDeployments": compact_section("metrics_deployments") if ${metricsEnabled ? "True" : "False"} else None,
|
||||
"metricsServices": compact_section("metrics_services") if ${metricsEnabled ? "True" : "False"} else None,
|
||||
"metricsPods": compact_section("metrics_pods") if ${metricsEnabled ? "True" : "False"} else None,
|
||||
},
|
||||
"probes": probe_results,
|
||||
"warnings": ([{"code": "prometheus-not-ready", "detail": item} for item in metrics_probes if item.get("ok") is not True] if ${metricsEnabled ? "True" : "False"} else [{"code": "prometheus-${metricsDisposition}"}]),
|
||||
"validationTrace": validation_trace,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2 if ${full ? "True" : "False"} else None))
|
||||
|
||||
@@ -32,9 +32,11 @@ export function configSummary(observability: ObservabilityConfig, target: Observ
|
||||
images: {
|
||||
collector: imageReference(observability.images.collector),
|
||||
traceBackend: imageReference(observability.images.tempo),
|
||||
metricsBackend: imageReference(observability.images.prometheus),
|
||||
},
|
||||
collector: observability.collector,
|
||||
traceBackend: observability.traceBackend,
|
||||
metricsBackend: observability.metricsBackend,
|
||||
sampling: observability.sampling,
|
||||
serviceConnections: observability.instrumentation.serviceConnections.map((item) => ({
|
||||
serviceName: item.serviceName,
|
||||
@@ -62,8 +64,8 @@ export function targetSummary(target: ObservabilityTarget): Record<string, unkno
|
||||
|
||||
export function policyChecks(yaml: string, target: ObservabilityTarget): Array<Record<string, unknown>> {
|
||||
return [
|
||||
{ name: "yaml-source-of-truth", ok: true, detail: "All concrete images, routes, namespace, ports, retention and sampling values are read from config/platform-infra/observability.yaml." },
|
||||
{ name: "clusterip-only", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(yaml), detail: "Collector and trace backend stay ClusterIP-only." },
|
||||
{ name: "yaml-source-of-truth", ok: true, detail: "All concrete images, routes, namespace, ports, retention, storage, scrape discovery and query budgets are read from config/platform-infra/observability.yaml." },
|
||||
{ name: "clusterip-only", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(yaml), detail: "Collector, Tempo and Prometheus stay ClusterIP-only." },
|
||||
{ name: "no-ingress", ok: !/^\s*kind:\s*Ingress\s*$/mu.test(yaml), detail: "No public ingress is rendered for the first tracing backend." },
|
||||
{ name: "no-host-network", ok: !/^\s*hostNetwork:\s*true\s*$/mu.test(yaml), detail: "Pods must not use host network." },
|
||||
{ name: "allow-all-network-policy", ok: yaml.includes("kind: NetworkPolicy") && yaml.includes("name: allow-all") && yaml.includes(`namespace: ${target.namespace}`), detail: `NetworkPolicy/allow-all is rendered in ${target.namespace}.` },
|
||||
@@ -76,6 +78,9 @@ export function statusSummary(payload: Record<string, unknown>): Record<string,
|
||||
const services = objectList(sectionJson(sections, "services"));
|
||||
const pods = objectList(sectionJson(sections, "pods"));
|
||||
const events = objectList(sectionJson(sections, "events"));
|
||||
const metricsDeployments = objectList(optionalSectionJson(sections, "metricsDeployments"));
|
||||
const metricsServices = objectList(optionalSectionJson(sections, "metricsServices"));
|
||||
const metricsPods = objectList(optionalSectionJson(sections, "metricsPods"));
|
||||
const podEvents = latestPodEvents(events);
|
||||
const probes = Array.isArray(payload.probes) ? payload.probes as Array<Record<string, unknown>> : [];
|
||||
const readyDeployments = deployments.map((item) => deploymentSummary(item));
|
||||
@@ -92,6 +97,12 @@ export function statusSummary(payload: Record<string, unknown>): Record<string,
|
||||
path: item.path,
|
||||
stderrTail: item.ok === true ? "" : item.stderrTail,
|
||||
})),
|
||||
warnings: Array.isArray(payload.warnings) ? payload.warnings : [],
|
||||
metrics: {
|
||||
deployments: metricsDeployments.map((item) => deploymentSummary(item)),
|
||||
services: metricsServices.map((item) => metadataName(item)),
|
||||
pods: metricsPods.map((item) => podSummary(item, podEvents)),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -106,6 +117,12 @@ export function sectionJson(sections: Record<string, unknown>, name: string): un
|
||||
return section.json;
|
||||
}
|
||||
|
||||
function optionalSectionJson(sections: Record<string, unknown>, name: string): unknown {
|
||||
const value = sections[name];
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
|
||||
return (value as Record<string, unknown>).json;
|
||||
}
|
||||
|
||||
export function objectList(value: unknown): Record<string, unknown>[] {
|
||||
if (typeof value !== "object" || value === null) return [];
|
||||
const items = (value as Record<string, unknown>).items;
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
import type { ObservabilityConfig, ObservabilityTarget } from "./types";
|
||||
import { tempoService } from "./search-script";
|
||||
import { imageReference, indent } from "./summary";
|
||||
import { prometheusManifests } from "./prometheus";
|
||||
|
||||
export function asArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
@@ -43,6 +44,7 @@ export function renderManifest(observability: ObservabilityConfig, target: Obser
|
||||
tempoConfigMap(observability, target),
|
||||
tempoDeployment(observability, target, tempoImage),
|
||||
tempoService(observability, target),
|
||||
...prometheusManifests(observability, target),
|
||||
].filter((item) => item.trim().length > 0).join("\n---\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ export interface ObservabilityConfig {
|
||||
images: {
|
||||
collector: ImageSpec;
|
||||
tempo: ImageSpec;
|
||||
prometheus: ImageSpec;
|
||||
};
|
||||
targets: ObservabilityTarget[];
|
||||
collector: {
|
||||
@@ -71,6 +72,7 @@ export interface ObservabilityConfig {
|
||||
otlp: OtlpPorts;
|
||||
storage: { mode: "emptyDir"; retention: string };
|
||||
};
|
||||
metricsBackend: PrometheusConfig;
|
||||
sampling: { mode: "parentbased_traceidratio"; ratio: number };
|
||||
instrumentation: {
|
||||
contextPropagation: string[];
|
||||
@@ -87,6 +89,31 @@ export interface ObservabilityConfig {
|
||||
};
|
||||
}
|
||||
|
||||
export interface PrometheusConfig {
|
||||
type: "prometheus";
|
||||
enabled: boolean;
|
||||
targetIds: string[];
|
||||
deploymentName: string;
|
||||
serviceName: string;
|
||||
configMapName: string;
|
||||
serviceAccountName: string;
|
||||
clusterRoleName: string;
|
||||
clusterRoleBindingName: string;
|
||||
persistentVolumeClaimName: string;
|
||||
replicas: number;
|
||||
httpPort: number;
|
||||
storage: { mode: "persistentVolumeClaim"; size: string; retention: string };
|
||||
discovery: {
|
||||
namespaces: string[];
|
||||
labelSelector: { key: string; value: string };
|
||||
annotationKeys: { scrape: string; path: string; port: string };
|
||||
defaultPath: string;
|
||||
scrapeInterval: string;
|
||||
scrapeTimeout: string;
|
||||
};
|
||||
query: { path: string; timeoutSeconds: number; maxSeries: number; maxResponseBytes: number };
|
||||
}
|
||||
|
||||
export interface ImageSpec {
|
||||
repository: string;
|
||||
tag: string;
|
||||
@@ -165,3 +192,7 @@ export interface DiagnoseCodeAgentOptions extends CommonOptions {
|
||||
candidateLimit: number;
|
||||
lookbackMinutes: number;
|
||||
}
|
||||
|
||||
export interface MetricsQueryOptions extends CommonOptions {
|
||||
query: string | null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { resolveToken } from "./gh/auth-and-safety";
|
||||
import { githubRequest, isGitHubError } from "./gh/client";
|
||||
|
||||
export interface PacDeliveryTimingBinding {
|
||||
target: Record<string, unknown>;
|
||||
consumer: {
|
||||
id: string;
|
||||
node: string;
|
||||
lane: string;
|
||||
repository: string;
|
||||
branch: string;
|
||||
pipeline: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface PacDeliveryTimingObservers {
|
||||
history: () => Promise<Record<string, unknown>>;
|
||||
status: () => Promise<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record) : [];
|
||||
}
|
||||
|
||||
function durationBetween(start: unknown, end: unknown): number | null {
|
||||
if (typeof start !== "string" || typeof end !== "string") return null;
|
||||
const a = Date.parse(start);
|
||||
const b = Date.parse(end);
|
||||
if (!Number.isFinite(a) || !Number.isFinite(b)) return null;
|
||||
return Math.max(0, Math.round((b - a) / 1000));
|
||||
}
|
||||
|
||||
export async function observePacDeliveryTiming(
|
||||
binding: PacDeliveryTimingBinding,
|
||||
observers: PacDeliveryTimingObservers,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const [sourceOwner, sourceRepo] = binding.consumer.repository.split("/");
|
||||
if (!sourceOwner || !sourceRepo) throw new Error(`invalid owning YAML source repository ${binding.consumer.repository}`);
|
||||
const [historyResult, statusResult] = await Promise.all([observers.history(), observers.status()]);
|
||||
const row = records(historyResult.rows)[0] ?? {};
|
||||
const statusSummary = record(statusResult.summary);
|
||||
const latest = record(statusSummary.latestPipelineRun);
|
||||
const commit = typeof row.commit === "string" ? row.commit : typeof latest.sourceCommit === "string" ? latest.sourceCommit : null;
|
||||
const evidenceGaps: string[] = [];
|
||||
let pullRequest: Record<string, unknown> | null = null;
|
||||
let githubEvidence: Record<string, unknown> = { repository: binding.consumer.repository, source: "github-commit-pulls", mutation: false };
|
||||
if (commit === null) {
|
||||
evidenceGaps.push("source-commit-missing");
|
||||
} else {
|
||||
const tokenInfo = resolveToken(binding.consumer.repository, false);
|
||||
if (tokenInfo.token === null) {
|
||||
evidenceGaps.push("github-token-unavailable");
|
||||
githubEvidence = { ...githubEvidence, ok: false, token: tokenInfo.probe };
|
||||
} else {
|
||||
const response = await githubRequest<unknown>(tokenInfo.token, "GET", `/repos/${sourceOwner}/${sourceRepo}/commits/${commit}/pulls`);
|
||||
if (isGitHubError(response)) {
|
||||
evidenceGaps.push("github-pr-lookup-failed");
|
||||
githubEvidence = { ...githubEvidence, ok: false, error: response };
|
||||
} else {
|
||||
const prs = Array.isArray(response) ? response : [];
|
||||
const merged = prs.find((item) => record(item).merged_at !== null && record(item).merged_at !== undefined) ?? prs[0];
|
||||
if (merged === undefined) {
|
||||
evidenceGaps.push("github-pr-not-found");
|
||||
} else {
|
||||
const item = record(merged);
|
||||
pullRequest = { number: item.number ?? null, title: item.title ?? null, url: item.html_url ?? null, mergedAt: item.merged_at ?? null, mergeCommit: commit };
|
||||
if (typeof item.merged_at !== "string") evidenceGaps.push("github-mergedAt-missing");
|
||||
}
|
||||
githubEvidence = { ...githubEvidence, ok: true, count: prs.length, token: tokenInfo.probe };
|
||||
}
|
||||
}
|
||||
}
|
||||
const mergedAt = pullRequest?.mergedAt ?? null;
|
||||
const startTime = row.startTime ?? null;
|
||||
const completionTime = row.completionTime ?? null;
|
||||
const argo = record(statusSummary.argo);
|
||||
const runtime = record(statusSummary.runtime);
|
||||
const provenance = record(statusSummary.provenance);
|
||||
if (mergedAt === null) evidenceGaps.push("mergedAt-missing");
|
||||
if (startTime === null || completionTime === null) evidenceGaps.push("pipeline-timestamps-missing");
|
||||
if (argo.sync === undefined || argo.health === undefined) evidenceGaps.push("argo-closeout-missing");
|
||||
if (runtime.readyReplicas === undefined) evidenceGaps.push("runtime-health-missing");
|
||||
if (provenance.relation === "unknown") evidenceGaps.push("runtime-provenance-unknown");
|
||||
return {
|
||||
ok: commit !== null && startTime !== null && completionTime !== null,
|
||||
action: "platform-infra-pipelines-as-code-delivery-timing",
|
||||
mutation: false,
|
||||
state: evidenceGaps.length === 0 ? "complete" : "partial",
|
||||
target: binding.target,
|
||||
consumer: binding.consumer,
|
||||
source: { commit, pullRequest, github: githubEvidence },
|
||||
timing: {
|
||||
mergedAt,
|
||||
pipelineStart: startTime,
|
||||
pipelineCompletion: completionTime,
|
||||
triggerWaitSeconds: durationBetween(mergedAt, startTime),
|
||||
pipelineDurationSeconds: row.durationSeconds ?? durationBetween(startTime, completionTime),
|
||||
endToEndSeconds: durationBetween(mergedAt, completionTime),
|
||||
},
|
||||
pipeline: { id: row.id ?? latest.name ?? null, status: row.status ?? latest.status ?? null, taskRuns: row.taskRuns ?? statusSummary.taskRuns ?? null },
|
||||
runtime: { argo, health: runtime, provenance },
|
||||
evidenceGaps: [...new Set(evidenceGaps)],
|
||||
};
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import type { GiteaConfig, GiteaMirrorRepository } from "./platform-infra-gitea-config";
|
||||
import { renderMirrorBootstrap } from "./platform-infra-gitea-render";
|
||||
import { resolvePacBootstrapMirrorRepository } from "./platform-infra-pipelines-as-code-bootstrap";
|
||||
import { pacBootstrapYamlMatchFailure, projectPacBootstrapResult, renderPacBootstrap, resolvePacBootstrapMirrorRepository } from "./platform-infra-pipelines-as-code-bootstrap";
|
||||
import { runPlatformInfraPipelinesAsCodeCommand } from "./platform-infra-pipelines-as-code";
|
||||
|
||||
const mirror: GiteaMirrorRepository = {
|
||||
@@ -78,3 +78,64 @@ test("platform bootstrap help advertises the composite entry before low-level ap
|
||||
expect(usage[0]).toContain("pipelines-as-code bootstrap");
|
||||
expect(usage).toContain("bun scripts/cli.ts platform-infra pipelines-as-code bootstrap --target <NODE> --consumer <consumer> --confirm");
|
||||
});
|
||||
|
||||
test("PaC bootstrap projects typed stages and compact JSON", () => {
|
||||
const result = {
|
||||
ok: true,
|
||||
action: "platform-infra-pipelines-as-code-bootstrap",
|
||||
mutation: false,
|
||||
mode: "dry-run",
|
||||
target: { id: "NC01" },
|
||||
consumer: { id: "widgets", node: "NC01", lane: "v01" },
|
||||
repository: { id: "widgets", namespace: "ci" },
|
||||
mirrorRepository: { key: mirror.key, upstreamRepository: mirror.upstream.repository, upstreamBranch: mirror.upstream.branch, owner: mirror.gitea.owner, repo: mirror.gitea.name },
|
||||
githubPreflight: { ok: true, mutation: false, repositories: [{ key: mirror.key, code: "github-upstream-available", ok: true }] },
|
||||
giteaBootstrap: { ok: true, mutation: false, mode: "dry-run", blockers: [] },
|
||||
pipelinesAsCodeApply: { ok: true, mutation: false, mode: "dry-run", remote: { ok: true, preflight: { repositoryExists: true } } },
|
||||
};
|
||||
const projection = projectPacBootstrapResult(result);
|
||||
expect(projection.ok).toBe(true);
|
||||
expect(JSON.stringify(projection)).not.toContain("pipelinesAsCodeApply");
|
||||
expect((projection.stages as Record<string, unknown>[]).map((item) => item.id)).toEqual(["gitea-init", "pac-controller", "tekton-consumer", "gitops-argo"]);
|
||||
const rendered = renderPacBootstrap(result).renderedText;
|
||||
expect(rendered).toContain("PRECONDITIONS");
|
||||
expect(rendered).toContain("yaml-repository-exact-match");
|
||||
expect(rendered).toContain("confirm:");
|
||||
});
|
||||
|
||||
test("PaC bootstrap returns fix-yaml next for zero YAML matches", () => {
|
||||
const result = pacBootstrapYamlMatchFailure("NC01", "widgets", new Error("cannot resolve repository: no exact match"));
|
||||
expect(projectPacBootstrapResult(result)).toBe(result);
|
||||
expect((result.blockers as Record<string, unknown>[])[0]?.code).toBe("yaml-repository-no-match");
|
||||
expect(result.next).toEqual({
|
||||
kind: "fix-yaml",
|
||||
command: "检查 config/platform-infra/gitea.yaml 与 config/platform-infra/pipelines-as-code.yaml 的 target、owner/name 和 read URL 精确匹配",
|
||||
});
|
||||
});
|
||||
|
||||
test("PaC bootstrap returns fix-yaml next for multiple YAML matches", () => {
|
||||
const result = pacBootstrapYamlMatchFailure("NC01", "widgets", new Error("cannot resolve repository: 2 exact matches"));
|
||||
expect(result.mutation).toBe(false);
|
||||
expect((result.blockers as Record<string, unknown>[])[0]?.code).toBe("yaml-repository-multiple-matches");
|
||||
expect((result.next as Record<string, unknown>).kind).toBe("fix-yaml");
|
||||
});
|
||||
|
||||
test("PaC bootstrap distinguishes GitHub access failure from Gitea and PaC stages", () => {
|
||||
const projection = projectPacBootstrapResult({
|
||||
ok: false,
|
||||
action: "platform-infra-pipelines-as-code-bootstrap",
|
||||
mutation: false,
|
||||
mode: "dry-run",
|
||||
target: { id: "NC01" },
|
||||
consumer: { id: "widgets", node: "NC01", lane: "v01" },
|
||||
repository: { id: "widgets", namespace: "ci" },
|
||||
mirrorRepository: { key: mirror.key, upstreamRepository: mirror.upstream.repository, upstreamBranch: mirror.upstream.branch, owner: mirror.gitea.owner, repo: mirror.gitea.name },
|
||||
githubPreflight: { ok: false, mutation: false, repositories: [{ key: mirror.key, code: "github-repository-not-found-or-no-access", errorTail: "repository unavailable" }] },
|
||||
giteaBootstrap: { ok: false, mutation: false, mode: "skipped", githubPreflight: { ok: false, repositories: [{ code: "github-repository-not-found-or-no-access", errorTail: "repository unavailable" }] } },
|
||||
pipelinesAsCodeApply: { ok: false, mutation: false, mode: "skipped" },
|
||||
});
|
||||
expect((projection.blockers as Record<string, unknown>[])).toEqual([{ code: "github-repository-not-found-or-no-access", stage: "github-upstream", reason: "repository unavailable" }]);
|
||||
expect((projection.preconditions as Record<string, unknown>[])[2]).toMatchObject({ ok: null, code: "gitea-bootstrap-skipped" });
|
||||
expect((projection.stages as Record<string, unknown>[]).map((item) => item.state)).toEqual(["skipped", "skipped", "skipped", "skipped"]);
|
||||
expect((projection.next as Record<string, unknown>).kind).toBe("read-only-status");
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { GiteaConfig, GiteaMirrorRepository } from "./platform-infra-gitea-config";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
|
||||
export interface PacBootstrapRepositoryIdentity {
|
||||
readonly id: string;
|
||||
@@ -28,6 +29,181 @@ export function resolvePacBootstrapMirrorRepository(
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
export function pacBootstrapYamlMatchFailure(
|
||||
targetId: string,
|
||||
consumerId: string,
|
||||
error: unknown,
|
||||
): Record<string, unknown> {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
const matchCount = reason.includes("no exact match") ? 0 : Number.parseInt(reason.match(/(\d+) exact matches/u)?.[1] ?? "-1", 10);
|
||||
return {
|
||||
ok: false,
|
||||
action: "platform-infra-pipelines-as-code-bootstrap",
|
||||
mutation: false,
|
||||
mode: "precondition-failed",
|
||||
target: { id: targetId },
|
||||
consumer: { id: consumerId },
|
||||
preconditions: [
|
||||
{ id: "yaml-exact-match", ok: false, code: matchCount === 0 ? "yaml-repository-no-match" : "yaml-repository-multiple-matches", matchCount, detail: reason },
|
||||
],
|
||||
stages: [],
|
||||
blockers: [{ code: matchCount === 0 ? "yaml-repository-no-match" : "yaml-repository-multiple-matches", stage: "yaml-exact-match", reason }],
|
||||
next: {
|
||||
kind: "fix-yaml",
|
||||
command: "检查 config/platform-infra/gitea.yaml 与 config/platform-infra/pipelines-as-code.yaml 的 target、owner/name 和 read URL 精确匹配",
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPacBootstrapResult(result: Record<string, unknown>): Record<string, unknown> {
|
||||
if (Array.isArray(result.preconditions) && Array.isArray(result.stages) && Array.isArray(result.blockers)) return result;
|
||||
const target = record(result.target);
|
||||
const consumer = record(result.consumer);
|
||||
const repository = record(result.repository);
|
||||
const mirror = record(result.mirrorRepository);
|
||||
const githubPreflight = record(result.githubPreflight);
|
||||
const githubRepository = records(githubPreflight.repositories)[0] ?? {};
|
||||
const gitea = record(result.giteaBootstrap);
|
||||
const apply = record(result.pipelinesAsCodeApply);
|
||||
const remote = record(apply.remote);
|
||||
const blockers = bootstrapBlockers(githubPreflight, gitea, apply, remote);
|
||||
const dryRun = result.mode === "dry-run";
|
||||
const repositoryMissing = remote.error === "gitea-repository-missing";
|
||||
const githubReady = githubPreflight.ok === true;
|
||||
const giteaReady = gitea.ok === true;
|
||||
const giteaSkipped = gitea.mode === "skipped";
|
||||
const applySkipped = apply.mode === "skipped";
|
||||
const applyReady = apply.ok === true || (dryRun && repositoryMissing);
|
||||
const next = bootstrapNext(result, blockers);
|
||||
return {
|
||||
ok: blockers.length === 0 && giteaReady && applyReady,
|
||||
action: result.action,
|
||||
mutation: result.mutation === true,
|
||||
mode: result.mode,
|
||||
target: { id: target.id },
|
||||
consumer: { id: consumer.id, node: consumer.node, lane: consumer.lane },
|
||||
sourceAuthority: {
|
||||
upstreamRepository: mirror.upstreamRepository,
|
||||
upstreamBranch: mirror.upstreamBranch,
|
||||
github: { ok: githubReady, code: stringValue(githubRepository.code, githubReady ? "github-upstream-available" : "github-upstream-preflight-failed") },
|
||||
giteaRepository: `${stringValue(mirror.owner)}/${stringValue(mirror.repo)}`,
|
||||
giteaKey: mirror.key,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
preconditions: [
|
||||
{ id: "yaml-exact-match", ok: true, code: "yaml-repository-exact-match", matchCount: 1 },
|
||||
{ id: "github-upstream", ok: githubReady, code: stringValue(githubRepository.code, githubReady ? "github-upstream-available" : "github-upstream-preflight-failed") },
|
||||
{ id: "gitea-repository", ok: giteaSkipped ? null : giteaReady, code: giteaSkipped ? "gitea-bootstrap-skipped" : giteaReady ? (dryRun ? "gitea-bootstrap-planned" : "gitea-bootstrap-ready") : "gitea-bootstrap-failed" },
|
||||
],
|
||||
stages: [
|
||||
{ id: "gitea-init", ok: giteaSkipped ? null : giteaReady, state: giteaSkipped ? "skipped" : giteaReady ? (dryRun ? "planned" : "ready") : "failed", mutation: gitea.mutation === true },
|
||||
{ id: "pac-controller", ok: applySkipped ? null : applyReady, state: applySkipped ? "skipped" : repositoryMissing ? "waiting-for-gitea-confirm" : applyReady ? (dryRun ? "planned" : "ready") : "failed", mutation: apply.mutation === true },
|
||||
{ id: "tekton-consumer", ok: applySkipped ? null : applyReady, state: applySkipped ? "skipped" : repositoryMissing ? "waiting-for-gitea-confirm" : applyReady ? (dryRun ? "planned" : "ready") : "failed", mutation: apply.mutation === true },
|
||||
{ id: "gitops-argo", ok: applySkipped ? null : applyReady, state: applySkipped ? "skipped" : repositoryMissing ? "waiting-for-gitea-confirm" : applyReady ? (dryRun ? "planned" : "ready") : "failed", mutation: apply.mutation === true },
|
||||
],
|
||||
repository: { id: repository.id, namespace: repository.namespace },
|
||||
blockers,
|
||||
next,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function renderPacBootstrap(result: Record<string, unknown>): RenderedCliResult {
|
||||
const projection = projectPacBootstrapResult(result);
|
||||
const target = record(projection.target);
|
||||
const consumer = record(projection.consumer);
|
||||
const source = record(projection.sourceAuthority);
|
||||
const preconditions = records(projection.preconditions);
|
||||
const stages = records(projection.stages);
|
||||
const blockers = records(projection.blockers);
|
||||
const next = record(projection.next);
|
||||
const lines = [
|
||||
"PLATFORM-INFRA PAC / TEKTON / GITOPS BOOTSTRAP",
|
||||
...table(["TARGET", "CONSUMER", "MODE", "MUTATION", "OK"], [[stringValue(target.id), stringValue(consumer.id), stringValue(projection.mode), boolText(projection.mutation), boolText(projection.ok)]]),
|
||||
"",
|
||||
"SOURCE AUTHORITY",
|
||||
...table(["UPSTREAM", "BRANCH", "GITEA_KEY", "GITEA_REPOSITORY"], [[stringValue(source.upstreamRepository), stringValue(source.upstreamBranch), stringValue(source.giteaKey), stringValue(source.giteaRepository)]]),
|
||||
"",
|
||||
"PRECONDITIONS",
|
||||
...table(["PRECONDITION", "OK", "CODE"], preconditions.map((item) => [stringValue(item.id), boolText(item.ok), stringValue(item.code)])),
|
||||
"",
|
||||
"STAGES",
|
||||
...table(["STAGE", "OK", "STATE", "MUTATION"], stages.map((item) => [stringValue(item.id), boolText(item.ok), stringValue(item.state), boolText(item.mutation)])),
|
||||
...(blockers.length === 0 ? [] : ["", "BLOCKERS", ...blockers.map((item) => ` ${stringValue(item.code)}: ${compactLine(stringValue(item.reason))}`)]),
|
||||
"",
|
||||
"NEXT",
|
||||
` ${stringValue(next.kind)}: ${stringValue(next.command)}`,
|
||||
];
|
||||
return { ok: projection.ok === true, command: "platform-infra pipelines-as-code bootstrap", renderedText: lines.join("\n"), contentType: "text/plain", projection };
|
||||
}
|
||||
|
||||
function bootstrapBlockers(githubPreflight: Record<string, unknown>, gitea: Record<string, unknown>, apply: Record<string, unknown>, remote: Record<string, unknown>): Record<string, unknown>[] {
|
||||
const blockers: Record<string, unknown>[] = [];
|
||||
const githubRepository = records(githubPreflight.repositories)[0] ?? {};
|
||||
if (githubPreflight.ok === false) {
|
||||
blockers.push({ code: stringValue(githubRepository.code, stringValue(githubPreflight.code, "github-upstream-preflight-failed")), stage: "github-upstream", reason: stringValue(githubRepository.errorTail, errorText(githubPreflight)) });
|
||||
return blockers;
|
||||
}
|
||||
for (const code of Array.isArray(gitea.blockers) ? gitea.blockers.map(String) : []) {
|
||||
blockers.push({ code: code.includes("credential") ? "source-credential-unavailable" : "github-upstream-unavailable", stage: "github-upstream", reason: code });
|
||||
}
|
||||
if (gitea.ok !== true && blockers.length === 0) blockers.push({ code: "gitea-bootstrap-failed", stage: "gitea-init", reason: errorText(gitea) });
|
||||
if (apply.ok !== true && remote.error !== "gitea-repository-missing") {
|
||||
blockers.push({ code: remote.error === "gitea-credential-unavailable" ? "gitea-credential-unavailable" : "pac-apply-failed", stage: "pac-controller", reason: errorText(remote, apply) });
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
function bootstrapNext(result: Record<string, unknown>, blockers: Record<string, unknown>[]): Record<string, unknown> {
|
||||
const target = record(result.target);
|
||||
const consumer = record(result.consumer);
|
||||
const source = record(result.mirrorRepository);
|
||||
if (blockers.length > 0) {
|
||||
const yamlFailure = blockers.some((item) => String(item.code).startsWith("yaml-"));
|
||||
return yamlFailure
|
||||
? { kind: "fix-yaml", command: "检查 config/platform-infra/gitea.yaml 与 config/platform-infra/pipelines-as-code.yaml 的 target、owner/name 和 read URL 精确匹配" }
|
||||
: { kind: "read-only-status", command: `bun scripts/cli.ts platform-infra pipelines-as-code status --target ${target.id} --consumer ${consumer.id}` };
|
||||
}
|
||||
if (result.mode === "dry-run") return { kind: "confirm", command: `bun scripts/cli.ts platform-infra pipelines-as-code bootstrap --target ${target.id} --consumer ${consumer.id} --confirm` };
|
||||
return { kind: "source-pr-merge", command: `合并 ${source.upstreamRepository}@${source.upstreamBranch} 的 GitHub PR，由自动链继续交付` };
|
||||
}
|
||||
|
||||
function errorText(...values: Record<string, unknown>[]): string {
|
||||
for (const value of values) {
|
||||
for (const candidate of [value.error, value.stderrTail, record(value.remote).error, record(value.remote).stderrTail]) {
|
||||
if (typeof candidate === "string" && candidate.length > 0) return candidate;
|
||||
}
|
||||
}
|
||||
return "unknown-bootstrap-failure";
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record) : [];
|
||||
}
|
||||
|
||||
function stringValue(value: unknown, fallback = "-"): string {
|
||||
return typeof value === "string" && value.length > 0 ? value : value === undefined || value === null ? fallback : String(value);
|
||||
}
|
||||
|
||||
function boolText(value: unknown): string {
|
||||
return value === true ? "true" : value === false ? "false" : "-";
|
||||
}
|
||||
|
||||
function compactLine(value: string): string {
|
||||
return value.replace(/\s+/gu, " ").trim().slice(0, 240) || "-";
|
||||
}
|
||||
|
||||
function table(headers: string[], rows: string[][]): string[] {
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length)));
|
||||
const format = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index])).join(" ").trimEnd();
|
||||
return [format(headers), format(widths.map((width) => "-".repeat(width))), ...rows.map(format)];
|
||||
}
|
||||
|
||||
function repositoryUrlIdentity(value: string): string {
|
||||
const parsed = new URL(value);
|
||||
const path = parsed.pathname.replace(/\/+$/u, "");
|
||||
|
||||
@@ -28,10 +28,16 @@ import {
|
||||
} from "./platform-infra-pipelines-as-code-source-artifact";
|
||||
import { pacAdmissionDesiredIdentity } from "./platform-infra-pac-provenance";
|
||||
import { pacConsumerRbacDesiredIdentity, renderPacConsumerRbacDesiredFragment } from "./platform-infra-pac-consumer-rbac";
|
||||
import { runPlatformInfraGiteaCommand } from "./platform-infra-gitea";
|
||||
import { runGiteaMirrorBootstrapPreflight, runPlatformInfraGiteaCommand } from "./platform-infra-gitea";
|
||||
import { readGiteaConfig } from "./platform-infra-gitea-config";
|
||||
import { resolvePacBootstrapMirrorRepository } from "./platform-infra-pipelines-as-code-bootstrap";
|
||||
import {
|
||||
pacBootstrapYamlMatchFailure,
|
||||
projectPacBootstrapResult,
|
||||
renderPacBootstrap,
|
||||
resolvePacBootstrapMirrorRepository,
|
||||
} from "./platform-infra-pipelines-as-code-bootstrap";
|
||||
import { featureConfigSchemaHistoryFields, renderFeatureConfigSchemaLine } from "./platform-infra-pac-feature-config-projection";
|
||||
import { observePacDeliveryTiming } from "./platform-infra-pac-delivery-timing";
|
||||
|
||||
const configFile = rootPath("config", "platform-infra", "pipelines-as-code.yaml");
|
||||
const configLabel = "config/platform-infra/pipelines-as-code.yaml";
|
||||
@@ -256,7 +262,7 @@ export async function runPlatformInfraPipelinesAsCodeCommand(config: UniDeskConf
|
||||
if (action === "bootstrap") {
|
||||
const options = parseApplyOptions(args.slice(1));
|
||||
const result = await bootstrap(config, options);
|
||||
return options.json || options.full || options.raw ? result : renderBootstrap(result);
|
||||
return options.full || options.raw ? result : options.json ? projectPacBootstrapResult(result) : renderPacBootstrap(result);
|
||||
}
|
||||
if (action === "status") {
|
||||
const options = parseCommonOptions(args.slice(1));
|
||||
@@ -273,6 +279,11 @@ export async function runPlatformInfraPipelinesAsCodeCommand(config: UniDeskConf
|
||||
const result = await history(config, options);
|
||||
return options.raw ? result : options.full ? compactHistoryJson(result, true) : options.json ? compactHistoryJson(result) : renderHistory(result);
|
||||
}
|
||||
if (action === "delivery-timing" || action === "timing") {
|
||||
const options = parseCommonOptions(args.slice(1));
|
||||
const result = await deliveryTiming(config, options);
|
||||
return options.full || options.raw || options.json ? result : renderDeliveryTiming(result);
|
||||
}
|
||||
if (action === "debug-step") {
|
||||
const options = parseHistoryOptions(args.slice(1));
|
||||
const result = await debugStep(config, options);
|
||||
@@ -395,12 +406,13 @@ function help(scope: string | null): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
return {
|
||||
command: "platform-infra pipelines-as-code plan|status|history|debug-step|source-artifact",
|
||||
command: "platform-infra pipelines-as-code plan|status|history|delivery-timing|debug-step|source-artifact",
|
||||
configTruth: configLabel,
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code plan --target JD01",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 [--json|--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 [--consumer hwlab-jd01-v03] [--limit 10]",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code delivery-timing --target NC01 --consumer selfmedia-nc01 [--json]",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code history --target JD01 --id <pipelinerun>",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code debug-step --target JD01 [--consumer <consumer>] [--id <pipelinerun>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra pipelines-as-code source-artifact check --target NC01 --consumer agentrun-nc01-v02 --source-worktree /abs/worktree",
|
||||
@@ -975,11 +987,20 @@ async function bootstrap(config: UniDeskConfig, options: ApplyOptions): Promise<
|
||||
throw new Error(`Pipelines-as-Code consumer ${consumer.id} belongs to ${consumer.node}, not target ${target.id}`);
|
||||
}
|
||||
const repository = resolveRepository(pac, consumer.repositoryRef);
|
||||
const mirror = resolvePacBootstrapMirrorRepository(readGiteaConfig(), target.id, repository);
|
||||
let mirror: ReturnType<typeof resolvePacBootstrapMirrorRepository>;
|
||||
try {
|
||||
mirror = resolvePacBootstrapMirrorRepository(readGiteaConfig(), target.id, repository);
|
||||
} catch (error) {
|
||||
return pacBootstrapYamlMatchFailure(target.id, consumer.id, error);
|
||||
}
|
||||
const githubPreflight = record(await runGiteaMirrorBootstrapPreflight(config, target.id, mirror.key));
|
||||
if (githubPreflight.ok !== true) {
|
||||
return bootstrapResult(target, consumer, repository, mirror, options, githubPreflight, { ok: false, mutation: false, mode: "skipped", githubPreflight, valuesPrinted: false }, { ok: false, mutation: false, mode: "skipped", valuesPrinted: false });
|
||||
}
|
||||
const giteaArgs = ["mirror", "bootstrap", "--target", target.id, "--repo", mirror.key, ...(options.confirm ? ["--confirm"] : []), "--full"];
|
||||
const giteaBootstrap = record(await runPlatformInfraGiteaCommand(config, giteaArgs));
|
||||
if (options.confirm && giteaBootstrap.ok !== true) {
|
||||
return bootstrapResult(target, consumer, repository, mirror, options, giteaBootstrap, {
|
||||
return bootstrapResult(target, consumer, repository, mirror, options, githubPreflight, giteaBootstrap, {
|
||||
ok: false,
|
||||
mutation: false,
|
||||
mode: "skipped",
|
||||
@@ -988,7 +1009,7 @@ async function bootstrap(config: UniDeskConfig, options: ApplyOptions): Promise<
|
||||
});
|
||||
}
|
||||
const pacApply = await apply(config, options);
|
||||
return bootstrapResult(target, consumer, repository, mirror, options, giteaBootstrap, pacApply);
|
||||
return bootstrapResult(target, consumer, repository, mirror, options, githubPreflight, giteaBootstrap, pacApply);
|
||||
}
|
||||
|
||||
function bootstrapResult(
|
||||
@@ -997,6 +1018,7 @@ function bootstrapResult(
|
||||
repository: PacRepository,
|
||||
mirror: ReturnType<typeof resolvePacBootstrapMirrorRepository>,
|
||||
options: ApplyOptions,
|
||||
githubPreflight: Record<string, unknown>,
|
||||
giteaBootstrap: Record<string, unknown>,
|
||||
pacApply: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
@@ -1021,6 +1043,7 @@ function bootstrapResult(
|
||||
repo: mirror.gitea.name,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
githubPreflight,
|
||||
steps: [
|
||||
{ id: "gitea-repository", ok: giteaBootstrap.ok === true, mutation: giteaBootstrap.mutation === true, mode: giteaBootstrap.mode, valuesPrinted: false },
|
||||
{ id: "pac-tekton-gitops", ok: pacReady, mutation: pacApply.mutation === true, mode: pacApply.mode, repositoryMissing, valuesPrinted: false },
|
||||
@@ -2208,43 +2231,6 @@ function renderApply(result: Record<string, unknown>): RenderedCliResult {
|
||||
return rendered(result, "platform-infra pipelines-as-code apply", lines);
|
||||
}
|
||||
|
||||
function renderBootstrap(result: Record<string, unknown>): RenderedCliResult {
|
||||
const target = record(result.target);
|
||||
const consumer = record(result.consumer);
|
||||
const mirror = record(result.mirrorRepository);
|
||||
const steps = arrayRecords(result.steps);
|
||||
const pacApply = record(result.pipelinesAsCodeApply);
|
||||
const pacRemote = record(pacApply.remote);
|
||||
const giteaBootstrap = record(result.giteaBootstrap);
|
||||
const next = record(result.next);
|
||||
const errorValue = [pacRemote.error, giteaBootstrap.error, pacApply.error]
|
||||
.find((value): value is string => typeof value === "string" && value.length > 0) ?? "";
|
||||
const error = errorValue.length === 0 ? "" : compactLine(errorValue);
|
||||
const lines = [
|
||||
"PLATFORM-INFRA PAC / TEKTON / GITOPS BOOTSTRAP",
|
||||
...table(["TARGET", "CONSUMER", "MODE", "MUTATION", "OK"], [[stringValue(target.id), stringValue(consumer.id), stringValue(result.mode), boolText(result.mutation), boolText(result.ok)]]),
|
||||
"",
|
||||
"SOURCE AUTHORITY",
|
||||
...table(["GITEA_KEY", "GITEA_REPOSITORY", "UPSTREAM", "BRANCH"], [[stringValue(mirror.key), `${stringValue(mirror.owner)}/${stringValue(mirror.repo)}`, stringValue(mirror.upstreamRepository), stringValue(mirror.upstreamBranch)]]),
|
||||
"",
|
||||
"STEPS",
|
||||
...table(["STEP", "OK", "MODE", "MUTATION", "DETAIL"], steps.map((step) => [
|
||||
stringValue(step.id),
|
||||
boolText(step.ok),
|
||||
stringValue(step.mode),
|
||||
boolText(step.mutation),
|
||||
step.repositoryMissing === true ? "repository-will-be-created-on-confirm" : "ready",
|
||||
])),
|
||||
...(error.length === 0 ? [] : ["", `ERROR ${error}`]),
|
||||
"",
|
||||
"NEXT",
|
||||
...(result.mode === "dry-run"
|
||||
? [` confirm: ${stringValue(next.confirm)}`, ` boundary: ${stringValue(next.boundary)}`]
|
||||
: [` delivery: ${stringValue(next.deliveryTrigger)}`, ` investigate: ${stringValue(next.investigate)}`]),
|
||||
];
|
||||
return rendered(result, "platform-infra pipelines-as-code bootstrap", lines);
|
||||
}
|
||||
|
||||
export function renderStatus(result: Record<string, unknown>): RenderedCliResult {
|
||||
const summary = record(result.summary);
|
||||
const consumer = record(result.consumer);
|
||||
@@ -2423,6 +2409,49 @@ function renderCloseout(result: Record<string, unknown>): RenderedCliResult {
|
||||
return rendered(result, "platform-infra pipelines-as-code closeout", lines);
|
||||
}
|
||||
|
||||
async function deliveryTiming(config: UniDeskConfig, options: CommonOptions): Promise<Record<string, unknown>> {
|
||||
const pac = readPacConfig();
|
||||
const target = resolveTarget(pac, options.targetId);
|
||||
const consumer = resolveConsumer(pac, options.consumerId);
|
||||
if (consumer.node.toLowerCase() !== target.id.toLowerCase()) throw new Error(`Pipelines-as-Code consumer ${consumer.id} belongs to ${consumer.node}, not target ${target.id}`);
|
||||
const authority = resolveCicdDeliveryAuthority({ consumerId: consumer.id, node: consumer.node, lane: consumer.lane });
|
||||
if (authority.kind !== "pac-pr-merge") throw new Error(`delivery authority is ${authority.kind}: ${authority.reason}`);
|
||||
const historyOptions: HistoryOptions = { ...options, limit: 1, detailId: null };
|
||||
return await observePacDeliveryTiming({
|
||||
target: targetSummary(target),
|
||||
consumer: {
|
||||
id: consumer.id,
|
||||
node: consumer.node,
|
||||
lane: consumer.lane,
|
||||
repository: authority.consumer.sourceRepository,
|
||||
branch: authority.consumer.sourceBranch,
|
||||
pipeline: consumer.pipeline,
|
||||
},
|
||||
}, {
|
||||
history: async () => await history(config, historyOptions),
|
||||
status: async () => await status(config, options),
|
||||
});
|
||||
}
|
||||
|
||||
function renderDeliveryTiming(result: Record<string, unknown>): RenderedCliResult {
|
||||
const timing = record(result.timing);
|
||||
const source = record(result.source);
|
||||
const pr = record(source.pullRequest);
|
||||
const runtime = record(result.runtime);
|
||||
const argo = record(runtime.argo);
|
||||
const health = record(runtime.health);
|
||||
const lines = [
|
||||
"PLATFORM-INFRA DELIVERY TIMING",
|
||||
`STATE: ${stringValue(result.state)} MUTATION: ${boolText(result.mutation)}`,
|
||||
`SOURCE: ${stringValue(record(result.consumer).repository)}@${stringValue(source.commit)} PR#${stringValue(pr.number)}`,
|
||||
`MERGED_AT: ${stringValue(timing.mergedAt)} PIPELINE_START: ${stringValue(timing.pipelineStart)} PIPELINE_DONE: ${stringValue(timing.pipelineCompletion)}`,
|
||||
`TRIGGER_WAIT_S: ${stringValue(timing.triggerWaitSeconds)} PIPELINE_S: ${stringValue(timing.pipelineDurationSeconds)} END_TO_END_S: ${stringValue(timing.endToEndSeconds)}`,
|
||||
`ARGO: ${stringValue(argo.sync)}/${stringValue(argo.health)} RUNTIME_READY: ${stringValue(health.readyReplicas)}/${stringValue(health.replicas)}`,
|
||||
`EVIDENCE_GAPS: ${(Array.isArray(result.evidenceGaps) ? result.evidenceGaps.map(String) : []).join(", ") || "-"}`,
|
||||
];
|
||||
return rendered(result, "platform-infra pipelines-as-code delivery-timing", lines);
|
||||
}
|
||||
|
||||
export function renderHistory(result: Record<string, unknown>): RenderedCliResult {
|
||||
const rows = arrayRecords(result.rows);
|
||||
const config = record(result.config);
|
||||
|
||||
@@ -0,0 +1,620 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import { CliInputError, type RenderedCliResult } from "../output";
|
||||
|
||||
import { boolField, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote";
|
||||
import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId } from "./runtime-target";
|
||||
|
||||
type FaultLevel = "P0" | "P1" | "P2";
|
||||
|
||||
interface FaultOptions {
|
||||
level: FaultLevel | null;
|
||||
group: string | null;
|
||||
account: string | null;
|
||||
model: string | null;
|
||||
stream: "sync" | "stream" | null;
|
||||
endpoint: string | null;
|
||||
requestId: string | null;
|
||||
pageToken: string | null;
|
||||
targetId: string;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
export async function codexPoolFaults(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
if (args.includes("--help")) return renderFaultHelp();
|
||||
const options = parseFaultOptions(args);
|
||||
const target = codexPoolRuntimeTarget(options.targetId);
|
||||
const scope = faultScope(options);
|
||||
const offset = decodePageToken(options.pageToken, scope);
|
||||
const payload = {
|
||||
filters: {
|
||||
level: options.level,
|
||||
group: options.group,
|
||||
account: options.account,
|
||||
model: options.model,
|
||||
stream: options.stream,
|
||||
endpoint: options.endpoint,
|
||||
requestId: options.requestId,
|
||||
},
|
||||
offset,
|
||||
pageSize: PAGE_SIZE,
|
||||
};
|
||||
const result = await runRemoteCodexPoolScript(config, "faults", faultScript(payload, target), target);
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && boolField(parsed, "ok", false);
|
||||
const data = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
||||
const pagination = data && typeof data.pagination === "object" && data.pagination !== null
|
||||
? data.pagination as Record<string, unknown>
|
||||
: null;
|
||||
if (pagination?.hasMore === true) pagination.nextPageToken = encodePageToken(offset + PAGE_SIZE, scope);
|
||||
if (pagination) {
|
||||
pagination.pageSize = PAGE_SIZE;
|
||||
pagination.pageToken = options.pageToken;
|
||||
}
|
||||
const response = {
|
||||
ok,
|
||||
action: "platform-infra-sub2api-codex-pool-faults",
|
||||
target: {
|
||||
id: target.id,
|
||||
route: target.route,
|
||||
runtimeMode: target.runtimeMode,
|
||||
endpoint: target.serviceDns,
|
||||
},
|
||||
source: {
|
||||
kind: "sub2api-native-admin-ops",
|
||||
versionBoundary: "Sub2API native admin Ops facts with explicit, version-neutral UniDesk CLI projection",
|
||||
mutation: false,
|
||||
},
|
||||
filters: payload.filters,
|
||||
faults: data,
|
||||
remote: compactCapture(result, { full: result.exitCode !== 0 || data === null }),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
if (options.json) return response;
|
||||
return renderFaults(response);
|
||||
}
|
||||
|
||||
function renderFaultHelp(): RenderedCliResult {
|
||||
return {
|
||||
ok: true,
|
||||
command: "platform-infra sub2api codex-pool faults --help",
|
||||
renderedText: [
|
||||
"SUB2API CODEX-POOL FAULTS",
|
||||
"Usage:",
|
||||
" bun scripts/cli.ts platform-infra sub2api codex-pool faults [options]",
|
||||
"Options:",
|
||||
" --level P0|P1|P2",
|
||||
" --group <name-or-id>",
|
||||
" --account <name-or-id>",
|
||||
" --model <model>",
|
||||
" --stream sync|stream",
|
||||
" --endpoint <path>",
|
||||
" --request-id <stable-id>",
|
||||
" --page-token <token>",
|
||||
" --target <id>",
|
||||
" --json",
|
||||
"Notes:",
|
||||
` Fixed page size: ${PAGE_SIZE}; --limit is intentionally unsupported.`,
|
||||
" Default output is a bounded Kubernetes-style table; JSON requires --json.",
|
||||
].join("\n"),
|
||||
contentType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
function parseFaultOptions(args: string[]): FaultOptions {
|
||||
let level: FaultLevel | null = null;
|
||||
let group: string | null = null;
|
||||
let account: string | null = null;
|
||||
let model: string | null = null;
|
||||
let stream: "sync" | "stream" | null = null;
|
||||
let endpoint: string | null = null;
|
||||
let requestId: string | null = null;
|
||||
let pageToken: string | null = null;
|
||||
let targetId = defaultCodexPoolRuntimeTargetId();
|
||||
let json = false;
|
||||
const readValue = (index: number, name: string): [string, number] => {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
return [validateSelector(value, name), index + 1];
|
||||
};
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index]!;
|
||||
if (arg === "--json") json = true;
|
||||
else if (arg === "--level") {
|
||||
const [value, next] = readValue(index, "--level");
|
||||
level = parseLevel(value);
|
||||
index = next;
|
||||
} else if (arg.startsWith("--level=")) level = parseLevel(arg.slice(8));
|
||||
else if (arg === "--group") [group, index] = readValue(index, "--group");
|
||||
else if (arg.startsWith("--group=")) group = validateSelector(arg.slice(8), "--group");
|
||||
else if (arg === "--account") [account, index] = readValue(index, "--account");
|
||||
else if (arg.startsWith("--account=")) account = validateSelector(arg.slice(10), "--account");
|
||||
else if (arg === "--model") [model, index] = readValue(index, "--model");
|
||||
else if (arg.startsWith("--model=")) model = validateSelector(arg.slice(8), "--model");
|
||||
else if (arg === "--stream") {
|
||||
const [value, next] = readValue(index, "--stream");
|
||||
stream = parseStream(value);
|
||||
index = next;
|
||||
} else if (arg.startsWith("--stream=")) stream = parseStream(arg.slice(9));
|
||||
else if (arg === "--endpoint") [endpoint, index] = readValue(index, "--endpoint");
|
||||
else if (arg.startsWith("--endpoint=")) endpoint = validateSelector(arg.slice(11), "--endpoint");
|
||||
else if (arg === "--request-id") [requestId, index] = readValue(index, "--request-id");
|
||||
else if (arg.startsWith("--request-id=")) requestId = validateSelector(arg.slice(13), "--request-id");
|
||||
else if (arg === "--page-token") [pageToken, index] = readValue(index, "--page-token");
|
||||
else if (arg.startsWith("--page-token=")) pageToken = validateSelector(arg.slice(13), "--page-token");
|
||||
else if (arg === "--target") [targetId, index] = readValue(index, "--target");
|
||||
else if (arg.startsWith("--target=")) targetId = validateSelector(arg.slice(9), "--target");
|
||||
else if (arg === "--limit" || arg.startsWith("--limit=")) throw new Error("faults uses a fixed page size; use --page-token for progressive disclosure");
|
||||
else throw new Error(`unsupported faults option: ${arg}`);
|
||||
}
|
||||
if (level !== null && group === null) {
|
||||
throw new CliInputError("--level requires --group; use the default command for the all-groups overview", {
|
||||
code: "missing-required-option",
|
||||
argument: "--group",
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool faults --target PK01 --level P0 --group <name-or-id>",
|
||||
],
|
||||
hint: "Run without --level for the all-groups overview, then drill down with --level and --group.",
|
||||
});
|
||||
}
|
||||
return { level, group, account, model, stream, endpoint, requestId, pageToken, targetId, json };
|
||||
}
|
||||
|
||||
function parseLevel(value: string): FaultLevel {
|
||||
const normalized = value.toUpperCase();
|
||||
if (normalized !== "P0" && normalized !== "P1" && normalized !== "P2") throw new Error("--level must be P0, P1, or P2");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseStream(value: string): "sync" | "stream" {
|
||||
if (value !== "sync" && value !== "stream") throw new Error("--stream must be sync or stream");
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateSelector(value: string, name: string): string {
|
||||
const normalized = value.trim();
|
||||
if (!normalized || normalized.length > 512 || /[\r\n\0]/u.test(normalized)) throw new Error(`${name} has an unsupported format`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function faultScope(options: FaultOptions): string {
|
||||
return createHash("sha256").update(JSON.stringify({
|
||||
level: options.level,
|
||||
group: options.group,
|
||||
account: options.account,
|
||||
model: options.model,
|
||||
stream: options.stream,
|
||||
endpoint: options.endpoint,
|
||||
requestId: options.requestId,
|
||||
targetId: options.targetId,
|
||||
})).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function encodePageToken(offset: number, scope: string): string {
|
||||
return Buffer.from(JSON.stringify({ v: 1, offset, scope }), "utf8").toString("base64url");
|
||||
}
|
||||
|
||||
function decodePageToken(token: string | null, scope: string): number {
|
||||
if (token === null) return 0;
|
||||
try {
|
||||
const parsed = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as Record<string, unknown>;
|
||||
if (parsed.v !== 1 || parsed.scope !== scope || !Number.isInteger(parsed.offset) || Number(parsed.offset) < 0) throw new Error("invalid");
|
||||
return Number(parsed.offset);
|
||||
} catch {
|
||||
throw new Error("--page-token is invalid or belongs to different filters");
|
||||
}
|
||||
}
|
||||
|
||||
function renderFaults(response: Record<string, unknown>): RenderedCliResult {
|
||||
const faults = response.faults && typeof response.faults === "object" ? response.faults as Record<string, unknown> : {};
|
||||
const lines = ["PLATFORM-INFRA SUB2API FAULTS"];
|
||||
const window = record(faults.window);
|
||||
lines.push(`WINDOW ${text(window.timeRange)} SOURCE native-admin-ops PROJECTION unidesk-cli`);
|
||||
lines.push("");
|
||||
const summary = arrayOfRecords(faults.summary);
|
||||
if (summary.length > 0) {
|
||||
lines.push("GROUPS");
|
||||
lines.push(table(["GROUP", "P0", "CUSTOMER_ERRS", "GROUP_ERR%", "P1", "TTFT_P99", "P2", "UPSTREAM_ERRS", "GROUP_UP_ERR%", "ABSORB%"], summary.map((row) => [
|
||||
`${text(row.groupName)} (${text(row.groupId)})`,
|
||||
text(row.p0), text(row.customerErrorCount), percent(row.customerErrorRatePercent),
|
||||
text(row.p1), milliseconds(row.ttftP99Ms),
|
||||
text(row.p2), text(row.upstreamErrorCount), percent(row.upstreamErrorRatePercent), percent(row.absorbedPercent),
|
||||
])));
|
||||
}
|
||||
const details = arrayOfRecords(faults.details);
|
||||
if (details.length > 0) {
|
||||
lines.push("", `DETAILS ${text(faults.level ?? "SUMMARY")}`);
|
||||
const columns = Array.isArray(faults.detailColumns) ? faults.detailColumns.map(String) : [];
|
||||
lines.push(table(columns, details.map((row) => columns.map((column) => text(row[column])))));
|
||||
}
|
||||
const boundary = record(faults.boundary);
|
||||
lines.push("", "BOUNDARY");
|
||||
for (const value of Array.isArray(boundary.notes) ? boundary.notes : []) lines.push(`- ${String(value)}`);
|
||||
const pagination = record(faults.pagination);
|
||||
if (pagination.nextPageToken) lines.push("", `NEXT_PAGE_TOKEN ${text(pagination.nextPageToken)}`);
|
||||
if (faults.next) lines.push("", "NEXT", ...String(faults.next).split("\n").map((line) => ` ${line}`));
|
||||
return {
|
||||
ok: response.ok === true,
|
||||
command: "platform-infra sub2api codex-pool faults",
|
||||
renderedText: lines.join("\n"),
|
||||
contentType: "text/plain",
|
||||
projection: response,
|
||||
};
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayOfRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
||||
}
|
||||
|
||||
function text(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
if (typeof value === "boolean") return value ? "yes" : "no";
|
||||
return String(value).replace(/[\r\n\t]+/gu, " ");
|
||||
}
|
||||
|
||||
function percent(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : text(value);
|
||||
}
|
||||
|
||||
function milliseconds(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${Math.round(value)}ms` : text(value);
|
||||
}
|
||||
|
||||
function table(headers: string[], rows: string[][]): string {
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length)));
|
||||
return [headers, ...rows].map((row) => row.map((cell, index) => cell.padEnd(widths[index]!)).join(" ").trimEnd()).join("\n");
|
||||
}
|
||||
|
||||
function pyJson(value: unknown): string {
|
||||
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
||||
}
|
||||
|
||||
function faultScript(payload: unknown, target: ReturnType<typeof codexPoolRuntimeTarget>): string {
|
||||
return `
|
||||
set -u
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import urlencode
|
||||
|
||||
RUNTIME_MODE = ${pyJson(target.runtimeMode)}
|
||||
NAMESPACE = ${pyJson(target.namespace)}
|
||||
HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
|
||||
APP_SECRET_NAME = ${pyJson(target.appSecretName)}
|
||||
PAYLOAD = ${pyJson(payload)}
|
||||
APP_POD = None
|
||||
|
||||
def run(cmd, input_bytes=None):
|
||||
return subprocess.run(cmd, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def docker(args):
|
||||
proc = run(["docker", *args])
|
||||
if proc.returncode == 0:
|
||||
return proc
|
||||
sudo_proc = run(["sudo", "-n", "docker", *args])
|
||||
return sudo_proc if sudo_proc.returncode == 0 else proc
|
||||
|
||||
def kubectl(args, input_bytes=None):
|
||||
return run(["kubectl", *args], input_bytes)
|
||||
|
||||
def kube_json(args, label):
|
||||
proc = kubectl([*args, "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(label + " failed")
|
||||
return json.loads(proc.stdout.decode("utf-8"))
|
||||
|
||||
if RUNTIME_MODE != "host-docker":
|
||||
pods = kube_json(["-n", NAMESPACE, "get", "pods", "-l", "app=sub2api"], "list pods").get("items") or []
|
||||
ready = [item for item in pods if item.get("status", {}).get("phase") == "Running"]
|
||||
if not ready:
|
||||
raise RuntimeError("sub2api app pod not found")
|
||||
APP_POD = ready[0]["metadata"]["name"]
|
||||
|
||||
def config_value(name, key, default=None):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = docker(["inspect", "sub2api-app", "--format", "{{json .Config.Env}}"])
|
||||
if proc.returncode != 0:
|
||||
return default
|
||||
for item in json.loads(proc.stdout.decode("utf-8")):
|
||||
if item.startswith(key + "="):
|
||||
return item.split("=", 1)[1]
|
||||
return default
|
||||
data = kube_json(["-n", NAMESPACE, "get", "configmap", name], "configmap/" + name).get("data") or {}
|
||||
return data.get(key, default)
|
||||
|
||||
def secret_value(name, key):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return config_value("", key)
|
||||
data = kube_json(["-n", NAMESPACE, "get", "secret", name], "secret/" + name).get("data") or {}
|
||||
return base64.b64decode(data[key]).decode("utf-8") if key in data else None
|
||||
|
||||
def curl_api(method, path, bearer=None, payload=None):
|
||||
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
script = r'''set -eu
|
||||
method="$1"; url="$2"; token="\${3:-}"; tmp="$(mktemp)"; trap 'rm -f "$tmp"' EXIT; cat > "$tmp"
|
||||
args=""; [ -n "$token" ] && args="Authorization: Bearer $token"
|
||||
if [ -n "$args" ] && [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
||||
elif [ -n "$args" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "$args" "$url"
|
||||
elif [ -s "$tmp" ]; then curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
||||
else curl -sS -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"; fi'''
|
||||
base = f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}" if RUNTIME_MODE == "host-docker" else "http://127.0.0.1:8080"
|
||||
cmd = ["sh", "-c", script, "sh", method, base + path, bearer or ""]
|
||||
proc = run(cmd, body) if RUNTIME_MODE == "host-docker" else run(["kubectl", "-n", NAMESPACE, "exec", "-i", APP_POD, "--", *cmd], body)
|
||||
output = proc.stdout.decode("utf-8", errors="replace")
|
||||
marker = "\\n__HTTP_CODE__:"
|
||||
pos = output.rfind(marker)
|
||||
status = int(output[pos + len(marker):].strip()[-3:]) if pos >= 0 else 0
|
||||
raw = output[:pos] if pos >= 0 else output
|
||||
try:
|
||||
parsed = json.loads(raw) if raw.strip() else None
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
return {"ok": proc.returncode == 0 and 200 <= status < 300, "status": status, "json": parsed, "body": raw[:300]}
|
||||
|
||||
def data_of(response, label):
|
||||
parsed = response.get("json")
|
||||
code = parsed.get("code") if isinstance(parsed, dict) else None
|
||||
if response.get("ok") is not True or (code is not None and code != 0):
|
||||
message = parsed.get("message") if isinstance(parsed, dict) else response.get("body")
|
||||
raise RuntimeError(f"{label} failed: http={response.get('status')} message={message}")
|
||||
return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
|
||||
|
||||
def login():
|
||||
email = config_value("sub2api-config", "ADMIN_EMAIL", "admin@example.com")
|
||||
password = secret_value(APP_SECRET_NAME, "ADMIN_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("ADMIN_PASSWORD missing")
|
||||
data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login")
|
||||
token = data.get("access_token") or data.get("token") if isinstance(data, dict) else None
|
||||
if not token:
|
||||
raise RuntimeError("admin login response has no token")
|
||||
return token
|
||||
|
||||
def get(token, path, params=None, label=None):
|
||||
suffix = "?" + urlencode({key: value for key, value in (params or {}).items() if value is not None}) if params else ""
|
||||
return data_of(curl_api("GET", path + suffix, bearer=token), label or path)
|
||||
|
||||
def items_of(data):
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
for key in ("items", "groups", "accounts"):
|
||||
if isinstance(data.get(key), list):
|
||||
return data[key]
|
||||
return []
|
||||
|
||||
def total_of(data):
|
||||
return int(data.get("total", len(items_of(data)))) if isinstance(data, dict) else len(items_of(data))
|
||||
|
||||
def all_items(token, path, params, label):
|
||||
page = 1
|
||||
page_size = 500
|
||||
rows = []
|
||||
while True:
|
||||
data = get(token, path, {**params, "page": page, "page_size": page_size}, label)
|
||||
batch = items_of(data)
|
||||
rows.extend(batch)
|
||||
total = total_of(data)
|
||||
if not batch or len(rows) >= total:
|
||||
return rows
|
||||
page += 1
|
||||
|
||||
def percent(value):
|
||||
if not isinstance(value, (int, float)):
|
||||
return None
|
||||
return round(value * 100 if value <= 1 else value, 4)
|
||||
|
||||
def selector_id(value):
|
||||
return int(value) if isinstance(value, str) and value.isdigit() else None
|
||||
|
||||
def scope_filters(group_id=None):
|
||||
result = {"time_range": "24h", "platform": "openai", "group_id": group_id}
|
||||
return result
|
||||
|
||||
def matches_detail_filters(item):
|
||||
filters = PAYLOAD["filters"]
|
||||
account = filters.get("account")
|
||||
if account:
|
||||
account_match = str(item.get("account_id") or "") == str(account) or str(item.get("account_name") or "") == str(account)
|
||||
if not account_match:
|
||||
return False
|
||||
if filters.get("model") and str(item.get("requested_model") or item.get("model") or "") != str(filters["model"]):
|
||||
return False
|
||||
if filters.get("stream"):
|
||||
expected_stream = filters["stream"] == "stream"
|
||||
if item.get("stream") is not expected_stream:
|
||||
return False
|
||||
if filters.get("endpoint") and str(item.get("inbound_endpoint") or item.get("request_path") or item.get("path") or "") != str(filters["endpoint"]):
|
||||
return False
|
||||
if filters.get("requestId"):
|
||||
request_id = filters["requestId"]
|
||||
if item.get("request_id") != request_id and item.get("client_request_id") != request_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
def has_detail_filters():
|
||||
filters = PAYLOAD["filters"]
|
||||
return any(filters.get(key) for key in ("account", "model", "stream", "endpoint", "requestId"))
|
||||
|
||||
DETAIL_ROWS = {}
|
||||
|
||||
def error_rows(token, path, group_id, apply_filters):
|
||||
cache_key = (path, group_id, apply_filters)
|
||||
if cache_key not in DETAIL_ROWS:
|
||||
rows = all_items(token, path, {**scope_filters(group_id), "view": "all", "include_detail": 1}, "fault detail scan")
|
||||
DETAIL_ROWS[cache_key] = [item for item in rows if matches_detail_filters(item)] if apply_filters else rows
|
||||
return DETAIL_ROWS[cache_key]
|
||||
|
||||
def selected_groups(groups):
|
||||
selector = PAYLOAD["filters"].get("group")
|
||||
if not selector:
|
||||
return groups
|
||||
selected = [item for item in groups if str(item.get("id")) == str(selector) or item.get("name") == selector]
|
||||
if len(selected) != 1:
|
||||
raise RuntimeError("group-not-found-or-ambiguous: " + str(selector))
|
||||
return selected
|
||||
|
||||
def threshold_state(value, threshold):
|
||||
if not isinstance(value, (int, float)) or not isinstance(threshold, (int, float)):
|
||||
return "unavailable"
|
||||
return "breach" if value >= threshold else "ok"
|
||||
|
||||
def availability_by_account(token, group_id):
|
||||
data = get(token, "/api/v1/admin/ops/account-availability", {"platform": "openai", "group_id": group_id}, "account availability")
|
||||
raw = data.get("account") if isinstance(data, dict) else None
|
||||
return raw if isinstance(raw, dict) else {}
|
||||
|
||||
def detail_row(item, group_metrics, availability, level, absorbed=None):
|
||||
account_id = item.get("account_id")
|
||||
state = availability.get(str(account_id), {}) if isinstance(availability, dict) else {}
|
||||
endpoint = item.get("inbound_endpoint") or item.get("request_path") or item.get("path")
|
||||
base = {
|
||||
"GROUP": f"{item.get('group_name') or group_metrics.get('groupName')} ({item.get('group_id') or group_metrics.get('groupId')})",
|
||||
"ACCOUNT": f"{item.get('account_name') or '-'} ({account_id or '-'})",
|
||||
"MODEL": item.get("requested_model") or item.get("model") or "-",
|
||||
"MODE": "stream" if item.get("stream") is True else "sync" if item.get("stream") is False else "-",
|
||||
"ENDPOINT": endpoint or "-",
|
||||
"STATUS": item.get("status_code") or "-",
|
||||
"REQUEST_ID": item.get("request_id") or item.get("client_request_id") or "-",
|
||||
}
|
||||
if level == "P0":
|
||||
base.update({"CUSTOMER_ERR%": group_metrics.get("customerErrorRatePercent"), "ABSORBED": False})
|
||||
else:
|
||||
unavailable_reason = str(state.get("unavailable_reason") or state.get("reason") or "").lower()
|
||||
if state.get("is_available") is False and ("temporary" in unavailable_reason or "cooldown" in unavailable_reason):
|
||||
temp_unsched = "active"
|
||||
elif state.get("is_available") is True:
|
||||
temp_unsched = "none"
|
||||
else:
|
||||
temp_unsched = "unavailable"
|
||||
base.update({"ROOT": item.get("error_source") or item.get("message") or "upstream", "TEMP_UNSCHED": temp_unsched, "ABSORBED": absorbed})
|
||||
return base
|
||||
|
||||
def execute():
|
||||
token = login()
|
||||
thresholds = get(token, "/api/v1/admin/ops/settings/metric-thresholds", label="metric thresholds")
|
||||
groups = selected_groups(items_of(get(token, "/api/v1/admin/groups/all", {"platform": "openai"}, "list groups")))
|
||||
summaries = []
|
||||
group_data = []
|
||||
for group in groups:
|
||||
group_id = group.get("id")
|
||||
native = get(token, "/api/v1/admin/ops/dashboard/overview", {"time_range": "24h", "platform": "openai", "group_id": group_id}, "dashboard overview")
|
||||
if has_detail_filters():
|
||||
customer_total = len(error_rows(token, "/api/v1/admin/ops/request-errors", group_id, True))
|
||||
upstream_total = len(error_rows(token, "/api/v1/admin/ops/upstream-errors", group_id, True))
|
||||
else:
|
||||
request_errors = get(token, "/api/v1/admin/ops/request-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "request errors")
|
||||
upstream_errors = get(token, "/api/v1/admin/ops/upstream-errors", {**scope_filters(group_id), "page": 1, "page_size": 1, "view": "all"}, "upstream errors")
|
||||
customer_total = total_of(request_errors)
|
||||
upstream_total = total_of(upstream_errors)
|
||||
absorbed = max(0, upstream_total - customer_total)
|
||||
absorbed_percent = round(absorbed * 100 / upstream_total, 2) if upstream_total else None
|
||||
customer_rate = percent(native.get("error_rate") if isinstance(native, dict) else None)
|
||||
upstream_rate = percent(native.get("upstream_error_rate") if isinstance(native, dict) else None)
|
||||
ttft = native.get("ttft") if isinstance(native, dict) and isinstance(native.get("ttft"), dict) else {}
|
||||
metrics = {
|
||||
"groupId": group_id,
|
||||
"groupName": group.get("name"),
|
||||
"customerErrorCount": customer_total,
|
||||
"customerErrorRatePercent": customer_rate,
|
||||
"upstreamErrorCount": upstream_total,
|
||||
"upstreamErrorRatePercent": upstream_rate,
|
||||
"absorbedCountProjection": absorbed,
|
||||
"absorbedPercent": absorbed_percent,
|
||||
"ttftP99Ms": ttft.get("p99_ms"),
|
||||
}
|
||||
metrics["p0"] = "active" if customer_total > 0 else "clear"
|
||||
metrics["p1"] = threshold_state(metrics["ttftP99Ms"], thresholds.get("ttft_p99_ms_max"))
|
||||
metrics["p2"] = threshold_state(upstream_rate, thresholds.get("upstream_error_rate_percent_max"))
|
||||
summaries.append(metrics)
|
||||
group_data.append((group, metrics))
|
||||
level = PAYLOAD["filters"].get("level")
|
||||
offset = int(PAYLOAD.get("offset") or 0)
|
||||
page = offset // int(PAYLOAD["pageSize"]) + 1
|
||||
details = []
|
||||
detail_columns = []
|
||||
total = 0
|
||||
if level in ("P0", "P2"):
|
||||
for group, metrics in group_data:
|
||||
group_id = group.get("id")
|
||||
endpoint = "/api/v1/admin/ops/request-errors" if level == "P0" else "/api/v1/admin/ops/upstream-errors"
|
||||
if has_detail_filters():
|
||||
rows = error_rows(token, endpoint, group_id, True)
|
||||
total += len(rows)
|
||||
rows = rows[offset:offset + PAYLOAD["pageSize"]]
|
||||
else:
|
||||
data = get(token, endpoint, {**scope_filters(group_id), "page": page, "page_size": PAYLOAD["pageSize"], "view": "all", "include_detail": 1}, level + " details")
|
||||
total += total_of(data)
|
||||
rows = items_of(data)
|
||||
availability = availability_by_account(token, group_id) if level == "P2" else {}
|
||||
customer_request_ids = set()
|
||||
if level == "P2":
|
||||
customer_request_ids = {
|
||||
item.get("request_id") or item.get("client_request_id")
|
||||
for item in error_rows(token, "/api/v1/admin/ops/request-errors", group_id, False)
|
||||
if item.get("request_id") or item.get("client_request_id")
|
||||
}
|
||||
for item in rows:
|
||||
absorbed = None
|
||||
if level == "P2" and item.get("request_id"):
|
||||
absorbed = item.get("request_id") not in customer_request_ids
|
||||
details.append(detail_row(item, metrics, availability, level, absorbed))
|
||||
detail_columns = ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "CUSTOMER_ERR%", "ABSORBED", "REQUEST_ID"] if level == "P0" else ["GROUP", "ACCOUNT", "MODEL", "MODE", "ENDPOINT", "STATUS", "ROOT", "TEMP_UNSCHED", "ABSORBED", "REQUEST_ID"]
|
||||
elif level == "P1":
|
||||
total = len(group_data) * 2
|
||||
endpoint_names = ["/v1/responses", "/v1/responses/compact"]
|
||||
for group, metrics in group_data:
|
||||
for endpoint in endpoint_names:
|
||||
if PAYLOAD["filters"].get("endpoint") and PAYLOAD["filters"].get("endpoint") != endpoint:
|
||||
continue
|
||||
details.append({
|
||||
"GROUP": f"{metrics.get('groupName')} ({metrics.get('groupId')})",
|
||||
"ENDPOINT": endpoint,
|
||||
"SAMPLES": "unavailable",
|
||||
"P50": "unavailable",
|
||||
"P95": "unavailable",
|
||||
"P99": "unavailable",
|
||||
"MAX": "unavailable",
|
||||
"OUTLIER_REQUEST_ID": "unavailable",
|
||||
"BOUNDARY": "native endpoint/request TTFT unavailable; total duration not substituted",
|
||||
})
|
||||
detail_columns = ["GROUP", "ENDPOINT", "SAMPLES", "P50", "P95", "P99", "MAX", "OUTLIER_REQUEST_ID", "BOUNDARY"]
|
||||
return {
|
||||
"ok": True,
|
||||
"level": level,
|
||||
"window": {"timeRange": "24h"},
|
||||
"thresholds": thresholds,
|
||||
"summary": summaries,
|
||||
"details": details[:PAYLOAD["pageSize"]],
|
||||
"detailColumns": detail_columns,
|
||||
"pagination": {"offset": offset, "total": total, "hasMore": offset + PAYLOAD["pageSize"] < total},
|
||||
"boundary": {"notes": [
|
||||
"P0/P2 facts come from native admin ops request-errors, upstream-errors, overview, and account-availability.",
|
||||
"P0/P1/P2 labels and failover/retry absorption are UniDesk CLI projections; Sub2API native severity is not rewritten.",
|
||||
"P1 endpoint-level samples and request-level TTFT are unavailable in the observed native Ops response; total duration is never used as TTFT.",
|
||||
"Absorption summary is a bounded count projection from native upstream/customer totals; P2 detail absorption is correlated by stable request ID.",
|
||||
]},
|
||||
"next": "Use --level P0|P1|P2 for details; reuse filters with --page-token for the next fixed page; use codex-pool trace --request-id <id> for trace disclosure.",
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
try:
|
||||
output = execute()
|
||||
except Exception as exc:
|
||||
output = {"ok": False, "error": str(exc), "valuesPrinted": False}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if output.get("ok") else 1)
|
||||
PY
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
import { readCodexPoolConfig } from "./config";
|
||||
import { capture, compactCapture, parseJsonOutput } from "./remote";
|
||||
import { renderedCliResult, renderTable, shorten, shortIso, textValue } from "./render";
|
||||
import { codexPoolRuntimeTarget } from "./runtime-target";
|
||||
import type { CodexPoolConfig, CodexPoolRuntimeTarget, FeedbackOptions } from "./types";
|
||||
|
||||
function pyJson(value: unknown): string {
|
||||
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
||||
}
|
||||
|
||||
export async function codexPoolFeedback(config: UniDeskConfig, options: FeedbackOptions): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const pool = readCodexPoolConfig();
|
||||
const target = codexPoolRuntimeTarget(options.targetId);
|
||||
const result = await capture(config, target.route, ["sh"], feedbackScript(pool, options, target));
|
||||
const report = parseJsonOutput(result.stdout);
|
||||
const ok = result.exitCode === 0 && report?.ok === true;
|
||||
const remote = compactCapture(result, { full: result.exitCode !== 0 || report === null });
|
||||
if (options.json) {
|
||||
return {
|
||||
ok,
|
||||
action: "platform-infra-sub2api-codex-pool-feedback",
|
||||
target: { id: target.id, route: target.route, runtimeMode: target.runtimeMode },
|
||||
report,
|
||||
remote,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
return renderedCliResult(ok, "platform-infra sub2api codex-pool feedback", renderFeedbackReport(report, options, remote));
|
||||
}
|
||||
|
||||
function feedbackScript(pool: CodexPoolConfig, options: FeedbackOptions, target: CodexPoolRuntimeTarget): string {
|
||||
return `
|
||||
set -u
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
RUNTIME_MODE = ${pyJson(target.runtimeMode)}
|
||||
NAMESPACE = ${pyJson(target.namespace)}
|
||||
APP_SECRET_NAME = ${pyJson(target.appSecretName)}
|
||||
ADMIN_EMAIL_DEFAULT = ${pyJson(pool.adminEmailDefault)}
|
||||
HOST_DOCKER_APP_PORT = ${pyJson(target.hostDockerAppPort)}
|
||||
HOST_DOCKER_ENV_PATH = ${pyJson(target.hostDockerEnvPath)}
|
||||
USER_SELECTOR = ${pyJson(options.user)}
|
||||
WINDOW = ${pyJson(options.window)}
|
||||
REQUEST_SELECTOR = ${pyJson(options.requestId)}
|
||||
PAGE_TOKEN = ${pyJson(options.pageToken)}
|
||||
APP_CONTAINER = "sub2api-app"
|
||||
PAGE_SIZE = 10
|
||||
|
||||
def run(command, input_bytes=None):
|
||||
return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def text(value, limit=1000):
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", errors="replace")
|
||||
return str(value or "")[-limit:]
|
||||
|
||||
def docker(args):
|
||||
proc = run(["docker", *args])
|
||||
if proc.returncode == 0:
|
||||
return proc
|
||||
fallback = run(["sudo", "-n", "docker", *args])
|
||||
return fallback if fallback.returncode == 0 else proc
|
||||
|
||||
def read_host_env():
|
||||
if RUNTIME_MODE != "host-docker":
|
||||
return {}
|
||||
try:
|
||||
with open(HOST_DOCKER_ENV_PATH, "r", encoding="utf-8") as handle:
|
||||
lines = handle.read().splitlines()
|
||||
except PermissionError:
|
||||
proc = run(["sudo", "-n", "cat", HOST_DOCKER_ENV_PATH])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("read host-docker env failed: " + text(proc.stderr))
|
||||
lines = proc.stdout.decode("utf-8", errors="replace").splitlines()
|
||||
values = {}
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
continue
|
||||
key, value = stripped.split("=", 1)
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
||||
value = value[1:-1]
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
def runtime_secret(key):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "get", "secret", APP_SECRET_NAME, "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("read app secret failed: " + text(proc.stderr))
|
||||
raw = (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
|
||||
return base64.b64decode(raw).decode("utf-8") if raw else None
|
||||
|
||||
def runtime_config(key):
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "get", "configmap", "sub2api-config", "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return (json.loads(proc.stdout.decode("utf-8")).get("data") or {}).get(key)
|
||||
|
||||
def parse_curl(proc):
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
marker = "\\n__HTTP_CODE__:"
|
||||
position = stdout.rfind(marker)
|
||||
if position < 0:
|
||||
return {"ok": False, "httpStatus": 0, "json": None, "message": text(proc.stderr)}
|
||||
body = stdout[:position]
|
||||
try:
|
||||
status = int(stdout[position + len(marker):].strip()[-3:])
|
||||
except Exception:
|
||||
status = 0
|
||||
try:
|
||||
parsed = json.loads(body) if body.strip() else None
|
||||
except Exception:
|
||||
parsed = None
|
||||
message = parsed.get("message") if isinstance(parsed, dict) else text(body)
|
||||
return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "message": message}
|
||||
|
||||
def curl_api(method, path, bearer=None, payload=None):
|
||||
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
script = r'''set -eu
|
||||
method="$1"
|
||||
url="$2"
|
||||
token="\${3:-}"
|
||||
tmp="$(mktemp)"
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
cat > "$tmp"
|
||||
args=""
|
||||
if [ -n "$token" ]; then args="Authorization: Bearer $token"; fi
|
||||
if [ "$method" = GET ] && [ ! -s "$tmp" ]; then
|
||||
if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -H "$args" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' "$url"; fi
|
||||
else
|
||||
if [ -n "$args" ]; then curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "$args" --data-binary @"$tmp" "$url"; else curl -sS -w '\\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"; fi
|
||||
fi'''
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = run(["sh", "-c", script, "sh", method, f"http://127.0.0.1:{HOST_DOCKER_APP_PORT}{path}", bearer or ""], body)
|
||||
else:
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "exec", "deploy/sub2api", "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""], body)
|
||||
return parse_curl(proc)
|
||||
|
||||
def api_data(method, path, token=None, payload=None, label=None):
|
||||
response = curl_api(method, path, token, payload)
|
||||
parsed = response.get("json")
|
||||
code = parsed.get("code") if isinstance(parsed, dict) else None
|
||||
if not response.get("ok") or (code is not None and code != 0):
|
||||
raise RuntimeError(f"{label or path} failed: http={response.get('httpStatus')} message={response.get('message')}")
|
||||
return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
|
||||
|
||||
def find_token(value):
|
||||
if isinstance(value, dict):
|
||||
for key in ("access_token", "token"):
|
||||
if isinstance(value.get(key), str) and value.get(key):
|
||||
return value[key]
|
||||
for item in value.values():
|
||||
token = find_token(item)
|
||||
if token:
|
||||
return token
|
||||
return None
|
||||
|
||||
def login():
|
||||
email = runtime_config("ADMIN_EMAIL") or ADMIN_EMAIL_DEFAULT
|
||||
password = runtime_secret("ADMIN_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("ADMIN_PASSWORD missing")
|
||||
token = find_token(api_data("POST", "/api/v1/auth/login", payload={"email": email, "password": password}, label="admin login"))
|
||||
if not token:
|
||||
raise RuntimeError("admin login returned no token")
|
||||
return email, token
|
||||
|
||||
def items(data):
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data.get("items"), list):
|
||||
return data["items"]
|
||||
for key in ("users", "api_keys", "keys", "groups", "accounts", "proxies", "logs"):
|
||||
if isinstance(data.get(key), list):
|
||||
return data[key]
|
||||
return []
|
||||
|
||||
def page_all(token, path, query=None, page_size=100):
|
||||
query = dict(query or {})
|
||||
collected = []
|
||||
page = 1
|
||||
while True:
|
||||
params = {**query, "page": page, "page_size": page_size}
|
||||
separator = "&" if "?" in path else "?"
|
||||
data = api_data("GET", path + separator + urlencode(params), token, label=path)
|
||||
batch = items(data)
|
||||
collected.extend(batch)
|
||||
total = data.get("total") if isinstance(data, dict) else None
|
||||
if not batch or (isinstance(total, int) and len(collected) >= total) or len(batch) < page_size:
|
||||
break
|
||||
page += 1
|
||||
return collected
|
||||
|
||||
def exact_user(token):
|
||||
if USER_SELECTOR.isdigit() and int(USER_SELECTOR) > 0:
|
||||
user = api_data("GET", f"/api/v1/admin/users/{int(USER_SELECTOR)}", token, label="get user")
|
||||
return user if isinstance(user, dict) else None
|
||||
candidates = page_all(token, "/api/v1/admin/users", {"search": USER_SELECTOR}, 100)
|
||||
exact = [user for user in candidates if isinstance(user, dict) and str(user.get("email") or "").lower() == USER_SELECTOR.lower()]
|
||||
if len(exact) > 1:
|
||||
raise RuntimeError("email matched multiple users")
|
||||
return exact[0] if exact else None
|
||||
|
||||
def safe_call(errors, name, callback, fallback):
|
||||
try:
|
||||
return callback()
|
||||
except Exception as exc:
|
||||
errors.append({"source": name, "error": redact_text(str(exc))})
|
||||
return fallback
|
||||
|
||||
def redact_text(value):
|
||||
value = str(value or "")
|
||||
value = re.sub(r"(?i)(bearer\\s+)[A-Za-z0-9._~+\\-/=]+", r"\\1<redacted>", value)
|
||||
value = re.sub(r"(?i)((?:api[_-]?key|token|password)\\s*[=:]\\s*)[^\\s,;]+", r"\\1<redacted>", value)
|
||||
value = re.sub(r"\\bsk-[A-Za-z0-9_-]{8,}\\b", "sk-<redacted>", value)
|
||||
return value[:500]
|
||||
|
||||
def iso_epoch(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
match = re.match(r"^(\\d{4}-\\d\\d-\\d\\d[T ]\\d\\d:\\d\\d:\\d\\d)(?:\\.(\\d+))?(Z|[+-]\\d\\d:?\\d\\d)$", value)
|
||||
if not match:
|
||||
return None
|
||||
base = datetime.strptime(match.group(1).replace(" ", "T"), "%Y-%m-%dT%H:%M:%S")
|
||||
fraction = int((match.group(2) or "0")[:6].ljust(6, "0"))
|
||||
zone = match.group(3)
|
||||
if zone == "Z":
|
||||
offset = timezone.utc
|
||||
else:
|
||||
sign = 1 if zone[0] == "+" else -1
|
||||
digits = zone[1:].replace(":", "")
|
||||
offset = timezone(sign * timedelta(hours=int(digits[:2]), minutes=int(digits[2:])))
|
||||
return base.replace(microsecond=fraction, tzinfo=offset).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def encode_page_token(row):
|
||||
payload = json.dumps({"v": 1, "at": row.get("at"), "requestId": row.get("requestId")}, separators=(",", ":")).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=")
|
||||
|
||||
def decode_page_token(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
padded = value + "=" * (-len(value) % 4)
|
||||
parsed = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8"))
|
||||
except Exception:
|
||||
raise RuntimeError("invalid page token")
|
||||
if not isinstance(parsed, dict) or parsed.get("v") != 1 or not parsed.get("at") or not parsed.get("requestId"):
|
||||
raise RuntimeError("invalid page token")
|
||||
return parsed
|
||||
|
||||
def dict_by_id(rows):
|
||||
return {str(row.get("id")): row for row in rows if isinstance(row, dict) and row.get("id") is not None}
|
||||
|
||||
def keyed_map(value, key):
|
||||
source = value.get(key) if isinstance(value, dict) else None
|
||||
return source if isinstance(source, dict) else {}
|
||||
|
||||
def runtime_health():
|
||||
if RUNTIME_MODE == "host-docker":
|
||||
proc = docker(["inspect", APP_CONTAINER, "sub2api-redis"])
|
||||
if proc.returncode != 0:
|
||||
return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
|
||||
rows = []
|
||||
for item in json.loads(proc.stdout.decode("utf-8")):
|
||||
state = item.get("State") or {}
|
||||
health = state.get("Health") or {}
|
||||
rows.append({"name": str(item.get("Name") or "").lstrip("/"), "running": state.get("Running"), "status": health.get("Status") or state.get("Status")})
|
||||
return {"ok": all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
|
||||
proc = run(["kubectl", "-n", NAMESPACE, "get", "pods", "-l", "app.kubernetes.io/name=sub2api", "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
return {"ok": False, "mode": RUNTIME_MODE, "error": text(proc.stderr)}
|
||||
rows = []
|
||||
for item in json.loads(proc.stdout.decode("utf-8")).get("items") or []:
|
||||
status = item.get("status") or {}
|
||||
containers = status.get("containerStatuses") or []
|
||||
rows.append({"name": (item.get("metadata") or {}).get("name"), "running": status.get("phase") == "Running", "status": "ready" if containers and all(entry.get("ready") is True for entry in containers) else status.get("phase")})
|
||||
return {"ok": bool(rows) and all(row.get("running") is True for row in rows), "mode": RUNTIME_MODE, "components": rows}
|
||||
|
||||
def compact_log(row):
|
||||
extra = row.get("extra") if isinstance(row.get("extra"), dict) else {}
|
||||
allow = {}
|
||||
for key in ("phase", "error_owner", "status_code", "duration_ms", "latency_ms", "time_to_first_token_ms", "upstream_status", "path", "stream"):
|
||||
if key in extra:
|
||||
allow[key] = extra.get(key)
|
||||
return {
|
||||
"id": row.get("id"),
|
||||
"at": row.get("created_at"),
|
||||
"level": row.get("level"),
|
||||
"component": row.get("component"),
|
||||
"message": redact_text(row.get("message")),
|
||||
"requestId": row.get("request_id"),
|
||||
"clientRequestId": row.get("client_request_id"),
|
||||
"accountId": row.get("account_id"),
|
||||
"extra": allow,
|
||||
}
|
||||
|
||||
def related_logs(request, logs):
|
||||
ids = {str(request.get("request_id") or "")}
|
||||
matched = []
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for row in logs:
|
||||
request_id = str(row.get("request_id") or "")
|
||||
client_id = str(row.get("client_request_id") or "")
|
||||
if row in matched or (request_id not in ids and client_id not in ids):
|
||||
continue
|
||||
matched.append(row)
|
||||
before = len(ids)
|
||||
ids.update(value for value in (request_id, client_id) if value)
|
||||
changed = changed or len(ids) != before
|
||||
return matched
|
||||
|
||||
def classify(request, logs, account, proxy, queued):
|
||||
evidence = " ".join([str(request.get("phase") or ""), str(request.get("message") or ""), *[str(log.get("component") or "") + " " + str(log.get("message") or "") for log in logs]]).lower()
|
||||
if any(marker in evidence for marker in ("client disconnect", "broken pipe", "context canceled", "client canceled")):
|
||||
return "client-connection"
|
||||
if any(marker in evidence for marker in ("proxy", "dial tcp", "dns", "connection refused", "network")) or (proxy and proxy.get("status") not in (None, "active")):
|
||||
return "proxy-egress"
|
||||
if any(marker in evidence for marker in ("upstream", "provider", "first token", "stream")) or str(request.get("phase") or "") == "upstream":
|
||||
return "upstream-first-byte-stream"
|
||||
if queued > 0:
|
||||
return "sub2api-queue"
|
||||
if request.get("kind") == "error" or (isinstance(request.get("status_code"), int) and request.get("status_code") >= 400):
|
||||
return "sub2api-or-upstream-unknown"
|
||||
return "completed"
|
||||
|
||||
def main():
|
||||
errors = []
|
||||
admin_email, token = login()
|
||||
user = exact_user(token)
|
||||
if not isinstance(user, dict):
|
||||
return {"ok": False, "mode": "feedback", "error": "user-not-found", "selector": USER_SELECTOR, "valuesPrinted": False}
|
||||
user_id = user.get("id")
|
||||
groups = safe_call(errors, "groups", lambda: items(api_data("GET", "/api/v1/admin/groups/all", token, label="groups")), [])
|
||||
keys = safe_call(errors, "api-keys", lambda: page_all(token, f"/api/v1/admin/users/{user_id}/api-keys", {}, 100), [])
|
||||
requests = safe_call(errors, "ops-requests", lambda: page_all(token, "/api/v1/admin/ops/requests", {"user_id": user_id, "time_range": WINDOW, "kind": "all", "sort": "created_at_desc"}, 100), [])
|
||||
logs = safe_call(errors, "ops-system-logs", lambda: page_all(token, "/api/v1/admin/ops/system-logs", {"user_id": user_id, "time_range": WINDOW}, 200), [])
|
||||
concurrency = safe_call(errors, "ops-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/concurrency", token, label="concurrency"), {})
|
||||
user_concurrency = safe_call(errors, "ops-user-concurrency", lambda: api_data("GET", "/api/v1/admin/ops/user-concurrency", token, label="user concurrency"), {})
|
||||
availability = safe_call(errors, "ops-account-availability", lambda: api_data("GET", "/api/v1/admin/ops/account-availability", token, label="account availability"), {})
|
||||
accounts = safe_call(errors, "accounts", lambda: page_all(token, "/api/v1/admin/accounts", {}, 100), [])
|
||||
proxies = safe_call(errors, "proxies", lambda: page_all(token, "/api/v1/admin/proxies", {}, 100), [])
|
||||
infrastructure = safe_call(errors, "infrastructure", runtime_health, {"ok": False, "mode": RUNTIME_MODE})
|
||||
|
||||
groups_by_id = dict_by_id(groups)
|
||||
keys_by_id = dict_by_id(keys)
|
||||
accounts_by_id = dict_by_id(accounts)
|
||||
proxies_by_id = dict_by_id(proxies)
|
||||
user_current_map = keyed_map(user_concurrency, "user")
|
||||
account_concurrency_map = keyed_map(concurrency, "account")
|
||||
account_availability_map = keyed_map(availability, "account")
|
||||
current_user = user_current_map.get(str(user_id)) or {}
|
||||
user_current = int(current_user.get("current_in_use") or user.get("current_concurrency") or 0)
|
||||
user_waiting = int(current_user.get("waiting_in_queue") or 0)
|
||||
|
||||
compact_keys = []
|
||||
for key in keys:
|
||||
group = groups_by_id.get(str(key.get("group_id"))) or {}
|
||||
compact_keys.append({
|
||||
"id": key.get("id"), "name": key.get("name"), "status": key.get("status"),
|
||||
"groupId": key.get("group_id"), "groupName": group.get("name"),
|
||||
"currentConcurrency": key.get("current_concurrency"), "keyPrinted": False,
|
||||
})
|
||||
|
||||
timeline = []
|
||||
ordered = sorted([row for row in requests if isinstance(row, dict)], key=lambda row: iso_epoch(row.get("created_at")) or 0)
|
||||
previous_epoch = None
|
||||
for row in ordered:
|
||||
matched_logs = related_logs(row, logs)
|
||||
client_id = next((log.get("client_request_id") for log in matched_logs if log.get("client_request_id")), None)
|
||||
account = accounts_by_id.get(str(row.get("account_id"))) or {}
|
||||
proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
|
||||
account_load = account_concurrency_map.get(str(row.get("account_id"))) or {}
|
||||
queued = int(account_load.get("waiting_in_queue") or 0)
|
||||
epoch = iso_epoch(row.get("created_at"))
|
||||
gap = round(epoch - previous_epoch, 1) if epoch is not None and previous_epoch is not None else None
|
||||
previous_epoch = epoch if epoch is not None else previous_epoch
|
||||
timeline.append({
|
||||
"at": row.get("created_at"), "gapSeconds": gap, "kind": row.get("kind"),
|
||||
"requestId": row.get("request_id"), "clientRequestId": client_id,
|
||||
"model": row.get("model"), "statusCode": row.get("status_code"), "durationMs": row.get("duration_ms"),
|
||||
"apiKeyId": row.get("api_key_id"), "apiKeyName": (keys_by_id.get(str(row.get("api_key_id"))) or {}).get("name"),
|
||||
"accountId": row.get("account_id"), "accountName": account.get("name"),
|
||||
"proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"),
|
||||
"attribution": classify(row, matched_logs, account, proxy, queued),
|
||||
"evidence": {"systemLogCount": len(matched_logs), "accountStatus": account.get("status"), "accountSchedulable": account.get("schedulable"), "proxyStatus": proxy.get("status")},
|
||||
})
|
||||
|
||||
all_timeline = timeline
|
||||
max_gap = max([row.get("gapSeconds") for row in all_timeline if isinstance(row.get("gapSeconds"), (int, float))], default=None)
|
||||
next_page_token = None
|
||||
more_available = False
|
||||
if REQUEST_SELECTOR:
|
||||
selected_ids = {REQUEST_SELECTOR}
|
||||
for row in all_timeline:
|
||||
if row.get("requestId") == REQUEST_SELECTOR or row.get("clientRequestId") == REQUEST_SELECTOR:
|
||||
selected_ids.update(value for value in (row.get("requestId"), row.get("clientRequestId")) if isinstance(value, str) and value)
|
||||
timeline = [row for row in all_timeline if any(isinstance(value, str) and value in selected_ids for value in (row.get("requestId"), row.get("clientRequestId")))]
|
||||
detail_logs = [compact_log(row) for row in logs if any(isinstance(value, str) and value in selected_ids for value in (row.get("request_id"), row.get("client_request_id")))]
|
||||
else:
|
||||
detail_logs = []
|
||||
ordered_desc = list(reversed(all_timeline))
|
||||
cursor = decode_page_token(PAGE_TOKEN)
|
||||
start = 0
|
||||
if cursor:
|
||||
start = next((index + 1 for index, row in enumerate(ordered_desc) if row.get("at") == cursor.get("at") and row.get("requestId") == cursor.get("requestId")), -1)
|
||||
if start < 0:
|
||||
raise RuntimeError("page token cursor is no longer present in the selected window")
|
||||
timeline = ordered_desc[start:start + PAGE_SIZE]
|
||||
more_available = start + len(timeline) < len(ordered_desc)
|
||||
if more_available and timeline:
|
||||
next_page_token = encode_page_token(timeline[-1])
|
||||
if not requests and user_waiting > 0:
|
||||
conclusion = "sub2api-queue"
|
||||
elif not requests and user_current > 0:
|
||||
conclusion = "in-flight"
|
||||
elif not requests:
|
||||
conclusion = "client-before-submit-or-no-inbound-evidence"
|
||||
elif timeline:
|
||||
conclusion = timeline[0].get("attribution")
|
||||
else:
|
||||
conclusion = "request-selector-not-found"
|
||||
|
||||
related_account_ids = {str(row.get("accountId")) for row in timeline if row.get("accountId") is not None}
|
||||
account_evidence = []
|
||||
for account_id in sorted(related_account_ids):
|
||||
account = accounts_by_id.get(account_id) or {}
|
||||
proxy = proxies_by_id.get(str(account.get("proxy_id"))) or {}
|
||||
load = account_concurrency_map.get(account_id) or {}
|
||||
available = account_availability_map.get(account_id) or {}
|
||||
account_evidence.append({
|
||||
"id": account.get("id"), "name": account.get("name"), "status": account.get("status"), "schedulable": account.get("schedulable"),
|
||||
"currentInUse": load.get("current_in_use"), "waitingInQueue": load.get("waiting_in_queue"), "maxCapacity": load.get("max_capacity"),
|
||||
"available": available.get("is_available"), "rateLimited": available.get("is_rate_limited"), "overloaded": available.get("is_overloaded"),
|
||||
"proxyId": account.get("proxy_id"), "proxyName": proxy.get("name"), "proxyStatus": proxy.get("status"),
|
||||
})
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "feedback",
|
||||
"selector": {"user": USER_SELECTOR, "requestId": REQUEST_SELECTOR, "pageToken": PAGE_TOKEN, "window": WINDOW},
|
||||
"user": {"id": user_id, "email": user.get("email"), "username": user.get("username"), "status": user.get("status"), "maxConcurrency": user.get("concurrency"), "currentInUse": user_current, "waitingInQueue": user_waiting},
|
||||
"apiKeys": compact_keys,
|
||||
"lifecycle": {"notInbound": not requests and user_current == 0 and user_waiting == 0, "queued": user_waiting, "inFlight": user_current, "completed": len(requests)},
|
||||
"timeline": timeline,
|
||||
"timelineSummary": {"returned": len(timeline), "total": len(all_timeline), "moreAvailable": more_available, "nextPageToken": next_page_token, "pageSize": PAGE_SIZE, "systemLogCount": len(logs), "maxGapSeconds": max_gap},
|
||||
"requestDetail": {"logs": detail_logs, "logCount": len(detail_logs)} if REQUEST_SELECTOR else None,
|
||||
"accounts": account_evidence,
|
||||
"infrastructure": infrastructure,
|
||||
"conclusion": {"attribution": conclusion, "boundary": "只读证据归因；无入站记录不等于客户端未调用，运行中请求仅由实时并发佐证。"},
|
||||
"toolCallReduction": {"manualInvestigationCalls": 9, "feedbackCliCalls": 1, "reducedCalls": 8, "reductionPercent": 88.9, "basis": "issue-2000: user, requests, system logs, two request-id traces, account, proxy, process/port, journal"},
|
||||
"sources": ["admin users", "admin user api-keys", "ops requests", "ops system-logs", "ops concurrency", "ops user-concurrency", "ops account-availability", "admin accounts", "admin proxies", "runtime health"],
|
||||
"errors": errors,
|
||||
"admin": {"email": admin_email, "tokenPrinted": False},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
|
||||
try:
|
||||
print(json.dumps(main(), ensure_ascii=False))
|
||||
except Exception as exc:
|
||||
print(json.dumps({"ok": False, "mode": "feedback", "error": redact_text(str(exc)), "valuesPrinted": False}, ensure_ascii=False))
|
||||
raise
|
||||
PY
|
||||
`;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
||||
}
|
||||
|
||||
function renderFeedbackReport(report: Record<string, unknown> | null, options: FeedbackOptions, remote: Record<string, unknown>): string {
|
||||
if (report === null) {
|
||||
return [
|
||||
`SUB2API FEEDBACK user=${options.user} unavailable`,
|
||||
`remote_exit=${remote.exitCode ?? "?"} stdout_bytes=${remote.stdoutBytes ?? "?"} stderr_bytes=${remote.stderrBytes ?? "?"}`,
|
||||
textValue(remote.stderrTail ?? remote.stdoutTail),
|
||||
].join("\n");
|
||||
}
|
||||
if (report.ok !== true) {
|
||||
return [
|
||||
`SUB2API FEEDBACK user=${options.user} ok=false`,
|
||||
`error=${textValue(report.error)}`,
|
||||
"JSON: add --json for the structured failure envelope.",
|
||||
].join("\n");
|
||||
}
|
||||
const user = record(report.user);
|
||||
const lifecycle = record(report.lifecycle);
|
||||
const summary = record(report.timelineSummary);
|
||||
const conclusion = record(report.conclusion);
|
||||
const reduction = record(report.toolCallReduction);
|
||||
const timeline = records(report.timeline);
|
||||
const accounts = records(report.accounts);
|
||||
const apiKeys = records(report.apiKeys);
|
||||
const detail = record(report.requestDetail);
|
||||
const detailLogs = records(detail.logs);
|
||||
const infrastructure = record(report.infrastructure);
|
||||
const components = records(infrastructure.components);
|
||||
const errors = records(report.errors);
|
||||
const lines: string[] = [];
|
||||
lines.push(`SUB2API FEEDBACK user=${textValue(user.email)}#${textValue(user.id)} window=${options.window} ok=true`);
|
||||
lines.push(`CONCLUSION attribution=${textValue(conclusion.attribution)} boundary=${textValue(conclusion.boundary)}`);
|
||||
lines.push(`LIFECYCLE not_inbound=${textValue(lifecycle.notInbound)} queued=${textValue(lifecycle.queued)} in_flight=${textValue(lifecycle.inFlight)} completed=${textValue(lifecycle.completed)} max_gap_s=${textValue(summary.maxGapSeconds)}`);
|
||||
lines.push(`TOOLS manual=${textValue(reduction.manualInvestigationCalls)} cli=${textValue(reduction.feedbackCliCalls)} reduced=${textValue(reduction.reducedCalls)} reduction=${textValue(reduction.reductionPercent)}%`);
|
||||
if (apiKeys.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("API KEYS");
|
||||
lines.push(renderTable([
|
||||
["ID", "NAME", "GROUP", "STATUS", "CURRENT"],
|
||||
...apiKeys.map((key) => [textValue(key.id), shorten(textValue(key.name), 30), shorten(textValue(key.groupName), 24), textValue(key.status), textValue(key.currentConcurrency)]),
|
||||
]));
|
||||
}
|
||||
lines.push("");
|
||||
lines.push(`REQUESTS returned=${textValue(summary.returned)} total=${textValue(summary.total)} moreAvailable=${textValue(summary.moreAvailable)} system_logs=${textValue(summary.systemLogCount)}`);
|
||||
if (timeline.length === 0) {
|
||||
lines.push("No matching completed request records in the selected window.");
|
||||
} else {
|
||||
lines.push(renderTable([
|
||||
["#", "AT", "GAP_S", "STATUS", "DURATION", "MODEL", "ACCOUNT", "ATTRIBUTION"],
|
||||
...timeline.map((item, index) => [
|
||||
String(index + 1), shortIso(item.at), textValue(item.gapSeconds), textValue(item.statusCode ?? item.kind), textValue(item.durationMs),
|
||||
shorten(textValue(item.model), 18), `${textValue(item.accountName)}#${textValue(item.accountId)}`, textValue(item.attribution),
|
||||
]),
|
||||
]));
|
||||
lines.push("");
|
||||
lines.push("REQUEST ID INDEX");
|
||||
lines.push(renderTable([
|
||||
["#", "REQUEST_ID", "CLIENT_ID"],
|
||||
...timeline.map((item, index) => [String(index + 1), textValue(item.requestId), textValue(item.clientRequestId)]),
|
||||
]));
|
||||
}
|
||||
if (accounts.length > 0 || components.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("ACCOUNT / INFRA");
|
||||
lines.push(renderTable([
|
||||
["ACCOUNT", "STATUS", "SCHED", "IN_USE", "QUEUE", "CAP", "AVAILABLE", "PROXY", "P_STATUS"],
|
||||
...accounts.map((item) => [
|
||||
`${textValue(item.name)}#${textValue(item.id)}`, textValue(item.status), textValue(item.schedulable),
|
||||
textValue(item.currentInUse), textValue(item.waitingInQueue), textValue(item.maxCapacity), textValue(item.available),
|
||||
shorten(`${textValue(item.proxyName)}#${textValue(item.proxyId)}`, 24), textValue(item.proxyStatus),
|
||||
]),
|
||||
...components.map((item) => [shorten(textValue(item.name), 32), textValue(item.status), "-", "-", "-", "-", textValue(item.running), "runtime", textValue(infrastructure.mode)]),
|
||||
]));
|
||||
}
|
||||
if (options.requestId !== null) {
|
||||
lines.push("");
|
||||
lines.push(`REQUEST DETAIL id=${options.requestId} logs=${textValue(detail.logCount)}`);
|
||||
if (detailLogs.length > 0) {
|
||||
lines.push(renderTable([
|
||||
["LOG_ID", "AT", "LEVEL", "COMPONENT", "REQUEST_ID", "CLIENT_ID", "MESSAGE"],
|
||||
...detailLogs.map((item) => [
|
||||
textValue(item.id), shortIso(item.at), textValue(item.level), shorten(textValue(item.component), 22),
|
||||
shorten(textValue(item.requestId), 18), shorten(textValue(item.clientRequestId), 18), shorten(textValue(item.message), 64),
|
||||
]),
|
||||
]));
|
||||
}
|
||||
}
|
||||
if (errors.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("PARTIAL EVIDENCE");
|
||||
lines.push(renderTable([["SOURCE", "ERROR"], ...errors.map((item) => [textValue(item.source), shorten(textValue(item.error), 90)])]));
|
||||
}
|
||||
lines.push("");
|
||||
if (summary.moreAvailable === true && typeof summary.nextPageToken === "string") {
|
||||
lines.push(`NEXT_PAGE_TOKEN ${summary.nextPageToken}`);
|
||||
lines.push(`Next: bun scripts/cli.ts platform-infra sub2api codex-pool feedback --target ${options.targetId} --user ${options.user} --window ${options.window} --page-token ${summary.nextPageToken}`);
|
||||
}
|
||||
lines.push("Disclosure: rerun with --request-id <client-or-internal-id> for correlated indexed logs.");
|
||||
lines.push("JSON: add --json for the complete redacted report.");
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -13,3 +13,4 @@ export * from "./remote-scripts";
|
||||
export * from "./accounts";
|
||||
export * from "./remote-python-sync";
|
||||
export * from "./remote";
|
||||
export * from "./feedback";
|
||||
|
||||
@@ -21,10 +21,12 @@ import {
|
||||
import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "../secrets";
|
||||
import { runSshCommandCapture, type SshCaptureResult } from "../ssh";
|
||||
|
||||
import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types";
|
||||
import type { ConfirmOptions, DisclosureOptions, FeedbackOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types";
|
||||
import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions";
|
||||
import { codexPoolFeedback } from "./feedback";
|
||||
import { renderCodexPoolPlan } from "./render";
|
||||
import { codexPoolRuntime } from "./runtime";
|
||||
import { codexPoolFaults } from "./faults";
|
||||
import { defaultCodexPoolRuntimeTargetId } from "./runtime-target";
|
||||
import { codexPoolHelp } from "./types";
|
||||
|
||||
@@ -38,7 +40,9 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[])
|
||||
if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1)));
|
||||
if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1)));
|
||||
if (action === "runtime") return await codexPoolRuntime(config, args.slice(1));
|
||||
if (action === "faults") return await codexPoolFaults(config, args.slice(1));
|
||||
if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1)));
|
||||
if (action === "feedback") return await codexPoolFeedback(config, parseFeedbackOptions(args.slice(1)));
|
||||
if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1)));
|
||||
if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1)));
|
||||
if (action === "sentinel-report") return await codexPoolSentinelReport(config, parseSentinelReportOptions(args.slice(1)));
|
||||
@@ -181,6 +185,7 @@ export function parseSentinelReportOptions(args: string[]): SentinelReportOption
|
||||
|
||||
export function parseTraceOptions(args: string[]): TraceOptions {
|
||||
let requestId: string | null = null;
|
||||
let pageToken: string | null = null;
|
||||
let since = "24h";
|
||||
let tail = 20_000;
|
||||
let contextSeconds = 300;
|
||||
@@ -258,6 +263,64 @@ export function parseTraceOptions(args: string[]): TraceOptions {
|
||||
return { ...disclosure, requestId, since, tail, contextSeconds, showLines };
|
||||
}
|
||||
|
||||
export function parseFeedbackOptions(args: string[]): FeedbackOptions {
|
||||
let user: string | null = null;
|
||||
let window: FeedbackOptions["window"] = "30m";
|
||||
let requestId: string | null = null;
|
||||
let pageToken: string | null = null;
|
||||
let json = false;
|
||||
const targetArgs: string[] = [];
|
||||
const windows = new Set<FeedbackOptions["window"]>(["5m", "30m", "1h", "6h", "24h", "7d", "30d"]);
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index]!;
|
||||
if (arg === "--json") {
|
||||
json = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--target" || arg.startsWith("--target=")) {
|
||||
targetArgs.push(arg);
|
||||
if (arg === "--target") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value");
|
||||
targetArgs.push(value);
|
||||
index += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const readValue = (option: string): string => {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${option} requires a value`);
|
||||
index += 1;
|
||||
return value.trim();
|
||||
};
|
||||
if (arg === "--user") user = readValue(arg);
|
||||
else if (arg.startsWith("--user=")) user = arg.slice("--user=".length).trim();
|
||||
else if (arg === "--window") {
|
||||
const value = readValue(arg) as FeedbackOptions["window"];
|
||||
if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d");
|
||||
window = value;
|
||||
} else if (arg.startsWith("--window=")) {
|
||||
const value = arg.slice("--window=".length) as FeedbackOptions["window"];
|
||||
if (!windows.has(value)) throw new Error("--window must be one of 5m, 30m, 1h, 6h, 24h, 7d, 30d");
|
||||
window = value;
|
||||
} else if (arg === "--request-id") requestId = readValue(arg);
|
||||
else if (arg.startsWith("--request-id=")) requestId = arg.slice("--request-id=".length).trim();
|
||||
else if (arg === "--page-token") pageToken = readValue(arg);
|
||||
else if (arg.startsWith("--page-token=")) pageToken = arg.slice("--page-token=".length).trim();
|
||||
else throw new Error(`unsupported option: ${arg}`);
|
||||
}
|
||||
if (user === null || user.length === 0) throw new Error("feedback requires --user <email-or-id>");
|
||||
if (user.length > 320 || /[\r\n]/u.test(user)) throw new Error("--user must be a single email or positive user id");
|
||||
if (requestId !== null && (requestId.length === 0 || requestId.length > 256 || /[\r\n]/u.test(requestId))) {
|
||||
throw new Error("--request-id must be a non-empty stable request id up to 256 characters");
|
||||
}
|
||||
if (pageToken !== null && (pageToken.length === 0 || pageToken.length > 1024 || !/^[A-Za-z0-9_-]+$/u.test(pageToken))) {
|
||||
throw new Error("--page-token must be a URL-safe cursor token");
|
||||
}
|
||||
if (requestId !== null && pageToken !== null) throw new Error("--request-id and --page-token cannot be combined");
|
||||
return { user, window, requestId, pageToken, json, targetId: parseTargetId(targetArgs) };
|
||||
}
|
||||
|
||||
export function parseKubectlDuration(raw: string): string {
|
||||
const value = raw.trim();
|
||||
if (!/^[1-9][0-9]*(?:s|m|h)$/u.test(value)) throw new Error("--since must be a kubectl duration such as 24h, 90m, or 300s");
|
||||
|
||||
@@ -71,6 +71,15 @@ export interface TraceOptions extends DisclosureOptions {
|
||||
showLines: boolean;
|
||||
}
|
||||
|
||||
export interface FeedbackOptions {
|
||||
user: string;
|
||||
window: "5m" | "30m" | "1h" | "6h" | "24h" | "7d" | "30d";
|
||||
requestId: string | null;
|
||||
pageToken: string | null;
|
||||
json: boolean;
|
||||
targetId: string;
|
||||
}
|
||||
|
||||
export interface SentinelImageOptions extends DisclosureOptions {
|
||||
action: "status" | "build";
|
||||
confirm: boolean;
|
||||
@@ -341,8 +350,8 @@ export function codexPoolHelp(): unknown {
|
||||
const pool = readCodexPoolConfig();
|
||||
const runtimeTarget = codexPoolRuntimeTarget();
|
||||
return {
|
||||
command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
|
||||
output: "json, except trace and sentinel-report default to low-noise text tables",
|
||||
command: "platform-infra sub2api codex-pool plan|sync|validate|runtime|faults|trace|feedback|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local",
|
||||
output: "json, except trace, feedback, and sentinel-report default to low-noise text tables",
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601",
|
||||
@@ -351,11 +360,13 @@ export function codexPoolHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime list [--target PK01] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime get --account <name-or-id> [--target PK01] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime errors [--account <name-or-id>] [--since 24h] [--tail 50000] [--target PK01] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool faults [--level P0|P1|P2] [--group <name-or-id>] [--account <name-or-id>] [--model <model>] [--stream sync|stream] [--endpoint <path>] [--request-id <id>] [--page-token <token>] [--target PK01] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --account <name-or-id> --template <id> [--target PK01] [--confirm]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime apply --accounts <name-or-id,...> --template <id> [--target PK01] [--confirm] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --account <name-or-id> --kind temp-unschedulable [--target PK01] [--confirm]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool runtime delete --accounts <name-or-id,...> --kind temp-unschedulable [--target PK01] [--confirm] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id <requestId> [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--target PK01] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm",
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import type { RenderedCliResult } from "../output";
|
||||
import { defaultSub2ApiTargetId } from "../platform-infra/config";
|
||||
|
||||
import { projectSub2ApiOps } from "./projection";
|
||||
import { querySub2ApiOps } from "./query";
|
||||
import { renderSub2ApiOps } from "./render";
|
||||
import type { Sub2ApiOpsOptions } from "./types";
|
||||
|
||||
export async function runSub2ApiOpsCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const options = parseSub2ApiOpsOptions(args);
|
||||
const raw = await querySub2ApiOps(config, options);
|
||||
const projected = projectSub2ApiOps(raw, options);
|
||||
return options.json ? projected : renderSub2ApiOps(projected);
|
||||
}
|
||||
|
||||
export function parseSub2ApiOpsOptions(args: string[]): Sub2ApiOpsOptions {
|
||||
const [actionRaw, ...rest] = args;
|
||||
if (actionRaw !== "diagnosis" && actionRaw !== "channels") {
|
||||
throw new Error("sub2api ops usage: diagnosis|channels [--target <id>] [--platform <name>] [--json]");
|
||||
}
|
||||
let targetId = defaultSub2ApiTargetId();
|
||||
let json = false;
|
||||
let platform: string | null = null;
|
||||
let group: string | null = null;
|
||||
let diagnosisId: string | null = null;
|
||||
let timeRange = "1h" as Sub2ApiOpsOptions["timeRange"];
|
||||
let channel: string | null = null;
|
||||
let model: string | null = null;
|
||||
let window = "7d" as Sub2ApiOpsOptions["window"];
|
||||
let pageToken: string | null = null;
|
||||
let recordId: string | null = null;
|
||||
for (let index = 0; index < rest.length; index += 1) {
|
||||
const arg = rest[index]!;
|
||||
const readValue = (name: string): string => {
|
||||
const value = rest[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
index += 1;
|
||||
return value.trim();
|
||||
};
|
||||
if (arg === "--json") json = true;
|
||||
else if (arg === "--target") targetId = readValue(arg);
|
||||
else if (arg === "--platform") platform = readValue(arg);
|
||||
else if (arg === "--group") group = readValue(arg);
|
||||
else if (arg === "--id") diagnosisId = readValue(arg);
|
||||
else if (arg === "--time-range") timeRange = parseTimeRange(readValue(arg));
|
||||
else if (arg === "--channel") channel = readValue(arg);
|
||||
else if (arg === "--model") model = readValue(arg);
|
||||
else if (arg === "--window") window = parseWindow(readValue(arg));
|
||||
else if (arg === "--page-token") pageToken = parseRecordId(readValue(arg), arg);
|
||||
else if (arg === "--record") recordId = parseRecordId(readValue(arg), arg);
|
||||
else throw new Error(`unknown sub2api ops option: ${arg}`);
|
||||
}
|
||||
if (actionRaw === "diagnosis" && (channel !== null || model !== null || window !== "7d" || pageToken !== null || recordId !== null)) throw new Error("ops diagnosis does not accept channel history options");
|
||||
if (actionRaw === "channels" && (group !== null || diagnosisId !== null || timeRange !== "1h")) throw new Error("ops channels does not accept --group, --id, or --time-range");
|
||||
if (actionRaw === "channels" && model !== null && channel === null) throw new Error("--model requires --channel");
|
||||
if (actionRaw === "channels" && (pageToken !== null || recordId !== null) && channel === null) throw new Error("--page-token and --record require --channel");
|
||||
if (pageToken !== null && recordId !== null) throw new Error("use only one of --page-token or --record");
|
||||
return { action: actionRaw, targetId, json, platform, group, diagnosisId, timeRange, channel, model, window, pageToken, recordId };
|
||||
}
|
||||
|
||||
function parseTimeRange(value: string): Sub2ApiOpsOptions["timeRange"] {
|
||||
if (value === "5m" || value === "30m" || value === "1h" || value === "6h" || value === "24h") return value;
|
||||
throw new Error("--time-range must be one of 5m, 30m, 1h, 6h, 24h");
|
||||
}
|
||||
|
||||
function parseWindow(value: string): Sub2ApiOpsOptions["window"] {
|
||||
if (value === "7d" || value === "15d" || value === "30d") return value;
|
||||
throw new Error("--window must be one of 7d, 15d, 30d");
|
||||
}
|
||||
|
||||
function parseRecordId(value: string, option: string): string {
|
||||
if (/^[1-9][0-9]*$/u.test(value)) return value;
|
||||
throw new Error(`${option} must be a positive native history record ID`);
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import type { Sub2ApiOpsOptions } from "./types";
|
||||
|
||||
const upstreamVersion = "v0.1.155";
|
||||
const diagnosisRuleSource = "frontend/src/views/admin/ops/components/OpsDashboardHeader.vue";
|
||||
const diagnosisTextSource = "frontend/src/i18n/locales/zh/admin/ops.ts";
|
||||
const channelHistoryPageSize = 20;
|
||||
const monitorResponseHeaderTimeoutMs = 30_000;
|
||||
const monitorTotalRequestTimeoutMs = 45_000;
|
||||
|
||||
export function projectSub2ApiOps(raw: Record<string, unknown>, options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
const target = record(raw.target);
|
||||
if (raw.ok !== true) {
|
||||
return {
|
||||
ok: false,
|
||||
action: `platform-infra-sub2api-ops-${options.action}`,
|
||||
target,
|
||||
query: querySummary(options),
|
||||
source: sourceSummary(options.action, "unavailable"),
|
||||
error: stringValue(raw.error, "Sub2API native source unavailable"),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
const data = record(raw.data);
|
||||
return options.action === "diagnosis"
|
||||
? projectDiagnosis(target, data, options)
|
||||
: projectChannels(target, data, options);
|
||||
}
|
||||
|
||||
function projectDiagnosis(target: Record<string, unknown>, data: Record<string, unknown>, options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
const runtimeVersion = record(data.runtimeVersion);
|
||||
const groups = arrayRecords(data.groups).map((entry) => {
|
||||
const group = record(entry.group);
|
||||
const overview = record(entry.overview);
|
||||
const scope = group.id === null || group.id === undefined ? "all" : `g${String(group.id)}`;
|
||||
return {
|
||||
id: scope,
|
||||
groupId: group.id ?? null,
|
||||
groupName: stringValue(group.name, "all"),
|
||||
platform: stringValue(group.platform, "all"),
|
||||
window: {
|
||||
kind: "native-ops-time-range",
|
||||
value: options.timeRange,
|
||||
startAt: overview.start_time ?? null,
|
||||
endAt: overview.end_time ?? null,
|
||||
},
|
||||
sourceStatus: "available",
|
||||
diagnoses: diagnosisItems(scope, overview),
|
||||
metrics: compactOverview(overview),
|
||||
};
|
||||
});
|
||||
const selectedDiagnosis = options.diagnosisId === null
|
||||
? null
|
||||
: groups.flatMap((group) => arrayRecords(group.diagnoses)).find((item) => item.id === options.diagnosisId) ?? null;
|
||||
return {
|
||||
ok: options.diagnosisId === null || selectedDiagnosis !== null,
|
||||
action: "platform-infra-sub2api-ops-diagnosis",
|
||||
target,
|
||||
query: querySummary(options),
|
||||
source: sourceSummary("diagnosis", "available", runtimeVersion),
|
||||
groups,
|
||||
selectedDiagnosis,
|
||||
evidence: options.diagnosisId === null ? null : projectEvidence(record(data.evidence)),
|
||||
error: options.diagnosisId !== null && selectedDiagnosis === null ? `diagnosis id not found: ${options.diagnosisId}` : null,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function diagnosisItems(scope: string, overview: Record<string, unknown>): Record<string, unknown>[] {
|
||||
const items: Record<string, unknown>[] = [];
|
||||
const qps = numberAt(record(overview.qps).current, 0);
|
||||
const errorRate = numberAt(overview.error_rate, 0);
|
||||
if (qps === 0 && errorRate === 0) {
|
||||
return [diagnosisItem(scope, "idle", "info", "system_activity", 0, "requests/s", "系统当前处于待机状态", "无活跃流量", null, overview)];
|
||||
}
|
||||
const systemMetrics = record(overview.system_metrics);
|
||||
if (systemMetrics.db_ok === false) items.push(diagnosisItem(scope, "db-down", "critical", "db_ok", false, "boolean", "数据库连接失败", "所有数据库操作将失败", "检查数据库服务状态、网络连接和连接配置", overview));
|
||||
if (systemMetrics.redis_ok === false) items.push(diagnosisItem(scope, "redis-down", "warning", "redis_ok", false, "boolean", "Redis连接失败", "缓存功能降级，性能可能下降", "检查Redis服务状态和网络连接", overview));
|
||||
const cpu = numberAt(systemMetrics.cpu_usage_percent, 0);
|
||||
if (cpu > 90) items.push(diagnosisItem(scope, "cpu-critical", "critical", "cpu_usage_percent", cpu, "%", `CPU使用率严重过高 (${cpu.toFixed(1)}%)`, "系统响应变慢，可能影响所有请求", "检查CPU密集型任务，考虑扩容或优化代码", overview));
|
||||
else if (cpu > 80) items.push(diagnosisItem(scope, "cpu-high", "warning", "cpu_usage_percent", cpu, "%", `CPU使用率偏高 (${cpu.toFixed(1)}%)`, "系统负载较高，需要关注", "监控CPU趋势，准备扩容方案", overview));
|
||||
const memory = numberAt(systemMetrics.memory_usage_percent, 0);
|
||||
if (memory > 90) items.push(diagnosisItem(scope, "memory-critical", "critical", "memory_usage_percent", memory, "%", `内存使用率严重过高 (${memory.toFixed(1)}%)`, "可能触发OOM，系统稳定性受威胁", "检查内存泄漏，考虑增加内存或优化内存使用", overview));
|
||||
else if (memory > 85) items.push(diagnosisItem(scope, "memory-high", "warning", "memory_usage_percent", memory, "%", `内存使用率偏高 (${memory.toFixed(1)}%)`, "内存压力较大，需要关注", "监控内存趋势，检查是否有内存泄漏", overview));
|
||||
const ttft = numberAt(record(overview.ttft).p99_ms, 0);
|
||||
if (ttft > 500) items.push(diagnosisItem(scope, "ttft-high", "warning", "ttft.p99_ms", ttft, "ms", `首 Token 时间偏高 (${ttft.toFixed(0)}ms)`, "用户感知时长增加", "优化请求处理流程，减少前置逻辑耗时", overview));
|
||||
const upstreamRate = numberAt(overview.upstream_error_rate, 0) * 100;
|
||||
if (upstreamRate > 5) items.push(diagnosisItem(scope, "upstream-critical", "critical", "upstream_error_rate", upstreamRate, "%", `上游错误率严重偏高 (${upstreamRate.toFixed(2)}%)`, "可能影响大量用户请求", "检查上游服务健康状态，启用降级策略", overview));
|
||||
else if (upstreamRate > 2) items.push(diagnosisItem(scope, "upstream-high", "warning", "upstream_error_rate", upstreamRate, "%", `上游错误率偏高 (${upstreamRate.toFixed(2)}%)`, "建议检查上游服务状态", "联系上游服务团队，准备降级方案", overview));
|
||||
const clientRate = errorRate * 100;
|
||||
if (clientRate > 3) items.push(diagnosisItem(scope, "error-high", "critical", "error_rate", clientRate, "%", `错误率过高 (${clientRate.toFixed(2)}%)`, "大量请求失败", "查看错误日志，定位错误根因，紧急修复", overview));
|
||||
else if (clientRate > 0.5) items.push(diagnosisItem(scope, "error-elevated", "warning", "error_rate", clientRate, "%", `错误率偏高 (${clientRate.toFixed(2)}%)`, "建议检查错误日志", "分析错误类型和分布，制定修复计划", overview));
|
||||
const sla = numberAt(overview.sla, 0) * 100;
|
||||
if (sla < 90) items.push(diagnosisItem(scope, "sla-critical", "critical", "sla", sla, "%", `SLA 严重低于目标 (${sla.toFixed(2)}%)`, "用户体验严重受损", "紧急排查错误原因，必要时采取限流保护", overview));
|
||||
else if (sla < 98) items.push(diagnosisItem(scope, "sla-low", "warning", "sla", sla, "%", `SLA 低于目标 (${sla.toFixed(2)}%)`, "需要关注服务质量", "分析SLA下降原因，优化系统性能", overview));
|
||||
const healthScore = finiteNumber(overview.health_score);
|
||||
if (healthScore !== null && healthScore < 60) items.push(diagnosisItem(scope, "health-critical", "critical", "health_score", healthScore, "score", `综合健康评分过低 (${healthScore})`, "多个指标可能同时异常，建议优先排查错误与资源使用情况", "全面检查系统状态，优先处理critical级别问题", overview));
|
||||
else if (healthScore !== null && healthScore < 90) items.push(diagnosisItem(scope, "health-low", "warning", "health_score", healthScore, "score", `综合健康评分偏低 (${healthScore})`, "可能存在轻度波动，建议关注 SLA 与错误率", "监控指标趋势，预防问题恶化", overview));
|
||||
if (items.length === 0) items.push(diagnosisItem(scope, "healthy", "info", "health", "normal", "state", "所有系统指标正常", "服务运行稳定", null, overview));
|
||||
return items;
|
||||
}
|
||||
|
||||
function diagnosisItem(scope: string, category: string, severity: string, metric: string, value: unknown, unit: string, message: string, impact: string, advice: string | null, overview: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
id: `${scope}:${category}`,
|
||||
category,
|
||||
severity,
|
||||
message,
|
||||
metric: { name: metric, value, unit },
|
||||
impact,
|
||||
advice,
|
||||
adviceKind: advice === null ? null : "native-frontend-advice",
|
||||
factRef: `${scope}.metrics`,
|
||||
inferenceStatus: "not-verified",
|
||||
};
|
||||
}
|
||||
|
||||
function compactOverview(overview: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
requestCount: overview.request_count_total ?? null,
|
||||
successCount: overview.success_count ?? null,
|
||||
errorCount: overview.error_count_total ?? null,
|
||||
errorRate: overview.error_rate ?? null,
|
||||
upstreamErrorRate: overview.upstream_error_rate ?? null,
|
||||
sla: overview.sla ?? null,
|
||||
healthScore: overview.health_score ?? null,
|
||||
qps: record(overview.qps).current ?? null,
|
||||
ttftP99Ms: record(overview.ttft).p99_ms ?? null,
|
||||
collectedAt: overview.end_time ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function projectEvidence(evidence: Record<string, unknown>): Record<string, unknown> | null {
|
||||
if (Object.keys(evidence).length === 0) return null;
|
||||
const records = arrayRecords(evidence.records).map((item) => ({
|
||||
requestId: item.request_id ?? null,
|
||||
createdAt: item.created_at ?? null,
|
||||
platform: item.platform ?? null,
|
||||
model: item.model ?? null,
|
||||
accountId: item.account_id ?? null,
|
||||
accountName: item.account_name ?? null,
|
||||
statusCode: item.status_code ?? item.upstream_status_code ?? null,
|
||||
durationMs: item.duration_ms ?? null,
|
||||
severity: item.severity ?? null,
|
||||
message: item.message ?? null,
|
||||
}));
|
||||
return { diagnosisId: evidence.diagnosisId ?? null, recordCount: records.length, records };
|
||||
}
|
||||
|
||||
function projectChannels(target: Record<string, unknown>, data: Record<string, unknown>, options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
const runtimeVersion = record(data.runtimeVersion);
|
||||
const channels = arrayRecords(data.channels).map((entry) => projectChannel(entry, options.window));
|
||||
const operational = channels.every((channel) => channel.status === "OPERATIONAL");
|
||||
const history = arrayRecords(data.history)
|
||||
.map((item) => ({
|
||||
id: item.id ?? null,
|
||||
model: item.model ?? null,
|
||||
status: item.status ?? null,
|
||||
latencyMs: item.latency_ms ?? null,
|
||||
pingLatencyMs: item.ping_latency_ms ?? null,
|
||||
message: item.message ?? null,
|
||||
checkedAt: item.checked_at ?? null,
|
||||
}))
|
||||
.sort((left, right) => String(left.checkedAt).localeCompare(String(right.checkedAt)));
|
||||
const historyProjection = options.channel === null ? null : paginateHistory(history, options, record(data.correlation), runtimeVersion);
|
||||
const historyError = historyProjection === null ? null : historyProjection.error;
|
||||
return {
|
||||
ok: historyError === null,
|
||||
action: "platform-infra-sub2api-ops-channels",
|
||||
target,
|
||||
query: querySummary(options),
|
||||
source: sourceSummary("channels", "available", runtimeVersion),
|
||||
timeoutPolicy: monitorTimeoutPolicy(runtimeVersion),
|
||||
overallStatus: operational ? "OPERATIONAL" : "DEGRADED",
|
||||
channels,
|
||||
history: historyProjection,
|
||||
error: historyError,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function paginateHistory(history: Record<string, unknown>[], options: Sub2ApiOpsOptions, correlation: Record<string, unknown>, runtimeVersion: Record<string, unknown>): Record<string, unknown> {
|
||||
if (options.recordId !== null) {
|
||||
const selectedRecordRaw = history.find((item) => String(item.id) === options.recordId) ?? null;
|
||||
const selectedRecord = selectedRecordRaw === null ? null : projectSelectedRecord(selectedRecordRaw, correlation, runtimeVersion);
|
||||
return {
|
||||
order: "PAST_TO_NOW",
|
||||
pageSize: channelHistoryPageSize,
|
||||
sourceRecordCount: history.length,
|
||||
sourceCapReached: history.length === 1000,
|
||||
records: [],
|
||||
moreAvailable: false,
|
||||
nextPageToken: null,
|
||||
selectedRecord,
|
||||
error: selectedRecord === null ? `channel history record not found: ${options.recordId}` : null,
|
||||
};
|
||||
}
|
||||
let end = history.length;
|
||||
if (options.pageToken !== null) {
|
||||
const cursorIndex = history.findIndex((item) => String(item.id) === options.pageToken);
|
||||
if (cursorIndex < 0) {
|
||||
return {
|
||||
order: "PAST_TO_NOW",
|
||||
pageSize: channelHistoryPageSize,
|
||||
sourceRecordCount: history.length,
|
||||
sourceCapReached: history.length === 1000,
|
||||
records: [],
|
||||
moreAvailable: false,
|
||||
nextPageToken: null,
|
||||
selectedRecord: null,
|
||||
error: `channel history page token not found: ${options.pageToken}`,
|
||||
};
|
||||
}
|
||||
end = cursorIndex;
|
||||
}
|
||||
const start = Math.max(0, end - channelHistoryPageSize);
|
||||
const records = history.slice(start, end);
|
||||
return {
|
||||
order: "PAST_TO_NOW",
|
||||
pageSize: channelHistoryPageSize,
|
||||
sourceRecordCount: history.length,
|
||||
sourceCapReached: history.length === 1000,
|
||||
records,
|
||||
moreAvailable: start > 0,
|
||||
nextPageToken: start > 0 && records.length > 0 ? records[0]?.id ?? null : null,
|
||||
selectedRecord: null,
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
|
||||
function projectSelectedRecord(item: Record<string, unknown>, correlation: Record<string, unknown>, runtimeVersion: Record<string, unknown>): Record<string, unknown> {
|
||||
const latencyMs = typeof item.latencyMs === "number" ? item.latencyMs : null;
|
||||
const selected = record(correlation.selected);
|
||||
const final = record(selected.final);
|
||||
const failovers = arrayRecords(selected.failovers);
|
||||
const selectFailures = arrayRecords(selected.selectFailures);
|
||||
const totalLatencyMs = typeof final.latencyMs === "number" ? final.latencyMs : null;
|
||||
const runtimeTag = typeof runtimeVersion.version === "string" && runtimeVersion.version.length > 0 ? runtimeVersion.version : null;
|
||||
const runtimeVersionMismatch = runtimeTag?.replace(/^v/u, "") !== upstreamVersion.replace(/^v/u, "");
|
||||
return {
|
||||
...item,
|
||||
timeout: {
|
||||
observedKind: String(item.message ?? "").toLowerCase().includes("timeout awaiting response headers") ? "response-header" : "unknown",
|
||||
responseHeaderTimeoutMs: monitorResponseHeaderTimeoutMs,
|
||||
totalRequestTimeoutMs: monitorTotalRequestTimeoutMs,
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersion: runtimeTag,
|
||||
runtimeVersionMismatch,
|
||||
runtimePolicyStatus: runtimeVersionMismatch ? "unverified" : "source-version-verified",
|
||||
remainingAfterResponseHeaderMs: monitorTotalRequestTimeoutMs - monitorResponseHeaderTimeoutMs,
|
||||
observedLatencyDeltaMs: latencyMs === null ? null : latencyMs - monitorResponseHeaderTimeoutMs,
|
||||
perUpstreamTimeout: { status: "unsupported", reason: "native monitor history does not expose per-upstream timeout" },
|
||||
overallGatewayDeadline: { status: "unsupported", reason: "native monitor history does not expose the gateway request deadline" },
|
||||
failoverReserve: { status: "unsupported", reason: "native monitor history does not expose a dedicated failover reserve" },
|
||||
},
|
||||
correlation: {
|
||||
status: correlation.status ?? "unsupported",
|
||||
reason: correlation.reason ?? null,
|
||||
requestId: selected.requestId ?? null,
|
||||
match: selected.match ?? null,
|
||||
apiKeyName: selected.apiKeyName ?? null,
|
||||
model: selected.model ?? null,
|
||||
accounts: selected.accounts ?? [],
|
||||
upstreamErrors: selected.upstreamErrors ?? [],
|
||||
failovers,
|
||||
selectFailures,
|
||||
final: Object.keys(final).length === 0 ? null : final,
|
||||
switchingObserved: failovers.length > 0,
|
||||
selectionFailureObserved: selectFailures.length > 0,
|
||||
sufficientForObservedCompletion: runtimeVersionMismatch || totalLatencyMs === null ? null : monitorTotalRequestTimeoutMs >= totalLatencyMs,
|
||||
sufficiencyStatus: runtimeVersionMismatch ? "unverified-runtime-version-mismatch" : totalLatencyMs === null ? "unsupported-no-final-latency" : "evaluated-against-verified-source-version",
|
||||
traceNext: selected.requestId ? `bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ${String(selected.requestId)}` : null,
|
||||
candidates: correlation.status === "ambiguous" ? correlation.candidates ?? [] : [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function projectChannel(entry: Record<string, unknown>, window: "7d" | "15d" | "30d"): Record<string, unknown> {
|
||||
const summary = record(entry.summary);
|
||||
const detail = record(entry.detail);
|
||||
const models = arrayRecords(detail.models).map((model) => ({
|
||||
model: model.model ?? null,
|
||||
status: upperStatus(model.latest_status),
|
||||
latencyMs: model.latest_latency_ms ?? null,
|
||||
availability: availabilityFor(model, window),
|
||||
availabilityWindow: window,
|
||||
avgLatency7dMs: model.avg_latency_7d_ms ?? null,
|
||||
}));
|
||||
const timeline = arrayRecords(summary.timeline)
|
||||
.map((point) => ({
|
||||
status: upperStatus(point.status),
|
||||
latencyMs: point.latency_ms ?? null,
|
||||
pingLatencyMs: point.ping_latency_ms ?? null,
|
||||
checkedAt: point.checked_at ?? null,
|
||||
}))
|
||||
.sort((left, right) => String(left.checkedAt).localeCompare(String(right.checkedAt)));
|
||||
const status = [summary.primary_status, ...arrayRecords(summary.extra_models).map((item) => item.status)]
|
||||
.every((value) => String(value ?? "").toLowerCase() === "operational") ? "OPERATIONAL" : "DEGRADED";
|
||||
return {
|
||||
id: summary.id ?? detail.id ?? null,
|
||||
name: summary.name ?? detail.name ?? null,
|
||||
groupName: summary.group_name ?? detail.group_name ?? null,
|
||||
provider: summary.provider ?? detail.provider ?? null,
|
||||
status,
|
||||
primary: {
|
||||
model: summary.primary_model ?? null,
|
||||
status: upperStatus(summary.primary_status),
|
||||
latencyMs: summary.primary_latency_ms ?? null,
|
||||
pingLatencyMs: summary.primary_ping_latency_ms ?? null,
|
||||
availability: availabilityFor(models.find((item) => item.model === summary.primary_model) ?? {}, window) ?? summary.availability_7d ?? null,
|
||||
availabilityWindow: window,
|
||||
},
|
||||
models,
|
||||
recentRecordCount: timeline.length,
|
||||
collectedAt: timeline.length === 0 ? null : timeline[timeline.length - 1]?.checkedAt ?? null,
|
||||
refreshRemainingSeconds: null,
|
||||
refreshSourceStatus: "unsupported-by-native-response",
|
||||
timeline: {
|
||||
order: "PAST_TO_NOW",
|
||||
recordCount: timeline.length,
|
||||
firstAt: timeline[0]?.checkedAt ?? null,
|
||||
lastAt: timeline[timeline.length - 1]?.checkedAt ?? null,
|
||||
records: [],
|
||||
detail: "use --channel <id-or-name> for native history pagination",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function monitorTimeoutPolicy(runtimeVersion: Record<string, unknown>): Record<string, unknown> {
|
||||
const runtimeTag = typeof runtimeVersion.version === "string" && runtimeVersion.version.length > 0 ? runtimeVersion.version : null;
|
||||
const runtimeVersionMismatch = runtimeTag?.replace(/^v/u, "") !== upstreamVersion.replace(/^v/u, "");
|
||||
return {
|
||||
runtime: { image: runtimeVersion.image ?? null, version: runtimeTag },
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersionMismatch,
|
||||
runtimePolicyStatus: runtimeVersionMismatch ? "unverified" : "source-version-verified",
|
||||
sourceReference: {
|
||||
responseHeaderTimeoutMs: monitorResponseHeaderTimeoutMs,
|
||||
totalRequestTimeoutMs: monitorTotalRequestTimeoutMs,
|
||||
sourceStatus: "official-source-constant-verified-for-reference-version",
|
||||
},
|
||||
perUpstreamTimeout: { status: "unsupported", reason: "native monitor API does not expose this field" },
|
||||
overallGatewayDeadline: { status: "unsupported", reason: "native monitor API does not expose this field" },
|
||||
failoverReserve: { status: "unsupported", reason: "native monitor API does not expose this field" },
|
||||
};
|
||||
}
|
||||
|
||||
function availabilityFor(model: Record<string, unknown>, window: "7d" | "15d" | "30d"): unknown {
|
||||
return model[`availability_${window}`] ?? model.availability ?? null;
|
||||
}
|
||||
|
||||
function sourceSummary(action: "diagnosis" | "channels", status: string, runtimeVersion: Record<string, unknown> = {}): Record<string, unknown> {
|
||||
if (action === "diagnosis") {
|
||||
return {
|
||||
status,
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersion,
|
||||
metricsEndpoint: "/api/v1/admin/ops/dashboard/overview",
|
||||
nativeDiagnosisEndpoint: { status: "unsupported", reason: "v0.1.155 has no diagnosis/advice response schema" },
|
||||
projectionKind: "native-frontend-projection",
|
||||
ruleSource: diagnosisRuleSource,
|
||||
textSource: diagnosisTextSource,
|
||||
evidenceBoundary: "metrics are native API facts; severity, impact and advice reproduce the official v0.1.155 frontend projection and are not server-returned root-cause proof",
|
||||
};
|
||||
}
|
||||
return {
|
||||
status,
|
||||
sourceReferenceVersion: upstreamVersion,
|
||||
runtimeVersion,
|
||||
listEndpoint: "/api/v1/channel-monitors",
|
||||
detailEndpoint: "/api/v1/channel-monitors/:id/status",
|
||||
historyEndpoint: "/api/v1/admin/channel-monitors/:id/history",
|
||||
correlationSource: "heuristic projection by native record checked_at, latency_ms and model; request/failover facts come from request rows and bounded runtime logs when available",
|
||||
windows: ["7d", "15d", "30d"],
|
||||
overallStatusKind: "native-frontend-projection",
|
||||
refreshRemaining: { status: "unsupported", reason: "native response does not include a refresh countdown" },
|
||||
};
|
||||
}
|
||||
|
||||
function querySummary(options: Sub2ApiOpsOptions): Record<string, unknown> {
|
||||
return {
|
||||
target: options.targetId,
|
||||
platform: options.platform,
|
||||
group: options.group,
|
||||
diagnosisId: options.diagnosisId,
|
||||
timeRange: options.action === "diagnosis" ? options.timeRange : undefined,
|
||||
channel: options.channel,
|
||||
model: options.model,
|
||||
window: options.action === "channels" ? options.window : undefined,
|
||||
pageToken: options.pageToken,
|
||||
recordId: options.recordId,
|
||||
snapshot: true,
|
||||
foregroundRefresh: false,
|
||||
};
|
||||
}
|
||||
|
||||
function upperStatus(value: unknown): string {
|
||||
const text = String(value ?? "unknown").trim();
|
||||
return text.length === 0 ? "UNKNOWN" : text.toUpperCase();
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record) : [];
|
||||
}
|
||||
|
||||
function stringValue(value: unknown, fallback: string): string {
|
||||
return typeof value === "string" && value.length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function finiteNumber(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function numberAt(value: unknown, fallback: number): number {
|
||||
return finiteNumber(value) ?? fallback;
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
import type { UniDeskConfig } from "../config";
|
||||
import { parseJsonOutput } from "../platform-infra-ops-library";
|
||||
import { runSshCommandCapture } from "../ssh";
|
||||
import { readSub2ApiConfig } from "../platform-infra/config";
|
||||
import { resolveTarget } from "../platform-infra/manifest";
|
||||
|
||||
import type { Sub2ApiOpsOptions, Sub2ApiOpsTarget } from "./types";
|
||||
|
||||
export async function querySub2ApiOps(config: UniDeskConfig, options: Sub2ApiOpsOptions): Promise<Record<string, unknown>> {
|
||||
const target = resolveOpsTarget(options.targetId);
|
||||
const result = await runSshCommandCapture(config, target.route, ["sh"], remoteQueryScript(target, options));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
if (result.exitCode !== 0 || parsed === null) {
|
||||
const stderr = result.stderr.trim().slice(-1200);
|
||||
const stdout = result.stdout.trim().slice(-1200);
|
||||
return {
|
||||
ok: false,
|
||||
sourceStatus: "unavailable",
|
||||
error: stderr || stdout || "remote Sub2API ops query returned no JSON",
|
||||
remote: { exitCode: result.exitCode, stdoutTail: stdout, stderrTail: stderr },
|
||||
target,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
return { ...parsed, target, valuesPrinted: false };
|
||||
}
|
||||
|
||||
function resolveOpsTarget(targetId: string): Sub2ApiOpsTarget {
|
||||
const config = readSub2ApiConfig();
|
||||
const target = resolveTarget(config, targetId);
|
||||
return {
|
||||
id: target.id,
|
||||
route: target.route,
|
||||
namespace: target.namespace,
|
||||
runtimeMode: target.runtimeMode,
|
||||
hostDockerAppPort: target.hostDocker?.appPort ?? null,
|
||||
hostDockerEnvPath: target.hostDocker?.envPath ?? null,
|
||||
appSecretName: config.runtime.database.secretName,
|
||||
};
|
||||
}
|
||||
|
||||
function pyJson(value: unknown): string {
|
||||
return `json.loads(${JSON.stringify(JSON.stringify(value))})`;
|
||||
}
|
||||
|
||||
function remoteQueryScript(target: Sub2ApiOpsTarget, options: Sub2ApiOpsOptions): string {
|
||||
return `
|
||||
set -u
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
|
||||
TARGET = ${pyJson(target)}
|
||||
OPTIONS = ${pyJson(options)}
|
||||
|
||||
def run(command, input_bytes=None):
|
||||
return subprocess.run(command, input=input_bytes, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
def docker(args):
|
||||
proc = run(["docker", *args])
|
||||
if proc.returncode == 0:
|
||||
return proc
|
||||
sudo_proc = run(["sudo", "-n", "docker", *args])
|
||||
return sudo_proc if sudo_proc.returncode == 0 else proc
|
||||
|
||||
def runtime_logs(since="24h", tail=20000):
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
return docker(["logs", f"--since={since}", f"--tail={tail}", "sub2api-app"])
|
||||
return run(["kubectl", "-n", TARGET["namespace"], "logs", "deployment/sub2api", f"--since={since}", f"--tail={tail}"])
|
||||
|
||||
def tail_text(value, limit=800):
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", errors="replace")
|
||||
return str(value)[-limit:]
|
||||
|
||||
def kube_json(args, label):
|
||||
proc = run(["kubectl", *args, "-o", "json"])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"{label} failed: {tail_text(proc.stderr)}")
|
||||
return json.loads(proc.stdout.decode("utf-8"))
|
||||
|
||||
def read_host_env():
|
||||
path = TARGET.get("hostDockerEnvPath")
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
lines = handle.read().splitlines()
|
||||
except PermissionError:
|
||||
proc = run(["sudo", "-n", "cat", path])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("read host env failed: " + tail_text(proc.stderr))
|
||||
lines = proc.stdout.decode("utf-8", errors="replace").splitlines()
|
||||
values = {}
|
||||
for line in lines:
|
||||
text = line.strip()
|
||||
if not text or text.startswith("#") or "=" not in text:
|
||||
continue
|
||||
key, value = text.split("=", 1)
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
||||
value = value[1:-1]
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
def select_app_pod():
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
return "sub2api-app"
|
||||
pods = kube_json(["-n", TARGET["namespace"], "get", "pods", "-l", "app.kubernetes.io/name=sub2api"], "sub2api pods").get("items") or []
|
||||
running = [item for item in pods if (item.get("status") or {}).get("phase") == "Running"]
|
||||
if not running:
|
||||
raise RuntimeError("sub2api app pod not found")
|
||||
return running[0]["metadata"]["name"]
|
||||
|
||||
APP_POD = select_app_pod()
|
||||
|
||||
def runtime_version():
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
proc = docker(["inspect", "--format", "{{.Config.Image}}", "sub2api-app"])
|
||||
image = proc.stdout.decode("utf-8", errors="replace").strip() if proc.returncode == 0 else ""
|
||||
else:
|
||||
pod = kube_json(["-n", TARGET["namespace"], "get", "pod", APP_POD], "sub2api pod")
|
||||
containers = ((pod.get("spec") or {}).get("containers") or [])
|
||||
image = containers[0].get("image") if containers else ""
|
||||
tag = image.rsplit(":", 1)[1] if isinstance(image, str) and ":" in image else None
|
||||
return {"image": image or None, "version": tag}
|
||||
|
||||
def config_value(key):
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
data = kube_json(["-n", TARGET["namespace"], "get", "configmap", "sub2api-config"], "configmap/sub2api-config").get("data") or {}
|
||||
return data.get(key)
|
||||
|
||||
def secret_value(key):
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
return read_host_env().get(key)
|
||||
data = kube_json(["-n", TARGET["namespace"], "get", "secret", TARGET["appSecretName"]], "sub2api secret").get("data") or {}
|
||||
return base64.b64decode(data[key]).decode("utf-8") if key in data else None
|
||||
|
||||
def parse_curl(proc):
|
||||
output = proc.stdout.decode("utf-8", errors="replace")
|
||||
marker = "\\n__HTTP_CODE__:"
|
||||
position = output.rfind(marker)
|
||||
if position < 0:
|
||||
return {"ok": False, "httpStatus": 0, "json": None, "body": output, "stderr": tail_text(proc.stderr)}
|
||||
body = output[:position]
|
||||
try:
|
||||
status = int(output[position + len(marker):].strip()[-3:])
|
||||
except ValueError:
|
||||
status = 0
|
||||
try:
|
||||
parsed = json.loads(body) if body.strip() else None
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
return {"ok": proc.returncode == 0 and 200 <= status < 300, "httpStatus": status, "json": parsed, "body": body, "stderr": tail_text(proc.stderr)}
|
||||
|
||||
def curl_api(method, path, bearer=None, payload=None):
|
||||
body = b"" if payload is None else json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
script = r'''
|
||||
set -eu
|
||||
method="$1"
|
||||
url="$2"
|
||||
token="\${3:-}"
|
||||
tmp="$(mktemp)"
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
cat > "$tmp"
|
||||
if [ -n "$token" ] && [ "$method" = GET ] && [ ! -s "$tmp" ]; then
|
||||
curl -sS --max-time 20 -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H "Authorization: Bearer $token" "$url"
|
||||
elif [ -n "$token" ]; then
|
||||
curl -sS --max-time 20 -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' -H "Authorization: Bearer $token" --data-binary @"$tmp" "$url"
|
||||
elif [ "$method" = GET ] && [ ! -s "$tmp" ]; then
|
||||
curl -sS --max-time 20 -w '\n__HTTP_CODE__:%{http_code}' -X "$method" "$url"
|
||||
else
|
||||
curl -sS --max-time 20 -w '\n__HTTP_CODE__:%{http_code}' -X "$method" -H 'Content-Type: application/json' --data-binary @"$tmp" "$url"
|
||||
fi
|
||||
'''
|
||||
if TARGET["runtimeMode"] == "host-docker":
|
||||
command = ["sh", "-c", script, "sh", method, f"http://127.0.0.1:{TARGET['hostDockerAppPort']}{path}", bearer or ""]
|
||||
else:
|
||||
command = ["kubectl", "-n", TARGET["namespace"], "exec", "-i", APP_POD, "--", "sh", "-c", script, "sh", method, f"http://127.0.0.1:8080{path}", bearer or ""]
|
||||
return parse_curl(run(command, body))
|
||||
|
||||
def data_of(response, label):
|
||||
parsed = response.get("json")
|
||||
code = parsed.get("code") if isinstance(parsed, dict) else None
|
||||
if response.get("ok") is not True or (code is not None and code != 0):
|
||||
message = parsed.get("message") if isinstance(parsed, dict) else tail_text(response.get("body"), 400)
|
||||
raise RuntimeError(f"{label} failed: http={response.get('httpStatus')} message={message}")
|
||||
return parsed.get("data") if isinstance(parsed, dict) and "data" in parsed else parsed
|
||||
|
||||
def items_of(data):
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict) and isinstance(data.get("items"), list):
|
||||
return data["items"]
|
||||
return []
|
||||
|
||||
def get(token, path, params=None, label="GET"):
|
||||
suffix = "?" + urlencode(params) if params else ""
|
||||
return data_of(curl_api("GET", path + suffix, bearer=token), label)
|
||||
|
||||
def login():
|
||||
email = config_value("ADMIN_EMAIL")
|
||||
password = secret_value("ADMIN_PASSWORD")
|
||||
if not email or not password:
|
||||
raise RuntimeError("Sub2API admin credentials are unavailable")
|
||||
data = data_of(curl_api("POST", "/api/v1/auth/login", payload={"email": email, "password": password}), "admin login")
|
||||
token = data.get("access_token") or data.get("token") if isinstance(data, dict) else None
|
||||
if not token:
|
||||
raise RuntimeError("admin login response has no token")
|
||||
return token
|
||||
|
||||
def find_named(items, selector, label):
|
||||
if selector is None:
|
||||
return items
|
||||
text = str(selector).strip().lower()
|
||||
numeric = int(text) if text.isdigit() else None
|
||||
matches = [item for item in items if (numeric is not None and item.get("id") == numeric) or str(item.get("id", "")).strip().lower() == text or str(item.get("name", "")).strip().lower() == text]
|
||||
if not matches:
|
||||
available = [f"{item.get('id')}:{item.get('name')}" for item in items]
|
||||
raise RuntimeError(f"unknown {label} {selector}; available={available}")
|
||||
return matches
|
||||
|
||||
def log_item(line):
|
||||
json_start = line.find("{")
|
||||
if json_start < 0:
|
||||
return None
|
||||
try:
|
||||
item = json.loads(line[json_start:])
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
prefix = line[:json_start].strip().split()
|
||||
item["_at"] = prefix[0] if prefix else None
|
||||
item["_message"] = " ".join(prefix[3:]) if len(prefix) >= 4 else " ".join(prefix[2:])
|
||||
return item
|
||||
|
||||
def epoch_of(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def account_names(token, platform):
|
||||
params = {"page": 1, "page_size": 500}
|
||||
if platform:
|
||||
params["platform"] = platform
|
||||
rows = items_of(get(token, "/api/v1/admin/accounts", params, "list accounts for monitor correlation"))
|
||||
return {item.get("id"): item.get("name") for item in rows if isinstance(item, dict) and item.get("id") is not None}
|
||||
|
||||
def correlate_record(token, record, monitor):
|
||||
checked_at = record.get("checked_at")
|
||||
checked_epoch = epoch_of(checked_at)
|
||||
latency_ms = record.get("latency_ms")
|
||||
if checked_epoch is None or not isinstance(latency_ms, (int, float)):
|
||||
return {"status": "unsupported", "reason": "native record has no usable checked_at/latency_ms", "candidates": []}
|
||||
expected_start = checked_epoch - latency_ms / 1000.0
|
||||
record_model = str(record.get("model") or "")
|
||||
request_rows = items_of(get(token, "/api/v1/admin/ops/requests", {"time_range": "24h", "page": 1, "page_size": 500, "kind": "all"}, "list request correlation candidates"))
|
||||
request_candidates = []
|
||||
for item in request_rows:
|
||||
request_id = item.get("request_id")
|
||||
created_epoch = epoch_of(item.get("created_at"))
|
||||
model_match = not record_model or str(item.get("model") or "") == record_model
|
||||
if not isinstance(request_id, str) or not request_id or created_epoch is None or not model_match:
|
||||
continue
|
||||
start_delta_ms = round(abs(created_epoch - expected_start) * 1000)
|
||||
if start_delta_ms <= 15000:
|
||||
request_candidates.append((request_id, item, start_delta_ms))
|
||||
proc = runtime_logs()
|
||||
names = account_names(token, monitor.get("provider"))
|
||||
grouped = {}
|
||||
candidate_ids = {item[0] for item in request_candidates}
|
||||
for line in proc.stdout.decode("utf-8", errors="replace").splitlines() if proc.returncode == 0 else []:
|
||||
item = log_item(line)
|
||||
if item is None:
|
||||
continue
|
||||
request_id = item.get("request_id")
|
||||
at_epoch = epoch_of(item.get("_at"))
|
||||
if not isinstance(request_id, str) or not request_id:
|
||||
continue
|
||||
if request_id not in candidate_ids and (at_epoch is None or at_epoch < expected_start - 15 or at_epoch > checked_epoch + 90):
|
||||
continue
|
||||
grouped.setdefault(request_id, []).append(item)
|
||||
candidates = []
|
||||
rows_by_id = {item[0]: (item[1], item[2]) for item in request_candidates}
|
||||
all_request_ids = set(grouped.keys()) | set(rows_by_id.keys())
|
||||
for request_id in all_request_ids:
|
||||
events = grouped.get(request_id, [])
|
||||
events.sort(key=lambda item: epoch_of(item.get("_at")) or 0)
|
||||
request_row, row_delta_ms = rows_by_id.get(request_id, ({}, None))
|
||||
first = events[0] if events else None
|
||||
models = [str(item.get("model")) for item in events if item.get("model")]
|
||||
first_epoch = epoch_of(first.get("_at")) if first is not None else epoch_of(request_row.get("created_at"))
|
||||
start_delta_ms = row_delta_ms if row_delta_ms is not None else round(abs((first_epoch or expected_start) - expected_start) * 1000)
|
||||
model_match = not record_model or record_model in models or str(request_row.get("model") or "") == record_model
|
||||
if start_delta_ms > 15000 or not model_match:
|
||||
continue
|
||||
failovers = [item for item in events if "upstream_failover_switching" in str(item.get("_message") or "")]
|
||||
select_failures = [item for item in events if "account_select_failed" in str(item.get("_message") or "")]
|
||||
upstream_errors = [item for item in events if "account_upstream_error" in str(item.get("_message") or "")]
|
||||
finals = [item for item in events if "http request completed" in str(item.get("_message") or "")]
|
||||
final = finals[-1] if finals else None
|
||||
account_ids = []
|
||||
for item in events:
|
||||
account_id = item.get("account_id")
|
||||
if isinstance(account_id, str) and account_id.isdigit():
|
||||
account_id = int(account_id)
|
||||
if isinstance(account_id, int) and account_id not in account_ids:
|
||||
account_ids.append(account_id)
|
||||
candidates.append({
|
||||
"requestId": request_id,
|
||||
"match": {"kind": "native-request-start-window", "startDeltaMs": start_delta_ms, "modelMatched": model_match},
|
||||
"firstAt": first.get("_at") if first is not None else request_row.get("created_at"),
|
||||
"lastAt": events[-1].get("_at") if events else request_row.get("created_at"),
|
||||
"model": next((item for item in models if item), request_row.get("model")),
|
||||
"apiKeyName": next((item.get("api_key_name") for item in events if item.get("api_key_name")), None),
|
||||
"accounts": [{"id": account_id, "name": names.get(account_id)} for account_id in account_ids] or ([{"id": request_row.get("account_id"), "name": request_row.get("account_name") or names.get(request_row.get("account_id"))}] if request_row.get("account_id") is not None else []),
|
||||
"upstreamErrors": [{"at": item.get("_at"), "accountId": item.get("account_id"), "error": item.get("error")} for item in upstream_errors],
|
||||
"failovers": [{"at": item.get("_at"), "accountId": item.get("account_id"), "upstreamStatus": item.get("upstream_status"), "switchCount": item.get("switch_count"), "maxSwitches": item.get("max_switches")} for item in failovers],
|
||||
"selectFailures": [{"at": item.get("_at"), "error": item.get("error"), "excludedAccountCount": item.get("excluded_account_count")} for item in select_failures],
|
||||
"final": {"at": request_row.get("created_at"), "statusCode": request_row.get("status_code") or request_row.get("upstream_status_code"), "accountId": request_row.get("account_id"), "latencyMs": request_row.get("duration_ms"), "path": request_row.get("path")} if final is None else {"at": final.get("_at"), "statusCode": final.get("status_code"), "accountId": final.get("account_id"), "latencyMs": final.get("latency_ms"), "path": final.get("path")},
|
||||
})
|
||||
candidates.sort(key=lambda item: item["match"]["startDeltaMs"])
|
||||
if not candidates:
|
||||
return {"status": "unsupported", "reason": "no unique native request_id is available in the record; bounded log correlation found no candidate", "candidates": []}
|
||||
best_delta = candidates[0]["match"]["startDeltaMs"]
|
||||
best = [item for item in candidates if item["match"]["startDeltaMs"] == best_delta]
|
||||
return {"status": "inferred" if len(best) == 1 else "ambiguous", "reason": "heuristic correlation by record start window and model; the native monitor record has no request_id" if len(best) == 1 else "multiple requests have the same nearest start time", "selected": best[0] if len(best) == 1 else None, "candidates": candidates[:5]}
|
||||
|
||||
def diagnosis(token):
|
||||
group_params = {"platform": OPTIONS["platform"]} if OPTIONS.get("platform") else None
|
||||
groups = items_of(get(token, "/api/v1/admin/groups/all", group_params, "list groups"))
|
||||
groups = find_named(groups, OPTIONS.get("group"), "group")
|
||||
snapshots = []
|
||||
selected = groups if groups else [{"id": None, "name": "all", "platform": OPTIONS.get("platform") or "all"}]
|
||||
for group in selected:
|
||||
params = {"time_range": OPTIONS["timeRange"]}
|
||||
platform = OPTIONS.get("platform") or group.get("platform")
|
||||
if platform and platform != "all":
|
||||
params["platform"] = platform
|
||||
if group.get("id"):
|
||||
params["group_id"] = group["id"]
|
||||
overview = get(token, "/api/v1/admin/ops/dashboard/overview", params, "ops dashboard overview")
|
||||
snapshots.append({"group": {"id": group.get("id"), "name": group.get("name"), "platform": platform or "all"}, "overview": overview})
|
||||
evidence = None
|
||||
diagnosis_id = OPTIONS.get("diagnosisId")
|
||||
if diagnosis_id:
|
||||
scope, _, category = diagnosis_id.partition(":")
|
||||
group = next((item["group"] for item in snapshots if ("g" + str(item["group"].get("id"))) == scope or scope == "all"), None)
|
||||
if group is None:
|
||||
raise RuntimeError(f"diagnosis id scope not found: {diagnosis_id}")
|
||||
params = {"time_range": OPTIONS["timeRange"], "page": 1, "page_size": 20}
|
||||
if group.get("id"):
|
||||
params["group_id"] = group["id"]
|
||||
if group.get("platform") and group.get("platform") != "all":
|
||||
params["platform"] = group["platform"]
|
||||
if category.startswith("upstream-"):
|
||||
endpoint = "/api/v1/admin/ops/upstream-errors"
|
||||
elif category.startswith("error-") or category.startswith("sla-"):
|
||||
endpoint = "/api/v1/admin/ops/request-errors"
|
||||
else:
|
||||
endpoint = "/api/v1/admin/ops/requests"
|
||||
params["kind"] = "all"
|
||||
params["sort"] = "duration_desc"
|
||||
evidence = {"diagnosisId": diagnosis_id, "records": items_of(get(token, endpoint, params, "diagnosis evidence"))}
|
||||
return {"groups": snapshots, "evidence": evidence}
|
||||
|
||||
def channels(token):
|
||||
monitors = items_of(get(token, "/api/v1/channel-monitors", None, "list channel monitors"))
|
||||
if OPTIONS.get("platform"):
|
||||
monitors = [item for item in monitors if str(item.get("provider", "")).lower() == OPTIONS["platform"].lower()]
|
||||
monitors = find_named(monitors, OPTIONS.get("channel"), "channel")
|
||||
output = []
|
||||
for monitor in monitors:
|
||||
detail = get(token, f"/api/v1/channel-monitors/{monitor['id']}/status", None, "channel monitor status")
|
||||
output.append({"summary": monitor, "detail": detail})
|
||||
history = None
|
||||
correlation = None
|
||||
if OPTIONS.get("channel") and output:
|
||||
monitor_id = output[0]["summary"]["id"]
|
||||
params = {"limit": 1000}
|
||||
if OPTIONS.get("model"):
|
||||
params["model"] = OPTIONS["model"]
|
||||
history = items_of(get(token, f"/api/v1/admin/channel-monitors/{monitor_id}/history", params, "channel monitor history"))
|
||||
record_id = OPTIONS.get("recordId")
|
||||
if record_id:
|
||||
selected = next((item for item in history if str(item.get("id")) == str(record_id)), None)
|
||||
if selected is not None:
|
||||
correlation = correlate_record(token, selected, output[0]["summary"])
|
||||
return {"channels": output, "history": history, "correlation": correlation}
|
||||
|
||||
try:
|
||||
token = login()
|
||||
data = diagnosis(token) if OPTIONS["action"] == "diagnosis" else channels(token)
|
||||
data["runtimeVersion"] = runtime_version()
|
||||
print(json.dumps({"ok": True, "sourceStatus": "available", "data": data}, ensure_ascii=False, separators=(",", ":")))
|
||||
except Exception as error:
|
||||
print(json.dumps({"ok": False, "sourceStatus": "unavailable", "error": str(error), "data": None}, ensure_ascii=False, separators=(",", ":")))
|
||||
sys.exit(1)
|
||||
PY
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
import type { RenderedCliResult } from "../output";
|
||||
|
||||
export function renderSub2ApiOps(result: Record<string, unknown>): RenderedCliResult {
|
||||
const action = String(result.action ?? "platform-infra-sub2api-ops");
|
||||
const lines = action.endsWith("diagnosis") ? renderDiagnosis(result) : renderChannels(result);
|
||||
return {
|
||||
ok: result.ok !== false,
|
||||
command: action.replaceAll("-", " "),
|
||||
renderedText: lines.join("\n"),
|
||||
contentType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
function renderDiagnosis(result: Record<string, unknown>): string[] {
|
||||
const source = record(result.source);
|
||||
const groups = arrayRecords(result.groups);
|
||||
const rows = groups.flatMap((group) => arrayRecords(group.diagnoses).map((item) => {
|
||||
const metric = record(item.metric);
|
||||
return [
|
||||
stringValue(item.id),
|
||||
stringValue(item.severity).toUpperCase(),
|
||||
`${stringValue(metric.value)} ${stringValue(metric.unit)}`.trim(),
|
||||
stringValue(item.message),
|
||||
`${stringValue(group.groupName)} / ${stringValue(group.platform)}`,
|
||||
];
|
||||
}));
|
||||
const lines = [
|
||||
"PLATFORM-INFRA SUB2API OPS DIAGNOSIS",
|
||||
...table(["ID", "SEVERITY", "VALUE", "DIAGNOSIS", "SCOPE"], rows),
|
||||
"",
|
||||
"SOURCE",
|
||||
...table(["FIELD", "VALUE"], [
|
||||
["status", stringValue(source.status)],
|
||||
["upstreamVersion", stringValue(source.upstreamVersion)],
|
||||
["metrics", stringValue(source.metricsEndpoint)],
|
||||
["advice", stringValue(source.projectionKind)],
|
||||
["nativeDiagnosisEndpoint", stringValue(record(source.nativeDiagnosisEndpoint).status)],
|
||||
]),
|
||||
];
|
||||
const selected = record(result.selectedDiagnosis);
|
||||
if (Object.keys(selected).length > 0) {
|
||||
const metric = record(selected.metric);
|
||||
lines.push(
|
||||
"",
|
||||
"DETAIL",
|
||||
...table(["FIELD", "VALUE"], [
|
||||
["id", stringValue(selected.id)],
|
||||
["category", stringValue(selected.category)],
|
||||
["severity", stringValue(selected.severity)],
|
||||
["metric", `${stringValue(metric.name)}=${stringValue(metric.value)} ${stringValue(metric.unit)}`.trim()],
|
||||
["impact", stringValue(selected.impact)],
|
||||
["nativeAdvice", stringValue(selected.advice, "-")],
|
||||
["inference", stringValue(selected.inferenceStatus)],
|
||||
]),
|
||||
);
|
||||
const evidence = record(result.evidence);
|
||||
const evidenceRows = arrayRecords(evidence.records).map((item) => [
|
||||
stringValue(item.requestId, "-"),
|
||||
stringValue(item.createdAt, "-"),
|
||||
stringValue(item.accountName, stringValue(item.accountId, "-")),
|
||||
stringValue(item.statusCode, "-"),
|
||||
stringValue(item.message, "-").replace(/\s+/gu, " ").slice(0, 80),
|
||||
]);
|
||||
lines.push("", "RELATED EVIDENCE", ...(evidenceRows.length === 0 ? ["-"] : table(["REQUEST_ID", "AT", "ACCOUNT", "STATUS", "SUMMARY"], evidenceRows)));
|
||||
}
|
||||
lines.push(
|
||||
"",
|
||||
"NEXT",
|
||||
" detail: bun scripts/cli.ts platform-infra sub2api ops diagnosis --id <diagnosis-id>",
|
||||
" json: bun scripts/cli.ts platform-infra sub2api ops diagnosis --json",
|
||||
"",
|
||||
`Evidence boundary: ${stringValue(source.evidenceBoundary)}`,
|
||||
"Disclosure: one-shot read-only snapshot; no foreground refresh, Secret, token, or full log output.",
|
||||
);
|
||||
if (result.error) lines.push(`Error: ${stringValue(result.error)}`);
|
||||
return lines;
|
||||
}
|
||||
|
||||
function renderChannels(result: Record<string, unknown>): string[] {
|
||||
const source = record(result.source);
|
||||
const channels = arrayRecords(result.channels);
|
||||
const rows = channels.map((channel) => {
|
||||
const primary = record(channel.primary);
|
||||
return [
|
||||
stringValue(channel.id),
|
||||
stringValue(channel.name),
|
||||
stringValue(channel.provider),
|
||||
stringValue(primary.model),
|
||||
stringValue(channel.status),
|
||||
stringValue(primary.latencyMs, "-"),
|
||||
stringValue(primary.pingLatencyMs, "-"),
|
||||
formatAvailability(primary.availability),
|
||||
stringValue(channel.recentRecordCount, "0"),
|
||||
stringValue(channel.collectedAt, "-"),
|
||||
];
|
||||
});
|
||||
const lines = [
|
||||
"PLATFORM-INFRA SUB2API OPS CHANNELS",
|
||||
...table(["ID", "CHANNEL", "PLATFORM", "MODEL", "STATUS", "LATENCY_MS", "PING_MS", "AVAIL", "RECORDS", "COLLECTED_AT"], rows),
|
||||
"",
|
||||
"SUMMARY",
|
||||
...table(["FIELD", "VALUE"], [
|
||||
["overallStatus", stringValue(result.overallStatus)],
|
||||
["window", stringValue(record(result.query).window)],
|
||||
["sourceStatus", stringValue(source.status)],
|
||||
["refreshRemaining", stringValue(record(source.refreshRemaining).status)],
|
||||
]),
|
||||
];
|
||||
const history = record(result.history);
|
||||
const historyRows = arrayRecords(history.records).map((item) => [
|
||||
stringValue(item.id),
|
||||
stringValue(item.model),
|
||||
stringValue(item.status),
|
||||
stringValue(item.latencyMs, "-"),
|
||||
stringValue(item.pingLatencyMs, "-"),
|
||||
stringValue(item.checkedAt, "-"),
|
||||
]);
|
||||
if (historyRows.length > 0) {
|
||||
lines.push(
|
||||
"",
|
||||
"PAST → NOW",
|
||||
...table(["ID", "MODEL", "STATUS", "LATENCY_MS", "PING_MS", "CHECKED_AT"], historyRows),
|
||||
`pageSize=${stringValue(history.pageSize)} moreAvailable=${stringValue(history.moreAvailable)} nextPageToken=${stringValue(history.nextPageToken)}`,
|
||||
);
|
||||
}
|
||||
const selectedRecord = record(history.selectedRecord);
|
||||
if (Object.keys(selectedRecord).length > 0) {
|
||||
const timeout = record(selectedRecord.timeout);
|
||||
const correlation = record(selectedRecord.correlation);
|
||||
const final = record(correlation.final);
|
||||
const accounts = arrayRecords(correlation.accounts);
|
||||
const failovers = arrayRecords(correlation.failovers);
|
||||
const selectFailures = arrayRecords(correlation.selectFailures);
|
||||
lines.push(
|
||||
"",
|
||||
"RECORD",
|
||||
...table(["FIELD", "VALUE"], [
|
||||
["id", stringValue(selectedRecord.id)],
|
||||
["model", stringValue(selectedRecord.model)],
|
||||
["status", stringValue(selectedRecord.status)],
|
||||
["latencyMs", stringValue(selectedRecord.latencyMs)],
|
||||
["pingLatencyMs", stringValue(selectedRecord.pingLatencyMs)],
|
||||
["checkedAt", stringValue(selectedRecord.checkedAt)],
|
||||
["message", stringValue(selectedRecord.message)],
|
||||
["runtimeVersion", stringValue(timeout.runtimeVersion)],
|
||||
["sourceReferenceVersion", stringValue(timeout.sourceReferenceVersion)],
|
||||
["runtimePolicyStatus", stringValue(timeout.runtimePolicyStatus)],
|
||||
["sourceResponseHeaderMs", stringValue(timeout.responseHeaderTimeoutMs)],
|
||||
["sourceTotalRequestMs", stringValue(timeout.totalRequestTimeoutMs)],
|
||||
["remainingAfterHeaderMs", stringValue(timeout.remainingAfterResponseHeaderMs)],
|
||||
["correlationStatus", stringValue(correlation.status)],
|
||||
["requestId", stringValue(correlation.requestId)],
|
||||
["accounts", accounts.map((item) => `${stringValue(item.id)}:${stringValue(item.name)}`).join(",") || "-"],
|
||||
["failoverCount", String(failovers.length)],
|
||||
["selectFailureCount", String(selectFailures.length)],
|
||||
["finalStatus", stringValue(final.statusCode)],
|
||||
["finalLatencyMs", stringValue(final.latencyMs)],
|
||||
["timeoutSufficient", stringValue(correlation.sufficientForObservedCompletion)],
|
||||
["sufficiencyStatus", stringValue(correlation.sufficiencyStatus)],
|
||||
]),
|
||||
);
|
||||
if (correlation.traceNext) lines.push(`Trace: ${stringValue(correlation.traceNext)}`);
|
||||
if (correlation.reason) lines.push(`Correlation: ${stringValue(correlation.reason)}`);
|
||||
}
|
||||
lines.push(
|
||||
"",
|
||||
"NEXT",
|
||||
" channel: bun scripts/cli.ts platform-infra sub2api ops channels --channel <id-or-name>",
|
||||
" window: bun scripts/cli.ts platform-infra sub2api ops channels --window 7d|15d|30d",
|
||||
" older page: bun scripts/cli.ts platform-infra sub2api ops channels --channel <id-or-name> --page-token <record-id>",
|
||||
" record: bun scripts/cli.ts platform-infra sub2api ops channels --channel <id-or-name> --record <record-id>",
|
||||
" json: bun scripts/cli.ts platform-infra sub2api ops channels --json",
|
||||
"",
|
||||
"Disclosure: one-shot read-only snapshot; native response has no refresh countdown, and the CLI does not simulate one.",
|
||||
);
|
||||
if (result.error) lines.push(`Error: ${stringValue(result.error)}`);
|
||||
return lines;
|
||||
}
|
||||
|
||||
function formatAvailability(value: unknown): string {
|
||||
return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(2)}%` : stringValue(value, "-");
|
||||
}
|
||||
|
||||
function table(headers: string[], rows: string[][]): string[] {
|
||||
if (rows.length === 0) return ["-"];
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => visibleLength(row[index] ?? ""))));
|
||||
const format = (row: string[]): string => row.map((cell, index) => `${cell}${" ".repeat(Math.max(0, widths[index]! - visibleLength(cell)))}`).join(" ").trimEnd();
|
||||
return [format(headers), format(widths.map((width) => "-".repeat(width))), ...rows.map(format)];
|
||||
}
|
||||
|
||||
function visibleLength(value: string): number {
|
||||
return [...value].length;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record) : [];
|
||||
}
|
||||
|
||||
function stringValue(value: unknown, fallback = "-"): string {
|
||||
if (typeof value === "string") return value.length > 0 ? value : fallback;
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
return fallback;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
export type Sub2ApiOpsAction = "diagnosis" | "channels";
|
||||
|
||||
export interface Sub2ApiOpsOptions {
|
||||
action: Sub2ApiOpsAction;
|
||||
targetId: string;
|
||||
json: boolean;
|
||||
platform: string | null;
|
||||
group: string | null;
|
||||
diagnosisId: string | null;
|
||||
timeRange: "5m" | "30m" | "1h" | "6h" | "24h";
|
||||
channel: string | null;
|
||||
model: string | null;
|
||||
window: "7d" | "15d" | "30d";
|
||||
pageToken: string | null;
|
||||
recordId: string | null;
|
||||
}
|
||||
|
||||
export interface Sub2ApiOpsTarget {
|
||||
id: string;
|
||||
route: string;
|
||||
namespace: string;
|
||||
runtimeMode: "k3s" | "host-docker";
|
||||
hostDockerAppPort: number | null;
|
||||
hostDockerEnvPath: string | null;
|
||||
appSecretName: string;
|
||||
}
|
||||
@@ -417,10 +417,13 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api apply [--target G14|D601] --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api status [--target G14|D601] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api validate [--target G14|D601] [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra sub2api ops diagnosis [--target PK01] [--platform <name>] [--group <id-or-name>] [--time-range 5m|30m|1h|6h|24h] [--id <diagnosis-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api ops channels [--target PK01] [--platform <name>] [--channel <id-or-name>] [--model <name>] [--window 7d|15d|30d] [--page-token <record-id>|--record <record-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool plan",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2rank plan [--target NC01]",
|
||||
@@ -503,6 +506,7 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool validate",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id <requestId>",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool feedback --user <email-or-id> [--window 30m] [--page-token <token>|--request-id <client-or-internal-id>] [--json]",
|
||||
"bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status",
|
||||
],
|
||||
module: "scripts/src/platform-infra-sub2api-codex.ts",
|
||||
|
||||
@@ -108,6 +108,10 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin
|
||||
return options.full || options.raw ? result : renderSub2ApiStatus(result);
|
||||
}
|
||||
if (action === "validate") return await validate(config, parseDisclosureOptions(args.slice(2)));
|
||||
if (action === "ops") {
|
||||
const { runSub2ApiOpsCommand } = await import("../platform-infra-sub2api-ops");
|
||||
return await runSub2ApiOpsCommand(config, args.slice(2));
|
||||
}
|
||||
if (action === "codex-pool") {
|
||||
const { runCodexPoolCommand } = await import("../platform-infra-sub2api-codex");
|
||||
return await runCodexPoolCommand(config, args.slice(2));
|
||||
|
||||
Reference in New Issue
Block a user