Codex
245 lines
10 KiB
TypeScript
245 lines
10 KiB
TypeScript
interface TimingSample {
|
|
label: string;
|
|
method: string;
|
|
url: string;
|
|
ok: boolean;
|
|
status: number;
|
|
durationMs: number;
|
|
bytes: number;
|
|
error: string | null;
|
|
}
|
|
|
|
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
|
|
|
|
interface CandidateTask {
|
|
id: string;
|
|
status: string;
|
|
stepCount: number | null;
|
|
updatedAt: string;
|
|
}
|
|
|
|
interface TraceCandidate {
|
|
seq: number | null;
|
|
total: number | null;
|
|
durationMs: number;
|
|
error: string | null;
|
|
}
|
|
|
|
interface PerfCheck {
|
|
name: string;
|
|
ok: boolean;
|
|
valueMs: number;
|
|
budgetMs: number;
|
|
hard: boolean;
|
|
}
|
|
|
|
export {};
|
|
|
|
function envNumber(name: string, fallback: number): number {
|
|
const raw = process.env[name];
|
|
if (raw === undefined || raw.length === 0) return fallback;
|
|
const value = Number(raw);
|
|
if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
|
|
return Math.floor(value);
|
|
}
|
|
|
|
function baseUrl(): string {
|
|
return (process.env.CI_CODE_QUEUE_URL ?? "http://code-queue-ci-read.unidesk-ci.svc.cluster.local:4222").replace(/\/+$/u, "");
|
|
}
|
|
|
|
function terminalStatus(status: string): boolean {
|
|
return status === "succeeded" || status === "failed" || status === "canceled";
|
|
}
|
|
|
|
async function fetchSample(label: string, url: string, timeoutMs = 30_000): Promise<TimingSample> {
|
|
const started = performance.now();
|
|
try {
|
|
const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
const text = await response.text();
|
|
return {
|
|
label,
|
|
method: "GET",
|
|
url,
|
|
ok: response.ok,
|
|
status: response.status,
|
|
durationMs: Math.round((performance.now() - started) * 10) / 10,
|
|
bytes: text.length,
|
|
error: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
label,
|
|
method: "GET",
|
|
url,
|
|
ok: false,
|
|
status: 0,
|
|
durationMs: Math.round((performance.now() - started) * 10) / 10,
|
|
bytes: 0,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
function percentile(values: number[], percentileValue: number): number {
|
|
if (values.length === 0) return 0;
|
|
const sorted = values.slice().sort((left, right) => left - right);
|
|
if (percentileValue <= 0) return sorted[0] ?? 0;
|
|
if (percentileValue >= 100) return sorted[sorted.length - 1] ?? 0;
|
|
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1));
|
|
return sorted[index] ?? 0;
|
|
}
|
|
|
|
async function candidateTasks(url: string): Promise<CandidateTask[]> {
|
|
const response = await fetch(`${url}/api/tasks/overview?limit=48&transcriptLimit=0&compact=1&selected=0&includeActive=0&stats=0&skipTrace=1`, {
|
|
signal: AbortSignal.timeout(30_000),
|
|
});
|
|
const body = await response.json() as { tasks?: Array<{ id?: string; status?: string; stepCount?: number; llmStepCount?: number; updatedAt?: string }> };
|
|
const tasks = (body.tasks ?? [])
|
|
.map((task): CandidateTask | null => {
|
|
if (typeof task.id !== "string" || task.id.length === 0) return null;
|
|
const stepCount = Number(task.stepCount ?? task.llmStepCount);
|
|
return {
|
|
id: task.id,
|
|
status: typeof task.status === "string" ? task.status : "",
|
|
stepCount: Number.isFinite(stepCount) && stepCount >= 0 ? Math.floor(stepCount) : null,
|
|
updatedAt: typeof task.updatedAt === "string" ? task.updatedAt : "",
|
|
};
|
|
})
|
|
.filter((task): task is CandidateTask => task !== null);
|
|
const ordered = [
|
|
...tasks.filter((task) => terminalStatus(task.status) && (task.stepCount ?? 0) > 0 && (task.stepCount ?? 0) <= 300),
|
|
...tasks.filter((task) => terminalStatus(task.status) && ((task.stepCount ?? 0) === 0 || task.stepCount === null)),
|
|
...tasks.filter((task) => terminalStatus(task.status)),
|
|
...tasks.filter((task) => !terminalStatus(task.status) && task.status !== "queued" && task.status !== "running" && task.status !== "judging"),
|
|
...tasks,
|
|
];
|
|
const seen = new Set<string>();
|
|
return ordered.filter((task) => {
|
|
if (seen.has(task.id)) return false;
|
|
seen.add(task.id);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
async function traceSeq(url: string, taskId: string, timeoutMs: number): Promise<TraceCandidate> {
|
|
const started = performance.now();
|
|
try {
|
|
const response = await fetch(`${url}/api/tasks/${encodeURIComponent(taskId)}/trace-steps?tail=1&limit=1`, {
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
});
|
|
const body = await response.json() as { total?: number; steps?: Array<{ seq?: number }> };
|
|
const durationMs = Math.round((performance.now() - started) * 10) / 10;
|
|
if (!response.ok) return { seq: null, total: null, durationMs, error: `status=${response.status}` };
|
|
const seq = body.steps?.find((step) => Number.isFinite(Number(step.seq)))?.seq;
|
|
return {
|
|
seq: Number.isFinite(Number(seq)) ? Number(seq) : null,
|
|
total: Number.isFinite(Number(body.total)) ? Number(body.total) : null,
|
|
durationMs,
|
|
error: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
seq: null,
|
|
total: null,
|
|
durationMs: Math.round((performance.now() - started) * 10) / 10,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
async function traceTarget(url: string): Promise<{ taskId: string; skippedTaskIds: string[]; selection: JsonValue }> {
|
|
const tasks = await candidateTasks(url);
|
|
if (tasks.length === 0) throw new Error("Code Queue CI perf could not find a task id in the production PostgreSQL task table");
|
|
const target = tasks[0];
|
|
if (target === undefined) throw new Error("Code Queue CI perf could not select a task from the production PostgreSQL task table");
|
|
return { taskId: target.id, skippedTaskIds: tasks.slice(1).map((task) => task.id), selection: target as unknown as JsonValue };
|
|
}
|
|
|
|
async function measureFirstPaint(url: string): Promise<Record<string, unknown>> {
|
|
const sample = await fetchSample("code-queue-read-first-paint-proxy", `${url}/api/tasks/overview?limit=12&transcriptLimit=1&compact=1&selected=0&includeActive=0&stats=0&skipTrace=1`, 60_000);
|
|
return {
|
|
ok: sample.ok,
|
|
url: sample.url,
|
|
firstPaintMs: sample.durationMs,
|
|
apiTimings: [sample],
|
|
consoleErrors: [],
|
|
note: "Code Queue service is API-only in k3s; this measures the first overview payload used by the frontend Code Queue page.",
|
|
};
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
const url = baseUrl();
|
|
const budgets = {
|
|
firstPaintMs: envNumber("FIRST_PAINT_BUDGET_MS", 2000),
|
|
traceSummaryMs: envNumber("TRACE_SUMMARY_BUDGET_MS", 10_000),
|
|
traceStepsMs: envNumber("TRACE_STEPS_BUDGET_MS", 900),
|
|
traceStepDetailMs: envNumber("TRACE_STEP_DETAIL_BUDGET_MS", 700),
|
|
overviewP95Ms: envNumber("OVERVIEW_P95_BUDGET_MS", 900),
|
|
};
|
|
const health = await fetchSample("health", `${url}/health`);
|
|
if (!health.ok) throw new Error(`Code Queue CI read health failed: ${JSON.stringify(health)}`);
|
|
const target = await traceTarget(url);
|
|
const { taskId } = target;
|
|
const firstPaint = await measureFirstPaint(url);
|
|
const traceSummary = await fetchSample("trace-summary", `${url}/api/tasks/${encodeURIComponent(taskId)}/trace-summary`);
|
|
const overviewSamples: TimingSample[] = [];
|
|
for (let index = 0; index < 10; index += 1) {
|
|
overviewSamples.push(await fetchSample("overview", `${url}/api/tasks/overview?limit=12&transcriptLimit=1&compact=1&selected=0&includeActive=0&stats=0&skipTrace=1&__ci=${Date.now()}-${index}`));
|
|
}
|
|
const traceProbe = await traceSeq(url, taskId, Math.max(10_000, Math.min(30_000, budgets.traceStepsMs)));
|
|
const seq = traceProbe.seq ?? 0;
|
|
const traceSteps = await fetchSample("trace-steps", `${url}/api/tasks/${encodeURIComponent(taskId)}/trace-steps?tail=1&limit=1`, Math.max(10_000, Math.min(30_000, budgets.traceStepsMs)));
|
|
const traceStepDetail = seq > 0
|
|
? await fetchSample("trace-step-detail", `${url}/api/tasks/${encodeURIComponent(taskId)}/trace-step?seq=${encodeURIComponent(String(seq))}`, Math.max(10_000, Math.min(30_000, budgets.traceStepDetailMs)))
|
|
: {
|
|
label: "trace-step-detail",
|
|
method: "GET",
|
|
url: `${url}/api/tasks/${encodeURIComponent(taskId)}/trace-step?seq=0`,
|
|
ok: false,
|
|
status: 0,
|
|
durationMs: 0,
|
|
bytes: 0,
|
|
error: traceProbe.error ?? "trace step seq unavailable",
|
|
};
|
|
const overviewSuccessful = overviewSamples.filter((sample) => sample.ok).map((sample) => sample.durationMs);
|
|
const overviewP95Ms = Math.round(percentile(overviewSuccessful, 95) * 10) / 10;
|
|
const firstPaintMs = Number((firstPaint as { firstPaintMs?: number }).firstPaintMs ?? 0);
|
|
const checks: PerfCheck[] = [
|
|
{ name: "first-paint", ok: firstPaintMs <= budgets.firstPaintMs, valueMs: firstPaintMs, budgetMs: budgets.firstPaintMs, hard: true },
|
|
{ name: "trace-summary", ok: traceSummary.ok && traceSummary.durationMs <= budgets.traceSummaryMs, valueMs: traceSummary.durationMs, budgetMs: budgets.traceSummaryMs, hard: true },
|
|
{ name: "overview-p95", ok: overviewSamples.every((sample) => sample.ok) && overviewP95Ms <= budgets.overviewP95Ms, valueMs: overviewP95Ms, budgetMs: budgets.overviewP95Ms, hard: true },
|
|
{ name: "trace-steps", ok: traceSteps.ok && traceSteps.durationMs <= budgets.traceStepsMs, valueMs: traceSteps.durationMs, budgetMs: budgets.traceStepsMs, hard: false },
|
|
{ name: "trace-step-detail", ok: traceStepDetail.ok && traceStepDetail.durationMs <= budgets.traceStepDetailMs, valueMs: traceStepDetail.durationMs, budgetMs: budgets.traceStepDetailMs, hard: false },
|
|
];
|
|
const hardChecks = checks.filter((check) => check.hard);
|
|
const result = {
|
|
ok: hardChecks.every((check) => check.ok),
|
|
measuredAt: new Date().toISOString(),
|
|
url,
|
|
taskId,
|
|
seq,
|
|
skippedTaskIds: target.skippedTaskIds,
|
|
selection: target.selection,
|
|
budgets,
|
|
checks,
|
|
diagnostics: {
|
|
nonBlockingChecks: checks.filter((check) => !check.hard).map((check) => check.name),
|
|
traceProbe,
|
|
},
|
|
health,
|
|
firstPaint,
|
|
traceSummary,
|
|
traceSteps,
|
|
traceStepDetail,
|
|
overview: {
|
|
p50Ms: Math.round(percentile(overviewSuccessful, 50) * 10) / 10,
|
|
p95Ms: overviewP95Ms,
|
|
samples: overviewSamples,
|
|
},
|
|
};
|
|
console.log(JSON.stringify(result, null, 2));
|
|
if (!result.ok) process.exitCode = 1;
|
|
}
|
|
|
|
await main();
|