fix: harden pgdata backup scheduler recovery

This commit is contained in:
Codex
2026-05-20 13:04:06 +00:00
parent b8cc847e48
commit 674cb59e85
7 changed files with 519 additions and 11 deletions
+60
View File
@@ -0,0 +1,60 @@
import { scheduleRetryRunObservation, scheduleRunObservation, scheduleRunsScope } from "./src/schedules";
type JsonRecord = Record<string, unknown>;
function assertCondition(condition: unknown, message: string, detail: JsonRecord = {}): void {
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
}
export function runScheduleCliContract(): JsonRecord {
const global = scheduleRunsScope(["runs", "--limit", "50"]);
assertCondition(global.scheduleId === null, "global schedule runs must not treat --limit value as schedule id", global);
assertCondition(global.limit === 50, "global schedule runs limit should be preserved", global);
const scoped = scheduleRunsScope(["runs", "unidesk-pgdata-baidu-daily", "--limit", "5"]);
assertCondition(scoped.scheduleId === "unidesk-pgdata-baidu-daily", "schedule-specific runs should preserve schedule id", scoped);
assertCondition(scoped.limit === 5, "schedule-specific runs limit should be preserved", scoped);
let numericScheduleRejected = false;
try {
scheduleRunsScope(["runs", "50"]);
} catch (error) {
numericScheduleRejected = String((error as Error).message).includes("schedule runs --limit N");
}
assertCondition(numericScheduleRejected, "numeric positional schedule id should point operators to global --limit syntax");
const timeoutObservation = scheduleRunObservation(
"unidesk-pgdata-baidu-daily",
{ body: { run: { id: "schedrun_new" } } },
{ ok: false, timedOut: true, timeoutMs: 1 },
);
assertCondition(timeoutObservation.newRunId === "schedrun_new", "schedule run output must expose newRunId even when wait times out", timeoutObservation);
assertCondition(String(timeoutObservation.observeCommand).includes("schedule runs unidesk-pgdata-baidu-daily --limit 20"), "schedule run output must expose observeCommand", timeoutObservation);
const retryObservation = scheduleRetryRunObservation("schedrun_failed", {
body: {
originalRunId: "schedrun_failed",
scheduleId: "unidesk-pgdata-baidu-daily",
newRunId: "schedrun_retry",
},
});
assertCondition(retryObservation.originalRunId === "schedrun_failed", "retry-run output must preserve originalRunId", retryObservation);
assertCondition(retryObservation.scheduleId === "unidesk-pgdata-baidu-daily", "retry-run output must expose scheduleId", retryObservation);
assertCondition(retryObservation.newRunId === "schedrun_retry", "retry-run output must expose newRunId", retryObservation);
assertCondition(String(retryObservation.observeCommand).includes("schedule runs unidesk-pgdata-baidu-daily --limit 20"), "retry-run output must expose observeCommand", retryObservation);
return {
ok: true,
checks: [
"global schedule runs limit parsing",
"schedule-specific runs parsing",
"numeric positional guard",
"run wait timeout observation",
"retry-run observation",
],
};
}
if (import.meta.main) {
process.stdout.write(`${JSON.stringify(runScheduleCliContract(), null, 2)}\n`);
}
+3
View File
@@ -284,6 +284,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
fileItem("scripts/src/ci.ts"),
fileItem("scripts/src/e2e.ts"),
fileItem("scripts/code-queue-prompt-observation-test.ts"),
fileItem("scripts/schedule-cli-contract-test.ts"),
fileItem("scripts/src/artifact-registry.ts"),
fileItem("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml"),
);
@@ -301,6 +302,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
items.push(commandItem("code-queue:control-plane-split-brain-diagnostics", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:control-plane-split-brain-diagnostics"], 30_000));
items.push(commandItem("code-queue:oa-publisher-degraded-visible", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:oa-publisher-degraded-visible"], 30_000));
items.push(commandItem("baidu-netdisk:artifact-guard-contract", ["bun", "scripts/baidu-netdisk-artifact-guard-contract-test.ts"], 30_000));
items.push(commandItem("schedule:cli-contract", ["bun", "scripts/schedule-cli-contract-test.ts"], 30_000));
} else {
items.push(skippedItem("typescript:scripts", "scripts TypeScript typecheck is opt-in", "--scripts-typecheck or --full"));
items.push(skippedItem("code-queue:prompt-observation-contract", "prompt observation contract is opt-in with script checks", "--scripts-typecheck or --full"));
@@ -308,6 +310,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
items.push(skippedItem("code-queue:trace-summary-contract", "Code Queue trace summary contract is opt-in with script checks", "--scripts-typecheck or --full"));
items.push(skippedItem("code-queue:liveness-diagnostics-fixtures", "Code Queue liveness diagnostics fixtures are opt-in with script checks", "--scripts-typecheck or --full"));
items.push(skippedItem("baidu-netdisk:artifact-guard-contract", "Baidu Netdisk artifact guard contract is opt-in with script checks", "--scripts-typecheck or --full"));
items.push(skippedItem("schedule:cli-contract", "Schedule CLI contract is opt-in with script checks", "--scripts-typecheck or --full"));
}
if (options.logs) {
items.push(unifiedLogRotationItem());
+2 -1
View File
@@ -191,13 +191,14 @@ function scheduleHelp(): unknown {
usage: [
"bun scripts/cli.ts schedule list",
"bun scripts/cli.ts schedule get <id>",
"bun scripts/cli.ts schedule runs --limit N",
"bun scripts/cli.ts schedule runs [scheduleId] [--limit N]",
"bun scripts/cli.ts schedule run <id> [--wait-ms N]",
"bun scripts/cli.ts schedule retry-run <failedRunId>",
"bun scripts/cli.ts schedule delete <id>",
"bun scripts/cli.ts schedule upsert-pgdata-backup [--time HH:MM] [--remote-base path]",
],
description: "Manage backend-core scheduled tasks and run history through the private core API.",
description: "Manage backend-core scheduled tasks and run history through the private core API. Global runs use schedule runs --limit N; schedule-specific runs pass a non-numeric schedule id.",
};
}
+68 -6
View File
@@ -35,9 +35,40 @@ function terminalRunStatus(status: unknown): boolean {
return ["succeeded", "failed", "skipped"].includes(String(status || ""));
}
function isPlainNumeric(value: string): boolean {
return /^[0-9]+$/.test(value);
}
export function scheduleRunsScope(args: string[]): { scheduleId: string | null; limit: number } {
const limit = numberOption(args, "--limit", 50);
let scheduleId: string | null = null;
for (let index = 1; index < args.length; index += 1) {
const arg = args[index];
if (arg === "--limit") {
index += 1;
continue;
}
if (arg.startsWith("--")) {
throw new Error(`unsupported schedule runs option: ${arg}`);
}
if (scheduleId !== null) {
throw new Error("schedule runs accepts at most one schedule id");
}
if (isPlainNumeric(arg)) {
throw new Error("schedule runs <scheduleId> requires a non-numeric schedule id; use schedule runs --limit N for the global history view");
}
scheduleId = arg;
}
return {
scheduleId,
limit,
};
}
async function waitForScheduleRun(scheduleId: string, runId: string, timeoutMs: number): Promise<unknown> {
const started = Date.now();
let latest: unknown = null;
const observeCommand = `bun scripts/cli.ts schedule runs ${scheduleId} --limit 20`;
while (Date.now() - started < timeoutMs) {
latest = coreInternalFetch(`/api/schedules/${encodeURIComponent(scheduleId)}/runs?limit=20`);
const runs = responseBody(latest).runs;
@@ -47,7 +78,33 @@ async function waitForScheduleRun(scheduleId: string, runId: string, timeoutMs:
}
await Bun.sleep(2000);
}
return { ok: false, timeoutMs, latest };
return { ok: false, timedOut: true, timeoutMs, runId, scheduleId, observeCommand, latest };
}
export function scheduleRunObservation(scheduleId: string, response: unknown, wait: unknown): Record<string, unknown> {
const run = responseBody(response).run as { id?: unknown } | undefined;
const runId = typeof run?.id === "string" ? run.id : null;
return {
trigger: response,
originalRunId: run?.id ?? null,
scheduleId,
newRunId: runId,
observeCommand: `bun scripts/cli.ts schedule runs ${scheduleId} --limit 20`,
wait,
};
}
export function scheduleRetryRunObservation(originalRunId: string, response: unknown): Record<string, unknown> {
const body = responseBody(response);
const scheduleId = typeof body.scheduleId === "string" ? body.scheduleId : null;
const newRunId = typeof body.newRunId === "string" ? body.newRunId : null;
return {
originalRunId: String(body.originalRunId ?? originalRunId),
scheduleId,
newRunId,
observeCommand: scheduleId !== null && newRunId !== null ? `bun scripts/cli.ts schedule runs ${scheduleId} --limit 20` : null,
response,
};
}
function pgdataBackupScheduleBody(config: UniDeskConfig, args: string[]): Record<string, unknown> {
@@ -84,9 +141,9 @@ export async function runScheduleCommand(config: UniDeskConfig, args: string[]):
return coreInternalFetch(`/api/schedules/${encodeURIComponent(idArg)}`);
}
if (action === "runs") {
const limit = numberOption(args, "--limit", 50);
return idArg
? coreInternalFetch(`/api/schedules/${encodeURIComponent(idArg)}/runs?limit=${limit}`)
const { scheduleId, limit } = scheduleRunsScope(args);
return scheduleId
? coreInternalFetch(`/api/schedules/${encodeURIComponent(scheduleId)}/runs?limit=${limit}`)
: coreInternalFetch(`/api/schedules/runs?limit=${limit}`);
}
if (action === "delete") {
@@ -99,12 +156,17 @@ export async function runScheduleCommand(config: UniDeskConfig, args: string[]):
const run = responseBody(response).run as { id?: unknown } | undefined;
const waitMs = numberOption(args, "--wait-ms", 0);
const wait = waitMs > 0 && typeof run?.id === "string" ? await waitForScheduleRun(idArg, run.id, waitMs) : null;
return { trigger: response, wait };
return scheduleRunObservation(idArg, response, wait);
}
if (action === "retry-run") {
if (!idArg) throw new Error("schedule retry-run requires failed run id");
const response = coreInternalFetch(`/api/schedules/runs/${encodeURIComponent(idArg)}/retry`, { method: "POST", body: {} });
return scheduleRetryRunObservation(idArg, response);
}
if (action === "upsert-pgdata-backup" || action === "pgdata-backup") {
const body = pgdataBackupScheduleBody(config, args);
const id = String(body.id || "unidesk-pgdata-baidu-daily");
return coreInternalFetch(`/api/schedules/${encodeURIComponent(id)}`, { method: "PUT", body });
}
throw new Error("schedule command must be one of: list, get, runs, run, delete, upsert-pgdata-backup");
throw new Error("schedule command must be one of: list, get, runs, run, retry-run, delete, upsert-pgdata-backup");
}