fix: surface pgdata backup read-only blockers
This commit is contained in:
+12
-2
@@ -94,6 +94,10 @@ function dispatchPayload(command: DebugDispatchCommand): Record<string, unknown>
|
||||
return { source: "cli-debug", ...explicit };
|
||||
}
|
||||
|
||||
function resultOk(result: unknown): boolean {
|
||||
return typeof result !== "object" || result === null || !("ok" in result) || (result as { ok?: unknown }).ok !== false;
|
||||
}
|
||||
|
||||
function latestJobId(): string {
|
||||
const jobs = listJobs();
|
||||
if (jobs.length === 0) throw new Error("No jobs found");
|
||||
@@ -238,7 +242,10 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (top === "microservice") {
|
||||
emitJson(commandName, await runMicroserviceCommand(config, args.slice(1)));
|
||||
const result = await runMicroserviceCommand(config, args.slice(1));
|
||||
const ok = resultOk(result);
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -261,7 +268,10 @@ async function main(): Promise<void> {
|
||||
}
|
||||
|
||||
if (top === "schedule") {
|
||||
emitJson(commandName, await runScheduleCommand(config, args.slice(1)));
|
||||
const result = await runScheduleCommand(config, args.slice(1));
|
||||
const ok = resultOk(result);
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { scheduleRetryRunObservation, scheduleRunObservation, scheduleRunsScope } from "./src/schedules";
|
||||
import { backendCoreUnavailableDiagnostic } from "./src/microservices";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
@@ -43,6 +44,31 @@ export function runScheduleCliContract(): JsonRecord {
|
||||
assertCondition(retryObservation.newRunId === "schedrun_retry", "retry-run output must expose newRunId", retryObservation);
|
||||
assertCondition(String(retryObservation.observeCommand).includes("schedule runs unidesk-pgdata-baidu-daily --limit 20"), "retry-run output must expose observeCommand", retryObservation);
|
||||
|
||||
const unavailable = backendCoreUnavailableDiagnostic({
|
||||
exitCode: 1,
|
||||
stdoutTail: "",
|
||||
stderrTail: "Error response from daemon: No such container: unidesk-backend-core\n",
|
||||
relatedContainers: [
|
||||
{ name: "unidesk-backend-core.verify-20260520T153456Z", image: "unidesk-backend-core:latest", status: "Exited (255)" },
|
||||
{ name: "unidesk-database.verify-20260520T153456Z", image: "postgres:16-alpine", status: "Exited (255)" },
|
||||
],
|
||||
envPath: "/tmp/docker-compose.env",
|
||||
baiduSecretPresence: {
|
||||
envPath: "/tmp/docker-compose.env",
|
||||
exists: true,
|
||||
keys: {
|
||||
UNIDESK_BAIDU_NETDISK_CLIENT_ID: { present: true, nonEmpty: false },
|
||||
UNIDESK_BAIDU_NETDISK_CLIENT_SECRET: { present: true, nonEmpty: false },
|
||||
UNIDESK_BAIDU_NETDISK_TOKEN_KEY: { present: true, nonEmpty: false },
|
||||
},
|
||||
},
|
||||
});
|
||||
assertCondition(unavailable.ok === false, "backend-core unavailable diagnostic must be a failed result", unavailable);
|
||||
assertCondition(unavailable.failureKind === "target-stack-not-running", "backend-core unavailable diagnostic must classify target stack absence", unavailable);
|
||||
assertCondition((unavailable.targetStack as JsonRecord).verifyOnlyObserved === true, "backend-core unavailable diagnostic must expose verify-only evidence", unavailable);
|
||||
assertCondition(Array.isArray(unavailable.authorizationRequiredForRecovery), "backend-core unavailable diagnostic must list authorization-gated recovery actions", unavailable);
|
||||
assertCondition(Array.isArray(unavailable.readOnlyCommands), "backend-core unavailable diagnostic must list read-only observation commands", unavailable);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
checks: [
|
||||
@@ -51,6 +77,7 @@ export function runScheduleCliContract(): JsonRecord {
|
||||
"numeric positional guard",
|
||||
"run wait timeout observation",
|
||||
"retry-run observation",
|
||||
"target stack unavailable diagnostic",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { runCommand } from "./command";
|
||||
import { type UniDeskConfig, repoRoot } from "./config";
|
||||
import { jsonByteLength, previewJson } from "./preview";
|
||||
@@ -27,11 +28,136 @@ function dockerCoreFetchCommand(path: string, init?: { method?: string; body?: u
|
||||
return ["docker", "exec", "unidesk-backend-core", "sh", "-lc", script];
|
||||
}
|
||||
|
||||
interface RelatedContainer {
|
||||
name: string;
|
||||
image: string;
|
||||
status: string;
|
||||
}
|
||||
|
||||
function unquoteEnvValue(value: string): string {
|
||||
return value.replace(/^['"]|['"]$/g, "");
|
||||
}
|
||||
|
||||
function baiduSecretPresence(envPath: string): Record<string, unknown> {
|
||||
const keys = [
|
||||
"UNIDESK_BAIDU_NETDISK_CLIENT_ID",
|
||||
"UNIDESK_BAIDU_NETDISK_CLIENT_SECRET",
|
||||
"UNIDESK_BAIDU_NETDISK_TOKEN_KEY",
|
||||
];
|
||||
if (!existsSync(envPath)) {
|
||||
return { envPath, exists: false, keys: Object.fromEntries(keys.map((key) => [key, { present: false, nonEmpty: false }])) };
|
||||
}
|
||||
const env = new Map<string, string>();
|
||||
for (const line of readFileSync(envPath, "utf8").split(/\r?\n/u)) {
|
||||
if (line.length === 0 || line.trimStart().startsWith("#")) continue;
|
||||
const index = line.indexOf("=");
|
||||
if (index === -1) continue;
|
||||
env.set(line.slice(0, index), unquoteEnvValue(line.slice(index + 1)));
|
||||
}
|
||||
return {
|
||||
envPath,
|
||||
exists: true,
|
||||
keys: Object.fromEntries(keys.map((key) => [key, { present: env.has(key), nonEmpty: Boolean((env.get(key) || "").trim()) }])),
|
||||
};
|
||||
}
|
||||
|
||||
function listRelatedStackContainers(): RelatedContainer[] {
|
||||
const result = runCommand(["docker", "ps", "-a", "--format", "{{json .}}"], repoRoot);
|
||||
if (result.exitCode !== 0 || result.stdout.trim().length === 0) return [];
|
||||
const names = ["unidesk-backend-core", "unidesk-database", "baidu-netdisk-backend"];
|
||||
return result.stdout
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line) as Record<string, string>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((row): row is Record<string, string> => row !== null)
|
||||
.filter((row) => names.some((name) => row.Names === name || String(row.Names || "").startsWith(`${name}.`)))
|
||||
.map((row) => ({ name: row.Names ?? "", image: row.Image ?? "", status: row.Status ?? "" }));
|
||||
}
|
||||
|
||||
export function backendCoreUnavailableDiagnostic(detail: {
|
||||
exitCode: number | null;
|
||||
stdoutTail: string;
|
||||
stderrTail: string;
|
||||
relatedContainers: RelatedContainer[];
|
||||
envPath: string;
|
||||
baiduSecretPresence: Record<string, unknown>;
|
||||
}): Record<string, unknown> {
|
||||
const expectedContainers = ["unidesk-backend-core", "unidesk-database", "baidu-netdisk-backend"];
|
||||
const presentExactNames = new Set(detail.relatedContainers.map((container) => container.name));
|
||||
const missingContainers = expectedContainers.filter((name) => !presentExactNames.has(name));
|
||||
const verifyOnlyObserved = detail.relatedContainers.some((container) => /\.verify-/u.test(container.name));
|
||||
return {
|
||||
ok: false,
|
||||
failureKind: "target-stack-not-running",
|
||||
degradedReason: "backend-core-container-missing",
|
||||
runnerDisposition: "infra-blocked",
|
||||
readOnlyReview: true,
|
||||
message: verifyOnlyObserved
|
||||
? "backend-core/database target containers are not running; only verify-only containers were observed."
|
||||
: "backend-core target container is not running, so private backend-core API checks cannot observe schedules or microservices.",
|
||||
targetStack: {
|
||||
expectedContainers,
|
||||
missingContainers,
|
||||
relatedContainers: detail.relatedContainers,
|
||||
verifyOnlyObserved,
|
||||
},
|
||||
observed: {
|
||||
commandExitCode: detail.exitCode,
|
||||
stdoutTail: detail.stdoutTail,
|
||||
stderrTail: detail.stderrTail,
|
||||
},
|
||||
baiduNetdiskConfigPresence: detail.baiduSecretPresence,
|
||||
readOnlyCommands: [
|
||||
"bun scripts/cli.ts server status",
|
||||
"bun scripts/cli.ts schedule list",
|
||||
"bun scripts/cli.ts schedule runs --limit 20",
|
||||
"bun scripts/cli.ts microservice health baidu-netdisk",
|
||||
"bun scripts/cli.ts microservice proxy baidu-netdisk /api/auth/status --raw",
|
||||
"bun scripts/cli.ts microservice proxy baidu-netdisk '/api/transfers?limit=20' --raw",
|
||||
],
|
||||
authorizationRequiredForRecovery: [
|
||||
"restore non-empty Baidu Netdisk runtime secrets in the controlled Compose env source without printing secret values",
|
||||
"start or deploy the production target stack containing backend-core, database, and baidu-netdisk",
|
||||
"after health is green, explicitly authorize schedule retry-run or schedule run before triggering a real PGDATA backup",
|
||||
],
|
||||
blockedWithoutAuthorization: [
|
||||
"server start",
|
||||
"server rebuild backend-core",
|
||||
"server rebuild baidu-netdisk",
|
||||
"deploy apply --env prod --service baidu-netdisk",
|
||||
"schedule run <id>",
|
||||
"schedule retry-run <failedRunId>",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function backendCoreContainerMissing(stderr: string): boolean {
|
||||
return stderr.includes("No such container: unidesk-backend-core");
|
||||
}
|
||||
|
||||
export function coreInternalFetch(path: string, init?: { method?: string; body?: unknown; maxResponseBytes?: number }): unknown {
|
||||
if (!path.startsWith("/")) throw new Error("core internal path must start with /");
|
||||
const command = dockerCoreFetchCommand(path, init);
|
||||
const result = runCommand(command, repoRoot);
|
||||
if (result.exitCode !== 0) {
|
||||
if (backendCoreContainerMissing(result.stderr)) {
|
||||
const envPath = join(repoRoot, ".state", "docker-compose.env");
|
||||
return backendCoreUnavailableDiagnostic({
|
||||
exitCode: result.exitCode,
|
||||
stdoutTail: result.stdout.slice(-1200),
|
||||
stderrTail: result.stderr.slice(-1200),
|
||||
relatedContainers: listRelatedStackContainers(),
|
||||
envPath,
|
||||
baiduSecretPresence: baiduSecretPresence(envPath),
|
||||
});
|
||||
}
|
||||
const parsedStdout = parseJsonRecord(result.stdout.trim());
|
||||
if (parsedStdout !== null) {
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user