feat: add github-backed decision storage

This commit is contained in:
Codex
2026-07-09 18:42:45 +02:00
parent 50a166a2ce
commit 6fc459e1fb
6 changed files with 629 additions and 12 deletions
@@ -2,6 +2,10 @@ ARG DECISION_CENTER_BASE_IMAGE=oven/bun:1-debian
FROM ${DECISION_CENTER_BASE_IMAGE}
WORKDIR /app/src/components/microservices/decision-center
RUN apt-get update \
&& apt-get install -y --no-install-recommends git openssh-client ca-certificates \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
COPY src/components/microservices/decision-center/package.json ./package.json
RUN bun install --production
COPY src/components/microservices/decision-center/tsconfig.json ./tsconfig.json
@@ -1,4 +1,7 @@
import { createHash, randomUUID } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { spawnSync } from "node:child_process";
import postgres from "postgres";
import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../../shared/src/rotating-jsonl";
import {
@@ -35,6 +38,19 @@ interface RuntimeConfig {
databaseUrl: string;
logFile: string;
databasePoolMax: number;
storage: StorageConfig;
}
interface StorageConfig {
primary: "postgres" | "github-repo";
repoSshUrl: string;
branch: string;
basePath: string;
worktreePath: string;
sshCommand: string;
authorName: string;
authorEmail: string;
commitMessagePrefix: string;
}
interface DecisionRecordRow {
@@ -164,6 +180,8 @@ let schemaReady = false;
let schemaLastError: JsonRecord | null = null;
let databaseHealth: JsonRecord = { ok: false, recordCount: 0, checkedAt: "", lastOkAt: "", degraded: false, error: null };
let databaseHealthRefreshInFlight = false;
let storageHealth: JsonRecord = { ok: false, primary: "postgres", checkedAt: "", repo: null, error: null };
let gitQueue: Promise<unknown> = Promise.resolve();
function envString(name: string, fallback: string): string {
const value = process.env[name];
@@ -180,12 +198,25 @@ function envNumber(name: string, fallback: number): number {
function configFromEnv(): RuntimeConfig {
const databaseUrl = process.env.DATABASE_URL || "";
if (!databaseUrl) throw new Error("DATABASE_URL is required");
const storagePrimary = envString("DECISION_CENTER_STORAGE_PRIMARY", "postgres");
if (storagePrimary !== "postgres" && storagePrimary !== "github-repo") throw new Error("DECISION_CENTER_STORAGE_PRIMARY must be postgres or github-repo");
return {
host: envString("HOST", "0.0.0.0"),
port: envNumber("PORT", 4277),
databaseUrl,
logFile: envString("LOG_FILE", "/var/log/unidesk/decision-center.jsonl"),
databasePoolMax: Math.max(1, Math.min(8, envNumber("DATABASE_POOL_MAX", 2))),
storage: {
primary: storagePrimary,
repoSshUrl: envString("DECISION_CENTER_GIT_REPO_SSH_URL", ""),
branch: envString("DECISION_CENTER_GIT_BRANCH", "main"),
basePath: envString("DECISION_CENTER_GIT_BASE_PATH", "."),
worktreePath: envString("DECISION_CENTER_GIT_WORKTREE", "/var/lib/unidesk/decision-center/repo"),
sshCommand: envString("GIT_SSH_COMMAND", ""),
authorName: envString("DECISION_CENTER_GIT_AUTHOR_NAME", "UniDesk Decision Center"),
authorEmail: envString("DECISION_CENTER_GIT_AUTHOR_EMAIL", "decision-center@unidesk.local"),
commitMessagePrefix: envString("DECISION_CENTER_GIT_COMMIT_PREFIX", "decision-center:"),
},
};
}
@@ -800,14 +831,40 @@ function deployInfo(): JsonRecord {
};
}
function storageConfigSummary(): JsonRecord {
return {
primary: config.storage.primary,
github: {
repoSshUrl: config.storage.repoSshUrl,
branch: config.storage.branch,
basePath: config.storage.basePath,
worktreePath: config.storage.worktreePath,
authorName: config.storage.authorName,
authorEmail: config.storage.authorEmail,
commitMessagePrefix: config.storage.commitMessagePrefix,
sshConfigured: config.storage.sshCommand.length > 0,
},
postgresIndex: {
enabled: true,
role: config.storage.primary === "github-repo" ? "runtime-index-cache" : "primary-store",
},
};
}
function health(): JsonRecord {
const dbOk = databaseHealth.ok === true;
const storageOk = config.storage.primary === "postgres" || storageHealth.ok === true;
return {
ok: schemaReady && dbOk,
ok: schemaReady && dbOk && storageOk,
service: "decision-center",
status: schemaReady && dbOk ? "ready" : "not-ready",
status: schemaReady && dbOk && storageOk ? "ready" : "not-ready",
startedAt: serviceStartedAt,
storage: "postgres",
storage: {
primary: config.storage.primary,
durableStore: config.storage.primary === "github-repo" ? "github-repo" : "postgres",
index: "postgres",
github: storageHealth,
},
schemaReady,
recordCount: Number(databaseHealth.recordCount ?? 0),
diaryEntryCount: Number(databaseHealth.diaryEntryCount ?? 0),
@@ -816,6 +873,308 @@ function health(): JsonRecord {
};
}
function runGit(args: string[], options: { cwd?: string; allowFailure?: boolean } = {}): { ok: boolean; stdout: string; stderr: string; exitCode: number } {
const env = { ...process.env };
if (config.storage.sshCommand) env.GIT_SSH_COMMAND = config.storage.sshCommand;
const result = spawnSync("git", args, {
cwd: options.cwd ?? config.storage.worktreePath,
env,
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
});
const exitCode = result.status ?? 1;
const stdout = typeof result.stdout === "string" ? result.stdout : "";
const stderr = typeof result.stderr === "string" ? result.stderr : "";
if (exitCode !== 0 && options.allowFailure !== true) {
throw new HttpError(503, "git storage command failed", {
command: ["git", ...args].join(" "),
exitCode,
stderr: stderr.slice(-2000),
});
}
return { ok: exitCode === 0, stdout, stderr, exitCode };
}
function gitFilePath(...parts: string[]): string {
return join(config.storage.worktreePath, ...parts);
}
function storageBasePath(): string {
const raw = config.storage.basePath.trim().replace(/^\/+/u, "").replace(/\/+$/u, "");
return raw === "." ? "" : raw;
}
function storagePath(...parts: string[]): string {
const base = storageBasePath();
return base ? join(base, ...parts) : join(...parts);
}
function safePathSegment(value: string, fallback: string): string {
const normalized = value
.trim()
.replace(/[\\/]+/gu, "-")
.replace(/[^A-Za-z0-9._-]+/gu, "-")
.replace(/^-+|-+$/gu, "")
.slice(0, 160);
return normalized || fallback;
}
function recordStoragePath(record: DecisionRecord): string {
const docType = safePathSegment(record.docType || record.type, "record");
const docPriority = safePathSegment(record.docPriority || record.level || "none", "none");
const name = safePathSegment(record.docNo || record.id, record.id);
return storagePath("records", docType, docPriority, `${name}.md`);
}
function diaryStoragePath(entry: DiaryEntry): string {
const source = safePathSegment(entry.sourceFile || "manual", "");
if (!source || source === "manual" || source === "inline" || source === "today") {
return storagePath("diary", entry.month, `${entry.date}.md`);
}
return storagePath("diary", entry.month, entry.date, `${source}.md`);
}
function frontmatter(meta: Record<string, unknown>, body: string): string {
const lines = Object.entries(meta).map(([key, value]) => `${key}: ${JSON.stringify(value ?? null)}`);
return `---\n${lines.join("\n")}\n---\n\n${body.replace(/\r\n?/gu, "\n").trimEnd()}\n`;
}
function writeTextFile(relativePath: string, text: string): void {
const path = gitFilePath(relativePath);
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, text, "utf8");
}
function recordMarkdown(record: DecisionRecord): string {
const { body: _body, summary: _summary, ...meta } = record;
return frontmatter({ kind: "decision-record", schemaVersion: 1, ...meta }, record.body || `# ${record.title}\n`);
}
function diaryMarkdown(entry: DiaryEntry): string {
const { body: _body, summary: _summary, ...meta } = entry;
return frontmatter({ kind: "decision-diary-entry", schemaVersion: 1, ...meta }, entry.body || defaultDiaryBody(entry.date));
}
async function readAllRecordsForStorage(): Promise<DecisionRecord[]> {
const rows = await withDatabaseRecovery("storage_read_all_records", () => sql<DecisionRecordRow[]>`
SELECT *
FROM decision_center_records
ORDER BY
CASE WHEN doc_no = '' THEN 1 ELSE 0 END ASC,
doc_type ASC NULLS LAST,
doc_priority ASC NULLS LAST,
doc_year ASC NULLS LAST,
doc_seq ASC NULLS LAST,
updated_at DESC
`, { retryRead: true });
return rows.map((row) => recordFromRow(row));
}
async function readAllDiaryForStorage(): Promise<DiaryEntry[]> {
const rows = await withDatabaseRecovery("storage_read_all_diary", () => sql<DiaryEntryRow[]>`
SELECT *
FROM decision_center_diary_entries
ORDER BY entry_date ASC, source_file ASC, updated_at ASC
`, { retryRead: true });
return rows.map((row) => diaryEntryFromRow(row));
}
function ensureGitWorktree(): void {
if (config.storage.primary !== "github-repo") return;
if (!config.storage.repoSshUrl) throw new HttpError(503, "GitHub storage repo is not configured", { code: "github_repo_missing" });
mkdirSync(dirname(config.storage.worktreePath), { recursive: true });
if (!existsSync(join(config.storage.worktreePath, ".git"))) {
if (existsSync(config.storage.worktreePath)) rmSync(config.storage.worktreePath, { recursive: true, force: true });
runGit(["clone", "--branch", config.storage.branch, config.storage.repoSshUrl, config.storage.worktreePath], { cwd: dirname(config.storage.worktreePath) });
}
runGit(["remote", "set-url", "origin", config.storage.repoSshUrl]);
runGit(["fetch", "origin", config.storage.branch]);
runGit(["checkout", config.storage.branch]);
runGit(["pull", "--ff-only", "origin", config.storage.branch]);
}
function gitHeadSummary(): JsonRecord {
if (!existsSync(join(config.storage.worktreePath, ".git"))) return { present: false };
const head = runGit(["rev-parse", "HEAD"], { allowFailure: true });
const branch = runGit(["rev-parse", "--abbrev-ref", "HEAD"], { allowFailure: true });
return {
present: head.ok,
branch: branch.stdout.trim(),
commit: head.stdout.trim().slice(0, 40),
shortCommit: head.stdout.trim().slice(0, 12),
};
}
async function updateStorageHealth(): Promise<void> {
const checkedAt = new Date().toISOString();
if (config.storage.primary !== "github-repo") {
storageHealth = { ok: true, primary: "postgres", checkedAt, repo: null, error: null };
return;
}
try {
ensureGitWorktree();
storageHealth = {
ok: true,
primary: "github-repo",
checkedAt,
repo: {
sshUrl: config.storage.repoSshUrl,
branch: config.storage.branch,
basePath: config.storage.basePath,
head: gitHeadSummary(),
},
error: null,
};
} catch (error) {
storageHealth = {
ok: false,
primary: "github-repo",
checkedAt,
repo: {
sshUrl: config.storage.repoSshUrl,
branch: config.storage.branch,
basePath: config.storage.basePath,
head: gitHeadSummary(),
},
error: errorToJson(error),
};
}
}
function enqueueGit<T>(operation: () => Promise<T>): Promise<T> {
const next = gitQueue.then(operation, operation);
gitQueue = next.catch(() => undefined);
return next;
}
async function writeSnapshotToGit(reason: string, dryRun: boolean): Promise<JsonRecord> {
const records = await readAllRecordsForStorage();
const diaryEntries = await readAllDiaryForStorage();
const recordFiles = records.map((record) => ({ path: recordStoragePath(record), record }));
const diaryFiles = diaryEntries.map((entry) => ({ path: diaryStoragePath(entry), entry }));
if (dryRun) {
return {
ok: true,
dryRun: true,
records: records.length,
diaryEntries: diaryEntries.length,
files: recordFiles.length + diaryFiles.length + 3,
samplePaths: [...recordFiles.slice(0, 5).map((item) => item.path), ...diaryFiles.slice(0, 5).map((item) => item.path)],
};
}
return await enqueueGit(async () => {
ensureGitWorktree();
const base = storageBasePath();
for (const dir of ["records", "diary", "indexes", ".decision-center"]) {
rmSync(gitFilePath(base ? join(base, dir) : dir), { recursive: true, force: true });
}
for (const item of recordFiles) writeTextFile(item.path, recordMarkdown(item.record));
for (const item of diaryFiles) writeTextFile(item.path, diaryMarkdown(item.entry));
writeTextFile(storagePath(".decision-center", "schema.json"), `${JSON.stringify({ schemaVersion: 1, generatedBy: "unidesk-decision-center", generatedAt: new Date().toISOString() }, null, 2)}\n`);
writeTextFile(storagePath("indexes", "records.json"), `${JSON.stringify(records.map((record) => {
const path = recordStoragePath(record);
const { body: _body, ...indexRecord } = record;
return { ...indexRecord, path };
}), null, 2)}\n`);
writeTextFile(storagePath("indexes", "diary.json"), `${JSON.stringify(diaryEntries.map((entry) => {
const path = diaryStoragePath(entry);
const { body: _body, ...indexEntry } = entry;
return { ...indexEntry, path };
}), null, 2)}\n`);
runGit(["add", base || "."]);
const diff = runGit(["diff", "--cached", "--quiet"], { allowFailure: true });
if (diff.exitCode === 0) {
await updateStorageHealth();
return { ok: true, dryRun: false, changed: false, records: records.length, diaryEntries: diaryEntries.length, head: gitHeadSummary() };
}
const message = `${config.storage.commitMessagePrefix} ${reason}`;
const env = {
GIT_AUTHOR_NAME: config.storage.authorName,
GIT_AUTHOR_EMAIL: config.storage.authorEmail,
GIT_COMMITTER_NAME: config.storage.authorName,
GIT_COMMITTER_EMAIL: config.storage.authorEmail,
...(config.storage.sshCommand ? { GIT_SSH_COMMAND: config.storage.sshCommand } : {}),
};
const commit = spawnSync("git", ["commit", "-m", message], {
cwd: config.storage.worktreePath,
env: { ...process.env, ...env },
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
});
if ((commit.status ?? 1) !== 0) throw new HttpError(503, "git commit failed", { stderr: String(commit.stderr).slice(-2000) });
runGit(["push", "origin", config.storage.branch]);
await updateStorageHealth();
return { ok: true, dryRun: false, changed: true, records: records.length, diaryEntries: diaryEntries.length, head: gitHeadSummary() };
});
}
async function persistStorageSnapshot(reason: string): Promise<void> {
if (config.storage.primary !== "github-repo") return;
const result = await writeSnapshotToGit(reason, false);
log("info", "github_storage_snapshot_pushed", { reason, result });
}
async function storagePlan(): Promise<JsonRecord> {
const dryRun = await writeSnapshotToGit("dry-run", true);
return {
ok: true,
action: "decision-center-storage-plan",
mutation: false,
config: storageConfigSummary(),
cache: databaseHealth,
plan: dryRun,
next: {
migrateDryRun: "bun scripts/cli.ts decision storage migrate --dry-run",
migrateConfirm: "bun scripts/cli.ts decision storage migrate --confirm",
verify: "bun scripts/cli.ts decision storage verify",
},
};
}
async function storageMigrate(confirm: boolean): Promise<JsonRecord> {
if (!confirm) {
const dryRun = await writeSnapshotToGit("migrate", true);
return { ok: true, action: "decision-center-storage-migrate", mutation: false, mode: "dry-run", ...dryRun };
}
const result = await writeSnapshotToGit("migrate postgres cache to github repo", false);
return { ok: result.ok === true, action: "decision-center-storage-migrate", mutation: true, mode: "confirmed", result };
}
async function storageVerify(): Promise<JsonRecord> {
await updateStorageHealth();
const records = await readAllRecordsForStorage();
const diaryEntries = await readAllDiaryForStorage();
let repoRecords = 0;
let repoDiary = 0;
try {
const recordsIndex = JSON.parse(readFileSync(gitFilePath(storagePath("indexes", "records.json")), "utf8")) as unknown;
const diaryIndex = JSON.parse(readFileSync(gitFilePath(storagePath("indexes", "diary.json")), "utf8")) as unknown;
repoRecords = Array.isArray(recordsIndex) ? recordsIndex.length : 0;
repoDiary = Array.isArray(diaryIndex) ? diaryIndex.length : 0;
} catch {
repoRecords = 0;
repoDiary = 0;
}
return {
ok: storageHealth.ok === true && repoRecords === records.length && repoDiary === diaryEntries.length,
action: "decision-center-storage-verify",
mutation: false,
storage: storageHealth,
cache: {
records: records.length,
diaryEntries: diaryEntries.length,
},
repoIndex: {
records: repoRecords,
diaryEntries: repoDiary,
},
diff: {
recordDelta: repoRecords - records.length,
diaryDelta: repoDiary - diaryEntries.length,
},
};
}
async function clearInvalidDocumentNumbers(): Promise<void> {
const rows = await sql<{ id: string; doc_no: string }[]>`
UPDATE decision_center_records
@@ -977,7 +1336,9 @@ async function createRecord(input: Record<string, unknown>): Promise<DecisionRec
`;
}));
log("info", "record_created", { id: rows[0]?.id ?? id, type: rows[0]?.type ?? "", level: rows[0]?.level ?? "", docNo: rows[0]?.doc_no ?? "" });
return recordFromRow(rows[0]!);
const record = recordFromRow(rows[0]!);
await persistStorageSnapshot(`record ${record.docNo || record.id} created`);
return record;
}
async function updateRecord(id: string, input: Record<string, unknown>): Promise<DecisionRecord> {
@@ -1022,7 +1383,9 @@ async function updateRecord(id: string, input: Record<string, unknown>): Promise
RETURNING *
`;
}));
return recordFromRow(rows[0]!);
const record = recordFromRow(rows[0]!);
await persistStorageSnapshot(`record ${record.docNo || record.id} updated`);
return record;
}
async function upsertRequirementRecord(input: Record<string, unknown>): Promise<JsonRecord> {
@@ -1266,7 +1629,9 @@ async function deleteRecord(id: string): Promise<JsonRecord> {
const rows = await withDatabaseRecovery("delete_record", () => sql<DecisionRecordRow[]>`DELETE FROM decision_center_records WHERE id = ${existing.id} RETURNING *`);
if (rows.length === 0) throw new HttpError(404, "decision record not found", { id });
log("info", "record_deleted", { id });
return { ok: true, deleted: recordFromRow(rows[0]!) };
const deleted = recordFromRow(rows[0]!);
await persistStorageSnapshot(`record ${deleted.docNo || deleted.id} deleted`);
return { ok: true, deleted };
}
function dateOnly(value: Date | string | null | undefined): string {
@@ -1466,6 +1831,7 @@ async function upsertDiaryEntry(draft: DiaryDraft, sourceFile: string, tags: str
`);
const row = rows[0];
if (row === undefined) throw new HttpError(500, "diary upsert returned no row", { date: draft.date });
await persistStorageSnapshot(`diary ${draft.date} upserted`);
return {
entry: diaryEntryFromRow(row, { includeBody: false }),
action: row.inserted ? "created" : row.changed ? "updated" : "unchanged",
@@ -1666,6 +2032,7 @@ async function upsertDiaryEntryByKey(key: string, input: Record<string, unknown>
if (row === undefined) throw new HttpError(500, "diary edit returned no row", { key });
const action = existing === undefined ? "created" : "updated";
log("info", "diary_entry_upserted", { id: row.id, date, action, sourceFile: row.source_file });
await persistStorageSnapshot(`diary ${date} ${action}`);
return { ok: true, action, entry: diaryEntryFromRow(row) };
}
@@ -1698,6 +2065,17 @@ async function route(req: Request): Promise<Response> {
return jsonResponse(body, body.ok === true ? 200 : 503);
}
if (url.pathname === "/logs" && method === "GET") return jsonResponse({ ok: true, logs: recentLogs.slice(-200) });
if (url.pathname === "/api/storage/plan" && method === "GET") return jsonResponse(await storagePlan());
if (url.pathname === "/api/storage/status" && method === "GET") {
await updateStorageHealth();
return jsonResponse({ ok: storageHealth.ok === true, action: "decision-center-storage-status", mutation: false, config: storageConfigSummary(), storage: storageHealth, cache: databaseHealth });
}
if (url.pathname === "/api/storage/migrate" && method === "POST") {
const input = await readJsonBody(req);
const confirm = input.confirm === true || url.searchParams.get("confirm") === "true";
return jsonResponse(await storageMigrate(confirm));
}
if (url.pathname === "/api/storage/verify" && method === "GET") return jsonResponse(await storageVerify());
if (url.pathname === "/api/records" && method === "GET") return jsonResponse({ ok: true, records: await listRecords(url) });
if (url.pathname === "/api/records" && method === "POST") return jsonResponse({ ok: true, record: await createRecord(await readJsonBody(req)) }, 201);
if (url.pathname === "/api/requirements" && method === "GET") return jsonResponse({ ok: true, requirements: await listRecords(url, { requirementOnly: true }) });
@@ -1770,11 +2148,17 @@ Bun.serve({
});
void waitForSchema()
.then(() => refreshDatabaseHealth())
.then(async () => {
await refreshDatabaseHealth();
await updateStorageHealth();
})
.catch((error) => {
log("error", "schema_init_failed", { error: errorToJson(error) });
});
setInterval(() => {
if (schemaReady) void refreshDatabaseHealth();
if (schemaReady) {
void refreshDatabaseHealth();
void updateStorageHealth();
}
}, 15_000).unref();
log("info", "service_started", { host: config.host, port: config.port, storage: "postgres" });
log("info", "service_started", { host: config.host, port: config.port, storage: storageConfigSummary() });