From 6fc459e1fbce898556f5eca65cc1aa272bae31c7 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 18:42:45 +0200 Subject: [PATCH] feat: add github-backed decision storage --- config/unidesk-host-k8s.yaml | 23 + .../native/deploy/render-unidesk-host-k8s.mjs | 99 +++++ scripts/src/decision-center.ts | 109 ++++- scripts/src/help.ts | 4 +- .../microservices/decision-center/Dockerfile | 4 + .../decision-center/src/index.ts | 402 +++++++++++++++++- 6 files changed, 629 insertions(+), 12 deletions(-) diff --git a/config/unidesk-host-k8s.yaml b/config/unidesk-host-k8s.yaml index f76cc2aa..1acc97d9 100644 --- a/config/unidesk-host-k8s.yaml +++ b/config/unidesk-host-k8s.yaml @@ -81,6 +81,29 @@ services: key: DATABASE_URL secretName: decision-center-runtime-secrets secretKey: DATABASE_URL + storage: + primary: github-repo + github: + repo: pikasTech/unidesk-decision-center-data + sshUrl: git@github.com:pikasTech/unidesk-decision-center-data.git + branch: main + basePath: data + worktreePath: /var/lib/unidesk/decision-center/repo + commitMessagePrefix: "decision-center:" + author: + name: UniDesk Decision Center + email: decision-center@unidesk.local + ssh: + secretName: decision-center-github-ssh + mountPath: /var/lib/unidesk/decision-center/ssh + privateKey: + sourceRef: + path: /root/.ssh/id_ed25519_github_codex + secretKey: ssh-privatekey + knownHosts: + sourceRef: + path: /root/.ssh/known_hosts + secretKey: known_hosts repository: url: https://github.com/pikasTech/unidesk dockerfile: src/components/microservices/decision-center/Dockerfile diff --git a/scripts/native/deploy/render-unidesk-host-k8s.mjs b/scripts/native/deploy/render-unidesk-host-k8s.mjs index f2863af7..f614cd2a 100755 --- a/scripts/native/deploy/render-unidesk-host-k8s.mjs +++ b/scripts/native/deploy/render-unidesk-host-k8s.mjs @@ -182,7 +182,59 @@ function readDecisionCenterDatabase(decision) { }; } +function readFileSource(sourceRef, path) { + const sourcePath = resolve(required(sourceRef.path, `${path}.path`)); + const value = readFileSync(sourcePath, "utf8"); + if (value.length === 0) throw new Error(`missing file content in ${sourcePath}`); + return { + sourcePath, + sourceRefPath: required(sourceRef.path, `${path}.path`), + value, + fingerprint: sha(value), + }; +} + +function readDecisionCenterStorage(decision) { + if (decision === null) return null; + const storage = optionalRecord(decision.storage, "services.decisionCenter.storage"); + if (storage === null) return null; + const primary = required(storage.primary, "services.decisionCenter.storage.primary"); + if (primary !== "postgres" && primary !== "github-repo") throw new Error("services.decisionCenter.storage.primary must be postgres or github-repo"); + const github = primary === "github-repo" ? record(storage.github, "services.decisionCenter.storage.github") : null; + if (github === null) return { primary }; + const ssh = record(github.ssh, "services.decisionCenter.storage.github.ssh"); + const privateKey = record(ssh.privateKey, "services.decisionCenter.storage.github.ssh.privateKey"); + const knownHosts = record(ssh.knownHosts, "services.decisionCenter.storage.github.ssh.knownHosts"); + const privateKeySource = readFileSource(record(privateKey.sourceRef, "services.decisionCenter.storage.github.ssh.privateKey.sourceRef"), "services.decisionCenter.storage.github.ssh.privateKey.sourceRef"); + const knownHostsSource = readFileSource(record(knownHosts.sourceRef, "services.decisionCenter.storage.github.ssh.knownHosts.sourceRef"), "services.decisionCenter.storage.github.ssh.knownHosts.sourceRef"); + return { + primary, + github: { + repo: required(github.repo, "services.decisionCenter.storage.github.repo"), + sshUrl: required(github.sshUrl, "services.decisionCenter.storage.github.sshUrl"), + branch: required(github.branch, "services.decisionCenter.storage.github.branch"), + basePath: required(github.basePath, "services.decisionCenter.storage.github.basePath"), + worktreePath: required(github.worktreePath, "services.decisionCenter.storage.github.worktreePath"), + commitMessagePrefix: required(github.commitMessagePrefix, "services.decisionCenter.storage.github.commitMessagePrefix"), + author: { + name: required(record(github.author, "services.decisionCenter.storage.github.author").name, "services.decisionCenter.storage.github.author.name"), + email: required(record(github.author, "services.decisionCenter.storage.github.author").email, "services.decisionCenter.storage.github.author.email"), + }, + ssh: { + secretName: required(ssh.secretName, "services.decisionCenter.storage.github.ssh.secretName"), + mountPath: required(ssh.mountPath, "services.decisionCenter.storage.github.ssh.mountPath"), + privateKeyKey: required(privateKey.secretKey, "services.decisionCenter.storage.github.ssh.privateKey.secretKey"), + knownHostsKey: required(knownHosts.secretKey, "services.decisionCenter.storage.github.ssh.knownHosts.secretKey"), + privateKeySource, + knownHostsSource, + fingerprint: sha(`${privateKeySource.fingerprint}\n${knownHostsSource.fingerprint}`), + }, + }, + }; +} + const decisionCenterDatabase = readDecisionCenterDatabase(decisionCenter); +const decisionCenterStorage = readDecisionCenterStorage(decisionCenter); const effectiveMicroservicesJson = decisionCenter === null ? required(runtimeConfig.microservicesJson, "runtimeConfig.microservicesJson") : JSON.stringify([ @@ -295,6 +347,24 @@ type: Opaque stringData: ${decisionCenterDatabase.secretKey}: ${yamlScalar(decisionCenterDatabase.value)}`); } +if (decisionCenterStorage?.github !== undefined) { +docs.push(`apiVersion: v1 +kind: Secret +metadata: + name: ${decisionCenterStorage.github.ssh.secretName} + namespace: ${namespace} + labels: +${indent(labels(config), 4)} + app.kubernetes.io/name: decision-center + annotations: + unidesk.ai/private-key-source-ref: ${yamlScalar(decisionCenterStorage.github.ssh.privateKeySource.sourceRefPath)} + unidesk.ai/known-hosts-source-ref: ${yamlScalar(decisionCenterStorage.github.ssh.knownHostsSource.sourceRefPath)} + unidesk.ai/fingerprint: ${yamlScalar(decisionCenterStorage.github.ssh.fingerprint)} +type: Opaque +stringData: + ${decisionCenterStorage.github.ssh.privateKeyKey}: ${yamlScalar(decisionCenterStorage.github.ssh.privateKeySource.value)} + ${decisionCenterStorage.github.ssh.knownHostsKey}: ${yamlScalar(decisionCenterStorage.github.ssh.knownHostsSource.value)}`); +} if (databaseMode === "k8s-postgres") { docs.push(`apiVersion: v1 kind: PersistentVolumeClaim @@ -552,6 +622,7 @@ spec: app.kubernetes.io/part-of: unidesk annotations: unidesk.ai/secret-fingerprint: ${yamlScalar(decisionCenterDatabase?.fingerprint ?? "")} + unidesk.ai/storage-secret-fingerprint: ${yamlScalar(decisionCenterStorage?.github?.ssh.fingerprint ?? "")} unidesk.ai/runtime-config-fingerprint: ${yamlScalar(runtimeConfigFingerprint)} spec: initContainers: @@ -594,9 +665,30 @@ spec: value: ${yamlScalar(required(runtime.commit, "runtime.commit"))} - name: LOG_FILE value: ${yamlScalar(required(decisionCenter.logFile, "services.decisionCenter.logFile"))} + - name: DECISION_CENTER_STORAGE_PRIMARY + value: ${yamlScalar(decisionCenterStorage?.primary ?? "postgres")} +${decisionCenterStorage?.github !== undefined ? indent(`- name: DECISION_CENTER_GIT_REPO_SSH_URL + value: ${yamlScalar(decisionCenterStorage.github.sshUrl)} +- name: DECISION_CENTER_GIT_BRANCH + value: ${yamlScalar(decisionCenterStorage.github.branch)} +- name: DECISION_CENTER_GIT_BASE_PATH + value: ${yamlScalar(decisionCenterStorage.github.basePath)} +- name: DECISION_CENTER_GIT_WORKTREE + value: ${yamlScalar(decisionCenterStorage.github.worktreePath)} +- name: DECISION_CENTER_GIT_AUTHOR_NAME + value: ${yamlScalar(decisionCenterStorage.github.author.name)} +- name: DECISION_CENTER_GIT_AUTHOR_EMAIL + value: ${yamlScalar(decisionCenterStorage.github.author.email)} +- name: DECISION_CENTER_GIT_COMMIT_PREFIX + value: ${yamlScalar(decisionCenterStorage.github.commitMessagePrefix)} +- name: GIT_SSH_COMMAND + value: ${yamlScalar(`ssh -i ${decisionCenterStorage.github.ssh.mountPath}/${decisionCenterStorage.github.ssh.privateKeyKey} -o UserKnownHostsFile=${decisionCenterStorage.github.ssh.mountPath}/${decisionCenterStorage.github.ssh.knownHostsKey} -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes`)}`, 12) : ""} volumeMounts: - name: logs mountPath: /var/log/unidesk +${decisionCenterStorage?.github !== undefined ? indent(`- name: github-ssh + mountPath: ${yamlScalar(decisionCenterStorage.github.ssh.mountPath)} + readOnly: true`, 12) : ""} readinessProbe: httpGet: path: ${required(decisionCenter.healthPath, "services.decisionCenter.healthPath")} @@ -622,6 +714,13 @@ ${indent(resourceBlock(decisionCenter.resources), 10)} volumes: - name: logs emptyDir: {}`); +if (decisionCenterStorage?.github !== undefined) { +docs[docs.length - 1] += ` + - name: github-ssh + secret: + secretName: ${decisionCenterStorage.github.ssh.secretName} + defaultMode: 0400`; +} } docs.push(`apiVersion: v1 kind: Service diff --git a/scripts/src/decision-center.ts b/scripts/src/decision-center.ts index 53bae0a4..5054fde8 100644 --- a/scripts/src/decision-center.ts +++ b/scripts/src/decision-center.ts @@ -76,6 +76,17 @@ function secretSourceSummary(sourcePath: string, key: string): Record { + const absolute = sourcePath.startsWith("/") ? sourcePath : resolve(repoRoot, sourcePath); + if (!existsSync(absolute)) return { sourceRef: sourcePath, present: false, fingerprint: null }; + const value = readFileSync(absolute, "utf8"); + return { + sourceRef: sourcePath, + present: value.length > 0, + fingerprint: value.length > 0 ? `sha256:${sha256Short(value)}` : null, + }; +} + function decisionDeploymentConfigSummary(configPath = hostK8sConfigPath): Record { const parsed = readYamlConfig(configPath); const target = asRecord(parsed.target, `${configPath}.target`); @@ -85,6 +96,13 @@ function decisionDeploymentConfigSummary(configPath = hostK8sConfigPath): Record const decision = asRecord(services.decisionCenter, `${configPath}.services.decisionCenter`); const database = asRecord(decision.database, `${configPath}.services.decisionCenter.database`); const sourceRef = asRecord(database.sourceRef, `${configPath}.services.decisionCenter.database.sourceRef`); + const storage = asRecord(decision.storage, `${configPath}.services.decisionCenter.storage`); + const github = asRecord(storage.github, `${configPath}.services.decisionCenter.storage.github`); + const ssh = asRecord(github.ssh, `${configPath}.services.decisionCenter.storage.github.ssh`); + const privateKey = asRecord(ssh.privateKey, `${configPath}.services.decisionCenter.storage.github.ssh.privateKey`); + const knownHosts = asRecord(ssh.knownHosts, `${configPath}.services.decisionCenter.storage.github.ssh.knownHosts`); + const privateKeySourceRef = asRecord(privateKey.sourceRef, `${configPath}.services.decisionCenter.storage.github.ssh.privateKey.sourceRef`); + const knownHostsSourceRef = asRecord(knownHosts.sourceRef, `${configPath}.services.decisionCenter.storage.github.ssh.knownHosts.sourceRef`); return { configPath, target: { @@ -113,6 +131,19 @@ function decisionDeploymentConfigSummary(configPath = hostK8sConfigPath): Record stringField(sourceRef, "key", `${configPath}.services.decisionCenter.database.sourceRef`), ), }, + storage: { + primary: stringField(storage, "primary", `${configPath}.services.decisionCenter.storage`), + github: { + repo: stringField(github, "repo", `${configPath}.services.decisionCenter.storage.github`), + sshUrl: stringField(github, "sshUrl", `${configPath}.services.decisionCenter.storage.github`), + branch: stringField(github, "branch", `${configPath}.services.decisionCenter.storage.github`), + basePath: stringField(github, "basePath", `${configPath}.services.decisionCenter.storage.github`), + worktreePath: stringField(github, "worktreePath", `${configPath}.services.decisionCenter.storage.github`), + sshSecret: stringField(ssh, "secretName", `${configPath}.services.decisionCenter.storage.github.ssh`), + privateKey: fileSourceSummary(stringField(privateKeySourceRef, "path", `${configPath}.services.decisionCenter.storage.github.ssh.privateKey.sourceRef`)), + knownHosts: fileSourceSummary(stringField(knownHostsSourceRef, "path", `${configPath}.services.decisionCenter.storage.github.ssh.knownHosts.sourceRef`)), + }, + }, }; } @@ -133,6 +164,23 @@ function manifestObjects(manifest: string): Array> { .filter((item) => item.kind.length > 0 && item.name.length > 0); } +function redactSecretManifestValues(manifest: string): string { + return manifest.split(/^---$/mu).map((doc) => { + if (!/^kind:\s*Secret\s*$/mu.test(doc)) return doc; + const lines = doc.split("\n"); + let inStringData = false; + return lines.map((line) => { + if (/^stringData:\s*$/u.test(line)) { + inStringData = true; + return line; + } + if (inStringData && /^\S/u.test(line)) inStringData = false; + if (inStringData && /^ [A-Za-z0-9_.-]+:\s*/u.test(line)) return line.replace(/:\s*.*$/u, ': ""'); + return line; + }).join("\n"); + }).join("---"); +} + function deploymentOptions(args: string[]): { configPath: string; confirm: boolean; dryRun: boolean; wait: boolean; full: boolean; raw: boolean } { return { configPath: optionValue(args, ["--config"]) ?? hostK8sConfigPath, @@ -175,7 +223,7 @@ function deploymentPlan(args: string[]): Record { function deploymentRender(args: string[]): Record | string { const options = deploymentOptions(args); const manifest = renderDecisionDeploymentManifest(options.configPath); - if (options.raw || options.full) return manifest; + if (options.raw || options.full) return redactSecretManifestValues(manifest); return { ok: true, action: "decision-center-deployment-render", @@ -315,6 +363,62 @@ async function runDeploymentCommand(config: UniDeskConfig, args: string[]): Prom throw new Error("decision deployment command must be one of: plan, render, apply, status"); } +function storageOptions(args: string[]): { configPath: string; confirm: boolean; dryRun: boolean } { + return { + configPath: optionValue(args, ["--config"]) ?? hostK8sConfigPath, + confirm: hasFlag(args, "--confirm"), + dryRun: hasFlag(args, "--dry-run"), + }; +} + +function storagePlan(args: string[]): Record { + const options = storageOptions(args); + const summary = decisionDeploymentConfigSummary(options.configPath); + return { + ok: true, + action: "decision-center-storage-plan", + mutation: false, + config: { + configPath: options.configPath, + target: asRecord(summary.target, "summary.target"), + storage: asRecord(summary.storage, "summary.storage"), + database: asRecord(summary.database, "summary.database"), + }, + next: { + servicePlan: "bun scripts/cli.ts decision storage status", + migrateDryRun: "bun scripts/cli.ts decision storage migrate --dry-run", + migrateConfirm: "bun scripts/cli.ts decision storage migrate --confirm", + verify: "bun scripts/cli.ts decision storage verify", + }, + }; +} + +function storageProxy(path: string, init?: { method?: string; body?: unknown }): unknown { + return unwrapProxyResponse(decisionProxy(path, init)); +} + +async function storageProxyAsync(fetcher: (path: string, init?: { method?: string; body?: unknown }) => Promise, path: string, init?: { method?: string; body?: unknown }): Promise { + return unwrapProxyResponse(await decisionProxyAsync(fetcher, path, init)); +} + +async function runStorageCommand(args: string[], fetcher?: (path: string, init?: { method?: string; body?: unknown }) => Promise): Promise { + const [action = "plan"] = args; + if (action === "plan") return storagePlan(args.slice(1)); + if (action === "status") { + return fetcher === undefined ? storageProxy("/api/storage/status") : storageProxyAsync(fetcher, "/api/storage/status"); + } + if (action === "migrate") { + const options = storageOptions(args.slice(1)); + if (!options.confirm && !options.dryRun) throw new Error("decision storage migrate requires --dry-run or --confirm"); + const init = { method: "POST", body: { confirm: options.confirm } }; + return fetcher === undefined ? storageProxy("/api/storage/migrate", init) : storageProxyAsync(fetcher, "/api/storage/migrate", init); + } + if (action === "verify") { + return fetcher === undefined ? storageProxy("/api/storage/verify") : storageProxyAsync(fetcher, "/api/storage/verify"); + } + throw new Error("decision storage command must be one of: plan, status, migrate, verify"); +} + function optionValues(args: string[], names: string[]): string[] { const values: string[] = []; for (let index = 0; index < args.length; index += 1) { @@ -951,6 +1055,7 @@ export async function runDecisionCenterCommandAsync( ): Promise { const [action = "list", id] = args; if (action === "deployment" || action === "deploy") return runDeploymentCommand(_config, args.slice(1)); + if (action === "storage") return runStorageCommand(args.slice(1), fetcher); if (action === "diary") { const [diaryAction = "list", diaryId] = args.slice(1); if (diaryAction === "import") return importDiaryAsync(args.slice(2), fetcher); @@ -978,5 +1083,5 @@ export async function runDecisionCenterCommandAsync( if (action === "list") return listRecordsAsync(args.slice(1), fetcher); if (action === "show") return showRecordAsync(id, fetcher); if (action === "health") return unwrapProxyResponse(await fetcher(`/api/microservices/${encodeURIComponent(serviceId)}/health`)); - throw new Error("decision command must be one of: upload, list, show, health, requirement, diary, deployment"); + throw new Error("decision command must be one of: upload, list, show, health, requirement, diary, deployment, storage"); } diff --git a/scripts/src/help.ts b/scripts/src/help.ts index c280fc80..05974e8d 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -278,7 +278,7 @@ function microserviceHelp(): unknown { function decisionHelp(): unknown { return { - command: "decision upload|list|show|health|diary|requirement|deployment", + command: "decision upload|list|show|health|diary|requirement|deployment|storage", output: "json", usage: [ "bun scripts/cli.ts decision upload [--title text] [--type meeting|decision] [--doc-no DC-...]", @@ -289,6 +289,8 @@ function decisionHelp(): unknown { "bun scripts/cli.ts decision requirement list|create|show|update|upsert ... [--doc-no DC-...] [--doc-type GOAL] [--doc-priority P0] [--signer text] [--issued-at ISO]", "bun scripts/cli.ts decision deployment plan|render|status [--config config/unidesk-host-k8s.yaml]", "bun scripts/cli.ts decision deployment apply --dry-run|--confirm [--config config/unidesk-host-k8s.yaml]", + "bun scripts/cli.ts decision storage plan|status|verify [--config config/unidesk-host-k8s.yaml]", + "bun scripts/cli.ts decision storage migrate --dry-run|--confirm [--config config/unidesk-host-k8s.yaml]", ], description: "Operate Decision Center through the registered user-service proxy and manage its NC01 YAML-first k8s deployment.", }; diff --git a/src/components/microservices/decision-center/Dockerfile b/src/components/microservices/decision-center/Dockerfile index 431b6dd2..e50345a7 100644 --- a/src/components/microservices/decision-center/Dockerfile +++ b/src/components/microservices/decision-center/Dockerfile @@ -2,6 +2,10 @@ ARG DECISION_CENTER_BASE_IMAGE=oven/bun:1-debian FROM ${DECISION_CENTER_BASE_IMAGE} WORKDIR /app/src/components/microservices/decision-center +RUN apt-get update \ + && apt-get install -y --no-install-recommends git openssh-client ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY src/components/microservices/decision-center/package.json ./package.json RUN bun install --production COPY src/components/microservices/decision-center/tsconfig.json ./tsconfig.json diff --git a/src/components/microservices/decision-center/src/index.ts b/src/components/microservices/decision-center/src/index.ts index 31eee877..0c9c4314 100644 --- a/src/components/microservices/decision-center/src/index.ts +++ b/src/components/microservices/decision-center/src/index.ts @@ -1,4 +1,7 @@ import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { spawnSync } from "node:child_process"; import postgres from "postgres"; import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../../shared/src/rotating-jsonl"; import { @@ -35,6 +38,19 @@ interface RuntimeConfig { databaseUrl: string; logFile: string; databasePoolMax: number; + storage: StorageConfig; +} + +interface StorageConfig { + primary: "postgres" | "github-repo"; + repoSshUrl: string; + branch: string; + basePath: string; + worktreePath: string; + sshCommand: string; + authorName: string; + authorEmail: string; + commitMessagePrefix: string; } interface DecisionRecordRow { @@ -164,6 +180,8 @@ let schemaReady = false; let schemaLastError: JsonRecord | null = null; let databaseHealth: JsonRecord = { ok: false, recordCount: 0, checkedAt: "", lastOkAt: "", degraded: false, error: null }; let databaseHealthRefreshInFlight = false; +let storageHealth: JsonRecord = { ok: false, primary: "postgres", checkedAt: "", repo: null, error: null }; +let gitQueue: Promise = Promise.resolve(); function envString(name: string, fallback: string): string { const value = process.env[name]; @@ -180,12 +198,25 @@ function envNumber(name: string, fallback: number): number { function configFromEnv(): RuntimeConfig { const databaseUrl = process.env.DATABASE_URL || ""; if (!databaseUrl) throw new Error("DATABASE_URL is required"); + const storagePrimary = envString("DECISION_CENTER_STORAGE_PRIMARY", "postgres"); + if (storagePrimary !== "postgres" && storagePrimary !== "github-repo") throw new Error("DECISION_CENTER_STORAGE_PRIMARY must be postgres or github-repo"); return { host: envString("HOST", "0.0.0.0"), port: envNumber("PORT", 4277), databaseUrl, logFile: envString("LOG_FILE", "/var/log/unidesk/decision-center.jsonl"), databasePoolMax: Math.max(1, Math.min(8, envNumber("DATABASE_POOL_MAX", 2))), + storage: { + primary: storagePrimary, + repoSshUrl: envString("DECISION_CENTER_GIT_REPO_SSH_URL", ""), + branch: envString("DECISION_CENTER_GIT_BRANCH", "main"), + basePath: envString("DECISION_CENTER_GIT_BASE_PATH", "."), + worktreePath: envString("DECISION_CENTER_GIT_WORKTREE", "/var/lib/unidesk/decision-center/repo"), + sshCommand: envString("GIT_SSH_COMMAND", ""), + authorName: envString("DECISION_CENTER_GIT_AUTHOR_NAME", "UniDesk Decision Center"), + authorEmail: envString("DECISION_CENTER_GIT_AUTHOR_EMAIL", "decision-center@unidesk.local"), + commitMessagePrefix: envString("DECISION_CENTER_GIT_COMMIT_PREFIX", "decision-center:"), + }, }; } @@ -800,14 +831,40 @@ function deployInfo(): JsonRecord { }; } +function storageConfigSummary(): JsonRecord { + return { + primary: config.storage.primary, + github: { + repoSshUrl: config.storage.repoSshUrl, + branch: config.storage.branch, + basePath: config.storage.basePath, + worktreePath: config.storage.worktreePath, + authorName: config.storage.authorName, + authorEmail: config.storage.authorEmail, + commitMessagePrefix: config.storage.commitMessagePrefix, + sshConfigured: config.storage.sshCommand.length > 0, + }, + postgresIndex: { + enabled: true, + role: config.storage.primary === "github-repo" ? "runtime-index-cache" : "primary-store", + }, + }; +} + function health(): JsonRecord { const dbOk = databaseHealth.ok === true; + const storageOk = config.storage.primary === "postgres" || storageHealth.ok === true; return { - ok: schemaReady && dbOk, + ok: schemaReady && dbOk && storageOk, service: "decision-center", - status: schemaReady && dbOk ? "ready" : "not-ready", + status: schemaReady && dbOk && storageOk ? "ready" : "not-ready", startedAt: serviceStartedAt, - storage: "postgres", + storage: { + primary: config.storage.primary, + durableStore: config.storage.primary === "github-repo" ? "github-repo" : "postgres", + index: "postgres", + github: storageHealth, + }, schemaReady, recordCount: Number(databaseHealth.recordCount ?? 0), diaryEntryCount: Number(databaseHealth.diaryEntryCount ?? 0), @@ -816,6 +873,308 @@ function health(): JsonRecord { }; } +function runGit(args: string[], options: { cwd?: string; allowFailure?: boolean } = {}): { ok: boolean; stdout: string; stderr: string; exitCode: number } { + const env = { ...process.env }; + if (config.storage.sshCommand) env.GIT_SSH_COMMAND = config.storage.sshCommand; + const result = spawnSync("git", args, { + cwd: options.cwd ?? config.storage.worktreePath, + env, + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }); + const exitCode = result.status ?? 1; + const stdout = typeof result.stdout === "string" ? result.stdout : ""; + const stderr = typeof result.stderr === "string" ? result.stderr : ""; + if (exitCode !== 0 && options.allowFailure !== true) { + throw new HttpError(503, "git storage command failed", { + command: ["git", ...args].join(" "), + exitCode, + stderr: stderr.slice(-2000), + }); + } + return { ok: exitCode === 0, stdout, stderr, exitCode }; +} + +function gitFilePath(...parts: string[]): string { + return join(config.storage.worktreePath, ...parts); +} + +function storageBasePath(): string { + const raw = config.storage.basePath.trim().replace(/^\/+/u, "").replace(/\/+$/u, ""); + return raw === "." ? "" : raw; +} + +function storagePath(...parts: string[]): string { + const base = storageBasePath(); + return base ? join(base, ...parts) : join(...parts); +} + +function safePathSegment(value: string, fallback: string): string { + const normalized = value + .trim() + .replace(/[\\/]+/gu, "-") + .replace(/[^A-Za-z0-9._-]+/gu, "-") + .replace(/^-+|-+$/gu, "") + .slice(0, 160); + return normalized || fallback; +} + +function recordStoragePath(record: DecisionRecord): string { + const docType = safePathSegment(record.docType || record.type, "record"); + const docPriority = safePathSegment(record.docPriority || record.level || "none", "none"); + const name = safePathSegment(record.docNo || record.id, record.id); + return storagePath("records", docType, docPriority, `${name}.md`); +} + +function diaryStoragePath(entry: DiaryEntry): string { + const source = safePathSegment(entry.sourceFile || "manual", ""); + if (!source || source === "manual" || source === "inline" || source === "today") { + return storagePath("diary", entry.month, `${entry.date}.md`); + } + return storagePath("diary", entry.month, entry.date, `${source}.md`); +} + +function frontmatter(meta: Record, body: string): string { + const lines = Object.entries(meta).map(([key, value]) => `${key}: ${JSON.stringify(value ?? null)}`); + return `---\n${lines.join("\n")}\n---\n\n${body.replace(/\r\n?/gu, "\n").trimEnd()}\n`; +} + +function writeTextFile(relativePath: string, text: string): void { + const path = gitFilePath(relativePath); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, text, "utf8"); +} + +function recordMarkdown(record: DecisionRecord): string { + const { body: _body, summary: _summary, ...meta } = record; + return frontmatter({ kind: "decision-record", schemaVersion: 1, ...meta }, record.body || `# ${record.title}\n`); +} + +function diaryMarkdown(entry: DiaryEntry): string { + const { body: _body, summary: _summary, ...meta } = entry; + return frontmatter({ kind: "decision-diary-entry", schemaVersion: 1, ...meta }, entry.body || defaultDiaryBody(entry.date)); +} + +async function readAllRecordsForStorage(): Promise { + const rows = await withDatabaseRecovery("storage_read_all_records", () => sql` + SELECT * + FROM decision_center_records + ORDER BY + CASE WHEN doc_no = '' THEN 1 ELSE 0 END ASC, + doc_type ASC NULLS LAST, + doc_priority ASC NULLS LAST, + doc_year ASC NULLS LAST, + doc_seq ASC NULLS LAST, + updated_at DESC + `, { retryRead: true }); + return rows.map((row) => recordFromRow(row)); +} + +async function readAllDiaryForStorage(): Promise { + const rows = await withDatabaseRecovery("storage_read_all_diary", () => sql` + SELECT * + FROM decision_center_diary_entries + ORDER BY entry_date ASC, source_file ASC, updated_at ASC + `, { retryRead: true }); + return rows.map((row) => diaryEntryFromRow(row)); +} + +function ensureGitWorktree(): void { + if (config.storage.primary !== "github-repo") return; + if (!config.storage.repoSshUrl) throw new HttpError(503, "GitHub storage repo is not configured", { code: "github_repo_missing" }); + mkdirSync(dirname(config.storage.worktreePath), { recursive: true }); + if (!existsSync(join(config.storage.worktreePath, ".git"))) { + if (existsSync(config.storage.worktreePath)) rmSync(config.storage.worktreePath, { recursive: true, force: true }); + runGit(["clone", "--branch", config.storage.branch, config.storage.repoSshUrl, config.storage.worktreePath], { cwd: dirname(config.storage.worktreePath) }); + } + runGit(["remote", "set-url", "origin", config.storage.repoSshUrl]); + runGit(["fetch", "origin", config.storage.branch]); + runGit(["checkout", config.storage.branch]); + runGit(["pull", "--ff-only", "origin", config.storage.branch]); +} + +function gitHeadSummary(): JsonRecord { + if (!existsSync(join(config.storage.worktreePath, ".git"))) return { present: false }; + const head = runGit(["rev-parse", "HEAD"], { allowFailure: true }); + const branch = runGit(["rev-parse", "--abbrev-ref", "HEAD"], { allowFailure: true }); + return { + present: head.ok, + branch: branch.stdout.trim(), + commit: head.stdout.trim().slice(0, 40), + shortCommit: head.stdout.trim().slice(0, 12), + }; +} + +async function updateStorageHealth(): Promise { + const checkedAt = new Date().toISOString(); + if (config.storage.primary !== "github-repo") { + storageHealth = { ok: true, primary: "postgres", checkedAt, repo: null, error: null }; + return; + } + try { + ensureGitWorktree(); + storageHealth = { + ok: true, + primary: "github-repo", + checkedAt, + repo: { + sshUrl: config.storage.repoSshUrl, + branch: config.storage.branch, + basePath: config.storage.basePath, + head: gitHeadSummary(), + }, + error: null, + }; + } catch (error) { + storageHealth = { + ok: false, + primary: "github-repo", + checkedAt, + repo: { + sshUrl: config.storage.repoSshUrl, + branch: config.storage.branch, + basePath: config.storage.basePath, + head: gitHeadSummary(), + }, + error: errorToJson(error), + }; + } +} + +function enqueueGit(operation: () => Promise): Promise { + const next = gitQueue.then(operation, operation); + gitQueue = next.catch(() => undefined); + return next; +} + +async function writeSnapshotToGit(reason: string, dryRun: boolean): Promise { + const records = await readAllRecordsForStorage(); + const diaryEntries = await readAllDiaryForStorage(); + const recordFiles = records.map((record) => ({ path: recordStoragePath(record), record })); + const diaryFiles = diaryEntries.map((entry) => ({ path: diaryStoragePath(entry), entry })); + if (dryRun) { + return { + ok: true, + dryRun: true, + records: records.length, + diaryEntries: diaryEntries.length, + files: recordFiles.length + diaryFiles.length + 3, + samplePaths: [...recordFiles.slice(0, 5).map((item) => item.path), ...diaryFiles.slice(0, 5).map((item) => item.path)], + }; + } + return await enqueueGit(async () => { + ensureGitWorktree(); + const base = storageBasePath(); + for (const dir of ["records", "diary", "indexes", ".decision-center"]) { + rmSync(gitFilePath(base ? join(base, dir) : dir), { recursive: true, force: true }); + } + for (const item of recordFiles) writeTextFile(item.path, recordMarkdown(item.record)); + for (const item of diaryFiles) writeTextFile(item.path, diaryMarkdown(item.entry)); + writeTextFile(storagePath(".decision-center", "schema.json"), `${JSON.stringify({ schemaVersion: 1, generatedBy: "unidesk-decision-center", generatedAt: new Date().toISOString() }, null, 2)}\n`); + writeTextFile(storagePath("indexes", "records.json"), `${JSON.stringify(records.map((record) => { + const path = recordStoragePath(record); + const { body: _body, ...indexRecord } = record; + return { ...indexRecord, path }; + }), null, 2)}\n`); + writeTextFile(storagePath("indexes", "diary.json"), `${JSON.stringify(diaryEntries.map((entry) => { + const path = diaryStoragePath(entry); + const { body: _body, ...indexEntry } = entry; + return { ...indexEntry, path }; + }), null, 2)}\n`); + runGit(["add", base || "."]); + const diff = runGit(["diff", "--cached", "--quiet"], { allowFailure: true }); + if (diff.exitCode === 0) { + await updateStorageHealth(); + return { ok: true, dryRun: false, changed: false, records: records.length, diaryEntries: diaryEntries.length, head: gitHeadSummary() }; + } + const message = `${config.storage.commitMessagePrefix} ${reason}`; + const env = { + GIT_AUTHOR_NAME: config.storage.authorName, + GIT_AUTHOR_EMAIL: config.storage.authorEmail, + GIT_COMMITTER_NAME: config.storage.authorName, + GIT_COMMITTER_EMAIL: config.storage.authorEmail, + ...(config.storage.sshCommand ? { GIT_SSH_COMMAND: config.storage.sshCommand } : {}), + }; + const commit = spawnSync("git", ["commit", "-m", message], { + cwd: config.storage.worktreePath, + env: { ...process.env, ...env }, + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }); + if ((commit.status ?? 1) !== 0) throw new HttpError(503, "git commit failed", { stderr: String(commit.stderr).slice(-2000) }); + runGit(["push", "origin", config.storage.branch]); + await updateStorageHealth(); + return { ok: true, dryRun: false, changed: true, records: records.length, diaryEntries: diaryEntries.length, head: gitHeadSummary() }; + }); +} + +async function persistStorageSnapshot(reason: string): Promise { + if (config.storage.primary !== "github-repo") return; + const result = await writeSnapshotToGit(reason, false); + log("info", "github_storage_snapshot_pushed", { reason, result }); +} + +async function storagePlan(): Promise { + const dryRun = await writeSnapshotToGit("dry-run", true); + return { + ok: true, + action: "decision-center-storage-plan", + mutation: false, + config: storageConfigSummary(), + cache: databaseHealth, + plan: dryRun, + next: { + migrateDryRun: "bun scripts/cli.ts decision storage migrate --dry-run", + migrateConfirm: "bun scripts/cli.ts decision storage migrate --confirm", + verify: "bun scripts/cli.ts decision storage verify", + }, + }; +} + +async function storageMigrate(confirm: boolean): Promise { + if (!confirm) { + const dryRun = await writeSnapshotToGit("migrate", true); + return { ok: true, action: "decision-center-storage-migrate", mutation: false, mode: "dry-run", ...dryRun }; + } + const result = await writeSnapshotToGit("migrate postgres cache to github repo", false); + return { ok: result.ok === true, action: "decision-center-storage-migrate", mutation: true, mode: "confirmed", result }; +} + +async function storageVerify(): Promise { + await updateStorageHealth(); + const records = await readAllRecordsForStorage(); + const diaryEntries = await readAllDiaryForStorage(); + let repoRecords = 0; + let repoDiary = 0; + try { + const recordsIndex = JSON.parse(readFileSync(gitFilePath(storagePath("indexes", "records.json")), "utf8")) as unknown; + const diaryIndex = JSON.parse(readFileSync(gitFilePath(storagePath("indexes", "diary.json")), "utf8")) as unknown; + repoRecords = Array.isArray(recordsIndex) ? recordsIndex.length : 0; + repoDiary = Array.isArray(diaryIndex) ? diaryIndex.length : 0; + } catch { + repoRecords = 0; + repoDiary = 0; + } + return { + ok: storageHealth.ok === true && repoRecords === records.length && repoDiary === diaryEntries.length, + action: "decision-center-storage-verify", + mutation: false, + storage: storageHealth, + cache: { + records: records.length, + diaryEntries: diaryEntries.length, + }, + repoIndex: { + records: repoRecords, + diaryEntries: repoDiary, + }, + diff: { + recordDelta: repoRecords - records.length, + diaryDelta: repoDiary - diaryEntries.length, + }, + }; +} + async function clearInvalidDocumentNumbers(): Promise { const rows = await sql<{ id: string; doc_no: string }[]>` UPDATE decision_center_records @@ -977,7 +1336,9 @@ async function createRecord(input: Record): Promise): Promise { @@ -1022,7 +1383,9 @@ async function updateRecord(id: string, input: Record): Promise RETURNING * `; })); - return recordFromRow(rows[0]!); + const record = recordFromRow(rows[0]!); + await persistStorageSnapshot(`record ${record.docNo || record.id} updated`); + return record; } async function upsertRequirementRecord(input: Record): Promise { @@ -1266,7 +1629,9 @@ async function deleteRecord(id: string): Promise { const rows = await withDatabaseRecovery("delete_record", () => sql`DELETE FROM decision_center_records WHERE id = ${existing.id} RETURNING *`); if (rows.length === 0) throw new HttpError(404, "decision record not found", { id }); log("info", "record_deleted", { id }); - return { ok: true, deleted: recordFromRow(rows[0]!) }; + const deleted = recordFromRow(rows[0]!); + await persistStorageSnapshot(`record ${deleted.docNo || deleted.id} deleted`); + return { ok: true, deleted }; } function dateOnly(value: Date | string | null | undefined): string { @@ -1466,6 +1831,7 @@ async function upsertDiaryEntry(draft: DiaryDraft, sourceFile: string, tags: str `); const row = rows[0]; if (row === undefined) throw new HttpError(500, "diary upsert returned no row", { date: draft.date }); + await persistStorageSnapshot(`diary ${draft.date} upserted`); return { entry: diaryEntryFromRow(row, { includeBody: false }), action: row.inserted ? "created" : row.changed ? "updated" : "unchanged", @@ -1666,6 +2032,7 @@ async function upsertDiaryEntryByKey(key: string, input: Record if (row === undefined) throw new HttpError(500, "diary edit returned no row", { key }); const action = existing === undefined ? "created" : "updated"; log("info", "diary_entry_upserted", { id: row.id, date, action, sourceFile: row.source_file }); + await persistStorageSnapshot(`diary ${date} ${action}`); return { ok: true, action, entry: diaryEntryFromRow(row) }; } @@ -1698,6 +2065,17 @@ async function route(req: Request): Promise { return jsonResponse(body, body.ok === true ? 200 : 503); } if (url.pathname === "/logs" && method === "GET") return jsonResponse({ ok: true, logs: recentLogs.slice(-200) }); + if (url.pathname === "/api/storage/plan" && method === "GET") return jsonResponse(await storagePlan()); + if (url.pathname === "/api/storage/status" && method === "GET") { + await updateStorageHealth(); + return jsonResponse({ ok: storageHealth.ok === true, action: "decision-center-storage-status", mutation: false, config: storageConfigSummary(), storage: storageHealth, cache: databaseHealth }); + } + if (url.pathname === "/api/storage/migrate" && method === "POST") { + const input = await readJsonBody(req); + const confirm = input.confirm === true || url.searchParams.get("confirm") === "true"; + return jsonResponse(await storageMigrate(confirm)); + } + if (url.pathname === "/api/storage/verify" && method === "GET") return jsonResponse(await storageVerify()); if (url.pathname === "/api/records" && method === "GET") return jsonResponse({ ok: true, records: await listRecords(url) }); if (url.pathname === "/api/records" && method === "POST") return jsonResponse({ ok: true, record: await createRecord(await readJsonBody(req)) }, 201); if (url.pathname === "/api/requirements" && method === "GET") return jsonResponse({ ok: true, requirements: await listRecords(url, { requirementOnly: true }) }); @@ -1770,11 +2148,17 @@ Bun.serve({ }); void waitForSchema() - .then(() => refreshDatabaseHealth()) + .then(async () => { + await refreshDatabaseHealth(); + await updateStorageHealth(); + }) .catch((error) => { log("error", "schema_init_failed", { error: errorToJson(error) }); }); setInterval(() => { - if (schemaReady) void refreshDatabaseHealth(); + if (schemaReady) { + void refreshDatabaseHealth(); + void updateStorageHealth(); + } }, 15_000).unref(); -log("info", "service_started", { host: config.host, port: config.port, storage: "postgres" }); +log("info", "service_started", { host: config.host, port: config.port, storage: storageConfigSummary() });