feat: add decision center diary imports
This commit is contained in:
Codex
@@ -1,4 +1,4 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import postgres from "postgres";
|
||||
import { createHourlyJsonlWriter, logRetentionBytesForService } from "../../../shared/src/rotating-jsonl";
|
||||
|
||||
@@ -52,6 +52,50 @@ interface DecisionRecord extends JsonRecord {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
interface DiaryEntryRow {
|
||||
id: string;
|
||||
entry_date: Date | string;
|
||||
month: string;
|
||||
title: string;
|
||||
body: string;
|
||||
source_file: string;
|
||||
markdown_path: string;
|
||||
tags: JsonValue;
|
||||
content_hash: string;
|
||||
created_at: Date | string;
|
||||
updated_at: Date | string;
|
||||
imported_at: Date | string;
|
||||
}
|
||||
|
||||
interface DiaryEntry extends JsonRecord {
|
||||
id: string;
|
||||
date: string;
|
||||
month: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
body: string;
|
||||
sourceFile: string;
|
||||
markdownPath: string;
|
||||
tags: string[];
|
||||
contentHash: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
importedAt: string;
|
||||
}
|
||||
|
||||
interface DiaryDraft {
|
||||
date: string;
|
||||
month: string;
|
||||
title: string;
|
||||
body: string;
|
||||
markdownPath: string;
|
||||
}
|
||||
|
||||
interface DiaryUpsertRow extends DiaryEntryRow {
|
||||
inserted: boolean;
|
||||
changed: boolean;
|
||||
}
|
||||
|
||||
class HttpError extends Error {
|
||||
readonly status: number;
|
||||
readonly detail: JsonRecord;
|
||||
@@ -271,7 +315,7 @@ function titleFromMarkdown(markdown: string, fallback: string): string {
|
||||
|
||||
async function readJsonBody(req: Request): Promise<Record<string, unknown>> {
|
||||
const text = await req.text();
|
||||
if (text.length > 1_000_000) throw new HttpError(413, "request body is too large", { maxBytes: 1_000_000 });
|
||||
if (text.length > 5_000_000) throw new HttpError(413, "request body is too large", { maxBytes: 5_000_000 });
|
||||
if (!text.trim()) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(text) as unknown;
|
||||
@@ -310,6 +354,27 @@ async function ensureSchema(): Promise<void> {
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_decision_center_records_type_status_level ON decision_center_records(type, status, level)`;
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_decision_center_records_linked_goal ON decision_center_records(linked_goal_id)`;
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_decision_center_records_updated ON decision_center_records(updated_at DESC)`;
|
||||
await sql`
|
||||
CREATE TABLE IF NOT EXISTS decision_center_diary_entries (
|
||||
id TEXT PRIMARY KEY,
|
||||
entry_date DATE NOT NULL,
|
||||
month TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
source_file TEXT NOT NULL DEFAULT '',
|
||||
markdown_path TEXT NOT NULL,
|
||||
tags JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
content_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
imported_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT decision_center_diary_entries_month_check CHECK (month ~ '^[0-9]{4}-[0-9]{2}$'),
|
||||
CONSTRAINT decision_center_diary_entries_path_check CHECK (markdown_path ~ '^[0-9]{4}-[0-9]{2}/[0-9]{4}-[0-9]{2}-[0-9]{2}[.]md$'),
|
||||
CONSTRAINT decision_center_diary_entries_source_date_unique UNIQUE (source_file, entry_date)
|
||||
)
|
||||
`;
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_decision_center_diary_entries_month_date ON decision_center_diary_entries(month DESC, entry_date DESC)`;
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_decision_center_diary_entries_updated ON decision_center_diary_entries(updated_at DESC)`;
|
||||
}, { retryRead: true });
|
||||
}
|
||||
|
||||
@@ -335,11 +400,16 @@ async function refreshDatabaseHealth(): Promise<void> {
|
||||
if (databaseHealthRefreshInFlight) return;
|
||||
databaseHealthRefreshInFlight = true;
|
||||
try {
|
||||
const rows = await withDatabaseRecovery("refresh_database_health", () => sql<{ count: string | number }[]>`SELECT count(*) AS count FROM decision_center_records`, { retryRead: true });
|
||||
const rows = await withDatabaseRecovery("refresh_database_health", () => sql<{ record_count: string | number; diary_entry_count: string | number }[]>`
|
||||
SELECT
|
||||
(SELECT count(*) FROM decision_center_records) AS record_count,
|
||||
(SELECT count(*) FROM decision_center_diary_entries) AS diary_entry_count
|
||||
`, { retryRead: true });
|
||||
const checkedAt = new Date().toISOString();
|
||||
databaseHealth = {
|
||||
ok: true,
|
||||
recordCount: Number(rows[0]?.count ?? 0),
|
||||
recordCount: Number(rows[0]?.record_count ?? 0),
|
||||
diaryEntryCount: Number(rows[0]?.diary_entry_count ?? 0),
|
||||
checkedAt,
|
||||
lastOkAt: checkedAt,
|
||||
degraded: false,
|
||||
@@ -353,6 +423,7 @@ async function refreshDatabaseHealth(): Promise<void> {
|
||||
databaseHealth = {
|
||||
ok: withinGrace,
|
||||
recordCount: Number(databaseHealth.recordCount ?? 0),
|
||||
diaryEntryCount: Number(databaseHealth.diaryEntryCount ?? 0),
|
||||
checkedAt,
|
||||
lastOkAt,
|
||||
degraded: withinGrace,
|
||||
@@ -382,6 +453,7 @@ function health(): JsonRecord {
|
||||
storage: "postgres",
|
||||
schemaReady,
|
||||
recordCount: Number(databaseHealth.recordCount ?? 0),
|
||||
diaryEntryCount: Number(databaseHealth.diaryEntryCount ?? 0),
|
||||
database: databaseHealth,
|
||||
deploy: deployInfo(),
|
||||
};
|
||||
@@ -528,6 +600,258 @@ async function deleteRecord(id: string): Promise<JsonRecord> {
|
||||
return { ok: true, deleted: recordFromRow(rows[0]!) };
|
||||
}
|
||||
|
||||
function dateOnly(value: Date | string | null | undefined): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
if (typeof value === "string") {
|
||||
const match = value.match(/^(\d{4}-\d{2}-\d{2})/u);
|
||||
if (match !== null) return match[1] ?? "";
|
||||
}
|
||||
return iso(value).slice(0, 10);
|
||||
}
|
||||
|
||||
function pad2(value: number): string {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function normalizedDate(year: number, month: number, day: number): string | null {
|
||||
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) return null;
|
||||
if (year < 1970 || year > 2100 || month < 1 || month > 12 || day < 1 || day > 31) return null;
|
||||
const date = new Date(Date.UTC(year, month - 1, day));
|
||||
if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day) return null;
|
||||
return `${year}-${pad2(month)}-${pad2(day)}`;
|
||||
}
|
||||
|
||||
function parseDiaryHeadingDate(line: string): string | null {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return null;
|
||||
const heading = trimmed.match(/^#{1,6}\s+(.+)$/u);
|
||||
const candidate = heading === null ? trimmed : heading[1]?.trim() ?? "";
|
||||
if (heading === null && !/^\d{4}/u.test(candidate)) return null;
|
||||
const match = candidate.match(/^(\d{4})\s*(?:年|[-/.])\s*(\d{1,2})\s*(?:月|[-/.])\s*(\d{1,2})\s*(?:日)?(?:\s*(?:[(（].*[)）]|星期[一二三四五六日天]|周[一二三四五六日天]|[-–—:：].*))?$/u);
|
||||
if (match === null) return null;
|
||||
return normalizedDate(Number(match[1]), Number(match[2]), Number(match[3]));
|
||||
}
|
||||
|
||||
function contentHash(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function diaryEntryFromRow(row: DiaryEntryRow, options: { includeBody?: boolean } = {}): DiaryEntry {
|
||||
const includeBody = options.includeBody !== false;
|
||||
const body = row.body || "";
|
||||
return {
|
||||
id: row.id,
|
||||
date: dateOnly(row.entry_date),
|
||||
month: row.month,
|
||||
title: row.title,
|
||||
summary: summaryFromBody(body),
|
||||
body: includeBody ? body : "",
|
||||
sourceFile: row.source_file,
|
||||
markdownPath: row.markdown_path,
|
||||
tags: Array.isArray(row.tags) ? row.tags.map(String) : [],
|
||||
contentHash: row.content_hash,
|
||||
createdAt: iso(row.created_at),
|
||||
updatedAt: iso(row.updated_at),
|
||||
importedAt: iso(row.imported_at),
|
||||
};
|
||||
}
|
||||
|
||||
function splitDiaryMarkdown(markdown: string): DiaryDraft[] {
|
||||
const normalized = markdown.replace(/\r\n?/gu, "\n");
|
||||
const lines = normalized.split("\n");
|
||||
const entries = new Map<string, DiaryDraft>();
|
||||
let currentDate = "";
|
||||
let currentLines: string[] = [];
|
||||
|
||||
const flush = (): void => {
|
||||
if (!currentDate) return;
|
||||
const body = currentLines.join("\n").trim();
|
||||
if (!body) return;
|
||||
const month = currentDate.slice(0, 7);
|
||||
const existing = entries.get(currentDate);
|
||||
const title = titleFromMarkdown(body, `${currentDate} 工作日志`);
|
||||
const draft: DiaryDraft = {
|
||||
date: currentDate,
|
||||
month,
|
||||
title,
|
||||
body: existing === undefined ? body : `${existing.body}\n\n---\n\n${body}`,
|
||||
markdownPath: `${month}/${currentDate}.md`,
|
||||
};
|
||||
entries.set(currentDate, draft);
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const date = parseDiaryHeadingDate(line);
|
||||
if (date !== null) {
|
||||
flush();
|
||||
currentDate = date;
|
||||
currentLines = [line];
|
||||
continue;
|
||||
}
|
||||
if (currentDate) currentLines.push(line);
|
||||
}
|
||||
flush();
|
||||
|
||||
return [...entries.values()].sort((left, right) => left.date.localeCompare(right.date));
|
||||
}
|
||||
|
||||
function validDateFilter(value: string, field: string): string {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/u.test(value)) throw new HttpError(400, `${field} must use YYYY-MM-DD`, { value });
|
||||
const [year, month, day] = value.split("-").map(Number);
|
||||
if (normalizedDate(year ?? 0, month ?? 0, day ?? 0) !== value) throw new HttpError(400, `${field} is not a valid date`, { value });
|
||||
return value;
|
||||
}
|
||||
|
||||
function validMonthFilter(value: string): string {
|
||||
if (!/^\d{4}-\d{2}$/u.test(value)) throw new HttpError(400, "month must use YYYY-MM", { value });
|
||||
const [year, month] = value.split("-").map(Number);
|
||||
if (normalizedDate(year ?? 0, month ?? 0, 1)?.slice(0, 7) !== value) throw new HttpError(400, "month is not valid", { value });
|
||||
return value;
|
||||
}
|
||||
|
||||
async function upsertDiaryEntry(draft: DiaryDraft, sourceFile: string, tags: string[]): Promise<{ entry: DiaryEntry; action: "created" | "updated" | "unchanged" }> {
|
||||
if (draft.body.length > 300_000) throw new HttpError(400, "diary day body must be at most 300000 characters", { date: draft.date, length: draft.body.length });
|
||||
const hash = contentHash(draft.body);
|
||||
const rows = await withDatabaseRecovery("upsert_diary_entry", () => sql<DiaryUpsertRow[]>`
|
||||
WITH existing AS (
|
||||
SELECT content_hash
|
||||
FROM decision_center_diary_entries
|
||||
WHERE source_file = ${sourceFile} AND entry_date = ${draft.date}::date
|
||||
),
|
||||
upsert AS (
|
||||
INSERT INTO decision_center_diary_entries (
|
||||
id, entry_date, month, title, body, source_file, markdown_path, tags, content_hash
|
||||
) VALUES (
|
||||
${`diary_${randomUUID()}`},
|
||||
${draft.date}::date,
|
||||
${draft.month},
|
||||
${draft.title},
|
||||
${draft.body},
|
||||
${sourceFile},
|
||||
${draft.markdownPath},
|
||||
${sql.json(tags)},
|
||||
${hash}
|
||||
)
|
||||
ON CONFLICT (source_file, entry_date) DO UPDATE
|
||||
SET
|
||||
month = EXCLUDED.month,
|
||||
title = EXCLUDED.title,
|
||||
body = EXCLUDED.body,
|
||||
markdown_path = EXCLUDED.markdown_path,
|
||||
tags = EXCLUDED.tags,
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
imported_at = now(),
|
||||
updated_at = CASE
|
||||
WHEN decision_center_diary_entries.content_hash IS DISTINCT FROM EXCLUDED.content_hash THEN now()
|
||||
ELSE decision_center_diary_entries.updated_at
|
||||
END
|
||||
RETURNING *
|
||||
)
|
||||
SELECT
|
||||
upsert.*,
|
||||
NOT EXISTS(SELECT 1 FROM existing) AS inserted,
|
||||
EXISTS(SELECT 1 FROM existing WHERE existing.content_hash IS DISTINCT FROM upsert.content_hash) AS changed
|
||||
FROM upsert
|
||||
`);
|
||||
const row = rows[0];
|
||||
if (row === undefined) throw new HttpError(500, "diary upsert returned no row", { date: draft.date });
|
||||
return {
|
||||
entry: diaryEntryFromRow(row, { includeBody: false }),
|
||||
action: row.inserted ? "created" : row.changed ? "updated" : "unchanged",
|
||||
};
|
||||
}
|
||||
|
||||
async function importDiary(input: Record<string, unknown>): Promise<JsonRecord> {
|
||||
const markdown = asString(input.markdown ?? input.body);
|
||||
if (!markdown) throw new HttpError(400, "markdown is required");
|
||||
if (markdown.length > 4_500_000) throw new HttpError(400, "markdown must be at most 4500000 characters");
|
||||
const sourceFile = asString(input.sourceFile ?? input.sourcePath ?? input.source) || "inline";
|
||||
const tags = asStringArray(input.tags, "tags");
|
||||
const drafts = splitDiaryMarkdown(markdown);
|
||||
if (drafts.length === 0) {
|
||||
throw new HttpError(400, "no dated diary sections found", {
|
||||
expectedHeading: "# 2025年2月12日 or # 2025-02-12",
|
||||
});
|
||||
}
|
||||
const entries: DiaryEntry[] = [];
|
||||
let createdCount = 0;
|
||||
let updatedCount = 0;
|
||||
let unchangedCount = 0;
|
||||
for (const draft of drafts) {
|
||||
const result = await upsertDiaryEntry(draft, sourceFile, tags);
|
||||
entries.push(result.entry);
|
||||
if (result.action === "created") createdCount += 1;
|
||||
else if (result.action === "updated") updatedCount += 1;
|
||||
else unchangedCount += 1;
|
||||
}
|
||||
log("info", "diary_imported", { sourceFile, days: drafts.length, createdCount, updatedCount, unchangedCount });
|
||||
return {
|
||||
ok: true,
|
||||
sourceFile,
|
||||
totalDays: drafts.length,
|
||||
createdCount,
|
||||
updatedCount,
|
||||
unchangedCount,
|
||||
firstDate: entries[0]?.date ?? "",
|
||||
lastDate: entries[entries.length - 1]?.date ?? "",
|
||||
pathPattern: "YYYY-MM/YYYY-MM-DD.md",
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
async function listDiaryEntries(url: URL): Promise<DiaryEntry[]> {
|
||||
const month = asString(url.searchParams.get("month"));
|
||||
const from = asString(url.searchParams.get("from"));
|
||||
const to = asString(url.searchParams.get("to"));
|
||||
const sourceFile = asString(url.searchParams.get("sourceFile"));
|
||||
const includeBody = url.searchParams.get("includeBody") === "true";
|
||||
const limit = Math.max(1, Math.min(1000, Number(url.searchParams.get("limit") || 240) || 240));
|
||||
if (month) validMonthFilter(month);
|
||||
if (from) validDateFilter(from, "from");
|
||||
if (to) validDateFilter(to, "to");
|
||||
const rows = await withDatabaseRecovery("list_diary_entries", () => sql<DiaryEntryRow[]>`
|
||||
SELECT *
|
||||
FROM decision_center_diary_entries
|
||||
WHERE (${month || null}::text IS NULL OR month = ${month || null})
|
||||
AND (${from || null}::date IS NULL OR entry_date >= ${from || null}::date)
|
||||
AND (${to || null}::date IS NULL OR entry_date <= ${to || null}::date)
|
||||
AND (${sourceFile || null}::text IS NULL OR source_file = ${sourceFile || null})
|
||||
ORDER BY entry_date DESC, updated_at DESC
|
||||
LIMIT ${limit}
|
||||
`, { retryRead: true });
|
||||
return rows.map((row) => diaryEntryFromRow(row, { includeBody }));
|
||||
}
|
||||
|
||||
async function listDiaryMonths(): Promise<JsonRecord[]> {
|
||||
const rows = await withDatabaseRecovery("list_diary_months", () => sql<Array<{ month: string; count: string | number; first_date: Date | string; last_date: Date | string; updated_at: Date | string }>>`
|
||||
SELECT month, count(*) AS count, min(entry_date) AS first_date, max(entry_date) AS last_date, max(updated_at) AS updated_at
|
||||
FROM decision_center_diary_entries
|
||||
GROUP BY month
|
||||
ORDER BY month DESC
|
||||
`, { retryRead: true });
|
||||
return rows.map((row) => ({
|
||||
month: row.month,
|
||||
count: Number(row.count),
|
||||
firstDate: dateOnly(row.first_date),
|
||||
lastDate: dateOnly(row.last_date),
|
||||
updatedAt: iso(row.updated_at),
|
||||
}));
|
||||
}
|
||||
|
||||
async function getDiaryEntry(key: string): Promise<DiaryEntry> {
|
||||
const date = /^\d{4}-\d{2}-\d{2}$/u.test(key) ? validDateFilter(key, "date") : "";
|
||||
const rows = await withDatabaseRecovery("get_diary_entry", () => sql<DiaryEntryRow[]>`
|
||||
SELECT *
|
||||
FROM decision_center_diary_entries
|
||||
WHERE (${date || null}::date IS NOT NULL AND entry_date = ${date || null}::date)
|
||||
OR (${date || null}::date IS NULL AND id = ${key})
|
||||
ORDER BY imported_at DESC
|
||||
LIMIT 1
|
||||
`, { retryRead: true });
|
||||
if (rows.length === 0) throw new HttpError(404, "diary entry not found", { key });
|
||||
return diaryEntryFromRow(rows[0]!);
|
||||
}
|
||||
|
||||
async function route(req: Request): Promise<Response> {
|
||||
const url = new URL(req.url);
|
||||
const method = req.method.toUpperCase();
|
||||
@@ -542,6 +866,16 @@ async function route(req: Request): Promise<Response> {
|
||||
if (url.pathname === "/api/records" && method === "GET") return jsonResponse({ ok: true, records: await listRecords(url) });
|
||||
if (url.pathname === "/api/records" && method === "POST") return jsonResponse({ ok: true, record: await createRecord(await readJsonBody(req)) }, 201);
|
||||
if (url.pathname === "/api/meetings/import" && method === "POST") return jsonResponse(await importMeeting(await readJsonBody(req)), 201);
|
||||
if (url.pathname === "/api/diary/months" && method === "GET") return jsonResponse({ ok: true, months: await listDiaryMonths() });
|
||||
if (url.pathname === "/api/diary/entries" && method === "GET") return jsonResponse({ ok: true, entries: await listDiaryEntries(url) });
|
||||
if (url.pathname === "/api/diary/import" && method === "POST") return jsonResponse(await importDiary(await readJsonBody(req)), 201);
|
||||
const diaryMatch = url.pathname.match(/^\/api\/diary\/entries\/([^/]+)$/u);
|
||||
if (diaryMatch !== null) {
|
||||
const key = decodeURIComponent(diaryMatch[1] ?? "");
|
||||
if (!key) throw new HttpError(400, "diary entry id or date is required");
|
||||
if (method === "GET") return jsonResponse({ ok: true, entry: await getDiaryEntry(key) });
|
||||
throw new HttpError(405, "diary entry route supports GET", { method });
|
||||
}
|
||||
const recordMatch = url.pathname.match(/^\/api\/records\/([^/]+)$/u);
|
||||
if (recordMatch !== null) {
|
||||
const id = decodeURIComponent(recordMatch[1] ?? "");
|
||||
|
||||
Reference in New Issue
Block a user