fix: complete monitor ingest migration contracts
This commit is contained in:
AgentRun Codex
@@ -1,146 +1,89 @@
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-12-p17-monitor-central-persistence.
|
||||
// Responsibility: Read-only legacy SQLite snapshot export and bounded, explicit one-shot migration verification.
|
||||
// Responsibility: Registry-resolved read-only legacy SQLite snapshots and one-shot canonical central-store import.
|
||||
import { createHash } from "node:crypto";
|
||||
import { closeSync, existsSync, openSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { Database } from "bun:sqlite";
|
||||
|
||||
import { stableJson } from "./monitor-terminal-ingest";
|
||||
|
||||
export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-1";
|
||||
|
||||
export interface MonitorSqliteSource {
|
||||
readonly sentinelId: string;
|
||||
readonly node: string;
|
||||
readonly lane: string;
|
||||
readonly path: string;
|
||||
readonly runtimeConfigRef: string;
|
||||
}
|
||||
import type { MonitorArtifactLocator, MonitorCentralIngest, MonitorCentralStore } from "./monitor-central-persistence";
|
||||
import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monitor-central-persistence";
|
||||
|
||||
export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-2";
|
||||
export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; }
|
||||
export interface MonitorSqliteTable { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record<string, unknown>[]; }
|
||||
export interface MonitorSqliteSnapshot {
|
||||
readonly schema: string;
|
||||
readonly createdAt: string;
|
||||
readonly source: MonitorSqliteSource;
|
||||
readonly fingerprint: string;
|
||||
readonly integrity: string;
|
||||
readonly byteLength: number;
|
||||
readonly tables: readonly { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record<string, unknown>[] }[];
|
||||
readonly artifactLocator: { readonly path: string; readonly sha256: string; readonly byteLength: number };
|
||||
}
|
||||
|
||||
export interface MonitorMigrationReconciliation {
|
||||
readonly runs: number;
|
||||
readonly findings: number;
|
||||
readonly metadata: number;
|
||||
readonly payloads: number;
|
||||
readonly locators: number;
|
||||
readonly latest: number;
|
||||
readonly schema: string; readonly createdAt: string; readonly source: MonitorSqliteSource; readonly fingerprint: string; readonly integrity: string; readonly byteLength: number;
|
||||
readonly tables: readonly MonitorSqliteTable[];
|
||||
readonly artifactLocator: { readonly path: string; readonly sha256: string; readonly sizeBytes: number; readonly kind: "legacy-sqlite" };
|
||||
readonly manifestContentHash: string;
|
||||
}
|
||||
export interface MonitorMigrationReconciliation { readonly runs: number; readonly findings: number; readonly metadata: number; readonly payloads: number; readonly locators: number; readonly latest: number; }
|
||||
|
||||
export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[]): Record<string, unknown> {
|
||||
const rows = sources.map((source) => {
|
||||
const path = resolve(source.path);
|
||||
const state = existsSync(path) ? statSync(path) : null;
|
||||
return {
|
||||
sentinelId: source.sentinelId,
|
||||
node: source.node,
|
||||
lane: source.lane,
|
||||
path,
|
||||
runtimeConfigRef: source.runtimeConfigRef,
|
||||
exists: state !== null,
|
||||
isFile: state?.isFile() ?? false,
|
||||
byteLength: state?.size ?? null,
|
||||
mutation: false,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
return { ok: true, command: "web-probe sentinel migration plan", sources: rows, sourceCount: rows.length, valuesRedacted: true };
|
||||
return { ok: true, command: "web-probe sentinel migration plan", sources: sources.map(sourcePlan), sourceCount: sources.length, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export function exportMonitorSqliteSnapshot(source: MonitorSqliteSource, outputPath: string): MonitorSqliteSnapshot {
|
||||
const path = assertReadOnlySqliteSource(source.path);
|
||||
const bytes = readFileSync(path);
|
||||
const fingerprint = `sha256:${sha256(bytes)}`;
|
||||
const sourceBytes = readFileSync(path);
|
||||
const database = new Database(path, { readonly: true });
|
||||
try {
|
||||
const integrity = String(database.query("PRAGMA integrity_check").get()?.integrity_check ?? "unknown");
|
||||
if (integrity !== "ok") throw new Error(`legacy SQLite integrity_check failed: ${integrity}`);
|
||||
const tables = database.query("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all() as { name: string; sql: string }[];
|
||||
const snapshot: MonitorSqliteSnapshot = {
|
||||
schema: MONITOR_SQLITE_MIGRATION_SCHEMA,
|
||||
createdAt: new Date().toISOString(),
|
||||
source: { ...source, path },
|
||||
fingerprint,
|
||||
integrity,
|
||||
byteLength: bytes.byteLength,
|
||||
tables: tables.map((table) => ({
|
||||
name: table.name,
|
||||
sql: table.sql,
|
||||
rowCount: Number(database.query(`SELECT COUNT(*) AS count FROM ${quoteIdentifier(table.name)}`).get()?.count ?? 0),
|
||||
rows: database.query(`SELECT * FROM ${quoteIdentifier(table.name)}`).all().map(normalizeRow),
|
||||
})),
|
||||
artifactLocator: { path: resolve(outputPath), sha256: "", byteLength: 0 },
|
||||
};
|
||||
const locatorSha256 = `sha256:${sha256(Buffer.from(stableJson({ ...snapshot, artifactLocator: { ...snapshot.artifactLocator, sha256: "", byteLength: 0 } })))}`;
|
||||
const materialized = `${stableJson({ ...snapshot, artifactLocator: { ...snapshot.artifactLocator, sha256: locatorSha256, byteLength: 0 } })}\n`;
|
||||
writeSnapshot(outputPath, materialized);
|
||||
const artifact = readFileSync(outputPath);
|
||||
return { ...snapshot, artifactLocator: { path: resolve(outputPath), sha256: locatorSha256, byteLength: artifact.byteLength } };
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
const tables = (database.query("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all() as { name: string; sql: string }[]).map((table) => ({ name: table.name, sql: table.sql, rowCount: Number(database.query(`SELECT COUNT(*) AS count FROM ${quoteIdentifier(table.name)}`).get()?.count ?? 0), rows: database.query(`SELECT * FROM ${quoteIdentifier(table.name)}`).all().map(normalizeRow) }));
|
||||
const unsigned = { schema: MONITOR_SQLITE_MIGRATION_SCHEMA, createdAt: new Date().toISOString(), source: { ...source, path }, fingerprint: `sha256:${sha256(sourceBytes)}`, integrity, byteLength: sourceBytes.byteLength, tables, artifactLocator: { path, sha256: `sha256:${sha256(sourceBytes)}`, sizeBytes: sourceBytes.byteLength, kind: "legacy-sqlite" } };
|
||||
const snapshot: MonitorSqliteSnapshot = { ...unsigned, manifestContentHash: `sha256:${sha256(Buffer.from(monitorCentralStableJson(unsigned)))}` };
|
||||
writeSnapshot(outputPath, `${monitorCentralStableJson(snapshot)}\n`);
|
||||
return snapshot;
|
||||
} finally { database.close(); }
|
||||
}
|
||||
|
||||
export function verifyMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot): Record<string, unknown> {
|
||||
const expected = `sha256:${sha256(Buffer.from(stableJson({ ...snapshot, artifactLocator: { ...snapshot.artifactLocator, sha256: "", byteLength: 0 } })))} ` .trim();
|
||||
const sourceBytes = readFileSync(assertReadOnlySqliteSource(snapshot.artifactLocator.path));
|
||||
const locatorValid = snapshot.artifactLocator.sha256 === `sha256:${sha256(sourceBytes)}` && snapshot.artifactLocator.sizeBytes === sourceBytes.byteLength;
|
||||
const fingerprintValid = snapshot.fingerprint === snapshot.artifactLocator.sha256 && snapshot.byteLength === snapshot.artifactLocator.sizeBytes;
|
||||
const reconciliation = reconcileTables(snapshot.tables);
|
||||
return {
|
||||
ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && snapshot.artifactLocator.sha256 === expected,
|
||||
command: "web-probe sentinel migration verify",
|
||||
fingerprint: snapshot.fingerprint,
|
||||
integrity: snapshot.integrity,
|
||||
reconciliation,
|
||||
artifactLocator: { ...snapshot.artifactLocator, valid: snapshot.artifactLocator.sha256 === expected },
|
||||
valuesRedacted: true,
|
||||
};
|
||||
const manifestContentHashValid = snapshot.manifestContentHash === `sha256:${sha256(Buffer.from(monitorCentralStableJson({ schema: snapshot.schema, createdAt: snapshot.createdAt, source: snapshot.source, fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, byteLength: snapshot.byteLength, tables: snapshot.tables, artifactLocator: snapshot.artifactLocator })))}`;
|
||||
const rowsValid = snapshot.tables.every((table) => table.rowCount === table.rows.length);
|
||||
return { ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && locatorValid && fingerprintValid && manifestContentHashValid && rowsValid, command: "web-probe sentinel migration verify", fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, reconciliation, artifactLocator: { ...snapshot.artifactLocator, valid: locatorValid }, schemaValid: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA, manifestContentHashValid, rowsValid, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export function reconcileTables(tables: readonly MonitorSqliteSnapshot["tables"][number][]): MonitorMigrationReconciliation {
|
||||
export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise<Record<string, unknown>> {
|
||||
const verified = verifyMonitorSqliteSnapshot(snapshot);
|
||||
if (verified.ok !== true) throw Object.assign(new Error("snapshot verification failed"), { code: "monitor-migration-snapshot-invalid", verification: verified });
|
||||
const ingests = canonicalIngests(snapshot);
|
||||
const dispositions = await Promise.all(ingests.map((input) => store.ingest(input)));
|
||||
const reconciliation = reconcileTables(snapshot.tables);
|
||||
return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export function reconcileTables(tables: readonly MonitorSqliteTable[]): MonitorMigrationReconciliation {
|
||||
const count = (names: readonly string[]) => tables.filter((table) => names.includes(table.name)).reduce((total, table) => total + table.rowCount, 0);
|
||||
return {
|
||||
runs: count(["runs", "run"]),
|
||||
findings: count(["findings", "finding"]),
|
||||
metadata: count(["metadata", "run_metadata"]),
|
||||
payloads: count(["payloads", "run_payloads"]),
|
||||
locators: count(["artifact_locators", "locators"]),
|
||||
latest: count(["latest", "latest_runs"]),
|
||||
};
|
||||
return { runs: count(["runs", "run"]), findings: count(["findings", "finding"]), metadata: count(["metadata", "run_metadata"]), payloads: count(["payloads", "run_payloads"]), locators: count(["artifact_locators", "locators"]), latest: count(["latest", "latest_runs"]) };
|
||||
}
|
||||
|
||||
function assertReadOnlySqliteSource(value: string): string {
|
||||
const path = resolve(value);
|
||||
const state = statSync(path);
|
||||
if (!state.isFile()) throw new Error(`legacy SQLite source is not a file: ${path}`);
|
||||
const descriptor = openSync(path, "r");
|
||||
closeSync(descriptor);
|
||||
return path;
|
||||
}
|
||||
|
||||
function writeSnapshot(path: string, content: string): void {
|
||||
const parent = dirname(resolve(path));
|
||||
if (!existsSync(parent)) throw new Error(`snapshot output directory does not exist: ${parent}`);
|
||||
writeFileSync(path, content);
|
||||
}
|
||||
|
||||
function quoteIdentifier(value: string): string {
|
||||
return `"${value.replaceAll("\"", "\"\"")}"`;
|
||||
}
|
||||
|
||||
function normalizeRow(value: unknown): Record<string, unknown> {
|
||||
const row = value as Record<string, unknown>;
|
||||
return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item]));
|
||||
}
|
||||
|
||||
function sha256(value: Uint8Array): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType<typeof normalizeMonitorCentralIngest>[] {
|
||||
const rows = tableRows(snapshot.tables, ["runs", "run"]);
|
||||
const findings = tableRows(snapshot.tables, ["findings", "finding"]);
|
||||
const metadata = tableRows(snapshot.tables, ["metadata", "run_metadata"]);
|
||||
const payloads = tableRows(snapshot.tables, ["payloads", "run_payloads"]);
|
||||
const locators = tableRows(snapshot.tables, ["artifact_locators", "locators"]);
|
||||
return rows.map((row) => {
|
||||
const runId = stringField(row, ["run_id", "id"]);
|
||||
const rowFindings = findings.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => jsonField(item, ["finding", "value_json"]) ?? item);
|
||||
const rowLocators = locators.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot));
|
||||
const payload = jsonField(payloads.find((item) => stringField(item, ["run_id"]) === runId), ["payload", "value_json"]) ?? { legacy: { run: row, metadata: metadata.filter((item) => String(item.key ?? "").includes(runId)), sourceFingerprint: snapshot.fingerprint } };
|
||||
return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: rowFindings, artifactLocators: rowLocators, timeline: [] });
|
||||
});
|
||||
}
|
||||
function locatorFromRow(row: Record<string, unknown>, source: MonitorSqliteSnapshot): MonitorArtifactLocator { return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc: stringField(row, ["owner_pvc", "ownerPvc"]) || "legacy-sqlite", stateDir: stringField(row, ["state_dir", "stateDir"]) || dirname(source.source.path), relativePath: stringField(row, ["relative_path", "relativePath"]) || source.source.path, sha256: stringField(row, ["sha256"]) || source.fingerprint, sizeBytes: numberField(row, ["size_bytes", "sizeBytes"]) ?? source.byteLength, kind: stringField(row, ["kind"]) || "legacy-sqlite", reachable: row.reachable !== false }; }
|
||||
function tableRows(tables: readonly MonitorSqliteTable[], names: readonly string[]): readonly Record<string, unknown>[] { return tables.filter((table) => names.includes(table.name)).flatMap((table) => table.rows); }
|
||||
function sourcePlan(source: MonitorSqliteSource): Record<string, unknown> { const path = resolve(source.path); const state = existsSync(path) ? statSync(path) : null; return { sentinelId: source.sentinelId, node: source.node, lane: source.lane, path, runtimeConfigRef: source.runtimeConfigRef, exists: state !== null, isFile: state?.isFile() ?? false, byteLength: state?.size ?? null, mutation: false, valuesRedacted: true }; }
|
||||
function assertReadOnlySqliteSource(value: string): string { const path = resolve(value); const state = statSync(path); if (!state.isFile()) throw new Error(`legacy SQLite source is not a file: ${path}`); const descriptor = openSync(path, "r"); closeSync(descriptor); return path; }
|
||||
function writeSnapshot(path: string, content: string): void { const parent = dirname(resolve(path)); if (!existsSync(parent)) throw new Error(`snapshot output directory does not exist: ${parent}`); writeFileSync(path, content); }
|
||||
function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; }
|
||||
function normalizeRow(value: unknown): Record<string, unknown> { const row = value as Record<string, unknown>; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); }
|
||||
function jsonField(row: Record<string, unknown> | undefined, fields: readonly string[]): Record<string, unknown> | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null; }
|
||||
function stringField(row: Record<string, unknown> | undefined, fields: readonly string[]): string { const value = row === undefined ? null : fields.map((field) => row[field]).find((item) => typeof item === "string" && item.length > 0); return typeof value === "string" ? value : ""; }
|
||||
function numberField(row: Record<string, unknown>, fields: readonly string[]): number | null { const value = fields.map((field) => row[field]).find((item) => typeof item === "number"); return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; }
|
||||
function sha256(value: Uint8Array): string { return createHash("sha256").update(value).digest("hex"); }
|
||||
|
||||
Reference in New Issue
Block a user