mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: address monitor ingest review blockers

Browse Source
This commit is contained in:
AgentRun Codex
2026-07-12 21:30:17 +00:00
parent 35a59b27c6
commit 9b9a460d37
8 changed files with 198 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/monitor-sqlite-migration.ts
+18 -9
View File
@@ -2,14 +2,14 @@
// Responsibility: Registry-resolved read-only legacy SQLite snapshots and one-shot canonical central-store import.
import { createHash } from "node:crypto";
import { closeSync, existsSync, openSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { dirname, join, resolve } from "node:path";
import { Database } from "bun:sqlite";
import type { MonitorArtifactLocator, MonitorCentralIngest, MonitorCentralStore } from "./monitor-central-persistence";
import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monitor-central-persistence";
export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-2";
export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; }
export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; }
export interface MonitorSqliteTable { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record<string, unknown>[]; }
export interface MonitorSqliteSnapshot {
readonly schema: string; readonly createdAt: string; readonly source: MonitorSqliteSource; readonly fingerprint: string; readonly integrity: string; readonly byteLength: number;
@@ -32,7 +32,7 @@ export function exportMonitorSqliteSnapshot(source: MonitorSqliteSource, outputP
if (integrity !== "ok") throw new Error(`legacy SQLite integrity_check failed: ${integrity}`);
const tables = (database.query("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all() as { name: string; sql: string }[]).map((table) => ({ name: table.name, sql: table.sql, rowCount: Number(database.query(`SELECT COUNT(*) AS count FROM ${quoteIdentifier(table.name)}`).get()?.count ?? 0), rows: database.query(`SELECT * FROM ${quoteIdentifier(table.name)}`).all().map(normalizeRow) }));
const unsigned = { schema: MONITOR_SQLITE_MIGRATION_SCHEMA, createdAt: new Date().toISOString(), source: { ...source, path }, fingerprint: `sha256:${sha256(sourceBytes)}`, integrity, byteLength: sourceBytes.byteLength, tables, artifactLocator: { path, sha256: `sha256:${sha256(sourceBytes)}`, sizeBytes: sourceBytes.byteLength, kind: "legacy-sqlite" } };
const snapshot: MonitorSqliteSnapshot = { ...unsigned, manifestContentHash: `sha256:${sha256(Buffer.from(monitorCentralStableJson(unsigned)))}` };
const snapshot: MonitorSqliteSnapshot = { ...unsigned, manifestContentHash: monitorSnapshotContentHash(unsigned) };
writeSnapshot(outputPath, `${monitorCentralStableJson(snapshot)}\n`);
return snapshot;
} finally { database.close(); }
@@ -43,7 +43,7 @@ export function verifyMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot): Re
const locatorValid = snapshot.artifactLocator.sha256 === `sha256:${sha256(sourceBytes)}` && snapshot.artifactLocator.sizeBytes === sourceBytes.byteLength;
const fingerprintValid = snapshot.fingerprint === snapshot.artifactLocator.sha256 && snapshot.byteLength === snapshot.artifactLocator.sizeBytes;
const reconciliation = reconcileTables(snapshot.tables);
const manifestContentHashValid = snapshot.manifestContentHash === `sha256:${sha256(Buffer.from(monitorCentralStableJson({ schema: snapshot.schema, createdAt: snapshot.createdAt, source: snapshot.source, fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, byteLength: snapshot.byteLength, tables: snapshot.tables, artifactLocator: snapshot.artifactLocator })))}`;
const manifestContentHashValid = snapshot.manifestContentHash === monitorSnapshotContentHash(snapshot);
const rowsValid = snapshot.tables.every((table) => table.rowCount === table.rows.length);
return { ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && locatorValid && fingerprintValid && manifestContentHashValid && rowsValid, command: "web-probe sentinel migration verify", fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, reconciliation, artifactLocator: { ...snapshot.artifactLocator, valid: locatorValid }, schemaValid: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA, manifestContentHashValid, rowsValid, valuesRedacted: true };
}
@@ -52,9 +52,11 @@ export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapsho
const verified = verifyMonitorSqliteSnapshot(snapshot);
if (verified.ok !== true) throw Object.assign(new Error("snapshot verification failed"), { code: "monitor-migration-snapshot-invalid", verification: verified });
const ingests = canonicalIngests(snapshot);
const dispositions = await Promise.all(ingests.map((input) => store.ingest(input)));
const manifest = { manifestId: snapshot.manifestContentHash, sourceFingerprint: snapshot.fingerprint, rowCount: snapshot.tables.reduce((total, table) => total + table.rowCount, 0) };
const batch = await store.importSource(manifest, ingests);
const dispositions = batch.runs;
const reconciliation = reconcileTables(snapshot.tables);
return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true };
return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, manifest, disposition: batch.disposition, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true };
}
export function reconcileTables(tables: readonly MonitorSqliteTable[]): MonitorMigrationReconciliation {
@@ -71,12 +73,17 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType<
return rows.map((row) => {
const runId = stringField(row, ["run_id", "id"]);
const rowFindings = findings.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => jsonField(item, ["finding", "value_json"]) ?? item);
const report = metadataValue(metadata, `run.report.${runId}`);
const rowLocators = locators.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot));
const payload = jsonField(payloads.find((item) => stringField(item, ["run_id"]) === runId), ["payload", "value_json"]) ?? { legacy: { run: row, metadata: metadata.filter((item) => String(item.key ?? "").includes(runId)), sourceFingerprint: snapshot.fingerprint } };
return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: rowFindings, artifactLocators: rowLocators, timeline: [] });
const reportLocator = reportLocatorFromRun(row, snapshot);
const payload = jsonField(payloads.find((item) => stringField(item, ["run_id"]) === runId), ["payload", "value_json"]) ?? { ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } };
const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : [];
return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: rowFindings.length ? rowFindings : reportFindings, artifactLocators: reportLocator ? [...rowLocators, reportLocator] : rowLocators, timeline: [] });
});
}
function locatorFromRow(row: Record<string, unknown>, source: MonitorSqliteSnapshot): MonitorArtifactLocator { return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc: stringField(row, ["owner_pvc", "ownerPvc"]) || "legacy-sqlite", stateDir: stringField(row, ["state_dir", "stateDir"]) || dirname(source.source.path), relativePath: stringField(row, ["relative_path", "relativePath"]) || source.source.path, sha256: stringField(row, ["sha256"]) || source.fingerprint, sizeBytes: numberField(row, ["size_bytes", "sizeBytes"]) ?? source.byteLength, kind: stringField(row, ["kind"]) || "legacy-sqlite", reachable: row.reachable !== false }; }
function locatorFromRow(row: Record<string, unknown>, source: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const hash = stringField(row, ["sha256"]); const sizeBytes = numberField(row, ["size_bytes", "sizeBytes"]); if (!ownerPvc || !relativePath || !hash || sizeBytes === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc, stateDir: stringField(row, ["state_dir", "stateDir"]), relativePath, sha256: hash, sizeBytes, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; }
function reportLocatorFromRun(row: Record<string, unknown>, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator | null { const stateDir = stringField(row, ["state_dir", "stateDir"]); const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!stateDir && !expectedHash) return null; if (!stateDir || !expectedHash) throw Object.assign(new Error("legacy report locator is incomplete"), { code: "monitor-migration-report-locator-invalid" }); const path = join(snapshot.source.stateRoot, stateDir, "analysis", "report.json"); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc: snapshot.source.ownerPvc, stateDir, relativePath: "analysis/report.json", sha256: actualHash, sizeBytes: bytes.byteLength, kind: "report-json", reachable: true }; }
function metadataValue(rows: readonly Record<string, unknown>[], key: string): Record<string, unknown> | null { return jsonField(rows.find((row) => stringField(row, ["key"]) === key), ["value_json", "value"]); }
function tableRows(tables: readonly MonitorSqliteTable[], names: readonly string[]): readonly Record<string, unknown>[] { return tables.filter((table) => names.includes(table.name)).flatMap((table) => table.rows); }
function sourcePlan(source: MonitorSqliteSource): Record<string, unknown> { const path = resolve(source.path); const state = existsSync(path) ? statSync(path) : null; return { sentinelId: source.sentinelId, node: source.node, lane: source.lane, path, runtimeConfigRef: source.runtimeConfigRef, exists: state !== null, isFile: state?.isFile() ?? false, byteLength: state?.size ?? null, mutation: false, valuesRedacted: true }; }
function assertReadOnlySqliteSource(value: string): string { const path = resolve(value); const state = statSync(path); if (!state.isFile()) throw new Error(`legacy SQLite source is not a file: ${path}`); const descriptor = openSync(path, "r"); closeSync(descriptor); return path; }
@@ -87,3 +94,5 @@ function jsonField(row: Record<string, unknown> | undefined, fields: readonly st
function stringField(row: Record<string, unknown> | undefined, fields: readonly string[]): string { const value = row === undefined ? null : fields.map((field) => row[field]).find((item) => typeof item === "string" && item.length > 0); return typeof value === "string" ? value : ""; }
function numberField(row: Record<string, unknown>, fields: readonly string[]): number | null { const value = fields.map((field) => row[field]).find((item) => typeof item === "number"); return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; }
function sha256(value: Uint8Array): string { return createHash("sha256").update(value).digest("hex"); }
export function monitorSnapshotContentHash(snapshot: Omit<MonitorSqliteSnapshot, "manifestContentHash"> | MonitorSqliteSnapshot): string { const { manifestContentHash: _ignored, ...unsigned } = snapshot as MonitorSqliteSnapshot; return `sha256:${sha256(Buffer.from(monitorCentralStableJson(unsigned)))}`; }
function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }