diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 766474b6..13675bfe 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -2176,7 +2176,7 @@ function compactWebObserveAnalyzePayloadForRaw(payload: Record, }; } -function compactWebObserveAnalyzeAnalysisForRaw(analysis: Record): Record { +export function compactWebObserveAnalyzeAnalysisForRaw(analysis: Record): Record { const counts = recordValue(analysis.counts); const archiveSummary = recordValue(analysis.archiveSummary); const requestRate = recordValue(analysis.requestRate); @@ -2217,8 +2217,10 @@ function compactWebObserveAnalyzeAnalysisForRaw(analysis: Record { +export function compactWebObserveAnalyzeAnalyzerForRaw(value: unknown): Record { const analyzer = recordValue(value); return { exitCode: numberOrNullValue(analyzer.exitCode), @@ -2365,6 +2367,7 @@ function compactWebObserveAnalyzeAnalyzerForRaw(value: unknown): Record { +test("stable JSON accepts optional fields and in-memory view preserves the #1878 contract", async () => { assert.equal(monitorCentralStableJson({ optional: undefined, value: "ok", list: [undefined] }), '{"list":[null],"value":"ok"}'); const store = createInMemoryMonitorCentralStore(); const normalized = normalizeMonitorCentralIngest({ ...input("top-level-view"), payload: { views: { summary: { state: "ready" } }, optional: undefined } }); await store.ingest(normalized); - assert.deepEqual(await store.view({ sentinelId: "sentinel-a", node: "NC01", lane: "v03" }, "top-level-view", "summary"), { run: { sentinelId: "sentinel-a", node: "NC01", lane: "v03", runId: "top-level-view", updatedAt: normalized.updatedAt, status: normalized.status, payloadHash: normalized.payloadHash }, view: "summary", value: { state: "ready" } }); + assert.deepEqual(await store.view({ sentinelId: "sentinel-a", node: "NC01", lane: "v03" }, "top-level-view", "summary"), { runId: "top-level-view", name: "summary", value: { state: "ready" } }); }); test("global and scoped latest use updatedAt then runId descending", async () => { diff --git a/scripts/src/monitor-central-persistence.ts b/scripts/src/monitor-central-persistence.ts index be289634..47a75450 100644 --- a/scripts/src/monitor-central-persistence.ts +++ b/scripts/src/monitor-central-persistence.ts @@ -245,7 +245,7 @@ export function createPostgresMonitorCentralStore(databaseUrl: string, options: if (!run) return null; const views = record(run.payload.views); const value = views[view]; - return value === undefined ? null : { run: runFrom(run), view, value }; + return value === undefined ? null : { runId, name: view, value }; }, async findings(scope, options) { const where = scopeWhere(scope); @@ -365,7 +365,7 @@ export function createInMemoryMonitorCentralStore(): MonitorCentralStore { const detail = await this.run(scope, runId); const views = detail?.payload && typeof detail.payload === "object" ? (detail.payload as Record).views : null; const value = views && typeof views === "object" ? (views as Record)[view] : undefined; - return value === undefined || !detail ? null : { run: runFrom(detail as MonitorRun), view, value }; + return value === undefined ? null : { runId, name: view, value }; }, async findings(scope, options) { return sortRuns([...records.values()].filter((item) => matches(scope, item)).map(runFrom)).flatMap((run) => { diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index fb2c6cba..f874f80b 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -77,20 +77,22 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< const locators = tableRows(snapshot.tables, ["artifact_locators", "locators"]); return rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); - const rowFindings = findings.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => jsonField(item, ["finding", "value_json"]) ?? item); + const rowFindings = findings.filter((item) => stringField(item, ["run_id"]) === runId).map(normalizeLegacyFinding); const report = metadataValue(metadata, `run.report.${runId}`); const rowLocators = locators.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); const reportLocator = reportLocatorFromRun(row, snapshot); const storedPayload = jsonField(payloads.find((item) => stringField(item, ["run_id"]) === runId), ["payload", "value_json"]); const payload = { ...(storedPayload ?? {}), ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } }; const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : []; - const canonicalFindings = reportFindings.length ? [...reportFindings, ...rowFindings.filter((item) => !reportFindings.some((reportFinding) => reportFinding.id === item.id))] : rowFindings; + const canonicalFindings = dedupeFindings(reportFindings, rowFindings); return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators: reportLocator ? [...rowLocators, reportLocator] : rowLocators, timeline: [] }); }); } function locatorFromRow(row: Record, source: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const hash = stringField(row, ["sha256"]); const sizeBytes = numberField(row, ["size_bytes", "sizeBytes"]); if (!ownerPvc || !relativePath || !hash || sizeBytes === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc, stateDir: stringField(row, ["state_dir", "stateDir"]), relativePath, sha256: hash, sizeBytes, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } function reportLocatorFromRun(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator | null { const stateDir = stringField(row, ["state_dir", "stateDir"]); const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!stateDir && !expectedHash) return null; if (!stateDir || !expectedHash) throw Object.assign(new Error("legacy report locator is incomplete"), { code: "monitor-migration-report-locator-invalid" }); const path = join(snapshot.source.stateRoot, stateDir, "analysis", "report.json"); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc: snapshot.source.ownerPvc, stateDir, relativePath: "analysis/report.json", sha256: actualHash, sizeBytes: bytes.byteLength, kind: "report-json", reachable: true }; } function metadataValue(rows: readonly Record[], key: string): Record | null { return jsonField(rows.find((row) => stringField(row, ["key"]) === key), ["value_json", "value"]); } +function normalizeLegacyFinding(row: Record): Record { return jsonField(row, ["finding", "value_json"]) ?? { id: stringField(row, ["finding_id", "id"]), severity: stringField(row, ["severity"]), count: numberField(row, ["count"]) ?? 1, summary: stringField(row, ["summary"]), reportJsonSha256: stringField(row, ["report_json_sha256", "reportJsonSha256"]), createdAt: stringField(row, ["created_at", "createdAt"]) }; } +function dedupeFindings(primary: readonly Record[], secondary: readonly Record[]): readonly Record[] { const seen = new Set(); return [...primary, ...secondary].filter((finding) => { const id = stringField(finding, ["id"]); const key = id || monitorCentralStableJson(finding); if (seen.has(key)) return false; seen.add(key); return true; }); } function tableRows(tables: readonly MonitorSqliteTable[], names: readonly string[]): readonly Record[] { return tables.filter((table) => names.includes(table.name)).flatMap((table) => table.rows); } function sourcePlan(source: MonitorSqliteSource): Record { const path = resolve(source.path); const state = existsSync(path) ? statSync(path) : null; return { sentinelId: source.sentinelId, node: source.node, lane: source.lane, path, runtimeConfigRef: source.runtimeConfigRef, exists: state !== null, isFile: state?.isFile() ?? false, byteLength: state?.size ?? null, mutation: false, valuesRedacted: true }; } function assertReadOnlySqliteSource(value: string): string { const path = resolve(value); const state = statSync(path); if (!state.isFile()) throw new Error(`legacy SQLite source is not a file: ${path}`); const descriptor = openSync(path, "r"); closeSync(descriptor); return path; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index da3e79c4..8a834366 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -9,7 +9,7 @@ import { Database } from "bun:sqlite"; import { createInMemoryMonitorCentralStore } from "./monitor-central-persistence"; import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot } from "./monitor-sqlite-migration"; import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter, submitMonitorTerminalIngest } from "./monitor-terminal-ingest"; -import { analysisSummaryFromAnalyzeResult } from "./hwlab-node-web-sentinel-p5-observe"; +import { compactWebObserveAnalyzeAnalysisForRaw } from "./hwlab-node/web-probe-observe"; test("terminal ingest payload is immutable and retries bounded failures", async () => { const report = { ok: false, findings: [{ id: "blocked" }], updatedAt: "2026-07-12T00:00:00.000Z" }; @@ -36,9 +36,9 @@ test("terminal writer requires explicit mutually exclusive authority", () => { assert.throws(() => resolveMonitorTerminalWriter({ sqlite: { path: "/state/legacy.sqlite" }, monitor: { terminalWriter: { mode: "central", ingest: { endpoint: "https://monitor.example.test/api/ingest", timeoutMs: 1_000, retry: { maxAttempts: 2, delayMs: 0 } } } } })); }); -test("runner-shaped analyze result projects nested report artifact facts", () => { - const summary = analysisSummaryFromAnalyzeResult({ ok: true, parsed: { data: { analysis: { analyzer: { stateDir: "run-1", reportJsonPath: "/state/run-1/analysis/report.json", reportJsonSha256: `sha256:${"c".repeat(64)}`, reportJsonBytes: 12, reportUpdatedAt: "2026-07-12T00:00:00.000Z" } } } }, result: {} } as never, null); - assert.deepEqual(summary && { stateDir: summary.stateDir, reportJsonSha256: summary.reportJsonSha256, reportJsonBytes: summary.reportJsonBytes, reportUpdatedAt: summary.reportUpdatedAt }, { stateDir: "run-1", reportJsonSha256: `sha256:${"c".repeat(64)}`, reportJsonBytes: 12, reportUpdatedAt: "2026-07-12T00:00:00.000Z" }); +test("real compact analyze projects stable report time and artifact facts", () => { + const compact = compactWebObserveAnalyzeAnalysisForRaw({ updatedAt: "2026-07-12T00:00:00.000Z", analyzer: { reportJsonPath: "/state/run-1/analysis/report.json", reportJsonSha256: `sha256:${"c".repeat(64)}`, reportJsonBytes: 12 } }); + assert.deepEqual({ reportUpdatedAt: compact.reportUpdatedAt, reportJsonSha256: compact.reportJsonSha256, reportJsonBytes: compact.reportJsonBytes }, { reportUpdatedAt: "2026-07-12T00:00:00.000Z", reportJsonSha256: `sha256:${"c".repeat(64)}`, reportJsonBytes: 12 }); }); test("SQLite snapshot reconciles the 228/242/234 fixture shape without production assumptions", async () => { @@ -53,10 +53,10 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio writeFileSync(reportPath, JSON.stringify({ updatedAt: "2026-07-12T00:00:00.000Z", views: { summary: { title: "legacy summary" } }, findings: [{ id: "report-finding" }] })); const reportHash = `sha256:${Bun.CryptoHasher.hash("sha256", readFileSync(reportPath), "hex")}`; database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, report_json_sha256 TEXT, updated_at TEXT)"); - database.run("CREATE TABLE findings (run_id TEXT, finding TEXT)"); + database.run("CREATE TABLE findings (run_id TEXT, sentinel_id TEXT, finding_id TEXT, severity TEXT, count INTEGER, summary TEXT, report_json_sha256 TEXT, created_at TEXT)"); database.run("CREATE TABLE metadata (id TEXT, key TEXT, value_json TEXT)"); for (let index = 0; index < 228; index += 1) database.run("INSERT INTO runs VALUES (?, ?, ?, ?)", [`run-${index}`, index === 0 ? "run-0" : null, index === 0 ? reportHash : null, "2026-07-12T00:00:00.000Z"]); - for (let index = 0; index < 242; index += 1) database.run("INSERT INTO findings VALUES (?, ?)", [index === 0 ? "run-0" : `run-${index % 228}`, JSON.stringify({ id: index === 0 ? "report-finding" : `finding-${index}`, summary: "indexed" })]); + for (let index = 0; index < 242; index += 1) database.run("INSERT INTO findings VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [index === 0 ? "run-0" : `run-${index % 228}`, "fixture", index === 0 ? "report-finding" : `finding-${index}`, "warning", 1, "indexed", reportHash, "2026-07-12T00:00:00.000Z"]); for (let index = 0; index < 234; index += 1) database.run("INSERT INTO metadata VALUES (?, ?, ?)", [`metadata-${index}`, index === 0 ? "run.report.run-0" : `metadata-${index}`, index === 0 ? JSON.stringify({ views: { summary: { title: "legacy summary" } }, findings: [{ id: "report-finding", rootCause: "complete metadata detail" }] }) : "{}"]); database.close(); const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot, ownerPvc: "fixture-pvc" }; @@ -73,6 +73,8 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio assert.deepEqual((await importMonitorSqliteSnapshot(snapshot, store)).imported, { inserted: 0, idempotent: 228 }); const importedRun = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0"); assert.equal(importedRun?.artifactLocators[0]?.sha256, reportHash); + const reportFindings = importedRun?.findings.filter((finding) => finding.id === "report-finding") ?? []; + assert.deepEqual(reportFindings, [{ id: "report-finding", rootCause: "complete metadata detail" }]); assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary"); assert.equal(verifyMonitorSqliteSnapshot({ ...snapshot, manifestContentHash: "sha256:tampered" }).ok, false); } finally {