diff --git a/config/hwlab-web-probe-monitor/runtime.yaml b/config/hwlab-web-probe-monitor/runtime.yaml index 32408ad6..2e19b054 100644 --- a/config/hwlab-web-probe-monitor/runtime.yaml +++ b/config/hwlab-web-probe-monitor/runtime.yaml @@ -42,7 +42,7 @@ monitor: - runId: sentinel-run-mrbjo83t-ef79c71d artifactCount: 1 stateDir: .state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5 - reportSha256: sha256:119c9e3ef4cbc41143121e5d1af8955a8454e5efdbb6600f64a207a0f148b33 + reportSha256: sha256:119c9e3ef4cbc41143121ee5d1af8955a8454e5efdbb6600f64a207a0f148b33 - runId: sentinel-run-mrbjsugk-764ff868 artifactCount: 4 stateDir: .state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4 diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index 3de863d0..0f0fe926 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -24,6 +24,7 @@ import { guardedSentinelDeliveryAction, migrationJobFailureSummary, runWebProbeSentinelCommand, + sameMigrationSource, webProbeSentinelDeliveryAuthorityFromCatalog, } from "./hwlab-node-web-sentinel-cicd"; import { parseNodeScopedDelegatedOptions } from "./hwlab-node/plan"; @@ -296,10 +297,16 @@ describe("migrated CLI 执行 guard 与提示清理", () => { }); test("migration Job 失败输出容器终态、有界日志和 Secret 脱敏", () => { - const failure = migrationJobFailureSummary({ podPhase: "Failed", containers: [{ name: "export", phase: "terminated", exitCode: 2, reason: "Error", message: "artifact source unavailable" }], logsTail: "DATABASE_URL=postgres://secret-value export failed" }, {}); + const failure = migrationJobFailureSummary({ podPhase: "Failed", containers: [{ name: "export", phase: "terminated", exitCode: 2, reason: "Error", message: "artifact source unavailable" }], logsTail: "DATABASE_URL=postgres://user:PW123@host/db Bearer token-value password=PW123" }, { ok: false, command: "export", error: { name: "Error", code: "monitor-migration-artifact-path-invalid", message: "cannot export postgres://user:PW123@host/db?token=abc", stack: "stack with password=PW123" } }); expect(failure).toMatchObject({ phase: "Failed", container: "export", exitCode: 2, status: "Error", valuesRedacted: true }); - expect(String(failure.logsTail)).toContain("DATABASE_URL="); - expect(String(failure.logsTail)).not.toContain("secret-value"); + expect(String(failure.message)).toContain("cannot export"); + expect(JSON.stringify(failure)).not.toMatch(/PW123|token-value|abc|stack/iu); + }); + + test("migration source 仅接受按 runId 重排的相同 gap 集合", () => { + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: "/state/index.sqlite", runtimeConfigRef: "fixture#runtime", stateRoot: "/state", ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: "/state" }], legacyArtifactGaps: [{ runId: "b", artifactCount: 2, stateDir: ".state/b", reportSha256: `sha256:${"b".repeat(64)}` }, { runId: "a", artifactCount: 1, stateDir: ".state/a", reportSha256: `sha256:${"a".repeat(64)}` }] }; + expect(sameMigrationSource(source, { ...source, legacyArtifactGaps: [...source.legacyArtifactGaps].reverse() })).toBe(true); + expect(sameMigrationSource(source, { ...source, legacyArtifactGaps: [{ ...source.legacyArtifactGaps[0], artifactCount: 3 }, source.legacyArtifactGaps[1]] })).toBe(false); }); test("PaC 外层事件与内层 deterministic publish 分开归类，只有外层可驱动 delivery", () => { diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index 0118baed..336b567f 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -49,9 +49,10 @@ test("one-shot migration helper accepts a unique control-plane suffix", () => { test("Release A declares exactly the two approved pre-migration artifact gaps", () => { expect(releaseAMonitorMigrationArtifactGaps(hwlabRuntimeLaneSpecForNode("v03", "NC01"), source)).toEqual([ - expect.objectContaining({ runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1 }), + expect.objectContaining({ runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1, reportSha256: "sha256:119c9e3ef4cbc41143121ee5d1af8955a8454e5efdbb6600f64a207a0f148b33" }), expect.objectContaining({ runId: "sentinel-run-mrbjsugk-764ff868", artifactCount: 4 }), ]); + expect(releaseAMonitorMigrationArtifactGaps(hwlabRuntimeLaneSpecForNode("v03", "NC01"), source).every((gap) => /^sha256:[a-f0-9]{64}$/u.test(gap.reportSha256))).toBe(true); }); test("Release A declares the controlled NC01 monitor database Secret projection", () => { diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index a6fddb38..af2eaea7 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -44,12 +44,14 @@ export function releaseAMonitorMigrationArtifactGaps(spec: HwlabRuntimeLaneSpec, const gap = item; return { runId: textAt(gap, "runId", resolved.ref), artifactCount: integerAt(gap, "artifactCount", resolved.ref), stateDir: textAt(gap, "stateDir", resolved.ref), reportSha256: textAt(gap, "reportSha256", resolved.ref) }; }); - const expected = [ - { runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5", reportSha256: "sha256:119c9e3ef4cbc41143121e5d1af8955a8454e5efdbb6600f64a207a0f148b33" }, - { runId: "sentinel-run-mrbjsugk-764ff868", artifactCount: 4, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4", reportSha256: "sha256:d2dbc0fc98f39d7373b9ae009e31fcd8d666014c2e32428f235150ee59666048" }, - ]; - if (JSON.stringify(gaps) !== JSON.stringify(expected)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps must declare exactly the two approved pre-migration artifact gaps`); - return gaps; + const runIds = new Set(); + for (const gap of gaps) { + if (runIds.has(gap.runId)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps runId values must be unique`); + if (gap.stateDir === "." || !gap.stateDir.startsWith(".") || gap.stateDir.includes("\\") || gap.stateDir.split("/").includes("..")) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps stateDir must be a safe relative path`); + if (!/^sha256:[a-f0-9]{64}$/u.test(gap.reportSha256)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps reportSha256 must be sha256:<64 lowercase hex>`); + runIds.add(gap.runId); + } + return [...gaps].sort((left, right) => left.runId.localeCompare(right.runId)); } function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly resolved: ReturnType; readonly runtime: Record; readonly migration: Record; readonly job: Record; readonly namespace: string; readonly deploymentName: string; readonly port: number; readonly secret: Record; readonly pool: Record; readonly labels: Record } { diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 159b0d0a..889ff35c 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -251,7 +251,7 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract, paylo const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord).map((item) => ({ name: stringAtNullable(item, "name"), phase: stringAtNullable(item, "phase"), exitCode: finiteNumberOrNull(item.exitCode), reason: stringAtNullable(item, "reason"), message: short(stringAtNullable(item, "message") ?? "") })).filter((item) => item.phase === "terminated" || item.exitCode !== null || item.reason !== null) : []; const failed = containers.find((item) => item.exitCode !== null && item.exitCode !== 0) ?? containers[0] ?? null; const logsTail = redactMigrationJobLogTail(short(String(probe.logsTail ?? ""))); - return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, status: failed?.reason ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactMigrationJobLogTail(short(stringAtNullable(payload, "error") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted")), logsTail, valuesRedacted: true }; + const nestedError = isRecord(payload.error) ? payload.error : null; + return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, status: failed?.reason ?? stringAtNullable(nestedError, "code") ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactMigrationJobLogTail(short(stringAtNullable(nestedError, "message") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted")), logsTail, valuesRedacted: true }; } function redactMigrationJobLogTail(value: string): string { - return value.replace(/(DATABASE_URL|password|token|secret)\s*[=:]\s*[^\s]+/giu, "$1="); + return value + .replace(/\b(?:postgres(?:ql)?|mysql):\/\/[^\s"']+/giu, "") + .replace(/\bBearer\s+[^\s"']+/giu, "Bearer ") + .replace(/(["']?(?:DATABASE_URL|password|token|secret)["']?\s*[=:]\s*["']?)[^\s,"'}\]]+/giu, "$1") + .replace(/([?&](?:password|token|secret)=[^&\s"']+)/giu, ""); } function migrationJobFailureNext(state: SentinelCicdState, namespace: string, jobName: string): Record { return { logs: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${jobName} --all-containers=true --tail=120`, describe: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${jobName}` }; } -function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean { - return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath); +export function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean { + return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath) && monitorGapSet(left.legacyArtifactGaps) === monitorGapSet(right.legacyArtifactGaps); +} + +function monitorGapSet(gaps: readonly { readonly runId: string; readonly artifactCount: number; readonly stateDir: string; readonly reportSha256: string }[] | undefined): string { + return JSON.stringify([...(gaps ?? [])].sort((left, right) => left.runId.localeCompare(right.runId))); } function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string | null): readonly MonitorSqliteSource[] { diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index 1b3e8290..22a296fa 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -26,6 +26,7 @@ export interface MonitorMigrationReconciliation { readonly runs: number; readonl export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise> { const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot); + verifySnapshotReferentialIntegrity(snapshot); verifyFrozenArtifactManifests(snapshot); const expected = canonicalIngests(snapshot); const expectedMetadata = canonicalRuntimeMetadata(snapshot); @@ -36,6 +37,7 @@ export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorS checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && actual?.updatedAt === input.updatedAt && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); checks.push({ name: "finding", runId: input.runId, ok: monitorCentralStableJson(actual?.findings) === monitorCentralStableJson(input.findings), valuesRedacted: true }); checks.push({ name: "artifactLocator", runId: input.runId, ok: monitorCentralStableJson(actual?.artifactLocators) === monitorCentralStableJson(input.artifactLocators), valuesRedacted: true }); + checks.push({ name: "timeline", runId: input.runId, ok: Array.isArray(actual?.timeline) && actual.timeline.length === input.timeline.length && monitorCentralStableJson(actual.timeline) === monitorCentralStableJson(input.timeline), valuesRedacted: true }); const views = isRecord(input.payload.views) ? input.payload.views : {}; for (const [name, value] of Object.entries(views)) { const actualView = await store.view(scope, input.runId, name); @@ -93,12 +95,15 @@ export function verifyMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot): Re const reconciliation = reconcileTables(snapshot.tables); const manifestContentHashValid = snapshot.manifestContentHash === monitorSnapshotContentHash(snapshot); const rowsValid = snapshot.tables.every((table) => table.rowCount === table.rows.length); - return { ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && locatorValid && fingerprintValid && manifestContentHashValid && rowsValid, command: "web-probe sentinel migration verify", fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, reconciliation, artifactLocator: { ...snapshot.artifactLocator, valid: locatorValid }, schemaValid: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA, manifestContentHashValid, rowsValid, valuesRedacted: true }; + let referentialValid = true; + try { verifySnapshotReferentialIntegrity(snapshot); } catch { referentialValid = false; } + return { ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && locatorValid && fingerprintValid && manifestContentHashValid && rowsValid && referentialValid, command: "web-probe sentinel migration verify", fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, reconciliation, artifactLocator: { ...snapshot.artifactLocator, valid: locatorValid }, schemaValid: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA, manifestContentHashValid, rowsValid, referentialValid, valuesRedacted: true }; } export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise> { const verified = verifyMonitorSqliteSnapshot(snapshot); if (verified.ok !== true) throw Object.assign(new Error("snapshot verification failed"), { code: "monitor-migration-snapshot-invalid", verification: verified }); + verifySnapshotReferentialIntegrity(snapshot); verifyFrozenArtifactManifests(snapshot); const ingests = canonicalIngests(snapshot); const runtimeMetadata = canonicalRuntimeMetadata(snapshot); @@ -152,9 +157,65 @@ function artifactSourceFor(source: MonitorSqliteSource, ownerPvc: string): Monit function artifactSourceForStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { const directory = resolve(stateDir); const matches = source.artifactSources.filter((item) => directory === resolve(item.mountPath) || directory.startsWith(`${resolve(item.mountPath)}/`)).sort((left, right) => resolve(right.mountPath).length - resolve(left.mountPath).length); if (matches.length === 0) throw Object.assign(new Error(`legacy artifact stateDir is not registry-mounted: ${stateDir}`), { code: "monitor-migration-artifact-source-missing" }); return matches[0]!; } function artifactPath(mountPath: string, stateDir: string, relativePath: string): string { const root = resolve(mountPath); const directory = stateDir.startsWith("/") ? resolve(stateDir) : resolve(root, stateDir); if (directory !== root && !directory.startsWith(`${root}/`)) throw Object.assign(new Error("legacy artifact stateDir escapes registry mount"), { code: "monitor-migration-artifact-path-invalid" }); return join(directory, relativePath); } function artifactFiles(directory: string): readonly string[] { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); if (entry.isDirectory()) return artifactFiles(path); return entry.isFile() ? [path] : []; }).sort(); } -function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void { for (const manifest of snapshot.artifactManifests) { if (manifest.missingBeforeMigration !== undefined) { if (manifest.locators.length !== 0 || !matchesArtifactGap(manifest.missingBeforeMigration, snapshot.source.legacyArtifactGaps ?? [])) throw Object.assign(new Error(`invalid frozen legacy artifact gap for ${manifest.runId}`), { code: "monitor-migration-artifact-gap-identity-invalid" }); continue; } for (const locator of manifest.locators) { const source = artifactSourceFor(snapshot.source, locator.ownerPvc); const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath); if (!existsSync(path) || !statSync(path).isFile()) throw Object.assign(new Error(`frozen artifact is unavailable for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: null, valuesRedacted: true } }); const bytes = readFileSync(path); if (`sha256:${sha256(bytes)}` !== locator.sha256 || bytes.byteLength !== locator.sizeBytes) throw Object.assign(new Error(`frozen artifact changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: manifest.locators.length, valuesRedacted: true } }); } } } +function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void { + for (const manifest of snapshot.artifactManifests) { + if (manifest.missingBeforeMigration !== undefined) { + const gap = manifest.missingBeforeMigration; + if (manifest.locators.length !== 0 || !matchesArtifactGap(gap, snapshot.source.legacyArtifactGaps ?? [])) throw Object.assign(new Error(`invalid frozen legacy artifact gap for ${manifest.runId}`), { code: "monitor-migration-artifact-gap-identity-invalid" }); + for (const source of snapshot.source.artifactSources) { + const directory = artifactPath(source.mountPath, gap.stateDir, ""); + if (existsSync(directory) && (statSync(directory).isFile() || (statSync(directory).isDirectory() && artifactFiles(directory).length > 0))) throw Object.assign(new Error(`legacy artifact bytes recovered for ${manifest.runId}; re-export is required`), { code: "monitor-migration-artifact-gap-recovered", diagnostic: { runId: manifest.runId, stateDir: gap.stateDir, valuesRedacted: true } }); + } + continue; + } + const expected = new Map(manifest.locators.map((locator) => [`${locator.ownerPvc}\u0000${locator.stateDir}\u0000${locator.relativePath}`, locator])); + const observed = new Map(); + for (const locator of manifest.locators) { + const source = artifactSourceFor(snapshot.source, locator.ownerPvc); + const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath); + if (!existsSync(path) || !statSync(path).isFile()) throw frozenArtifactInvalid(manifest, null); + const bytes = readFileSync(path); + const actual = { ...locator, sha256: `sha256:${sha256(bytes)}`, sizeBytes: bytes.byteLength }; + if (actual.sha256 !== locator.sha256 || actual.sizeBytes !== locator.sizeBytes) throw frozenArtifactInvalid(manifest, manifest.locators.length); + observed.set(`${locator.ownerPvc}\u0000${locator.stateDir}\u0000${locator.relativePath}`, actual); + } + for (const locator of manifest.locators) { + const source = artifactSourceFor(snapshot.source, locator.ownerPvc); + const directory = artifactPath(source.mountPath, locator.stateDir, ""); + if (!existsSync(directory) || !statSync(directory).isDirectory()) throw frozenArtifactInvalid(manifest, null); + for (const path of artifactFiles(directory)) { + const relativePath = path.slice(directory.length + 1).replaceAll("\\", "/"); + const key = `${locator.ownerPvc}\u0000${locator.stateDir}\u0000${relativePath}`; + if (!expected.has(key)) throw frozenArtifactInvalid(manifest, manifest.locators.length + 1); + } + } + if (observed.size !== expected.size) throw frozenArtifactInvalid(manifest, observed.size); + } +} + +function frozenArtifactInvalid(manifest: MonitorArtifactManifest, observedCount: number | null): Error { return Object.assign(new Error(`frozen artifact set changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: manifest.locators[0]?.stateDir ?? manifest.missingBeforeMigration?.stateDir ?? "", expectedCount: manifest.locators.length, observedCount, valuesRedacted: true } }); } +function verifySnapshotReferentialIntegrity(snapshot: MonitorSqliteSnapshot): void { + const rows = tableRows(snapshot.tables, ["runs", "run"]); + const rowByRunId = new Map>(); + for (const row of rows) { const runId = stringField(row, ["run_id", "id"]); if (!runId || rowByRunId.has(runId)) throw Object.assign(new Error("snapshot run identities are invalid"), { code: "monitor-migration-snapshot-referential-invalid" }); rowByRunId.set(runId, row); } + const manifestByRunId = new Map(); + for (const manifest of snapshot.artifactManifests) { if (!manifest.runId || manifestByRunId.has(manifest.runId) || !rowByRunId.has(manifest.runId)) throw Object.assign(new Error("snapshot artifact manifests are invalid"), { code: "monitor-migration-snapshot-referential-invalid" }); manifestByRunId.set(manifest.runId, manifest); } + if (manifestByRunId.size !== rowByRunId.size) throw Object.assign(new Error("snapshot artifact manifests must match SQLite runs"), { code: "monitor-migration-snapshot-referential-invalid" }); + const declared = snapshot.source.legacyArtifactGaps ?? []; + const declaredByRunId = new Map(); + for (const gap of declared) { if (!gap.runId || declaredByRunId.has(gap.runId)) throw Object.assign(new Error("snapshot legacy artifact gaps are invalid"), { code: "monitor-migration-snapshot-referential-invalid" }); declaredByRunId.set(gap.runId, gap); } + for (const [runId, manifest] of manifestByRunId) { + const row = rowByRunId.get(runId)!; + const gap = manifest.missingBeforeMigration; + if (gap === undefined) { if (declaredByRunId.has(runId)) throw Object.assign(new Error("declared gap is missing its frozen manifest"), { code: "monitor-migration-snapshot-referential-invalid" }); continue; } + const declaredGap = declaredByRunId.get(runId); + if (gap.runId !== runId || declaredGap === undefined || !sameArtifactGap(gap, declaredGap) || gap.artifactCount !== numberField(row, ["artifact_count", "artifactCount"]) || gap.stateDir !== stringField(row, ["state_dir", "stateDir"]) || gap.reportSha256 !== stringField(row, ["report_json_sha256", "reportJsonSha256"])) throw Object.assign(new Error("snapshot legacy artifact gap does not match run identity"), { code: "monitor-migration-snapshot-referential-invalid" }); + } + if ([...declaredByRunId.keys()].some((runId) => manifestByRunId.get(runId)?.missingBeforeMigration === undefined)) throw Object.assign(new Error("snapshot declared gap has no matching frozen manifest"), { code: "monitor-migration-snapshot-referential-invalid" }); +} function artifactCountMismatch(runId: string, stateDir: string, expectedCount: number, observedCount: number): Error { return Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch", diagnostic: { runId, stateDir, expectedCount, observedCount, valuesRedacted: true } }); } function artifactGapIdentityInvalid(runId: string, expected: MonitorLegacyArtifactGap, observed: Record): Error { return Object.assign(new Error(`legacy artifact gap identity mismatch for ${runId}`), { code: "monitor-migration-artifact-gap-identity-invalid", diagnostic: { runId, expected, observed, valuesRedacted: true } }); } +function sameArtifactGap(left: MonitorLegacyArtifactGap, right: MonitorLegacyArtifactGap): boolean { return left.runId === right.runId && left.artifactCount === right.artifactCount && left.stateDir === right.stateDir && left.reportSha256 === right.reportSha256; } function matchesArtifactGap(gap: MonitorLegacyArtifactGap, declared: readonly MonitorLegacyArtifactGap[]): boolean { return declared.some((item) => item.runId === gap.runId && item.artifactCount === gap.artifactCount && item.stateDir === gap.stateDir && item.reportSha256 === gap.reportSha256); } function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; } function normalizeRow(value: unknown): Record { const row = value as Record; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index e0d997e2..718c05c8 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -8,7 +8,7 @@ import test from "node:test"; import { Database } from "bun:sqlite"; import { createInMemoryMonitorCentralStore } from "./monitor-central-persistence"; -import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration"; +import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, monitorSnapshotContentHash, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration"; import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter, submitMonitorTerminalIngest } from "./monitor-terminal-ingest"; import { compactWebObserveAnalyzeAnalysisForRaw } from "./hwlab-node/web-probe-observe"; @@ -87,7 +87,7 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary"); const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store); assert.equal(pgVerification.ok, true); - assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalOverview", "run", "runtimeMetadata", "scopedOverview", "view"]); + assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalOverview", "run", "runtimeMetadata", "scopedOverview", "timeline", "view"]); assert.deepEqual((pgVerification.reconciliation as { target: Record }).target, { runs: 228, findings: 242, payloads: 228, artifactLocators: 1, views: 1, runtimeMetadata: 234 }); await store.importSource({ manifestId: "sha256:orphan-metadata", sourceFingerprint: "sha256:orphan-metadata", rowCount: 0 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "orphan", value: { unexpected: true }, updatedAt: snapshot.createdAt }]); assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, false); @@ -192,6 +192,9 @@ test("workspace artifact source verifies locator identity hash and size", async writeFileSync(join(stateDir, "artifacts", "request.txt"), "changed\n"); await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid"); writeFileSync(join(stateDir, "artifacts", "request.txt"), artifacts[2][1]); + writeFileSync(join(stateDir, "artifacts", "new-after-export.txt"), "new\n"); + await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid"); + rmSync(join(stateDir, "artifacts", "new-after-export.txt")); rmSync(join(stateDir, "artifacts", "response.txt")); await assert.rejects(() => verifyMonitorSqliteSnapshotAgainstStore(snapshot, store), (error: unknown) => { const migrationError = error as { code?: string; diagnostic?: Record }; return migrationError.code === "monitor-migration-frozen-artifact-invalid" && migrationError.diagnostic?.runId === "run-1" && migrationError.diagnostic?.stateDir === stateDir && migrationError.diagnostic?.expectedCount === 5; }); } finally { @@ -204,7 +207,7 @@ test("declared pre-migration artifact gaps import empty locators and fail closed try { const sqlitePath = join(directory, "index.sqlite"); const gaps = [ - { runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5", reportSha256: "sha256:119c9e3ef4cbc41143121e5d1af8955a8454e5efdbb6600f64a207a0f148b33" }, + { runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5", reportSha256: "sha256:119c9e3ef4cbc41143121ee5d1af8955a8454e5efdbb6600f64a207a0f148b33" }, { runId: "sentinel-run-mrbjsugk-764ff868", artifactCount: 4, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4", reportSha256: "sha256:d2dbc0fc98f39d7373b9ae009e31fcd8d666014c2e32428f235150ee59666048" }, ]; const database = new Database(sqlitePath); @@ -223,6 +226,17 @@ test("declared pre-migration artifact gaps import empty locators and fail closed assert.equal((run?.timeline.at(-1) as { bytesVerified?: boolean }).bytesVerified, false); } assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true); + const timelineTamperedStore = { ...store, async run(scope: { sentinelId: string; node: string; lane: string }, runId: string) { const run = await store.run(scope, runId); return run === null ? null : { ...run, timeline: [] }; } }; + assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, timelineTamperedStore)).ok, false); + const swappedManifests = snapshot.artifactManifests.map((manifest, index) => ({ ...manifest, missingBeforeMigration: { ...gaps[(index + 1) % gaps.length] } })); + const swappedUnsigned = { ...snapshot, artifactManifests: swappedManifests }; + const swappedSnapshot = { ...swappedUnsigned, manifestContentHash: monitorSnapshotContentHash(swappedUnsigned) }; + assert.equal(verifyMonitorSqliteSnapshot(swappedSnapshot).ok, false); + const recoveredPath = join(directory, gaps[0].stateDir, "recovered.bin"); + mkdirSync(join(recoveredPath, ".."), { recursive: true }); + writeFileSync(recoveredPath, "recovered\n"); + await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-recovered"); + rmSync(recoveredPath); assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], artifactCount: 2 }, gaps[1]] }, join(directory, "changed.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid"); assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], stateDir: "wrong-state-dir" }, gaps[1]] }, join(directory, "wrong-state-dir.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid"); assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], reportSha256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, gaps[1]] }, join(directory, "wrong-report.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid");