From 3aff835c4af5a0c92014966c6d3b3d56c2aaaba7 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 01:08:53 +0000 Subject: [PATCH] fix: expose monitor migration artifact gaps --- config/hwlab-web-probe-monitor/runtime.yaml | 9 ++++ scripts/src/cicd-delivery-authority.test.ts | 8 ++++ .../src/hwlab-node-web-probe-monitor.test.ts | 9 +++- scripts/src/hwlab-node-web-probe-monitor.ts | 22 ++++++++-- .../src/hwlab-node-web-sentinel-cicd-jobs.ts | 11 +++-- scripts/src/hwlab-node-web-sentinel-cicd.ts | 22 ++++++++-- scripts/src/monitor-sqlite-migration.ts | 20 +++++---- .../monitor-terminal-ingest-migration.test.ts | 42 +++++++++++++++++++ 8 files changed, 125 insertions(+), 18 deletions(-) diff --git a/config/hwlab-web-probe-monitor/runtime.yaml b/config/hwlab-web-probe-monitor/runtime.yaml index 057fa8eb..32408ad6 100644 --- a/config/hwlab-web-probe-monitor/runtime.yaml +++ b/config/hwlab-web-probe-monitor/runtime.yaml @@ -38,6 +38,15 @@ monitor: workflow: export-import-verify readOnlySource: true enabled: false + legacyArtifactGaps: + - runId: sentinel-run-mrbjo83t-ef79c71d + artifactCount: 1 + stateDir: .state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5 + reportSha256: sha256:119c9e3ef4cbc41143121e5d1af8955a8454e5efdbb6600f64a207a0f148b33 + - runId: sentinel-run-mrbjsugk-764ff868 + artifactCount: 4 + stateDir: .state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4 + reportSha256: sha256:d2dbc0fc98f39d7373b9ae009e31fcd8d666014c2e32428f235150ee59666048 publicExposure: enabled: false mode: prepared-not-routed diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index 991be5ea..3de863d0 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -22,6 +22,7 @@ import { hwlabNodeHelp } from "./hwlab-node-help"; import { runHwlabNodeCommand } from "./hwlab-node/entry"; import { guardedSentinelDeliveryAction, + migrationJobFailureSummary, runWebProbeSentinelCommand, webProbeSentinelDeliveryAuthorityFromCatalog, } from "./hwlab-node-web-sentinel-cicd"; @@ -294,6 +295,13 @@ describe("migrated CLI 执行 guard 与提示清理", () => { expect(execute.sentinel).toMatchObject({ kind: "migration", action: "job-run", confirm: true, wait: true }); }); + test("migration Job 失败输出容器终态、有界日志和 Secret 脱敏", () => { + const failure = migrationJobFailureSummary({ podPhase: "Failed", containers: [{ name: "export", phase: "terminated", exitCode: 2, reason: "Error", message: "artifact source unavailable" }], logsTail: "DATABASE_URL=postgres://secret-value export failed" }, {}); + expect(failure).toMatchObject({ phase: "Failed", container: "export", exitCode: 2, status: "Error", valuesRedacted: true }); + expect(String(failure.logsTail)).toContain("DATABASE_URL="); + expect(String(failure.logsTail)).not.toContain("secret-value"); + }); + test("PaC 外层事件与内层 deterministic publish 分开归类，只有外层可驱动 delivery", () => { const consumer = { repository: "sentinel-nc01-v03", pipelineRunPrefix: "hwlab-web-probe-sentinel-nc01-", pipeline: "hwlab-web-probe-sentinel-nc01" }; const outer = { diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index c409b507..0118baed 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import { rootPath } from "./config"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; -import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; +import { releaseAMonitorManifests, releaseAMonitorMigrationArtifactGaps, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; import { readYamlRecord } from "./platform-infra-ops-library"; const source = { sentinelId: "nc01-web-probe-sentinel", node: "NC01", lane: "v03", path: "/var/lib/web-probe-sentinel-nc01/index.sqlite", runtimeConfigRef: "config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime", stateRoot: "/var/lib/web-probe-sentinel-nc01", ownerPvc: "hwlab-web-probe-sentinel-nc01-state", artifactSources: [{ ownerPvc: "hwlab-web-probe-sentinel-nc01-state", mountPath: "/var/lib/web-probe-sentinel-nc01" }, { ownerPvc: "legacy-web-observe-workspace-nc01", mountPath: "/root/hwlab-v03", hostPath: "/root/hwlab-v03" }] }; @@ -47,6 +47,13 @@ test("one-shot migration helper accepts a unique control-plane suffix", () => { expect(job.metadata.name).toContain("source-commit-run-42"); }); +test("Release A declares exactly the two approved pre-migration artifact gaps", () => { + expect(releaseAMonitorMigrationArtifactGaps(hwlabRuntimeLaneSpecForNode("v03", "NC01"), source)).toEqual([ + expect.objectContaining({ runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1 }), + expect.objectContaining({ runId: "sentinel-run-mrbjsugk-764ff868", artifactCount: 4 }), + ]); +}); + test("Release A declares the controlled NC01 monitor database Secret projection", () => { const config = readYamlRecord>(rootPath("config/secrets-distribution.yaml"), "unidesk-secret-distribution"); expect(config.sources.files).toEqual(expect.arrayContaining([expect.objectContaining({ sourceRef: "hwlab/web-probe-monitor-db.env", type: "env", requiredKeys: ["DATABASE_URL"], createIfMissing: { enabled: false } })])); diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index 5ef87c17..a6fddb38 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -2,7 +2,7 @@ // Responsibility: YAML-first PG-capable central Monitor manifests while SQLite remains authoritative. import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; -import type { MonitorSqliteSource } from "./monitor-sqlite-migration"; +import type { MonitorLegacyArtifactGap, MonitorSqliteSource } from "./monitor-sqlite-migration"; export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly migrationJobEnabled?: boolean } = {}): readonly Record[] { const config = releaseAMonitorConfig(spec, resolvedSource); @@ -37,7 +37,22 @@ export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolve return { poolMax: integerAt(pool, "max", resolved.ref), idleTimeoutSeconds: integerAt(pool, "idleTimeoutSeconds", resolved.ref) }; } -function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly resolved: ReturnType; readonly runtime: Record; readonly job: Record; readonly namespace: string; readonly deploymentName: string; readonly port: number; readonly secret: Record; readonly pool: Record; readonly labels: Record } { +export function releaseAMonitorMigrationArtifactGaps(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): readonly MonitorLegacyArtifactGap[] { + const { resolved, migration } = releaseAMonitorConfig(spec, resolvedSource); + const gaps = arrayAt(migration, "legacyArtifactGaps", resolved.ref).map((item, index) => { + if (!isRecord(item)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps[${index}] must be an object`); + const gap = item; + return { runId: textAt(gap, "runId", resolved.ref), artifactCount: integerAt(gap, "artifactCount", resolved.ref), stateDir: textAt(gap, "stateDir", resolved.ref), reportSha256: textAt(gap, "reportSha256", resolved.ref) }; + }); + const expected = [ + { runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5", reportSha256: "sha256:119c9e3ef4cbc41143121e5d1af8955a8454e5efdbb6600f64a207a0f148b33" }, + { runId: "sentinel-run-mrbjsugk-764ff868", artifactCount: 4, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4", reportSha256: "sha256:d2dbc0fc98f39d7373b9ae009e31fcd8d666014c2e32428f235150ee59666048" }, + ]; + if (JSON.stringify(gaps) !== JSON.stringify(expected)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps must declare exactly the two approved pre-migration artifact gaps`); + return gaps; +} + +function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly resolved: ReturnType; readonly runtime: Record; readonly migration: Record; readonly job: Record; readonly namespace: string; readonly deploymentName: string; readonly port: number; readonly secret: Record; readonly pool: Record; readonly labels: Record } { const configRef = spec.observability.webProbe?.monitor?.configRef; if (!configRef) throw new Error(`lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.monitor.configRef is required`); const resolved = readWebProbeSentinelConfigRef(spec, configRef); @@ -62,10 +77,11 @@ function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: Monit textAt(secret, "key", resolved.ref); textAt(secret, "sourceRef", resolved.ref); const pool = recordAt(runtime, "pool", resolved.ref); - return { resolved, runtime, job, namespace, deploymentName, port, secret, pool, labels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" } }; + return { resolved, runtime, migration, job, namespace, deploymentName, port, secret, pool, labels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" } }; } function recordAt(value: Record, key: string, path: string): Record { if (!isRecord(value[key])) throw new Error(`${path}.${key} must be an object`); return value[key] as Record; } +function arrayAt(value: Record, key: string, path: string): readonly unknown[] { if (!Array.isArray(value[key])) throw new Error(`${path}.${key} must be an array`); return value[key] as readonly unknown[]; } function textAt(value: Record, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; } function integerAt(value: Record, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts index 2adca9c9..d6280444 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -937,12 +937,15 @@ export function probeK8sJobScript(namespace: string, jobName: string): string { "active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)", "pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)", "pod_phase=''", - "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", + "reason=''", + "containers_b64=''", + "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); reason=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.reason}' 2>/dev/null); containers_b64=$(kubectl -n \"$namespace\" get pod \"$pod\" -o json 2>/dev/null | node -e 'let body=\"\";process.stdin.on(\"data\",c=>body+=c).on(\"end\",()=>{try{const pod=JSON.parse(body);const all=[...(pod.status?.initContainerStatuses||[]),...(pod.status?.containerStatuses||[])];console.log(Buffer.from(JSON.stringify(all.map(item=>({name:item.name,phase:item.state?.terminated?\"terminated\":item.state?.waiting?\"waiting\":item.state?.running?\"running\":\"unknown\",exitCode:item.state?.terminated?.exitCode??null,reason:item.state?.terminated?.reason??item.state?.waiting?.reason??null,message:item.state?.terminated?.message??item.state?.waiting?.message??null})))).toString(\"base64\"))}catch{console.log(\"\")}})' | tr -d '\\n'); fi", "logs_tail=''", "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi", - "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", - "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", - "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", + "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$reason\" \"$containers_b64\" \"$logs_tail\" <<'NODE'", + "const [succeeded, failed, active, pod, podPhase, reason, containersB64, logsB64] = process.argv.slice(2);", + "let containers=[];try{containers=JSON.parse(Buffer.from(containersB64 || '', 'base64').toString('utf8'))}catch{}", + "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, reason: reason || null, containers, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", "NODE", ].join("\n"); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 6eba65f0..159b0d0a 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -37,7 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration"; -import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; +import { releaseAMonitorManifests, releaseAMonitorMigrationArtifactGaps, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; import { arrayAt, arrayAtNullable, @@ -193,7 +193,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: } function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract): RenderedCliResult { - const sources = resolvedMigrationSources(spec, options.sentinelId); + const sources = resolvedMigrationSources(spec, options.sentinelId).map((source) => ({ ...source, legacyArtifactGaps: releaseAMonitorMigrationArtifactGaps(spec, source) })); const command = `web-probe sentinel migration ${options.action}`; if (options.action === "job-plan" || options.action === "job-run") return runSentinelMigrationJob(spec, options, sources, command); if (options.action === "plan") { @@ -250,13 +250,29 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract, payload: Record): Record { + const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord).map((item) => ({ name: stringAtNullable(item, "name"), phase: stringAtNullable(item, "phase"), exitCode: finiteNumberOrNull(item.exitCode), reason: stringAtNullable(item, "reason"), message: short(stringAtNullable(item, "message") ?? "") })).filter((item) => item.phase === "terminated" || item.exitCode !== null || item.reason !== null) : []; + const failed = containers.find((item) => item.exitCode !== null && item.exitCode !== 0) ?? containers[0] ?? null; + const logsTail = redactMigrationJobLogTail(short(String(probe.logsTail ?? ""))); + return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, status: failed?.reason ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactMigrationJobLogTail(short(stringAtNullable(payload, "error") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted")), logsTail, valuesRedacted: true }; +} + +function redactMigrationJobLogTail(value: string): string { + return value.replace(/(DATABASE_URL|password|token|secret)\s*[=:]\s*[^\s]+/giu, "$1="); +} + +function migrationJobFailureNext(state: SentinelCicdState, namespace: string, jobName: string): Record { + return { logs: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${jobName} --all-containers=true --tail=120`, describe: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${jobName}` }; +} + function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean { return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath); } diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index 75009d60..1b3e8290 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -10,13 +10,15 @@ import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monit export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-3"; export interface MonitorArtifactSource { readonly ownerPvc: string; readonly mountPath: string; readonly hostPath?: string; } -function freezeArtifactManifests(snapshot: Pick): readonly { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[] }[] { const rows = tableRows(snapshot.tables, ["runs", "run"]); const locatorRows = tableRows(snapshot.tables, ["artifact_locators", "locators"]); return rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); const rowLocators = locatorRows.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); const locators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); const expectedCount = numberField(row, ["artifact_count", "artifactCount"]); if (expectedCount !== null && expectedCount !== locators.length) throw artifactCountMismatch(runId, stringField(row, ["state_dir", "stateDir"]), expectedCount, locators.length); verifyReportLocator(row, locators); return { runId, locators }; }); } -export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; readonly artifactSources: readonly MonitorArtifactSource[]; } +export interface MonitorLegacyArtifactGap { readonly runId: string; readonly artifactCount: number; readonly stateDir: string; readonly reportSha256: string; } +export interface MonitorArtifactManifest { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[]; readonly missingBeforeMigration?: MonitorLegacyArtifactGap; } +function freezeArtifactManifests(snapshot: Pick): readonly MonitorArtifactManifest[] { const rows = tableRows(snapshot.tables, ["runs", "run"]); const locatorRows = tableRows(snapshot.tables, ["artifact_locators", "locators"]); const declared = snapshot.source.legacyArtifactGaps ?? []; const used = new Set(); const manifests = rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); const gap = declared.find((item) => item.runId === runId); const expectedCount = numberField(row, ["artifact_count", "artifactCount"]); const stateDir = stringField(row, ["state_dir", "stateDir"]); const reportSha256 = stringField(row, ["report_json_sha256", "reportJsonSha256"]); const rowLocators = locatorRows.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); if (gap !== undefined) { if (gap.artifactCount !== expectedCount || gap.stateDir !== stateDir || gap.reportSha256 !== reportSha256) throw artifactGapIdentityInvalid(runId, gap, { artifactCount: expectedCount, stateDir, reportSha256 }); if (rowLocators.length !== 0) throw Object.assign(new Error(`declared missing artifact gap unexpectedly has locators for ${runId}`), { code: "monitor-migration-artifact-gap-locator-unexpected" }); used.add(runId); return { runId, locators: [], missingBeforeMigration: gap }; } const locators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); if (expectedCount !== null && expectedCount !== locators.length) throw artifactCountMismatch(runId, stateDir, expectedCount, locators.length); verifyReportLocator(row, locators); return { runId, locators }; }); if (used.size !== declared.length) throw Object.assign(new Error("declared legacy artifact gap was not found in the SQLite snapshot"), { code: "monitor-migration-artifact-gap-unmatched" }); return manifests; } +export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; readonly artifactSources: readonly MonitorArtifactSource[]; readonly legacyArtifactGaps?: readonly MonitorLegacyArtifactGap[]; } export interface MonitorSqliteTable { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record[]; } export interface MonitorSqliteSnapshot { readonly schema: string; readonly createdAt: string; readonly source: MonitorSqliteSource; readonly fingerprint: string; readonly integrity: string; readonly byteLength: number; readonly tables: readonly MonitorSqliteTable[]; - readonly artifactManifests: readonly { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[] }[]; + readonly artifactManifests: readonly MonitorArtifactManifest[]; readonly artifactLocator: { readonly path: string; readonly sha256: string; readonly sizeBytes: number; readonly kind: "legacy-sqlite" }; readonly manifestContentHash: string; } @@ -125,9 +127,11 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< const payload = { ...(storedPayload ?? {}), ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } }; const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : []; const canonicalFindings = dedupeFindings(reportFindings, rowFindings); - const artifactLocators = snapshot.artifactManifests.find((item) => item.runId === runId)?.locators; - if (artifactLocators === undefined) throw Object.assign(new Error(`snapshot artifact manifest is missing ${runId}`), { code: "monitor-migration-artifact-manifest-missing" }); - return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators, timeline: [] }); + const artifactManifest = snapshot.artifactManifests.find((item) => item.runId === runId); + if (artifactManifest === undefined) throw Object.assign(new Error(`snapshot artifact manifest is missing ${runId}`), { code: "monitor-migration-artifact-manifest-missing" }); + const missing = artifactManifest.missingBeforeMigration; + const missingFact = missing === undefined ? null : { code: "legacy-artifact-bytes-missing-before-migration", runId, artifactCount: missing.artifactCount, stateDir: missing.stateDir, reportSha256: missing.reportSha256, locator: null, bytesVerified: false, valuesRedacted: true }; + return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload: missingFact === null ? payload : { ...payload, migration: { legacyArtifactBytesMissingBeforeMigration: missingFact } }, findings: missingFact === null ? canonicalFindings : [...canonicalFindings, missingFact], artifactLocators: artifactManifest.locators, timeline: missingFact === null ? [] : [missingFact] }); }); } function targetReconciliation(ingests: readonly ReturnType[], runtimeMetadata: readonly MonitorRuntimeMetadata[]): Record { return { runs: ingests.length, findings: ingests.reduce((total, item) => total + item.findings.length, 0), payloads: ingests.length, artifactLocators: ingests.reduce((total, item) => total + item.artifactLocators.length, 0), views: ingests.reduce((total, item) => total + (isRecord(item.payload.views) ? Object.keys(item.payload.views).length : 0), 0), runtimeMetadata: runtimeMetadata.length }; } @@ -148,8 +152,10 @@ function artifactSourceFor(source: MonitorSqliteSource, ownerPvc: string): Monit function artifactSourceForStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { const directory = resolve(stateDir); const matches = source.artifactSources.filter((item) => directory === resolve(item.mountPath) || directory.startsWith(`${resolve(item.mountPath)}/`)).sort((left, right) => resolve(right.mountPath).length - resolve(left.mountPath).length); if (matches.length === 0) throw Object.assign(new Error(`legacy artifact stateDir is not registry-mounted: ${stateDir}`), { code: "monitor-migration-artifact-source-missing" }); return matches[0]!; } function artifactPath(mountPath: string, stateDir: string, relativePath: string): string { const root = resolve(mountPath); const directory = stateDir.startsWith("/") ? resolve(stateDir) : resolve(root, stateDir); if (directory !== root && !directory.startsWith(`${root}/`)) throw Object.assign(new Error("legacy artifact stateDir escapes registry mount"), { code: "monitor-migration-artifact-path-invalid" }); return join(directory, relativePath); } function artifactFiles(directory: string): readonly string[] { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); if (entry.isDirectory()) return artifactFiles(path); return entry.isFile() ? [path] : []; }).sort(); } -function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void { for (const manifest of snapshot.artifactManifests) for (const locator of manifest.locators) { const source = artifactSourceFor(snapshot.source, locator.ownerPvc); const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath); if (!existsSync(path) || !statSync(path).isFile()) throw Object.assign(new Error(`frozen artifact is unavailable for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: null, valuesRedacted: true } }); const bytes = readFileSync(path); if (`sha256:${sha256(bytes)}` !== locator.sha256 || bytes.byteLength !== locator.sizeBytes) throw Object.assign(new Error(`frozen artifact changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: manifest.locators.length, valuesRedacted: true } }); } } +function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void { for (const manifest of snapshot.artifactManifests) { if (manifest.missingBeforeMigration !== undefined) { if (manifest.locators.length !== 0 || !matchesArtifactGap(manifest.missingBeforeMigration, snapshot.source.legacyArtifactGaps ?? [])) throw Object.assign(new Error(`invalid frozen legacy artifact gap for ${manifest.runId}`), { code: "monitor-migration-artifact-gap-identity-invalid" }); continue; } for (const locator of manifest.locators) { const source = artifactSourceFor(snapshot.source, locator.ownerPvc); const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath); if (!existsSync(path) || !statSync(path).isFile()) throw Object.assign(new Error(`frozen artifact is unavailable for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: null, valuesRedacted: true } }); const bytes = readFileSync(path); if (`sha256:${sha256(bytes)}` !== locator.sha256 || bytes.byteLength !== locator.sizeBytes) throw Object.assign(new Error(`frozen artifact changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: manifest.locators.length, valuesRedacted: true } }); } } } function artifactCountMismatch(runId: string, stateDir: string, expectedCount: number, observedCount: number): Error { return Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch", diagnostic: { runId, stateDir, expectedCount, observedCount, valuesRedacted: true } }); } +function artifactGapIdentityInvalid(runId: string, expected: MonitorLegacyArtifactGap, observed: Record): Error { return Object.assign(new Error(`legacy artifact gap identity mismatch for ${runId}`), { code: "monitor-migration-artifact-gap-identity-invalid", diagnostic: { runId, expected, observed, valuesRedacted: true } }); } +function matchesArtifactGap(gap: MonitorLegacyArtifactGap, declared: readonly MonitorLegacyArtifactGap[]): boolean { return declared.some((item) => item.runId === gap.runId && item.artifactCount === gap.artifactCount && item.stateDir === gap.stateDir && item.reportSha256 === gap.reportSha256); } function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; } function normalizeRow(value: unknown): Record { const row = value as Record; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); } function jsonField(row: Record | undefined, fields: readonly string[]): Record | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 3cfc2203..e0d997e2 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -199,6 +199,48 @@ test("workspace artifact source verifies locator identity hash and size", async } }); +test("declared pre-migration artifact gaps import empty locators and fail closed on identity drift", async () => { + const directory = mkdtempSync(join(tmpdir(), "monitor-artifact-gaps-")); + try { + const sqlitePath = join(directory, "index.sqlite"); + const gaps = [ + { runId: "sentinel-run-mrbjo83t-ef79c71d", artifactCount: 1, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5", reportSha256: "sha256:119c9e3ef4cbc41143121e5d1af8955a8454e5efdbb6600f64a207a0f148b33" }, + { runId: "sentinel-run-mrbjsugk-764ff868", artifactCount: 4, stateDir: ".state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4", reportSha256: "sha256:d2dbc0fc98f39d7373b9ae009e31fcd8d666014c2e32428f235150ee59666048" }, + ]; + const database = new Database(sqlitePath); + database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)"); + for (const gap of gaps) database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", [gap.runId, gap.stateDir, gap.artifactCount, gap.reportSha256, "2026-07-08T03:58:39.000Z"]); + database.close(); + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }], legacyArtifactGaps: gaps }; + const snapshot = exportMonitorSqliteSnapshot(source, join(directory, "snapshot.json")); + const store = createInMemoryMonitorCentralStore(); + await importMonitorSqliteSnapshot(snapshot, store); + for (const gap of gaps) { + const run = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, gap.runId); + assert.equal(run?.artifactLocators.length, 0); + assert.equal((run?.timeline.at(-1) as { code?: string; locator?: unknown; bytesVerified?: boolean } | undefined)?.code, "legacy-artifact-bytes-missing-before-migration"); + assert.equal((run?.timeline.at(-1) as { locator?: unknown }).locator, null); + assert.equal((run?.timeline.at(-1) as { bytesVerified?: boolean }).bytesVerified, false); + } + assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true); + assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], artifactCount: 2 }, gaps[1]] }, join(directory, "changed.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid"); + assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], stateDir: "wrong-state-dir" }, gaps[1]] }, join(directory, "wrong-state-dir.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid"); + assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], reportSha256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, gaps[1]] }, join(directory, "wrong-report.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid"); + assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [...gaps, { runId: "third", artifactCount: 1, stateDir: "missing", reportSha256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }] }, join(directory, "third.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-unmatched"); + const unexpectedArtifact = Buffer.from("unexpected\n"); + const unexpectedPath = join(directory, gaps[0].stateDir, "extra.bin"); + mkdirSync(join(unexpectedPath, ".."), { recursive: true }); + writeFileSync(unexpectedPath, unexpectedArtifact); + const locatorDatabase = new Database(sqlitePath); + locatorDatabase.run("CREATE TABLE artifact_locators (run_id TEXT, owner_pvc TEXT, state_dir TEXT, relative_path TEXT, sha256 TEXT, size_bytes INTEGER)"); + locatorDatabase.run("INSERT INTO artifact_locators VALUES (?, ?, ?, ?, ?, ?)", [gaps[0].runId, "fixture-pvc", gaps[0].stateDir, "extra.bin", `sha256:${createHash("sha256").update(unexpectedArtifact).digest("hex")}`, unexpectedArtifact.byteLength]); + locatorDatabase.close(); + assert.throws(() => exportMonitorSqliteSnapshot(source, join(directory, "unexpected-locator.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-locator-unexpected"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + test("artifact count without a registry-backed locator fails closed", async () => { const directory = mkdtempSync(join(tmpdir(), "monitor-artifact-count-")); try {