From e3d21ed4e17dd22dc7f3fa1ec3e446dde29901f4 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Sun, 12 Jul 2026 22:51:50 +0000 Subject: [PATCH 1/6] feat: add Release A monitor PG capability --- config/hwlab-node-lanes.yaml | 2 + .../runtime.nc01-v03.yaml | 66 +++++++++++++++++++ config/secrets-distribution.yaml | 8 +++ scripts/monitor-sqlite-migration-verify.ts | 18 +++++ scripts/src/hwlab-node-lanes.ts | 20 +++++- .../src/hwlab-node-web-probe-monitor.test.ts | 16 +++++ scripts/src/hwlab-node-web-probe-monitor.ts | 35 ++++++++++ scripts/src/hwlab-node-web-sentinel-cicd.ts | 7 +- scripts/src/monitor-sqlite-migration.ts | 30 +++++++++ .../monitor-terminal-ingest-migration.test.ts | 5 +- 10 files changed, 203 insertions(+), 4 deletions(-) create mode 100644 config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml create mode 100644 scripts/monitor-sqlite-migration-verify.ts create mode 100644 scripts/src/hwlab-node-web-probe-monitor.test.ts create mode 100644 scripts/src/hwlab-node-web-probe-monitor.ts diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 7f3aea15..cefbab01 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -162,6 +162,8 @@ lanes: observability: prometheusOperator: false webProbe: + monitor: + configRef: config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml#monitor monitorRoot: enabled: true sentinelId: nc01-web-probe-sentinel diff --git a/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml b/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml new file mode 100644 index 00000000..6b75707d --- /dev/null +++ b/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml @@ -0,0 +1,66 @@ +version: 1 +kind: HwlabWebProbeMonitorRuntime +metadata: + id: hwlab-web-probe-monitor-nc01-v03 + owner: UniDesk + specRef: PJ2026-01060508 + relatedIssues: + - 1868 + - 1873 +monitor: + runtime: + node: NC01 + lane: v03 + namespace: hwlab-v03 + deploymentName: hwlab-web-probe-monitor + serviceName: hwlab-web-probe-monitor + servicePort: 8090 + replicas: 1 + imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel:p3-service + healthPath: /api/health + databaseSecret: + name: hwlab-web-probe-monitor-db + key: DATABASE_URL + sourceRef: hwlab/web-probe-monitor-db.env + timeout: + requestSeconds: 15 + startupSeconds: 30 + pool: + max: 4 + idleTimeoutSeconds: 30 + pageSize: 100 + persistence: + mode: host-postgres + authority: prepared-central-only + database: web_probe_monitor + schema: monitor + migration: + mode: legacy-authority + source: + registry: config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01.observability.webProbe.sentinels + sentinelId: nc01-web-probe-sentinel + runtimeConfigRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.sentinel.configRefs.runtime + sqlitePath: /var/lib/web-probe-sentinel-nc01/index.sqlite + pvcName: hwlab-web-probe-sentinel-nc01-state + stateRoot: /var/lib/web-probe-sentinel-nc01 + job: + name: hwlab-web-probe-monitor-migration + command: plan + readOnly: true + publicExposure: + enabled: false + mode: prepared-not-routed + hostname: monitor.pikapython.com + cicd: + consumer: sentinel-nc01-v03 + mode: existing-pac-consumer + capabilities: + boundedReadModel: true + health: true + schema: true + schedulerDue: true + maintenance: true + runnerAvailability: true + runnerTriggerFacts: true + schedulerAuthority: legacy-sqlite + clientQueryAuthority: legacy-sqlite diff --git a/config/secrets-distribution.yaml b/config/secrets-distribution.yaml index ff82b27f..837d825d 100644 --- a/config/secrets-distribution.yaml +++ b/config/secrets-distribution.yaml @@ -109,6 +109,14 @@ targets: enabled: true kubernetesSecrets: + - name: hwlab-web-probe-monitor-db + targetId: hwlab-nc01-v03 + secretName: hwlab-web-probe-monitor-db + type: Opaque + data: + - sourceRef: hwlab/web-probe-monitor-db.env + sourceKey: DATABASE_URL + targetKey: DATABASE_URL - name: hwlab-nc01-v03-opencode-server-auth targetId: hwlab-nc01-v03 secretName: hwlab-v03-opencode-server-auth diff --git a/scripts/monitor-sqlite-migration-verify.ts b/scripts/monitor-sqlite-migration-verify.ts new file mode 100644 index 00000000..13fab513 --- /dev/null +++ b/scripts/monitor-sqlite-migration-verify.ts @@ -0,0 +1,18 @@ +import { readFileSync } from "node:fs"; + +import { createPostgresMonitorCentralStore, structuredMonitorCentralError } from "./src/monitor-central-persistence"; +import { verifyMonitorSqliteSnapshotAgainstStore, type MonitorSqliteSnapshot } from "./src/monitor-sqlite-migration"; + +function required(name: string): string { const value = process.env[name]; if (!value) throw Object.assign(new Error(`${name} is required`), { code: "monitor_migration_runtime_config_missing" }); return value; } +function positive(name: string): number { const value = Number(required(name)); if (!Number.isSafeInteger(value) || value < 1) throw Object.assign(new Error(`${name} must be a positive integer`), { code: "monitor_migration_runtime_config_invalid" }); return value; } + +let store: ReturnType | null = null; +try { + const snapshotPath = process.argv[2]; + if (!snapshotPath) throw Object.assign(new Error("snapshot path is required"), { code: "monitor_migration_snapshot_missing" }); + store = createPostgresMonitorCentralStore(required("DATABASE_URL"), { poolMax: positive("MONITOR_CENTRAL_PG_POOL_MAX"), idleTimeoutSeconds: positive("MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS") }); + console.log(JSON.stringify(await verifyMonitorSqliteSnapshotAgainstStore(JSON.parse(readFileSync(snapshotPath, "utf8")) as MonitorSqliteSnapshot, store))); +} catch (error) { + console.log(JSON.stringify({ ok: false, error: structuredMonitorCentralError(error), valuesRedacted: true })); + process.exitCode = 2; +} finally { await store?.close(); } diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 8f02280b..484fa843 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -336,9 +336,14 @@ export interface HwlabRuntimeObservabilitySpec { export interface HwlabRuntimeObservabilityWebProbeSpec { readonly sentinel?: HwlabRuntimeWebProbeSentinelSpec; readonly sentinels?: readonly HwlabRuntimeWebProbeSentinelRegistryItemSpec[]; + readonly monitor?: HwlabRuntimeWebProbeMonitorSpec; readonly monitorRoot?: HwlabRuntimeWebProbeMonitorRootSpec; } +export interface HwlabRuntimeWebProbeMonitorSpec { + readonly configRef: string; +} + export interface HwlabRuntimeObservabilityMetricsEndpointSpec { readonly serviceName: string; readonly containerName: string; @@ -1867,13 +1872,14 @@ function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservab function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntimeObservabilityWebProbeSpec | undefined { if (value === undefined) return undefined; const raw = asRecord(value, path); - const allowed = new Set(["sentinel", "sentinels", "monitorRoot"]); + const allowed = new Set(["sentinel", "sentinels", "monitor", "monitorRoot"]); for (const key of Object.keys(raw)) { - if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; observability.webProbe currently only owns sentinel/sentinels/monitorRoot`); + if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; observability.webProbe currently only owns sentinel/sentinels/monitor/monitorRoot`); } if (raw.sentinel !== undefined && raw.sentinels !== undefined) throw new Error(`${path} may declare sentinel or sentinels, not both`); const sentinel = raw.sentinel === undefined ? undefined : webProbeSentinelConfig(raw.sentinel, `${path}.sentinel`); const sentinels = raw.sentinels === undefined ? undefined : webProbeSentinelRegistryConfig(raw.sentinels, `${path}.sentinels`); + const monitor = raw.monitor === undefined ? undefined : webProbeMonitorConfig(raw.monitor, `${path}.monitor`); const monitorRoot = raw.monitorRoot === undefined ? undefined : webProbeMonitorRootConfig(raw.monitorRoot, `${path}.monitorRoot`); if (monitorRoot !== undefined) { if (sentinels !== undefined && !sentinels.some((item) => item.id === monitorRoot.sentinelId)) { @@ -1886,10 +1892,20 @@ function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntime return { ...(sentinel === undefined ? {} : { sentinel }), ...(sentinels === undefined ? {} : { sentinels }), + ...(monitor === undefined ? {} : { monitor }), ...(monitorRoot === undefined ? {} : { monitorRoot }), }; } +function webProbeMonitorConfig(value: unknown, path: string): HwlabRuntimeWebProbeMonitorSpec { + const raw = asRecord(value, path); + const allowed = new Set(["configRef"]); + for (const key of Object.keys(raw)) if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; monitor may only contain configRef`); + const configRef = stringField(raw, "configRef", path); + if (!configRef.startsWith("config/") || !configRef.includes("#")) throw new Error(`${path}.configRef must be a config YAML selector`); + return { configRef }; +} + function webProbeMonitorRootConfig(value: unknown, path: string): HwlabRuntimeWebProbeMonitorRootSpec { const raw = asRecord(value, path); const allowed = new Set(["enabled", "sentinelId", "publicBaseUrl", "routePrefix", "caddyManagedBlockOwner"]); diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts new file mode 100644 index 00000000..6882e143 --- /dev/null +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -0,0 +1,16 @@ +import { expect, test } from "bun:test"; + +import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; +import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor"; + +test("Release A central monitor renders internal service, deployment and read-only PVC migration job", () => { + const manifests = releaseAMonitorManifests(hwlabRuntimeLaneSpecForNode("v03", "NC01")); + const service = manifests.find((item) => item.kind === "Service") as any; + const deployment = manifests.find((item) => item.kind === "Deployment") as any; + const job = manifests.find((item) => item.kind === "Job") as any; + expect(service.spec.type).toBe("ClusterIP"); + expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } })])); + expect(job.spec.template.spec.containers[0].args).toContain("plan"); + expect(job.spec.template.spec.containers[0].volumeMounts).toEqual([{ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }]); + expect(job.spec.template.spec.volumes).toEqual([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }]); +}); diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts new file mode 100644 index 00000000..d476fbb6 --- /dev/null +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -0,0 +1,35 @@ +// SPEC: PJ2026-01060508 Web哨兵 Release A. +// Responsibility: YAML-first PG-capable central Monitor manifests while SQLite remains authoritative. +import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref"; +import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; + +export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec): readonly Record[] { + const configRef = spec.observability.webProbe?.monitor?.configRef; + if (!configRef) return []; + const resolved = readWebProbeSentinelConfigRef(spec, configRef); + if (!isRecord(resolved.target)) throw new Error(`${resolved.ref} must resolve to an object`); + const runtime = recordAt(resolved.target, "runtime", resolved.ref); + const migration = recordAt(resolved.target, "migration", resolved.ref); + const source = recordAt(migration, "source", resolved.ref); + const job = recordAt(migration, "job", resolved.ref); + if (textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref}.migration.mode must be legacy-authority`); + if (textAt(source, "sentinelId", resolved.ref) !== "nc01-web-probe-sentinel" || textAt(source, "sqlitePath", resolved.ref) !== "/var/lib/web-probe-sentinel-nc01/index.sqlite") throw new Error(`${resolved.ref}.migration.source must declare the resolved NC01/v03 source`); + if (textAt(job, "command", resolved.ref) !== "plan" || job.readOnly !== true) throw new Error(`${resolved.ref}.migration.job must be a read-only plan`); + const namespace = textAt(runtime, "namespace", resolved.ref); + const deploymentName = textAt(runtime, "deploymentName", resolved.ref); + const port = integerAt(runtime, "servicePort", resolved.ref); + const secret = recordAt(runtime, "databaseSecret", resolved.ref); + const pool = recordAt(runtime, "pool", resolved.ref); + const labels = { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" }; + const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; + return [ + { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port, targetPort: "http" }] } }, + { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels }, spec: { replicas: integerAt(runtime, "replicas", resolved.ref), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { metadata: { labels }, spec: { containers: [{ name: "monitor", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/monitor-central-service.ts"], env, ports: [{ name: "http", containerPort: port }], readinessProbe: { httpGet: { path: textAt(runtime, "healthPath", resolved.ref), port: "http" } } }] } } } }, + { apiVersion: "batch/v1", kind: "Job", metadata: { name: textAt(job, "name", resolved.ref), namespace, labels }, spec: { backoffLimit: 0, template: { metadata: { labels }, spec: { restartPolicy: "Never", containers: [{ name: "migration", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/cli.ts", "web-probe", "sentinel", "migration", "plan", "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", textAt(source, "sentinelId", resolved.ref), "--json"], env: [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }], volumeMounts: [{ name: "legacy-sqlite", mountPath: textAt(source, "stateRoot", resolved.ref), readOnly: true }] }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: textAt(source, "pvcName", resolved.ref), readOnly: true } }] } } } }, + ]; +} + +function recordAt(value: Record, key: string, path: string): Record { if (!isRecord(value[key])) throw new Error(`${path}.${key} must be an object`); return value[key] as Record; } +function textAt(value: Record, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; } +function integerAt(value: Record, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; } +function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 8b230d1c..0920ad57 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -37,6 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration"; +import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor"; import { arrayAt, arrayAtNullable, @@ -221,7 +222,10 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract> { + const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot); + const expected = canonicalIngests(snapshot); + const checks: Record[] = []; + for (const input of expected) { + const scope = { sentinelId: input.sentinelId, node: input.node, lane: input.lane }; + const actual = await store.run(scope, input.runId); + checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); + checks.push({ name: "finding", runId: input.runId, ok: monitorCentralStableJson(actual?.findings) === monitorCentralStableJson(input.findings), valuesRedacted: true }); + checks.push({ name: "metadata", runId: input.runId, ok: monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); + checks.push({ name: "artifactLocator", runId: input.runId, ok: monitorCentralStableJson(actual?.artifactLocators) === monitorCentralStableJson(input.artifactLocators), valuesRedacted: true }); + const views = isRecord(input.payload.views) ? input.payload.views : {}; + for (const [name, value] of Object.entries(views)) { + const actualView = await store.view(scope, input.runId, name); + checks.push({ name: "view", runId: input.runId, view: name, ok: monitorCentralStableJson(actualView?.value) === monitorCentralStableJson(value), valuesRedacted: true }); + } + } + const latestFor = (items: readonly typeof expected) => [...items].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.runId.localeCompare(left.runId))[0] ?? null; + const globalLatest = latestFor(expected); + const global = await store.overview({}); + checks.push({ name: "globalLatest", ok: globalLatest === null ? global.latest === null : isRecord(global.latest) && global.latest.runId === globalLatest.runId && global.latest.updatedAt === globalLatest.updatedAt, valuesRedacted: true }); + const scopes = [...new Map(expected.map((item) => [`${item.sentinelId}/${item.node}/${item.lane}`, { sentinelId: item.sentinelId, node: item.node, lane: item.lane }])).values()]; + for (const scope of scopes) { + const latest = latestFor(expected.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane)); + const overview = await store.overview(scope); + checks.push({ name: "scopedLatest", scope, ok: latest === null ? overview.latest === null : isRecord(overview.latest) && overview.latest.runId === latest.runId && overview.latest.updatedAt === latest.updatedAt, valuesRedacted: true }); + } + return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: snapshotVerification.reconciliation, checks, valuesRedacted: true }; +} + export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[]): Record { return { ok: true, command: "web-probe sentinel migration plan", sources: sources.map(sourcePlan), sourceCount: sources.length, valuesRedacted: true }; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 4b31575f..fb427594 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -7,7 +7,7 @@ import test from "node:test"; import { Database } from "bun:sqlite"; import { createInMemoryMonitorCentralStore } from "./monitor-central-persistence"; -import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot } from "./monitor-sqlite-migration"; +import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration"; import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter, submitMonitorTerminalIngest } from "./monitor-terminal-ingest"; import { compactWebObserveAnalyzeAnalysisForRaw } from "./hwlab-node/web-probe-observe"; @@ -78,6 +78,9 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio const reportFindings = importedRun?.findings.filter((finding) => finding.id === "report-finding") ?? []; assert.deepEqual(reportFindings, [{ id: "report-finding", rootCause: "complete metadata detail" }]); assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary"); + const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store); + assert.equal(pgVerification.ok, true); + assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalLatest", "metadata", "run", "scopedLatest", "view"]); assert.equal(verifyMonitorSqliteSnapshot({ ...snapshot, manifestContentHash: "sha256:tampered" }).ok, false); } finally { rmSync(directory, { recursive: true, force: true }); From c101b2543e5b7bbba16b44ab178b911099db4bca Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Sun, 12 Jul 2026 23:24:41 +0000 Subject: [PATCH 2/6] fix: close Release A monitor migration blockers --- config/hwlab-node-lanes.yaml | 2 +- .../{runtime.nc01-v03.yaml => runtime.yaml} | 23 ++------ config/secrets-distribution.yaml | 6 ++ .../src/hwlab-node-web-probe-monitor.test.ts | 44 ++++++++++++--- scripts/src/hwlab-node-web-probe-monitor.ts | 56 ++++++++++++++----- scripts/src/hwlab-node-web-sentinel-cicd.ts | 23 ++++---- scripts/src/monitor-central-persistence.ts | 33 +++++++++-- scripts/src/monitor-sqlite-migration.ts | 24 +++++--- .../monitor-terminal-ingest-migration.test.ts | 15 ++++- 9 files changed, 162 insertions(+), 64 deletions(-) rename config/hwlab-web-probe-monitor/{runtime.nc01-v03.yaml => runtime.yaml} (69%) diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index cefbab01..aa831aad 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -163,7 +163,7 @@ lanes: prometheusOperator: false webProbe: monitor: - configRef: config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml#monitor + configRef: config/hwlab-web-probe-monitor/runtime.yaml#monitor monitorRoot: enabled: true sentinelId: nc01-web-probe-sentinel diff --git a/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml b/config/hwlab-web-probe-monitor/runtime.yaml similarity index 69% rename from config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml rename to config/hwlab-web-probe-monitor/runtime.yaml index 6b75707d..965a8d81 100644 --- a/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml +++ b/config/hwlab-web-probe-monitor/runtime.yaml @@ -1,7 +1,7 @@ version: 1 kind: HwlabWebProbeMonitorRuntime metadata: - id: hwlab-web-probe-monitor-nc01-v03 + id: hwlab-web-probe-monitor-runtime owner: UniDesk specRef: PJ2026-01060508 relatedIssues: @@ -16,22 +16,18 @@ monitor: serviceName: hwlab-web-probe-monitor servicePort: 8090 replicas: 1 - imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel:p3-service - healthPath: /api/health + healthPath: /health databaseSecret: name: hwlab-web-probe-monitor-db key: DATABASE_URL sourceRef: hwlab/web-probe-monitor-db.env - timeout: - requestSeconds: 15 - startupSeconds: 30 pool: max: 4 idleTimeoutSeconds: 30 pageSize: 100 persistence: mode: host-postgres - authority: prepared-central-only + authority: central-prepared-non-authoritative database: web_probe_monitor schema: monitor migration: @@ -39,7 +35,7 @@ monitor: source: registry: config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01.observability.webProbe.sentinels sentinelId: nc01-web-probe-sentinel - runtimeConfigRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.sentinel.configRefs.runtime + runtimeConfigRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime sqlitePath: /var/lib/web-probe-sentinel-nc01/index.sqlite pvcName: hwlab-web-probe-sentinel-nc01-state stateRoot: /var/lib/web-probe-sentinel-nc01 @@ -47,6 +43,7 @@ monitor: name: hwlab-web-probe-monitor-migration command: plan readOnly: true + enabled: false publicExposure: enabled: false mode: prepared-not-routed @@ -54,13 +51,3 @@ monitor: cicd: consumer: sentinel-nc01-v03 mode: existing-pac-consumer - capabilities: - boundedReadModel: true - health: true - schema: true - schedulerDue: true - maintenance: true - runnerAvailability: true - runnerTriggerFacts: true - schedulerAuthority: legacy-sqlite - clientQueryAuthority: legacy-sqlite diff --git a/config/secrets-distribution.yaml b/config/secrets-distribution.yaml index 837d825d..da64f964 100644 --- a/config/secrets-distribution.yaml +++ b/config/secrets-distribution.yaml @@ -90,6 +90,12 @@ sources: OPENCODE_SERVER_PASSWORD: bytes: 32 prefix: oc_ + - sourceRef: hwlab/web-probe-monitor-db.env + type: env + requiredKeys: + - DATABASE_URL + createIfMissing: + enabled: false targets: - id: platform-infra-g14 diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index 6882e143..c55f892f 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -1,16 +1,46 @@ import { expect, test } from "bun:test"; +import { rootPath } from "./config"; import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; -import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor"; +import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; +import { readYamlRecord } from "./platform-infra-ops-library"; -test("Release A central monitor renders internal service, deployment and read-only PVC migration job", () => { - const manifests = releaseAMonitorManifests(hwlabRuntimeLaneSpecForNode("v03", "NC01")); +const source = { sentinelId: "nc01-web-probe-sentinel", node: "NC01", lane: "v03", path: "/var/lib/web-probe-sentinel-nc01/index.sqlite", runtimeConfigRef: "config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime", stateRoot: "/var/lib/web-probe-sentinel-nc01", ownerPvc: "hwlab-web-probe-sentinel-nc01-state" }; +const imageRef = "127.0.0.1:5000/hwlab/web-probe-sentinel-nc01:source-commit"; + +test("Release A live manifests exclude disabled migration Job and select only monitor service Pods", () => { + const spec = hwlabRuntimeLaneSpecForNode("v03", "NC01"); + const manifests = releaseAMonitorManifests(spec, imageRef, source); const service = manifests.find((item) => item.kind === "Service") as any; const deployment = manifests.find((item) => item.kind === "Deployment") as any; - const job = manifests.find((item) => item.kind === "Job") as any; + expect(manifests.find((item) => item.kind === "Job")).toBeUndefined(); expect(service.spec.type).toBe("ClusterIP"); + expect(service.spec.selector).toEqual({ "app.kubernetes.io/name": "hwlab-web-probe-monitor", "app.kubernetes.io/component": "monitor-service" }); + expect(deployment.spec.template.metadata.labels["app.kubernetes.io/component"]).toBe("monitor-service"); + expect(deployment.spec.template.spec.containers[0].image).toBe(imageRef); + expect(deployment.spec.template.spec.containers[0].readinessProbe.httpGet.path).toBe("/health"); expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } })])); - expect(job.spec.template.spec.containers[0].args).toContain("plan"); - expect(job.spec.template.spec.containers[0].volumeMounts).toEqual([{ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }]); - expect(job.spec.template.spec.volumes).toEqual([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }]); + expect(releaseAMonitorMigrationPool(spec, source)).toEqual({ poolMax: 4, idleTimeoutSeconds: 30 }); +}); + +test("explicit migration helper renders ordered read-only export import verify Job", () => { + const job = releaseAMonitorMigrationJob(hwlabRuntimeLaneSpecForNode("v03", "NC01"), imageRef, source, { enabled: true }) as any; + expect(job.metadata.name).toContain("source-commit"); + expect(job.metadata.labels["app.kubernetes.io/component"]).toBe("migration"); + expect(job.spec.template.metadata.labels["app.kubernetes.io/component"]).toBe("migration"); + expect(job.spec.template.spec.initContainers.map((container: any) => container.name)).toEqual(["export", "import"]); + expect(job.spec.template.spec.initContainers[0].args).toEqual(expect.arrayContaining(["export", "--snapshot", "/migration/snapshot.json", "--confirm"])); + expect(job.spec.template.spec.initContainers[1].args).toEqual(expect.arrayContaining(["scripts/cli.ts", "import", "--snapshot", "/migration/snapshot.json", "--confirm"])); + expect(job.spec.template.spec.containers[0].args).toEqual(expect.arrayContaining(["scripts/cli.ts", "verify", "--snapshot", "/migration/snapshot.json"])); + for (const container of [...job.spec.template.spec.initContainers, ...job.spec.template.spec.containers]) { + expect(container.volumeMounts).toEqual(expect.arrayContaining([expect.objectContaining({ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }), expect.objectContaining({ name: "snapshot", mountPath: "/migration" })])); + } + expect(job.spec.template.spec.volumes).toEqual([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }, { name: "snapshot", emptyDir: {} }]); + expect(job.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } }), expect.objectContaining({ name: "MONITOR_CENTRAL_PG_POOL_MAX", value: "4" })])); +}); + +test("Release A declares the controlled NC01 monitor database Secret projection", () => { + const config = readYamlRecord>(rootPath("config/secrets-distribution.yaml"), "unidesk-secret-distribution"); + expect(config.sources.files).toEqual(expect.arrayContaining([expect.objectContaining({ sourceRef: "hwlab/web-probe-monitor-db.env", type: "env", requiredKeys: ["DATABASE_URL"], createIfMissing: { enabled: false } })])); + expect(config.kubernetesSecrets).toEqual(expect.arrayContaining([expect.objectContaining({ name: "hwlab-web-probe-monitor-db", targetId: "hwlab-nc01-v03", secretName: "hwlab-web-probe-monitor-db", data: [{ sourceRef: "hwlab/web-probe-monitor-db.env", sourceKey: "DATABASE_URL", targetKey: "DATABASE_URL" }] })])); }); diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index d476fbb6..42823f51 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -2,34 +2,62 @@ // Responsibility: YAML-first PG-capable central Monitor manifests while SQLite remains authoritative. import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; +import type { MonitorSqliteSource } from "./monitor-sqlite-migration"; -export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec): readonly Record[] { +export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource): readonly Record[] { + const config = releaseAMonitorConfig(spec, resolvedSource); + const { resolved, runtime, job, namespace, deploymentName, port, secret, pool, labels } = config; + const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; + const monitorLabels = { ...labels, "app.kubernetes.io/component": "monitor-service" }; + return [ + { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels: monitorLabels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/component": "monitor-service" }, ports: [{ name: "http", port, targetPort: "http" }] } }, + { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels: monitorLabels }, spec: { replicas: integerAt(runtime, "replicas", resolved.ref), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/component": "monitor-service" } }, template: { metadata: { labels: monitorLabels }, spec: { containers: [{ name: "monitor", image: imageRef, command: ["bun"], args: ["scripts/monitor-central-service.ts"], env, ports: [{ name: "http", containerPort: port }], readinessProbe: { httpGet: { path: textAt(runtime, "healthPath", resolved.ref), port: "http" } } }] } } } }, + ]; +} + +export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly enabled?: boolean } = {}): Record | null { + const config = releaseAMonitorConfig(spec, resolvedSource); + const { resolved, runtime, job, namespace, secret, pool, labels } = config; + if ((options.enabled ?? job.enabled) !== true) return null; + const migrationLabels = { ...labels, "app.kubernetes.io/component": "migration" }; + const mounts = [{ name: "legacy-sqlite", mountPath: resolvedSource.stateRoot, readOnly: true }, { name: "snapshot", mountPath: "/migration" }]; + const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }]; + const cli = ["scripts/cli.ts", "web-probe", "sentinel", "migration"]; + const sourceArgs = ["--node", spec.nodeId, "--lane", spec.lane, "--sentinel", resolvedSource.sentinelId, "--snapshot", "/migration/snapshot.json"]; + return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs, "--confirm"], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: resolvedSource.ownerPvc, readOnly: true } }, { name: "snapshot", emptyDir: {} }] } } } }; +} + +export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly poolMax: number; readonly idleTimeoutSeconds: number } { + const { resolved, pool } = releaseAMonitorConfig(spec, resolvedSource); + return { poolMax: integerAt(pool, "max", resolved.ref), idleTimeoutSeconds: integerAt(pool, "idleTimeoutSeconds", resolved.ref) }; +} + +function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly resolved: ReturnType; readonly runtime: Record; readonly job: Record; readonly namespace: string; readonly deploymentName: string; readonly port: number; readonly secret: Record; readonly pool: Record; readonly labels: Record } { const configRef = spec.observability.webProbe?.monitor?.configRef; - if (!configRef) return []; + if (!configRef) throw new Error(`lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.monitor.configRef is required`); const resolved = readWebProbeSentinelConfigRef(spec, configRef); if (!isRecord(resolved.target)) throw new Error(`${resolved.ref} must resolve to an object`); - const runtime = recordAt(resolved.target, "runtime", resolved.ref); - const migration = recordAt(resolved.target, "migration", resolved.ref); + const monitor = resolved.target; + const runtime = recordAt(monitor, "runtime", resolved.ref); + const persistence = recordAt(monitor, "persistence", resolved.ref); + const migration = recordAt(monitor, "migration", resolved.ref); const source = recordAt(migration, "source", resolved.ref); const job = recordAt(migration, "job", resolved.ref); - if (textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref}.migration.mode must be legacy-authority`); - if (textAt(source, "sentinelId", resolved.ref) !== "nc01-web-probe-sentinel" || textAt(source, "sqlitePath", resolved.ref) !== "/var/lib/web-probe-sentinel-nc01/index.sqlite") throw new Error(`${resolved.ref}.migration.source must declare the resolved NC01/v03 source`); - if (textAt(job, "command", resolved.ref) !== "plan" || job.readOnly !== true) throw new Error(`${resolved.ref}.migration.job must be a read-only plan`); + const exposure = recordAt(monitor, "publicExposure", resolved.ref); + const cicd = recordAt(monitor, "cicd", resolved.ref); + if (textAt(runtime, "node", resolved.ref) !== spec.nodeId || textAt(runtime, "lane", resolved.ref) !== spec.lane || textAt(persistence, "mode", resolved.ref) !== "host-postgres" || textAt(persistence, "authority", resolved.ref) !== "central-prepared-non-authoritative" || exposure.enabled !== false || textAt(cicd, "mode", resolved.ref) !== "existing-pac-consumer" || textAt(cicd, "consumer", resolved.ref) !== "sentinel-nc01-v03" || textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref} must remain the NC01/v03 non-authoritative Release A configuration`); + if (textAt(source, "sentinelId", resolved.ref) !== resolvedSource.sentinelId || textAt(source, "runtimeConfigRef", resolved.ref) !== resolvedSource.runtimeConfigRef || textAt(source, "sqlitePath", resolved.ref) !== resolvedSource.path || textAt(source, "pvcName", resolved.ref) !== resolvedSource.ownerPvc || textAt(source, "stateRoot", resolved.ref) !== resolvedSource.stateRoot || textAt(job, "command", resolved.ref) !== "plan" || job.readOnly !== true || typeof job.enabled !== "boolean") throw new Error(`${resolved.ref}.migration must match the resolved active registry and declare a read-only plan`); const namespace = textAt(runtime, "namespace", resolved.ref); const deploymentName = textAt(runtime, "deploymentName", resolved.ref); const port = integerAt(runtime, "servicePort", resolved.ref); const secret = recordAt(runtime, "databaseSecret", resolved.ref); + if (textAt(secret, "name", resolved.ref) !== "hwlab-web-probe-monitor-db" || textAt(secret, "key", resolved.ref) !== "DATABASE_URL" || textAt(secret, "sourceRef", resolved.ref) !== "hwlab/web-probe-monitor-db.env") throw new Error(`${resolved.ref}.runtime.databaseSecret must be the controlled monitor DATABASE_URL projection`); const pool = recordAt(runtime, "pool", resolved.ref); - const labels = { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" }; - const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; - return [ - { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port, targetPort: "http" }] } }, - { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels }, spec: { replicas: integerAt(runtime, "replicas", resolved.ref), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { metadata: { labels }, spec: { containers: [{ name: "monitor", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/monitor-central-service.ts"], env, ports: [{ name: "http", containerPort: port }], readinessProbe: { httpGet: { path: textAt(runtime, "healthPath", resolved.ref), port: "http" } } }] } } } }, - { apiVersion: "batch/v1", kind: "Job", metadata: { name: textAt(job, "name", resolved.ref), namespace, labels }, spec: { backoffLimit: 0, template: { metadata: { labels }, spec: { restartPolicy: "Never", containers: [{ name: "migration", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/cli.ts", "web-probe", "sentinel", "migration", "plan", "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", textAt(source, "sentinelId", resolved.ref), "--json"], env: [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }], volumeMounts: [{ name: "legacy-sqlite", mountPath: textAt(source, "stateRoot", resolved.ref), readOnly: true }] }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: textAt(source, "pvcName", resolved.ref), readOnly: true } }] } } } }, - ]; + return { resolved, runtime, job, namespace, deploymentName, port, secret, pool, labels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" } }; } function recordAt(value: Record, key: string, path: string): Record { if (!isRecord(value[key])) throw new Error(`${path}.${key} must be an object`); return value[key] as Record; } function textAt(value: Record, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; } function integerAt(value: Record, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } +function versionedJobName(base: string, imageRef: string): string { const suffix = imageRef.split(":").at(-1)?.replace(/[^a-z0-9-]/gu, "-").slice(0, 20) || "image"; return `${base}-${suffix}`.slice(0, 63).replace(/-+$/u, ""); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 0920ad57..3817ed2b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -37,7 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration"; -import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor"; +import { releaseAMonitorManifests, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; import { arrayAt, arrayAtNullable, @@ -209,25 +209,28 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract[0]; + const source = sources.find((item) => sameMigrationSource(item, snapshot.source)); + if (source === undefined) throw new Error("migration snapshot source is not in the current resolved registry"); const verification = verifyMonitorSqliteSnapshot(snapshot); + if (verification.ok !== true) throw new Error("migration snapshot verification failed"); if (options.action === "import") { - const source = sources.find((item) => item.sentinelId === snapshot.source.sentinelId && item.runtimeConfigRef === snapshot.source.runtimeConfigRef); - if (source === undefined) throw new Error("migration snapshot source is not in the current resolved registry"); - const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, source.runtimeConfigRef), source.runtimeConfigRef); - const poolMax = numberAtNullable(runtime, "monitor.migration.poolMax"); - const idleTimeoutSeconds = numberAtNullable(runtime, "monitor.migration.idleTimeoutSeconds"); - if (stringAtNullable(runtime, "monitor.migration.databaseUrlSourceRef") === null || stringAtNullable(runtime, "monitor.migration.databaseUrlTargetKey") !== "DATABASE_URL" || poolMax === null || idleTimeoutSeconds === null) throw new Error("monitor.migration must declare databaseUrlSourceRef, databaseUrlTargetKey=DATABASE_URL, poolMax, and idleTimeoutSeconds"); if (!process.env.DATABASE_URL) throw new Error("monitor migration DATABASE_URL must be injected by the controlled Secret consumer"); - const result = runCommand(["bun", "scripts/monitor-sqlite-migration-import.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: String(poolMax), MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: String(idleTimeoutSeconds) } }); + const pool = releaseAMonitorMigrationPool(spec, source); + const result = runCommand(["bun", "scripts/monitor-sqlite-migration-import.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: String(pool.poolMax), MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: String(pool.idleTimeoutSeconds) } }); const projection = parseJsonObject(result.stdout) ?? { ok: false, command, error: "monitor-migration-import-unparseable", stderr: short(result.stderr), valuesRedacted: true }; return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection); } if (!process.env.DATABASE_URL) throw new Error("monitor migration verify requires DATABASE_URL projected from hwlab-web-probe-monitor-db/DATABASE_URL"); - const result = runCommand(["bun", "scripts/monitor-sqlite-migration-verify.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: "4", MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: "30" } }); + const pool = releaseAMonitorMigrationPool(spec, source); + const result = runCommand(["bun", "scripts/monitor-sqlite-migration-verify.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: String(pool.poolMax), MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: String(pool.idleTimeoutSeconds) } }); const projection = parseJsonObject(result.stdout) ?? { ok: false, command, error: "monitor-migration-verify-unparseable", stderr: short(result.stderr), valuesRedacted: true }; return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection); } +function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean { + return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc; +} + function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string | null): readonly MonitorSqliteSource[] { const ids = sentinelId === null ? webProbeSentinelRegistryRows(spec).filter((item) => item.enabled).map((item) => item.id) : [sentinelId]; return ids.map((id) => { @@ -1302,7 +1305,7 @@ function renderSentinelManifests( spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] }, }, ...(cadenceJob === null ? [] : [cadenceJob]), - ...releaseAMonitorManifests(spec), + ...releaseAMonitorManifests(spec, image.ref, resolvedMigrationSources(spec, sentinelId)[0] ?? (() => { throw new Error("Release A monitor requires one resolved migration source"); })()), { apiVersion: "apps/v1", kind: "Deployment", diff --git a/scripts/src/monitor-central-persistence.ts b/scripts/src/monitor-central-persistence.ts index 47a75450..6c4d9d09 100644 --- a/scripts/src/monitor-central-persistence.ts +++ b/scripts/src/monitor-central-persistence.ts @@ -4,7 +4,7 @@ import { createHash } from "node:crypto"; import postgres from "postgres"; export const MONITOR_CENTRAL_SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-12-p17-monitor-central-persistence"; -export const MONITOR_CENTRAL_SCHEMA_VERSION = "2026-07-12-p17-2"; +export const MONITOR_CENTRAL_SCHEMA_VERSION = "2026-07-12-p17-3"; export interface MonitorScope { readonly sentinelId?: string; @@ -52,7 +52,8 @@ export interface MonitorCentralStore { schemaVersion(): Promise; capability(): Promise<{ readonly ingest: boolean; readonly query: boolean }>; ingest(input: NormalizedMonitorCentralIngest): Promise<{ readonly disposition: "inserted" | "idempotent" }>; - importSource(manifest: MonitorImportManifest, inputs: readonly NormalizedMonitorCentralIngest[]): Promise<{ readonly disposition: "inserted" | "idempotent"; readonly runs: readonly { readonly disposition: "inserted" | "idempotent" }[] }>; + importSource(manifest: MonitorImportManifest, inputs: readonly NormalizedMonitorCentralIngest[], runtimeMetadata?: readonly MonitorRuntimeMetadata[]): Promise<{ readonly disposition: "inserted" | "idempotent"; readonly runs: readonly { readonly disposition: "inserted" | "idempotent" }[] }>; + runtimeMetadata(scope: Required): Promise; updateArtifactAvailability(scope: Required, runId: string, locatorIndex: number, reachable: boolean): Promise; overview(scope: MonitorScope): Promise>; runs(scope: MonitorScope, options: RunQueryOptions): Promise; @@ -71,6 +72,12 @@ export interface MonitorImportManifest { readonly rowCount: number; } +export interface MonitorRuntimeMetadata extends Required { + readonly key: string; + readonly value: unknown; + readonly updatedAt: string; +} + interface RunCursor { readonly updatedAt: string; readonly runId: string; @@ -197,7 +204,7 @@ export function createPostgresMonitorCentralStore(databaseUrl: string, options: async ingest(input) { return sql.begin((transaction) => writePostgresIngest(transaction, input)); }, - async importSource(manifest, inputs) { + async importSource(manifest, inputs, runtimeMetadata = []) { return sql.begin(async (transaction) => { const claimed = await transaction<{ manifest_id: string }[]>` insert into monitor.import_manifests (manifest_id, source_fingerprint, row_count, imported_at) @@ -213,9 +220,23 @@ export function createPostgresMonitorCentralStore(databaseUrl: string, options: } const runs = [] as { disposition: "inserted" | "idempotent" }[]; for (const input of inputs) runs.push(await writePostgresIngest(transaction, input)); + for (const item of runtimeMetadata) { + await transaction` + insert into monitor.runners (sentinel_id, node, lane, updated_at) + values (${item.sentinelId}, ${item.node}, ${item.lane}, ${item.updatedAt}) + on conflict (sentinel_id, node, lane) do update set updated_at = greatest(monitor.runners.updated_at, excluded.updated_at)`; + await transaction` + insert into monitor.runtime_metadata (sentinel_id, node, lane, key, value, updated_at) + values (${item.sentinelId}, ${item.node}, ${item.lane}, ${item.key}, ${JSON.stringify(item.value)}::jsonb, ${item.updatedAt}) + on conflict (sentinel_id, node, lane, key) do update set value = excluded.value, updated_at = excluded.updated_at`; + } return { disposition: "inserted" as const, runs }; }); }, + async runtimeMetadata(scope) { + const rows = await sql<({ key: string; value: unknown; updated_at: string })[]>`select key, value, updated_at from monitor.runtime_metadata where sentinel_id = ${scope.sentinelId} and node = ${scope.node} and lane = ${scope.lane} order by key`; + return rows.map((item) => ({ ...scope, key: item.key, value: item.value, updatedAt: new Date(item.updated_at).toISOString() })); + }, async updateArtifactAvailability(scope, runId, locatorIndex, reachable) { await sql` update monitor.artifact_locators @@ -303,12 +324,14 @@ export const MONITOR_CENTRAL_MIGRATIONS = [ "create table if not exists monitor.findings (sentinel_id text not null, node text not null, lane text not null, run_id text not null, finding_index integer not null, finding jsonb not null, primary key (sentinel_id, node, lane, run_id, finding_index), foreign key (sentinel_id, node, lane, run_id) references monitor.runs on delete cascade)", "create table if not exists monitor.artifact_locators (sentinel_id text not null, node text not null, lane text not null, run_id text not null, locator_index integer not null, owner_node text not null, owner_lane text not null, owner_pvc text not null, state_dir text not null, relative_path text not null, sha256 text not null, size_bytes bigint not null, kind text not null, reachable boolean not null, primary key (sentinel_id, node, lane, run_id, locator_index), foreign key (sentinel_id, node, lane, run_id) references monitor.runs on delete cascade)", "create table if not exists monitor.timeline_events (sentinel_id text not null, node text not null, lane text not null, run_id text not null, event_index integer not null, event jsonb not null, primary key (sentinel_id, node, lane, run_id, event_index), foreign key (sentinel_id, node, lane, run_id) references monitor.runs on delete cascade)", + "create table if not exists monitor.runtime_metadata (sentinel_id text not null, node text not null, lane text not null, key text not null, value jsonb not null, updated_at timestamptz not null, primary key (sentinel_id, node, lane, key), foreign key (sentinel_id, node, lane) references monitor.runners (sentinel_id, node, lane))", "create table if not exists monitor.import_manifests (manifest_id text primary key, source_fingerprint text not null, row_count bigint not null, imported_at timestamptz not null)", ]; export function createInMemoryMonitorCentralStore(): MonitorCentralStore { const records = new Map(); const manifests = new Map(); + const metadata = new Map(); return { async close() {}, async migrate() {}, @@ -326,7 +349,7 @@ export function createInMemoryMonitorCentralStore(): MonitorCentralStore { records.set(key, structuredClone(input)); return { disposition: "inserted" }; }, - async importSource(manifest, inputs) { + async importSource(manifest, inputs, runtimeMetadata = []) { const previous = manifests.get(manifest.manifestId); if (previous) { if (previous.sourceFingerprint !== manifest.sourceFingerprint || previous.rowCount !== manifest.rowCount) throw Object.assign(new Error("monitor import manifest conflict"), { code: "monitor_import_manifest_conflict" }); @@ -342,9 +365,11 @@ export function createInMemoryMonitorCentralStore(): MonitorCentralStore { } records.clear(); for (const [key, value] of staged) records.set(key, value); + for (const item of runtimeMetadata) metadata.set(`${item.sentinelId}/${item.node}/${item.lane}/${item.key}`, structuredClone(item)); manifests.set(manifest.manifestId, { ...manifest }); return { disposition: "inserted", runs }; }, + async runtimeMetadata(scope) { return [...metadata.values()].filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane).sort((left, right) => left.key.localeCompare(right.key)).map(structuredClone); }, async updateArtifactAvailability(scope, runId, locatorIndex, reachable) { const key = identityKey({ ...scope, runId }); const existing = records.get(key); diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index 9c858e99..9850385e 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -5,7 +5,7 @@ import { closeSync, existsSync, openSync, readFileSync, statSync, writeFileSync import { dirname, join, resolve } from "node:path"; import { Database } from "bun:sqlite"; -import type { MonitorArtifactLocator, MonitorCentralIngest, MonitorCentralStore } from "./monitor-central-persistence"; +import type { MonitorArtifactLocator, MonitorCentralIngest, MonitorCentralStore, MonitorRuntimeMetadata } from "./monitor-central-persistence"; import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monitor-central-persistence"; export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-2"; @@ -22,13 +22,13 @@ export interface MonitorMigrationReconciliation { readonly runs: number; readonl export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise> { const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot); const expected = canonicalIngests(snapshot); + const expectedMetadata = canonicalRuntimeMetadata(snapshot); const checks: Record[] = []; for (const input of expected) { const scope = { sentinelId: input.sentinelId, node: input.node, lane: input.lane }; const actual = await store.run(scope, input.runId); - checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); + checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && actual?.updatedAt === input.updatedAt && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); checks.push({ name: "finding", runId: input.runId, ok: monitorCentralStableJson(actual?.findings) === monitorCentralStableJson(input.findings), valuesRedacted: true }); - checks.push({ name: "metadata", runId: input.runId, ok: monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); checks.push({ name: "artifactLocator", runId: input.runId, ok: monitorCentralStableJson(actual?.artifactLocators) === monitorCentralStableJson(input.artifactLocators), valuesRedacted: true }); const views = isRecord(input.payload.views) ? input.payload.views : {}; for (const [name, value] of Object.entries(views)) { @@ -39,14 +39,18 @@ export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorS const latestFor = (items: readonly typeof expected) => [...items].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.runId.localeCompare(left.runId))[0] ?? null; const globalLatest = latestFor(expected); const global = await store.overview({}); - checks.push({ name: "globalLatest", ok: globalLatest === null ? global.latest === null : isRecord(global.latest) && global.latest.runId === globalLatest.runId && global.latest.updatedAt === globalLatest.updatedAt, valuesRedacted: true }); + checks.push({ name: "globalOverview", ok: global.runCount === expected.length && (globalLatest === null ? global.latest === null : sameRun(global.latest, globalLatest)), valuesRedacted: true }); const scopes = [...new Map(expected.map((item) => [`${item.sentinelId}/${item.node}/${item.lane}`, { sentinelId: item.sentinelId, node: item.node, lane: item.lane }])).values()]; for (const scope of scopes) { const latest = latestFor(expected.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane)); const overview = await store.overview(scope); - checks.push({ name: "scopedLatest", scope, ok: latest === null ? overview.latest === null : isRecord(overview.latest) && overview.latest.runId === latest.runId && overview.latest.updatedAt === latest.updatedAt, valuesRedacted: true }); + const expectedCount = expected.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane).length; + checks.push({ name: "scopedOverview", scope, ok: overview.runCount === expectedCount && (latest === null ? overview.latest === null : sameRun(overview.latest, latest)), valuesRedacted: true }); + const actualMetadata = await store.runtimeMetadata(scope); + const scopedMetadata = expectedMetadata.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane); + checks.push({ name: "runtimeMetadata", scope, ok: monitorCentralStableJson(actualMetadata) === monitorCentralStableJson(scopedMetadata), sourceCount: scopedMetadata.length, targetCount: actualMetadata.length, valuesRedacted: true }); } - return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: snapshotVerification.reconciliation, checks, valuesRedacted: true }; + return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: { source: reconcileTables(snapshot.tables), target: { runs: expected.length, runtimeMetadata: expectedMetadata.length } }, checks, valuesRedacted: true }; } export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[]): Record { @@ -87,11 +91,12 @@ export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapsho const verified = verifyMonitorSqliteSnapshot(snapshot); if (verified.ok !== true) throw Object.assign(new Error("snapshot verification failed"), { code: "monitor-migration-snapshot-invalid", verification: verified }); const ingests = canonicalIngests(snapshot); + const runtimeMetadata = canonicalRuntimeMetadata(snapshot); const manifest = { manifestId: snapshot.manifestContentHash, sourceFingerprint: snapshot.fingerprint, rowCount: snapshot.tables.reduce((total, table) => total + table.rowCount, 0) }; - const batch = await store.importSource(manifest, ingests); + const batch = await store.importSource(manifest, ingests, runtimeMetadata); const dispositions = batch.runs; const reconciliation = reconcileTables(snapshot.tables); - return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, manifest, disposition: batch.disposition, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true }; + return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, manifest, disposition: batch.disposition, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation: { source: reconciliation, target: { runs: ingests.length, runtimeMetadata: runtimeMetadata.length } }, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true }; } export function reconcileTables(tables: readonly MonitorSqliteTable[]): MonitorMigrationReconciliation { @@ -118,6 +123,8 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators: reportLocator ? [...rowLocators, reportLocator] : rowLocators, timeline: [] }); }); } +function canonicalRuntimeMetadata(snapshot: MonitorSqliteSnapshot): readonly MonitorRuntimeMetadata[] { return tableRows(snapshot.tables, ["metadata", "run_metadata"]).map((row) => ({ sentinelId: snapshot.source.sentinelId, node: snapshot.source.node, lane: snapshot.source.lane, key: stringField(row, ["key"]), value: jsonValue(row, ["value_json", "value"]), updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt })).filter((item) => item.key.length > 0).sort((left, right) => left.key.localeCompare(right.key)); } +function sameRun(value: unknown, expected: { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly runId: string; readonly updatedAt: string }): boolean { return isRecord(value) && value.sentinelId === expected.sentinelId && value.node === expected.node && value.lane === expected.lane && value.runId === expected.runId && value.updatedAt === expected.updatedAt; } function locatorFromRow(row: Record, source: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const hash = stringField(row, ["sha256"]); const sizeBytes = numberField(row, ["size_bytes", "sizeBytes"]); if (!ownerPvc || !relativePath || !hash || sizeBytes === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc, stateDir: stringField(row, ["state_dir", "stateDir"]), relativePath, sha256: hash, sizeBytes, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } function reportLocatorFromRun(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator | null { const stateDir = stringField(row, ["state_dir", "stateDir"]); const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!stateDir && !expectedHash) return null; if (!stateDir || !expectedHash) throw Object.assign(new Error("legacy report locator is incomplete"), { code: "monitor-migration-report-locator-invalid" }); const path = join(snapshot.source.stateRoot, stateDir, "analysis", "report.json"); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc: snapshot.source.ownerPvc, stateDir, relativePath: "analysis/report.json", sha256: actualHash, sizeBytes: bytes.byteLength, kind: "report-json", reachable: true }; } function metadataValue(rows: readonly Record[], key: string): Record | null { return jsonField(rows.find((row) => stringField(row, ["key"]) === key), ["value_json", "value"]); } @@ -130,6 +137,7 @@ function writeSnapshot(path: string, content: string): void { const parent = dir function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; } function normalizeRow(value: unknown): Record { const row = value as Record; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); } function jsonField(row: Record | undefined, fields: readonly string[]): Record | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } +function jsonValue(row: Record | undefined, fields: readonly string[]): unknown { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value !== "string") return value ?? null; try { return JSON.parse(value); } catch { return value; } } function stringField(row: Record | undefined, fields: readonly string[]): string { const value = row === undefined ? null : fields.map((field) => row[field]).find((item) => typeof item === "string" && item.length > 0); return typeof value === "string" ? value : ""; } function numberField(row: Record, fields: readonly string[]): number | null { const value = fields.map((field) => row[field]).find((item) => typeof item === "number"); return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; } function sha256(value: Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index fb427594..1862ea94 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -59,7 +59,7 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio database.run("CREATE TABLE metadata (id TEXT, key TEXT, value_json TEXT)"); for (let index = 0; index < 228; index += 1) database.run("INSERT INTO runs VALUES (?, ?, ?, ?)", [`run-${index}`, index === 0 ? "run-0" : null, index === 0 ? reportHash : null, "2026-07-12T00:00:00.000Z"]); for (let index = 0; index < 242; index += 1) database.run("INSERT INTO findings VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [index === 0 ? "run-0" : `run-${index % 228}`, "fixture", index === 0 ? "report-finding" : `finding-${index}`, "warning", 1, "indexed", reportHash, "2026-07-12T00:00:00.000Z"]); - for (let index = 0; index < 234; index += 1) database.run("INSERT INTO metadata VALUES (?, ?, ?)", [`metadata-${index}`, index === 0 ? "run.report.run-0" : `metadata-${index}`, index === 0 ? JSON.stringify({ views: { summary: { title: "legacy summary" } }, findings: [{ id: "report-finding", rootCause: "complete metadata detail" }] }) : "{}"]); + for (let index = 0; index < 234; index += 1) database.run("INSERT INTO metadata VALUES (?, ?, ?)", [`metadata-${index}`, index === 0 ? "run.report.run-0" : index === 1 ? "scheduler.heartbeat" : index === 2 ? "maintenance" : index === 3 ? "manual-trigger.latest" : `metadata-${index}`, index === 0 ? JSON.stringify({ views: { summary: { title: "legacy summary" } }, findings: [{ id: "report-finding", rootCause: "complete metadata detail" }] }) : index === 2 ? JSON.stringify(false) : JSON.stringify({ index })]); database.close(); const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot, ownerPvc: "fixture-pvc" }; const plan = planMonitorSqliteSources([source]); @@ -73,6 +73,9 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio const store = createInMemoryMonitorCentralStore(); assert.deepEqual((await importMonitorSqliteSnapshot(snapshot, store)).imported, { inserted: 228, idempotent: 0 }); assert.deepEqual((await importMonitorSqliteSnapshot(snapshot, store)).imported, { inserted: 0, idempotent: 228 }); + const importedMetadata = await store.runtimeMetadata({ sentinelId: "fixture", node: "NC01", lane: "v03" }); + assert.equal(importedMetadata.length, 234); + assert.equal(importedMetadata.find((item) => item.key === "scheduler.heartbeat")?.updatedAt, snapshot.createdAt); const importedRun = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0"); assert.equal(importedRun?.artifactLocators[0]?.sha256, reportHash); const reportFindings = importedRun?.findings.filter((finding) => finding.id === "report-finding") ?? []; @@ -80,7 +83,9 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary"); const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store); assert.equal(pgVerification.ok, true); - assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalLatest", "metadata", "run", "scopedLatest", "view"]); + assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalOverview", "run", "runtimeMetadata", "scopedOverview", "view"]); + await store.importSource({ manifestId: "sha256:orphan-metadata", sourceFingerprint: "sha256:orphan-metadata", rowCount: 0 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "orphan", value: { unexpected: true }, updatedAt: snapshot.createdAt }]); + assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, false); assert.equal(verifyMonitorSqliteSnapshot({ ...snapshot, manifestContentHash: "sha256:tampered" }).ok, false); } finally { rmSync(directory, { recursive: true, force: true }); @@ -101,3 +106,9 @@ test("single-source batch rolls back conflicts and preserves manifest idempotenc assert.equal((await store.importSource(manifest, [first])).disposition, "inserted"); assert.equal((await store.importSource(manifest, [first])).disposition, "idempotent"); }); + +test("runtime metadata import is scoped and does not require a legacy run", async () => { + const store = createInMemoryMonitorCentralStore(); + await store.importSource({ manifestId: "sha256:metadata-only", sourceFingerprint: "sha256:metadata-only", rowCount: 1 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "scheduler.heartbeat", value: { healthy: true }, updatedAt: "2026-07-12T00:00:00.000Z" }]); + assert.deepEqual(await store.runtimeMetadata({ sentinelId: "fixture", node: "NC01", lane: "v03" }), [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "scheduler.heartbeat", value: { healthy: true }, updatedAt: "2026-07-12T00:00:00.000Z" }]); +}); From 1a3e95e8d92d1f9bfb2fe2ead993f919281fd550 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Sun, 12 Jul 2026 23:35:21 +0000 Subject: [PATCH 3/6] fix: align Release A migration semantics --- config/hwlab-web-probe-monitor/runtime.yaml | 4 ++-- .../src/hwlab-node-web-probe-monitor.test.ts | 4 +++- scripts/src/hwlab-node-web-probe-monitor.ts | 19 ++++++++++----- scripts/src/hwlab-node-web-sentinel-cicd.ts | 1 + scripts/src/hwlab-node/web-probe-observe.ts | 2 +- scripts/src/monitor-sqlite-migration.ts | 7 +++--- .../monitor-terminal-ingest-migration.test.ts | 24 +++++++++++++++---- 7 files changed, 44 insertions(+), 17 deletions(-) diff --git a/config/hwlab-web-probe-monitor/runtime.yaml b/config/hwlab-web-probe-monitor/runtime.yaml index 965a8d81..68df1fab 100644 --- a/config/hwlab-web-probe-monitor/runtime.yaml +++ b/config/hwlab-web-probe-monitor/runtime.yaml @@ -41,8 +41,8 @@ monitor: stateRoot: /var/lib/web-probe-sentinel-nc01 job: name: hwlab-web-probe-monitor-migration - command: plan - readOnly: true + workflow: export-import-verify + readOnlySource: true enabled: false publicExposure: enabled: false diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index c55f892f..3f65d2ba 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -21,6 +21,7 @@ test("Release A live manifests exclude disabled migration Job and select only mo expect(deployment.spec.template.spec.containers[0].readinessProbe.httpGet.path).toBe("/health"); expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } })])); expect(releaseAMonitorMigrationPool(spec, source)).toEqual({ poolMax: 4, idleTimeoutSeconds: 30 }); + expect(releaseAMonitorManifests(spec, imageRef, source, { migrationJobEnabled: true }).map((item) => item.kind)).toEqual(["Service", "Deployment", "Job"]); }); test("explicit migration helper renders ordered read-only export import verify Job", () => { @@ -29,7 +30,8 @@ test("explicit migration helper renders ordered read-only export import verify J expect(job.metadata.labels["app.kubernetes.io/component"]).toBe("migration"); expect(job.spec.template.metadata.labels["app.kubernetes.io/component"]).toBe("migration"); expect(job.spec.template.spec.initContainers.map((container: any) => container.name)).toEqual(["export", "import"]); - expect(job.spec.template.spec.initContainers[0].args).toEqual(expect.arrayContaining(["export", "--snapshot", "/migration/snapshot.json", "--confirm"])); + expect(job.spec.template.spec.initContainers[0].args).toEqual(expect.arrayContaining(["export", "--snapshot", "/migration/snapshot.json"])); + expect(job.spec.template.spec.initContainers[0].args).not.toContain("--confirm"); expect(job.spec.template.spec.initContainers[1].args).toEqual(expect.arrayContaining(["scripts/cli.ts", "import", "--snapshot", "/migration/snapshot.json", "--confirm"])); expect(job.spec.template.spec.containers[0].args).toEqual(expect.arrayContaining(["scripts/cli.ts", "verify", "--snapshot", "/migration/snapshot.json"])); for (const container of [...job.spec.template.spec.initContainers, ...job.spec.template.spec.containers]) { diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index 42823f51..3192c139 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -4,27 +4,31 @@ import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config- import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { MonitorSqliteSource } from "./monitor-sqlite-migration"; -export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource): readonly Record[] { +export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly migrationJobEnabled?: boolean } = {}): readonly Record[] { const config = releaseAMonitorConfig(spec, resolvedSource); - const { resolved, runtime, job, namespace, deploymentName, port, secret, pool, labels } = config; + const { resolved, runtime, namespace, deploymentName, port, secret, pool, labels } = config; + requireImageRef(imageRef); const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; const monitorLabels = { ...labels, "app.kubernetes.io/component": "monitor-service" }; - return [ + const manifests = [ { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels: monitorLabels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/component": "monitor-service" }, ports: [{ name: "http", port, targetPort: "http" }] } }, { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels: monitorLabels }, spec: { replicas: integerAt(runtime, "replicas", resolved.ref), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/component": "monitor-service" } }, template: { metadata: { labels: monitorLabels }, spec: { containers: [{ name: "monitor", image: imageRef, command: ["bun"], args: ["scripts/monitor-central-service.ts"], env, ports: [{ name: "http", containerPort: port }], readinessProbe: { httpGet: { path: textAt(runtime, "healthPath", resolved.ref), port: "http" } } }] } } } }, ]; + const migrationJob = releaseAMonitorMigrationJob(spec, imageRef, resolvedSource, { enabled: options.migrationJobEnabled }); + return migrationJob === null ? manifests : [...manifests, migrationJob]; } export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly enabled?: boolean } = {}): Record | null { const config = releaseAMonitorConfig(spec, resolvedSource); const { resolved, runtime, job, namespace, secret, pool, labels } = config; + requireImageRef(imageRef); if ((options.enabled ?? job.enabled) !== true) return null; const migrationLabels = { ...labels, "app.kubernetes.io/component": "migration" }; const mounts = [{ name: "legacy-sqlite", mountPath: resolvedSource.stateRoot, readOnly: true }, { name: "snapshot", mountPath: "/migration" }]; const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }]; const cli = ["scripts/cli.ts", "web-probe", "sentinel", "migration"]; const sourceArgs = ["--node", spec.nodeId, "--lane", spec.lane, "--sentinel", resolvedSource.sentinelId, "--snapshot", "/migration/snapshot.json"]; - return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs, "--confirm"], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: resolvedSource.ownerPvc, readOnly: true } }, { name: "snapshot", emptyDir: {} }] } } } }; + return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: resolvedSource.ownerPvc, readOnly: true } }, { name: "snapshot", emptyDir: {} }] } } } }; } export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly poolMax: number; readonly idleTimeoutSeconds: number } { @@ -45,8 +49,10 @@ function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: Monit const job = recordAt(migration, "job", resolved.ref); const exposure = recordAt(monitor, "publicExposure", resolved.ref); const cicd = recordAt(monitor, "cicd", resolved.ref); - if (textAt(runtime, "node", resolved.ref) !== spec.nodeId || textAt(runtime, "lane", resolved.ref) !== spec.lane || textAt(persistence, "mode", resolved.ref) !== "host-postgres" || textAt(persistence, "authority", resolved.ref) !== "central-prepared-non-authoritative" || exposure.enabled !== false || textAt(cicd, "mode", resolved.ref) !== "existing-pac-consumer" || textAt(cicd, "consumer", resolved.ref) !== "sentinel-nc01-v03" || textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref} must remain the NC01/v03 non-authoritative Release A configuration`); - if (textAt(source, "sentinelId", resolved.ref) !== resolvedSource.sentinelId || textAt(source, "runtimeConfigRef", resolved.ref) !== resolvedSource.runtimeConfigRef || textAt(source, "sqlitePath", resolved.ref) !== resolvedSource.path || textAt(source, "pvcName", resolved.ref) !== resolvedSource.ownerPvc || textAt(source, "stateRoot", resolved.ref) !== resolvedSource.stateRoot || textAt(job, "command", resolved.ref) !== "plan" || job.readOnly !== true || typeof job.enabled !== "boolean") throw new Error(`${resolved.ref}.migration must match the resolved active registry and declare a read-only plan`); + const expectedRegistry = `config/hwlab-node-lanes.yaml#lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.sentinels`; + const expectedConsumer = `sentinel-${spec.nodeId.toLowerCase()}-${spec.lane}`; + if (textAt(runtime, "node", resolved.ref) !== spec.nodeId || textAt(runtime, "lane", resolved.ref) !== spec.lane || textAt(persistence, "mode", resolved.ref) !== "host-postgres" || textAt(persistence, "authority", resolved.ref) !== "central-prepared-non-authoritative" || textAt(persistence, "database", resolved.ref) !== "web_probe_monitor" || textAt(persistence, "schema", resolved.ref) !== "monitor" || exposure.enabled !== false || textAt(exposure, "mode", resolved.ref) !== "prepared-not-routed" || textAt(exposure, "hostname", resolved.ref).length === 0 || textAt(cicd, "mode", resolved.ref) !== "existing-pac-consumer" || textAt(cicd, "consumer", resolved.ref) !== expectedConsumer || textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref} must remain a non-authoritative Release A configuration`); + if (textAt(source, "registry", resolved.ref) !== expectedRegistry || textAt(source, "sentinelId", resolved.ref) !== resolvedSource.sentinelId || textAt(source, "runtimeConfigRef", resolved.ref) !== resolvedSource.runtimeConfigRef || textAt(source, "sqlitePath", resolved.ref) !== resolvedSource.path || textAt(source, "pvcName", resolved.ref) !== resolvedSource.ownerPvc || textAt(source, "stateRoot", resolved.ref) !== resolvedSource.stateRoot || textAt(job, "workflow", resolved.ref) !== "export-import-verify" || job.readOnlySource !== true || typeof job.enabled !== "boolean" || !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(textAt(job, "name", resolved.ref))) throw new Error(`${resolved.ref}.migration must match the resolved active registry and declare an export-import-verify read-only workflow`); const namespace = textAt(runtime, "namespace", resolved.ref); const deploymentName = textAt(runtime, "deploymentName", resolved.ref); const port = integerAt(runtime, "servicePort", resolved.ref); @@ -61,3 +67,4 @@ function textAt(value: Record, key: string, path: string): stri function integerAt(value: Record, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function versionedJobName(base: string, imageRef: string): string { const suffix = imageRef.split(":").at(-1)?.replace(/[^a-z0-9-]/gu, "-").slice(0, 20) || "image"; return `${base}-${suffix}`.slice(0, 63).replace(/-+$/u, ""); } +function requireImageRef(imageRef: string): void { if (typeof imageRef !== "string" || imageRef.trim().length === 0) throw new Error("Release A monitor imageRef must be a non-empty current PaC image reference"); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 3817ed2b..fa82b1cc 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -214,6 +214,7 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract [`${item.sentinelId}/${item.node}/${item.lane}`, { sentinelId: item.sentinelId, node: item.node, lane: item.lane }])).values()]; + const scopes = [...new Map([{ sentinelId: snapshot.source.sentinelId, node: snapshot.source.node, lane: snapshot.source.lane }, ...expected.map((item) => ({ sentinelId: item.sentinelId, node: item.node, lane: item.lane })), ...expectedMetadata.map((item) => ({ sentinelId: item.sentinelId, node: item.node, lane: item.lane }))].map((scope) => [`${scope.sentinelId}/${scope.node}/${scope.lane}`, scope])).values()]; for (const scope of scopes) { const latest = latestFor(expected.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane)); const overview = await store.overview(scope); @@ -50,7 +50,7 @@ export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorS const scopedMetadata = expectedMetadata.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane); checks.push({ name: "runtimeMetadata", scope, ok: monitorCentralStableJson(actualMetadata) === monitorCentralStableJson(scopedMetadata), sourceCount: scopedMetadata.length, targetCount: actualMetadata.length, valuesRedacted: true }); } - return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: { source: reconcileTables(snapshot.tables), target: { runs: expected.length, runtimeMetadata: expectedMetadata.length } }, checks, valuesRedacted: true }; + return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: { source: reconcileTables(snapshot.tables), target: targetReconciliation(expected, expectedMetadata) }, checks, valuesRedacted: true }; } export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[]): Record { @@ -96,7 +96,7 @@ export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapsho const batch = await store.importSource(manifest, ingests, runtimeMetadata); const dispositions = batch.runs; const reconciliation = reconcileTables(snapshot.tables); - return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, manifest, disposition: batch.disposition, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation: { source: reconciliation, target: { runs: ingests.length, runtimeMetadata: runtimeMetadata.length } }, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true }; + return { ok: true, command: "web-probe sentinel migration import", source: snapshot.source, manifest, disposition: batch.disposition, imported: { inserted: dispositions.filter((item) => item.disposition === "inserted").length, idempotent: dispositions.filter((item) => item.disposition === "idempotent").length }, reconciliation: { source: reconciliation, target: targetReconciliation(ingests, runtimeMetadata) }, payloadHashCount: new Set(ingests.map((item) => item.payloadHash)).size, valuesRedacted: true }; } export function reconcileTables(tables: readonly MonitorSqliteTable[]): MonitorMigrationReconciliation { @@ -123,6 +123,7 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators: reportLocator ? [...rowLocators, reportLocator] : rowLocators, timeline: [] }); }); } +function targetReconciliation(ingests: readonly ReturnType[], runtimeMetadata: readonly MonitorRuntimeMetadata[]): Record { return { runs: ingests.length, findings: ingests.reduce((total, item) => total + item.findings.length, 0), payloads: ingests.length, artifactLocators: ingests.reduce((total, item) => total + item.artifactLocators.length, 0), views: ingests.reduce((total, item) => total + (isRecord(item.payload.views) ? Object.keys(item.payload.views).length : 0), 0), runtimeMetadata: runtimeMetadata.length }; } function canonicalRuntimeMetadata(snapshot: MonitorSqliteSnapshot): readonly MonitorRuntimeMetadata[] { return tableRows(snapshot.tables, ["metadata", "run_metadata"]).map((row) => ({ sentinelId: snapshot.source.sentinelId, node: snapshot.source.node, lane: snapshot.source.lane, key: stringField(row, ["key"]), value: jsonValue(row, ["value_json", "value"]), updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt })).filter((item) => item.key.length > 0).sort((left, right) => left.key.localeCompare(right.key)); } function sameRun(value: unknown, expected: { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly runId: string; readonly updatedAt: string }): boolean { return isRecord(value) && value.sentinelId === expected.sentinelId && value.node === expected.node && value.lane === expected.lane && value.runId === expected.runId && value.updatedAt === expected.updatedAt; } function locatorFromRow(row: Record, source: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const hash = stringField(row, ["sha256"]); const sizeBytes = numberField(row, ["size_bytes", "sizeBytes"]); if (!ownerPvc || !relativePath || !hash || sizeBytes === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc, stateDir: stringField(row, ["state_dir", "stateDir"]), relativePath, sha256: hash, sizeBytes, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 1862ea94..17749a5d 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -84,6 +84,7 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store); assert.equal(pgVerification.ok, true); assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalOverview", "run", "runtimeMetadata", "scopedOverview", "view"]); + assert.deepEqual((pgVerification.reconciliation as { target: Record }).target, { runs: 228, findings: 242, payloads: 228, artifactLocators: 1, views: 1, runtimeMetadata: 234 }); await store.importSource({ manifestId: "sha256:orphan-metadata", sourceFingerprint: "sha256:orphan-metadata", rowCount: 0 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "orphan", value: { unexpected: true }, updatedAt: snapshot.createdAt }]); assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, false); assert.equal(verifyMonitorSqliteSnapshot({ ...snapshot, manifestContentHash: "sha256:tampered" }).ok, false); @@ -107,8 +108,23 @@ test("single-source batch rolls back conflicts and preserves manifest idempotenc assert.equal((await store.importSource(manifest, [first])).disposition, "idempotent"); }); -test("runtime metadata import is scoped and does not require a legacy run", async () => { - const store = createInMemoryMonitorCentralStore(); - await store.importSource({ manifestId: "sha256:metadata-only", sourceFingerprint: "sha256:metadata-only", rowCount: 1 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "scheduler.heartbeat", value: { healthy: true }, updatedAt: "2026-07-12T00:00:00.000Z" }]); - assert.deepEqual(await store.runtimeMetadata({ sentinelId: "fixture", node: "NC01", lane: "v03" }), [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "scheduler.heartbeat", value: { healthy: true }, updatedAt: "2026-07-12T00:00:00.000Z" }]); +test("metadata-only snapshot verifies scoped runtime metadata and rejects extras", async () => { + const directory = mkdtempSync(join(tmpdir(), "monitor-metadata-only-")); + try { + const sqlitePath = join(directory, "index.sqlite"); + const snapshotPath = join(directory, "snapshot.json"); + const database = new Database(sqlitePath); + database.run("CREATE TABLE metadata (key TEXT, value_json TEXT)"); + database.run("INSERT INTO metadata VALUES (?, ?)", ["scheduler.heartbeat", JSON.stringify({ healthy: true })]); + database.close(); + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc" }; + const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); + const store = createInMemoryMonitorCentralStore(); + await importMonitorSqliteSnapshot(snapshot, store); + assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true); + await store.importSource({ manifestId: "sha256:metadata-only-extra", sourceFingerprint: "sha256:metadata-only-extra", rowCount: 0 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "orphan", value: true, updatedAt: snapshot.createdAt }]); + assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } }); From 9a32ead76fba59831d79eb7a3b5aa9b39b655282 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 00:12:35 +0000 Subject: [PATCH 4/6] fix: preserve immutable monitor migration sources --- config/hwlab-web-probe-monitor/runtime.yaml | 7 -- config/hwlab-web-probe-sentinel/profiles.yaml | 6 ++ .../src/hwlab-node-web-probe-monitor.test.ts | 5 +- scripts/src/hwlab-node-web-probe-monitor.ts | 13 +-- scripts/src/hwlab-node-web-sentinel-cicd.ts | 12 ++- .../src/monitor-central-persistence.test.ts | 8 ++ scripts/src/monitor-central-persistence.ts | 5 +- scripts/src/monitor-sqlite-migration.ts | 21 +++-- .../monitor-terminal-ingest-migration.test.ts | 82 +++++++++++++++++-- 9 files changed, 130 insertions(+), 29 deletions(-) diff --git a/config/hwlab-web-probe-monitor/runtime.yaml b/config/hwlab-web-probe-monitor/runtime.yaml index 68df1fab..ed6651eb 100644 --- a/config/hwlab-web-probe-monitor/runtime.yaml +++ b/config/hwlab-web-probe-monitor/runtime.yaml @@ -28,17 +28,10 @@ monitor: persistence: mode: host-postgres authority: central-prepared-non-authoritative - database: web_probe_monitor - schema: monitor migration: mode: legacy-authority source: registry: config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01.observability.webProbe.sentinels - sentinelId: nc01-web-probe-sentinel - runtimeConfigRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime - sqlitePath: /var/lib/web-probe-sentinel-nc01/index.sqlite - pvcName: hwlab-web-probe-sentinel-nc01-state - stateRoot: /var/lib/web-probe-sentinel-nc01 job: name: hwlab-web-probe-monitor-migration workflow: export-import-verify diff --git a/config/hwlab-web-probe-sentinel/profiles.yaml b/config/hwlab-web-probe-sentinel/profiles.yaml index 5e389f96..aa682282 100644 --- a/config/hwlab-web-probe-sentinel/profiles.yaml +++ b/config/hwlab-web-probe-sentinel/profiles.yaml @@ -452,6 +452,12 @@ templates: serviceName: "hwlab-web-probe-sentinel-${nodeLower}" pvcName: "hwlab-web-probe-sentinel-${nodeLower}-state" stateRoot: "/var/lib/web-probe-sentinel-${nodeLower}" + artifactSources: + - ownerPvc: "hwlab-web-probe-sentinel-${nodeLower}-state" + mountPath: "/var/lib/web-probe-sentinel-${nodeLower}" + - ownerPvc: "legacy-web-observe-workspace-${nodeLower}" + mountPath: "/root/hwlab-v03" + hostPath: "/root/hwlab-v03" imageRef: "127.0.0.1:5000/hwlab/web-probe-sentinel-${nodeLower}:source-commit" sqlite: busyTimeoutMs: 2000 diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index 3f65d2ba..9473bb63 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -5,7 +5,7 @@ import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; import { readYamlRecord } from "./platform-infra-ops-library"; -const source = { sentinelId: "nc01-web-probe-sentinel", node: "NC01", lane: "v03", path: "/var/lib/web-probe-sentinel-nc01/index.sqlite", runtimeConfigRef: "config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime", stateRoot: "/var/lib/web-probe-sentinel-nc01", ownerPvc: "hwlab-web-probe-sentinel-nc01-state" }; +const source = { sentinelId: "nc01-web-probe-sentinel", node: "NC01", lane: "v03", path: "/var/lib/web-probe-sentinel-nc01/index.sqlite", runtimeConfigRef: "config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime", stateRoot: "/var/lib/web-probe-sentinel-nc01", ownerPvc: "hwlab-web-probe-sentinel-nc01-state", artifactSources: [{ ownerPvc: "hwlab-web-probe-sentinel-nc01-state", mountPath: "/var/lib/web-probe-sentinel-nc01" }, { ownerPvc: "legacy-web-observe-workspace-nc01", mountPath: "/root/hwlab-v03", hostPath: "/root/hwlab-v03" }] }; const imageRef = "127.0.0.1:5000/hwlab/web-probe-sentinel-nc01:source-commit"; test("Release A live manifests exclude disabled migration Job and select only monitor service Pods", () => { @@ -37,7 +37,8 @@ test("explicit migration helper renders ordered read-only export import verify J for (const container of [...job.spec.template.spec.initContainers, ...job.spec.template.spec.containers]) { expect(container.volumeMounts).toEqual(expect.arrayContaining([expect.objectContaining({ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }), expect.objectContaining({ name: "snapshot", mountPath: "/migration" })])); } - expect(job.spec.template.spec.volumes).toEqual([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }, { name: "snapshot", emptyDir: {} }]); + expect(job.spec.template.spec.volumes).toEqual(expect.arrayContaining([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }, { name: "artifact-1", hostPath: { path: "/root/hwlab-v03", type: "Directory" } }, { name: "snapshot", emptyDir: {} }])); + expect(job.spec.template.spec.containers[0].volumeMounts).toEqual(expect.arrayContaining([expect.objectContaining({ name: "artifact-1", mountPath: "/root/hwlab-v03", readOnly: true })])); expect(job.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } }), expect.objectContaining({ name: "MONITOR_CENTRAL_PG_POOL_MAX", value: "4" })])); }); diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index 3192c139..6798c996 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -24,11 +24,12 @@ export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef requireImageRef(imageRef); if ((options.enabled ?? job.enabled) !== true) return null; const migrationLabels = { ...labels, "app.kubernetes.io/component": "migration" }; - const mounts = [{ name: "legacy-sqlite", mountPath: resolvedSource.stateRoot, readOnly: true }, { name: "snapshot", mountPath: "/migration" }]; + const sourceVolumes = resolvedSource.artifactSources.map((source, index) => ({ name: source.ownerPvc === resolvedSource.ownerPvc ? "legacy-sqlite" : `artifact-${index}`, source })); + const mounts = [...sourceVolumes.map((item) => ({ name: item.name, mountPath: item.source.mountPath, readOnly: true })), { name: "snapshot", mountPath: "/migration" }]; const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }]; const cli = ["scripts/cli.ts", "web-probe", "sentinel", "migration"]; const sourceArgs = ["--node", spec.nodeId, "--lane", spec.lane, "--sentinel", resolvedSource.sentinelId, "--snapshot", "/migration/snapshot.json"]; - return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: resolvedSource.ownerPvc, readOnly: true } }, { name: "snapshot", emptyDir: {} }] } } } }; + return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [...sourceVolumes.map((item) => item.source.hostPath === undefined ? { name: item.name, persistentVolumeClaim: { claimName: item.source.ownerPvc, readOnly: true } } : { name: item.name, hostPath: { path: item.source.hostPath, type: "Directory" } }), { name: "snapshot", emptyDir: {} }] } } } }; } export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly poolMax: number; readonly idleTimeoutSeconds: number } { @@ -51,13 +52,15 @@ function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: Monit const cicd = recordAt(monitor, "cicd", resolved.ref); const expectedRegistry = `config/hwlab-node-lanes.yaml#lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.sentinels`; const expectedConsumer = `sentinel-${spec.nodeId.toLowerCase()}-${spec.lane}`; - if (textAt(runtime, "node", resolved.ref) !== spec.nodeId || textAt(runtime, "lane", resolved.ref) !== spec.lane || textAt(persistence, "mode", resolved.ref) !== "host-postgres" || textAt(persistence, "authority", resolved.ref) !== "central-prepared-non-authoritative" || textAt(persistence, "database", resolved.ref) !== "web_probe_monitor" || textAt(persistence, "schema", resolved.ref) !== "monitor" || exposure.enabled !== false || textAt(exposure, "mode", resolved.ref) !== "prepared-not-routed" || textAt(exposure, "hostname", resolved.ref).length === 0 || textAt(cicd, "mode", resolved.ref) !== "existing-pac-consumer" || textAt(cicd, "consumer", resolved.ref) !== expectedConsumer || textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref} must remain a non-authoritative Release A configuration`); - if (textAt(source, "registry", resolved.ref) !== expectedRegistry || textAt(source, "sentinelId", resolved.ref) !== resolvedSource.sentinelId || textAt(source, "runtimeConfigRef", resolved.ref) !== resolvedSource.runtimeConfigRef || textAt(source, "sqlitePath", resolved.ref) !== resolvedSource.path || textAt(source, "pvcName", resolved.ref) !== resolvedSource.ownerPvc || textAt(source, "stateRoot", resolved.ref) !== resolvedSource.stateRoot || textAt(job, "workflow", resolved.ref) !== "export-import-verify" || job.readOnlySource !== true || typeof job.enabled !== "boolean" || !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(textAt(job, "name", resolved.ref))) throw new Error(`${resolved.ref}.migration must match the resolved active registry and declare an export-import-verify read-only workflow`); + if (textAt(runtime, "node", resolved.ref) !== spec.nodeId || textAt(runtime, "lane", resolved.ref) !== spec.lane || textAt(persistence, "mode", resolved.ref) !== "host-postgres" || textAt(persistence, "authority", resolved.ref) !== "central-prepared-non-authoritative" || exposure.enabled !== false || textAt(exposure, "mode", resolved.ref) !== "prepared-not-routed" || textAt(exposure, "hostname", resolved.ref).length === 0 || textAt(cicd, "mode", resolved.ref) !== "existing-pac-consumer" || textAt(cicd, "consumer", resolved.ref) !== expectedConsumer || textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref} must remain a non-authoritative Release A configuration`); + if (textAt(source, "registry", resolved.ref) !== expectedRegistry || textAt(job, "workflow", resolved.ref) !== "export-import-verify" || job.readOnlySource !== true || typeof job.enabled !== "boolean" || !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(textAt(job, "name", resolved.ref))) throw new Error(`${resolved.ref}.migration must reference the resolved active registry and declare an export-import-verify read-only workflow`); const namespace = textAt(runtime, "namespace", resolved.ref); const deploymentName = textAt(runtime, "deploymentName", resolved.ref); const port = integerAt(runtime, "servicePort", resolved.ref); const secret = recordAt(runtime, "databaseSecret", resolved.ref); - if (textAt(secret, "name", resolved.ref) !== "hwlab-web-probe-monitor-db" || textAt(secret, "key", resolved.ref) !== "DATABASE_URL" || textAt(secret, "sourceRef", resolved.ref) !== "hwlab/web-probe-monitor-db.env") throw new Error(`${resolved.ref}.runtime.databaseSecret must be the controlled monitor DATABASE_URL projection`); + textAt(secret, "name", resolved.ref); + textAt(secret, "key", resolved.ref); + textAt(secret, "sourceRef", resolved.ref); const pool = recordAt(runtime, "pool", resolved.ref); return { resolved, runtime, job, namespace, deploymentName, port, secret, pool, labels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" } }; } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index fa82b1cc..261bbfad 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -229,7 +229,7 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath); } function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string | null): readonly MonitorSqliteSource[] { @@ -237,7 +237,15 @@ function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string return ids.map((id) => { const sentinel = resolveWebProbeSentinel(spec, id); const runtime = recordTarget(readWebProbeSentinelConfigRefTarget(spec, sentinel.configRefs.runtime), sentinel.configRefs.runtime); - return { sentinelId: sentinel.id, node: spec.nodeId, lane: spec.lane, path: stringAt(runtime, "sqlite.path"), runtimeConfigRef: sentinel.configRefs.runtime, stateRoot: stringAt(runtime, "stateRoot"), ownerPvc: stringAt(runtime, "pvcName") }; + const stateRoot = stringAt(runtime, "stateRoot"); + const ownerPvc = stringAt(runtime, "pvcName"); + const artifactSources = arrayAt(runtime, "artifactSources").map((item, index) => { + const source = recordTarget(item, `${sentinel.configRefs.runtime}.artifactSources[${index}]`); + const hostPath = stringAtNullable(source, "hostPath"); + return { ownerPvc: stringAt(source, "ownerPvc"), mountPath: stringAt(source, "mountPath"), ...(hostPath === null ? {} : { hostPath }) }; + }); + if (!artifactSources.some((item) => item.ownerPvc === ownerPvc && item.mountPath === stateRoot)) throw new Error(`${sentinel.configRefs.runtime}.artifactSources must include the resolved index PVC and stateRoot`); + return { sentinelId: sentinel.id, node: spec.nodeId, lane: spec.lane, path: stringAt(runtime, "sqlite.path"), runtimeConfigRef: sentinel.configRefs.runtime, stateRoot, ownerPvc, artifactSources }; }); } diff --git a/scripts/src/monitor-central-persistence.test.ts b/scripts/src/monitor-central-persistence.test.ts index d5801c33..5c4a0d35 100644 --- a/scripts/src/monitor-central-persistence.test.ts +++ b/scripts/src/monitor-central-persistence.test.ts @@ -32,6 +32,14 @@ test("HTTP terminal ingest is unbound-safe, ignores caller hash, and maps confli assert.equal((await fetch(new Request("http://local/health"))).status, 200); }); +test("health returns 503 when the central store is not ready", async () => { + const store = createInMemoryMonitorCentralStore(); + const target = createMonitorCentralService({ store: { ...store, async ping() { throw new Error("postgres unavailable"); } }, pageSize: 50 }); + const response = await target.fetch(new Request("http://local/health")); + assert.equal(response.status, 503); + assert.equal((await response.json() as { ok: boolean }).ok, false); +}); + test("full terminal envelope controls hash and concurrent same identity converges", async () => { const target = service(); const base = { ...input("run-envelope"), payloadHash: "sha256:caller-controlled" }; diff --git a/scripts/src/monitor-central-persistence.ts b/scripts/src/monitor-central-persistence.ts index 6c4d9d09..e9a8c760 100644 --- a/scripts/src/monitor-central-persistence.ts +++ b/scripts/src/monitor-central-persistence.ts @@ -158,7 +158,10 @@ export function createMonitorCentralService(options: MonitorCentralServiceOption const result = await ingest(await requestRecord(request)); return json(result, result.disposition === "inserted" ? 201 : 200); } - if (request.method === "GET" && url.pathname === "/health") return json(await health()); + if (request.method === "GET" && url.pathname === "/health") { + const result = await health(); + return json(result, result.ok === true ? 200 : 503); + } if (request.method === "GET" && url.pathname === "/api/overview") return json({ ok: true, overview: await store.overview(scopeFrom(url)), valuesRedacted: true }); if (request.method === "GET" && url.pathname === "/api/runs") return json(runPage(await store.runs(scopeFrom(url), runQueryOptions(url, pageSize)))); if (request.method === "GET" && url.pathname === "/api/findings") return json(findingPage(await store.findings(scopeFrom(url), findingQueryOptions(url, pageSize)))); diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index 30f555c7..c0abd30f 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -9,7 +9,8 @@ import type { MonitorArtifactLocator, MonitorCentralIngest, MonitorCentralStore, import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monitor-central-persistence"; export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-2"; -export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; } +export interface MonitorArtifactSource { readonly ownerPvc: string; readonly mountPath: string; readonly hostPath?: string; } +export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; readonly artifactSources: readonly MonitorArtifactSource[]; } export interface MonitorSqliteTable { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record[]; } export interface MonitorSqliteSnapshot { readonly schema: string; readonly createdAt: string; readonly source: MonitorSqliteSource; readonly fingerprint: string; readonly integrity: string; readonly byteLength: number; @@ -59,7 +60,6 @@ export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[] export function exportMonitorSqliteSnapshot(source: MonitorSqliteSource, outputPath: string): MonitorSqliteSnapshot { const path = assertReadOnlySqliteSource(source.path); - const sourceBytes = readFileSync(path); const database = new Database(path, { readonly: true }); let transactionOpen = false; try { @@ -68,7 +68,10 @@ export function exportMonitorSqliteSnapshot(source: MonitorSqliteSource, outputP const integrity = String(database.query("PRAGMA integrity_check").get()?.integrity_check ?? "unknown"); if (integrity !== "ok") throw new Error(`legacy SQLite integrity_check failed: ${integrity}`); const tables = (database.query("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all() as { name: string; sql: string }[]).map((table) => ({ name: table.name, sql: table.sql, rowCount: Number(database.query(`SELECT COUNT(*) AS count FROM ${quoteIdentifier(table.name)}`).get()?.count ?? 0), rows: database.query(`SELECT * FROM ${quoteIdentifier(table.name)}`).all().map(normalizeRow) })); - const unsigned = { schema: MONITOR_SQLITE_MIGRATION_SCHEMA, createdAt: new Date().toISOString(), source: { ...source, path }, fingerprint: `sha256:${sha256(sourceBytes)}`, integrity, byteLength: sourceBytes.byteLength, tables, artifactLocator: { path, sha256: `sha256:${sha256(sourceBytes)}`, sizeBytes: sourceBytes.byteLength, kind: "legacy-sqlite" } }; + const immutablePath = sqliteSnapshotPath(outputPath); + const immutableBytes = (database as unknown as { serialize(): Uint8Array }).serialize(); + writeFileSync(immutablePath, immutableBytes); + const unsigned = { schema: MONITOR_SQLITE_MIGRATION_SCHEMA, createdAt: new Date().toISOString(), source: { ...source, path }, fingerprint: `sha256:${sha256(immutableBytes)}`, integrity, byteLength: immutableBytes.byteLength, tables, artifactLocator: { path: immutablePath, sha256: `sha256:${sha256(immutableBytes)}`, sizeBytes: immutableBytes.byteLength, kind: "legacy-sqlite" } }; const snapshot: MonitorSqliteSnapshot = { ...unsigned, manifestContentHash: monitorSnapshotContentHash(unsigned) }; writeSnapshot(outputPath, `${monitorCentralStableJson(snapshot)}\n`); database.exec("COMMIT"); @@ -120,14 +123,17 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< const payload = { ...(storedPayload ?? {}), ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } }; const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : []; const canonicalFindings = dedupeFindings(reportFindings, rowFindings); - return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators: reportLocator ? [...rowLocators, reportLocator] : rowLocators, timeline: [] }); + const artifactLocators = reportLocator ? [...rowLocators, reportLocator] : rowLocators; + const artifactCount = numberField(row, ["artifact_count", "artifactCount"]); + if (artifactCount !== null && artifactCount !== artifactLocators.length) throw Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch" }); + return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators, timeline: [] }); }); } function targetReconciliation(ingests: readonly ReturnType[], runtimeMetadata: readonly MonitorRuntimeMetadata[]): Record { return { runs: ingests.length, findings: ingests.reduce((total, item) => total + item.findings.length, 0), payloads: ingests.length, artifactLocators: ingests.reduce((total, item) => total + item.artifactLocators.length, 0), views: ingests.reduce((total, item) => total + (isRecord(item.payload.views) ? Object.keys(item.payload.views).length : 0), 0), runtimeMetadata: runtimeMetadata.length }; } function canonicalRuntimeMetadata(snapshot: MonitorSqliteSnapshot): readonly MonitorRuntimeMetadata[] { return tableRows(snapshot.tables, ["metadata", "run_metadata"]).map((row) => ({ sentinelId: snapshot.source.sentinelId, node: snapshot.source.node, lane: snapshot.source.lane, key: stringField(row, ["key"]), value: jsonValue(row, ["value_json", "value"]), updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt })).filter((item) => item.key.length > 0).sort((left, right) => left.key.localeCompare(right.key)); } function sameRun(value: unknown, expected: { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly runId: string; readonly updatedAt: string }): boolean { return isRecord(value) && value.sentinelId === expected.sentinelId && value.node === expected.node && value.lane === expected.lane && value.runId === expected.runId && value.updatedAt === expected.updatedAt; } -function locatorFromRow(row: Record, source: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const hash = stringField(row, ["sha256"]); const sizeBytes = numberField(row, ["size_bytes", "sizeBytes"]); if (!ownerPvc || !relativePath || !hash || sizeBytes === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || source.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || source.source.lane, ownerPvc, stateDir: stringField(row, ["state_dir", "stateDir"]), relativePath, sha256: hash, sizeBytes, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } -function reportLocatorFromRun(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator | null { const stateDir = stringField(row, ["state_dir", "stateDir"]); const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!stateDir && !expectedHash) return null; if (!stateDir || !expectedHash) throw Object.assign(new Error("legacy report locator is incomplete"), { code: "monitor-migration-report-locator-invalid" }); const path = join(snapshot.source.stateRoot, stateDir, "analysis", "report.json"); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc: snapshot.source.ownerPvc, stateDir, relativePath: "analysis/report.json", sha256: actualHash, sizeBytes: bytes.byteLength, kind: "report-json", reachable: true }; } +function locatorFromRow(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const expectedHash = stringField(row, ["sha256"]); const expectedSize = numberField(row, ["size_bytes", "sizeBytes"]); const stateDir = stringField(row, ["state_dir", "stateDir"]); if (!ownerPvc || !stateDir || !relativePath || !expectedHash || expectedSize === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); const source = artifactSourceFor(snapshot.source, ownerPvc); const path = artifactPath(source.mountPath, stateDir, relativePath); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash || bytes.byteLength !== expectedSize) throw Object.assign(new Error("legacy artifact locator does not match source artifact"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || snapshot.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || snapshot.source.lane, ownerPvc, stateDir, relativePath, sha256: actualHash, sizeBytes: bytes.byteLength, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } +function reportLocatorFromRun(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator | null { const stateDir = stringField(row, ["state_dir", "stateDir"]); const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!stateDir && !expectedHash) return null; if (!stateDir || !expectedHash) throw Object.assign(new Error("legacy report locator is incomplete"), { code: "monitor-migration-report-locator-invalid" }); const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc", "artifact_owner_pvc", "artifactOwnerPvc"]) || snapshot.source.ownerPvc; const source = artifactSourceFor(snapshot.source, ownerPvc); const path = artifactPath(source.mountPath, stateDir, "analysis/report.json"); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc, stateDir, relativePath: "analysis/report.json", sha256: actualHash, sizeBytes: bytes.byteLength, kind: "report-json", reachable: true }; } function metadataValue(rows: readonly Record[], key: string): Record | null { return jsonField(rows.find((row) => stringField(row, ["key"]) === key), ["value_json", "value"]); } function normalizeLegacyFinding(row: Record): Record { return jsonField(row, ["finding", "value_json"]) ?? { id: stringField(row, ["finding_id", "id"]), severity: stringField(row, ["severity"]), count: numberField(row, ["count"]) ?? 1, summary: stringField(row, ["summary"]), reportJsonSha256: stringField(row, ["report_json_sha256", "reportJsonSha256"]), createdAt: stringField(row, ["created_at", "createdAt"]) }; } function dedupeFindings(primary: readonly Record[], secondary: readonly Record[]): readonly Record[] { const seen = new Set(); return [...primary, ...secondary].filter((finding) => { const id = stringField(finding, ["id"]); const key = id || monitorCentralStableJson(finding); if (seen.has(key)) return false; seen.add(key); return true; }); } @@ -135,6 +141,9 @@ function tableRows(tables: readonly MonitorSqliteTable[], names: readonly string function sourcePlan(source: MonitorSqliteSource): Record { const path = resolve(source.path); const state = existsSync(path) ? statSync(path) : null; return { sentinelId: source.sentinelId, node: source.node, lane: source.lane, path, runtimeConfigRef: source.runtimeConfigRef, exists: state !== null, isFile: state?.isFile() ?? false, byteLength: state?.size ?? null, mutation: false, valuesRedacted: true }; } function assertReadOnlySqliteSource(value: string): string { const path = resolve(value); const state = statSync(path); if (!state.isFile()) throw new Error(`legacy SQLite source is not a file: ${path}`); const descriptor = openSync(path, "r"); closeSync(descriptor); return path; } function writeSnapshot(path: string, content: string): void { const parent = dirname(resolve(path)); if (!existsSync(parent)) throw new Error(`snapshot output directory does not exist: ${parent}`); writeFileSync(path, content); } +function sqliteSnapshotPath(path: string): string { const resolved = resolve(path); return resolved.endsWith(".json") ? `${resolved.slice(0, -5)}.sqlite` : `${resolved}.sqlite`; } +function artifactSourceFor(source: MonitorSqliteSource, ownerPvc: string): MonitorArtifactSource { const artifactSource = source.artifactSources.find((item) => item.ownerPvc === ownerPvc); if (!artifactSource) throw Object.assign(new Error(`legacy artifact owner is not registry-mounted: ${ownerPvc}`), { code: "monitor-migration-artifact-source-missing" }); return artifactSource; } +function artifactPath(mountPath: string, stateDir: string, relativePath: string): string { const root = resolve(mountPath); const directory = stateDir.startsWith("/") ? resolve(stateDir) : resolve(root, stateDir); if (directory !== root && !directory.startsWith(`${root}/`)) throw Object.assign(new Error("legacy artifact stateDir escapes registry mount"), { code: "monitor-migration-artifact-path-invalid" }); return join(directory, relativePath); } function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; } function normalizeRow(value: unknown): Record { const row = value as Record; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); } function jsonField(row: Record | undefined, fields: readonly string[]): Record | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 17749a5d..88aed141 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -1,5 +1,6 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-12-p17-monitor-central-persistence. import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -50,18 +51,20 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio const snapshotPath = join(directory, "snapshot.json"); const database = new Database(sqlitePath); const stateRoot = join(directory, "state"); - const reportPath = join(stateRoot, "run-0", "analysis", "report.json"); - mkdirSync(join(stateRoot, "run-0", "analysis"), { recursive: true }); + const workspaceRoot = join(directory, "hwlab-v03"); + const workspaceRun = join(workspaceRoot, "web-observe", "run-0"); + const reportPath = join(workspaceRun, "analysis", "report.json"); + mkdirSync(join(workspaceRun, "analysis"), { recursive: true }); writeFileSync(reportPath, JSON.stringify({ updatedAt: "2026-07-12T00:00:00.000Z", views: { summary: { title: "legacy summary" } }, findings: [{ id: "report-finding" }] })); const reportHash = `sha256:${Bun.CryptoHasher.hash("sha256", readFileSync(reportPath), "hex")}`; - database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, report_json_sha256 TEXT, updated_at TEXT)"); + database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_owner_pvc TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)"); database.run("CREATE TABLE findings (run_id TEXT, sentinel_id TEXT, finding_id TEXT, severity TEXT, count INTEGER, summary TEXT, report_json_sha256 TEXT, created_at TEXT)"); database.run("CREATE TABLE metadata (id TEXT, key TEXT, value_json TEXT)"); - for (let index = 0; index < 228; index += 1) database.run("INSERT INTO runs VALUES (?, ?, ?, ?)", [`run-${index}`, index === 0 ? "run-0" : null, index === 0 ? reportHash : null, "2026-07-12T00:00:00.000Z"]); + for (let index = 0; index < 228; index += 1) database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?)", [`run-${index}`, index === 0 ? workspaceRun : null, index === 0 ? "legacy-web-observe-workspace-nc01" : null, index === 0 ? 1 : 0, index === 0 ? reportHash : null, "2026-07-12T00:00:00.000Z"]); for (let index = 0; index < 242; index += 1) database.run("INSERT INTO findings VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [index === 0 ? "run-0" : `run-${index % 228}`, "fixture", index === 0 ? "report-finding" : `finding-${index}`, "warning", 1, "indexed", reportHash, "2026-07-12T00:00:00.000Z"]); for (let index = 0; index < 234; index += 1) database.run("INSERT INTO metadata VALUES (?, ?, ?)", [`metadata-${index}`, index === 0 ? "run.report.run-0" : index === 1 ? "scheduler.heartbeat" : index === 2 ? "maintenance" : index === 3 ? "manual-trigger.latest" : `metadata-${index}`, index === 0 ? JSON.stringify({ views: { summary: { title: "legacy summary" } }, findings: [{ id: "report-finding", rootCause: "complete metadata detail" }] }) : index === 2 ? JSON.stringify(false) : JSON.stringify({ index })]); database.close(); - const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot, ownerPvc: "fixture-pvc" }; + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: stateRoot }, { ownerPvc: "legacy-web-observe-workspace-nc01", mountPath: workspaceRoot, hostPath: workspaceRoot }] }; const plan = planMonitorSqliteSources([source]); assert.equal((plan.sources as { exists: boolean }[])[0].exists, true); const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); @@ -78,6 +81,7 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio assert.equal(importedMetadata.find((item) => item.key === "scheduler.heartbeat")?.updatedAt, snapshot.createdAt); const importedRun = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0"); assert.equal(importedRun?.artifactLocators[0]?.sha256, reportHash); + assert.equal(importedRun?.artifactLocators[0]?.ownerPvc, "legacy-web-observe-workspace-nc01"); const reportFindings = importedRun?.findings.filter((finding) => finding.id === "report-finding") ?? []; assert.deepEqual(reportFindings, [{ id: "report-finding", rootCause: "complete metadata detail" }]); assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary"); @@ -117,7 +121,7 @@ test("metadata-only snapshot verifies scoped runtime metadata and rejects extras database.run("CREATE TABLE metadata (key TEXT, value_json TEXT)"); database.run("INSERT INTO metadata VALUES (?, ?)", ["scheduler.heartbeat", JSON.stringify({ healthy: true })]); database.close(); - const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc" }; + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] }; const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); const store = createInMemoryMonitorCentralStore(); await importMonitorSqliteSnapshot(snapshot, store); @@ -128,3 +132,69 @@ test("metadata-only snapshot verifies scoped runtime metadata and rejects extras rmSync(directory, { recursive: true, force: true }); } }); + +test("WAL export freezes a serialized SQLite snapshot independent of later live writes", () => { + const directory = mkdtempSync(join(tmpdir(), "monitor-wal-snapshot-")); + try { + const sqlitePath = join(directory, "index.sqlite"); + const snapshotPath = join(directory, "snapshot.json"); + const database = new Database(sqlitePath); + database.exec("PRAGMA journal_mode = WAL"); + database.run("CREATE TABLE metadata (key TEXT, value_json TEXT)"); + database.run("INSERT INTO metadata VALUES (?, ?)", ["scheduler.heartbeat", JSON.stringify({ sequence: 1 })]); + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] }; + const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); + database.run("INSERT INTO metadata VALUES (?, ?)", ["scheduler.heartbeat.late", JSON.stringify({ sequence: 2 })]); + database.close(); + + assert.equal(snapshot.artifactLocator.path, join(directory, "snapshot.sqlite")); + assert.equal(snapshot.tables.find((table) => table.name === "metadata")?.rowCount, 1); + assert.equal(verifyMonitorSqliteSnapshot(snapshot).ok, true); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("workspace artifact source verifies locator identity hash and size", async () => { + const directory = mkdtempSync(join(tmpdir(), "monitor-workspace-artifact-")); + try { + const sqlitePath = join(directory, "index.sqlite"); + const snapshotPath = join(directory, "snapshot.json"); + const workspaceRoot = join(directory, "hwlab-v03"); + const stateDir = join(workspaceRoot, "web-observe", "run-1"); + const reportBytes = Buffer.from('{"ok":true}\n'); + mkdirSync(join(stateDir, "analysis"), { recursive: true }); + writeFileSync(join(stateDir, "analysis", "report.json"), reportBytes); + const reportHash = `sha256:${createHash("sha256").update(reportBytes).digest("hex")}`; + const database = new Database(sqlitePath); + database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_owner_pvc TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)"); + database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?)", ["run-1", stateDir, "legacy-web-observe-workspace-nc01", 1, reportHash, "2026-07-12T00:00:00.000Z"]); + database.close(); + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }, { ownerPvc: "legacy-web-observe-workspace-nc01", mountPath: workspaceRoot, hostPath: workspaceRoot }] }; + const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); + const store = createInMemoryMonitorCentralStore(); + await importMonitorSqliteSnapshot(snapshot, store); + const run = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-1"); + assert.deepEqual(run?.artifactLocators, [{ ownerNode: "NC01", ownerLane: "v03", ownerPvc: "legacy-web-observe-workspace-nc01", stateDir, relativePath: "analysis/report.json", sha256: reportHash, sizeBytes: reportBytes.byteLength, kind: "report-json", reachable: true }]); + assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("artifact count without a registry-backed locator fails closed", async () => { + const directory = mkdtempSync(join(tmpdir(), "monitor-artifact-count-")); + try { + const sqlitePath = join(directory, "index.sqlite"); + const snapshotPath = join(directory, "snapshot.json"); + const database = new Database(sqlitePath); + database.run("CREATE TABLE runs (id TEXT, artifact_count INTEGER, updated_at TEXT)"); + database.run("INSERT INTO runs VALUES (?, ?, ?)", ["run-1", 1, "2026-07-12T00:00:00.000Z"]); + database.close(); + const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] }; + const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); + await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-count-mismatch"); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); From 3a63da0fa8a2d55cb021c5b2de23884083e8a59c Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 00:25:44 +0000 Subject: [PATCH 5/6] fix: complete Release A migration execution --- config/hwlab-web-probe-monitor/runtime.yaml | 1 + scripts/src/cicd-delivery-authority.test.ts | 8 +++++ .../src/hwlab-node-web-probe-monitor.test.ts | 7 ++++- scripts/src/hwlab-node-web-probe-monitor.ts | 8 ++--- .../hwlab-node-web-sentinel-cicd-shared.ts | 4 ++- scripts/src/hwlab-node-web-sentinel-cicd.ts | 31 ++++++++++++++++++- scripts/src/hwlab-node/web-probe-observe.ts | 7 +++-- scripts/src/monitor-sqlite-migration.ts | 13 +++++--- .../monitor-terminal-ingest-migration.test.ts | 24 +++++++++++--- 9 files changed, 83 insertions(+), 20 deletions(-) diff --git a/config/hwlab-web-probe-monitor/runtime.yaml b/config/hwlab-web-probe-monitor/runtime.yaml index ed6651eb..057fa8eb 100644 --- a/config/hwlab-web-probe-monitor/runtime.yaml +++ b/config/hwlab-web-probe-monitor/runtime.yaml @@ -15,6 +15,7 @@ monitor: deploymentName: hwlab-web-probe-monitor serviceName: hwlab-web-probe-monitor servicePort: 8090 + bindHost: 0.0.0.0 replicas: 1 healthPath: /health databaseSecret: diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index 4ef06656..991be5ea 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -286,6 +286,14 @@ describe("migrated CLI 执行 guard 与提示清理", () => { expect(parsed.sentinel).toMatchObject({ kind: "publish", internalExecution: "pac-controller", manualRecovery: false }); }); + test("migration one-shot Job 仅允许显式计划或双确认等待执行", () => { + const plan = parseNodeWebProbeSentinelOptions(["migration", "job-plan", "--node", "NC01", "--lane", "v03", "--sentinel", "nc01-web-probe-sentinel"]); + expect(plan.sentinel).toMatchObject({ kind: "migration", action: "job-plan", confirm: false, wait: false }); + expect(() => parseNodeWebProbeSentinelOptions(["migration", "job-run", "--node", "NC01", "--lane", "v03", "--sentinel", "nc01-web-probe-sentinel", "--confirm"])).toThrow("requires --confirm --wait"); + const execute = parseNodeWebProbeSentinelOptions(["migration", "job-run", "--node", "NC01", "--lane", "v03", "--sentinel", "nc01-web-probe-sentinel", "--confirm", "--wait"]); + expect(execute.sentinel).toMatchObject({ kind: "migration", action: "job-run", confirm: true, wait: true }); + }); + test("PaC 外层事件与内层 deterministic publish 分开归类，只有外层可驱动 delivery", () => { const consumer = { repository: "sentinel-nc01-v03", pipelineRunPrefix: "hwlab-web-probe-sentinel-nc01-", pipeline: "hwlab-web-probe-sentinel-nc01" }; const outer = { diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index 9473bb63..c409b507 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -19,7 +19,7 @@ test("Release A live manifests exclude disabled migration Job and select only mo expect(deployment.spec.template.metadata.labels["app.kubernetes.io/component"]).toBe("monitor-service"); expect(deployment.spec.template.spec.containers[0].image).toBe(imageRef); expect(deployment.spec.template.spec.containers[0].readinessProbe.httpGet.path).toBe("/health"); - expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } })])); + expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } }), { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }])); expect(releaseAMonitorMigrationPool(spec, source)).toEqual({ poolMax: 4, idleTimeoutSeconds: 30 }); expect(releaseAMonitorManifests(spec, imageRef, source, { migrationJobEnabled: true }).map((item) => item.kind)).toEqual(["Service", "Deployment", "Job"]); }); @@ -42,6 +42,11 @@ test("explicit migration helper renders ordered read-only export import verify J expect(job.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } }), expect.objectContaining({ name: "MONITOR_CENTRAL_PG_POOL_MAX", value: "4" })])); }); +test("one-shot migration helper accepts a unique control-plane suffix", () => { + const job = releaseAMonitorMigrationJob(hwlabRuntimeLaneSpecForNode("v03", "NC01"), imageRef, source, { enabled: true, nameSuffix: "run-42" }) as any; + expect(job.metadata.name).toContain("source-commit-run-42"); +}); + test("Release A declares the controlled NC01 monitor database Secret projection", () => { const config = readYamlRecord>(rootPath("config/secrets-distribution.yaml"), "unidesk-secret-distribution"); expect(config.sources.files).toEqual(expect.arrayContaining([expect.objectContaining({ sourceRef: "hwlab/web-probe-monitor-db.env", type: "env", requiredKeys: ["DATABASE_URL"], createIfMissing: { enabled: false } })])); diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index 6798c996..5ef87c17 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -8,7 +8,7 @@ export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: s const config = releaseAMonitorConfig(spec, resolvedSource); const { resolved, runtime, namespace, deploymentName, port, secret, pool, labels } = config; requireImageRef(imageRef); - const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; + const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: textAt(runtime, "bindHost", resolved.ref) }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; const monitorLabels = { ...labels, "app.kubernetes.io/component": "monitor-service" }; const manifests = [ { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels: monitorLabels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/component": "monitor-service" }, ports: [{ name: "http", port, targetPort: "http" }] } }, @@ -18,7 +18,7 @@ export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: s return migrationJob === null ? manifests : [...manifests, migrationJob]; } -export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly enabled?: boolean } = {}): Record | null { +export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly enabled?: boolean; readonly nameSuffix?: string } = {}): Record | null { const config = releaseAMonitorConfig(spec, resolvedSource); const { resolved, runtime, job, namespace, secret, pool, labels } = config; requireImageRef(imageRef); @@ -29,7 +29,7 @@ export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }]; const cli = ["scripts/cli.ts", "web-probe", "sentinel", "migration"]; const sourceArgs = ["--node", spec.nodeId, "--lane", spec.lane, "--sentinel", resolvedSource.sentinelId, "--snapshot", "/migration/snapshot.json"]; - return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [...sourceVolumes.map((item) => item.source.hostPath === undefined ? { name: item.name, persistentVolumeClaim: { claimName: item.source.ownerPvc, readOnly: true } } : { name: item.name, hostPath: { path: item.source.hostPath, type: "Directory" } }), { name: "snapshot", emptyDir: {} }] } } } }; + return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef, options.nameSuffix), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [...sourceVolumes.map((item) => item.source.hostPath === undefined ? { name: item.name, persistentVolumeClaim: { claimName: item.source.ownerPvc, readOnly: true } } : { name: item.name, hostPath: { path: item.source.hostPath, type: "Directory" } }), { name: "snapshot", emptyDir: {} }] } } } }; } export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly poolMax: number; readonly idleTimeoutSeconds: number } { @@ -69,5 +69,5 @@ function recordAt(value: Record, key: string, path: string): Re function textAt(value: Record, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; } function integerAt(value: Record, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function versionedJobName(base: string, imageRef: string): string { const suffix = imageRef.split(":").at(-1)?.replace(/[^a-z0-9-]/gu, "-").slice(0, 20) || "image"; return `${base}-${suffix}`.slice(0, 63).replace(/-+$/u, ""); } +function versionedJobName(base: string, imageRef: string, nameSuffix?: string): string { const imageSuffix = imageRef.split(":").at(-1)?.replace(/[^a-z0-9-]/gu, "-").slice(0, 20) || "image"; const suffix = nameSuffix === undefined ? imageSuffix : `${imageSuffix}-${nameSuffix.replace(/[^a-z0-9-]/gu, "-").slice(0, 12)}`; return `${base}-${suffix}`.slice(0, 63).replace(/-+$/u, ""); } function requireImageRef(imageRef: string): void { if (typeof imageRef !== "string" || imageRef.trim().length === 0) throw new Error("Release A monitor imageRef must be a non-empty current PaC image reference"); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts b/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts index f2464c2f..a4a33d1c 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts @@ -16,7 +16,7 @@ export type WebProbeSentinelPublishAction = "publish-current"; export type WebProbeSentinelMaintenanceAction = "status" | "start" | "stop"; export type WebProbeSentinelDashboardAction = "verify" | "screenshot" | "trigger"; export type WebProbeSentinelReportView = "summary" | "turn-summary" | "findings" | "trace-frame" | "auth-session-switch-summary"; -export type WebProbeSentinelMigrationAction = "plan" | "export" | "import" | "verify"; +export type WebProbeSentinelMigrationAction = "plan" | "export" | "import" | "verify" | "job-plan" | "job-run"; export type WebProbeSentinelSourceAuthority = "git-mirror-snapshot" | "gitea-snapshot"; export interface WebProbeSentinelSourceOverrideOptions { @@ -35,6 +35,8 @@ export type WebProbeSentinelOptions = readonly sentinelId: string | null; readonly snapshotPath: string | null; readonly confirm: boolean; + readonly wait: boolean; + readonly timeoutSeconds: number; readonly json: boolean; } | { diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 261bbfad..6eba65f0 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -37,7 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration"; -import { releaseAMonitorManifests, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; +import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor"; import { arrayAt, arrayAtNullable, @@ -195,6 +195,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract): RenderedCliResult { const sources = resolvedMigrationSources(spec, options.sentinelId); const command = `web-probe sentinel migration ${options.action}`; + if (options.action === "job-plan" || options.action === "job-run") return runSentinelMigrationJob(spec, options, sources, command); if (options.action === "plan") { const projection = planMonitorSqliteSources(sources); return rendered(true, command, renderMigrationCompact(projection), projection); @@ -228,6 +229,34 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract, sources: readonly MonitorSqliteSource[], command: string): RenderedCliResult { + if (sources.length !== 1) throw new Error(`web-probe sentinel migration ${options.action} requires --sentinel when more than one resolved registry source exists`); + const state = loadSentinelCicdState(spec, sources[0]!.sentinelId, options.timeoutSeconds, "cached"); + const manifest = releaseAMonitorMigrationJob(spec, state.image.ref, sources[0]!, { enabled: true, nameSuffix: Date.now().toString(36) }); + if (manifest === null) throw new Error("Release A migration Job helper did not render an enabled Job"); + const metadata = manifest.metadata as Record; + const namespace = String(metadata.namespace); + const jobName = String(metadata.name); + const plan = { ok: true, command, mutation: false, namespace, jobName, image: state.image.ref, source: sources[0], manifest: { apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { name: jobName, namespace } }, valuesRedacted: true }; + if (options.action === "job-plan") return rendered(true, command, renderMigrationCompact(plan), plan); + if (!options.confirm || !options.wait) throw new Error("web-probe sentinel migration job-run requires --confirm --wait"); + const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 }); + if (created.exitCode !== 0) return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase: "create-failed", create: compactCommand(created) }); + const startedAt = Date.now(); + let lastProbe = created; + while (Date.now() - startedAt < Math.max(5_000, Math.min(options.timeoutSeconds * 1000, 120_000))) { + lastProbe = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 }); + const probe = resolveSentinelChildJson(lastProbe, "web-probe-sentinel-migration-job-probe").parsed ?? {}; + if (probe.succeeded === true || probe.failed === true) { + const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? "")); + const ok = probe.succeeded === true && payload.ok === true; + return rendered(ok, command, `ok=${ok}`, { ...plan, ok, mutation: true, phase: probe.succeeded === true ? "succeeded" : "failed", result: payload, valuesRedacted: true }); + } + runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); + } + return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase: "timeout", probe: compactCommand(lastProbe), valuesRedacted: true }); +} + function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean { return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath); } diff --git a/scripts/src/hwlab-node/web-probe-observe.ts b/scripts/src/hwlab-node/web-probe-observe.ts index 4e94905f..ab554273 100644 --- a/scripts/src/hwlab-node/web-probe-observe.ts +++ b/scripts/src/hwlab-node/web-probe-observe.ts @@ -220,11 +220,12 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe sentinel = { kind: "error", action: "get", node, lane, sentinelId, errorId, raw: args.includes("--raw"), full: args.includes("--full"), timeoutSeconds }; } else if (sentinelActionRaw === "migration") { const migrationAction = args[1]; - if (migrationAction !== "plan" && migrationAction !== "export" && migrationAction !== "import" && migrationAction !== "verify") { - throw new Error("web-probe sentinel migration usage: migration plan|export|import|verify --node NODE --lane vNN [--sentinel ID] [--snapshot PATH] [--confirm] [--json]"); + if (migrationAction !== "plan" && migrationAction !== "export" && migrationAction !== "import" && migrationAction !== "verify" && migrationAction !== "job-plan" && migrationAction !== "job-run") { + throw new Error("web-probe sentinel migration usage: migration plan|export|import|verify|job-plan|job-run --node NODE --lane vNN [--sentinel ID] [--snapshot PATH] [--confirm --wait] [--json]"); } if (migrationAction === "import" && !confirm) throw new Error("web-probe sentinel migration import requires --confirm"); - sentinel = { kind: "migration", action: migrationAction, node, lane, sentinelId, snapshotPath: optionValue(args, "--snapshot") ?? null, confirm, json: args.includes("--json") }; + if (migrationAction === "job-run" && (!confirm || !args.includes("--wait"))) throw new Error("web-probe sentinel migration job-run requires --confirm --wait"); + sentinel = { kind: "migration", action: migrationAction, node, lane, sentinelId, snapshotPath: optionValue(args, "--snapshot") ?? null, confirm, wait: args.includes("--wait"), timeoutSeconds, json: args.includes("--json") }; } else if (sentinelActionRaw === "report") { const view = parseWebProbeSentinelReportView(optionValue(args, "--view") ?? "summary"); const latest = args.includes("--latest"); diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index c0abd30f..a6dfa297 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -1,7 +1,7 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-12-p17-monitor-central-persistence. // Responsibility: Registry-resolved read-only legacy SQLite snapshots and one-shot canonical central-store import. import { createHash } from "node:crypto"; -import { closeSync, existsSync, openSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { closeSync, existsSync, openSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { Database } from "bun:sqlite"; @@ -118,14 +118,14 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< const rowFindings = findings.filter((item) => stringField(item, ["run_id"]) === runId).map(normalizeLegacyFinding); const report = metadataValue(metadata, `run.report.${runId}`); const rowLocators = locators.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); - const reportLocator = reportLocatorFromRun(row, snapshot); const storedPayload = jsonField(payloads.find((item) => stringField(item, ["run_id"]) === runId), ["payload", "value_json"]); const payload = { ...(storedPayload ?? {}), ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } }; const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : []; const canonicalFindings = dedupeFindings(reportFindings, rowFindings); - const artifactLocators = reportLocator ? [...rowLocators, reportLocator] : rowLocators; + const artifactLocators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); const artifactCount = numberField(row, ["artifact_count", "artifactCount"]); - if (artifactCount !== null && artifactCount !== artifactLocators.length) throw Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch" }); + if (artifactCount !== null && artifactCount !== artifactLocators.length) throw Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch", diagnostic: { runId, stateDir: stringField(row, ["state_dir", "stateDir"]), expectedCount: artifactCount, observedCount: artifactLocators.length, valuesRedacted: true } }); + verifyReportLocator(row, artifactLocators); return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators, timeline: [] }); }); } @@ -133,7 +133,8 @@ function targetReconciliation(ingests: readonly ReturnType ({ sentinelId: snapshot.source.sentinelId, node: snapshot.source.node, lane: snapshot.source.lane, key: stringField(row, ["key"]), value: jsonValue(row, ["value_json", "value"]), updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt })).filter((item) => item.key.length > 0).sort((left, right) => left.key.localeCompare(right.key)); } function sameRun(value: unknown, expected: { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly runId: string; readonly updatedAt: string }): boolean { return isRecord(value) && value.sentinelId === expected.sentinelId && value.node === expected.node && value.lane === expected.lane && value.runId === expected.runId && value.updatedAt === expected.updatedAt; } function locatorFromRow(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const expectedHash = stringField(row, ["sha256"]); const expectedSize = numberField(row, ["size_bytes", "sizeBytes"]); const stateDir = stringField(row, ["state_dir", "stateDir"]); if (!ownerPvc || !stateDir || !relativePath || !expectedHash || expectedSize === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); const source = artifactSourceFor(snapshot.source, ownerPvc); const path = artifactPath(source.mountPath, stateDir, relativePath); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash || bytes.byteLength !== expectedSize) throw Object.assign(new Error("legacy artifact locator does not match source artifact"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || snapshot.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || snapshot.source.lane, ownerPvc, stateDir, relativePath, sha256: actualHash, sizeBytes: bytes.byteLength, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } -function reportLocatorFromRun(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator | null { const stateDir = stringField(row, ["state_dir", "stateDir"]); const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!stateDir && !expectedHash) return null; if (!stateDir || !expectedHash) throw Object.assign(new Error("legacy report locator is incomplete"), { code: "monitor-migration-report-locator-invalid" }); const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc", "artifact_owner_pvc", "artifactOwnerPvc"]) || snapshot.source.ownerPvc; const source = artifactSourceFor(snapshot.source, ownerPvc); const path = artifactPath(source.mountPath, stateDir, "analysis/report.json"); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc, stateDir, relativePath: "analysis/report.json", sha256: actualHash, sizeBytes: bytes.byteLength, kind: "report-json", reachable: true }; } +function artifactLocatorsFromRun(row: Record, snapshot: MonitorSqliteSnapshot): readonly MonitorArtifactLocator[] { const stateDir = stringField(row, ["state_dir", "stateDir"]); if (!stateDir) return []; const source = artifactSourceForStateDir(snapshot.source, stateDir); const directory = artifactPath(source.mountPath, stateDir, ""); if (!existsSync(directory) || !statSync(directory).isDirectory()) throw Object.assign(new Error("legacy artifact stateDir is unavailable"), { code: "monitor-migration-artifact-path-invalid" }); return artifactFiles(directory).map((path) => { const bytes = readFileSync(path); const relativePath = path.slice(directory.length + 1).replaceAll("\\", "/"); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc: source.ownerPvc, stateDir, relativePath, sha256: `sha256:${sha256(bytes)}`, sizeBytes: bytes.byteLength, kind: relativePath === "analysis/report.json" ? "report-json" : "artifact", reachable: true }; }); } +function verifyReportLocator(row: Record, locators: readonly MonitorArtifactLocator[]): void { const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!expectedHash) return; const report = locators.find((locator) => locator.relativePath === "analysis/report.json"); if (!report || report.sha256 !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); } function metadataValue(rows: readonly Record[], key: string): Record | null { return jsonField(rows.find((row) => stringField(row, ["key"]) === key), ["value_json", "value"]); } function normalizeLegacyFinding(row: Record): Record { return jsonField(row, ["finding", "value_json"]) ?? { id: stringField(row, ["finding_id", "id"]), severity: stringField(row, ["severity"]), count: numberField(row, ["count"]) ?? 1, summary: stringField(row, ["summary"]), reportJsonSha256: stringField(row, ["report_json_sha256", "reportJsonSha256"]), createdAt: stringField(row, ["created_at", "createdAt"]) }; } function dedupeFindings(primary: readonly Record[], secondary: readonly Record[]): readonly Record[] { const seen = new Set(); return [...primary, ...secondary].filter((finding) => { const id = stringField(finding, ["id"]); const key = id || monitorCentralStableJson(finding); if (seen.has(key)) return false; seen.add(key); return true; }); } @@ -143,7 +144,9 @@ function assertReadOnlySqliteSource(value: string): string { const path = resolv function writeSnapshot(path: string, content: string): void { const parent = dirname(resolve(path)); if (!existsSync(parent)) throw new Error(`snapshot output directory does not exist: ${parent}`); writeFileSync(path, content); } function sqliteSnapshotPath(path: string): string { const resolved = resolve(path); return resolved.endsWith(".json") ? `${resolved.slice(0, -5)}.sqlite` : `${resolved}.sqlite`; } function artifactSourceFor(source: MonitorSqliteSource, ownerPvc: string): MonitorArtifactSource { const artifactSource = source.artifactSources.find((item) => item.ownerPvc === ownerPvc); if (!artifactSource) throw Object.assign(new Error(`legacy artifact owner is not registry-mounted: ${ownerPvc}`), { code: "monitor-migration-artifact-source-missing" }); return artifactSource; } +function artifactSourceForStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { const directory = resolve(stateDir); const matches = source.artifactSources.filter((item) => directory === resolve(item.mountPath) || directory.startsWith(`${resolve(item.mountPath)}/`)).sort((left, right) => resolve(right.mountPath).length - resolve(left.mountPath).length); if (matches.length === 0) throw Object.assign(new Error(`legacy artifact stateDir is not registry-mounted: ${stateDir}`), { code: "monitor-migration-artifact-source-missing" }); return matches[0]!; } function artifactPath(mountPath: string, stateDir: string, relativePath: string): string { const root = resolve(mountPath); const directory = stateDir.startsWith("/") ? resolve(stateDir) : resolve(root, stateDir); if (directory !== root && !directory.startsWith(`${root}/`)) throw Object.assign(new Error("legacy artifact stateDir escapes registry mount"), { code: "monitor-migration-artifact-path-invalid" }); return join(directory, relativePath); } +function artifactFiles(directory: string): readonly string[] { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); if (entry.isDirectory()) return artifactFiles(path); return entry.isFile() ? [path] : []; }).sort(); } function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; } function normalizeRow(value: unknown): Record { const row = value as Record; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); } function jsonField(row: Record | undefined, fields: readonly string[]): Record | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 88aed141..1546a0a2 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -162,21 +162,35 @@ test("workspace artifact source verifies locator identity hash and size", async const snapshotPath = join(directory, "snapshot.json"); const workspaceRoot = join(directory, "hwlab-v03"); const stateDir = join(workspaceRoot, "web-observe", "run-1"); - const reportBytes = Buffer.from('{"ok":true}\n'); + const artifacts = [ + ["analysis/report.json", Buffer.from('{"ok":true}\n')], + ["analysis/details.json", Buffer.from('{"details":true}\n')], + ["artifacts/request.txt", Buffer.from("request\n")], + ["artifacts/response.txt", Buffer.from("response\n")], + ["screenshots/page.png", Buffer.from([0x89, 0x50, 0x4e, 0x47])], + ] as const; + const reportBytes = artifacts[0][1]; mkdirSync(join(stateDir, "analysis"), { recursive: true }); - writeFileSync(join(stateDir, "analysis", "report.json"), reportBytes); + for (const [relativePath, bytes] of artifacts) { + const path = join(stateDir, relativePath); + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, bytes); + } const reportHash = `sha256:${createHash("sha256").update(reportBytes).digest("hex")}`; const database = new Database(sqlitePath); - database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_owner_pvc TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)"); - database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?)", ["run-1", stateDir, "legacy-web-observe-workspace-nc01", 1, reportHash, "2026-07-12T00:00:00.000Z"]); + database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)"); + database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", ["run-1", stateDir, 5, reportHash, "2026-07-12T00:00:00.000Z"]); database.close(); const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }, { ownerPvc: "legacy-web-observe-workspace-nc01", mountPath: workspaceRoot, hostPath: workspaceRoot }] }; const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); const store = createInMemoryMonitorCentralStore(); await importMonitorSqliteSnapshot(snapshot, store); const run = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-1"); - assert.deepEqual(run?.artifactLocators, [{ ownerNode: "NC01", ownerLane: "v03", ownerPvc: "legacy-web-observe-workspace-nc01", stateDir, relativePath: "analysis/report.json", sha256: reportHash, sizeBytes: reportBytes.byteLength, kind: "report-json", reachable: true }]); + assert.equal(run?.artifactLocators.length, 5); + assert.deepEqual(run?.artifactLocators.map((locator) => ({ ownerPvc: locator.ownerPvc, stateDir: locator.stateDir, relativePath: locator.relativePath, sha256: locator.sha256, sizeBytes: locator.sizeBytes })), artifacts.map(([relativePath, bytes]) => ({ ownerPvc: "legacy-web-observe-workspace-nc01", stateDir, relativePath, sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, sizeBytes: bytes.byteLength })).sort((left, right) => left.relativePath.localeCompare(right.relativePath))); assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true); + rmSync(join(stateDir, "artifacts", "response.txt")); + await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => { const migrationError = error as { code?: string; diagnostic?: Record }; return migrationError.code === "monitor-migration-artifact-count-mismatch" && migrationError.diagnostic?.runId === "run-1" && migrationError.diagnostic?.stateDir === stateDir && migrationError.diagnostic?.expectedCount === 5 && migrationError.diagnostic?.observedCount === 4; }); } finally { rmSync(directory, { recursive: true, force: true }); } From 036b29b59b8bd0d1c7a8f2cd1ba11b73c1d4ad06 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 00:32:36 +0000 Subject: [PATCH 6/6] fix: freeze migration artifact manifests --- scripts/src/monitor-sqlite-migration.ts | 21 +++++++++++-------- .../monitor-terminal-ingest-migration.test.ts | 8 ++++--- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/src/monitor-sqlite-migration.ts b/scripts/src/monitor-sqlite-migration.ts index a6dfa297..75009d60 100644 --- a/scripts/src/monitor-sqlite-migration.ts +++ b/scripts/src/monitor-sqlite-migration.ts @@ -8,13 +8,15 @@ import { Database } from "bun:sqlite"; import type { MonitorArtifactLocator, MonitorCentralIngest, MonitorCentralStore, MonitorRuntimeMetadata } from "./monitor-central-persistence"; import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monitor-central-persistence"; -export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-2"; +export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-3"; export interface MonitorArtifactSource { readonly ownerPvc: string; readonly mountPath: string; readonly hostPath?: string; } +function freezeArtifactManifests(snapshot: Pick): readonly { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[] }[] { const rows = tableRows(snapshot.tables, ["runs", "run"]); const locatorRows = tableRows(snapshot.tables, ["artifact_locators", "locators"]); return rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); const rowLocators = locatorRows.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); const locators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); const expectedCount = numberField(row, ["artifact_count", "artifactCount"]); if (expectedCount !== null && expectedCount !== locators.length) throw artifactCountMismatch(runId, stringField(row, ["state_dir", "stateDir"]), expectedCount, locators.length); verifyReportLocator(row, locators); return { runId, locators }; }); } export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; readonly artifactSources: readonly MonitorArtifactSource[]; } export interface MonitorSqliteTable { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record[]; } export interface MonitorSqliteSnapshot { readonly schema: string; readonly createdAt: string; readonly source: MonitorSqliteSource; readonly fingerprint: string; readonly integrity: string; readonly byteLength: number; readonly tables: readonly MonitorSqliteTable[]; + readonly artifactManifests: readonly { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[] }[]; readonly artifactLocator: { readonly path: string; readonly sha256: string; readonly sizeBytes: number; readonly kind: "legacy-sqlite" }; readonly manifestContentHash: string; } @@ -22,6 +24,7 @@ export interface MonitorMigrationReconciliation { readonly runs: number; readonl export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise> { const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot); + verifyFrozenArtifactManifests(snapshot); const expected = canonicalIngests(snapshot); const expectedMetadata = canonicalRuntimeMetadata(snapshot); const checks: Record[] = []; @@ -71,7 +74,8 @@ export function exportMonitorSqliteSnapshot(source: MonitorSqliteSource, outputP const immutablePath = sqliteSnapshotPath(outputPath); const immutableBytes = (database as unknown as { serialize(): Uint8Array }).serialize(); writeFileSync(immutablePath, immutableBytes); - const unsigned = { schema: MONITOR_SQLITE_MIGRATION_SCHEMA, createdAt: new Date().toISOString(), source: { ...source, path }, fingerprint: `sha256:${sha256(immutableBytes)}`, integrity, byteLength: immutableBytes.byteLength, tables, artifactLocator: { path: immutablePath, sha256: `sha256:${sha256(immutableBytes)}`, sizeBytes: immutableBytes.byteLength, kind: "legacy-sqlite" } }; + const artifactManifests = freezeArtifactManifests({ source: { ...source, path }, tables }); + const unsigned = { schema: MONITOR_SQLITE_MIGRATION_SCHEMA, createdAt: new Date().toISOString(), source: { ...source, path }, fingerprint: `sha256:${sha256(immutableBytes)}`, integrity, byteLength: immutableBytes.byteLength, tables, artifactManifests, artifactLocator: { path: immutablePath, sha256: `sha256:${sha256(immutableBytes)}`, sizeBytes: immutableBytes.byteLength, kind: "legacy-sqlite" } }; const snapshot: MonitorSqliteSnapshot = { ...unsigned, manifestContentHash: monitorSnapshotContentHash(unsigned) }; writeSnapshot(outputPath, `${monitorCentralStableJson(snapshot)}\n`); database.exec("COMMIT"); @@ -93,6 +97,7 @@ export function verifyMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot): Re export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise> { const verified = verifyMonitorSqliteSnapshot(snapshot); if (verified.ok !== true) throw Object.assign(new Error("snapshot verification failed"), { code: "monitor-migration-snapshot-invalid", verification: verified }); + verifyFrozenArtifactManifests(snapshot); const ingests = canonicalIngests(snapshot); const runtimeMetadata = canonicalRuntimeMetadata(snapshot); const manifest = { manifestId: snapshot.manifestContentHash, sourceFingerprint: snapshot.fingerprint, rowCount: snapshot.tables.reduce((total, table) => total + table.rowCount, 0) }; @@ -112,27 +117,23 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType< const findings = tableRows(snapshot.tables, ["findings", "finding"]); const metadata = tableRows(snapshot.tables, ["metadata", "run_metadata"]); const payloads = tableRows(snapshot.tables, ["payloads", "run_payloads"]); - const locators = tableRows(snapshot.tables, ["artifact_locators", "locators"]); return rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); const rowFindings = findings.filter((item) => stringField(item, ["run_id"]) === runId).map(normalizeLegacyFinding); const report = metadataValue(metadata, `run.report.${runId}`); - const rowLocators = locators.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); const storedPayload = jsonField(payloads.find((item) => stringField(item, ["run_id"]) === runId), ["payload", "value_json"]); const payload = { ...(storedPayload ?? {}), ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } }; const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : []; const canonicalFindings = dedupeFindings(reportFindings, rowFindings); - const artifactLocators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); - const artifactCount = numberField(row, ["artifact_count", "artifactCount"]); - if (artifactCount !== null && artifactCount !== artifactLocators.length) throw Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch", diagnostic: { runId, stateDir: stringField(row, ["state_dir", "stateDir"]), expectedCount: artifactCount, observedCount: artifactLocators.length, valuesRedacted: true } }); - verifyReportLocator(row, artifactLocators); + const artifactLocators = snapshot.artifactManifests.find((item) => item.runId === runId)?.locators; + if (artifactLocators === undefined) throw Object.assign(new Error(`snapshot artifact manifest is missing ${runId}`), { code: "monitor-migration-artifact-manifest-missing" }); return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators, timeline: [] }); }); } function targetReconciliation(ingests: readonly ReturnType[], runtimeMetadata: readonly MonitorRuntimeMetadata[]): Record { return { runs: ingests.length, findings: ingests.reduce((total, item) => total + item.findings.length, 0), payloads: ingests.length, artifactLocators: ingests.reduce((total, item) => total + item.artifactLocators.length, 0), views: ingests.reduce((total, item) => total + (isRecord(item.payload.views) ? Object.keys(item.payload.views).length : 0), 0), runtimeMetadata: runtimeMetadata.length }; } function canonicalRuntimeMetadata(snapshot: MonitorSqliteSnapshot): readonly MonitorRuntimeMetadata[] { return tableRows(snapshot.tables, ["metadata", "run_metadata"]).map((row) => ({ sentinelId: snapshot.source.sentinelId, node: snapshot.source.node, lane: snapshot.source.lane, key: stringField(row, ["key"]), value: jsonValue(row, ["value_json", "value"]), updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt })).filter((item) => item.key.length > 0).sort((left, right) => left.key.localeCompare(right.key)); } function sameRun(value: unknown, expected: { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly runId: string; readonly updatedAt: string }): boolean { return isRecord(value) && value.sentinelId === expected.sentinelId && value.node === expected.node && value.lane === expected.lane && value.runId === expected.runId && value.updatedAt === expected.updatedAt; } -function locatorFromRow(row: Record, snapshot: MonitorSqliteSnapshot): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const expectedHash = stringField(row, ["sha256"]); const expectedSize = numberField(row, ["size_bytes", "sizeBytes"]); const stateDir = stringField(row, ["state_dir", "stateDir"]); if (!ownerPvc || !stateDir || !relativePath || !expectedHash || expectedSize === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); const source = artifactSourceFor(snapshot.source, ownerPvc); const path = artifactPath(source.mountPath, stateDir, relativePath); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash || bytes.byteLength !== expectedSize) throw Object.assign(new Error("legacy artifact locator does not match source artifact"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || snapshot.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || snapshot.source.lane, ownerPvc, stateDir, relativePath, sha256: actualHash, sizeBytes: bytes.byteLength, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } +function locatorFromRow(row: Record, snapshot: Pick): MonitorArtifactLocator { const ownerPvc = stringField(row, ["owner_pvc", "ownerPvc"]); const relativePath = stringField(row, ["relative_path", "relativePath"]); const expectedHash = stringField(row, ["sha256"]); const expectedSize = numberField(row, ["size_bytes", "sizeBytes"]); const stateDir = stringField(row, ["state_dir", "stateDir"]); if (!ownerPvc || !stateDir || !relativePath || !expectedHash || expectedSize === null) throw Object.assign(new Error("legacy artifact locator is incomplete"), { code: "monitor-migration-locator-invalid" }); const source = artifactSourceFor(snapshot.source, ownerPvc); const path = artifactPath(source.mountPath, stateDir, relativePath); const bytes = readFileSync(path); const actualHash = `sha256:${sha256(bytes)}`; if (actualHash !== expectedHash || bytes.byteLength !== expectedSize) throw Object.assign(new Error("legacy artifact locator does not match source artifact"), { code: "monitor-migration-locator-invalid" }); return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || snapshot.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || snapshot.source.lane, ownerPvc, stateDir, relativePath, sha256: actualHash, sizeBytes: bytes.byteLength, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; } function artifactLocatorsFromRun(row: Record, snapshot: MonitorSqliteSnapshot): readonly MonitorArtifactLocator[] { const stateDir = stringField(row, ["state_dir", "stateDir"]); if (!stateDir) return []; const source = artifactSourceForStateDir(snapshot.source, stateDir); const directory = artifactPath(source.mountPath, stateDir, ""); if (!existsSync(directory) || !statSync(directory).isDirectory()) throw Object.assign(new Error("legacy artifact stateDir is unavailable"), { code: "monitor-migration-artifact-path-invalid" }); return artifactFiles(directory).map((path) => { const bytes = readFileSync(path); const relativePath = path.slice(directory.length + 1).replaceAll("\\", "/"); return { ownerNode: snapshot.source.node, ownerLane: snapshot.source.lane, ownerPvc: source.ownerPvc, stateDir, relativePath, sha256: `sha256:${sha256(bytes)}`, sizeBytes: bytes.byteLength, kind: relativePath === "analysis/report.json" ? "report-json" : "artifact", reachable: true }; }); } function verifyReportLocator(row: Record, locators: readonly MonitorArtifactLocator[]): void { const expectedHash = stringField(row, ["report_json_sha256", "reportJsonSha256"]); if (!expectedHash) return; const report = locators.find((locator) => locator.relativePath === "analysis/report.json"); if (!report || report.sha256 !== expectedHash) throw Object.assign(new Error("legacy report hash does not match artifact"), { code: "monitor-migration-report-locator-invalid" }); } function metadataValue(rows: readonly Record[], key: string): Record | null { return jsonField(rows.find((row) => stringField(row, ["key"]) === key), ["value_json", "value"]); } @@ -147,6 +148,8 @@ function artifactSourceFor(source: MonitorSqliteSource, ownerPvc: string): Monit function artifactSourceForStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { const directory = resolve(stateDir); const matches = source.artifactSources.filter((item) => directory === resolve(item.mountPath) || directory.startsWith(`${resolve(item.mountPath)}/`)).sort((left, right) => resolve(right.mountPath).length - resolve(left.mountPath).length); if (matches.length === 0) throw Object.assign(new Error(`legacy artifact stateDir is not registry-mounted: ${stateDir}`), { code: "monitor-migration-artifact-source-missing" }); return matches[0]!; } function artifactPath(mountPath: string, stateDir: string, relativePath: string): string { const root = resolve(mountPath); const directory = stateDir.startsWith("/") ? resolve(stateDir) : resolve(root, stateDir); if (directory !== root && !directory.startsWith(`${root}/`)) throw Object.assign(new Error("legacy artifact stateDir escapes registry mount"), { code: "monitor-migration-artifact-path-invalid" }); return join(directory, relativePath); } function artifactFiles(directory: string): readonly string[] { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); if (entry.isDirectory()) return artifactFiles(path); return entry.isFile() ? [path] : []; }).sort(); } +function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void { for (const manifest of snapshot.artifactManifests) for (const locator of manifest.locators) { const source = artifactSourceFor(snapshot.source, locator.ownerPvc); const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath); if (!existsSync(path) || !statSync(path).isFile()) throw Object.assign(new Error(`frozen artifact is unavailable for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: null, valuesRedacted: true } }); const bytes = readFileSync(path); if (`sha256:${sha256(bytes)}` !== locator.sha256 || bytes.byteLength !== locator.sizeBytes) throw Object.assign(new Error(`frozen artifact changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: manifest.locators.length, valuesRedacted: true } }); } } +function artifactCountMismatch(runId: string, stateDir: string, expectedCount: number, observedCount: number): Error { return Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch", diagnostic: { runId, stateDir, expectedCount, observedCount, valuesRedacted: true } }); } function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; } function normalizeRow(value: unknown): Record { const row = value as Record; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); } function jsonField(row: Record | undefined, fields: readonly string[]): Record | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 1546a0a2..3cfc2203 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -189,8 +189,11 @@ test("workspace artifact source verifies locator identity hash and size", async assert.equal(run?.artifactLocators.length, 5); assert.deepEqual(run?.artifactLocators.map((locator) => ({ ownerPvc: locator.ownerPvc, stateDir: locator.stateDir, relativePath: locator.relativePath, sha256: locator.sha256, sizeBytes: locator.sizeBytes })), artifacts.map(([relativePath, bytes]) => ({ ownerPvc: "legacy-web-observe-workspace-nc01", stateDir, relativePath, sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, sizeBytes: bytes.byteLength })).sort((left, right) => left.relativePath.localeCompare(right.relativePath))); assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true); + writeFileSync(join(stateDir, "artifacts", "request.txt"), "changed\n"); + await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid"); + writeFileSync(join(stateDir, "artifacts", "request.txt"), artifacts[2][1]); rmSync(join(stateDir, "artifacts", "response.txt")); - await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => { const migrationError = error as { code?: string; diagnostic?: Record }; return migrationError.code === "monitor-migration-artifact-count-mismatch" && migrationError.diagnostic?.runId === "run-1" && migrationError.diagnostic?.stateDir === stateDir && migrationError.diagnostic?.expectedCount === 5 && migrationError.diagnostic?.observedCount === 4; }); + await assert.rejects(() => verifyMonitorSqliteSnapshotAgainstStore(snapshot, store), (error: unknown) => { const migrationError = error as { code?: string; diagnostic?: Record }; return migrationError.code === "monitor-migration-frozen-artifact-invalid" && migrationError.diagnostic?.runId === "run-1" && migrationError.diagnostic?.stateDir === stateDir && migrationError.diagnostic?.expectedCount === 5; }); } finally { rmSync(directory, { recursive: true, force: true }); } @@ -206,8 +209,7 @@ test("artifact count without a registry-backed locator fails closed", async () = database.run("INSERT INTO runs VALUES (?, ?, ?)", ["run-1", 1, "2026-07-12T00:00:00.000Z"]); database.close(); const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] }; - const snapshot = exportMonitorSqliteSnapshot(source, snapshotPath); - await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-count-mismatch"); + assert.throws(() => exportMonitorSqliteSnapshot(source, snapshotPath), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-count-mismatch"); } finally { rmSync(directory, { recursive: true, force: true }); }