diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 7f3aea15..cefbab01 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -162,6 +162,8 @@ lanes: observability: prometheusOperator: false webProbe: + monitor: + configRef: config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml#monitor monitorRoot: enabled: true sentinelId: nc01-web-probe-sentinel diff --git a/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml b/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml new file mode 100644 index 00000000..6b75707d --- /dev/null +++ b/config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml @@ -0,0 +1,66 @@ +version: 1 +kind: HwlabWebProbeMonitorRuntime +metadata: + id: hwlab-web-probe-monitor-nc01-v03 + owner: UniDesk + specRef: PJ2026-01060508 + relatedIssues: + - 1868 + - 1873 +monitor: + runtime: + node: NC01 + lane: v03 + namespace: hwlab-v03 + deploymentName: hwlab-web-probe-monitor + serviceName: hwlab-web-probe-monitor + servicePort: 8090 + replicas: 1 + imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel:p3-service + healthPath: /api/health + databaseSecret: + name: hwlab-web-probe-monitor-db + key: DATABASE_URL + sourceRef: hwlab/web-probe-monitor-db.env + timeout: + requestSeconds: 15 + startupSeconds: 30 + pool: + max: 4 + idleTimeoutSeconds: 30 + pageSize: 100 + persistence: + mode: host-postgres + authority: prepared-central-only + database: web_probe_monitor + schema: monitor + migration: + mode: legacy-authority + source: + registry: config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01.observability.webProbe.sentinels + sentinelId: nc01-web-probe-sentinel + runtimeConfigRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.sentinel.configRefs.runtime + sqlitePath: /var/lib/web-probe-sentinel-nc01/index.sqlite + pvcName: hwlab-web-probe-sentinel-nc01-state + stateRoot: /var/lib/web-probe-sentinel-nc01 + job: + name: hwlab-web-probe-monitor-migration + command: plan + readOnly: true + publicExposure: + enabled: false + mode: prepared-not-routed + hostname: monitor.pikapython.com + cicd: + consumer: sentinel-nc01-v03 + mode: existing-pac-consumer + capabilities: + boundedReadModel: true + health: true + schema: true + schedulerDue: true + maintenance: true + runnerAvailability: true + runnerTriggerFacts: true + schedulerAuthority: legacy-sqlite + clientQueryAuthority: legacy-sqlite diff --git a/config/secrets-distribution.yaml b/config/secrets-distribution.yaml index ff82b27f..837d825d 100644 --- a/config/secrets-distribution.yaml +++ b/config/secrets-distribution.yaml @@ -109,6 +109,14 @@ targets: enabled: true kubernetesSecrets: + - name: hwlab-web-probe-monitor-db + targetId: hwlab-nc01-v03 + secretName: hwlab-web-probe-monitor-db + type: Opaque + data: + - sourceRef: hwlab/web-probe-monitor-db.env + sourceKey: DATABASE_URL + targetKey: DATABASE_URL - name: hwlab-nc01-v03-opencode-server-auth targetId: hwlab-nc01-v03 secretName: hwlab-v03-opencode-server-auth diff --git a/scripts/monitor-sqlite-migration-verify.ts b/scripts/monitor-sqlite-migration-verify.ts new file mode 100644 index 00000000..13fab513 --- /dev/null +++ b/scripts/monitor-sqlite-migration-verify.ts @@ -0,0 +1,18 @@ +import { readFileSync } from "node:fs"; + +import { createPostgresMonitorCentralStore, structuredMonitorCentralError } from "./src/monitor-central-persistence"; +import { verifyMonitorSqliteSnapshotAgainstStore, type MonitorSqliteSnapshot } from "./src/monitor-sqlite-migration"; + +function required(name: string): string { const value = process.env[name]; if (!value) throw Object.assign(new Error(`${name} is required`), { code: "monitor_migration_runtime_config_missing" }); return value; } +function positive(name: string): number { const value = Number(required(name)); if (!Number.isSafeInteger(value) || value < 1) throw Object.assign(new Error(`${name} must be a positive integer`), { code: "monitor_migration_runtime_config_invalid" }); return value; } + +let store: ReturnType | null = null; +try { + const snapshotPath = process.argv[2]; + if (!snapshotPath) throw Object.assign(new Error("snapshot path is required"), { code: "monitor_migration_snapshot_missing" }); + store = createPostgresMonitorCentralStore(required("DATABASE_URL"), { poolMax: positive("MONITOR_CENTRAL_PG_POOL_MAX"), idleTimeoutSeconds: positive("MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS") }); + console.log(JSON.stringify(await verifyMonitorSqliteSnapshotAgainstStore(JSON.parse(readFileSync(snapshotPath, "utf8")) as MonitorSqliteSnapshot, store))); +} catch (error) { + console.log(JSON.stringify({ ok: false, error: structuredMonitorCentralError(error), valuesRedacted: true })); + process.exitCode = 2; +} finally { await store?.close(); } diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 8f02280b..484fa843 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -336,9 +336,14 @@ export interface HwlabRuntimeObservabilitySpec { export interface HwlabRuntimeObservabilityWebProbeSpec { readonly sentinel?: HwlabRuntimeWebProbeSentinelSpec; readonly sentinels?: readonly HwlabRuntimeWebProbeSentinelRegistryItemSpec[]; + readonly monitor?: HwlabRuntimeWebProbeMonitorSpec; readonly monitorRoot?: HwlabRuntimeWebProbeMonitorRootSpec; } +export interface HwlabRuntimeWebProbeMonitorSpec { + readonly configRef: string; +} + export interface HwlabRuntimeObservabilityMetricsEndpointSpec { readonly serviceName: string; readonly containerName: string; @@ -1867,13 +1872,14 @@ function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservab function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntimeObservabilityWebProbeSpec | undefined { if (value === undefined) return undefined; const raw = asRecord(value, path); - const allowed = new Set(["sentinel", "sentinels", "monitorRoot"]); + const allowed = new Set(["sentinel", "sentinels", "monitor", "monitorRoot"]); for (const key of Object.keys(raw)) { - if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; observability.webProbe currently only owns sentinel/sentinels/monitorRoot`); + if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; observability.webProbe currently only owns sentinel/sentinels/monitor/monitorRoot`); } if (raw.sentinel !== undefined && raw.sentinels !== undefined) throw new Error(`${path} may declare sentinel or sentinels, not both`); const sentinel = raw.sentinel === undefined ? undefined : webProbeSentinelConfig(raw.sentinel, `${path}.sentinel`); const sentinels = raw.sentinels === undefined ? undefined : webProbeSentinelRegistryConfig(raw.sentinels, `${path}.sentinels`); + const monitor = raw.monitor === undefined ? undefined : webProbeMonitorConfig(raw.monitor, `${path}.monitor`); const monitorRoot = raw.monitorRoot === undefined ? undefined : webProbeMonitorRootConfig(raw.monitorRoot, `${path}.monitorRoot`); if (monitorRoot !== undefined) { if (sentinels !== undefined && !sentinels.some((item) => item.id === monitorRoot.sentinelId)) { @@ -1886,10 +1892,20 @@ function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntime return { ...(sentinel === undefined ? {} : { sentinel }), ...(sentinels === undefined ? {} : { sentinels }), + ...(monitor === undefined ? {} : { monitor }), ...(monitorRoot === undefined ? {} : { monitorRoot }), }; } +function webProbeMonitorConfig(value: unknown, path: string): HwlabRuntimeWebProbeMonitorSpec { + const raw = asRecord(value, path); + const allowed = new Set(["configRef"]); + for (const key of Object.keys(raw)) if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; monitor may only contain configRef`); + const configRef = stringField(raw, "configRef", path); + if (!configRef.startsWith("config/") || !configRef.includes("#")) throw new Error(`${path}.configRef must be a config YAML selector`); + return { configRef }; +} + function webProbeMonitorRootConfig(value: unknown, path: string): HwlabRuntimeWebProbeMonitorRootSpec { const raw = asRecord(value, path); const allowed = new Set(["enabled", "sentinelId", "publicBaseUrl", "routePrefix", "caddyManagedBlockOwner"]); diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts new file mode 100644 index 00000000..6882e143 --- /dev/null +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -0,0 +1,16 @@ +import { expect, test } from "bun:test"; + +import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes"; +import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor"; + +test("Release A central monitor renders internal service, deployment and read-only PVC migration job", () => { + const manifests = releaseAMonitorManifests(hwlabRuntimeLaneSpecForNode("v03", "NC01")); + const service = manifests.find((item) => item.kind === "Service") as any; + const deployment = manifests.find((item) => item.kind === "Deployment") as any; + const job = manifests.find((item) => item.kind === "Job") as any; + expect(service.spec.type).toBe("ClusterIP"); + expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } })])); + expect(job.spec.template.spec.containers[0].args).toContain("plan"); + expect(job.spec.template.spec.containers[0].volumeMounts).toEqual([{ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }]); + expect(job.spec.template.spec.volumes).toEqual([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }]); +}); diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts new file mode 100644 index 00000000..d476fbb6 --- /dev/null +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -0,0 +1,35 @@ +// SPEC: PJ2026-01060508 Web哨兵 Release A. +// Responsibility: YAML-first PG-capable central Monitor manifests while SQLite remains authoritative. +import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref"; +import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; + +export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec): readonly Record[] { + const configRef = spec.observability.webProbe?.monitor?.configRef; + if (!configRef) return []; + const resolved = readWebProbeSentinelConfigRef(spec, configRef); + if (!isRecord(resolved.target)) throw new Error(`${resolved.ref} must resolve to an object`); + const runtime = recordAt(resolved.target, "runtime", resolved.ref); + const migration = recordAt(resolved.target, "migration", resolved.ref); + const source = recordAt(migration, "source", resolved.ref); + const job = recordAt(migration, "job", resolved.ref); + if (textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref}.migration.mode must be legacy-authority`); + if (textAt(source, "sentinelId", resolved.ref) !== "nc01-web-probe-sentinel" || textAt(source, "sqlitePath", resolved.ref) !== "/var/lib/web-probe-sentinel-nc01/index.sqlite") throw new Error(`${resolved.ref}.migration.source must declare the resolved NC01/v03 source`); + if (textAt(job, "command", resolved.ref) !== "plan" || job.readOnly !== true) throw new Error(`${resolved.ref}.migration.job must be a read-only plan`); + const namespace = textAt(runtime, "namespace", resolved.ref); + const deploymentName = textAt(runtime, "deploymentName", resolved.ref); + const port = integerAt(runtime, "servicePort", resolved.ref); + const secret = recordAt(runtime, "databaseSecret", resolved.ref); + const pool = recordAt(runtime, "pool", resolved.ref); + const labels = { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" }; + const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }]; + return [ + { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port, targetPort: "http" }] } }, + { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels }, spec: { replicas: integerAt(runtime, "replicas", resolved.ref), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { metadata: { labels }, spec: { containers: [{ name: "monitor", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/monitor-central-service.ts"], env, ports: [{ name: "http", containerPort: port }], readinessProbe: { httpGet: { path: textAt(runtime, "healthPath", resolved.ref), port: "http" } } }] } } } }, + { apiVersion: "batch/v1", kind: "Job", metadata: { name: textAt(job, "name", resolved.ref), namespace, labels }, spec: { backoffLimit: 0, template: { metadata: { labels }, spec: { restartPolicy: "Never", containers: [{ name: "migration", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/cli.ts", "web-probe", "sentinel", "migration", "plan", "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", textAt(source, "sentinelId", resolved.ref), "--json"], env: [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }], volumeMounts: [{ name: "legacy-sqlite", mountPath: textAt(source, "stateRoot", resolved.ref), readOnly: true }] }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: textAt(source, "pvcName", resolved.ref), readOnly: true } }] } } } }, + ]; +} + +function recordAt(value: Record, key: string, path: string): Record { if (!isRecord(value[key])) throw new Error(`${path}.${key} must be an object`); return value[key] as Record; } +function textAt(value: Record, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; } +function integerAt(value: Record, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; } +function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 8b230d1c..0920ad57 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -37,6 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration"; +import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor"; import { arrayAt, arrayAtNullable, @@ -221,7 +222,10 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract> { + const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot); + const expected = canonicalIngests(snapshot); + const checks: Record[] = []; + for (const input of expected) { + const scope = { sentinelId: input.sentinelId, node: input.node, lane: input.lane }; + const actual = await store.run(scope, input.runId); + checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); + checks.push({ name: "finding", runId: input.runId, ok: monitorCentralStableJson(actual?.findings) === monitorCentralStableJson(input.findings), valuesRedacted: true }); + checks.push({ name: "metadata", runId: input.runId, ok: monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true }); + checks.push({ name: "artifactLocator", runId: input.runId, ok: monitorCentralStableJson(actual?.artifactLocators) === monitorCentralStableJson(input.artifactLocators), valuesRedacted: true }); + const views = isRecord(input.payload.views) ? input.payload.views : {}; + for (const [name, value] of Object.entries(views)) { + const actualView = await store.view(scope, input.runId, name); + checks.push({ name: "view", runId: input.runId, view: name, ok: monitorCentralStableJson(actualView?.value) === monitorCentralStableJson(value), valuesRedacted: true }); + } + } + const latestFor = (items: readonly typeof expected) => [...items].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.runId.localeCompare(left.runId))[0] ?? null; + const globalLatest = latestFor(expected); + const global = await store.overview({}); + checks.push({ name: "globalLatest", ok: globalLatest === null ? global.latest === null : isRecord(global.latest) && global.latest.runId === globalLatest.runId && global.latest.updatedAt === globalLatest.updatedAt, valuesRedacted: true }); + const scopes = [...new Map(expected.map((item) => [`${item.sentinelId}/${item.node}/${item.lane}`, { sentinelId: item.sentinelId, node: item.node, lane: item.lane }])).values()]; + for (const scope of scopes) { + const latest = latestFor(expected.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane)); + const overview = await store.overview(scope); + checks.push({ name: "scopedLatest", scope, ok: latest === null ? overview.latest === null : isRecord(overview.latest) && overview.latest.runId === latest.runId && overview.latest.updatedAt === latest.updatedAt, valuesRedacted: true }); + } + return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: snapshotVerification.reconciliation, checks, valuesRedacted: true }; +} + export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[]): Record { return { ok: true, command: "web-probe sentinel migration plan", sources: sources.map(sourcePlan), sourceCount: sources.length, valuesRedacted: true }; } diff --git a/scripts/src/monitor-terminal-ingest-migration.test.ts b/scripts/src/monitor-terminal-ingest-migration.test.ts index 4b31575f..fb427594 100644 --- a/scripts/src/monitor-terminal-ingest-migration.test.ts +++ b/scripts/src/monitor-terminal-ingest-migration.test.ts @@ -7,7 +7,7 @@ import test from "node:test"; import { Database } from "bun:sqlite"; import { createInMemoryMonitorCentralStore } from "./monitor-central-persistence"; -import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot } from "./monitor-sqlite-migration"; +import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration"; import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter, submitMonitorTerminalIngest } from "./monitor-terminal-ingest"; import { compactWebObserveAnalyzeAnalysisForRaw } from "./hwlab-node/web-probe-observe"; @@ -78,6 +78,9 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio const reportFindings = importedRun?.findings.filter((finding) => finding.id === "report-finding") ?? []; assert.deepEqual(reportFindings, [{ id: "report-finding", rootCause: "complete metadata detail" }]); assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary"); + const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store); + assert.equal(pgVerification.ok, true); + assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalLatest", "metadata", "run", "scopedLatest", "view"]); assert.equal(verifyMonitorSqliteSnapshot({ ...snapshot, manifestContentHash: "sha256:tampered" }).ok, false); } finally { rmSync(directory, { recursive: true, force: true });