feat: add Release A monitor PG capability

config/hwlab-node-lanes.yaml

@@ -162,6 +162,8 @@ lanes:
         observability:
           prometheusOperator: false
           webProbe:
+            monitor:
+              configRef: config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml#monitor
             monitorRoot:
               enabled: true
               sentinelId: nc01-web-probe-sentinel

config/hwlab-web-probe-monitor/runtime.nc01-v03.yaml

@@ -0,0 +1,66 @@
+version: 1
+kind: HwlabWebProbeMonitorRuntime
+metadata:
+  id: hwlab-web-probe-monitor-nc01-v03
+  owner: UniDesk
+  specRef: PJ2026-01060508
+  relatedIssues:
+    - 1868
+    - 1873
+monitor:
+  runtime:
+    node: NC01
+    lane: v03
+    namespace: hwlab-v03
+    deploymentName: hwlab-web-probe-monitor
+    serviceName: hwlab-web-probe-monitor
+    servicePort: 8090
+    replicas: 1
+    imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel:p3-service
+    healthPath: /api/health
+    databaseSecret:
+      name: hwlab-web-probe-monitor-db
+      key: DATABASE_URL
+      sourceRef: hwlab/web-probe-monitor-db.env
+    timeout:
+      requestSeconds: 15
+      startupSeconds: 30
+    pool:
+      max: 4
+      idleTimeoutSeconds: 30
+    pageSize: 100
+  persistence:
+    mode: host-postgres
+    authority: prepared-central-only
+    database: web_probe_monitor
+    schema: monitor
+  migration:
+    mode: legacy-authority
+    source:
+      registry: config/hwlab-node-lanes.yaml#lanes.v03.targets.NC01.observability.webProbe.sentinels
+      sentinelId: nc01-web-probe-sentinel
+      runtimeConfigRef: config/hwlab-web-probe-sentinel/profiles.yaml#nodes.${NODE}.sentinels.${nodeLower}-web-probe-sentinel.sentinel.configRefs.runtime
+      sqlitePath: /var/lib/web-probe-sentinel-nc01/index.sqlite
+      pvcName: hwlab-web-probe-sentinel-nc01-state
+      stateRoot: /var/lib/web-probe-sentinel-nc01
+    job:
+      name: hwlab-web-probe-monitor-migration
+      command: plan
+      readOnly: true
+  publicExposure:
+    enabled: false
+    mode: prepared-not-routed
+    hostname: monitor.pikapython.com
+  cicd:
+    consumer: sentinel-nc01-v03
+    mode: existing-pac-consumer
+  capabilities:
+    boundedReadModel: true
+    health: true
+    schema: true
+    schedulerDue: true
+    maintenance: true
+    runnerAvailability: true
+    runnerTriggerFacts: true
+    schedulerAuthority: legacy-sqlite
+    clientQueryAuthority: legacy-sqlite

config/secrets-distribution.yaml

@@ -109,6 +109,14 @@ targets:
     enabled: true
 
 kubernetesSecrets:
+  - name: hwlab-web-probe-monitor-db
+    targetId: hwlab-nc01-v03
+    secretName: hwlab-web-probe-monitor-db
+    type: Opaque
+    data:
+      - sourceRef: hwlab/web-probe-monitor-db.env
+        sourceKey: DATABASE_URL
+        targetKey: DATABASE_URL
   - name: hwlab-nc01-v03-opencode-server-auth
     targetId: hwlab-nc01-v03
     secretName: hwlab-v03-opencode-server-auth

scripts/monitor-sqlite-migration-verify.ts

@@ -0,0 +1,18 @@
+import { readFileSync } from "node:fs";
+
+import { createPostgresMonitorCentralStore, structuredMonitorCentralError } from "./src/monitor-central-persistence";
+import { verifyMonitorSqliteSnapshotAgainstStore, type MonitorSqliteSnapshot } from "./src/monitor-sqlite-migration";
+
+function required(name: string): string { const value = process.env[name]; if (!value) throw Object.assign(new Error(`${name} is required`), { code: "monitor_migration_runtime_config_missing" }); return value; }
+function positive(name: string): number { const value = Number(required(name)); if (!Number.isSafeInteger(value) || value < 1) throw Object.assign(new Error(`${name} must be a positive integer`), { code: "monitor_migration_runtime_config_invalid" }); return value; }
+
+let store: ReturnType<typeof createPostgresMonitorCentralStore> | null = null;
+try {
+  const snapshotPath = process.argv[2];
+  if (!snapshotPath) throw Object.assign(new Error("snapshot path is required"), { code: "monitor_migration_snapshot_missing" });
+  store = createPostgresMonitorCentralStore(required("DATABASE_URL"), { poolMax: positive("MONITOR_CENTRAL_PG_POOL_MAX"), idleTimeoutSeconds: positive("MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS") });
+  console.log(JSON.stringify(await verifyMonitorSqliteSnapshotAgainstStore(JSON.parse(readFileSync(snapshotPath, "utf8")) as MonitorSqliteSnapshot, store)));
+} catch (error) {
+  console.log(JSON.stringify({ ok: false, error: structuredMonitorCentralError(error), valuesRedacted: true }));
+  process.exitCode = 2;
+} finally { await store?.close(); }

scripts/src/hwlab-node-lanes.ts

@@ -336,9 +336,14 @@ export interface HwlabRuntimeObservabilitySpec {
 export interface HwlabRuntimeObservabilityWebProbeSpec {
   readonly sentinel?: HwlabRuntimeWebProbeSentinelSpec;
   readonly sentinels?: readonly HwlabRuntimeWebProbeSentinelRegistryItemSpec[];
+  readonly monitor?: HwlabRuntimeWebProbeMonitorSpec;
   readonly monitorRoot?: HwlabRuntimeWebProbeMonitorRootSpec;
 }
 
+export interface HwlabRuntimeWebProbeMonitorSpec {
+  readonly configRef: string;
+}
+
 export interface HwlabRuntimeObservabilityMetricsEndpointSpec {
   readonly serviceName: string;
   readonly containerName: string;
@@ -1867,13 +1872,14 @@ function observabilityConfig(value: unknown, path: string): HwlabRuntimeObservab
 function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntimeObservabilityWebProbeSpec | undefined {
   if (value === undefined) return undefined;
   const raw = asRecord(value, path);
-  const allowed = new Set(["sentinel", "sentinels", "monitorRoot"]);
+  const allowed = new Set(["sentinel", "sentinels", "monitor", "monitorRoot"]);
   for (const key of Object.keys(raw)) {
-    if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; observability.webProbe currently only owns sentinel/sentinels/monitorRoot`);
+    if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; observability.webProbe currently only owns sentinel/sentinels/monitor/monitorRoot`);
   }
   if (raw.sentinel !== undefined && raw.sentinels !== undefined) throw new Error(`${path} may declare sentinel or sentinels, not both`);
   const sentinel = raw.sentinel === undefined ? undefined : webProbeSentinelConfig(raw.sentinel, `${path}.sentinel`);
   const sentinels = raw.sentinels === undefined ? undefined : webProbeSentinelRegistryConfig(raw.sentinels, `${path}.sentinels`);
+  const monitor = raw.monitor === undefined ? undefined : webProbeMonitorConfig(raw.monitor, `${path}.monitor`);
   const monitorRoot = raw.monitorRoot === undefined ? undefined : webProbeMonitorRootConfig(raw.monitorRoot, `${path}.monitorRoot`);
   if (monitorRoot !== undefined) {
     if (sentinels !== undefined && !sentinels.some((item) => item.id === monitorRoot.sentinelId)) {
@@ -1886,10 +1892,20 @@ function observabilityWebProbeConfig(value: unknown, path: string): HwlabRuntime
   return {
     ...(sentinel === undefined ? {} : { sentinel }),
     ...(sentinels === undefined ? {} : { sentinels }),
+    ...(monitor === undefined ? {} : { monitor }),
     ...(monitorRoot === undefined ? {} : { monitorRoot }),
   };
 }
 
+function webProbeMonitorConfig(value: unknown, path: string): HwlabRuntimeWebProbeMonitorSpec {
+  const raw = asRecord(value, path);
+  const allowed = new Set(["configRef"]);
+  for (const key of Object.keys(raw)) if (!allowed.has(key)) throw new Error(`${path}.${key} is not allowed; monitor may only contain configRef`);
+  const configRef = stringField(raw, "configRef", path);
+  if (!configRef.startsWith("config/") || !configRef.includes("#")) throw new Error(`${path}.configRef must be a config YAML selector`);
+  return { configRef };
+}
+
 function webProbeMonitorRootConfig(value: unknown, path: string): HwlabRuntimeWebProbeMonitorRootSpec {
   const raw = asRecord(value, path);
   const allowed = new Set(["enabled", "sentinelId", "publicBaseUrl", "routePrefix", "caddyManagedBlockOwner"]);

scripts/src/hwlab-node-web-probe-monitor.test.ts

@@ -0,0 +1,16 @@
+import { expect, test } from "bun:test";
+
+import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
+import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor";
+
+test("Release A central monitor renders internal service, deployment and read-only PVC migration job", () => {
+  const manifests = releaseAMonitorManifests(hwlabRuntimeLaneSpecForNode("v03", "NC01"));
+  const service = manifests.find((item) => item.kind === "Service") as any;
+  const deployment = manifests.find((item) => item.kind === "Deployment") as any;
+  const job = manifests.find((item) => item.kind === "Job") as any;
+  expect(service.spec.type).toBe("ClusterIP");
+  expect(deployment.spec.template.spec.containers[0].env).toEqual(expect.arrayContaining([expect.objectContaining({ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: "hwlab-web-probe-monitor-db", key: "DATABASE_URL" } } })]));
+  expect(job.spec.template.spec.containers[0].args).toContain("plan");
+  expect(job.spec.template.spec.containers[0].volumeMounts).toEqual([{ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }]);
+  expect(job.spec.template.spec.volumes).toEqual([{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: "hwlab-web-probe-sentinel-nc01-state", readOnly: true } }]);
+});

scripts/src/hwlab-node-web-probe-monitor.ts

@@ -0,0 +1,35 @@
+// SPEC: PJ2026-01060508 Web哨兵 Release A.
+// Responsibility: YAML-first PG-capable central Monitor manifests while SQLite remains authoritative.
+import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref";
+import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
+
+export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec): readonly Record<string, unknown>[] {
+  const configRef = spec.observability.webProbe?.monitor?.configRef;
+  if (!configRef) return [];
+  const resolved = readWebProbeSentinelConfigRef(spec, configRef);
+  if (!isRecord(resolved.target)) throw new Error(`${resolved.ref} must resolve to an object`);
+  const runtime = recordAt(resolved.target, "runtime", resolved.ref);
+  const migration = recordAt(resolved.target, "migration", resolved.ref);
+  const source = recordAt(migration, "source", resolved.ref);
+  const job = recordAt(migration, "job", resolved.ref);
+  if (textAt(migration, "mode", resolved.ref) !== "legacy-authority") throw new Error(`${resolved.ref}.migration.mode must be legacy-authority`);
+  if (textAt(source, "sentinelId", resolved.ref) !== "nc01-web-probe-sentinel" || textAt(source, "sqlitePath", resolved.ref) !== "/var/lib/web-probe-sentinel-nc01/index.sqlite") throw new Error(`${resolved.ref}.migration.source must declare the resolved NC01/v03 source`);
+  if (textAt(job, "command", resolved.ref) !== "plan" || job.readOnly !== true) throw new Error(`${resolved.ref}.migration.job must be a read-only plan`);
+  const namespace = textAt(runtime, "namespace", resolved.ref);
+  const deploymentName = textAt(runtime, "deploymentName", resolved.ref);
+  const port = integerAt(runtime, "servicePort", resolved.ref);
+  const secret = recordAt(runtime, "databaseSecret", resolved.ref);
+  const pool = recordAt(runtime, "pool", resolved.ref);
+  const labels = { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" };
+  const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_HOST", value: "0.0.0.0" }, { name: "MONITOR_CENTRAL_PORT", value: String(port) }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }, { name: "MONITOR_CENTRAL_PAGE_SIZE", value: String(integerAt(runtime, "pageSize", resolved.ref)) }];
+  return [
+    { apiVersion: "v1", kind: "Service", metadata: { name: textAt(runtime, "serviceName", resolved.ref), namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port, targetPort: "http" }] } },
+    { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: deploymentName, namespace, labels }, spec: { replicas: integerAt(runtime, "replicas", resolved.ref), selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } }, template: { metadata: { labels }, spec: { containers: [{ name: "monitor", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/monitor-central-service.ts"], env, ports: [{ name: "http", containerPort: port }], readinessProbe: { httpGet: { path: textAt(runtime, "healthPath", resolved.ref), port: "http" } } }] } } } },
+    { apiVersion: "batch/v1", kind: "Job", metadata: { name: textAt(job, "name", resolved.ref), namespace, labels }, spec: { backoffLimit: 0, template: { metadata: { labels }, spec: { restartPolicy: "Never", containers: [{ name: "migration", image: textAt(runtime, "imageRef", resolved.ref), command: ["bun"], args: ["scripts/cli.ts", "web-probe", "sentinel", "migration", "plan", "--node", spec.nodeId, "--lane", spec.lane, "--sentinel", textAt(source, "sentinelId", resolved.ref), "--json"], env: [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }], volumeMounts: [{ name: "legacy-sqlite", mountPath: textAt(source, "stateRoot", resolved.ref), readOnly: true }] }], volumes: [{ name: "legacy-sqlite", persistentVolumeClaim: { claimName: textAt(source, "pvcName", resolved.ref), readOnly: true } }] } } } },
+  ];
+}
+
+function recordAt(value: Record<string, unknown>, key: string, path: string): Record<string, unknown> { if (!isRecord(value[key])) throw new Error(`${path}.${key} must be an object`); return value[key] as Record<string, unknown>; }
+function textAt(value: Record<string, unknown>, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; }
+function integerAt(value: Record<string, unknown>, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; }
+function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }

scripts/src/hwlab-node-web-sentinel-cicd.ts

@@ -37,6 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr
 import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
 import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
 import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration";
+import { releaseAMonitorManifests } from "./hwlab-node-web-probe-monitor";
 import {
   arrayAt,
   arrayAtNullable,
@@ -221,7 +222,10 @@ function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract<WebPr
     const projection = parseJsonObject(result.stdout) ?? { ok: false, command, error: "monitor-migration-import-unparseable", stderr: short(result.stderr), valuesRedacted: true };
     return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection);
   }
-  return rendered(verification.ok === true, command, renderMigrationCompact(verification), verification);
+  if (!process.env.DATABASE_URL) throw new Error("monitor migration verify requires DATABASE_URL projected from hwlab-web-probe-monitor-db/DATABASE_URL");
+  const result = runCommand(["bun", "scripts/monitor-sqlite-migration-verify.ts", options.snapshotPath], repoRoot, { timeoutMs: 120_000, env: { ...process.env, MONITOR_CENTRAL_PG_POOL_MAX: "4", MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS: "30" } });
+  const projection = parseJsonObject(result.stdout) ?? { ok: false, command, error: "monitor-migration-verify-unparseable", stderr: short(result.stderr), valuesRedacted: true };
+  return rendered(projection.ok === true, command, renderMigrationCompact(projection), projection);
 }
 
 function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string | null): readonly MonitorSqliteSource[] {
@@ -1298,6 +1302,7 @@ function renderSentinelManifests(
       spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
     },
     ...(cadenceJob === null ? [] : [cadenceJob]),
+    ...releaseAMonitorManifests(spec),
     {
       apiVersion: "apps/v1",
       kind: "Deployment",

scripts/src/monitor-sqlite-migration.ts

@@ -19,6 +19,36 @@ export interface MonitorSqliteSnapshot {
 }
 export interface MonitorMigrationReconciliation { readonly runs: number; readonly findings: number; readonly metadata: number; readonly payloads: number; readonly locators: number; readonly latest: number; }
 
+export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise<Record<string, unknown>> {
+  const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot);
+  const expected = canonicalIngests(snapshot);
+  const checks: Record<string, unknown>[] = [];
+  for (const input of expected) {
+    const scope = { sentinelId: input.sentinelId, node: input.node, lane: input.lane };
+    const actual = await store.run(scope, input.runId);
+    checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true });
+    checks.push({ name: "finding", runId: input.runId, ok: monitorCentralStableJson(actual?.findings) === monitorCentralStableJson(input.findings), valuesRedacted: true });
+    checks.push({ name: "metadata", runId: input.runId, ok: monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true });
+    checks.push({ name: "artifactLocator", runId: input.runId, ok: monitorCentralStableJson(actual?.artifactLocators) === monitorCentralStableJson(input.artifactLocators), valuesRedacted: true });
+    const views = isRecord(input.payload.views) ? input.payload.views : {};
+    for (const [name, value] of Object.entries(views)) {
+      const actualView = await store.view(scope, input.runId, name);
+      checks.push({ name: "view", runId: input.runId, view: name, ok: monitorCentralStableJson(actualView?.value) === monitorCentralStableJson(value), valuesRedacted: true });
+    }
+  }
+  const latestFor = (items: readonly typeof expected) => [...items].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.runId.localeCompare(left.runId))[0] ?? null;
+  const globalLatest = latestFor(expected);
+  const global = await store.overview({});
+  checks.push({ name: "globalLatest", ok: globalLatest === null ? global.latest === null : isRecord(global.latest) && global.latest.runId === globalLatest.runId && global.latest.updatedAt === globalLatest.updatedAt, valuesRedacted: true });
+  const scopes = [...new Map(expected.map((item) => [`${item.sentinelId}/${item.node}/${item.lane}`, { sentinelId: item.sentinelId, node: item.node, lane: item.lane }])).values()];
+  for (const scope of scopes) {
+    const latest = latestFor(expected.filter((item) => item.sentinelId === scope.sentinelId && item.node === scope.node && item.lane === scope.lane));
+    const overview = await store.overview(scope);
+    checks.push({ name: "scopedLatest", scope, ok: latest === null ? overview.latest === null : isRecord(overview.latest) && overview.latest.runId === latest.runId && overview.latest.updatedAt === latest.updatedAt, valuesRedacted: true });
+  }
+  return { ok: snapshotVerification.ok === true && checks.every((item) => item.ok === true), command: "web-probe sentinel migration verify", source: "resolved-active-registry", snapshot: snapshotVerification, reconciliation: snapshotVerification.reconciliation, checks, valuesRedacted: true };
+}
+
 export function planMonitorSqliteSources(sources: readonly MonitorSqliteSource[]): Record<string, unknown> {
   return { ok: true, command: "web-probe sentinel migration plan", sources: sources.map(sourcePlan), sourceCount: sources.length, valuesRedacted: true };
 }

scripts/src/monitor-terminal-ingest-migration.test.ts

@@ -7,7 +7,7 @@ import test from "node:test";
 import { Database } from "bun:sqlite";
 
 import { createInMemoryMonitorCentralStore } from "./monitor-central-persistence";
-import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot } from "./monitor-sqlite-migration";
+import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration";
 import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter, submitMonitorTerminalIngest } from "./monitor-terminal-ingest";
 import { compactWebObserveAnalyzeAnalysisForRaw } from "./hwlab-node/web-probe-observe";
 
@@ -78,6 +78,9 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio
     const reportFindings = importedRun?.findings.filter((finding) => finding.id === "report-finding") ?? [];
     assert.deepEqual(reportFindings, [{ id: "report-finding", rootCause: "complete metadata detail" }]);
     assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary");
+    const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store);
+    assert.equal(pgVerification.ok, true);
+    assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalLatest", "metadata", "run", "scopedLatest", "view"]);
     assert.equal(verifyMonitorSqliteSnapshot({ ...snapshot, manifestContentHash: "sha256:tampered" }).ok, false);
   } finally {
     rmSync(directory, { recursive: true, force: true });