Merge pull request #1882 from pikasTech/fix/web-sentinel-migration-reliability
修复 Web Sentinel 迁移缺口与失败可见性
This commit is contained in:
Lyon
@@ -38,6 +38,15 @@ monitor:
|
||||
workflow: export-import-verify
|
||||
readOnlySource: true
|
||||
enabled: false
|
||||
legacyArtifactGaps:
|
||||
- runId: sentinel-run-mrbjo83t-ef79c71d
|
||||
artifactCount: 1
|
||||
stateDir: .state/web-observe/NC01/v03/2026/07/08/20260708T035504Z_workbench_webobs-mrbjo88n-c5bed5
|
||||
reportSha256: sha256:119c9e3ef4cbc41143121ee5d1af8955a8454e5efdbb6600f64a207a0f148b33
|
||||
- runId: sentinel-run-mrbjsugk-764ff868
|
||||
artifactCount: 4
|
||||
stateDir: .state/web-observe/NC01/v03/2026/07/08/20260708T035839Z_workbench_webobs-mrbjsul8-5952f4
|
||||
reportSha256: sha256:d2dbc0fc98f39d7373b9ae009e31fcd8d666014c2e32428f235150ee59666048
|
||||
publicExposure:
|
||||
enabled: false
|
||||
mode: prepared-not-routed
|
||||
|
||||
@@ -22,7 +22,10 @@ import { hwlabNodeHelp } from "./hwlab-node-help";
|
||||
import { runHwlabNodeCommand } from "./hwlab-node/entry";
|
||||
import {
|
||||
guardedSentinelDeliveryAction,
|
||||
migrationJobFailed,
|
||||
migrationJobFailureSummary,
|
||||
runWebProbeSentinelCommand,
|
||||
sameMigrationSource,
|
||||
webProbeSentinelDeliveryAuthorityFromCatalog,
|
||||
} from "./hwlab-node-web-sentinel-cicd";
|
||||
import { parseNodeScopedDelegatedOptions } from "./hwlab-node/plan";
|
||||
@@ -294,6 +297,39 @@ describe("migrated CLI 执行 guard 与提示清理", () => {
|
||||
expect(execute.sentinel).toMatchObject({ kind: "migration", action: "job-run", confirm: true, wait: true });
|
||||
});
|
||||
|
||||
test("migration Job 失败输出容器终态、有界日志和 Secret 脱敏", () => {
|
||||
const failure = migrationJobFailureSummary({ podPhase: "Failed", containers: [{ name: "export", phase: "terminated", exitCode: 2, reason: "Error", message: "artifact source unavailable" }], logsTail: "DATABASE_URL=https://user:PW123@host/db Bearer token-value password=PW123" }, { ok: false, command: "export", error: { name: "Error", code: "monitor-migration-artifact-path-invalid", message: "cannot export https://user:PW123@host/db?token=abc", stack: "stack with password=PW123" } });
|
||||
expect(failure).toMatchObject({ phase: "Failed", container: "export", exitCode: 2, status: "Error", valuesRedacted: true });
|
||||
expect(String(failure.message)).toContain("cannot export");
|
||||
expect(JSON.stringify(failure)).not.toMatch(/PW123|token-value|abc|stack/iu);
|
||||
});
|
||||
|
||||
test("migration Job 先脱敏长 userinfo 再截断 nested message 和日志", () => {
|
||||
const credential = "LONG-USERINFO-CREDENTIAL-MARKER";
|
||||
const longUrl = `custom+ssh://u:${credential.repeat(12)}@host.example.test/path`;
|
||||
const failure = migrationJobFailureSummary({ podPhase: "Failed", logsTail: `${longUrl} logs` }, { ok: false, error: { message: `${longUrl} nested message` } });
|
||||
expect(String(failure.message)).toContain("<redacted-url>");
|
||||
expect(String(failure.logsTail)).toContain("<redacted-url>");
|
||||
expect(JSON.stringify(failure)).not.toContain(credential);
|
||||
});
|
||||
|
||||
test("migration Job 业务失败、创建失败和超时均只返回安全 failure 投影", () => {
|
||||
const business = migrationJobFailed("migration", { jobName: "job" }, "result-failed", { ok: false, error: { message: "https://user:PW123@host failed", stack: "secret stack" } }, { podPhase: "Succeeded", logsTail: "Bearer abc" }) as { projection?: Record<string, unknown> };
|
||||
const createFailed = migrationJobFailed("migration", { jobName: "job" }, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "DATABASE_URL=https://user:PW123@host/db" }) as { projection?: Record<string, unknown> };
|
||||
const timeout = migrationJobFailed("migration", { jobName: "job" }, "timeout", {}, { podPhase: "Timeout", logsTail: "password=PW123" }) as { projection?: Record<string, unknown> };
|
||||
for (const result of [business, createFailed, timeout]) {
|
||||
expect(result.projection).toMatchObject({ ok: false, valuesRedacted: true });
|
||||
expect(result.projection?.result).toBeUndefined();
|
||||
expect(JSON.stringify(result.projection)).not.toMatch(/PW123|abc|stack/iu);
|
||||
}
|
||||
});
|
||||
|
||||
test("migration source 仅接受按 runId 重排的相同 gap 集合", () => {
|
||||
const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: "/state/index.sqlite", runtimeConfigRef: "fixture#runtime", stateRoot: "/state", ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: "/state" }], legacyArtifactGaps: [{ runId: "b", artifactCount: 2, stateDir: ".state/b", reportSha256: `sha256:${"b".repeat(64)}` }, { runId: "a", artifactCount: 1, stateDir: ".state/a", reportSha256: `sha256:${"a".repeat(64)}` }] };
|
||||
expect(sameMigrationSource(source, { ...source, legacyArtifactGaps: [...source.legacyArtifactGaps].reverse() })).toBe(true);
|
||||
expect(sameMigrationSource(source, { ...source, legacyArtifactGaps: [{ ...source.legacyArtifactGaps[0], artifactCount: 3 }, source.legacyArtifactGaps[1]] })).toBe(false);
|
||||
});
|
||||
|
||||
test("PaC 外层事件与内层 deterministic publish 分开归类，只有外层可驱动 delivery", () => {
|
||||
const consumer = { repository: "sentinel-nc01-v03", pipelineRunPrefix: "hwlab-web-probe-sentinel-nc01-", pipeline: "hwlab-web-probe-sentinel-nc01" };
|
||||
const outer = {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { expect, test } from "bun:test";
|
||||
|
||||
import { rootPath } from "./config";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "./hwlab-node-lanes";
|
||||
import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
|
||||
import { releaseAMonitorManifests, releaseAMonitorMigrationArtifactGaps, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
|
||||
import { readYamlRecord } from "./platform-infra-ops-library";
|
||||
|
||||
const source = { sentinelId: "nc01-web-probe-sentinel", node: "NC01", lane: "v03", path: "/var/lib/web-probe-sentinel-nc01/index.sqlite", runtimeConfigRef: "config/hwlab-web-probe-sentinel/profiles.yaml#nodes.NC01.sentinels.nc01-web-probe-sentinel.runtime", stateRoot: "/var/lib/web-probe-sentinel-nc01", ownerPvc: "hwlab-web-probe-sentinel-nc01-state", artifactSources: [{ ownerPvc: "hwlab-web-probe-sentinel-nc01-state", mountPath: "/var/lib/web-probe-sentinel-nc01" }, { ownerPvc: "legacy-web-observe-workspace-nc01", mountPath: "/root/hwlab-v03", hostPath: "/root/hwlab-v03" }] };
|
||||
@@ -47,6 +47,14 @@ test("one-shot migration helper accepts a unique control-plane suffix", () => {
|
||||
expect(job.metadata.name).toContain("source-commit-run-42");
|
||||
});
|
||||
|
||||
test("Release A declares exactly the two approved pre-migration artifact gaps", () => {
|
||||
const config = readYamlRecord<Record<string, any>>(rootPath("config/hwlab-web-probe-monitor/runtime.yaml"), "HwlabWebProbeMonitorRuntime");
|
||||
const declared = config.monitor.migration.legacyArtifactGaps;
|
||||
const resolved = releaseAMonitorMigrationArtifactGaps(hwlabRuntimeLaneSpecForNode("v03", "NC01"), source);
|
||||
expect(resolved).toEqual([...declared].sort((left: { runId: string }, right: { runId: string }) => left.runId.localeCompare(right.runId)));
|
||||
expect(resolved.every((gap) => gap.artifactCount > 0 && gap.stateDir.startsWith(".") && /^sha256:[a-f0-9]{64}$/u.test(gap.reportSha256))).toBe(true);
|
||||
});
|
||||
|
||||
test("Release A declares the controlled NC01 monitor database Secret projection", () => {
|
||||
const config = readYamlRecord<Record<string, any>>(rootPath("config/secrets-distribution.yaml"), "unidesk-secret-distribution");
|
||||
expect(config.sources.files).toEqual(expect.arrayContaining([expect.objectContaining({ sourceRef: "hwlab/web-probe-monitor-db.env", type: "env", requiredKeys: ["DATABASE_URL"], createIfMissing: { enabled: false } })]));
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Responsibility: YAML-first PG-capable central Monitor manifests while SQLite remains authoritative.
|
||||
import { readWebProbeSentinelConfigRef } from "./hwlab-node-web-sentinel-config-ref";
|
||||
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import type { MonitorSqliteSource } from "./monitor-sqlite-migration";
|
||||
import type { MonitorLegacyArtifactGap, MonitorSqliteSource } from "./monitor-sqlite-migration";
|
||||
|
||||
export function releaseAMonitorManifests(spec: HwlabRuntimeLaneSpec, imageRef: string, resolvedSource: MonitorSqliteSource, options: { readonly migrationJobEnabled?: boolean } = {}): readonly Record<string, unknown>[] {
|
||||
const config = releaseAMonitorConfig(spec, resolvedSource);
|
||||
@@ -37,7 +37,24 @@ export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolve
|
||||
return { poolMax: integerAt(pool, "max", resolved.ref), idleTimeoutSeconds: integerAt(pool, "idleTimeoutSeconds", resolved.ref) };
|
||||
}
|
||||
|
||||
function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly resolved: ReturnType<typeof readWebProbeSentinelConfigRef>; readonly runtime: Record<string, unknown>; readonly job: Record<string, unknown>; readonly namespace: string; readonly deploymentName: string; readonly port: number; readonly secret: Record<string, unknown>; readonly pool: Record<string, unknown>; readonly labels: Record<string, string> } {
|
||||
export function releaseAMonitorMigrationArtifactGaps(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): readonly MonitorLegacyArtifactGap[] {
|
||||
const { resolved, migration } = releaseAMonitorConfig(spec, resolvedSource);
|
||||
const gaps = arrayAt(migration, "legacyArtifactGaps", resolved.ref).map((item, index) => {
|
||||
if (!isRecord(item)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps[${index}] must be an object`);
|
||||
const gap = item;
|
||||
return { runId: textAt(gap, "runId", resolved.ref), artifactCount: integerAt(gap, "artifactCount", resolved.ref), stateDir: textAt(gap, "stateDir", resolved.ref), reportSha256: textAt(gap, "reportSha256", resolved.ref) };
|
||||
});
|
||||
const runIds = new Set<string>();
|
||||
for (const gap of gaps) {
|
||||
if (runIds.has(gap.runId)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps runId values must be unique`);
|
||||
if (gap.stateDir === "." || !gap.stateDir.startsWith(".") || gap.stateDir.includes("\\") || gap.stateDir.split("/").includes("..")) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps stateDir must be a safe relative path`);
|
||||
if (!/^sha256:[a-f0-9]{64}$/u.test(gap.reportSha256)) throw new Error(`${resolved.ref}.migration.legacyArtifactGaps reportSha256 must be sha256:<64 lowercase hex>`);
|
||||
runIds.add(gap.runId);
|
||||
}
|
||||
return [...gaps].sort((left, right) => left.runId.localeCompare(right.runId));
|
||||
}
|
||||
|
||||
function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly resolved: ReturnType<typeof readWebProbeSentinelConfigRef>; readonly runtime: Record<string, unknown>; readonly migration: Record<string, unknown>; readonly job: Record<string, unknown>; readonly namespace: string; readonly deploymentName: string; readonly port: number; readonly secret: Record<string, unknown>; readonly pool: Record<string, unknown>; readonly labels: Record<string, string> } {
|
||||
const configRef = spec.observability.webProbe?.monitor?.configRef;
|
||||
if (!configRef) throw new Error(`lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.monitor.configRef is required`);
|
||||
const resolved = readWebProbeSentinelConfigRef(spec, configRef);
|
||||
@@ -62,10 +79,11 @@ function releaseAMonitorConfig(spec: HwlabRuntimeLaneSpec, resolvedSource: Monit
|
||||
textAt(secret, "key", resolved.ref);
|
||||
textAt(secret, "sourceRef", resolved.ref);
|
||||
const pool = recordAt(runtime, "pool", resolved.ref);
|
||||
return { resolved, runtime, job, namespace, deploymentName, port, secret, pool, labels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" } };
|
||||
return { resolved, runtime, migration, job, namespace, deploymentName, port, secret, pool, labels: { "app.kubernetes.io/name": deploymentName, "app.kubernetes.io/part-of": "hwlab-web-probe-monitor", "app.kubernetes.io/managed-by": "unidesk", "unidesk.ai/migration-mode": "legacy-authority" } };
|
||||
}
|
||||
|
||||
function recordAt(value: Record<string, unknown>, key: string, path: string): Record<string, unknown> { if (!isRecord(value[key])) throw new Error(`${path}.${key} must be an object`); return value[key] as Record<string, unknown>; }
|
||||
function arrayAt(value: Record<string, unknown>, key: string, path: string): readonly unknown[] { if (!Array.isArray(value[key])) throw new Error(`${path}.${key} must be an array`); return value[key] as readonly unknown[]; }
|
||||
function textAt(value: Record<string, unknown>, key: string, path: string): string { const item = value[key]; if (typeof item !== "string" || item.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return item; }
|
||||
function integerAt(value: Record<string, unknown>, key: string, path: string): number { const item = value[key]; if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 1) throw new Error(`${path}.${key} must be a positive integer`); return item; }
|
||||
function isRecord(value: unknown): value is Record<string, unknown> { return typeof value === "object" && value !== null && !Array.isArray(value); }
|
||||
|
||||
@@ -937,12 +937,15 @@ export function probeK8sJobScript(namespace: string, jobName: string): string {
|
||||
"active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)",
|
||||
"pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)",
|
||||
"pod_phase=''",
|
||||
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi",
|
||||
"reason=''",
|
||||
"containers_b64=''",
|
||||
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); reason=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.reason}' 2>/dev/null); containers_b64=$(kubectl -n \"$namespace\" get pod \"$pod\" -o json 2>/dev/null | node -e 'let body=\"\";process.stdin.on(\"data\",c=>body+=c).on(\"end\",()=>{try{const pod=JSON.parse(body);const all=[...(pod.status?.initContainerStatuses||[]),...(pod.status?.containerStatuses||[])];console.log(Buffer.from(JSON.stringify(all.map(item=>({name:item.name,phase:item.state?.terminated?\"terminated\":item.state?.waiting?\"waiting\":item.state?.running?\"running\":\"unknown\",exitCode:item.state?.terminated?.exitCode??null,reason:item.state?.terminated?.reason??item.state?.waiting?.reason??null,message:item.state?.terminated?.message??item.state?.waiting?.message??null})))).toString(\"base64\"))}catch{console.log(\"\")}})' | tr -d '\\n'); fi",
|
||||
"logs_tail=''",
|
||||
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
|
||||
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'",
|
||||
"const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);",
|
||||
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
||||
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$reason\" \"$containers_b64\" \"$logs_tail\" <<'NODE'",
|
||||
"const [succeeded, failed, active, pod, podPhase, reason, containersB64, logsB64] = process.argv.slice(2);",
|
||||
"let containers=[];try{containers=JSON.parse(Buffer.from(containersB64 || '', 'base64').toString('utf8'))}catch{}",
|
||||
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, reason: reason || null, containers, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelEr
|
||||
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
|
||||
import { exportMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, type MonitorSqliteSource } from "./monitor-sqlite-migration";
|
||||
import { releaseAMonitorManifests, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
|
||||
import { releaseAMonitorManifests, releaseAMonitorMigrationArtifactGaps, releaseAMonitorMigrationJob, releaseAMonitorMigrationPool } from "./hwlab-node-web-probe-monitor";
|
||||
import {
|
||||
arrayAt,
|
||||
arrayAtNullable,
|
||||
@@ -193,7 +193,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
|
||||
}
|
||||
|
||||
function runSentinelMigration(spec: HwlabRuntimeLaneSpec, options: Extract<WebProbeSentinelOptions, { kind: "migration" }>): RenderedCliResult {
|
||||
const sources = resolvedMigrationSources(spec, options.sentinelId);
|
||||
const sources = resolvedMigrationSources(spec, options.sentinelId).map((source) => ({ ...source, legacyArtifactGaps: releaseAMonitorMigrationArtifactGaps(spec, source) }));
|
||||
const command = `web-probe sentinel migration ${options.action}`;
|
||||
if (options.action === "job-plan" || options.action === "job-run") return runSentinelMigrationJob(spec, options, sources, command);
|
||||
if (options.action === "plan") {
|
||||
@@ -241,7 +241,7 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract<We
|
||||
if (options.action === "job-plan") return rendered(true, command, renderMigrationCompact(plan), plan);
|
||||
if (!options.confirm || !options.wait) throw new Error("web-probe sentinel migration job-run requires --confirm --wait");
|
||||
const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 });
|
||||
if (created.exitCode !== 0) return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase: "create-failed", create: compactCommand(created) });
|
||||
if (created.exitCode !== 0) return migrationJobFailed(command, plan, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "migration Job creation failed" });
|
||||
const startedAt = Date.now();
|
||||
let lastProbe = created;
|
||||
while (Date.now() - startedAt < Math.max(5_000, Math.min(options.timeoutSeconds * 1000, 120_000))) {
|
||||
@@ -249,16 +249,48 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract<We
|
||||
const probe = resolveSentinelChildJson(lastProbe, "web-probe-sentinel-migration-job-probe").parsed ?? {};
|
||||
if (probe.succeeded === true || probe.failed === true) {
|
||||
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
|
||||
const ok = probe.succeeded === true && payload.ok === true;
|
||||
return rendered(ok, command, `ok=${ok}`, { ...plan, ok, mutation: true, phase: probe.succeeded === true ? "succeeded" : "failed", result: payload, valuesRedacted: true });
|
||||
if (probe.succeeded === true && payload.ok === true) return rendered(true, command, "ok=true", { ...plan, ok: true, mutation: true, phase: "succeeded", result: payload, valuesRedacted: true });
|
||||
return migrationJobFailed(command, plan, probe.succeeded === true ? "result-failed" : "failed", payload, probe, migrationJobFailureNext(state, namespace, jobName));
|
||||
}
|
||||
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
|
||||
}
|
||||
return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase: "timeout", probe: compactCommand(lastProbe), valuesRedacted: true });
|
||||
return migrationJobFailed(command, plan, "timeout", {}, { podPhase: "Timeout", logsTail: "migration Job did not reach a successful result before timeout" }, migrationJobFailureNext(state, namespace, jobName));
|
||||
}
|
||||
|
||||
function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean {
|
||||
return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath);
|
||||
export function migrationJobFailed(command: string, plan: Record<string, unknown>, phase: string, payload: Record<string, unknown>, probe: Record<string, unknown>, next?: Record<string, string>): RenderedCliResult {
|
||||
return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase, failure: migrationJobFailureSummary(probe, payload), next, valuesRedacted: true });
|
||||
}
|
||||
|
||||
export function migrationJobFailureSummary(probe: Record<string, unknown>, payload: Record<string, unknown>): Record<string, unknown> {
|
||||
const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord).map((item) => ({ name: stringAtNullable(item, "name"), phase: stringAtNullable(item, "phase"), exitCode: finiteNumberOrNull(item.exitCode), reason: stringAtNullable(item, "reason"), message: redactedMigrationJobText(stringAtNullable(item, "message") ?? "") })).filter((item) => item.phase === "terminated" || item.exitCode !== null || item.reason !== null) : [];
|
||||
const failed = containers.find((item) => item.exitCode !== null && item.exitCode !== 0) ?? containers[0] ?? null;
|
||||
const logsTail = redactedMigrationJobText(String(probe.logsTail ?? ""));
|
||||
const nestedError = isRecord(payload.error) ? payload.error : null;
|
||||
return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, status: failed?.reason ?? stringAtNullable(nestedError, "code") ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactedMigrationJobText(stringAtNullable(nestedError, "message") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted"), logsTail, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function redactedMigrationJobText(value: string): string {
|
||||
return short(redactMigrationJobLogTail(value));
|
||||
}
|
||||
|
||||
function redactMigrationJobLogTail(value: string): string {
|
||||
return value
|
||||
.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s"']*@[^\s"']+/giu, "<redacted-url>")
|
||||
.replace(/\bBearer\s+[^\s"']+/giu, "Bearer <redacted>")
|
||||
.replace(/(["']?(?:DATABASE_URL|password|token|secret)["']?\s*[=:]\s*["']?)[^\s,"'}\]]+/giu, "$1<redacted>")
|
||||
.replace(/([?&](?:password|token|secret)=[^&\s"']+)/giu, "<redacted-query>");
|
||||
}
|
||||
|
||||
function migrationJobFailureNext(state: SentinelCicdState, namespace: string, jobName: string): Record<string, string> {
|
||||
return { logs: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${jobName} --all-containers=true --tail=120`, describe: `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${jobName}` };
|
||||
}
|
||||
|
||||
export function sameMigrationSource(left: MonitorSqliteSource, right: MonitorSqliteSource): boolean {
|
||||
return left.sentinelId === right.sentinelId && left.node === right.node && left.lane === right.lane && left.path === right.path && left.runtimeConfigRef === right.runtimeConfigRef && left.stateRoot === right.stateRoot && left.ownerPvc === right.ownerPvc && left.artifactSources.length === right.artifactSources.length && left.artifactSources.every((source, index) => source.ownerPvc === right.artifactSources[index]?.ownerPvc && source.mountPath === right.artifactSources[index]?.mountPath && source.hostPath === right.artifactSources[index]?.hostPath) && monitorGapSet(left.legacyArtifactGaps) === monitorGapSet(right.legacyArtifactGaps);
|
||||
}
|
||||
|
||||
function monitorGapSet(gaps: readonly { readonly runId: string; readonly artifactCount: number; readonly stateDir: string; readonly reportSha256: string }[] | undefined): string {
|
||||
return [...(gaps ?? [])].map((gap) => [gap.runId, gap.artifactCount, gap.stateDir, gap.reportSha256] as const).sort((left, right) => left[0].localeCompare(right[0])).map((gap) => JSON.stringify(gap)).join("\n");
|
||||
}
|
||||
|
||||
function resolvedMigrationSources(spec: HwlabRuntimeLaneSpec, sentinelId: string | null): readonly MonitorSqliteSource[] {
|
||||
|
||||
@@ -10,13 +10,15 @@ import { monitorCentralStableJson, normalizeMonitorCentralIngest } from "./monit
|
||||
|
||||
export const MONITOR_SQLITE_MIGRATION_SCHEMA = "2026-07-12-p17-3";
|
||||
export interface MonitorArtifactSource { readonly ownerPvc: string; readonly mountPath: string; readonly hostPath?: string; }
|
||||
function freezeArtifactManifests(snapshot: Pick<MonitorSqliteSnapshot, "source" | "tables">): readonly { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[] }[] { const rows = tableRows(snapshot.tables, ["runs", "run"]); const locatorRows = tableRows(snapshot.tables, ["artifact_locators", "locators"]); return rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); const rowLocators = locatorRows.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); const locators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); const expectedCount = numberField(row, ["artifact_count", "artifactCount"]); if (expectedCount !== null && expectedCount !== locators.length) throw artifactCountMismatch(runId, stringField(row, ["state_dir", "stateDir"]), expectedCount, locators.length); verifyReportLocator(row, locators); return { runId, locators }; }); }
|
||||
export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; readonly artifactSources: readonly MonitorArtifactSource[]; }
|
||||
export interface MonitorLegacyArtifactGap { readonly runId: string; readonly artifactCount: number; readonly stateDir: string; readonly reportSha256: string; }
|
||||
export interface MonitorArtifactManifest { readonly runId: string; readonly artifactCount: number; readonly stateDir: string; readonly reportSha256: string; readonly ownerPvc: string | null; readonly locators: readonly MonitorArtifactLocator[]; readonly missingBeforeMigration?: MonitorLegacyArtifactGap; }
|
||||
function freezeArtifactManifests(snapshot: Pick<MonitorSqliteSnapshot, "source" | "tables">): readonly MonitorArtifactManifest[] { const rows = tableRows(snapshot.tables, ["runs", "run"]); const locatorRows = tableRows(snapshot.tables, ["artifact_locators", "locators"]); const declared = snapshot.source.legacyArtifactGaps ?? []; const used = new Set<string>(); const manifests = rows.map((row) => { const runId = stringField(row, ["run_id", "id"]); const gap = declared.find((item) => item.runId === runId); const rowCount = numberField(row, ["artifact_count", "artifactCount"]); const stateDir = stringField(row, ["state_dir", "stateDir"]); const reportSha256 = stringField(row, ["report_json_sha256", "reportJsonSha256"]); const rowLocators = locatorRows.filter((item) => stringField(item, ["run_id"]) === runId).map((item) => locatorFromRow(item, snapshot)); if (gap !== undefined) { if (gap.artifactCount !== rowCount || gap.stateDir !== stateDir || gap.reportSha256 !== reportSha256) throw artifactGapIdentityInvalid(runId, gap, { artifactCount: rowCount, stateDir, reportSha256 }); if (rowLocators.length !== 0) throw Object.assign(new Error(`declared missing artifact gap unexpectedly has locators for ${runId}`), { code: "monitor-migration-artifact-gap-locator-unexpected" }); used.add(runId); return { runId, artifactCount: gap.artifactCount, stateDir, reportSha256, ownerPvc: null, locators: [], missingBeforeMigration: gap }; } const locators = rowLocators.length > 0 ? rowLocators : artifactLocatorsFromRun(row, snapshot); const artifactCount = rowCount ?? locators.length; if (artifactCount !== locators.length) throw artifactCountMismatch(runId, stateDir, artifactCount, locators.length); verifyReportLocator(row, locators); const ownerPvc = locators[0]?.ownerPvc ?? (stateDir ? artifactSourceForRunStateDir(snapshot.source, stateDir).ownerPvc : null); return { runId, artifactCount, stateDir, reportSha256, ownerPvc, locators }; }); if (used.size !== declared.length) throw Object.assign(new Error("declared legacy artifact gap was not found in the SQLite snapshot"), { code: "monitor-migration-artifact-gap-unmatched" }); return manifests; }
|
||||
export interface MonitorSqliteSource { readonly sentinelId: string; readonly node: string; readonly lane: string; readonly path: string; readonly runtimeConfigRef: string; readonly stateRoot: string; readonly ownerPvc: string; readonly artifactSources: readonly MonitorArtifactSource[]; readonly legacyArtifactGaps?: readonly MonitorLegacyArtifactGap[]; }
|
||||
export interface MonitorSqliteTable { readonly name: string; readonly sql: string; readonly rowCount: number; readonly rows: readonly Record<string, unknown>[]; }
|
||||
export interface MonitorSqliteSnapshot {
|
||||
readonly schema: string; readonly createdAt: string; readonly source: MonitorSqliteSource; readonly fingerprint: string; readonly integrity: string; readonly byteLength: number;
|
||||
readonly tables: readonly MonitorSqliteTable[];
|
||||
readonly artifactManifests: readonly { readonly runId: string; readonly locators: readonly MonitorArtifactLocator[] }[];
|
||||
readonly artifactManifests: readonly MonitorArtifactManifest[];
|
||||
readonly artifactLocator: { readonly path: string; readonly sha256: string; readonly sizeBytes: number; readonly kind: "legacy-sqlite" };
|
||||
readonly manifestContentHash: string;
|
||||
}
|
||||
@@ -24,6 +26,7 @@ export interface MonitorMigrationReconciliation { readonly runs: number; readonl
|
||||
|
||||
export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise<Record<string, unknown>> {
|
||||
const snapshotVerification = verifyMonitorSqliteSnapshot(snapshot);
|
||||
verifySnapshotReferentialIntegrity(snapshot);
|
||||
verifyFrozenArtifactManifests(snapshot);
|
||||
const expected = canonicalIngests(snapshot);
|
||||
const expectedMetadata = canonicalRuntimeMetadata(snapshot);
|
||||
@@ -34,6 +37,7 @@ export async function verifyMonitorSqliteSnapshotAgainstStore(snapshot: MonitorS
|
||||
checks.push({ name: "run", runId: input.runId, ok: actual?.payloadHash === input.payloadHash && actual?.status === input.status && actual?.updatedAt === input.updatedAt && monitorCentralStableJson(actual?.payload) === monitorCentralStableJson(input.payload), valuesRedacted: true });
|
||||
checks.push({ name: "finding", runId: input.runId, ok: monitorCentralStableJson(actual?.findings) === monitorCentralStableJson(input.findings), valuesRedacted: true });
|
||||
checks.push({ name: "artifactLocator", runId: input.runId, ok: monitorCentralStableJson(actual?.artifactLocators) === monitorCentralStableJson(input.artifactLocators), valuesRedacted: true });
|
||||
checks.push({ name: "timeline", runId: input.runId, ok: Array.isArray(actual?.timeline) && actual.timeline.length === input.timeline.length && monitorCentralStableJson(actual.timeline) === monitorCentralStableJson(input.timeline), valuesRedacted: true });
|
||||
const views = isRecord(input.payload.views) ? input.payload.views : {};
|
||||
for (const [name, value] of Object.entries(views)) {
|
||||
const actualView = await store.view(scope, input.runId, name);
|
||||
@@ -91,12 +95,15 @@ export function verifyMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot): Re
|
||||
const reconciliation = reconcileTables(snapshot.tables);
|
||||
const manifestContentHashValid = snapshot.manifestContentHash === monitorSnapshotContentHash(snapshot);
|
||||
const rowsValid = snapshot.tables.every((table) => table.rowCount === table.rows.length);
|
||||
return { ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && locatorValid && fingerprintValid && manifestContentHashValid && rowsValid, command: "web-probe sentinel migration verify", fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, reconciliation, artifactLocator: { ...snapshot.artifactLocator, valid: locatorValid }, schemaValid: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA, manifestContentHashValid, rowsValid, valuesRedacted: true };
|
||||
let referentialValid = true;
|
||||
try { verifySnapshotReferentialIntegrity(snapshot); } catch { referentialValid = false; }
|
||||
return { ok: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA && snapshot.integrity === "ok" && locatorValid && fingerprintValid && manifestContentHashValid && rowsValid && referentialValid, command: "web-probe sentinel migration verify", fingerprint: snapshot.fingerprint, integrity: snapshot.integrity, reconciliation, artifactLocator: { ...snapshot.artifactLocator, valid: locatorValid }, schemaValid: snapshot.schema === MONITOR_SQLITE_MIGRATION_SCHEMA, manifestContentHashValid, rowsValid, referentialValid, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export async function importMonitorSqliteSnapshot(snapshot: MonitorSqliteSnapshot, store: MonitorCentralStore): Promise<Record<string, unknown>> {
|
||||
const verified = verifyMonitorSqliteSnapshot(snapshot);
|
||||
if (verified.ok !== true) throw Object.assign(new Error("snapshot verification failed"), { code: "monitor-migration-snapshot-invalid", verification: verified });
|
||||
verifySnapshotReferentialIntegrity(snapshot);
|
||||
verifyFrozenArtifactManifests(snapshot);
|
||||
const ingests = canonicalIngests(snapshot);
|
||||
const runtimeMetadata = canonicalRuntimeMetadata(snapshot);
|
||||
@@ -125,9 +132,11 @@ function canonicalIngests(snapshot: MonitorSqliteSnapshot): readonly ReturnType<
|
||||
const payload = { ...(storedPayload ?? {}), ...(report ?? {}), legacy: { run: row, sourceFingerprint: snapshot.fingerprint } };
|
||||
const reportFindings = Array.isArray(report?.findings) ? report.findings.filter(isRecord) : [];
|
||||
const canonicalFindings = dedupeFindings(reportFindings, rowFindings);
|
||||
const artifactLocators = snapshot.artifactManifests.find((item) => item.runId === runId)?.locators;
|
||||
if (artifactLocators === undefined) throw Object.assign(new Error(`snapshot artifact manifest is missing ${runId}`), { code: "monitor-migration-artifact-manifest-missing" });
|
||||
return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload, findings: canonicalFindings, artifactLocators, timeline: [] });
|
||||
const artifactManifest = snapshot.artifactManifests.find((item) => item.runId === runId);
|
||||
if (artifactManifest === undefined) throw Object.assign(new Error(`snapshot artifact manifest is missing ${runId}`), { code: "monitor-migration-artifact-manifest-missing" });
|
||||
const missing = artifactManifest.missingBeforeMigration;
|
||||
const missingFact = missing === undefined ? null : { code: "legacy-artifact-bytes-missing-before-migration", runId, artifactCount: missing.artifactCount, stateDir: missing.stateDir, reportSha256: missing.reportSha256, locator: null, bytesVerified: false, valuesRedacted: true };
|
||||
return normalizeMonitorCentralIngest({ sentinelId: stringField(row, ["sentinel_id", "sentinelId"]) || snapshot.source.sentinelId, node: stringField(row, ["node"]) || snapshot.source.node, lane: stringField(row, ["lane"]) || snapshot.source.lane, runId, updatedAt: stringField(row, ["updated_at", "updatedAt", "created_at", "createdAt"]) || snapshot.createdAt, status: stringField(row, ["status"]) || "legacy", payload: missingFact === null ? payload : { ...payload, migration: { legacyArtifactBytesMissingBeforeMigration: missingFact } }, findings: missingFact === null ? canonicalFindings : [...canonicalFindings, missingFact], artifactLocators: artifactManifest.locators, timeline: missingFact === null ? [] : [missingFact] });
|
||||
});
|
||||
}
|
||||
function targetReconciliation(ingests: readonly ReturnType<typeof normalizeMonitorCentralIngest>[], runtimeMetadata: readonly MonitorRuntimeMetadata[]): Record<string, number> { return { runs: ingests.length, findings: ingests.reduce((total, item) => total + item.findings.length, 0), payloads: ingests.length, artifactLocators: ingests.reduce((total, item) => total + item.artifactLocators.length, 0), views: ingests.reduce((total, item) => total + (isRecord(item.payload.views) ? Object.keys(item.payload.views).length : 0), 0), runtimeMetadata: runtimeMetadata.length }; }
|
||||
@@ -145,11 +154,81 @@ function assertReadOnlySqliteSource(value: string): string { const path = resolv
|
||||
function writeSnapshot(path: string, content: string): void { const parent = dirname(resolve(path)); if (!existsSync(parent)) throw new Error(`snapshot output directory does not exist: ${parent}`); writeFileSync(path, content); }
|
||||
function sqliteSnapshotPath(path: string): string { const resolved = resolve(path); return resolved.endsWith(".json") ? `${resolved.slice(0, -5)}.sqlite` : `${resolved}.sqlite`; }
|
||||
function artifactSourceFor(source: MonitorSqliteSource, ownerPvc: string): MonitorArtifactSource { const artifactSource = source.artifactSources.find((item) => item.ownerPvc === ownerPvc); if (!artifactSource) throw Object.assign(new Error(`legacy artifact owner is not registry-mounted: ${ownerPvc}`), { code: "monitor-migration-artifact-source-missing" }); return artifactSource; }
|
||||
function artifactSourceForStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { const directory = resolve(stateDir); const matches = source.artifactSources.filter((item) => directory === resolve(item.mountPath) || directory.startsWith(`${resolve(item.mountPath)}/`)).sort((left, right) => resolve(right.mountPath).length - resolve(left.mountPath).length); if (matches.length === 0) throw Object.assign(new Error(`legacy artifact stateDir is not registry-mounted: ${stateDir}`), { code: "monitor-migration-artifact-source-missing" }); return matches[0]!; }
|
||||
function artifactSourceForStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { if (!stateDir.startsWith("/") && source.artifactSources.length === 1) return source.artifactSources[0]!; const directory = resolve(stateDir); const matches = source.artifactSources.filter((item) => directory === resolve(item.mountPath) || directory.startsWith(`${resolve(item.mountPath)}/`)).sort((left, right) => resolve(right.mountPath).length - resolve(left.mountPath).length); if (matches.length === 0) throw Object.assign(new Error(`legacy artifact stateDir is not registry-mounted: ${stateDir}`), { code: "monitor-migration-artifact-source-missing" }); return matches[0]!; }
|
||||
function artifactSourceForRunStateDir(source: MonitorSqliteSource, stateDir: string): MonitorArtifactSource { return artifactSourceForStateDir(source, stateDir); }
|
||||
function artifactPath(mountPath: string, stateDir: string, relativePath: string): string { const root = resolve(mountPath); const directory = stateDir.startsWith("/") ? resolve(stateDir) : resolve(root, stateDir); if (directory !== root && !directory.startsWith(`${root}/`)) throw Object.assign(new Error("legacy artifact stateDir escapes registry mount"), { code: "monitor-migration-artifact-path-invalid" }); return join(directory, relativePath); }
|
||||
function artifactFiles(directory: string): readonly string[] { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); if (entry.isDirectory()) return artifactFiles(path); return entry.isFile() ? [path] : []; }).sort(); }
|
||||
function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void { for (const manifest of snapshot.artifactManifests) for (const locator of manifest.locators) { const source = artifactSourceFor(snapshot.source, locator.ownerPvc); const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath); if (!existsSync(path) || !statSync(path).isFile()) throw Object.assign(new Error(`frozen artifact is unavailable for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: null, valuesRedacted: true } }); const bytes = readFileSync(path); if (`sha256:${sha256(bytes)}` !== locator.sha256 || bytes.byteLength !== locator.sizeBytes) throw Object.assign(new Error(`frozen artifact changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: locator.stateDir, expectedCount: manifest.locators.length, observedCount: manifest.locators.length, valuesRedacted: true } }); } }
|
||||
function verifyFrozenArtifactManifests(snapshot: MonitorSqliteSnapshot): void {
|
||||
for (const manifest of snapshot.artifactManifests) {
|
||||
if (manifest.missingBeforeMigration !== undefined) {
|
||||
const gap = manifest.missingBeforeMigration;
|
||||
if (manifest.locators.length !== 0 || !matchesArtifactGap(gap, snapshot.source.legacyArtifactGaps ?? [])) throw Object.assign(new Error(`invalid frozen legacy artifact gap for ${manifest.runId}`), { code: "monitor-migration-artifact-gap-identity-invalid" });
|
||||
for (const source of snapshot.source.artifactSources) {
|
||||
const directory = artifactPath(source.mountPath, gap.stateDir, "");
|
||||
if (existsSync(directory) && (statSync(directory).isFile() || (statSync(directory).isDirectory() && artifactFiles(directory).length > 0))) throw Object.assign(new Error(`legacy artifact bytes recovered for ${manifest.runId}; re-export is required`), { code: "monitor-migration-artifact-gap-recovered", diagnostic: { runId: manifest.runId, stateDir: gap.stateDir, valuesRedacted: true } });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const expected = new Map(manifest.locators.map((locator) => [`${locator.ownerPvc}\u0000${locator.stateDir}\u0000${locator.relativePath}`, locator]));
|
||||
const observed = new Map<string, MonitorArtifactLocator>();
|
||||
for (const locator of manifest.locators) {
|
||||
const source = artifactSourceFor(snapshot.source, locator.ownerPvc);
|
||||
const path = artifactPath(source.mountPath, locator.stateDir, locator.relativePath);
|
||||
if (!existsSync(path) || !statSync(path).isFile()) throw frozenArtifactInvalid(manifest, null);
|
||||
const bytes = readFileSync(path);
|
||||
const actual = { ...locator, sha256: `sha256:${sha256(bytes)}`, sizeBytes: bytes.byteLength };
|
||||
if (actual.sha256 !== locator.sha256 || actual.sizeBytes !== locator.sizeBytes) throw frozenArtifactInvalid(manifest, manifest.locators.length);
|
||||
observed.set(`${locator.ownerPvc}\u0000${locator.stateDir}\u0000${locator.relativePath}`, actual);
|
||||
}
|
||||
if (!manifest.stateDir) { if (manifest.locators.length !== 0 || manifest.artifactCount !== 0) throw frozenArtifactInvalid(manifest, null); continue; }
|
||||
if (manifest.ownerPvc === null) throw frozenArtifactInvalid(manifest, null);
|
||||
const source = artifactSourceFor(snapshot.source, manifest.ownerPvc);
|
||||
const directory = artifactPath(source.mountPath, manifest.stateDir, "");
|
||||
if (!existsSync(directory) || !statSync(directory).isDirectory()) throw frozenArtifactInvalid(manifest, null);
|
||||
for (const path of artifactFiles(directory)) {
|
||||
const relativePath = path.slice(directory.length + 1).replaceAll("\\", "/");
|
||||
const key = `${manifest.ownerPvc}\u0000${manifest.stateDir}\u0000${relativePath}`;
|
||||
if (!expected.has(key)) throw frozenArtifactInvalid(manifest, manifest.locators.length + 1);
|
||||
}
|
||||
if (observed.size !== expected.size) throw frozenArtifactInvalid(manifest, observed.size);
|
||||
}
|
||||
}
|
||||
|
||||
function frozenArtifactInvalid(manifest: MonitorArtifactManifest, observedCount: number | null): Error { return Object.assign(new Error(`frozen artifact set changed for ${manifest.runId}`), { code: "monitor-migration-frozen-artifact-invalid", diagnostic: { runId: manifest.runId, stateDir: manifest.stateDir, expectedCount: manifest.locators.length, observedCount, valuesRedacted: true } }); }
|
||||
function verifySnapshotReferentialIntegrity(snapshot: MonitorSqliteSnapshot): void {
|
||||
const rows = tableRows(snapshot.tables, ["runs", "run"]);
|
||||
const rowByRunId = new Map<string, Record<string, unknown>>();
|
||||
for (const row of rows) { const runId = stringField(row, ["run_id", "id"]); if (!runId || rowByRunId.has(runId)) throw Object.assign(new Error("snapshot run identities are invalid"), { code: "monitor-migration-snapshot-referential-invalid" }); rowByRunId.set(runId, row); }
|
||||
const manifestByRunId = new Map<string, MonitorArtifactManifest>();
|
||||
for (const manifest of snapshot.artifactManifests) { if (!manifest.runId || manifestByRunId.has(manifest.runId) || !rowByRunId.has(manifest.runId)) throw Object.assign(new Error("snapshot artifact manifests are invalid"), { code: "monitor-migration-snapshot-referential-invalid" }); manifestByRunId.set(manifest.runId, manifest); }
|
||||
if (manifestByRunId.size !== rowByRunId.size) throw Object.assign(new Error("snapshot artifact manifests must match SQLite runs"), { code: "monitor-migration-snapshot-referential-invalid" });
|
||||
const declared = snapshot.source.legacyArtifactGaps ?? [];
|
||||
const declaredByRunId = new Map<string, MonitorLegacyArtifactGap>();
|
||||
for (const gap of declared) { if (!gap.runId || declaredByRunId.has(gap.runId)) throw Object.assign(new Error("snapshot legacy artifact gaps are invalid"), { code: "monitor-migration-snapshot-referential-invalid" }); declaredByRunId.set(gap.runId, gap); }
|
||||
for (const [runId, manifest] of manifestByRunId) {
|
||||
const row = rowByRunId.get(runId)!;
|
||||
const gap = manifest.missingBeforeMigration;
|
||||
if (gap === undefined) {
|
||||
const rowCount = numberField(row, ["artifact_count", "artifactCount"]);
|
||||
const stateDir = stringField(row, ["state_dir", "stateDir"]);
|
||||
const reportSha256 = stringField(row, ["report_json_sha256", "reportJsonSha256"]);
|
||||
const locatorRows = tableRows(snapshot.tables, ["artifact_locators", "locators"]);
|
||||
const expectedLocators = locatorRows.filter((locator) => stringField(locator, ["run_id"]) === runId).map((locator) => locatorBindingFromRow(locator, snapshot));
|
||||
if (declaredByRunId.has(runId) || manifest.artifactCount !== (rowCount ?? manifest.locators.length) || manifest.stateDir !== stateDir || manifest.reportSha256 !== reportSha256 || (stateDir.length > 0 && manifest.ownerPvc === null) || manifest.locators.some((locator) => locator.ownerPvc !== manifest.ownerPvc || locator.stateDir !== stateDir) || manifest.locators.length !== manifest.artifactCount || (expectedLocators.length > 0 && !sameLocatorBindings(manifest.locators, expectedLocators))) throw Object.assign(new Error("snapshot normal artifact manifest does not match run identity"), { code: "monitor-migration-snapshot-referential-invalid" });
|
||||
verifyReportLocator(row, manifest.locators);
|
||||
continue;
|
||||
}
|
||||
const declaredGap = declaredByRunId.get(runId);
|
||||
if (gap.runId !== runId || declaredGap === undefined || !sameArtifactGap(gap, declaredGap) || gap.artifactCount !== numberField(row, ["artifact_count", "artifactCount"]) || gap.stateDir !== stringField(row, ["state_dir", "stateDir"]) || gap.reportSha256 !== stringField(row, ["report_json_sha256", "reportJsonSha256"])) throw Object.assign(new Error("snapshot legacy artifact gap does not match run identity"), { code: "monitor-migration-snapshot-referential-invalid" });
|
||||
}
|
||||
if ([...declaredByRunId.keys()].some((runId) => manifestByRunId.get(runId)?.missingBeforeMigration === undefined)) throw Object.assign(new Error("snapshot declared gap has no matching frozen manifest"), { code: "monitor-migration-snapshot-referential-invalid" });
|
||||
}
|
||||
function locatorBindingFromRow(row: Record<string, unknown>, snapshot: Pick<MonitorSqliteSnapshot, "source">): MonitorArtifactLocator { return { ownerNode: stringField(row, ["owner_node", "ownerNode"]) || snapshot.source.node, ownerLane: stringField(row, ["owner_lane", "ownerLane"]) || snapshot.source.lane, ownerPvc: stringField(row, ["owner_pvc", "ownerPvc"]), stateDir: stringField(row, ["state_dir", "stateDir"]), relativePath: stringField(row, ["relative_path", "relativePath"]), sha256: stringField(row, ["sha256"]), sizeBytes: numberField(row, ["size_bytes", "sizeBytes"]) ?? -1, kind: stringField(row, ["kind"]) || "artifact", reachable: row.reachable !== false }; }
|
||||
function sameLocatorBindings(left: readonly MonitorArtifactLocator[], right: readonly MonitorArtifactLocator[]): boolean { const key = (locator: MonitorArtifactLocator): string => [locator.ownerNode, locator.ownerLane, locator.ownerPvc, locator.stateDir, locator.relativePath, locator.sha256, locator.sizeBytes, locator.kind, locator.reachable].join("\u0000"); return left.length === right.length && left.map(key).sort().every((value, index) => value === right.map(key).sort()[index]); }
|
||||
function artifactCountMismatch(runId: string, stateDir: string, expectedCount: number, observedCount: number): Error { return Object.assign(new Error(`legacy artifact locator count mismatch for ${runId}`), { code: "monitor-migration-artifact-count-mismatch", diagnostic: { runId, stateDir, expectedCount, observedCount, valuesRedacted: true } }); }
|
||||
function artifactGapIdentityInvalid(runId: string, expected: MonitorLegacyArtifactGap, observed: Record<string, unknown>): Error { return Object.assign(new Error(`legacy artifact gap identity mismatch for ${runId}`), { code: "monitor-migration-artifact-gap-identity-invalid", diagnostic: { runId, expected, observed, valuesRedacted: true } }); }
|
||||
function sameArtifactGap(left: MonitorLegacyArtifactGap, right: MonitorLegacyArtifactGap): boolean { return left.runId === right.runId && left.artifactCount === right.artifactCount && left.stateDir === right.stateDir && left.reportSha256 === right.reportSha256; }
|
||||
function matchesArtifactGap(gap: MonitorLegacyArtifactGap, declared: readonly MonitorLegacyArtifactGap[]): boolean { return declared.some((item) => item.runId === gap.runId && item.artifactCount === gap.artifactCount && item.stateDir === gap.stateDir && item.reportSha256 === gap.reportSha256); }
|
||||
function quoteIdentifier(value: string): string { return `"${value.replaceAll("\"", "\"\"")}"`; }
|
||||
function normalizeRow(value: unknown): Record<string, unknown> { const row = value as Record<string, unknown>; return Object.fromEntries(Object.entries(row).map(([key, item]) => [key, item instanceof Uint8Array ? Buffer.from(item).toString("base64") : item])); }
|
||||
function jsonField(row: Record<string, unknown> | undefined, fields: readonly string[]): Record<string, unknown> | null { if (!row) return null; const value = fields.map((field) => row[field]).find((item) => item !== undefined); if (typeof value === "string") try { const parsed = JSON.parse(value); return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null; } catch { return null; } return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null; }
|
||||
|
||||
@@ -8,7 +8,7 @@ import test from "node:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
|
||||
import { createInMemoryMonitorCentralStore } from "./monitor-central-persistence";
|
||||
import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration";
|
||||
import { exportMonitorSqliteSnapshot, importMonitorSqliteSnapshot, monitorSnapshotContentHash, planMonitorSqliteSources, verifyMonitorSqliteSnapshot, verifyMonitorSqliteSnapshotAgainstStore } from "./monitor-sqlite-migration";
|
||||
import { buildMonitorTerminalIngest, MonitorIngestFailedError, resolveMonitorTerminalWriter, submitMonitorTerminalIngest } from "./monitor-terminal-ingest";
|
||||
import { compactWebObserveAnalyzeAnalysisForRaw } from "./hwlab-node/web-probe-observe";
|
||||
|
||||
@@ -87,7 +87,7 @@ test("SQLite snapshot reconciles the 228/242/234 fixture shape without productio
|
||||
assert.equal((await store.view({ sentinelId: "fixture", node: "NC01", lane: "v03" }, "run-0", "summary"))?.value.title, "legacy summary");
|
||||
const pgVerification = await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store);
|
||||
assert.equal(pgVerification.ok, true);
|
||||
assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalOverview", "run", "runtimeMetadata", "scopedOverview", "view"]);
|
||||
assert.deepEqual([...new Set((pgVerification.checks as { name: string }[]).map((item) => item.name))].sort(), ["artifactLocator", "finding", "globalOverview", "run", "runtimeMetadata", "scopedOverview", "timeline", "view"]);
|
||||
assert.deepEqual((pgVerification.reconciliation as { target: Record<string, number> }).target, { runs: 228, findings: 242, payloads: 228, artifactLocators: 1, views: 1, runtimeMetadata: 234 });
|
||||
await store.importSource({ manifestId: "sha256:orphan-metadata", sourceFingerprint: "sha256:orphan-metadata", rowCount: 0 }, [], [{ sentinelId: "fixture", node: "NC01", lane: "v03", key: "orphan", value: { unexpected: true }, updatedAt: snapshot.createdAt }]);
|
||||
assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, false);
|
||||
@@ -192,6 +192,9 @@ test("workspace artifact source verifies locator identity hash and size", async
|
||||
writeFileSync(join(stateDir, "artifacts", "request.txt"), "changed\n");
|
||||
await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid");
|
||||
writeFileSync(join(stateDir, "artifacts", "request.txt"), artifacts[2][1]);
|
||||
writeFileSync(join(stateDir, "artifacts", "new-after-export.txt"), "new\n");
|
||||
await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid");
|
||||
rmSync(join(stateDir, "artifacts", "new-after-export.txt"));
|
||||
rmSync(join(stateDir, "artifacts", "response.txt"));
|
||||
await assert.rejects(() => verifyMonitorSqliteSnapshotAgainstStore(snapshot, store), (error: unknown) => { const migrationError = error as { code?: string; diagnostic?: Record<string, unknown> }; return migrationError.code === "monitor-migration-frozen-artifact-invalid" && migrationError.diagnostic?.runId === "run-1" && migrationError.diagnostic?.stateDir === stateDir && migrationError.diagnostic?.expectedCount === 5; });
|
||||
} finally {
|
||||
@@ -199,6 +202,158 @@ test("workspace artifact source verifies locator identity hash and size", async
|
||||
}
|
||||
});
|
||||
|
||||
test("declared pre-migration artifact gaps import empty locators and fail closed on identity drift", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "monitor-artifact-gaps-"));
|
||||
try {
|
||||
const sqlitePath = join(directory, "index.sqlite");
|
||||
const gaps = [
|
||||
{ runId: "gap-run-a", artifactCount: 1, stateDir: ".state/missing/a", reportSha256: `sha256:${"a".repeat(64)}` },
|
||||
{ runId: "gap-run-b", artifactCount: 4, stateDir: ".state/missing/b", reportSha256: `sha256:${"b".repeat(64)}` },
|
||||
];
|
||||
const database = new Database(sqlitePath);
|
||||
database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)");
|
||||
for (const gap of gaps) database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", [gap.runId, gap.stateDir, gap.artifactCount, gap.reportSha256, "2026-07-08T03:58:39.000Z"]);
|
||||
database.close();
|
||||
const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }], legacyArtifactGaps: gaps };
|
||||
const snapshot = exportMonitorSqliteSnapshot(source, join(directory, "snapshot.json"));
|
||||
const store = createInMemoryMonitorCentralStore();
|
||||
await importMonitorSqliteSnapshot(snapshot, store);
|
||||
for (const gap of gaps) {
|
||||
const run = await store.run({ sentinelId: "fixture", node: "NC01", lane: "v03" }, gap.runId);
|
||||
assert.equal(run?.artifactLocators.length, 0);
|
||||
assert.equal((run?.timeline.at(-1) as { code?: string; locator?: unknown; bytesVerified?: boolean } | undefined)?.code, "legacy-artifact-bytes-missing-before-migration");
|
||||
assert.equal((run?.timeline.at(-1) as { locator?: unknown }).locator, null);
|
||||
assert.equal((run?.timeline.at(-1) as { bytesVerified?: boolean }).bytesVerified, false);
|
||||
}
|
||||
assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, store)).ok, true);
|
||||
const timelineTamperedStore = { ...store, async run(scope: { sentinelId: string; node: string; lane: string }, runId: string) { const run = await store.run(scope, runId); return run === null ? null : { ...run, timeline: [] }; } };
|
||||
assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(snapshot, timelineTamperedStore)).ok, false);
|
||||
const swappedManifests = snapshot.artifactManifests.map((manifest, index) => ({ ...manifest, missingBeforeMigration: { ...gaps[(index + 1) % gaps.length] } }));
|
||||
const swappedUnsigned = { ...snapshot, artifactManifests: swappedManifests };
|
||||
const swappedSnapshot = { ...swappedUnsigned, manifestContentHash: monitorSnapshotContentHash(swappedUnsigned) };
|
||||
assert.equal(verifyMonitorSqliteSnapshot(swappedSnapshot).ok, false);
|
||||
const recoveredPath = join(directory, gaps[0].stateDir, "recovered.bin");
|
||||
mkdirSync(join(recoveredPath, ".."), { recursive: true });
|
||||
writeFileSync(recoveredPath, "recovered\n");
|
||||
await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-recovered");
|
||||
rmSync(recoveredPath);
|
||||
assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], artifactCount: 2 }, gaps[1]] }, join(directory, "changed.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid");
|
||||
assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], stateDir: "wrong-state-dir" }, gaps[1]] }, join(directory, "wrong-state-dir.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid");
|
||||
assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [{ ...gaps[0], reportSha256: `sha256:${"c".repeat(64)}` }, gaps[1]] }, join(directory, "wrong-report.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-identity-invalid");
|
||||
assert.throws(() => exportMonitorSqliteSnapshot({ ...source, legacyArtifactGaps: [...gaps, { runId: "third", artifactCount: 1, stateDir: "missing", reportSha256: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }] }, join(directory, "third.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-unmatched");
|
||||
const unexpectedArtifact = Buffer.from("unexpected\n");
|
||||
const unexpectedPath = join(directory, gaps[0].stateDir, "extra.bin");
|
||||
mkdirSync(join(unexpectedPath, ".."), { recursive: true });
|
||||
writeFileSync(unexpectedPath, unexpectedArtifact);
|
||||
const locatorDatabase = new Database(sqlitePath);
|
||||
locatorDatabase.run("CREATE TABLE artifact_locators (run_id TEXT, owner_pvc TEXT, state_dir TEXT, relative_path TEXT, sha256 TEXT, size_bytes INTEGER)");
|
||||
locatorDatabase.run("INSERT INTO artifact_locators VALUES (?, ?, ?, ?, ?, ?)", [gaps[0].runId, "fixture-pvc", gaps[0].stateDir, "extra.bin", `sha256:${createHash("sha256").update(unexpectedArtifact).digest("hex")}`, unexpectedArtifact.byteLength]);
|
||||
locatorDatabase.close();
|
||||
assert.throws(() => exportMonitorSqliteSnapshot(source, join(directory, "unexpected-locator.json")), (error: unknown) => (error as { code?: string }).code === "monitor-migration-artifact-gap-locator-unexpected");
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("empty normal artifact directories freeze owner state and normal manifests cannot be exchanged", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "monitor-empty-artifact-"));
|
||||
try {
|
||||
const sqlitePath = join(directory, "index.sqlite");
|
||||
const emptyStateDir = ".state/empty";
|
||||
const fullStateDir = ".state/full";
|
||||
mkdirSync(join(directory, emptyStateDir), { recursive: true });
|
||||
const fullDirectory = join(directory, fullStateDir);
|
||||
mkdirSync(join(fullDirectory, "analysis"), { recursive: true });
|
||||
const report = Buffer.from("report\n");
|
||||
writeFileSync(join(fullDirectory, "analysis", "report.json"), report);
|
||||
const database = new Database(sqlitePath);
|
||||
database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)");
|
||||
database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", ["empty", emptyStateDir, 0, "", "2026-07-12T00:00:00.000Z"]);
|
||||
database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", ["full", fullStateDir, 1, `sha256:${createHash("sha256").update(report).digest("hex")}`, "2026-07-12T00:00:00.000Z"]);
|
||||
database.close();
|
||||
const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] };
|
||||
const snapshot = exportMonitorSqliteSnapshot(source, join(directory, "snapshot.json"));
|
||||
const parsedSnapshot = JSON.parse(readFileSync(join(directory, "snapshot.json"), "utf8"));
|
||||
const roundtripStore = createInMemoryMonitorCentralStore();
|
||||
await importMonitorSqliteSnapshot(parsedSnapshot, roundtripStore);
|
||||
assert.equal((await verifyMonitorSqliteSnapshotAgainstStore(parsedSnapshot, roundtripStore)).ok, true);
|
||||
const emptyManifest = snapshot.artifactManifests.find((manifest) => manifest.runId === "empty");
|
||||
assert.deepEqual({ artifactCount: emptyManifest?.artifactCount, stateDir: emptyManifest?.stateDir, ownerPvc: emptyManifest?.ownerPvc, locatorCount: emptyManifest?.locators.length }, { artifactCount: 0, stateDir: emptyStateDir, ownerPvc: "fixture-pvc", locatorCount: 0 });
|
||||
const swappedManifests = snapshot.artifactManifests.map((manifest, index, manifests) => ({ ...manifests[(index + 1) % manifests.length], runId: manifest.runId }));
|
||||
const swappedUnsigned = { ...snapshot, artifactManifests: swappedManifests };
|
||||
assert.equal(verifyMonitorSqliteSnapshot({ ...swappedUnsigned, manifestContentHash: monitorSnapshotContentHash(swappedUnsigned) }).ok, false);
|
||||
const emptyDirectory = join(directory, emptyStateDir);
|
||||
mkdirSync(emptyDirectory, { recursive: true });
|
||||
writeFileSync(join(emptyDirectory, "added-after-export.bin"), "added\n");
|
||||
await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid");
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("normal report locator hash cannot drift from its SQLite row after a rehashed snapshot", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "monitor-report-locator-binding-"));
|
||||
try {
|
||||
const sqlitePath = join(directory, "index.sqlite");
|
||||
const stateDir = ".state/run";
|
||||
const reportPath = join(directory, stateDir, "analysis", "report.json");
|
||||
const reportA = Buffer.from("report A\n");
|
||||
const reportB = Buffer.from("report B with different size\n");
|
||||
mkdirSync(join(reportPath, ".."), { recursive: true });
|
||||
writeFileSync(reportPath, reportA);
|
||||
const reportAHash = `sha256:${createHash("sha256").update(reportA).digest("hex")}`;
|
||||
const reportBHash = `sha256:${createHash("sha256").update(reportB).digest("hex")}`;
|
||||
const database = new Database(sqlitePath);
|
||||
database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)");
|
||||
database.run("CREATE TABLE artifact_locators (run_id TEXT, owner_pvc TEXT, state_dir TEXT, relative_path TEXT, sha256 TEXT, size_bytes INTEGER)");
|
||||
database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", ["run", stateDir, 1, reportAHash, "2026-07-12T00:00:00.000Z"]);
|
||||
database.run("INSERT INTO artifact_locators VALUES (?, ?, ?, ?, ?, ?)", ["run", "fixture-pvc", stateDir, "analysis/report.json", reportAHash, reportA.byteLength]);
|
||||
database.close();
|
||||
const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] };
|
||||
const snapshot = exportMonitorSqliteSnapshot(source, join(directory, "snapshot.json"));
|
||||
writeFileSync(reportPath, reportB);
|
||||
const tamperedUnsigned = { ...snapshot, artifactManifests: snapshot.artifactManifests.map((manifest) => ({ ...manifest, locators: manifest.locators.map((locator) => ({ ...locator, sha256: reportBHash, sizeBytes: reportB.byteLength })) })) };
|
||||
const tampered = { ...tamperedUnsigned, manifestContentHash: monitorSnapshotContentHash(tamperedUnsigned) };
|
||||
assert.equal(verifyMonitorSqliteSnapshot(tampered).ok, false);
|
||||
await assert.rejects(() => importMonitorSqliteSnapshot(tampered, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-snapshot-invalid");
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("mixed per-run locator rows keep fallback runs valid and reject rebinds", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "monitor-mixed-locator-rows-"));
|
||||
try {
|
||||
const sqlitePath = join(directory, "index.sqlite");
|
||||
const reports = new Map([["a", Buffer.from("report A\n")], ["b", Buffer.from("report B\n")]]);
|
||||
const hashes = new Map([...reports].map(([runId, report]) => [runId, `sha256:${createHash("sha256").update(report).digest("hex")}`]));
|
||||
for (const [runId, report] of reports) {
|
||||
const reportPath = join(directory, ".state", runId, "analysis", "report.json");
|
||||
mkdirSync(join(reportPath, ".."), { recursive: true });
|
||||
writeFileSync(reportPath, report);
|
||||
}
|
||||
const database = new Database(sqlitePath);
|
||||
database.run("CREATE TABLE runs (id TEXT, state_dir TEXT, artifact_count INTEGER, report_json_sha256 TEXT, updated_at TEXT)");
|
||||
database.run("CREATE TABLE artifact_locators (run_id TEXT, owner_pvc TEXT, state_dir TEXT, relative_path TEXT, sha256 TEXT, size_bytes INTEGER)");
|
||||
for (const [runId, report] of reports) database.run("INSERT INTO runs VALUES (?, ?, ?, ?, ?)", [runId, `.state/${runId}`, 1, hashes.get(runId), "2026-07-12T00:00:00.000Z"]);
|
||||
database.run("INSERT INTO artifact_locators VALUES (?, ?, ?, ?, ?, ?)", ["a", "fixture-pvc", ".state/a", "analysis/report.json", hashes.get("a"), reports.get("a")?.byteLength]);
|
||||
database.close();
|
||||
const source = { sentinelId: "fixture", node: "NC01", lane: "v03", path: sqlitePath, runtimeConfigRef: "fixture#runtime", stateRoot: directory, ownerPvc: "fixture-pvc", artifactSources: [{ ownerPvc: "fixture-pvc", mountPath: directory }] };
|
||||
const snapshot = exportMonitorSqliteSnapshot(source, join(directory, "snapshot.json"));
|
||||
assert.equal(verifyMonitorSqliteSnapshot(snapshot).ok, true);
|
||||
const manifestA = snapshot.artifactManifests.find((manifest) => manifest.runId === "a")!;
|
||||
const manifestB = snapshot.artifactManifests.find((manifest) => manifest.runId === "b")!;
|
||||
const reboundUnsigned = { ...snapshot, artifactManifests: snapshot.artifactManifests.map((manifest) => manifest.runId === "a" ? { ...manifest, locators: manifestB.locators } : manifest) };
|
||||
assert.equal(verifyMonitorSqliteSnapshot({ ...reboundUnsigned, manifestContentHash: monitorSnapshotContentHash(reboundUnsigned) }).ok, false);
|
||||
const swappedUnsigned = { ...snapshot, artifactManifests: [{ ...manifestB, runId: "a" }, { ...manifestA, runId: "b" }] };
|
||||
assert.equal(verifyMonitorSqliteSnapshot({ ...swappedUnsigned, manifestContentHash: monitorSnapshotContentHash(swappedUnsigned) }).ok, false);
|
||||
writeFileSync(join(directory, ".state", "b", "added-after-export.bin"), "added\n");
|
||||
await assert.rejects(() => importMonitorSqliteSnapshot(snapshot, createInMemoryMonitorCentralStore()), (error: unknown) => (error as { code?: string }).code === "monitor-migration-frozen-artifact-invalid");
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("artifact count without a registry-backed locator fails closed", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "monitor-artifact-count-"));
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user