From e37b4b2afefe265f5b1bd48e3569b27ffca532de Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 02:30:55 +0000 Subject: [PATCH 1/3] fix: expose migration job failures --- scripts/src/cicd-delivery-authority.test.ts | 8 ++++++++ scripts/src/hwlab-node-web-probe-monitor.test.ts | 3 +++ scripts/src/hwlab-node-web-probe-monitor.ts | 2 +- scripts/src/hwlab-node-web-sentinel-cicd.ts | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index 9ffa5397..a0b2debf 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -313,6 +313,14 @@ describe("migrated CLI 执行 guard 与提示清理", () => { expect(JSON.stringify(failure)).not.toContain(credential); }); + test("migration Job verify 的结构化失败直接投影语义 code 和 message", () => { + const failure = migrationJobFailed("web-probe sentinel migration job-run", { jobName: "migration-job" }, "failed", { ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "timeline count does not match snapshot", stack: "DATABASE_URL=postgres://user:PW123@host/db" } }, { podPhase: "Failed", containers: [{ name: "verify", phase: "terminated", exitCode: 1, reason: "Error", message: "-" }], logsTail: '{"ok":false,"command":"web-probe sentinel migration verify","error":{"code":"monitor-migration-pg-verify-mismatch","message":"timeline count does not match snapshot","stack":"secret stack"}}' }) as { projection?: Record }; + expect(failure.projection?.phase).toBe("failed"); + expect(failure.projection?.result).toBeUndefined(); + expect(failure.projection?.failure).toMatchObject({ phase: "Failed", container: "verify", exitCode: 1, code: "monitor-migration-pg-verify-mismatch", message: "timeline count does not match snapshot", valuesRedacted: true }); + expect(JSON.stringify(failure.projection)).not.toMatch(/PW123|stack|runs=-|findings=-/iu); + }); + test("migration Job 业务失败、创建失败和超时均只返回安全 failure 投影", () => { const business = migrationJobFailed("migration", { jobName: "job" }, "result-failed", { ok: false, error: { message: "https://user:PW123@host failed", stack: "secret stack" } }, { podPhase: "Succeeded", logsTail: "Bearer abc" }) as { projection?: Record }; const createFailed = migrationJobFailed("migration", { jobName: "job" }, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "DATABASE_URL=https://user:PW123@host/db" }) as { projection?: Record }; diff --git a/scripts/src/hwlab-node-web-probe-monitor.test.ts b/scripts/src/hwlab-node-web-probe-monitor.test.ts index a0a254eb..d90ba1cc 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.test.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.test.ts @@ -31,9 +31,12 @@ test("explicit migration helper renders ordered read-only export import verify J expect(job.spec.template.metadata.labels["app.kubernetes.io/component"]).toBe("migration"); expect(job.spec.template.spec.initContainers.map((container: any) => container.name)).toEqual(["export", "import"]); expect(job.spec.template.spec.initContainers[0].args).toEqual(expect.arrayContaining(["export", "--snapshot", "/migration/snapshot.json"])); + expect(job.spec.template.spec.initContainers[0].args).toContain("--json"); expect(job.spec.template.spec.initContainers[0].args).not.toContain("--confirm"); expect(job.spec.template.spec.initContainers[1].args).toEqual(expect.arrayContaining(["scripts/cli.ts", "import", "--snapshot", "/migration/snapshot.json", "--confirm"])); + expect(job.spec.template.spec.initContainers[1].args).toContain("--json"); expect(job.spec.template.spec.containers[0].args).toEqual(expect.arrayContaining(["scripts/cli.ts", "verify", "--snapshot", "/migration/snapshot.json"])); + expect(job.spec.template.spec.containers[0].args).toContain("--json"); for (const container of [...job.spec.template.spec.initContainers, ...job.spec.template.spec.containers]) { expect(container.volumeMounts).toEqual(expect.arrayContaining([expect.objectContaining({ name: "legacy-sqlite", mountPath: "/var/lib/web-probe-sentinel-nc01", readOnly: true }), expect.objectContaining({ name: "snapshot", mountPath: "/migration" })])); } diff --git a/scripts/src/hwlab-node-web-probe-monitor.ts b/scripts/src/hwlab-node-web-probe-monitor.ts index af2eaea7..6ea3d56a 100644 --- a/scripts/src/hwlab-node-web-probe-monitor.ts +++ b/scripts/src/hwlab-node-web-probe-monitor.ts @@ -29,7 +29,7 @@ export function releaseAMonitorMigrationJob(spec: HwlabRuntimeLaneSpec, imageRef const env = [{ name: "DATABASE_URL", valueFrom: { secretKeyRef: { name: textAt(secret, "name", resolved.ref), key: textAt(secret, "key", resolved.ref) } } }, { name: "MONITOR_CENTRAL_PG_POOL_MAX", value: String(integerAt(pool, "max", resolved.ref)) }, { name: "MONITOR_CENTRAL_PG_IDLE_TIMEOUT_SECONDS", value: String(integerAt(pool, "idleTimeoutSeconds", resolved.ref)) }]; const cli = ["scripts/cli.ts", "web-probe", "sentinel", "migration"]; const sourceArgs = ["--node", spec.nodeId, "--lane", spec.lane, "--sentinel", resolvedSource.sentinelId, "--snapshot", "/migration/snapshot.json"]; - return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef, options.nameSuffix), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs], env, volumeMounts: mounts }], volumes: [...sourceVolumes.map((item) => item.source.hostPath === undefined ? { name: item.name, persistentVolumeClaim: { claimName: item.source.ownerPvc, readOnly: true } } : { name: item.name, hostPath: { path: item.source.hostPath, type: "Directory" } }), { name: "snapshot", emptyDir: {} }] } } } }; + return { apiVersion: "batch/v1", kind: "Job", metadata: { name: versionedJobName(textAt(job, "name", resolved.ref), imageRef, options.nameSuffix), namespace, labels: migrationLabels }, spec: { backoffLimit: 0, template: { metadata: { labels: migrationLabels }, spec: { restartPolicy: "Never", initContainers: [{ name: "export", image: imageRef, command: ["bun"], args: [...cli, "export", ...sourceArgs, "--json"], volumeMounts: mounts }, { name: "import", image: imageRef, command: ["bun"], args: [...cli, "import", ...sourceArgs, "--confirm", "--json"], env, volumeMounts: mounts }], containers: [{ name: "verify", image: imageRef, command: ["bun"], args: [...cli, "verify", ...sourceArgs, "--json"], env, volumeMounts: mounts }], volumes: [...sourceVolumes.map((item) => item.source.hostPath === undefined ? { name: item.name, persistentVolumeClaim: { claimName: item.source.ownerPvc, readOnly: true } } : { name: item.name, hostPath: { path: item.source.hostPath, type: "Directory" } }), { name: "snapshot", emptyDir: {} }] } } } }; } export function releaseAMonitorMigrationPool(spec: HwlabRuntimeLaneSpec, resolvedSource: MonitorSqliteSource): { readonly poolMax: number; readonly idleTimeoutSeconds: number } { diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 9a9f6427..e4bc8a4a 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -266,7 +266,7 @@ export function migrationJobFailureSummary(probe: Record, paylo const failed = containers.find((item) => item.exitCode !== null && item.exitCode !== 0) ?? containers[0] ?? null; const logsTail = redactedMigrationJobText(String(probe.logsTail ?? "")); const nestedError = isRecord(payload.error) ? payload.error : null; - return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, status: failed?.reason ?? stringAtNullable(nestedError, "code") ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactedMigrationJobText(stringAtNullable(nestedError, "message") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted"), logsTail, valuesRedacted: true }; + return { phase: stringAtNullable(probe, "podPhase") ?? "Failed", container: failed?.name ?? null, exitCode: failed?.exitCode ?? null, code: stringAtNullable(nestedError, "code") ?? null, status: failed?.reason ?? stringAtNullable(nestedError, "code") ?? stringAtNullable(probe, "reason") ?? "JobFailed", message: redactedMigrationJobText(stringAtNullable(nestedError, "message") ?? stringAtNullable(payload, "message") ?? failed?.message ?? "migration Job failed before a structured command result was emitted"), logsTail, valuesRedacted: true }; } function redactedMigrationJobText(value: string): string { From 8d091e3de89cb0ecff4e98ad5856539832858bc6 Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 02:46:45 +0000 Subject: [PATCH 2/3] fix: parse migration job envelopes --- scripts/src/cicd-delivery-authority.test.ts | 18 +++++++ .../src/hwlab-node-web-sentinel-cicd-jobs.ts | 49 ++++++++++++++----- scripts/src/hwlab-node-web-sentinel-cicd.ts | 17 ++++++- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index a0b2debf..82acd31a 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -24,6 +24,7 @@ import { guardedSentinelDeliveryAction, migrationJobFailed, migrationJobFailureSummary, + migrationJobPayloadFromProbe, runWebProbeSentinelCommand, sameMigrationSource, webProbeSentinelDeliveryAuthorityFromCatalog, @@ -321,6 +322,23 @@ describe("migrated CLI 执行 guard 与提示清理", () => { expect(JSON.stringify(failure.projection)).not.toMatch(/PW123|stack|runs=-|findings=-/iu); }); + test("migration Job 只从目标容器读取多行 CLI envelope，忽略合并日志尾随结果", () => { + const verifyFailure = JSON.stringify({ ok: false, data: { ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } } }, null, 2); + const importSuccess = JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration import" } }, null, 2); + const probe = { failed: true, podPhase: "Failed", containers: [{ name: "verify", phase: "terminated", exitCode: 1, reason: "Error", message: "-" }], containerLogs: { verify: `verify prefix\n${verifyFailure}`, import: importSuccess }, logsTail: `${verifyFailure}\n${importSuccess}` }; + const payload = migrationJobPayloadFromProbe(probe); + expect(payload).toMatchObject({ ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } }); + const failure = migrationJobFailed("migration", { jobName: "job" }, "failed", payload, probe) as { projection?: Record }; + expect(failure.projection?.failure).toMatchObject({ container: "verify", code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" }); + expect(failure.projection?.result).toBeUndefined(); + }); + + test("migration Job 成功固定读取 verify，缺结构化 verify 结果显式失败", () => { + const importSuccess = JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration import" } }, null, 2); + expect(migrationJobPayloadFromProbe({ succeeded: true, containers: [{ name: "verify", phase: "terminated", exitCode: 0 }], containerLogs: { import: importSuccess }, logsTail: importSuccess })).toMatchObject({ ok: false, error: { code: "monitor-migration-job-result-missing" } }); + expect(migrationJobPayloadFromProbe({ succeeded: true, containers: [{ name: "verify", phase: "terminated", exitCode: 0 }], containerLogs: { verify: JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration verify" } }), import: importSuccess } })).toMatchObject({ ok: true, command: "web-probe sentinel migration verify" }); + }); + test("migration Job 业务失败、创建失败和超时均只返回安全 failure 投影", () => { const business = migrationJobFailed("migration", { jobName: "job" }, "result-failed", { ok: false, error: { message: "https://user:PW123@host failed", stack: "secret stack" } }, { podPhase: "Succeeded", logsTail: "Bearer abc" }) as { projection?: Record }; const createFailed = migrationJobFailed("migration", { jobName: "job" }, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "DATABASE_URL=https://user:PW123@host/db" }) as { projection?: Record }; diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts index d6280444..a19484ad 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -8,6 +8,7 @@ import { arrayAt, compactCommand, finiteNumberOrNull, + isRecord, monitorWebBuildkitStatePlan, nonEmptyString, numberAt, @@ -941,11 +942,13 @@ export function probeK8sJobScript(namespace: string, jobName: string): string { "containers_b64=''", "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); reason=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.reason}' 2>/dev/null); containers_b64=$(kubectl -n \"$namespace\" get pod \"$pod\" -o json 2>/dev/null | node -e 'let body=\"\";process.stdin.on(\"data\",c=>body+=c).on(\"end\",()=>{try{const pod=JSON.parse(body);const all=[...(pod.status?.initContainerStatuses||[]),...(pod.status?.containerStatuses||[])];console.log(Buffer.from(JSON.stringify(all.map(item=>({name:item.name,phase:item.state?.terminated?\"terminated\":item.state?.waiting?\"waiting\":item.state?.running?\"running\":\"unknown\",exitCode:item.state?.terminated?.exitCode??null,reason:item.state?.terminated?.reason??item.state?.waiting?.reason??null,message:item.state?.terminated?.message??item.state?.waiting?.message??null})))).toString(\"base64\"))}catch{console.log(\"\")}})' | tr -d '\\n'); fi", "logs_tail=''", - "if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi", - "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$reason\" \"$containers_b64\" \"$logs_tail\" <<'NODE'", - "const [succeeded, failed, active, pod, podPhase, reason, containersB64, logsB64] = process.argv.slice(2);", + "container_logs_b64=''", + "if [ -n \"$pod\" ]; then logs_tail=$(kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null | tail -c 6000 | base64 | tr -d '\\n'); container_logs_b64=$(for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' 2>/dev/null); do printf '%s\\t%s\\n' \"$container\" \"$(kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null | tail -c 6000 | base64 | tr -d '\\n')\"; done | base64 | tr -d '\\n'); fi", + "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$reason\" \"$containers_b64\" \"$logs_tail\" \"$container_logs_b64\" <<'NODE'", + "const [succeeded, failed, active, pod, podPhase, reason, containersB64, logsB64, containerLogsB64] = process.argv.slice(2);", "let containers=[];try{containers=JSON.parse(Buffer.from(containersB64 || '', 'base64').toString('utf8'))}catch{}", - "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, reason: reason || null, containers, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", + "const containerLogs={};for(const line of Buffer.from(containerLogsB64 || '', 'base64').toString('utf8').split(/\\r?\\n/)){const tab=line.indexOf('\\t');if(tab>0)containerLogs[line.slice(0,tab)]=Buffer.from(line.slice(tab+1),'base64').toString('utf8')}", + "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, reason: reason || null, containers, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), containerLogs, valuesRedacted: true }));", "NODE", ].join("\n"); } @@ -977,14 +980,38 @@ function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string } export function sentinelPayloadFromLogs(logsTail: string): Record { - const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); - for (let index = lines.length - 1; index >= 0; index -= 1) { - const line = lines[index]; - if (!line.startsWith("{") || !line.endsWith("}")) continue; - const parsed = parseJsonObject(line); - if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed; + const parsed = lastJsonObjectFromText(logsTail); + if (parsed === null) return {}; + const data = isRecord(parsed.data) ? parsed.data : null; + return data !== null && (data.ok === true || data.ok === false) ? data : parsed.ok === true || parsed.ok === false ? parsed : {}; +} + +function lastJsonObjectFromText(text: string): Record | null { + let last: Record | null = null; + for (let start = text.indexOf("{"); start >= 0; start = text.indexOf("{", start + 1)) { + const candidate = jsonObjectAt(text, start); + if (candidate !== null && (candidate.ok === true || candidate.ok === false)) last = candidate; } - return {}; + return last; +} + +function jsonObjectAt(text: string, start: number): Record | null { + let depth = 0; + let inString = false; + let escaped = false; + for (let index = start; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === "\"") inString = false; + continue; + } + if (char === "\"") { inString = true; continue; } + if (char === "{") { depth += 1; continue; } + if (char === "}" && --depth === 0) return parseJsonObject(text.slice(start, index + 1)); + } + return null; } function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult { diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index e4bc8a4a..519df4ca 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -248,7 +248,7 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract): Record { + const failedContainer = migrationFailedContainerName(probe); + const container = probe.succeeded === true ? "verify" : failedContainer; + const expectedCommand = container === "export" ? "web-probe sentinel migration export" : container === "import" ? "web-probe sentinel migration import" : container === "verify" ? "web-probe sentinel migration verify" : null; + const containerLogs = isRecord(probe.containerLogs) ? probe.containerLogs : {}; + const payload = container === null ? {} : sentinelPayloadFromLogs(String(containerLogs[container] ?? "")); + if (expectedCommand !== null && payload.command === expectedCommand && (payload.ok === true || payload.ok === false)) return payload; + return { ok: false, error: { code: "monitor-migration-job-result-missing", message: expectedCommand === null ? "migration Job failed without an identifiable terminated container result" : `migration Job ${container} container did not emit the expected structured result` }, valuesRedacted: true }; +} + +function migrationFailedContainerName(probe: Record): string | null { + const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord) : []; + return stringAtNullable(containers.find((container) => finiteNumberOrNull(container.exitCode) !== null && finiteNumberOrNull(container.exitCode) !== 0), "name"); +} + export function migrationJobFailed(command: string, plan: Record, phase: string, payload: Record, probe: Record, next?: Record): RenderedCliResult { return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase, failure: migrationJobFailureSummary(probe, payload), next, valuesRedacted: true }); } From 0757464fb63389c96138d66cb87cdaeb6584566e Mon Sep 17 00:00:00 2001 From: AgentRun Codex Date: Mon, 13 Jul 2026 02:54:31 +0000 Subject: [PATCH 3/3] fix: accept migration command arguments --- scripts/src/cicd-delivery-authority.test.ts | 11 +++++++++-- scripts/src/hwlab-node-web-sentinel-cicd.ts | 6 +++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index 82acd31a..ba3e0ee8 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -323,11 +323,11 @@ describe("migrated CLI 执行 guard 与提示清理", () => { }); test("migration Job 只从目标容器读取多行 CLI envelope，忽略合并日志尾随结果", () => { - const verifyFailure = JSON.stringify({ ok: false, data: { ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } } }, null, 2); + const verifyFailure = JSON.stringify({ ok: false, command: "web-probe sentinel migration verify --node NC01 --lane v03 --json", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } }, null, 2); const importSuccess = JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration import" } }, null, 2); const probe = { failed: true, podPhase: "Failed", containers: [{ name: "verify", phase: "terminated", exitCode: 1, reason: "Error", message: "-" }], containerLogs: { verify: `verify prefix\n${verifyFailure}`, import: importSuccess }, logsTail: `${verifyFailure}\n${importSuccess}` }; const payload = migrationJobPayloadFromProbe(probe); - expect(payload).toMatchObject({ ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } }); + expect(payload).toMatchObject({ ok: false, command: "web-probe sentinel migration verify --node NC01 --lane v03 --json", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } }); const failure = migrationJobFailed("migration", { jobName: "job" }, "failed", payload, probe) as { projection?: Record }; expect(failure.projection?.failure).toMatchObject({ container: "verify", code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" }); expect(failure.projection?.result).toBeUndefined(); @@ -339,6 +339,13 @@ describe("migrated CLI 执行 guard 与提示清理", () => { expect(migrationJobPayloadFromProbe({ succeeded: true, containers: [{ name: "verify", phase: "terminated", exitCode: 0 }], containerLogs: { verify: JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration verify" } }), import: importSuccess } })).toMatchObject({ ok: true, command: "web-probe sentinel migration verify" }); }); + test("migration Job 接受带参数的 direct export 包络并拒绝相似 command 前缀", () => { + const direct = (command: string) => JSON.stringify({ ok: false, command, error: { code: "monitor-migration-artifact-path-invalid", message: "export artifact is unavailable" } }, null, 2); + const failedExport = (command: string) => migrationJobPayloadFromProbe({ failed: true, containers: [{ name: "export", phase: "terminated", exitCode: 1 }], containerLogs: { export: direct(command) } }); + expect(failedExport("web-probe sentinel migration export --node NC01 --lane v03 --json")).toMatchObject({ ok: false, command: "web-probe sentinel migration export --node NC01 --lane v03 --json", error: { code: "monitor-migration-artifact-path-invalid" } }); + for (const command of ["web-probe sentinel migration export-other --json", "web-probe sentinel migration exportfoo --json", "web-probe sentinel migration import --json"]) expect(failedExport(command)).toMatchObject({ ok: false, error: { code: "monitor-migration-job-result-missing" } }); + }); + test("migration Job 业务失败、创建失败和超时均只返回安全 failure 投影", () => { const business = migrationJobFailed("migration", { jobName: "job" }, "result-failed", { ok: false, error: { message: "https://user:PW123@host failed", stack: "secret stack" } }, { podPhase: "Succeeded", logsTail: "Bearer abc" }) as { projection?: Record }; const createFailed = migrationJobFailed("migration", { jobName: "job" }, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "DATABASE_URL=https://user:PW123@host/db" }) as { projection?: Record }; diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 519df4ca..e8146a9b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -263,10 +263,14 @@ export function migrationJobPayloadFromProbe(probe: Record): Re const expectedCommand = container === "export" ? "web-probe sentinel migration export" : container === "import" ? "web-probe sentinel migration import" : container === "verify" ? "web-probe sentinel migration verify" : null; const containerLogs = isRecord(probe.containerLogs) ? probe.containerLogs : {}; const payload = container === null ? {} : sentinelPayloadFromLogs(String(containerLogs[container] ?? "")); - if (expectedCommand !== null && payload.command === expectedCommand && (payload.ok === true || payload.ok === false)) return payload; + if (expectedCommand !== null && migrationPayloadCommandMatches(payload.command, expectedCommand) && (payload.ok === true || payload.ok === false)) return payload; return { ok: false, error: { code: "monitor-migration-job-result-missing", message: expectedCommand === null ? "migration Job failed without an identifiable terminated container result" : `migration Job ${container} container did not emit the expected structured result` }, valuesRedacted: true }; } +function migrationPayloadCommandMatches(value: unknown, expected: string): boolean { + return typeof value === "string" && (value === expected || value.startsWith(`${expected} `)); +} + function migrationFailedContainerName(probe: Record): string | null { const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord) : []; return stringAtNullable(containers.find((container) => finiteNumberOrNull(container.exitCode) !== null && finiteNumberOrNull(container.exitCode) !== 0), "name");