mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: parse migration job envelopes

Browse Source
This commit is contained in:
AgentRun Codex
2026-07-13 02:46:45 +00:00
parent e37b4b2afe
commit 8d091e3de8
3 changed files with 72 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/cicd-delivery-authority.test.ts
+18
View File
@@ -24,6 +24,7 @@ import {
guardedSentinelDeliveryAction,
migrationJobFailed,
migrationJobFailureSummary,
migrationJobPayloadFromProbe,
runWebProbeSentinelCommand,
sameMigrationSource,
webProbeSentinelDeliveryAuthorityFromCatalog,
@@ -321,6 +322,23 @@ describe("migrated CLI 执行 guard 与提示清理", () => {
expect(JSON.stringify(failure.projection)).not.toMatch(/PW123|stack|runs=-|findings=-/iu);
});
test("migration Job 只从目标容器读取多行 CLI envelope，忽略合并日志尾随结果", () => {
const verifyFailure = JSON.stringify({ ok: false, data: { ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } } }, null, 2);
const importSuccess = JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration import" } }, null, 2);
const probe = { failed: true, podPhase: "Failed", containers: [{ name: "verify", phase: "terminated", exitCode: 1, reason: "Error", message: "-" }], containerLogs: { verify: `verify prefix\n${verifyFailure}`, import: importSuccess }, logsTail: `${verifyFailure}\n${importSuccess}` };
const payload = migrationJobPayloadFromProbe(probe);
expect(payload).toMatchObject({ ok: false, command: "web-probe sentinel migration verify", error: { code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" } });
const failure = migrationJobFailed("migration", { jobName: "job" }, "failed", payload, probe) as { projection?: Record<string, unknown> };
expect(failure.projection?.failure).toMatchObject({ container: "verify", code: "monitor-migration-pg-verify-mismatch", message: "verify saw {braces} and an escaped \"quote\"" });
expect(failure.projection?.result).toBeUndefined();
});
test("migration Job 成功固定读取 verify，缺结构化 verify 结果显式失败", () => {
const importSuccess = JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration import" } }, null, 2);
expect(migrationJobPayloadFromProbe({ succeeded: true, containers: [{ name: "verify", phase: "terminated", exitCode: 0 }], containerLogs: { import: importSuccess }, logsTail: importSuccess })).toMatchObject({ ok: false, error: { code: "monitor-migration-job-result-missing" } });
expect(migrationJobPayloadFromProbe({ succeeded: true, containers: [{ name: "verify", phase: "terminated", exitCode: 0 }], containerLogs: { verify: JSON.stringify({ ok: true, data: { ok: true, command: "web-probe sentinel migration verify" } }), import: importSuccess } })).toMatchObject({ ok: true, command: "web-probe sentinel migration verify" });
});
test("migration Job 业务失败、创建失败和超时均只返回安全 failure 投影", () => {
const business = migrationJobFailed("migration", { jobName: "job" }, "result-failed", { ok: false, error: { message: "https://user:PW123@host failed", stack: "secret stack" } }, { podPhase: "Succeeded", logsTail: "Bearer abc" }) as { projection?: Record<string, unknown> };
const createFailed = migrationJobFailed("migration", { jobName: "job" }, "create-failed", {}, { podPhase: "CreateFailed", logsTail: "DATABASE_URL=https://user:PW123@host/db" }) as { projection?: Record<string, unknown> };
scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts
+38 -11
View File
@@ -8,6 +8,7 @@ import {
arrayAt,
compactCommand,
finiteNumberOrNull,
isRecord,
monitorWebBuildkitStatePlan,
nonEmptyString,
numberAt,
@@ -941,11 +942,13 @@ export function probeK8sJobScript(namespace: string, jobName: string): string {
"containers_b64=''",
"if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); reason=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.reason}' 2>/dev/null); containers_b64=$(kubectl -n \"$namespace\" get pod \"$pod\" -o json 2>/dev/null | node -e 'let body=\"\";process.stdin.on(\"data\",c=>body+=c).on(\"end\",()=>{try{const pod=JSON.parse(body);const all=[...(pod.status?.initContainerStatuses||[]),...(pod.status?.containerStatuses||[])];console.log(Buffer.from(JSON.stringify(all.map(item=>({name:item.name,phase:item.state?.terminated?\"terminated\":item.state?.waiting?\"waiting\":item.state?.running?\"running\":\"unknown\",exitCode:item.state?.terminated?.exitCode??null,reason:item.state?.terminated?.reason??item.state?.waiting?.reason??null,message:item.state?.terminated?.message??item.state?.waiting?.message??null})))).toString(\"base64\"))}catch{console.log(\"\")}})' | tr -d '\\n'); fi",
"logs_tail=''",
"if [ -n \"$pod\" ]; then logs_tail=$({ kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null || true; for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null); do kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=60 2>/dev/null || true; done; } | tail -c 6000 | base64 | tr -d '\\n'); fi",
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$reason\" \"$containers_b64\" \"$logs_tail\" <<'NODE'",
"const [succeeded, failed, active, pod, podPhase, reason, containersB64, logsB64] = process.argv.slice(2);",
"container_logs_b64=''",
"if [ -n \"$pod\" ]; then logs_tail=$(kubectl -n \"$namespace\" logs \"$pod\" --all-containers=true --tail=80 2>/dev/null | tail -c 6000 | base64 | tr -d '\\n'); container_logs_b64=$(for container in $(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}' 2>/dev/null); do printf '%s\\t%s\\n' \"$container\" \"$(kubectl -n \"$namespace\" logs \"$pod\" -c \"$container\" --tail=80 2>/dev/null | tail -c 6000 | base64 | tr -d '\\n')\"; done | base64 | tr -d '\\n'); fi",
"node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$reason\" \"$containers_b64\" \"$logs_tail\" \"$container_logs_b64\" <<'NODE'",
"const [succeeded, failed, active, pod, podPhase, reason, containersB64, logsB64, containerLogsB64] = process.argv.slice(2);",
"let containers=[];try{containers=JSON.parse(Buffer.from(containersB64 || '', 'base64').toString('utf8'))}catch{}",
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, reason: reason || null, containers, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));",
"const containerLogs={};for(const line of Buffer.from(containerLogsB64 || '', 'base64').toString('utf8').split(/\\r?\\n/)){const tab=line.indexOf('\\t');if(tab>0)containerLogs[line.slice(0,tab)]=Buffer.from(line.slice(tab+1),'base64').toString('utf8')}",
"console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, reason: reason || null, containers, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), containerLogs, valuesRedacted: true }));",
"NODE",
].join("\n");
}
@@ -977,14 +980,38 @@ function probeTektonPipelineRunScript(namespace: string, pipelineRunName: string
}
export function sentinelPayloadFromLogs(logsTail: string): Record<string, unknown> {
const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
for (let index = lines.length - 1; index >= 0; index -= 1) {
const line = lines[index];
if (!line.startsWith("{") || !line.endsWith("}")) continue;
const parsed = parseJsonObject(line);
if (parsed !== null && (parsed.ok === true || parsed.ok === false)) return parsed;
const parsed = lastJsonObjectFromText(logsTail);
if (parsed === null) return {};
const data = isRecord(parsed.data) ? parsed.data : null;
return data !== null && (data.ok === true || data.ok === false) ? data : parsed.ok === true || parsed.ok === false ? parsed : {};
}
function lastJsonObjectFromText(text: string): Record<string, unknown> | null {
let last: Record<string, unknown> | null = null;
for (let start = text.indexOf("{"); start >= 0; start = text.indexOf("{", start + 1)) {
const candidate = jsonObjectAt(text, start);
if (candidate !== null && (candidate.ok === true || candidate.ok === false)) last = candidate;
}
return {};
return last;
}
function jsonObjectAt(text: string, start: number): Record<string, unknown> | null {
let depth = 0;
let inString = false;
let escaped = false;
for (let index = start; index < text.length; index += 1) {
const char = text[index];
if (inString) {
if (escaped) escaped = false;
else if (char === "\\") escaped = true;
else if (char === "\"") inString = false;
continue;
}
if (char === "\"") { inString = true; continue; }
if (char === "{") { depth += 1; continue; }
if (char === "}" && --depth === 0) return parseJsonObject(text.slice(start, index + 1));
}
return null;
}
function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult {
scripts/src/hwlab-node-web-sentinel-cicd.ts
+16 -1
View File
@@ -248,7 +248,7 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract<We
lastProbe = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", probeK8sJobScript(namespace, jobName)], repoRoot, { timeoutMs: Math.min(options.timeoutSeconds, 60) * 1000 });
const probe = resolveSentinelChildJson(lastProbe, "web-probe-sentinel-migration-job-probe").parsed ?? {};
if (probe.succeeded === true || probe.failed === true) {
const payload = sentinelPayloadFromLogs(String(probe.logsTail ?? ""));
const payload = migrationJobPayloadFromProbe(probe);
if (probe.succeeded === true && payload.ok === true) return rendered(true, command, "ok=true", { ...plan, ok: true, mutation: true, phase: "succeeded", result: payload, valuesRedacted: true });
return migrationJobFailed(command, plan, probe.succeeded === true ? "result-failed" : "failed", payload, probe, migrationJobFailureNext(state, namespace, jobName));
}
@@ -257,6 +257,21 @@ function runSentinelMigrationJob(spec: HwlabRuntimeLaneSpec, options: Extract<We
return migrationJobFailed(command, plan, "timeout", {}, { podPhase: "Timeout", logsTail: "migration Job did not reach a successful result before timeout" }, migrationJobFailureNext(state, namespace, jobName));
}
export function migrationJobPayloadFromProbe(probe: Record<string, unknown>): Record<string, unknown> {
const failedContainer = migrationFailedContainerName(probe);
const container = probe.succeeded === true ? "verify" : failedContainer;
const expectedCommand = container === "export" ? "web-probe sentinel migration export" : container === "import" ? "web-probe sentinel migration import" : container === "verify" ? "web-probe sentinel migration verify" : null;
const containerLogs = isRecord(probe.containerLogs) ? probe.containerLogs : {};
const payload = container === null ? {} : sentinelPayloadFromLogs(String(containerLogs[container] ?? ""));
if (expectedCommand !== null && payload.command === expectedCommand && (payload.ok === true || payload.ok === false)) return payload;
return { ok: false, error: { code: "monitor-migration-job-result-missing", message: expectedCommand === null ? "migration Job failed without an identifiable terminated container result" : `migration Job ${container} container did not emit the expected structured result` }, valuesRedacted: true };
}
function migrationFailedContainerName(probe: Record<string, unknown>): string | null {
const containers = Array.isArray(probe.containers) ? probe.containers.filter(isRecord) : [];
return stringAtNullable(containers.find((container) => finiteNumberOrNull(container.exitCode) !== null && finiteNumberOrNull(container.exitCode) !== 0), "name");
}
export function migrationJobFailed(command: string, plan: Record<string, unknown>, phase: string, payload: Record<string, unknown>, probe: Record<string, unknown>, next?: Record<string, string>): RenderedCliResult {
return rendered(false, command, "ok=false", { ...plan, ok: false, mutation: true, phase, failure: migrationJobFailureSummary(probe, payload), next, valuesRedacted: true });
}