From fe23d35c6e2cee352cd29682a1e89a2de9ffa347 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 10:52:15 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20PaC=20=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E5=91=BD=E4=BB=A4=E7=A9=BA=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/src/agentrun/control-plane.ts | 20 ++- scripts/src/agentrun/public-exposure.ts | 15 +- scripts/src/agentrun/utils.ts | 4 +- ...latform-infra-pac-failure-evidence.test.ts | 31 +++- .../src/platform-infra-pipelines-as-code.ts | 163 ++++++++++++++++-- 5 files changed, 208 insertions(+), 25 deletions(-) diff --git a/scripts/src/agentrun/control-plane.ts b/scripts/src/agentrun/control-plane.ts index 25a76ffc..7ecde0e8 100644 --- a/scripts/src/agentrun/control-plane.ts +++ b/scripts/src/agentrun/control-plane.ts @@ -47,6 +47,8 @@ import { compactAgentRunLaneStatusTarget, compactLaneSecretsStatus } from "./tri import { capture, captureJsonPayload, compactCapture, record, stringOrNull, timedStatusStage } from "./utils"; import { yamlLaneK3sSourceStatusScript, yamlLaneRuntimeStatusScript, yamlLaneSourceStatusScript } from "./yaml-lane"; +const agentRunStatusCaptureTimeoutMs = 20_000; + export function parseSecretSyncOptions(args: string[]): SecretSyncOptions { const base = parseConfirmOptions(args); let node: string | null = null; @@ -409,13 +411,15 @@ function giteaMirrorStatusCommand(spec: AgentRunLaneSpec): string { export async function statusPipelinesAsCodeLane(config: UniDeskConfig, options: StatusOptions, target: { configPath: string; spec: AgentRunLaneSpec }, authority: Extract): Promise> { const spec = target.spec; const consumerId = authority.consumer.consumerId; - const pacStatus = record(await runPlatformInfraPipelinesAsCodeCommand(config, ["status", "--target", spec.nodeId, "--consumer", consumerId, "--full"])); + const pacProbe = await timedStatusStage("pac", () => runPlatformInfraPipelinesAsCodeCommand(config, ["status", "--target", spec.nodeId, "--consumer", consumerId, "--full"])); + const pacStatus = record(pacProbe.value); const pacSummary = record(pacStatus.summary); + const pacReadObservation = record(pacStatus.observation); const latestPipelineRun = record(pacSummary.latestPipelineRun); const artifact = record(pacSummary.artifact); const pacArgo = record(pacSummary.argo); const pipelineRunName = options.pipelineRun ?? stringOrNull(latestPipelineRun.name); - const runtimeProbe = await timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])); + const runtimeProbe = await timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)], { runtimeTimeoutMs: agentRunStatusCaptureTimeoutMs })); const runtimePayload = captureJsonPayload(runtimeProbe.value); const manager = record(runtimePayload.manager); const database = record(runtimePayload.database); @@ -479,6 +483,7 @@ export async function statusPipelinesAsCodeLane(config: UniDeskConfig, options: pac: { ready: pacSummary.ready ?? false, webhookCount: pacSummary.webhookCount ?? null, + readObservation: pacReadObservation, latestPipelineRun: { name: pipelineRunName, status: latestPipelineRun.status ?? null, @@ -539,8 +544,9 @@ export async function statusPipelinesAsCodeLane(config: UniDeskConfig, options: migration, }, timings: { + pacMs: pacProbe.elapsedMs, runtimeMs: runtimeProbe.elapsedMs, - totalMs: runtimeProbe.elapsedMs, + totalMs: pacProbe.elapsedMs + runtimeProbe.elapsedMs, }, disclosure: { output: options.full || options.raw ? "full" : "compact-summary", @@ -636,8 +642,8 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio if (deliveryAuthority.kind === "pac-pr-merge") return await statusPipelinesAsCodeLane(config, options, target, deliveryAuthority); if (deliveryAuthority.kind === "unknown") return unknownDeliveryAuthorityStatus(options, target, deliveryAuthority); const sourceProbe = await timedStatusStage("source", () => spec.source.statusMode === "k3s-git-mirror" - ? capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)]) - : capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)])); + ? capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneK3sSourceStatusScript(spec)], { runtimeTimeoutMs: agentRunStatusCaptureTimeoutMs }) + : capture(config, `${spec.nodeRoute}:${spec.source.workspace}`, ["sh", "--", yamlLaneSourceStatusScript(spec)], { runtimeTimeoutMs: agentRunStatusCaptureTimeoutMs })); const sourcePayload = captureJsonPayload(sourceProbe.value); const branchTipCommit = stringOrNull(sourcePayload.remoteBranchCommit) ?? stringOrNull(sourcePayload.localHead); const initialSourceCommit = options.sourceCommit @@ -645,8 +651,8 @@ export async function statusYamlLane(config: UniDeskConfig, options: StatusOptio ?? stringOrNull(sourcePayload.localHead); const pipelineRunName = options.pipelineRun ?? (initialSourceCommit ? agentRunPipelineRunName(spec, initialSourceCommit) : null); const [runtimeProbe, mirrorProbe] = await Promise.all([ - timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)])), - timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)])), + timedStatusStage("runtime", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneRuntimeStatusScript(spec, pipelineRunName)], { runtimeTimeoutMs: agentRunStatusCaptureTimeoutMs })), + timedStatusStage("git-mirror", () => capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneGitMirrorStatusScript(spec)], { runtimeTimeoutMs: agentRunStatusCaptureTimeoutMs })), ]); const runtimePayload = captureJsonPayload(runtimeProbe.value); const mirrorPayload = captureJsonPayload(mirrorProbe.value); diff --git a/scripts/src/agentrun/public-exposure.ts b/scripts/src/agentrun/public-exposure.ts index 7c3feb81..2a203f1b 100644 --- a/scripts/src/agentrun/public-exposure.ts +++ b/scripts/src/agentrun/public-exposure.ts @@ -80,6 +80,8 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record `- ${warning}`)].join("\n"), "", "NEXT", @@ -127,7 +140,7 @@ export function renderAgentRunControlPlaneStatusSummary(result: Record line !== null); return renderedCliResult(result.ok !== false, "agentrun control-plane status", `${lines.join("\n")}\n`); } diff --git a/scripts/src/agentrun/utils.ts b/scripts/src/agentrun/utils.ts index b22068c7..83d5a770 100644 --- a/scripts/src/agentrun/utils.ts +++ b/scripts/src/agentrun/utils.ts @@ -137,8 +137,8 @@ export type AgentRunBridgeCaptureResult = SshCaptureResult & { bridgeExecution?: export let localBackendCoreStatusCache: LocalBackendCoreStatus | null = null; -export async function capture(config: UniDeskConfig, target: string, args: string[]): Promise { - const result = await runSshCommandCapture(config, target, args); +export async function capture(config: UniDeskConfig, target: string, args: string[], options: { runtimeTimeoutMs?: number } = {}): Promise { + const result = await runSshCommandCapture(config, target, args, undefined, options); const plan = agentRunBridgeCapturePlan(config, target); return attachBridgeExecution(result, { ...plan, reason: `unified-ssh-capture:${plan.reason}` }); } diff --git a/scripts/src/platform-infra-pac-failure-evidence.test.ts b/scripts/src/platform-infra-pac-failure-evidence.test.ts index 63c28769..bf2a89f5 100644 --- a/scripts/src/platform-infra-pac-failure-evidence.test.ts +++ b/scripts/src/platform-infra-pac-failure-evidence.test.ts @@ -5,7 +5,7 @@ import { resolve } from "node:path"; import { createRequire } from "node:module"; import { sentinelPublishImageBuildShell } from "./hwlab-node-web-sentinel-cicd-jobs"; import type { SentinelCicdState } from "./hwlab-node-web-sentinel-cicd-shared"; -import { compactHistoryJson, compactStatusSummary, renderDebugStep, renderHistory, renderStatus } from "./platform-infra-pipelines-as-code"; +import { compactHistoryJson, compactStatusSummary, pacReadOnlyCaptureObservation, renderDebugStep, renderHistory, renderStatus } from "./platform-infra-pipelines-as-code"; const require = createRequire(import.meta.url); const { parsePacLogRecords } = require("../native/cicd/pac-status-evaluator.cjs") as { @@ -55,6 +55,35 @@ describe("PaC 失败证据合同", () => { expect(compactStatus).toMatchObject({ artifact: { traceId: row.traceId, firstBreak: row.firstBreak, error: row.error } }); }); + test("只读 capture timeout 保留阶段、预算、首断点与下一步", () => { + const observation = pacReadOnlyCaptureObservation({ + action: "history", + targetId: "NC01", + consumerId: "agentrun-nc01-v02", + result: { exitCode: 124, stdout: "", stderr: "UNIDESK_SSH_RUNTIME_TIMEOUT" }, + parsed: null, + elapsedMs: 25001, + timeoutMs: 25000, + next: "bun scripts/cli.ts platform-infra pipelines-as-code history --target NC01 --consumer agentrun-nc01-v02 --limit 5", + }); + expect(observation).toMatchObject({ + ok: false, + status: "timeout", + target: "NC01", + consumer: "agentrun-nc01-v02", + stage: "history-remote-observe", + elapsedMs: 25001, + timeoutMs: 25000, + exitCode: 124, + firstBreak: { code: "pac-read-timeout", phase: "remote-read" }, + }); + const output = renderHistory({ rows: [], config: {}, historyErrors: [], deliveryBudget: {}, observation, next: {} }).renderedText; + expect(output).toContain("READ OBSERVATION"); + expect(output).toContain("history-remote-observe"); + expect(output).toContain("pac-read-timeout"); + expect(output).toContain("next-read-only:"); + }); + test("feature-config warning 在 status、history、debug-step 同构有界投影", () => { const observation = extractPacSourceObservation([ { diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index d04a0d42..8219e4a1 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -52,6 +52,7 @@ const remoteScriptFile = rootPath("scripts", "src", "platform-infra-pipelines-as const evaluatorFile = rootPath("scripts", "native", "cicd", "pac-status-evaluator.cjs"); const sourceArtifactRuntimeObserverFile = rootPath("scripts", "native", "cicd", "pac-source-artifact-runtime.mjs"); const fieldManager = "unidesk-platform-infra-pipelines-as-code"; +const pacReadOnlyCaptureTimeoutMs = 25_000; const y = createYamlFieldReader(configLabel); function kubernetesLabelValue(value: string): string { @@ -293,6 +294,25 @@ interface SecretMaterial { webhookPath: string; } +type PacReadOnlyAction = "status" | "history" | "debug-step"; + +interface PacReadOnlyCaptureObservation { + readonly ok: boolean; + readonly status: "succeeded" | "timeout" | "remote-failure" | "empty-response" | "parse-failure"; + readonly target: string; + readonly consumer: string; + readonly stage: string; + readonly elapsedMs: number; + readonly timeoutMs: number; + readonly exitCode: number; + readonly stdoutBytes: number; + readonly stderrBytes: number; + readonly partial: boolean; + readonly firstBreak: { readonly code: string; readonly phase: string }; + readonly next: string; + readonly valuesPrinted: false; +} + export async function runPlatformInfraPipelinesAsCodeCommand(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { const [action = "plan"] = args; if (action === "help" || action === "--help") return help(args[1] ?? null); @@ -1350,14 +1370,15 @@ async function status(config: UniDeskConfig, options: CommonOptions): Promise record(observation.warning)) .filter((warning) => Object.keys(warning).length > 0); return { - ok: result.exitCode === 0 && parsed?.ok !== false && historyErrors.length === 0, + ok: observation.ok && parsed?.ok !== false && historyErrors.length === 0, action: "platform-infra-pipelines-as-code-history", mutation: false, target: targetSummary(target), @@ -1633,6 +1658,7 @@ async function history(config: UniDeskConfig, options: HistoryOptions): Promise< detailId: options.detailId, rows, deliveryBudget, + observation, historyErrors, warnings: [...pac.validationWarnings, ...deliveryWarnings], controlPlane: parsed === null ? undefined : { @@ -1646,9 +1672,7 @@ async function history(config: UniDeskConfig, options: HistoryOptions): Promise< valuesPrinted: false, }, remote: parsed === null || options.raw ? remote : undefined, - next: selection.nextConsumerId === null - ? pacNodeReadOnlyNext(target.id) - : pacReadOnlyNext(target.id, selection.nextConsumerId), + next, }; } @@ -1698,12 +1722,13 @@ async function debugStep(config: UniDeskConfig, options: HistoryOptions): Promis } const consumer = resolveConsumer(pac, selection.selectedConsumerIds[0] ?? null); const repository = resolveRepository(pac, consumer.repositoryRef); - const result = await capture(config, target.route, ["sh"], remoteScript("debug-step", pac, target, repository, consumer, { ...options, confirm: false, dryRun: true, wait: false }, emptySecretMaterial(), "")); - const parsed = parseJsonOutput(result.stdout); + const next = nextCommands(target.id, consumer.id, pac.defaults.consumerId); + const observed = await capturePacReadOnly(config, "debug-step", target, consumer, remoteScript("debug-step", pac, target, repository, consumer, { ...options, confirm: false, dryRun: true, wait: false }, emptySecretMaterial(), ""), stringValue(next.status)); + const { result, parsed, observation } = observed; const fixtureChecks = parsed === null ? [] : arrayRecords(parsed.checks); const fixtureDisclosure = pacDebugFixtureDisclosure(fixtureChecks, options.detailId); return { - ok: result.exitCode === 0 && parsed?.ok === true, + ok: observation.ok && parsed?.ok === true, action: "platform-infra-pipelines-as-code-debug-step", mutation: false, target: targetSummary(target), @@ -1713,12 +1738,97 @@ async function debugStep(config: UniDeskConfig, options: HistoryOptions): Promis fixtureGate: { ...fixtureDisclosure.fixtureGate, ok: parsed !== null && fixtureDisclosure.fixtureGate.ok === true }, checks: fixtureDisclosure.checks, realRun: parsed === null ? null : parsed.realRun ?? null, + observation, remote: parsed === null || options.raw ? parsed ?? compactCapture(result, { full: true }) : undefined, - next: nextCommands(target.id, consumer.id, pac.defaults.consumerId), + next, valuesPrinted: false, }; } +async function capturePacReadOnly( + config: UniDeskConfig, + action: PacReadOnlyAction, + target: PacTarget, + consumer: PacConsumer, + script: string, + next: string, +): Promise<{ readonly result: Awaited>; readonly parsed: Record | null; readonly observation: PacReadOnlyCaptureObservation }> { + const stage = `${action}-remote-observe`; + const startedAt = Date.now(); + printPacReadOnlyProgress(target, consumer, stage, "started", startedAt, { timeoutMs: pacReadOnlyCaptureTimeoutMs }); + const result = await capture(config, target.route, ["sh"], script, { runtimeTimeoutMs: pacReadOnlyCaptureTimeoutMs }); + const parsed = parseJsonOutput(result.stdout); + const observation = pacReadOnlyCaptureObservation({ + action, + targetId: target.id, + consumerId: consumer.id, + result, + parsed, + elapsedMs: Date.now() - startedAt, + timeoutMs: pacReadOnlyCaptureTimeoutMs, + next, + }); + printPacReadOnlyProgress(target, consumer, stage, observation.status, startedAt, { + exitCode: result.exitCode, + firstBreak: observation.firstBreak.code, + }); + return { result, parsed, observation }; +} + +export function pacReadOnlyCaptureObservation(options: { + readonly action: PacReadOnlyAction; + readonly targetId: string; + readonly consumerId: string; + readonly result: { readonly exitCode: number; readonly stdout: string; readonly stderr: string }; + readonly parsed: Record | null; + readonly elapsedMs: number; + readonly timeoutMs: number; + readonly next: string; +}): PacReadOnlyCaptureObservation { + const stdoutBytes = Buffer.byteLength(options.result.stdout, "utf8"); + const stderrBytes = Buffer.byteLength(options.result.stderr, "utf8"); + const timedOut = options.result.exitCode === 124 || /ssh-runtime-timeout|top-level runtime limit/iu.test(options.result.stderr); + const status = timedOut + ? "timeout" + : options.result.exitCode !== 0 + ? "remote-failure" + : options.result.stdout.trim().length === 0 + ? "empty-response" + : options.parsed === null + ? "parse-failure" + : "succeeded"; + const firstBreakCode = status === "succeeded" ? "none" : `pac-read-${status}`; + return { + ok: status === "succeeded", + status, + target: options.targetId, + consumer: options.consumerId, + stage: `${options.action}-remote-observe`, + elapsedMs: options.elapsedMs, + timeoutMs: options.timeoutMs, + exitCode: options.result.exitCode, + stdoutBytes, + stderrBytes, + partial: stdoutBytes > 0 || stderrBytes > 0, + firstBreak: { code: firstBreakCode, phase: "remote-read" }, + next: options.next, + valuesPrinted: false, + }; +} + +function printPacReadOnlyProgress(target: PacTarget, consumer: PacConsumer, stage: string, status: string, startedAt: number, detail: Record): void { + process.stderr.write(`${JSON.stringify({ + event: "platform-infra.pipelines-as-code.read.progress", + target: target.id, + consumer: consumer.id, + stage, + status, + elapsedMs: Date.now() - startedAt, + ...detail, + valuesPrinted: false, + })}\n`); +} + export function pacDebugFixtureDisclosure(fixtureChecks: readonly Record[], detailId: string | null): { readonly fixtureGate: Record; readonly checks: readonly Record[]; @@ -2718,6 +2828,7 @@ export function renderStatus(result: Record): RenderedCliResult ]]), ` hint: ${compactLine(stringValue(diagnostics.hint))}`, ` pipeline-run: ${latest.name === undefined ? "-" : `bun scripts/cli.ts platform-infra pipelines-as-code history --target ${stringValue(record(result.target).id)} --id ${stringValue(latest.name)}`}`, + ...renderPacReadObservation(result), "", "NEXT", ` full: ${stringValue(record(result.next).status)} --full`, @@ -2912,6 +3023,7 @@ export function renderHistory(result: Record): RenderedCliResul ]; }))), ...(detailId === "-" || rows.length !== 1 ? [] : renderHistoryDetail(rows[0] ?? {})), + ...renderPacReadObservation(result), "", "NEXT", rows.length > 0 && detailId === "-" ? ` detail: ${stringValue(record(result.next).history)} --id ${stringValue(rows[0]?.id ?? rows[0]?.pipelineRun)}` : null, @@ -2987,6 +3099,7 @@ export function renderDebugStep(result: Record): RenderedCliRes `${stringValue(item.expectedOk)}/${stringValue(item.expectedCode)}`, `${stringValue(item.actualOk)}/${stringValue(item.actualCode)}`, ]))), + ...renderPacReadObservation(result), "", "NEXT", ` status: ${stringValue(record(result.next).status)}`, @@ -2994,6 +3107,28 @@ export function renderDebugStep(result: Record): RenderedCliRes return rendered(result, "platform-infra pipelines-as-code debug-step", lines); } +function renderPacReadObservation(result: Record): string[] { + const observation = record(result.observation); + if (Object.keys(observation).length === 0) return []; + const firstBreak = record(observation.firstBreak); + return [ + "", + "READ OBSERVATION", + ...table(["TARGET", "CONSUMER", "STAGE", "STATUS", "ELAPSED_MS", "TIMEOUT_MS", "EXIT", "PARTIAL", "FIRST_BREAK"], [[ + stringValue(observation.target), + stringValue(observation.consumer), + stringValue(observation.stage), + stringValue(observation.status), + stringValue(observation.elapsedMs), + stringValue(observation.timeoutMs), + stringValue(observation.exitCode), + boolText(observation.partial), + stringValue(firstBreak.code), + ]]), + ` next-read-only: ${stringValue(observation.next)}`, + ]; +} + function parseApplyOptions(args: string[]): ApplyOptions { const commonArgs: string[] = []; let confirm = false;