From 06f0c43867ad4e5a90ce57ffc0c1ba8d2333bd56 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 11 Jun 2026 09:51:44 +0000 Subject: [PATCH] fix: make CI logs use stream capture --- scripts/ci-logs-visibility-contract-test.ts | 56 ++++++ scripts/src/ci.ts | 200 ++++++++++++++++---- 2 files changed, 219 insertions(+), 37 deletions(-) create mode 100644 scripts/ci-logs-visibility-contract-test.ts diff --git a/scripts/ci-logs-visibility-contract-test.ts b/scripts/ci-logs-visibility-contract-test.ts new file mode 100644 index 00000000..da1ab479 --- /dev/null +++ b/scripts/ci-logs-visibility-contract-test.ts @@ -0,0 +1,56 @@ +import { readFileSync } from "node:fs"; +import { ciHelp } from "./src/ci"; + +function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { + if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); +} + +const help = JSON.stringify(ciHelp()); +const source = readFileSync("scripts/src/ci.ts", "utf8"); + +assertCondition( + help.includes("ci logs [--provider-id G14] [--tail-lines 80]"), + "ci help must expose bounded log tail control", + ciHelp(), +); + +assertCondition( + help.includes('"defaultCapture":"ssh-stream"') || help.includes('"defaultCapture": "ssh-stream"'), + "ci help must declare ssh-stream as the default logs capture path", + ciHelp(), +); + +assertCondition( + source.includes("interface CiLogsOptions") + && source.includes("function ciLogsOptions(args: string[]): CiLogsOptions") + && source.includes("ci logs --tail-lines must be an integer between 1 and 2000") + && source.includes('capture: args.includes("--dispatch-task") ? "dispatch-task" : "ssh-stream"'), + "ci logs must parse bounded tail options", +); + +assertCondition( + source.includes("function extractCiLogConditionHints(value: string): string[]") + && source.includes("function extractCiLogFailureHints(value: string): string[]") + && source.includes("conditionHints") + && source.includes("failureHints"), + "ci logs must expose condition and failure hints without requiring raw output", +); + +assertCondition( + source.includes("runSshCommandCapture(config, `${target.providerId}:k3s`, [\"script\"], script)") + && source.includes("runRemoteKubectl(script, 60_000, 45_000, target)") + && source.includes("kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o jsonpath") + && source.includes("kubectl logs -n unidesk-ci \"$pod\" --all-containers=true --tail=\"$tail_lines\""), + "ci logs must default to SSH stream capture while retaining dispatch-task fallback for Tekton conditions and bounded pod log tails", +); + +console.log(JSON.stringify({ + ok: true, + checks: [ + "ci logs help exposes bounded tail control", + "ci logs help declares ssh-stream default capture", + "ci logs parses bounded tail options", + "ci logs exposes condition and failure hints", + "ci logs defaults to SSH stream capture and retains dispatch-task fallback", + ], +})); diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index 26e23947..ffa28da9 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -15,6 +15,7 @@ import { type ArtifactRegistryReadonlyProbe, } from "./artifact-registry"; import { d601K3sGuardShellLines, d601NativeKubeconfig } from "./d601-k3s-guard"; +import { runSshCommandCapture } from "./ssh"; const d601ProviderId = "D601"; const d601Kubeconfig = d601NativeKubeconfig; @@ -142,6 +143,11 @@ interface DispatchResult { raw: unknown; } +interface CiLogsOptions { + tailLines: number; + capture: "dispatch-task" | "ssh-stream"; +} + type PublishPreflightFailureClassification = "auth-missing" | "remote-proxy-missing" | "provider-unreachable" | "local-docker-required" | "registry-not-installed" | "registry-unhealthy" | "remote-command-timeout" | "ssh-helper-command-shape-incompatible" | "ci-runner-not-ready"; type PublishPreflightControlChannel = "backend-core" | "database" | "provider" | "registry"; type PublishPreflightDetailedChannel = "backend-core-api" | "provider-dispatch" | "provider-host-ssh" | "database" | "artifact-registry"; @@ -253,6 +259,56 @@ function numberOption(args: string[], name: string, fallback: number): number { return value; } +function ciLogsOptions(args: string[]): CiLogsOptions { + const rawTail = stringOption(args, "--tail-lines") ?? stringOption(args, "--tail"); + const tailLines = rawTail === null ? 160 : Number(rawTail); + if (!Number.isInteger(tailLines) || tailLines <= 0 || tailLines > 2000) { + throw new Error("ci logs --tail-lines must be an integer between 1 and 2000"); + } + return { tailLines, capture: args.includes("--dispatch-task") ? "dispatch-task" : "ssh-stream" }; +} + +function countTextLines(value: string): number { + if (value.length === 0) return 0; + const breaks = value.match(/\r\n|\r|\n/gu)?.length ?? 0; + return breaks + (/(\r\n|\r|\n)$/u.test(value) ? 0 : 1); +} + +function tailTextLines(value: string, limit: number): string { + if (value.length === 0) return ""; + return value.split(/\r?\n/u).slice(-limit).join("\n"); +} + +function boundedHintLine(line: string): string { + const normalized = line.trim(); + if (normalized.length <= 1000) return normalized; + return `${normalized.slice(0, 1000)}`; +} + +function boundedHeadTail(value: string, headChars: number, tailChars: number): string { + if (value.length <= headChars + tailChars + 80) return value; + return `${value.slice(0, headChars)}\n...[omitted ${value.length - headChars - tailChars} chars]...\n${value.slice(-tailChars)}`; +} + +function extractCiLogConditionHints(value: string): string[] { + return value + .split(/\r?\n/u) + .filter((line) => /^(condition=|taskrun=)/u.test(line.trim())) + .map(boundedHintLine) + .filter((line) => line.length > 0) + .slice(-30); +} + +function extractCiLogFailureHints(value: string): string[] { + const failurePattern = /\b(error|failed|failure|stepfailed|exit code|exited|denied|forbidden|unauthorized|timeout|timed out|panic|exception|could not|cannot|no such|killed|oom|back-off|imagepull|errimagepull|invalid|not found)\b/iu; + return value + .split(/\r?\n/u) + .filter((line) => failurePattern.test(line)) + .map(boundedHintLine) + .filter((line) => line.length > 0) + .slice(-40); +} + function requireRevision(value: string | null): string { if (value === null || value.length === 0) throw new Error("ci run requires --revision "); if (!/^[A-Za-z0-9._/@:-]{1,160}$/u.test(value) || value.startsWith("-") || value.includes("..")) throw new Error("ci --revision contains unsupported characters"); @@ -2701,47 +2757,111 @@ async function runDevE2E(options: CiDevE2EOptions): Promise> { - if (name.length === 0) throw new Error("ci logs requires run id or PipelineRun name"); - if (/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(name)) { - const result = await dispatchSsh([ - "set -euo pipefail", - `run_id=${shellQuote(name)}`, - `result_root=${shellQuote(`${target.homeDir}/.unidesk/runs`)}`, - "result_dir=\"$result_root/$run_id\"", - "printf 'result_dir=%s\\n' \"$result_dir\"", - "found=0", - "if [ -f \"$result_dir/result.json\" ]; then found=1; echo '===== result.json'; cat \"$result_dir/result.json\"; fi", - "if [ -f \"$result_dir/launcher.log\" ]; then found=1; echo '===== launcher.log'; tail -n 160 \"$result_dir/launcher.log\"; fi", - "if [ -f \"$result_dir/runner.log\" ]; then found=1; echo '===== runner.log'; tail -n 240 \"$result_dir/runner.log\"; fi", - "if [ -f \"$result_dir/pods.log\" ]; then found=1; echo '===== pods.log'; tail -n 240 \"$result_dir/pods.log\"; fi", - "if [ \"$found\" = \"0\" ]; then echo \"no_run_files=$result_dir\" >&2; exit 42; fi", - ].join("\n"), 60_000, 45_000, true, target); - if (result.ok || (result.exitCode !== 42 && !result.stderr.includes("no_run_files="))) { - return { - ok: result.ok, - runId: name, - output: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - }; - } - } - const result = await runRemoteKubectl([ - "set -euo pipefail", +function remoteRunLogsScript(name: string, target: CiTarget, options: CiLogsOptions): string { + return [ + "set -eu", + `run_id=${shellQuote(name)}`, + `result_root=${shellQuote(`${target.homeDir}/.unidesk/runs`)}`, + `tail_lines=${shellQuote(String(options.tailLines))}`, + "result_dir=\"$result_root/$run_id\"", + "printf 'result_dir=%s\\n' \"$result_dir\"", + "found=0", + "if [ -f \"$result_dir/result.json\" ]; then found=1; echo '===== result.json'; cat \"$result_dir/result.json\"; fi", + "if [ -f \"$result_dir/launcher.log\" ]; then found=1; echo '===== launcher.log'; tail -n \"$tail_lines\" \"$result_dir/launcher.log\"; fi", + "if [ -f \"$result_dir/runner.log\" ]; then found=1; echo '===== runner.log'; tail -n \"$tail_lines\" \"$result_dir/runner.log\"; fi", + "if [ -f \"$result_dir/pods.log\" ]; then found=1; echo '===== pods.log'; tail -n \"$tail_lines\" \"$result_dir/pods.log\"; fi", + "if [ \"$found\" = \"0\" ]; then echo \"no_run_files=$result_dir\" >&2; exit 42; fi", + ].join("\n"); +} + +function pipelineRunLogsScript(name: string, options: CiLogsOptions): string { + return [ + "set -eu", + `tail_lines=${shellQuote(String(options.tailLines))}`, `kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o wide`, `kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o wide`, - `for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail=160 || true; done`, - ].join("\n"), 60_000, 45_000, target); + "echo '===== pipelinerun conditions'", + `kubectl get pipelinerun/${shellQuote(name)} -n unidesk-ci -o jsonpath='{range .status.conditions[*]}condition={.type} status={.status} reason={.reason} message={.message}{"\\n"}{end}' || true`, + "echo '===== taskrun conditions'", + `kubectl get taskrun -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o jsonpath='{range .items[*]}taskrun={.metadata.name} phase={.status.conditions[0].type} status={.status.conditions[0].status} reason={.status.conditions[0].reason} message={.status.conditions[0].message}{"\\n"}{end}' || true`, + "echo '===== pod log tails'", + `for pod in $(kubectl get pods -n unidesk-ci -l tekton.dev/pipelineRun=${shellQuote(name)} -o name); do echo "===== $pod"; kubectl logs -n unidesk-ci "$pod" --all-containers=true --tail="$tail_lines" || true; done`, + ].join("\n"); +} + +function ciLogsResultFromCapture(args: { + ok: boolean; + providerId: string; + kind: "run" | "pipelinerun"; + name: string; + options: CiLogsOptions; + stdout: string; + stderr: string; + exitCode: number | null; + dispatchTaskId?: string | null; +}): Record { + const combined = `${args.stdout}\n${args.stderr}`.trim(); + const nextName = args.kind === "run" ? "runId" : "pipelineRun"; return { - ok: true, - providerId: target.providerId, - pipelineRun: name, - output: result.stdout, - stderr: result.stderr, + ok: args.ok, + providerId: args.providerId, + [nextName]: args.name, + capture: args.options.capture, + tailLines: args.options.tailLines, + lineCount: countTextLines(combined), + outputTail: boundedHeadTail(tailTextLines(args.stdout, args.options.tailLines * 4), 2400, 2400), + stderrTail: boundedHeadTail(tailTextLines(args.stderr, Math.min(args.options.tailLines, 80)), 1200, 1200), + conditionHints: extractCiLogConditionHints(combined), + failureHints: extractCiLogFailureHints(combined), + exitCode: args.exitCode, + ...(args.dispatchTaskId === undefined ? {} : { dispatchTaskId: args.dispatchTaskId }), + fullOutputAvailable: args.options.capture === "ssh-stream", + next: [ + `bun scripts/cli.ts ci logs ${args.name} --provider-id ${args.providerId} --tail-lines ${Math.min(args.options.tailLines * 2, 2000)}`, + `bun scripts/cli.ts ci status --provider-id ${args.providerId}`, + ], }; } +async function logs(config: UniDeskConfig, name: string, target = ciTarget(null), options: CiLogsOptions = { tailLines: 160, capture: "ssh-stream" }): Promise> { + if (name.length === 0) throw new Error("ci logs requires run id or PipelineRun name"); + if (/^[a-z0-9]([-a-z0-9]{0,46}[a-z0-9])?$/u.test(name)) { + const runScript = remoteRunLogsScript(name, target, options); + const result = options.capture === "ssh-stream" + ? await runSshCommandCapture(config, `${target.providerId}:k3s`, ["script"], runScript) + : await dispatchSsh(runScript, 60_000, 45_000, true, target); + const resultOk = "ok" in result ? result.ok : result.exitCode === 0; + if (resultOk || (result.exitCode !== 42 && !result.stderr.includes("no_run_files="))) { + return ciLogsResultFromCapture({ + ok: resultOk, + providerId: target.providerId, + kind: "run", + name, + options, + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + dispatchTaskId: "taskId" in result ? result.taskId : undefined, + }); + } + } + const script = pipelineRunLogsScript(name, options); + const result = options.capture === "ssh-stream" + ? await runSshCommandCapture(config, `${target.providerId}:k3s`, ["script"], script) + : await runRemoteKubectl(script, 60_000, 45_000, target); + return ciLogsResultFromCapture({ + ok: result.exitCode === 0, + providerId: target.providerId, + kind: "pipelinerun", + name, + options, + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + dispatchTaskId: "taskId" in result ? result.taskId : undefined, + }); +} + function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record { if (artifact.kind === "source-build") { return { @@ -2784,7 +2904,7 @@ export function ciHelp(): Record { "bun scripts/cli.ts ci publish-user-service --service decision-center --commit ", "bun scripts/cli.ts ci publish-user-service --service frontend --commit ", "bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000", - "bun scripts/cli.ts ci logs [--provider-id G14]", + "bun scripts/cli.ts ci logs [--provider-id G14] [--tail-lines 80]", ], tekton: { pipelineVersion: tektonPipelineVersion, @@ -2834,6 +2954,12 @@ export function ciHelp(): Record { desiredState: "origin/master:deploy.json#environments.dev", scriptSource: "origin/master:deploy.json#environments.dev.ci", }, + logs: { + defaultCapture: "ssh-stream", + dispatchTaskFallback: "--dispatch-task", + tailLines: "1..2000", + reason: "stream capture avoids backend-core task-result compactJson truncating CI log text before the CLI can extract failure hints", + }, }; } @@ -2931,7 +3057,7 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi waitMs, }); } - if (action === "logs") return logs(nameArg ?? "", ciTarget(providerIdOption(args))); + if (action === "logs") return logs(config, nameArg ?? "", ciTarget(providerIdOption(args)), ciLogsOptions(args)); throw new Error("ci command must be one of: install, status, run, publish-backend-core, publish-user-service, run-dev-e2e, logs"); }