fix: 收敛 PaC OTel renderer 配置

This commit is contained in:
Codex
2026-07-13 15:41:37 +02:00
parent 0b73068c3d
commit b740d5d31e
3 changed files with 393 additions and 19 deletions
+234
View File
@@ -0,0 +1,234 @@
import { rootPath } from "../../src/config";
import { readYamlRecord } from "../../src/platform-infra-ops-library";
import { materializeYamlComposition } from "../../src/yaml-composition";
const pacConfigPath = "config/platform-infra/pipelines-as-code.yaml";
export interface CicdOtelRuntimeConfig {
readonly endpoint: string;
readonly endpointHostname: string;
readonly serviceName: string;
readonly samplingRatio: number;
readonly timeoutMs: number;
readonly fallbackCodes: readonly string[];
}
export interface CicdOtelExportWarning {
readonly event: "cicd.otel.export";
readonly warning: true;
readonly blocking: false;
readonly causeCode: string;
readonly endpointHostname: string;
readonly httpStatus?: number;
readonly timedOut?: boolean;
readonly errorType?: string;
readonly timeoutMs: number;
readonly traceId: string;
readonly valuesRedacted: true;
}
export function loadCicdOtelRuntimeConfig(
node: string,
lane: string,
env: Record<string, string | undefined> = process.env,
): CicdOtelRuntimeConfig {
const root = materializeYamlComposition(
readYamlRecord<Record<string, unknown>>(rootPath(pacConfigPath), "platform-infra-pipelines-as-code"),
{ label: "platform-infra-pipelines-as-code", stripTemplateKeys: true },
).value;
const observability = record(root.observability, `${pacConfigPath}#observability`);
const repository = repositoryFor(root.repositories, node, lane);
const params = record(repository.params, `${pacConfigPath}#repositories.${repository.id}.params`);
const fallbackCodes: string[] = [];
const endpoint = authoritativeValue(
[
[params.otel_traces_endpoint, "INVALID_ENDPOINT_PARAM"],
[observability.tracesEndpoint, "INVALID_ENDPOINT_AUTHORITY"],
],
validEndpoint,
fallbackCodes,
);
validateInput(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, endpoint, validEndpoint, "ENDPOINT", fallbackCodes);
const endpointUrl = new URL(endpoint);
const serviceName = authoritativeValue(
[
[params.otel_service_name, "INVALID_SERVICE_PARAM"],
[observability.serviceName, "INVALID_SERVICE_AUTHORITY"],
],
validServiceName,
fallbackCodes,
);
validateInput(env.OTEL_SERVICE_NAME, serviceName, validServiceName, "SERVICE", fallbackCodes);
const sampling = record(observability.sampling, `${pacConfigPath}#observability.sampling`);
const samplingRatio = authoritativeNumber(
[
[params.otel_sampling_ratio, "INVALID_SAMPLING_PARAM"],
[sampling.ratio, "INVALID_SAMPLING_AUTHORITY"],
],
(value) => value >= 0 && value <= 1,
fallbackCodes,
);
validateNumberInput(env.OTEL_TRACES_SAMPLER_ARG, samplingRatio, "SAMPLING", fallbackCodes);
const exporterFailure = record(observability.exporterFailure, `${pacConfigPath}#observability.exporterFailure`);
const timeoutMs = authoritativeNumber(
[
[params.otel_exporter_timeout_ms, "INVALID_TIMEOUT_PARAM"],
[exporterFailure.timeoutMs, "INVALID_TIMEOUT_AUTHORITY"],
],
(value) => Number.isInteger(value) && value >= 50 && value <= 2_000,
fallbackCodes,
);
validateNumberInput(env.OTEL_EXPORTER_TIMEOUT_MS, timeoutMs, "TIMEOUT", fallbackCodes);
return {
endpoint: endpointUrl.toString(),
endpointHostname: endpointUrl.hostname,
serviceName,
samplingRatio,
timeoutMs,
fallbackCodes,
};
}
export async function exportCicdOtelPayload(
config: CicdOtelRuntimeConfig,
traceparent: string,
traceId: string,
payload: unknown,
fetcher: typeof fetch = fetch,
): Promise<CicdOtelExportWarning | null> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), config.timeoutMs);
try {
const response = await fetcher(config.endpoint, {
method: "POST",
headers: { "content-type": "application/json", traceparent },
body: JSON.stringify(payload),
signal: controller.signal,
});
if (response.ok) return null;
return warning(config, traceId, `HTTP_${response.status}`, { httpStatus: response.status });
} catch (error) {
return warning(config, traceId, errorCauseCode(error), {
timedOut: error instanceof Error && error.name === "AbortError",
errorType: error instanceof Error ? error.name : "Error",
});
} finally {
clearTimeout(timer);
}
}
function repositoryFor(value: unknown, node: string, lane: string): Record<string, unknown> {
if (!Array.isArray(value)) throw new Error(`${pacConfigPath}#repositories must be an array`);
const expectedId = `sentinel-${node.toLowerCase()}-${lane.toLowerCase()}`;
const repository = value.find((item) => recordOrNull(item)?.id === expectedId);
if (repository === undefined) throw new Error(`${pacConfigPath} has no repository ${expectedId}`);
return record(repository, `${pacConfigPath}#repositories.${expectedId}`);
}
function authoritativeValue(
candidates: ReadonlyArray<readonly [unknown, string]>,
validate: (value: string) => boolean,
fallbackCodes: string[],
): string {
for (const [candidate, code] of candidates) {
if (candidate === undefined || candidate === null || candidate === "") {
fallbackCodes.push(code);
continue;
}
if (typeof candidate === "string" && validate(candidate)) return candidate;
fallbackCodes.push(code);
}
throw new Error(`invalid owning OTel configuration: ${fallbackCodes.at(-1) ?? "UNKNOWN"}`);
}
function authoritativeNumber(
candidates: ReadonlyArray<readonly [unknown, string]>,
validate: (value: number) => boolean,
fallbackCodes: string[],
): number {
for (const [candidate, code] of candidates) {
if (candidate === undefined || candidate === null || candidate === "") {
fallbackCodes.push(code);
continue;
}
const value = typeof candidate === "number" ? candidate : Number(candidate);
if (Number.isFinite(value) && validate(value)) return value;
fallbackCodes.push(code);
}
throw new Error(`invalid owning OTel configuration: ${fallbackCodes.at(-1) ?? "UNKNOWN"}`);
}
function validateInput(
input: string | undefined,
desired: string,
validate: (value: string) => boolean,
label: string,
fallbackCodes: string[],
): void {
if (input === undefined || input === "") return;
if (!validate(input)) fallbackCodes.push(`INVALID_${label}_INPUT`);
else if (input !== desired) fallbackCodes.push(`MISMATCH_${label}_INPUT`);
}
function validateNumberInput(input: string | undefined, desired: number, label: string, fallbackCodes: string[]): void {
if (input === undefined || input === "") return;
const value = Number(input);
if (!Number.isFinite(value)) fallbackCodes.push(`INVALID_${label}_INPUT`);
else if (input !== String(desired)) fallbackCodes.push(`MISMATCH_${label}_INPUT`);
}
function validEndpoint(value: string): boolean {
if (hasTemplatePlaceholder(value)) return false;
try {
const url = new URL(value);
return (url.protocol === "http:" || url.protocol === "https:")
&& url.hostname.length > 0
&& url.username.length === 0
&& url.password.length === 0;
} catch {
return false;
}
}
function validServiceName(value: string): boolean {
return !hasTemplatePlaceholder(value) && /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(value);
}
function hasTemplatePlaceholder(value: string): boolean {
return /\{\{[^}]+\}\}|\$\{[^}]+\}/u.test(value);
}
function warning(
config: CicdOtelRuntimeConfig,
traceId: string,
causeCode: string,
details: Pick<CicdOtelExportWarning, "httpStatus" | "timedOut" | "errorType">,
): CicdOtelExportWarning {
return {
event: "cicd.otel.export",
warning: true,
blocking: false,
causeCode,
endpointHostname: config.endpointHostname,
...details,
timeoutMs: config.timeoutMs,
traceId,
valuesRedacted: true,
};
}
function errorCauseCode(error: unknown): string {
if (error instanceof Error && error.name === "AbortError") return "EXPORT_TIMEOUT";
const cause = error instanceof Error && typeof error.cause === "object" && error.cause !== null ? error.cause as Record<string, unknown> : null;
return typeof cause?.code === "string" && /^[A-Za-z0-9_.-]{1,80}$/u.test(cause.code) ? cause.code : "EXPORT_FAILED";
}
function record(value: unknown, path: string): Record<string, unknown> {
const result = recordOrNull(value);
if (result === null) throw new Error(`${path} must be an object`);
return result;
}
function recordOrNull(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
@@ -0,0 +1,133 @@
import { afterEach, expect, test } from "bun:test";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rootPath } from "../../src/config";
import { exportCicdOtelPayload, loadCicdOtelRuntimeConfig } from "./otel-runtime-config";
const roots: string[] = [];
afterEach(() => {
while (roots.length > 0) rmSync(roots.pop()!, { recursive: true, force: true });
});
test("unexpanded Repository inputs fall back to materialized owning YAML", () => {
const config = loadCicdOtelRuntimeConfig("NC01", "v03", {
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "{{otel_traces_endpoint}}",
OTEL_SERVICE_NAME: "{{otel_service_name}}",
OTEL_TRACES_SAMPLER_ARG: "{{otel_sampling_ratio}}",
OTEL_EXPORTER_TIMEOUT_MS: "{{otel_exporter_timeout_ms}}",
});
expect(config).toMatchObject({
endpoint: "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces",
endpointHostname: "otel-collector.platform-infra.svc.cluster.local",
serviceName: "unidesk-cicd-sentinel-nc01",
samplingRatio: 1,
timeoutMs: 300,
});
expect(config.fallbackCodes).toEqual([
"INVALID_ENDPOINT_INPUT",
"INVALID_SERVICE_INPUT",
"INVALID_SAMPLING_INPUT",
"INVALID_TIMEOUT_INPUT",
]);
});
test("valid but mismatched runtime inputs cannot override owning YAML", () => {
const config = loadCicdOtelRuntimeConfig("NC01", "v03", {
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://collector.example.invalid:4318/v1/traces",
OTEL_SERVICE_NAME: "different-service",
OTEL_TRACES_SAMPLER_ARG: "0.5",
OTEL_EXPORTER_TIMEOUT_MS: "600",
});
expect(config.endpointHostname).toBe("otel-collector.platform-infra.svc.cluster.local");
expect(config.serviceName).toBe("unidesk-cicd-sentinel-nc01");
expect(config.samplingRatio).toBe(1);
expect(config.timeoutMs).toBe(300);
expect(config.fallbackCodes).toEqual([
"MISMATCH_ENDPOINT_INPUT",
"MISMATCH_SERVICE_INPUT",
"MISMATCH_SAMPLING_INPUT",
"MISMATCH_TIMEOUT_INPUT",
]);
});
test("legal owning endpoint and non-blocking exporter failure share the bounded helper", async () => {
const config = loadCicdOtelRuntimeConfig("NC01", "v03", {
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces",
OTEL_SERVICE_NAME: "unidesk-cicd-sentinel-nc01",
OTEL_TRACES_SAMPLER_ARG: "1",
OTEL_EXPORTER_TIMEOUT_MS: "300",
});
expect(config.fallbackCodes).toEqual([]);
const warning = await exportCicdOtelPayload(
config,
"00-11111111111111111111111111111111-2222222222222222-01",
"11111111111111111111111111111111",
{ resourceSpans: [] },
async () => { throw Object.assign(new TypeError("fetch failed"), { cause: { code: "ENOTFOUND" } }); },
);
expect(warning).toEqual({
event: "cicd.otel.export",
warning: true,
blocking: false,
causeCode: "ENOTFOUND",
endpointHostname: "otel-collector.platform-infra.svc.cluster.local",
timedOut: false,
errorType: "TypeError",
timeoutMs: 300,
traceId: "11111111111111111111111111111111",
valuesRedacted: true,
});
});
test("actual renderer replaces unexpanded inputs without changing business exit", async () => {
const result = await render({ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "{{otel_traces_endpoint}}" }, false);
expect(result.exitCode).toBe(0);
expect(result.stderr).toContain('"event":"cicd.otel.config"');
expect(result.stderr).toContain('"blocking":false');
expect(result.stderr).toContain('"endpointHostname":"otel-collector.platform-infra.svc.cluster.local"');
expect(result.stderr).not.toContain("{{otel_traces_endpoint}}");
expect(readFileSync(join(result.outputDir, "source.sh"), "utf8")).toContain("http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces");
});
async function render(env: Record<string, string>, traced = true) {
const outputDir = mkdtempSync(join(tmpdir(), "unidesk-render-sentinel-publish-"));
roots.push(outputDir);
const child = Bun.spawn({
cmd: [
process.execPath,
"scripts/native/cicd/render-sentinel-publish-task.ts",
"--node", "NC01",
"--lane", "v03",
"--sentinel", "nc01-web-probe-sentinel",
"--pipeline-run", "hwlab-web-probe-sentinel-nc01-test",
"--source-commit", "a".repeat(40),
"--source-stage-ref", `refs/unidesk/snapshots/test/${"a".repeat(40)}`,
"--source-authority", "gitea-snapshot",
"--output-dir", outputDir,
],
cwd: rootPath(),
env: {
...process.env,
TRACEPARENT: traced ? "00-11111111111111111111111111111111-2222222222222222-01" : "",
OTEL_SERVICE_NAME: "unidesk-cicd-test",
OTEL_TRACES_SAMPLER_ARG: "1",
OTEL_EXPORTER_TIMEOUT_MS: "300",
...env,
},
stdout: "pipe",
stderr: "pipe",
});
const [exitCode, stdout, stderr] = await Promise.all([
child.exited,
new Response(child.stdout).text(),
new Response(child.stderr).text(),
]);
return {
exitCode,
stdout,
stderr,
outputDir,
};
}
@@ -8,6 +8,7 @@ import {
sentinelPublishShell,
sentinelPublishSourceShell,
} from "../../src/hwlab-node-web-sentinel-cicd-jobs";
import { exportCicdOtelPayload, loadCicdOtelRuntimeConfig } from "./otel-runtime-config";
const options = parseOptions(process.argv.slice(2));
const sourceAuthority = options.sourceAuthority === "gitea-snapshot" ? "gitea-snapshot" : fail("--source-authority must be gitea-snapshot");
@@ -25,6 +26,13 @@ const state = loadSentinelCicdState(
);
const outputDir = resolve(options.outputDir);
mkdirSync(outputDir, { recursive: true });
const otelConfig = loadCicdOtelRuntimeConfig(options.node, options.lane);
const otelSetup = [
`export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${shellQuote(otelConfig.endpoint)}`,
`export OTEL_EXPORTER_TIMEOUT_MS=${shellQuote(String(otelConfig.timeoutMs))}`,
`export OTEL_SERVICE_NAME=${shellQuote(otelConfig.serviceName)}`,
`export OTEL_TRACES_SAMPLER_ARG=${shellQuote(String(otelConfig.samplingRatio))}`,
].join("\n");
const traceSetup = [
"if [ -f /workspace/meta/traceparent ]; then TRACEPARENT=$(cat /workspace/meta/traceparent); export TRACEPARENT; fi",
"if [ -f /workspace/meta/tracestate ]; then TRACESTATE=$(cat /workspace/meta/tracestate); export TRACESTATE; fi",
@@ -33,14 +41,16 @@ const proxySetup = sentinelImageBuildProxyEnv(state).map(({ name, value }) => `e
writeExecutable("source.sh", [
"#!/bin/sh",
"set -eu",
otelSetup,
`incoming_traceparent=${shellQuote(process.env.TRACEPARENT ?? "")}`,
`incoming_tracestate=${shellQuote(process.env.TRACESTATE ?? "")}`,
sentinelPublishSourceShell(state, options.pipelineRun),
"if [ -n \"$incoming_traceparent\" ]; then printf '%s' \"$incoming_traceparent\" > /workspace/meta/traceparent; fi",
"if [ -n \"$incoming_tracestate\" ]; then printf '%s' \"$incoming_tracestate\" > /workspace/meta/tracestate; fi",
].join("\n"));
writeExecutable("image-build.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishImageBuildShell(state, options.pipelineRun)].join("\n"));
writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true)].join("\n"));
writeExecutable("image-build.sh", ["#!/bin/sh", "set -eu", otelSetup, traceSetup, proxySetup, sentinelPublishImageBuildShell(state, options.pipelineRun)].join("\n"));
writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", otelSetup, traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true)].join("\n"));
emitConfigFallbackWarning();
await emitOuterSpan();
process.stdout.write(`${JSON.stringify({ ok: true, action: "render-sentinel-publish-task", node: options.node, lane: options.lane, sentinel: options.sentinel, pipelineRun: options.pipelineRun, sourceCommit: options.sourceCommit, outputDir, traceId: traceId(process.env.TRACEPARENT), valuesRedacted: true })}\n`);
@@ -80,13 +90,12 @@ function traceId(traceparent: string | undefined): string | null {
}
async function emitOuterSpan(): Promise<void> {
const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
const context = process.env.TRACEPARENT?.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/iu);
if (endpoint === undefined || context === undefined || context === null) return;
if (context === undefined || context === null || !shouldSample(context[1] ?? "", otelConfig.samplingRatio)) return;
const startedAtMs = Number(process.env.CICD_TRACE_STARTED_AT_MS ?? Date.now());
const payload = {
resourceSpans: [{
resource: { attributes: attributes({ "service.name": process.env.OTEL_SERVICE_NAME ?? "unidesk-cicd", "unidesk.node": options.node, "hwlab.lane": options.lane, "unidesk.values_redacted": true }) },
resource: { attributes: attributes({ "service.name": otelConfig.serviceName, "unidesk.node": options.node, "hwlab.lane": options.lane, "unidesk.values_redacted": true }) },
scopeSpans: [{
scope: { name: "unidesk.cicd.pac_outer", version: "1" },
spans: [{
@@ -115,22 +124,20 @@ async function emitOuterSpan(): Promise<void> {
}],
}],
};
const controller = new AbortController();
const timeoutMs = exporterTimeoutMs();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: process.env.TRACEPARENT ?? "" }, body: JSON.stringify(payload), signal: controller.signal });
if (!response.ok) process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`);
} catch (error) {
process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, timedOut: error instanceof Error && error.name === "AbortError", errorType: error instanceof Error ? error.name : "Error", timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`);
} finally {
clearTimeout(timer);
}
const warning = await exportCicdOtelPayload(otelConfig, process.env.TRACEPARENT ?? "", context[1]?.toLowerCase() ?? "", payload);
if (warning !== null) process.stderr.write(`${JSON.stringify(warning)}\n`);
}
function exporterTimeoutMs(): number {
const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS ?? "300");
return Number.isInteger(value) && value >= 50 && value <= 2_000 ? value : 300;
function emitConfigFallbackWarning(): void {
if (otelConfig.fallbackCodes.length === 0) return;
process.stderr.write(`${JSON.stringify({ event: "cicd.otel.config", warning: true, blocking: false, causeCodes: otelConfig.fallbackCodes, endpointHostname: otelConfig.endpointHostname, valuesRedacted: true })}\n`);
}
function shouldSample(traceIdValue: string, ratio: number): boolean {
if (ratio <= 0) return false;
if (ratio >= 1) return true;
const bucket = Number.parseInt(traceIdValue.slice(0, 8), 16) / 0xffffffff;
return bucket < ratio;
}
function attributes(values: Record<string, unknown>): Array<Record<string, unknown>> {