feat: 补齐 CI/CD OTel 与失败投影
This commit is contained in:
Codex
@@ -0,0 +1,152 @@
|
||||
import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { hwlabRuntimeLaneSpecForNode } from "../../src/hwlab-node-lanes";
|
||||
import { loadSentinelCicdState } from "../../src/hwlab-node-web-sentinel-cicd";
|
||||
import {
|
||||
sentinelImageBuildProxyEnv,
|
||||
sentinelPublishImageBuildShell,
|
||||
sentinelPublishShell,
|
||||
sentinelPublishSourceShell,
|
||||
} from "../../src/hwlab-node-web-sentinel-cicd-jobs";
|
||||
|
||||
const options = parseOptions(process.argv.slice(2));
|
||||
const sourceAuthority = options.sourceAuthority === "gitea-snapshot" ? "gitea-snapshot" : fail("--source-authority must be gitea-snapshot");
|
||||
const state = loadSentinelCicdState(
|
||||
hwlabRuntimeLaneSpecForNode(options.lane, options.node),
|
||||
options.sentinel,
|
||||
30,
|
||||
"cached",
|
||||
{
|
||||
commit: options.sourceCommit,
|
||||
stageRef: options.sourceStageRef,
|
||||
mirrorCommit: options.sourceCommit,
|
||||
sourceAuthority,
|
||||
},
|
||||
);
|
||||
const outputDir = resolve(options.outputDir);
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
const traceSetup = [
|
||||
"if [ -f /workspace/meta/traceparent ]; then TRACEPARENT=$(cat /workspace/meta/traceparent); export TRACEPARENT; fi",
|
||||
"if [ -f /workspace/meta/tracestate ]; then TRACESTATE=$(cat /workspace/meta/tracestate); export TRACESTATE; fi",
|
||||
].join("\n");
|
||||
const proxySetup = sentinelImageBuildProxyEnv(state).map(({ name, value }) => `export ${name}=${shellQuote(value)}`).join("\n");
|
||||
writeExecutable("source.sh", [
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
`incoming_traceparent=${shellQuote(process.env.TRACEPARENT ?? "")}`,
|
||||
`incoming_tracestate=${shellQuote(process.env.TRACESTATE ?? "")}`,
|
||||
sentinelPublishSourceShell(state, options.pipelineRun),
|
||||
"if [ -n \"$incoming_traceparent\" ]; then printf '%s' \"$incoming_traceparent\" > /workspace/meta/traceparent; fi",
|
||||
"if [ -n \"$incoming_tracestate\" ]; then printf '%s' \"$incoming_tracestate\" > /workspace/meta/tracestate; fi",
|
||||
].join("\n"));
|
||||
writeExecutable("image-build.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishImageBuildShell(state, options.pipelineRun)].join("\n"));
|
||||
writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true)].join("\n"));
|
||||
await emitOuterSpan();
|
||||
process.stdout.write(`${JSON.stringify({ ok: true, action: "render-sentinel-publish-task", node: options.node, lane: options.lane, sentinel: options.sentinel, pipelineRun: options.pipelineRun, sourceCommit: options.sourceCommit, outputDir, traceId: traceId(process.env.TRACEPARENT), valuesRedacted: true })}\n`);
|
||||
|
||||
function writeExecutable(name: string, content: string): void {
|
||||
const path = resolve(outputDir, name);
|
||||
writeFileSync(path, `${content}\n`, "utf8");
|
||||
chmodSync(path, 0o755);
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): Record<"node" | "lane" | "sentinel" | "pipelineRun" | "sourceCommit" | "sourceStageRef" | "sourceAuthority" | "outputDir", string> {
|
||||
const values = new Map<string, string>();
|
||||
for (let index = 0; index < args.length; index += 2) {
|
||||
const key = args[index];
|
||||
const value = args[index + 1];
|
||||
if (key === undefined || value === undefined || !key.startsWith("--")) fail("render-sentinel-publish-task requires key/value options");
|
||||
values.set(key.slice(2), value);
|
||||
}
|
||||
const required = (key: string): string => values.get(key) || fail(`--${key} is required`);
|
||||
const result = {
|
||||
node: required("node"),
|
||||
lane: required("lane"),
|
||||
sentinel: required("sentinel"),
|
||||
pipelineRun: required("pipeline-run"),
|
||||
sourceCommit: required("source-commit"),
|
||||
sourceStageRef: required("source-stage-ref"),
|
||||
sourceAuthority: required("source-authority"),
|
||||
outputDir: required("output-dir"),
|
||||
};
|
||||
if (!/^[0-9a-f]{40}$/u.test(result.sourceCommit)) fail("--source-commit must be a 40-character lowercase Git commit");
|
||||
for (const key of ["node", "lane", "sentinel", "pipelineRun"] as const) if (!/^[A-Za-z0-9._-]+$/u.test(result[key])) fail(`--${key} must be a simple id`);
|
||||
if (!result.sourceStageRef.startsWith("refs/unidesk/snapshots/")) fail("--source-stage-ref must be an immutable UniDesk snapshot ref");
|
||||
return result;
|
||||
}
|
||||
|
||||
function traceId(traceparent: string | undefined): string | null {
|
||||
return traceparent?.match(/^00-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$/iu)?.[1]?.toLowerCase() ?? null;
|
||||
}
|
||||
|
||||
async function emitOuterSpan(): Promise<void> {
|
||||
const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
|
||||
const context = process.env.TRACEPARENT?.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/iu);
|
||||
if (endpoint === undefined || context === undefined || context === null) return;
|
||||
const startedAtMs = Number(process.env.CICD_TRACE_STARTED_AT_MS ?? Date.now());
|
||||
const payload = {
|
||||
resourceSpans: [{
|
||||
resource: { attributes: attributes({ "service.name": process.env.OTEL_SERVICE_NAME ?? "unidesk-cicd", "unidesk.node": options.node, "hwlab.lane": options.lane, "unidesk.values_redacted": true }) },
|
||||
scopeSpans: [{
|
||||
scope: { name: "unidesk.cicd.pac_outer", version: "1" },
|
||||
spans: [{
|
||||
traceId: context[1]?.toLowerCase(),
|
||||
spanId: context[2]?.toLowerCase(),
|
||||
name: "cicd.pac.outer_event",
|
||||
kind: 2,
|
||||
startTimeUnixNano: (BigInt(startedAtMs) * 1_000_000n).toString(),
|
||||
endTimeUnixNano: (BigInt(Date.now()) * 1_000_000n).toString(),
|
||||
attributes: attributes({
|
||||
"cicd.target": options.node,
|
||||
"cicd.lane": options.lane,
|
||||
"cicd.consumer": `sentinel-${options.node.toLowerCase()}-v03`,
|
||||
"cicd.repository": "pikasTech/unidesk",
|
||||
"cicd.branch": "master",
|
||||
"cicd.source_commit": options.sourceCommit,
|
||||
"cicd.pipeline_run": options.pipelineRun,
|
||||
"cicd.delivery_class": "outer-pac-event",
|
||||
"cicd.phase": "task-rendered",
|
||||
"warning": false,
|
||||
"blocking": false,
|
||||
"valuesRedacted": true,
|
||||
}),
|
||||
status: { code: 1 },
|
||||
}],
|
||||
}],
|
||||
}],
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const timeoutMs = exporterTimeoutMs();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: process.env.TRACEPARENT ?? "" }, body: JSON.stringify(payload), signal: controller.signal });
|
||||
if (!response.ok) process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, timedOut: error instanceof Error && error.name === "AbortError", errorType: error instanceof Error ? error.name : "Error", timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function exporterTimeoutMs(): number {
|
||||
const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS ?? "300");
|
||||
return Number.isInteger(value) && value >= 50 && value <= 2_000 ? value : 300;
|
||||
}
|
||||
|
||||
function attributes(values: Record<string, unknown>): Array<Record<string, unknown>> {
|
||||
return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ key, value: attributeValue(value) }));
|
||||
}
|
||||
|
||||
function attributeValue(value: unknown): Record<string, unknown> {
|
||||
if (typeof value === "boolean") return { boolValue: value };
|
||||
if (typeof value === "number" && Number.isFinite(value)) return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
|
||||
return { stringValue: typeof value === "string" ? value : JSON.stringify(value) };
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function fail(message: string): never {
|
||||
throw new Error(message);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import https from "node:https";
|
||||
|
||||
const namespace = process.env.NAMESPACE || "";
|
||||
@@ -14,10 +15,12 @@ const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
|
||||
const manifest = JSON.parse(Buffer.from(readFileSync(0, "utf8").replace(/\s+/g, ""), "base64").toString("utf8"));
|
||||
const startedAt = Date.now();
|
||||
const traceContext = resolveTraceContext();
|
||||
|
||||
function request(method, path, body, contentType = "application/json") {
|
||||
return new Promise((resolve, reject) => {
|
||||
const headers = { authorization: `Bearer ${token}` };
|
||||
const headers = { authorization: `Bearer ${token}`, traceparent: traceContext.traceparent };
|
||||
if (traceContext.tracestate !== null) headers.tracestate = traceContext.tracestate;
|
||||
const payload = body === undefined ? null : typeof body === "string" ? body : JSON.stringify(body);
|
||||
if (payload !== null) {
|
||||
headers["content-type"] = contentType;
|
||||
@@ -86,7 +89,33 @@ if (latest.found) {
|
||||
} else if (result.status >= 200 && result.status < 300) {
|
||||
created = true;
|
||||
} else {
|
||||
process.stderr.write(result.text || `kube api POST pipelinerun status ${result.status}`);
|
||||
const error = kubernetesApiError(result, "pipelinerun-create");
|
||||
const firstBreak = { code: "kubernetes-api-request-failed", phase: "pipelinerun-create", reason: error.message, valuesRedacted: true };
|
||||
process.stdout.write(JSON.stringify({
|
||||
ok: false,
|
||||
submitted: false,
|
||||
created: false,
|
||||
reused: false,
|
||||
wait: shouldWait,
|
||||
polls: 0,
|
||||
completed: false,
|
||||
failed: true,
|
||||
terminal: true,
|
||||
stillRunning: false,
|
||||
timedOutWait: false,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
pipelineRun: compact(null),
|
||||
traceId: traceContext.traceId,
|
||||
traceparent: traceContext.traceparent,
|
||||
firstBreak,
|
||||
error,
|
||||
statusAuthority: "kubernetes-api-serviceaccount",
|
||||
parsedDownstreamCliOutput: false,
|
||||
warning: false,
|
||||
blocking: true,
|
||||
valuesRedacted: true,
|
||||
}));
|
||||
await emitSpan("cicd.tekton.pipelinerun.create", false, { error, firstBreak, httpStatus: error.httpStatus, exitCode: 1 });
|
||||
process.exit(1);
|
||||
}
|
||||
latest = await getPipelineRun();
|
||||
@@ -122,6 +151,8 @@ const output = {
|
||||
stillRunning: !terminal,
|
||||
timedOutWait: shouldWait && !terminal,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
traceId: traceContext.traceId,
|
||||
traceparent: traceContext.traceparent,
|
||||
pipelineRun: compact(latest.object),
|
||||
logsTail,
|
||||
statusAuthority: "kubernetes-api-serviceaccount",
|
||||
@@ -129,6 +160,12 @@ const output = {
|
||||
valuesRedacted: true,
|
||||
};
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
await emitSpan("cicd.tekton.pipelinerun.wait", !failed, {
|
||||
phase: failed ? "pipelinerun-terminal-failed" : completed ? "pipelinerun-terminal-succeeded" : "pipelinerun-wait",
|
||||
status: condition?.status || null,
|
||||
durationMs: Date.now() - startedAt,
|
||||
exitCode: failed ? 1 : 0,
|
||||
});
|
||||
if (failed) process.exit(1);
|
||||
|
||||
function requiredPositiveNumber(name) {
|
||||
@@ -137,6 +174,128 @@ function requiredPositiveNumber(name) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveTraceContext() {
|
||||
const incoming = process.env.TRACEPARENT || process.env.OTEL_TRACEPARENT || "";
|
||||
const matched = incoming.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/i);
|
||||
const traceId = matched?.[1]?.toLowerCase() || randomBytes(16).toString("hex");
|
||||
const parentSpanId = matched?.[2]?.toLowerCase() || null;
|
||||
const spanId = randomBytes(8).toString("hex");
|
||||
return {
|
||||
traceId,
|
||||
parentSpanId,
|
||||
spanId,
|
||||
traceparent: `00-${traceId}-${spanId}-${matched?.[3]?.toLowerCase() || "01"}`,
|
||||
tracestate: boundedHeader(process.env.TRACESTATE),
|
||||
};
|
||||
}
|
||||
|
||||
function boundedHeader(value) {
|
||||
return typeof value === "string" && value.length > 0 && value.length <= 512 ? value : null;
|
||||
}
|
||||
|
||||
function kubernetesApiError(result, phase) {
|
||||
const body = parseBody(result);
|
||||
const code = safeToken(body?.reason) || `HTTP_${result.status || 0}`;
|
||||
const message = redactSummary(body?.message || result.text || `Kubernetes API request failed with HTTP ${result.status || 0}`);
|
||||
return {
|
||||
type: "KubernetesApiError",
|
||||
code,
|
||||
phase,
|
||||
httpStatus: result.status || null,
|
||||
exitCode: 1,
|
||||
stderrSummary: message,
|
||||
message,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function safeToken(value) {
|
||||
return typeof value === "string" && /^[A-Za-z0-9_.-]{1,80}$/.test(value) ? value : null;
|
||||
}
|
||||
|
||||
function redactSummary(value) {
|
||||
return String(value || "")
|
||||
.replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/gi, "$1[REDACTED]")
|
||||
.replace(/(token|password|secret)(\s*[:=]\s*)\S+/gi, "$1$2[REDACTED]")
|
||||
.replace(/https?:\/\/[^\s/]+@/gi, "https://[REDACTED]@")
|
||||
.replace(/([?&][^=\s]+)=([^&\s]+)/g, "$1=[REDACTED]")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 480);
|
||||
}
|
||||
|
||||
async function emitSpan(name, ok, attributes) {
|
||||
const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
|
||||
if (!endpoint) return;
|
||||
const started = BigInt(startedAt) * 1_000_000n;
|
||||
const ended = BigInt(Date.now()) * 1_000_000n;
|
||||
const payload = {
|
||||
resourceSpans: [{
|
||||
resource: { attributes: otelAttributes({
|
||||
"service.name": process.env.OTEL_SERVICE_NAME || "unidesk-cicd",
|
||||
"deployment.environment": process.env.UNIDESK_LANE || null,
|
||||
"unidesk.node": process.env.UNIDESK_NODE || null,
|
||||
"unidesk.values_redacted": true,
|
||||
}) },
|
||||
scopeSpans: [{
|
||||
scope: { name: "unidesk.cicd.submit_pipelinerun", version: "1" },
|
||||
spans: [{
|
||||
traceId: traceContext.traceId,
|
||||
spanId: traceContext.spanId,
|
||||
...(traceContext.parentSpanId === null ? {} : { parentSpanId: traceContext.parentSpanId }),
|
||||
name,
|
||||
kind: 3,
|
||||
startTimeUnixNano: started.toString(),
|
||||
endTimeUnixNano: ended.toString(),
|
||||
attributes: otelAttributes({
|
||||
"cicd.consumer": process.env.UNIDESK_PAC_CONSUMER || null,
|
||||
"cicd.pipeline_run": pipelineRun,
|
||||
"k8s.namespace.name": namespace,
|
||||
"cicd.source_commit": process.env.SOURCE_COMMIT || null,
|
||||
"cicd.phase": attributes.phase || null,
|
||||
"cicd.status": attributes.status || null,
|
||||
"cicd.duration_ms": attributes.durationMs || null,
|
||||
"process.exit.code": attributes.exitCode ?? null,
|
||||
"http.response.status_code": attributes.httpStatus ?? null,
|
||||
"error.type": attributes.error?.type || null,
|
||||
"error.code": attributes.error?.code || null,
|
||||
"cicd.first_break": attributes.firstBreak?.code || null,
|
||||
"warning": false,
|
||||
"blocking": attributes.exitCode === 1,
|
||||
"valuesRedacted": true,
|
||||
}),
|
||||
status: { code: ok ? 1 : 2 },
|
||||
}],
|
||||
}],
|
||||
}],
|
||||
};
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutMs = exporterTimeoutMs();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: traceContext.traceparent }, body: JSON.stringify(payload), signal: controller.signal });
|
||||
clearTimeout(timeout);
|
||||
if (!response.ok) process.stderr.write(JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, traceId: traceContext.traceId, valuesRedacted: true }) + "\n");
|
||||
} catch (error) {
|
||||
process.stderr.write(JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, errorType: error instanceof Error ? error.name : "Error", timedOut: error instanceof Error && error.name === "AbortError", timeoutMs: exporterTimeoutMs(), traceId: traceContext.traceId, valuesRedacted: true }) + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
function exporterTimeoutMs() {
|
||||
const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS || "300");
|
||||
return Number.isInteger(value) && value >= 50 && value <= 2000 ? value : 300;
|
||||
}
|
||||
|
||||
function otelAttributes(values) {
|
||||
return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ key, value: otelValue(value) }));
|
||||
}
|
||||
|
||||
function otelValue(value) {
|
||||
if (typeof value === "boolean") return { boolValue: value };
|
||||
if (typeof value === "number" && Number.isFinite(value)) return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
|
||||
return { stringValue: typeof value === "string" ? value : JSON.stringify(value) };
|
||||
}
|
||||
|
||||
async function pipelineRunLogsTail() {
|
||||
const selector = encodeURIComponent(`tekton.dev/pipelineRun=${pipelineRun}`);
|
||||
const podsResult = await request("GET", `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${selector}`);
|
||||
|
||||
Reference in New Issue
Block a user