mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 补齐 CI/CD OTel 与失败投影

Browse Source
This commit is contained in:
Codex
2026-07-13 12:16:55 +02:00
parent 67cb6032f8
commit 083e63bff0
13 changed files with 734 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.tekton/web-probe-sentinel-jd01-pac.yaml
+97 -21
View File
@@ -13,43 +13,119 @@ metadata:
hwlab.pikastech.local/source-commit: "{{revision}}"
spec:
timeouts:
pipeline: 300s
pipeline: 900s
taskRunTemplate:
serviceAccountName: default
podTemplate:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
securityContext:
fsGroup: 1000
pipelineSpec:
tasks:
- name: publish
taskSpec:
volumes:
- name: cache
persistentVolumeClaim:
claimName: hwlab-git-mirror-cache
- name: git-ssh
secret:
secretName: git-mirror-github-ssh
defaultMode: 256
- name: workspace
emptyDir:
sizeLimit: 8Gi
- name: buildkit-state
hostPath:
path: /var/lib/unidesk/web-probe-sentinel/buildkit-jd01
type: DirectoryOrCreate
- name: tmp
emptyDir: {}
steps:
- name: sentinel-publish
- name: source
image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1
imagePullPolicy: IfNotPresent
workingDir: /workspace
env:
- name: GIT_READ_URL
value: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git
- name: SOURCE_COMMIT
value: "{{revision}}"
- name: SOURCE_STAGE_REF
value: refs/unidesk/snapshots/gitea-actions/unidesk-master/{{revision}}
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
value: "{{otel_traces_endpoint}}"
- name: OTEL_EXPORTER_TIMEOUT_MS
value: "{{otel_exporter_timeout_ms}}"
- name: OTEL_SERVICE_NAME
value: "{{otel_service_name}}"
- name: OTEL_TRACES_SAMPLER_ARG
value: "{{otel_sampling_ratio}}"
script: |
#!/bin/sh
set -eu
rm -rf source
git clone --filter=blob:none --no-checkout "$GIT_READ_URL" source
cd source
git fetch --depth=1 --filter=blob:none origin "+$SOURCE_STAGE_REF:refs/remotes/origin/unidesk-source-snapshot"
git checkout --detach "$SOURCE_COMMIT"
bun scripts/cli.ts web-probe sentinel publish-current \
export CICD_TRACE_STARTED_AT_MS=$(($(date +%s) * 1000))
export TRACEPARENT=$(node -e 'const c=require("node:crypto");process.stdout.write(`00-${c.randomBytes(16).toString("hex")}-${c.randomBytes(8).toString("hex")}-01`)')
rm -rf /workspace/bootstrap /workspace/generated
git clone --filter=blob:none --no-checkout http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git /workspace/bootstrap
cd /workspace/bootstrap
git fetch --depth=1 --filter=blob:none origin "+refs/unidesk/snapshots/gitea-actions/unidesk-master/{{revision}}:refs/remotes/origin/unidesk-source-snapshot"
git checkout --detach "{{revision}}"
bun scripts/native/cicd/render-sentinel-publish-task.ts \
--node JD01 \
--lane v03 \
--sentinel jd01-web-probe-sentinel \
--confirm \
--wait \
--timeout-seconds 180 \
--source-commit "$SOURCE_COMMIT" \
--source-stage-ref "$SOURCE_STAGE_REF" \
--source-authority gitea-snapshot
--pipeline-run "$(context.pipelineRun.name)" \
--source-commit "{{revision}}" \
--source-stage-ref "refs/unidesk/snapshots/gitea-actions/unidesk-master/{{revision}}" \
--source-authority gitea-snapshot \
--output-dir /workspace/generated
exec /workspace/generated/source.sh
volumeMounts:
- name: workspace
mountPath: /workspace
- name: git-ssh
mountPath: /git-ssh
readOnly: true
- name: prepare-buildkit-state
image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1
imagePullPolicy: IfNotPresent
script: |
#!/bin/sh
set -eu
mkdir -p /home/user/.local/share/buildkit
chown -R 1000:1000 /home/user/.local/share/buildkit
securityContext:
runAsUser: 0
runAsGroup: 0
volumeMounts:
- name: buildkit-state
mountPath: /home/user/.local/share/buildkit
- name: image-build
image: 127.0.0.1:5000/hwlab/buildkit:rootless
imagePullPolicy: IfNotPresent
env:
- name: BUILDKITD_FLAGS
value: --oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host
script: |
#!/bin/sh
exec /workspace/generated/image-build.sh
securityContext:
privileged: true
runAsUser: 1000
runAsGroup: 1000
volumeMounts:
- name: workspace
mountPath: /workspace
- name: buildkit-state
mountPath: /home/user/.local/share/buildkit
- name: tmp
mountPath: /tmp
- name: publish
image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1
imagePullPolicy: IfNotPresent
script: |
#!/bin/sh
exec /workspace/generated/publish.sh
volumeMounts:
- name: workspace
mountPath: /workspace
- name: cache
mountPath: /cache
- name: git-ssh
mountPath: /git-ssh
readOnly: true
.tekton/web-probe-sentinel-nc01-pac.yaml
+97 -21
View File
@@ -13,43 +13,119 @@ metadata:
hwlab.pikastech.local/source-commit: "{{revision}}"
spec:
timeouts:
pipeline: 300s
pipeline: 900s
taskRunTemplate:
serviceAccountName: default
podTemplate:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
securityContext:
fsGroup: 1000
pipelineSpec:
tasks:
- name: publish
taskSpec:
volumes:
- name: cache
persistentVolumeClaim:
claimName: hwlab-git-mirror-cache
- name: git-ssh
secret:
secretName: git-mirror-github-ssh
defaultMode: 256
- name: workspace
emptyDir:
sizeLimit: 8Gi
- name: buildkit-state
hostPath:
path: /var/lib/unidesk/web-probe-sentinel/buildkit-nc01
type: DirectoryOrCreate
- name: tmp
emptyDir: {}
steps:
- name: sentinel-publish
- name: source
image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1
imagePullPolicy: IfNotPresent
workingDir: /workspace
env:
- name: GIT_READ_URL
value: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git
- name: SOURCE_COMMIT
value: "{{revision}}"
- name: SOURCE_STAGE_REF
value: refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01/{{revision}}
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
value: "{{otel_traces_endpoint}}"
- name: OTEL_EXPORTER_TIMEOUT_MS
value: "{{otel_exporter_timeout_ms}}"
- name: OTEL_SERVICE_NAME
value: "{{otel_service_name}}"
- name: OTEL_TRACES_SAMPLER_ARG
value: "{{otel_sampling_ratio}}"
script: |
#!/bin/sh
set -eu
rm -rf source
git clone --filter=blob:none --no-checkout "$GIT_READ_URL" source
cd source
git fetch --depth=1 --filter=blob:none origin "+$SOURCE_STAGE_REF:refs/remotes/origin/unidesk-source-snapshot"
git checkout --detach "$SOURCE_COMMIT"
bun scripts/cli.ts web-probe sentinel publish-current \
export CICD_TRACE_STARTED_AT_MS=$(($(date +%s) * 1000))
export TRACEPARENT=$(node -e 'const c=require("node:crypto");process.stdout.write(`00-${c.randomBytes(16).toString("hex")}-${c.randomBytes(8).toString("hex")}-01`)')
rm -rf /workspace/bootstrap /workspace/generated
git clone --filter=blob:none --no-checkout http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git /workspace/bootstrap
cd /workspace/bootstrap
git fetch --depth=1 --filter=blob:none origin "+refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01/{{revision}}:refs/remotes/origin/unidesk-source-snapshot"
git checkout --detach "{{revision}}"
bun scripts/native/cicd/render-sentinel-publish-task.ts \
--node NC01 \
--lane v03 \
--sentinel nc01-web-probe-sentinel \
--confirm \
--wait \
--timeout-seconds 180 \
--source-commit "$SOURCE_COMMIT" \
--source-stage-ref "$SOURCE_STAGE_REF" \
--source-authority gitea-snapshot
--pipeline-run "$(context.pipelineRun.name)" \
--source-commit "{{revision}}" \
--source-stage-ref "refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01/{{revision}}" \
--source-authority gitea-snapshot \
--output-dir /workspace/generated
exec /workspace/generated/source.sh
volumeMounts:
- name: workspace
mountPath: /workspace
- name: git-ssh
mountPath: /git-ssh
readOnly: true
- name: prepare-buildkit-state
image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1
imagePullPolicy: IfNotPresent
script: |
#!/bin/sh
set -eu
mkdir -p /home/user/.local/share/buildkit
chown -R 1000:1000 /home/user/.local/share/buildkit
securityContext:
runAsUser: 0
runAsGroup: 0
volumeMounts:
- name: buildkit-state
mountPath: /home/user/.local/share/buildkit
- name: image-build
image: 127.0.0.1:5000/hwlab/buildkit:rootless
imagePullPolicy: IfNotPresent
env:
- name: BUILDKITD_FLAGS
value: --oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host
script: |
#!/bin/sh
exec /workspace/generated/image-build.sh
securityContext:
privileged: true
runAsUser: 1000
runAsGroup: 1000
volumeMounts:
- name: workspace
mountPath: /workspace
- name: buildkit-state
mountPath: /home/user/.local/share/buildkit
- name: tmp
mountPath: /tmp
- name: publish
image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1
imagePullPolicy: IfNotPresent
script: |
#!/bin/sh
exec /workspace/generated/publish.sh
volumeMounts:
- name: workspace
mountPath: /workspace
- name: cache
mountPath: /cache
- name: git-ssh
mountPath: /git-ssh
readOnly: true
config/platform-infra/pipelines-as-code.yaml
+18
View File
@@ -19,6 +19,20 @@ defaults:
consumerId: hwlab-nc01-v03
display:
timeZone: Asia/Shanghai
observability:
configRef: config/platform-infra/observability.yaml
tracesEndpoint: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces
serviceName: unidesk-cicd
propagation:
- tracecontext
- baggage
sampling:
mode: parentbased_traceidratio
ratio: 1
exporterFailure:
warning: true
blocking: false
timeoutMs: 300
release:
version: v0.48.0
manifestUrl: https://github.com/openshift-pipelines/pipelines-as-code/releases/download/v0.48.0/release.k8s.yaml
@@ -411,6 +425,10 @@ templates:
pipeline_name: "hwlab-web-probe-sentinel-${nodeLower}-pac"
pipeline_run_prefix: "hwlab-web-probe-sentinel-${nodeLower}"
service_account: default
otel_traces_endpoint: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces
otel_service_name: "unidesk-cicd-sentinel-${nodeLower}"
otel_sampling_ratio: "1"
otel_exporter_timeout_ms: "300"
hwlabV03:
id: "hwlab-${nodeLower}-v03"
name: "hwlab-${nodeLower}-v03"
docs/reference/platform-infra.md
+2 -1
View File
@@ -25,7 +25,8 @@
- Durable inbox 只是 webhook delivery 的持久接收与重试 journal，不是业务 source/ref 的第二份真相，也不得被 PaC、Tekton、GitOps、Argo 或 runtime 当作源码/read model。最终 source authority 仍只有 Gitea authority branch 与 immutable snapshot；禁止新增 polling/read-model fallback 或第二 source authority。
- 默认 PaC 入口是 `bun scripts/cli.ts platform-infra pipelines-as-code status|history --target <node>`。`status` 应显示 webhook、PipelineRun/TaskRun duration、image status、env identity、digest、GitOps commit、Argo revision 和 runtime provenance；`history` 必须读取 target node 上的实时对象、显式报告 read error，并支持 consumer 与 PipelineRun id 下钻。`history --id` 必须通过运行面 provenance 与 PipelineRun prefix 唯一解析实际 consumer；零匹配、多匹配或显式 consumer 不一致都 fail-closed，不能回退到默认 consumer 或默认 node。
- Sentinel 内部 publish capability 由 `config/platform-infra/pipelines-as-code.yaml#capabilities.sentinelInternalPublish` 单一拥有，默认 `enabled=false`。Pod env、ownerReferences、ServiceAccount、label 与 annotation 不能证明 creator；#1769 的 admission-owned provenance 与最小权限边界落地前不得启用。`.tekton` 继续走既有自动 `publish-current`，不能退化成人工 publish。
- PaC 观察必须把 PipelineRun 分类为外层 `outer-pac-event`、内层 `inner-deterministic-publish` 或 `unknown`。只有唯一外层 PaC push event 能驱动 latest、status、debug-step 与兼容 closeout；内层只能展示为未绑定执行观察，缺少 admission-owned 父子证据时标记 `unproven`，禁止按名称前缀推断关系。
- PaC 观察必须把 PipelineRun 分类为外层 `outer-pac-event`、历史内层 `inner-deterministic-publish` 或 `unknown`。只有唯一外层 PaC push event 能驱动 latest、status、debug-step 与兼容 closeout；Web sentinel 的 source、image、artifact、GitOps 步骤必须直接位于该 outer PipelineRun，禁止再创建 nested PipelineRun。历史内层运行只作为未绑定执行观察，缺少 admission-owned 父子证据时标记 `unproven`，禁止按名称前缀推断关系。
- CI/CD OTel endpoint、service/resource 属性、采样与 exporter timeout/failure policy 由 `config/platform-infra/pipelines-as-code.yaml` 引用现有 `config/platform-infra/observability.yaml` 并负责渲染。outer PaC 在最早可控 step 生成 W3C trace context，后续 source、image、artifact、GitOps 与 helper 复用同一 traceId；export 失败或超时只输出 `warning=true`、`blocking=false` 的有界事件，不改变 PipelineRun、TaskRun 或业务 step 终态。
- `config/platform-infra/pipelines-as-code.yaml` 可以声明多个 repository 和 consumer。`cicd status --node <NODE>` 用于 node 汇总，`history --target <NODE>` 用于全 consumer 审计，`status --target <NODE> --consumer <id>` 用于 consumer 只读状态；不得跨 repository 混合 PipelineRun 或 env reuse 证据。
- PaC Repository CR `spec.url` 必须匹配 Gitea webhook payload 的 public repository URL；ClusterIP/service URL 只属于 `cloneUrl` 与 `params.git_read_url`。Repository params 必须使用该 consumer 的 node-specific snapshot prefix。
- GitHub 始终是 upstream write authority。Bridge 只允许 GitHub -> Gitea 更新 YAML 声明的 branch 与 immutable snapshot，不得反向写 GitHub，也不得增加 polling 或第二 trigger path。
scripts/native/cicd/render-sentinel-publish-task.ts
+152
View File
@@ -0,0 +1,152 @@
import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { hwlabRuntimeLaneSpecForNode } from "../../src/hwlab-node-lanes";
import { loadSentinelCicdState } from "../../src/hwlab-node-web-sentinel-cicd";
import {
sentinelImageBuildProxyEnv,
sentinelPublishImageBuildShell,
sentinelPublishShell,
sentinelPublishSourceShell,
} from "../../src/hwlab-node-web-sentinel-cicd-jobs";
const options = parseOptions(process.argv.slice(2));
const sourceAuthority = options.sourceAuthority === "gitea-snapshot" ? "gitea-snapshot" : fail("--source-authority must be gitea-snapshot");
const state = loadSentinelCicdState(
hwlabRuntimeLaneSpecForNode(options.lane, options.node),
options.sentinel,
30,
"cached",
{
commit: options.sourceCommit,
stageRef: options.sourceStageRef,
mirrorCommit: options.sourceCommit,
sourceAuthority,
},
);
const outputDir = resolve(options.outputDir);
mkdirSync(outputDir, { recursive: true });
const traceSetup = [
"if [ -f /workspace/meta/traceparent ]; then TRACEPARENT=$(cat /workspace/meta/traceparent); export TRACEPARENT; fi",
"if [ -f /workspace/meta/tracestate ]; then TRACESTATE=$(cat /workspace/meta/tracestate); export TRACESTATE; fi",
].join("\n");
const proxySetup = sentinelImageBuildProxyEnv(state).map(({ name, value }) => `export ${name}=${shellQuote(value)}`).join("\n");
writeExecutable("source.sh", [
"#!/bin/sh",
"set -eu",
`incoming_traceparent=${shellQuote(process.env.TRACEPARENT ?? "")}`,
`incoming_tracestate=${shellQuote(process.env.TRACESTATE ?? "")}`,
sentinelPublishSourceShell(state, options.pipelineRun),
"if [ -n \"$incoming_traceparent\" ]; then printf '%s' \"$incoming_traceparent\" > /workspace/meta/traceparent; fi",
"if [ -n \"$incoming_tracestate\" ]; then printf '%s' \"$incoming_tracestate\" > /workspace/meta/tracestate; fi",
].join("\n"));
writeExecutable("image-build.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishImageBuildShell(state, options.pipelineRun)].join("\n"));
writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true)].join("\n"));
await emitOuterSpan();
process.stdout.write(`${JSON.stringify({ ok: true, action: "render-sentinel-publish-task", node: options.node, lane: options.lane, sentinel: options.sentinel, pipelineRun: options.pipelineRun, sourceCommit: options.sourceCommit, outputDir, traceId: traceId(process.env.TRACEPARENT), valuesRedacted: true })}\n`);
function writeExecutable(name: string, content: string): void {
const path = resolve(outputDir, name);
writeFileSync(path, `${content}\n`, "utf8");
chmodSync(path, 0o755);
}
function parseOptions(args: string[]): Record<"node" | "lane" | "sentinel" | "pipelineRun" | "sourceCommit" | "sourceStageRef" | "sourceAuthority" | "outputDir", string> {
const values = new Map<string, string>();
for (let index = 0; index < args.length; index += 2) {
const key = args[index];
const value = args[index + 1];
if (key === undefined || value === undefined || !key.startsWith("--")) fail("render-sentinel-publish-task requires key/value options");
values.set(key.slice(2), value);
}
const required = (key: string): string => values.get(key) || fail(`--${key} is required`);
const result = {
node: required("node"),
lane: required("lane"),
sentinel: required("sentinel"),
pipelineRun: required("pipeline-run"),
sourceCommit: required("source-commit"),
sourceStageRef: required("source-stage-ref"),
sourceAuthority: required("source-authority"),
outputDir: required("output-dir"),
};
if (!/^[0-9a-f]{40}$/u.test(result.sourceCommit)) fail("--source-commit must be a 40-character lowercase Git commit");
for (const key of ["node", "lane", "sentinel", "pipelineRun"] as const) if (!/^[A-Za-z0-9._-]+$/u.test(result[key])) fail(`--${key} must be a simple id`);
if (!result.sourceStageRef.startsWith("refs/unidesk/snapshots/")) fail("--source-stage-ref must be an immutable UniDesk snapshot ref");
return result;
}
function traceId(traceparent: string | undefined): string | null {
return traceparent?.match(/^00-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$/iu)?.[1]?.toLowerCase() ?? null;
}
async function emitOuterSpan(): Promise<void> {
const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
const context = process.env.TRACEPARENT?.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/iu);
if (endpoint === undefined || context === undefined || context === null) return;
const startedAtMs = Number(process.env.CICD_TRACE_STARTED_AT_MS ?? Date.now());
const payload = {
resourceSpans: [{
resource: { attributes: attributes({ "service.name": process.env.OTEL_SERVICE_NAME ?? "unidesk-cicd", "unidesk.node": options.node, "hwlab.lane": options.lane, "unidesk.values_redacted": true }) },
scopeSpans: [{
scope: { name: "unidesk.cicd.pac_outer", version: "1" },
spans: [{
traceId: context[1]?.toLowerCase(),
spanId: context[2]?.toLowerCase(),
name: "cicd.pac.outer_event",
kind: 2,
startTimeUnixNano: (BigInt(startedAtMs) * 1_000_000n).toString(),
endTimeUnixNano: (BigInt(Date.now()) * 1_000_000n).toString(),
attributes: attributes({
"cicd.target": options.node,
"cicd.lane": options.lane,
"cicd.consumer": `sentinel-${options.node.toLowerCase()}-v03`,
"cicd.repository": "pikasTech/unidesk",
"cicd.branch": "master",
"cicd.source_commit": options.sourceCommit,
"cicd.pipeline_run": options.pipelineRun,
"cicd.delivery_class": "outer-pac-event",
"cicd.phase": "task-rendered",
"warning": false,
"blocking": false,
"valuesRedacted": true,
}),
status: { code: 1 },
}],
}],
}],
};
const controller = new AbortController();
const timeoutMs = exporterTimeoutMs();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: process.env.TRACEPARENT ?? "" }, body: JSON.stringify(payload), signal: controller.signal });
if (!response.ok) process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`);
} catch (error) {
process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, timedOut: error instanceof Error && error.name === "AbortError", errorType: error instanceof Error ? error.name : "Error", timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`);
} finally {
clearTimeout(timer);
}
}
function exporterTimeoutMs(): number {
const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS ?? "300");
return Number.isInteger(value) && value >= 50 && value <= 2_000 ? value : 300;
}
function attributes(values: Record<string, unknown>): Array<Record<string, unknown>> {
return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ key, value: attributeValue(value) }));
}
function attributeValue(value: unknown): Record<string, unknown> {
if (typeof value === "boolean") return { boolValue: value };
if (typeof value === "number" && Number.isFinite(value)) return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
return { stringValue: typeof value === "string" ? value : JSON.stringify(value) };
}
function shellQuote(value: string): string {
return `'${value.replaceAll("'", `'"'"'`)}'`;
}
function fail(message: string): never {
throw new Error(message);
}
scripts/native/cicd/submit-pipelinerun.mjs
+161 -2
View File
@@ -1,4 +1,5 @@
import { readFileSync } from "node:fs";
import { randomBytes } from "node:crypto";
import https from "node:https";
const namespace = process.env.NAMESPACE || "";
@@ -14,10 +15,12 @@ const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token"
const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
const manifest = JSON.parse(Buffer.from(readFileSync(0, "utf8").replace(/\s+/g, ""), "base64").toString("utf8"));
const startedAt = Date.now();
const traceContext = resolveTraceContext();
function request(method, path, body, contentType = "application/json") {
return new Promise((resolve, reject) => {
const headers = { authorization: `Bearer ${token}` };
const headers = { authorization: `Bearer ${token}`, traceparent: traceContext.traceparent };
if (traceContext.tracestate !== null) headers.tracestate = traceContext.tracestate;
const payload = body === undefined ? null : typeof body === "string" ? body : JSON.stringify(body);
if (payload !== null) {
headers["content-type"] = contentType;
@@ -86,7 +89,33 @@ if (latest.found) {
} else if (result.status >= 200 && result.status < 300) {
created = true;
} else {
process.stderr.write(result.text || `kube api POST pipelinerun status ${result.status}`);
const error = kubernetesApiError(result, "pipelinerun-create");
const firstBreak = { code: "kubernetes-api-request-failed", phase: "pipelinerun-create", reason: error.message, valuesRedacted: true };
process.stdout.write(JSON.stringify({
ok: false,
submitted: false,
created: false,
reused: false,
wait: shouldWait,
polls: 0,
completed: false,
failed: true,
terminal: true,
stillRunning: false,
timedOutWait: false,
elapsedMs: Date.now() - startedAt,
pipelineRun: compact(null),
traceId: traceContext.traceId,
traceparent: traceContext.traceparent,
firstBreak,
error,
statusAuthority: "kubernetes-api-serviceaccount",
parsedDownstreamCliOutput: false,
warning: false,
blocking: true,
valuesRedacted: true,
}));
await emitSpan("cicd.tekton.pipelinerun.create", false, { error, firstBreak, httpStatus: error.httpStatus, exitCode: 1 });
process.exit(1);
}
latest = await getPipelineRun();
@@ -122,6 +151,8 @@ const output = {
stillRunning: !terminal,
timedOutWait: shouldWait && !terminal,
elapsedMs: Date.now() - startedAt,
traceId: traceContext.traceId,
traceparent: traceContext.traceparent,
pipelineRun: compact(latest.object),
logsTail,
statusAuthority: "kubernetes-api-serviceaccount",
@@ -129,6 +160,12 @@ const output = {
valuesRedacted: true,
};
process.stdout.write(JSON.stringify(output));
await emitSpan("cicd.tekton.pipelinerun.wait", !failed, {
phase: failed ? "pipelinerun-terminal-failed" : completed ? "pipelinerun-terminal-succeeded" : "pipelinerun-wait",
status: condition?.status || null,
durationMs: Date.now() - startedAt,
exitCode: failed ? 1 : 0,
});
if (failed) process.exit(1);
function requiredPositiveNumber(name) {
@@ -137,6 +174,128 @@ function requiredPositiveNumber(name) {
return value;
}
function resolveTraceContext() {
const incoming = process.env.TRACEPARENT || process.env.OTEL_TRACEPARENT || "";
const matched = incoming.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/i);
const traceId = matched?.[1]?.toLowerCase() || randomBytes(16).toString("hex");
const parentSpanId = matched?.[2]?.toLowerCase() || null;
const spanId = randomBytes(8).toString("hex");
return {
traceId,
parentSpanId,
spanId,
traceparent: `00-${traceId}-${spanId}-${matched?.[3]?.toLowerCase() || "01"}`,
tracestate: boundedHeader(process.env.TRACESTATE),
};
}
function boundedHeader(value) {
return typeof value === "string" && value.length > 0 && value.length <= 512 ? value : null;
}
function kubernetesApiError(result, phase) {
const body = parseBody(result);
const code = safeToken(body?.reason) || `HTTP_${result.status || 0}`;
const message = redactSummary(body?.message || result.text || `Kubernetes API request failed with HTTP ${result.status || 0}`);
return {
type: "KubernetesApiError",
code,
phase,
httpStatus: result.status || null,
exitCode: 1,
stderrSummary: message,
message,
valuesRedacted: true,
};
}
function safeToken(value) {
return typeof value === "string" && /^[A-Za-z0-9_.-]{1,80}$/.test(value) ? value : null;
}
function redactSummary(value) {
return String(value || "")
.replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/gi, "$1[REDACTED]")
.replace(/(token|password|secret)(\s*[:=]\s*)\S+/gi, "$1$2[REDACTED]")
.replace(/https?:\/\/[^\s/]+@/gi, "https://[REDACTED]@")
.replace(/([?&][^=\s]+)=([^&\s]+)/g, "$1=[REDACTED]")
.replace(/\s+/g, " ")
.trim()
.slice(0, 480);
}
async function emitSpan(name, ok, attributes) {
const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
if (!endpoint) return;
const started = BigInt(startedAt) * 1_000_000n;
const ended = BigInt(Date.now()) * 1_000_000n;
const payload = {
resourceSpans: [{
resource: { attributes: otelAttributes({
"service.name": process.env.OTEL_SERVICE_NAME || "unidesk-cicd",
"deployment.environment": process.env.UNIDESK_LANE || null,
"unidesk.node": process.env.UNIDESK_NODE || null,
"unidesk.values_redacted": true,
}) },
scopeSpans: [{
scope: { name: "unidesk.cicd.submit_pipelinerun", version: "1" },
spans: [{
traceId: traceContext.traceId,
spanId: traceContext.spanId,
...(traceContext.parentSpanId === null ? {} : { parentSpanId: traceContext.parentSpanId }),
name,
kind: 3,
startTimeUnixNano: started.toString(),
endTimeUnixNano: ended.toString(),
attributes: otelAttributes({
"cicd.consumer": process.env.UNIDESK_PAC_CONSUMER || null,
"cicd.pipeline_run": pipelineRun,
"k8s.namespace.name": namespace,
"cicd.source_commit": process.env.SOURCE_COMMIT || null,
"cicd.phase": attributes.phase || null,
"cicd.status": attributes.status || null,
"cicd.duration_ms": attributes.durationMs || null,
"process.exit.code": attributes.exitCode ?? null,
"http.response.status_code": attributes.httpStatus ?? null,
"error.type": attributes.error?.type || null,
"error.code": attributes.error?.code || null,
"cicd.first_break": attributes.firstBreak?.code || null,
"warning": false,
"blocking": attributes.exitCode === 1,
"valuesRedacted": true,
}),
status: { code: ok ? 1 : 2 },
}],
}],
}],
};
try {
const controller = new AbortController();
const timeoutMs = exporterTimeoutMs();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: traceContext.traceparent }, body: JSON.stringify(payload), signal: controller.signal });
clearTimeout(timeout);
if (!response.ok) process.stderr.write(JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, traceId: traceContext.traceId, valuesRedacted: true }) + "\n");
} catch (error) {
process.stderr.write(JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, errorType: error instanceof Error ? error.name : "Error", timedOut: error instanceof Error && error.name === "AbortError", timeoutMs: exporterTimeoutMs(), traceId: traceContext.traceId, valuesRedacted: true }) + "\n");
}
}
function exporterTimeoutMs() {
const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS || "300");
return Number.isInteger(value) && value >= 50 && value <= 2000 ? value : 300;
}
function otelAttributes(values) {
return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ key, value: otelValue(value) }));
}
function otelValue(value) {
if (typeof value === "boolean") return { boolValue: value };
if (typeof value === "number" && Number.isFinite(value)) return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
return { stringValue: typeof value === "string" ? value : JSON.stringify(value) };
}
async function pipelineRunLogsTail() {
const selector = encodeURIComponent(`tekton.dev/pipelineRun=${pipelineRun}`);
const podsResult = await request("GET", `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${selector}`);
scripts/src/cicd-delivery-authority.test.ts
+21 -2
View File
@@ -262,7 +262,7 @@ describe("migrated CLI 执行 guard 与提示清理", () => {
});
});
test("Web sentinel 保持旧自动入口，不把 default SA、Pod env 或可写 metadata 当 creator proof", () => {
test("Web sentinel 自动入口保持唯一 outer PaC，不把 default SA、Pod env 或可写 metadata 当 creator proof", () => {
const pacYaml = readFileSync("config/platform-infra/pipelines-as-code.yaml", "utf8");
expect(pacYaml).toContain("sentinelInternalPublish:");
expect(pacYaml).toContain("enabled: false");
@@ -271,7 +271,8 @@ describe("migrated CLI 执行 guard 与提示清理", () => {
for (const node of ["jd01", "nc01"] as const) {
const source = readFileSync(`.tekton/web-probe-sentinel-${node}-pac.yaml`, "utf8");
expect(source).toContain("serviceAccountName: default");
expect(source).toContain("web-probe sentinel publish-current \\");
expect(source).toContain("render-sentinel-publish-task.ts");
expect(source).not.toContain("web-probe sentinel publish-current");
expect(source).not.toContain("pac-publish-current");
expect(source).not.toContain("UNIDESK_PAC_INTERNAL_ENTRYPOINT");
expect(source).not.toContain("UNIDESK_PAC_POD_UID");
@@ -532,6 +533,24 @@ describe("migrated CLI 执行 guard 与提示清理", () => {
expect(() => resolvePacHistoryConsumerSelection(consumers, "NC01", "hwlab-nc01-v03", "hwlab-web-probe-sentinel-nc01-fixture")).toThrow("belongs to sentinel-nc01-v03");
});
test("Web sentinel PaC 使用唯一 outer PipelineRun 并贯穿有界 OTel 合同", () => {
for (const node of ["nc01", "jd01"]) {
const pipeline = readFileSync(`.tekton/web-probe-sentinel-${node}-pac.yaml`, "utf8");
expect(pipeline).toContain("render-sentinel-publish-task.ts");
expect(pipeline).toContain("OTEL_EXPORTER_TIMEOUT_MS");
expect(pipeline).toContain("TRACEPARENT");
expect(pipeline).not.toContain("publish-current");
expect(pipeline).not.toContain("submit-pipelinerun.mjs");
}
const submitHelper = readFileSync("scripts/native/cicd/submit-pipelinerun.mjs", "utf8");
expect(submitHelper).toContain("parentSpanId");
expect(submitHelper).toContain("AbortController");
expect(submitHelper).toContain("warning: true, blocking: false");
const historyCollector = readFileSync("scripts/src/platform-infra-pipelines-as-code-remote.sh", "utf8");
expect(historyCollector).toContain("state.firstBreak");
expect(historyCollector).toContain("state.traceId");
});
test("PaC、HWLAB scoped help 的真实 CLI 路由可达且默认入口不泄漏 delivery mutation", () => {
const pacDefault = runCliJson(["platform-infra", "pipelines-as-code", "--help"]);
const pacBootstrap = runCliJson(["platform-infra", "pipelines-as-code", "help", "platform-bootstrap"]);
scripts/src/cicd-node-status.ts
+4 -1
View File
@@ -90,6 +90,9 @@ function renderNodeStatus(result: Record<string, unknown>, full: boolean): Rende
`${stringValue(record(row.argo).sync)}/${stringValue(record(row.argo).health)}`,
stringValue(record(record(row.argo).revisionRelation).relation),
runtimeText(record(row.runtime)),
short(stringValue(row.traceId), 10),
stringValue(record(row.firstBreak).code),
stringValue(record(row.error).code),
stringValue(row.reason),
]);
const detailRows = consumers.map((row) => [
@@ -111,7 +114,7 @@ function renderNodeStatus(result: Record<string, unknown>, full: boolean): Rende
]]),
"",
"CONSUMERS",
...(rows.length === 0 ? ["-"] : table(["CONSUMER", "READY", "PIPELINE", "DUR", "SOURCE", "GITOPS", "ARGO", "RELATION", "RUNTIME", "REASON"], rows)),
...(rows.length === 0 ? ["-"] : table(["CONSUMER", "READY", "PIPELINE", "DUR", "SOURCE", "GITOPS", "ARGO", "RELATION", "RUNTIME", "TRACE", "FIRST_BREAK", "ERROR", "REASON"], rows)),
"",
...(full ? [
"DETAIL",
scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts
+14 -10
View File
@@ -403,6 +403,10 @@ function runSentinelPublishPipelineRunInCluster(state: SentinelCicdState, manife
payload: Object.keys(payload).length === 0 ? { ok: false, status: completed ? "result-missing" : failed ? "failed" : "timeout", valuesRedacted: true } : payload,
polls: parsed.polls ?? null,
elapsedMs: parsed.elapsedMs ?? result.durationMs ?? null,
traceId: parsed.traceId ?? null,
traceparent: parsed.traceparent ?? null,
firstBreak: parsed.firstBreak ?? null,
error: parsed.error ?? null,
probe: {
nativeSubmit: parsed,
capture: compactCommand(result),
@@ -560,7 +564,7 @@ function requireSentinelBuildkitImage(state: SentinelCicdState): string {
return image;
}
function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> {
export function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> {
const proxy = state.spec.networkProfile.imageBuildProxy;
const noProxy = proxy.noProxy.join(",");
return [
@@ -575,7 +579,7 @@ function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: str
];
}
function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string {
export function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string {
const monitorWeb = record(state.image.monitorWeb);
const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => {
if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths");
@@ -604,8 +608,8 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
": > \"$event_log\"",
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
"emit_stage() { stage=$1; status=$2; started=$3; current_phase=$stage; export current_phase; finished=$(now_ms); elapsed=$((finished - started)); trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"traceId\":\"%s\",\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" \"$trace_id\" | tee -a \"$event_log\"; }",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"child-process-exit-nonzero\",\"phase\":\"%s\",\"reason\":\"publish step command exited non-zero\",\"valuesRedacted\":true},\"error\":{\"type\":\"ChildProcessError\",\"code\":\"EXIT_NONZERO\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"${current_phase:-source}\" \"${current_phase:-source}\" \"$code\" \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
"trap emit_failed EXIT",
"started_ms=$(now_ms)",
"write_meta started_ms \"$started_ms\"",
@@ -703,7 +707,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string):
].join("\n");
}
function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string {
export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string {
const monitorWeb = record(state.image.monitorWeb);
const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode");
const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode");
@@ -727,8 +731,8 @@ function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: strin
"build_log=/workspace/image-build.log",
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
"write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }",
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
"emit_stage() { stage=$1; status=$2; started=$3; current_phase=$stage; export current_phase; finished=$(now_ms); elapsed=$((finished - started)); trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"traceId\":\"%s\",\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" \"$trace_id\" | tee -a \"$event_log\"; }",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"child-process-exit-nonzero\",\"phase\":\"%s\",\"reason\":\"image build command exited non-zero\",\"valuesRedacted\":true},\"error\":{\"type\":\"ChildProcessError\",\"code\":\"EXIT_NONZERO\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"${current_phase:-image-build}\" \"${current_phase:-image-build}\" \"$code\" \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }",
"trap emit_failed EXIT",
"cd /workspace/source",
"image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi",
@@ -767,7 +771,7 @@ function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: strin
].join("\n");
}
function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
export function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string {
const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : [];
const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({
path: file.path,
@@ -786,8 +790,8 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish
"build_log=/workspace/image-build.log",
"now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }",
"read_meta() { cat \"$meta_dir/$1\"; }",
"emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }",
"emit_stage() { stage=$1; status=$2; started=$3; current_phase=$stage; export current_phase; finished=$(now_ms); elapsed=$((finished - started)); trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"traceId\":\"%s\",\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" \"$trace_id\"; }",
"emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"child-process-exit-nonzero\",\"phase\":\"%s\",\"reason\":\"GitOps publish command exited non-zero\",\"valuesRedacted\":true},\"error\":{\"type\":\"ChildProcessError\",\"code\":\"EXIT_NONZERO\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"${current_phase:-gitops-publish}\" \"${current_phase:-gitops-publish}\" \"$code\" \"$code\" \"$job_name\"; fi; exit \"$code\"; }",
"trap emit_failed EXIT",
"if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi",
"mkdir -p /root/.ssh",
scripts/src/hwlab-node-web-sentinel-cicd-shared.ts
+18
View File
@@ -515,6 +515,8 @@ export function renderPublishCurrentResult(result: Record<string, unknown>): str
const stageBudgets = record(result.stageBudgets);
const validationPlan = record(result.validationPlan);
const blocker = record(result.blocker);
const firstBreak = record(result.firstBreak);
const typedError = record(result.error);
const recoveryNext = record(controlPlane.recoveryNext);
const next = record(result.next);
const drillDown = record(result.drillDown);
@@ -609,6 +611,15 @@ export function renderPublishCurrentResult(result: Record<string, unknown>): str
"",
warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),
"",
table(["TRACE_ID", "FIRST_BREAK", "ERROR_TYPE", "ERROR_CODE", "HTTP", "EXIT"], [[
result.traceId ?? "-",
firstBreak.code ?? "-",
typedError.type ?? "-",
typedError.code ?? "-",
typedError.httpStatus ?? "-",
typedError.exitCode ?? "-",
]]),
"",
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"),
"",
renderSentinelDrillDown(drillDown),
@@ -685,6 +696,9 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
const argoApply = record(result.argoApply);
const blocker = record(result.blocker);
const statusDiagnosis = record(result.statusDiagnosis);
const traceId = result.traceId ?? deliveryStatus.traceId ?? record(publish).traceId ?? "-";
const firstBreak = record(result.firstBreak);
const typedError = record(result.error);
const drillDown = record(result.drillDown);
const targetValidation = record(result.targetValidation);
const targetValidationBusiness = record(targetValidation.businessStatus);
@@ -742,6 +756,8 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
warnings.length,
]]),
"",
table(["TRACE_ID", "FIRST_BREAK", "ERROR_TYPE", "ERROR_CODE"], [[traceId, firstBreak.code ?? "-", typedError.type ?? "-", typedError.code ?? "-"]]),
"",
flush.mode === "async-job" ? `ASYNC_FLUSH_STATUS\n ${flushNext.status ?? "-"}` : "ASYNC_FLUSH_STATUS\n-",
"",
renderSentinelNext({ ...next, pipelineRunStatus: drillDown.pipelineRunStatus }),
@@ -786,6 +802,8 @@ export function renderControlPlaneResult(result: Record<string, unknown>): strin
]]),
].join("\n"),
"",
table(["TRACE_ID", "FIRST_BREAK", "ERROR_TYPE", "ERROR_CODE"], [[traceId, firstBreak.code ?? statusDiagnosis.code ?? "-", typedError.type ?? "-", typedError.code ?? "-"]]),
"",
renderSentinelDrillDown(drillDown),
"",
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]),
scripts/src/hwlab-node-web-sentinel-cicd.ts
+22 -1
View File
@@ -578,6 +578,21 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio
const publish = runSentinelPublishJob(state, true, Math.max(1, remainingBudgetSeconds()), options.rerun);
const elapsedMs = Date.now() - startedAt;
const ok = record(publish).ok === true;
const publishFirstBreak = record(record(publish).firstBreak);
const publishError = record(record(publish).error);
if (!ok) {
sentinelProgressEvent("sentinel.publish.failure", {
phase: publishFirstBreak.phase ?? record(publish).phase ?? "sentinel-publish",
status: "failed",
traceId: record(publish).traceId ?? null,
firstBreak: Object.keys(publishFirstBreak).length === 0 ? null : publishFirstBreak,
error: Object.keys(publishError).length === 0 ? null : publishError,
pipelineRun: record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun),
sourceCommit: state.sourceHead.commit,
node: state.spec.nodeId,
lane: state.spec.lane,
});
}
const result = {
ok,
command,
@@ -590,6 +605,9 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio
source: state.sourceHead,
image: state.image,
pipelineRun: record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun),
traceId: record(publish).traceId ?? null,
firstBreak: Object.keys(publishFirstBreak).length === 0 ? null : publishFirstBreak,
error: Object.keys(publishError).length === 0 ? null : publishError,
controlPlane: {
ok,
mode: "pac-publish-only",
@@ -613,7 +631,10 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio
...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", controlPlaneWaitWarningSeconds(state)),
...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"),
],
blocker: ok ? null : { code: "sentinel-pac-publish-not-ready", reason: "sentinel publish PipelineRun did not complete successfully" },
blocker: ok ? null : {
code: publishFirstBreak.code ?? publishError.code ?? "sentinel-pac-publish-not-ready",
reason: publishFirstBreak.reason ?? publishError.message ?? "sentinel publish PipelineRun did not complete successfully",
},
recoveryNext: controlPlaneRecoveryNext(state, ok, publish, null, false),
next: publishCurrentNext(state),
valuesRedacted: true,
scripts/src/platform-infra-pipelines-as-code-remote.sh
+56 -1
View File
@@ -712,11 +712,37 @@ function recursiveEnvReuse(value, state) {
if (typeof env.serviceReusedCount === 'number') state.serviceReusedCount = env.serviceReusedCount;
}
if (typeof value.status === 'string' && /reus|skip|built|publish/u.test(value.status)) state.status = value.status;
if (state.traceId === null && typeof value.traceId === 'string' && /^[0-9a-f]{32}$/u.test(value.traceId)) state.traceId = value.traceId;
if (state.firstBreak === null && value.firstBreak && typeof value.firstBreak === 'object') state.firstBreak = boundedFailureRecord(value.firstBreak);
if (state.error === null && value.error && typeof value.error === 'object') state.error = boundedFailureRecord(value.error);
for (const item of Object.values(value)) recursiveEnvReuse(item, state);
}
function boundedFailureRecord(value) {
const input = value && typeof value === 'object' && !Array.isArray(value) ? value : {};
const token = (item) => typeof item === 'string' && /^[A-Za-z0-9_.-]{1,100}$/u.test(item) ? item : null;
const summary = (item) => typeof item === 'string'
? item.replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/giu, '$1[REDACTED]')
.replace(/(token|password|secret)(\s*[:=]\s*)\S+/giu, '$1$2[REDACTED]')
.replace(/https?:\/\/[^\s/]+@/giu, 'https://[REDACTED]@')
.replace(/([?&][^=\s]+)=([^&\s]+)/gu, '$1=[REDACTED]')
.replace(/\s+/gu, ' ').trim().slice(0, 480)
: null;
return {
type: token(input.type),
code: token(input.code),
phase: token(input.phase),
httpStatus: Number.isInteger(input.httpStatus) ? input.httpStatus : null,
exitCode: Number.isInteger(input.exitCode) ? input.exitCode : null,
reason: summary(input.reason),
message: summary(input.message),
stderrSummary: summary(input.stderrSummary),
valuesPrinted: false,
};
}
function envReuseForPipelineRun(namespace, name, taskRuns) {
const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null };
const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null };
const matching = (taskRuns.items || []).filter((item) => item.metadata?.labels?.['tekton.dev/pipelineRun'] === name);
const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null);
const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean);
@@ -870,6 +896,9 @@ function rowFor(consumer, item, taskRuns) {
sourceObservation,
artifact: evidence.artifact,
collector: evidence.collector,
traceId: evidence.traceId,
firstBreak: evidence.firstBreak,
error: evidence.error,
taskRuns: tasks,
source: classification.deliveryClass === 'outer-pac-event'
? 'gitea-pac-event-observed'
@@ -1007,6 +1036,32 @@ if (contractTaskRuns.length === 0) {
}
const records = [...parsePacLogRecords(logText), ...terminalRecords];
const out = extractPacArtifactEvidence(records, logText);
const diagnostic = { traceId: null, firstBreak: null, error: null };
function visit(value) {
if (value === null || value === undefined) return;
if (Array.isArray(value)) { for (const item of value) visit(item); return; }
if (typeof value !== 'object') return;
if (diagnostic.traceId === null && typeof value.traceId === 'string' && /^[0-9a-f]{32}$/u.test(value.traceId)) diagnostic.traceId = value.traceId;
if (diagnostic.firstBreak === null && value.firstBreak && typeof value.firstBreak === 'object') diagnostic.firstBreak = boundedFailureRecord(value.firstBreak);
if (diagnostic.error === null && value.error && typeof value.error === 'object') diagnostic.error = boundedFailureRecord(value.error);
for (const item of Object.values(value)) visit(item);
}
function boundedFailureRecord(value) {
const input = value && typeof value === 'object' && !Array.isArray(value) ? value : {};
const token = (item) => typeof item === 'string' && /^[A-Za-z0-9_.-]{1,100}$/u.test(item) ? item : null;
const summary = (item) => typeof item === 'string'
? item.replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/giu, '$1[REDACTED]')
.replace(/(token|password|secret)(\s*[:=]\s*)\S+/giu, '$1$2[REDACTED]')
.replace(/https?:\/\/[^\s/]+@/giu, 'https://[REDACTED]@')
.replace(/([?&][^=\s]+)=([^&\s]+)/gu, '$1=[REDACTED]')
.replace(/\s+/gu, ' ').trim().slice(0, 480)
: null;
return { type: token(input.type), code: token(input.code), phase: token(input.phase), httpStatus: Number.isInteger(input.httpStatus) ? input.httpStatus : null, exitCode: Number.isInteger(input.exitCode) ? input.exitCode : null, reason: summary(input.reason), message: summary(input.message), stderrSummary: summary(input.stderrSummary), valuesPrinted: false };
}
for (const record of records) visit(record);
out.traceId = diagnostic.traceId;
out.firstBreak = diagnostic.firstBreak;
out.error = diagnostic.error;
out.collector = {
mode,
contractTaskCount: contractTaskRuns.length,
scripts/src/platform-infra-pipelines-as-code.ts
+72 -1
View File
@@ -73,6 +73,15 @@ interface PacConfig {
display: {
timeZone: string;
};
observability: {
configRef: string;
tracesEndpoint: string;
serviceName: string;
propagation: string[];
samplingMode: "parentbased_traceidratio";
samplingRatio: number;
exporterFailure: { warning: true; blocking: false; timeoutMs: number };
};
release: {
version: string;
manifestUrl: string;
@@ -414,6 +423,9 @@ function readPacConfig(): PacConfig {
const gitOpsMirrorFlush = y.objectField(closeout, "gitOpsMirrorFlush", "closeout");
const gitea = y.objectField(root, "gitea", "");
const display = y.objectField(root, "display", "");
const observability = y.objectField(root, "observability", "");
const observabilitySampling = y.objectField(observability, "sampling", "observability");
const exporterFailure = y.objectField(observability, "exporterFailure", "observability");
const admin = y.objectField(gitea, "admin", "gitea");
const webhook = y.objectField(gitea, "webhook", "gitea");
const repositories = root.repositories === undefined
@@ -439,6 +451,19 @@ function readPacConfig(): PacConfig {
display: {
timeZone: y.stringField(display, "timeZone", "display"),
},
observability: {
configRef: y.stringField(observability, "configRef", "observability"),
tracesEndpoint: urlField(observability, "tracesEndpoint", "observability"),
serviceName: y.stringField(observability, "serviceName", "observability"),
propagation: y.stringArrayField(observability, "propagation", "observability"),
samplingMode: y.enumField(observabilitySampling, "mode", "observability.sampling", ["parentbased_traceidratio"] as const),
samplingRatio: numberInRange(observabilitySampling.ratio, 0, 1, "observability.sampling.ratio"),
exporterFailure: {
warning: literalBoolean(exporterFailure, "warning", true, "observability.exporterFailure"),
blocking: literalBoolean(exporterFailure, "blocking", false, "observability.exporterFailure"),
timeoutMs: integerInRange(exporterFailure.timeoutMs, 50, 2_000, "observability.exporterFailure.timeoutMs"),
},
},
release: {
version: y.stringField(release, "version", "release"),
manifestUrl: urlField(release, "manifestUrl", "release"),
@@ -802,6 +827,17 @@ function validateConfig(config: PacConfig): void {
if (consumerIds.has(consumer.id)) throw new Error(`${configLabel}.consumers id must be unique: ${consumer.id}`);
consumerIds.add(consumer.id);
const repository = resolveRepository(config, consumer.repositoryRef);
if (consumer.id.startsWith("sentinel-")) {
if (repository.params.otel_traces_endpoint !== config.observability.tracesEndpoint) {
throw new Error(`${configLabel}.repositories.${repository.id}.params.otel_traces_endpoint must match observability.tracesEndpoint`);
}
if (repository.params.otel_sampling_ratio !== String(config.observability.samplingRatio)) {
throw new Error(`${configLabel}.repositories.${repository.id}.params.otel_sampling_ratio must match observability.sampling.ratio`);
}
if (repository.params.otel_exporter_timeout_ms !== String(config.observability.exporterFailure.timeoutMs)) {
throw new Error(`${configLabel}.repositories.${repository.id}.params.otel_exporter_timeout_ms must match observability.exporterFailure.timeoutMs`);
}
}
if (repository.namespace !== consumer.namespace) throw new Error(`${configLabel}.consumers.${consumer.id}.namespace must match repository namespace`);
if (consumer.deliveryProvenance !== null) {
if (consumer.sourceArtifact === null) throw new Error(`${configLabel}.consumers.${consumer.id}.deliveryProvenance requires sourceArtifact ownership`);
@@ -845,6 +881,24 @@ function validateConfig(config: PacConfig): void {
validateTimeZone(config.display.timeZone, `${configLabel}.display.timeZone`);
}
function numberInRange(value: unknown, minimum: number, maximum: number, path: string): number {
if (typeof value !== "number" || !Number.isFinite(value) || value < minimum || value > maximum) {
throw new Error(`${configLabel}.${path} must be a number from ${minimum} to ${maximum}`);
}
return value;
}
function integerInRange(value: unknown, minimum: number, maximum: number, path: string): number {
const parsed = numberInRange(value, minimum, maximum, path);
if (!Number.isInteger(parsed)) throw new Error(`${configLabel}.${path} must be an integer`);
return parsed;
}
function literalBoolean<T extends boolean>(value: Record<string, unknown>, key: string, expected: T, path: string): T {
if (value[key] !== expected) throw new Error(`${configLabel}.${path}.${key} must be ${String(expected)}`);
return expected;
}
function resolveTarget(config: PacConfig, targetId: string | null): PacTarget {
const id = targetId ?? config.defaults.targetId;
const target = config.targets.find((item) => item.id.toLowerCase() === id.toLowerCase());
@@ -1281,7 +1335,11 @@ export function resolvePacHistoryConsumerSelection(
async function debugStep(config: UniDeskConfig, options: HistoryOptions): Promise<Record<string, unknown>> {
const pac = readPacConfig();
const target = resolveTarget(pac, options.targetId);
const consumer = resolveConsumer(pac, options.consumerId);
const selection = resolvePacHistoryConsumerSelection(pac.consumers, target.id, options.consumerId, options.detailId);
if (selection.selectedConsumerIds.length !== 1) {
throw new Error("debug-step requires --consumer or --id resolving exactly one Pipelines-as-Code consumer");
}
const consumer = resolveConsumer(pac, selection.selectedConsumerIds[0] ?? null);
const repository = resolveRepository(pac, consumer.repositoryRef);
const result = await capture(config, target.route, ["sh"], remoteScript("debug-step", pac, target, repository, consumer, { ...options, confirm: false, dryRun: true, wait: false }, emptySecretMaterial(), ""));
const parsed = parseJsonOutput(result.stdout);
@@ -1681,6 +1739,9 @@ function nodeStatusRow(targetId: string, consumer: PacConsumer, repository: PacR
phase: stringValue(diagnostics.phase),
hint: compactLine(stringValue(diagnostics.hint)),
},
traceId: stringValue(artifact.traceId),
firstBreak: record(artifact.firstBreak),
error: record(artifact.error),
reason: ready ? "ready" : nodeStatusReason({ ciReady, argoReady, runtimeReady, diagnosticsReady, pipelineStatus: statusText(latest), diagnostics }),
statusCommand: `bun scripts/cli.ts platform-infra pipelines-as-code status --target ${targetId} --consumer ${consumer.id}`,
historyCommand: `bun scripts/cli.ts platform-infra pipelines-as-code history --target ${targetId} --consumer ${consumer.id} --limit 10`,
@@ -2803,6 +2864,8 @@ function renderHistoryDetail(row: Record<string, unknown>): string[] {
const longest = record(taskRuns.longest);
const failed = record(taskRuns.failed);
const failedStep = record(failed.failedStep);
const firstBreak = record(row.firstBreak);
const typedError = record(row.error);
return [
"",
"DETAIL",
@@ -2826,6 +2889,14 @@ function renderHistoryDetail(row: Record<string, unknown>): string[] {
["failedStepExit", stringValue(failedStep.exitCode)],
["failedStepTermination", stringValue(failedStep.terminationReason ?? failedStep.reason)],
["failure", compactLine(stringValue(failed.message))],
["traceId", stringValue(row.traceId)],
["firstBreak", stringValue(firstBreak.code)],
["firstBreakPhase", stringValue(firstBreak.phase)],
["errorType", stringValue(typedError.type)],
["errorCode", stringValue(typedError.code)],
["errorHttpStatus", stringValue(typedError.httpStatus)],
["errorExitCode", stringValue(typedError.exitCode)],
["errorSummary", compactLine(stringValue(typedError.stderrSummary ?? typedError.message))],
["envReuseSource", stringValue(envReuse.source)],
["deliveryMode", stringValue(sourceObservation.mode)],
["deliveryValid", stringValue(sourceObservation.valid)],