From 083e63bff0b70659fd8b0b986102c65e3b9b8bad Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 12:16:55 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=A1=A5=E9=BD=90=20CI/CD=20OTel=20?= =?UTF-8?q?=E4=B8=8E=E5=A4=B1=E8=B4=A5=E6=8A=95=E5=BD=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .tekton/web-probe-sentinel-jd01-pac.yaml | 118 ++++++++++--- .tekton/web-probe-sentinel-nc01-pac.yaml | 118 ++++++++++--- config/platform-infra/pipelines-as-code.yaml | 18 ++ docs/reference/platform-infra.md | 3 +- .../cicd/render-sentinel-publish-task.ts | 152 ++++++++++++++++ scripts/native/cicd/submit-pipelinerun.mjs | 163 +++++++++++++++++- scripts/src/cicd-delivery-authority.test.ts | 23 ++- scripts/src/cicd-node-status.ts | 5 +- .../src/hwlab-node-web-sentinel-cicd-jobs.ts | 24 +-- .../hwlab-node-web-sentinel-cicd-shared.ts | 18 ++ scripts/src/hwlab-node-web-sentinel-cicd.ts | 23 ++- ...platform-infra-pipelines-as-code-remote.sh | 57 +++++- .../src/platform-infra-pipelines-as-code.ts | 73 +++++++- 13 files changed, 734 insertions(+), 61 deletions(-) create mode 100644 scripts/native/cicd/render-sentinel-publish-task.ts diff --git a/.tekton/web-probe-sentinel-jd01-pac.yaml b/.tekton/web-probe-sentinel-jd01-pac.yaml index 35ff0169..83fafa54 100644 --- a/.tekton/web-probe-sentinel-jd01-pac.yaml +++ b/.tekton/web-probe-sentinel-jd01-pac.yaml @@ -13,43 +13,119 @@ metadata: hwlab.pikastech.local/source-commit: "{{revision}}" spec: timeouts: - pipeline: 300s + pipeline: 900s taskRunTemplate: serviceAccountName: default podTemplate: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet + securityContext: + fsGroup: 1000 pipelineSpec: tasks: - name: publish taskSpec: + volumes: + - name: cache + persistentVolumeClaim: + claimName: hwlab-git-mirror-cache + - name: git-ssh + secret: + secretName: git-mirror-github-ssh + defaultMode: 256 + - name: workspace + emptyDir: + sizeLimit: 8Gi + - name: buildkit-state + hostPath: + path: /var/lib/unidesk/web-probe-sentinel/buildkit-jd01 + type: DirectoryOrCreate + - name: tmp + emptyDir: {} steps: - - name: sentinel-publish + - name: source image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: IfNotPresent - workingDir: /workspace env: - - name: GIT_READ_URL - value: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git - - name: SOURCE_COMMIT - value: "{{revision}}" - - name: SOURCE_STAGE_REF - value: refs/unidesk/snapshots/gitea-actions/unidesk-master/{{revision}} + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: "{{otel_traces_endpoint}}" + - name: OTEL_EXPORTER_TIMEOUT_MS + value: "{{otel_exporter_timeout_ms}}" + - name: OTEL_SERVICE_NAME + value: "{{otel_service_name}}" + - name: OTEL_TRACES_SAMPLER_ARG + value: "{{otel_sampling_ratio}}" script: | #!/bin/sh set -eu - rm -rf source - git clone --filter=blob:none --no-checkout "$GIT_READ_URL" source - cd source - git fetch --depth=1 --filter=blob:none origin "+$SOURCE_STAGE_REF:refs/remotes/origin/unidesk-source-snapshot" - git checkout --detach "$SOURCE_COMMIT" - bun scripts/cli.ts web-probe sentinel publish-current \ + export CICD_TRACE_STARTED_AT_MS=$(($(date +%s) * 1000)) + export TRACEPARENT=$(node -e 'const c=require("node:crypto");process.stdout.write(`00-${c.randomBytes(16).toString("hex")}-${c.randomBytes(8).toString("hex")}-01`)') + rm -rf /workspace/bootstrap /workspace/generated + git clone --filter=blob:none --no-checkout http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git /workspace/bootstrap + cd /workspace/bootstrap + git fetch --depth=1 --filter=blob:none origin "+refs/unidesk/snapshots/gitea-actions/unidesk-master/{{revision}}:refs/remotes/origin/unidesk-source-snapshot" + git checkout --detach "{{revision}}" + bun scripts/native/cicd/render-sentinel-publish-task.ts \ --node JD01 \ --lane v03 \ --sentinel jd01-web-probe-sentinel \ - --confirm \ - --wait \ - --timeout-seconds 180 \ - --source-commit "$SOURCE_COMMIT" \ - --source-stage-ref "$SOURCE_STAGE_REF" \ - --source-authority gitea-snapshot + --pipeline-run "$(context.pipelineRun.name)" \ + --source-commit "{{revision}}" \ + --source-stage-ref "refs/unidesk/snapshots/gitea-actions/unidesk-master/{{revision}}" \ + --source-authority gitea-snapshot \ + --output-dir /workspace/generated + exec /workspace/generated/source.sh + volumeMounts: + - name: workspace + mountPath: /workspace + - name: git-ssh + mountPath: /git-ssh + readOnly: true + - name: prepare-buildkit-state + image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: IfNotPresent + script: | + #!/bin/sh + set -eu + mkdir -p /home/user/.local/share/buildkit + chown -R 1000:1000 /home/user/.local/share/buildkit + securityContext: + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: buildkit-state + mountPath: /home/user/.local/share/buildkit + - name: image-build + image: 127.0.0.1:5000/hwlab/buildkit:rootless + imagePullPolicy: IfNotPresent + env: + - name: BUILDKITD_FLAGS + value: --oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host + script: | + #!/bin/sh + exec /workspace/generated/image-build.sh + securityContext: + privileged: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - name: workspace + mountPath: /workspace + - name: buildkit-state + mountPath: /home/user/.local/share/buildkit + - name: tmp + mountPath: /tmp + - name: publish + image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: IfNotPresent + script: | + #!/bin/sh + exec /workspace/generated/publish.sh + volumeMounts: + - name: workspace + mountPath: /workspace + - name: cache + mountPath: /cache + - name: git-ssh + mountPath: /git-ssh + readOnly: true diff --git a/.tekton/web-probe-sentinel-nc01-pac.yaml b/.tekton/web-probe-sentinel-nc01-pac.yaml index c2ecee0e..6ec53bce 100644 --- a/.tekton/web-probe-sentinel-nc01-pac.yaml +++ b/.tekton/web-probe-sentinel-nc01-pac.yaml @@ -13,43 +13,119 @@ metadata: hwlab.pikastech.local/source-commit: "{{revision}}" spec: timeouts: - pipeline: 300s + pipeline: 900s taskRunTemplate: serviceAccountName: default podTemplate: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet + securityContext: + fsGroup: 1000 pipelineSpec: tasks: - name: publish taskSpec: + volumes: + - name: cache + persistentVolumeClaim: + claimName: hwlab-git-mirror-cache + - name: git-ssh + secret: + secretName: git-mirror-github-ssh + defaultMode: 256 + - name: workspace + emptyDir: + sizeLimit: 8Gi + - name: buildkit-state + hostPath: + path: /var/lib/unidesk/web-probe-sentinel/buildkit-nc01 + type: DirectoryOrCreate + - name: tmp + emptyDir: {} steps: - - name: sentinel-publish + - name: source image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: IfNotPresent - workingDir: /workspace env: - - name: GIT_READ_URL - value: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git - - name: SOURCE_COMMIT - value: "{{revision}}" - - name: SOURCE_STAGE_REF - value: refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01/{{revision}} + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: "{{otel_traces_endpoint}}" + - name: OTEL_EXPORTER_TIMEOUT_MS + value: "{{otel_exporter_timeout_ms}}" + - name: OTEL_SERVICE_NAME + value: "{{otel_service_name}}" + - name: OTEL_TRACES_SAMPLER_ARG + value: "{{otel_sampling_ratio}}" script: | #!/bin/sh set -eu - rm -rf source - git clone --filter=blob:none --no-checkout "$GIT_READ_URL" source - cd source - git fetch --depth=1 --filter=blob:none origin "+$SOURCE_STAGE_REF:refs/remotes/origin/unidesk-source-snapshot" - git checkout --detach "$SOURCE_COMMIT" - bun scripts/cli.ts web-probe sentinel publish-current \ + export CICD_TRACE_STARTED_AT_MS=$(($(date +%s) * 1000)) + export TRACEPARENT=$(node -e 'const c=require("node:crypto");process.stdout.write(`00-${c.randomBytes(16).toString("hex")}-${c.randomBytes(8).toString("hex")}-01`)') + rm -rf /workspace/bootstrap /workspace/generated + git clone --filter=blob:none --no-checkout http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-unidesk.git /workspace/bootstrap + cd /workspace/bootstrap + git fetch --depth=1 --filter=blob:none origin "+refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01/{{revision}}:refs/remotes/origin/unidesk-source-snapshot" + git checkout --detach "{{revision}}" + bun scripts/native/cicd/render-sentinel-publish-task.ts \ --node NC01 \ --lane v03 \ --sentinel nc01-web-probe-sentinel \ - --confirm \ - --wait \ - --timeout-seconds 180 \ - --source-commit "$SOURCE_COMMIT" \ - --source-stage-ref "$SOURCE_STAGE_REF" \ - --source-authority gitea-snapshot + --pipeline-run "$(context.pipelineRun.name)" \ + --source-commit "{{revision}}" \ + --source-stage-ref "refs/unidesk/snapshots/gitea-actions/unidesk-master-nc01/{{revision}}" \ + --source-authority gitea-snapshot \ + --output-dir /workspace/generated + exec /workspace/generated/source.sh + volumeMounts: + - name: workspace + mountPath: /workspace + - name: git-ssh + mountPath: /git-ssh + readOnly: true + - name: prepare-buildkit-state + image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: IfNotPresent + script: | + #!/bin/sh + set -eu + mkdir -p /home/user/.local/share/buildkit + chown -R 1000:1000 /home/user/.local/share/buildkit + securityContext: + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: buildkit-state + mountPath: /home/user/.local/share/buildkit + - name: image-build + image: 127.0.0.1:5000/hwlab/buildkit:rootless + imagePullPolicy: IfNotPresent + env: + - name: BUILDKITD_FLAGS + value: --oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host + script: | + #!/bin/sh + exec /workspace/generated/image-build.sh + securityContext: + privileged: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - name: workspace + mountPath: /workspace + - name: buildkit-state + mountPath: /home/user/.local/share/buildkit + - name: tmp + mountPath: /tmp + - name: publish + image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: IfNotPresent + script: | + #!/bin/sh + exec /workspace/generated/publish.sh + volumeMounts: + - name: workspace + mountPath: /workspace + - name: cache + mountPath: /cache + - name: git-ssh + mountPath: /git-ssh + readOnly: true diff --git a/config/platform-infra/pipelines-as-code.yaml b/config/platform-infra/pipelines-as-code.yaml index 9f00c1f0..b798ac44 100644 --- a/config/platform-infra/pipelines-as-code.yaml +++ b/config/platform-infra/pipelines-as-code.yaml @@ -19,6 +19,20 @@ defaults: consumerId: hwlab-nc01-v03 display: timeZone: Asia/Shanghai +observability: + configRef: config/platform-infra/observability.yaml + tracesEndpoint: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces + serviceName: unidesk-cicd + propagation: + - tracecontext + - baggage + sampling: + mode: parentbased_traceidratio + ratio: 1 + exporterFailure: + warning: true + blocking: false + timeoutMs: 300 release: version: v0.48.0 manifestUrl: https://github.com/openshift-pipelines/pipelines-as-code/releases/download/v0.48.0/release.k8s.yaml @@ -411,6 +425,10 @@ templates: pipeline_name: "hwlab-web-probe-sentinel-${nodeLower}-pac" pipeline_run_prefix: "hwlab-web-probe-sentinel-${nodeLower}" service_account: default + otel_traces_endpoint: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces + otel_service_name: "unidesk-cicd-sentinel-${nodeLower}" + otel_sampling_ratio: "1" + otel_exporter_timeout_ms: "300" hwlabV03: id: "hwlab-${nodeLower}-v03" name: "hwlab-${nodeLower}-v03" diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 5410a299..175de5b3 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -25,7 +25,8 @@ - Durable inbox 只是 webhook delivery 的持久接收与重试 journal，不是业务 source/ref 的第二份真相，也不得被 PaC、Tekton、GitOps、Argo 或 runtime 当作源码/read model。最终 source authority 仍只有 Gitea authority branch 与 immutable snapshot；禁止新增 polling/read-model fallback 或第二 source authority。 - 默认 PaC 入口是 `bun scripts/cli.ts platform-infra pipelines-as-code status|history --target `。`status` 应显示 webhook、PipelineRun/TaskRun duration、image status、env identity、digest、GitOps commit、Argo revision 和 runtime provenance；`history` 必须读取 target node 上的实时对象、显式报告 read error，并支持 consumer 与 PipelineRun id 下钻。`history --id` 必须通过运行面 provenance 与 PipelineRun prefix 唯一解析实际 consumer；零匹配、多匹配或显式 consumer 不一致都 fail-closed，不能回退到默认 consumer 或默认 node。 - Sentinel 内部 publish capability 由 `config/platform-infra/pipelines-as-code.yaml#capabilities.sentinelInternalPublish` 单一拥有，默认 `enabled=false`。Pod env、ownerReferences、ServiceAccount、label 与 annotation 不能证明 creator；#1769 的 admission-owned provenance 与最小权限边界落地前不得启用。`.tekton` 继续走既有自动 `publish-current`，不能退化成人工 publish。 -- PaC 观察必须把 PipelineRun 分类为外层 `outer-pac-event`、内层 `inner-deterministic-publish` 或 `unknown`。只有唯一外层 PaC push event 能驱动 latest、status、debug-step 与兼容 closeout；内层只能展示为未绑定执行观察，缺少 admission-owned 父子证据时标记 `unproven`，禁止按名称前缀推断关系。 +- PaC 观察必须把 PipelineRun 分类为外层 `outer-pac-event`、历史内层 `inner-deterministic-publish` 或 `unknown`。只有唯一外层 PaC push event 能驱动 latest、status、debug-step 与兼容 closeout；Web sentinel 的 source、image、artifact、GitOps 步骤必须直接位于该 outer PipelineRun，禁止再创建 nested PipelineRun。历史内层运行只作为未绑定执行观察，缺少 admission-owned 父子证据时标记 `unproven`，禁止按名称前缀推断关系。 +- CI/CD OTel endpoint、service/resource 属性、采样与 exporter timeout/failure policy 由 `config/platform-infra/pipelines-as-code.yaml` 引用现有 `config/platform-infra/observability.yaml` 并负责渲染。outer PaC 在最早可控 step 生成 W3C trace context，后续 source、image、artifact、GitOps 与 helper 复用同一 traceId；export 失败或超时只输出 `warning=true`、`blocking=false` 的有界事件，不改变 PipelineRun、TaskRun 或业务 step 终态。 - `config/platform-infra/pipelines-as-code.yaml` 可以声明多个 repository 和 consumer。`cicd status --node ` 用于 node 汇总，`history --target ` 用于全 consumer 审计，`status --target --consumer ` 用于 consumer 只读状态；不得跨 repository 混合 PipelineRun 或 env reuse 证据。 - PaC Repository CR `spec.url` 必须匹配 Gitea webhook payload 的 public repository URL；ClusterIP/service URL 只属于 `cloneUrl` 与 `params.git_read_url`。Repository params 必须使用该 consumer 的 node-specific snapshot prefix。 - GitHub 始终是 upstream write authority。Bridge 只允许 GitHub -> Gitea 更新 YAML 声明的 branch 与 immutable snapshot，不得反向写 GitHub，也不得增加 polling 或第二 trigger path。 diff --git a/scripts/native/cicd/render-sentinel-publish-task.ts b/scripts/native/cicd/render-sentinel-publish-task.ts new file mode 100644 index 00000000..8fd52029 --- /dev/null +++ b/scripts/native/cicd/render-sentinel-publish-task.ts @@ -0,0 +1,152 @@ +import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { hwlabRuntimeLaneSpecForNode } from "../../src/hwlab-node-lanes"; +import { loadSentinelCicdState } from "../../src/hwlab-node-web-sentinel-cicd"; +import { + sentinelImageBuildProxyEnv, + sentinelPublishImageBuildShell, + sentinelPublishShell, + sentinelPublishSourceShell, +} from "../../src/hwlab-node-web-sentinel-cicd-jobs"; + +const options = parseOptions(process.argv.slice(2)); +const sourceAuthority = options.sourceAuthority === "gitea-snapshot" ? "gitea-snapshot" : fail("--source-authority must be gitea-snapshot"); +const state = loadSentinelCicdState( + hwlabRuntimeLaneSpecForNode(options.lane, options.node), + options.sentinel, + 30, + "cached", + { + commit: options.sourceCommit, + stageRef: options.sourceStageRef, + mirrorCommit: options.sourceCommit, + sourceAuthority, + }, +); +const outputDir = resolve(options.outputDir); +mkdirSync(outputDir, { recursive: true }); +const traceSetup = [ + "if [ -f /workspace/meta/traceparent ]; then TRACEPARENT=$(cat /workspace/meta/traceparent); export TRACEPARENT; fi", + "if [ -f /workspace/meta/tracestate ]; then TRACESTATE=$(cat /workspace/meta/tracestate); export TRACESTATE; fi", +].join("\n"); +const proxySetup = sentinelImageBuildProxyEnv(state).map(({ name, value }) => `export ${name}=${shellQuote(value)}`).join("\n"); +writeExecutable("source.sh", [ + "#!/bin/sh", + "set -eu", + `incoming_traceparent=${shellQuote(process.env.TRACEPARENT ?? "")}`, + `incoming_tracestate=${shellQuote(process.env.TRACESTATE ?? "")}`, + sentinelPublishSourceShell(state, options.pipelineRun), + "if [ -n \"$incoming_traceparent\" ]; then printf '%s' \"$incoming_traceparent\" > /workspace/meta/traceparent; fi", + "if [ -n \"$incoming_tracestate\" ]; then printf '%s' \"$incoming_tracestate\" > /workspace/meta/tracestate; fi", +].join("\n")); +writeExecutable("image-build.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishImageBuildShell(state, options.pipelineRun)].join("\n")); +writeExecutable("publish.sh", ["#!/bin/sh", "set -eu", traceSetup, proxySetup, sentinelPublishShell(state, options.pipelineRun, true)].join("\n")); +await emitOuterSpan(); +process.stdout.write(`${JSON.stringify({ ok: true, action: "render-sentinel-publish-task", node: options.node, lane: options.lane, sentinel: options.sentinel, pipelineRun: options.pipelineRun, sourceCommit: options.sourceCommit, outputDir, traceId: traceId(process.env.TRACEPARENT), valuesRedacted: true })}\n`); + +function writeExecutable(name: string, content: string): void { + const path = resolve(outputDir, name); + writeFileSync(path, `${content}\n`, "utf8"); + chmodSync(path, 0o755); +} + +function parseOptions(args: string[]): Record<"node" | "lane" | "sentinel" | "pipelineRun" | "sourceCommit" | "sourceStageRef" | "sourceAuthority" | "outputDir", string> { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const key = args[index]; + const value = args[index + 1]; + if (key === undefined || value === undefined || !key.startsWith("--")) fail("render-sentinel-publish-task requires key/value options"); + values.set(key.slice(2), value); + } + const required = (key: string): string => values.get(key) || fail(`--${key} is required`); + const result = { + node: required("node"), + lane: required("lane"), + sentinel: required("sentinel"), + pipelineRun: required("pipeline-run"), + sourceCommit: required("source-commit"), + sourceStageRef: required("source-stage-ref"), + sourceAuthority: required("source-authority"), + outputDir: required("output-dir"), + }; + if (!/^[0-9a-f]{40}$/u.test(result.sourceCommit)) fail("--source-commit must be a 40-character lowercase Git commit"); + for (const key of ["node", "lane", "sentinel", "pipelineRun"] as const) if (!/^[A-Za-z0-9._-]+$/u.test(result[key])) fail(`--${key} must be a simple id`); + if (!result.sourceStageRef.startsWith("refs/unidesk/snapshots/")) fail("--source-stage-ref must be an immutable UniDesk snapshot ref"); + return result; +} + +function traceId(traceparent: string | undefined): string | null { + return traceparent?.match(/^00-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$/iu)?.[1]?.toLowerCase() ?? null; +} + +async function emitOuterSpan(): Promise { + const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + const context = process.env.TRACEPARENT?.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/iu); + if (endpoint === undefined || context === undefined || context === null) return; + const startedAtMs = Number(process.env.CICD_TRACE_STARTED_AT_MS ?? Date.now()); + const payload = { + resourceSpans: [{ + resource: { attributes: attributes({ "service.name": process.env.OTEL_SERVICE_NAME ?? "unidesk-cicd", "unidesk.node": options.node, "hwlab.lane": options.lane, "unidesk.values_redacted": true }) }, + scopeSpans: [{ + scope: { name: "unidesk.cicd.pac_outer", version: "1" }, + spans: [{ + traceId: context[1]?.toLowerCase(), + spanId: context[2]?.toLowerCase(), + name: "cicd.pac.outer_event", + kind: 2, + startTimeUnixNano: (BigInt(startedAtMs) * 1_000_000n).toString(), + endTimeUnixNano: (BigInt(Date.now()) * 1_000_000n).toString(), + attributes: attributes({ + "cicd.target": options.node, + "cicd.lane": options.lane, + "cicd.consumer": `sentinel-${options.node.toLowerCase()}-v03`, + "cicd.repository": "pikasTech/unidesk", + "cicd.branch": "master", + "cicd.source_commit": options.sourceCommit, + "cicd.pipeline_run": options.pipelineRun, + "cicd.delivery_class": "outer-pac-event", + "cicd.phase": "task-rendered", + "warning": false, + "blocking": false, + "valuesRedacted": true, + }), + status: { code: 1 }, + }], + }], + }], + }; + const controller = new AbortController(); + const timeoutMs = exporterTimeoutMs(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: process.env.TRACEPARENT ?? "" }, body: JSON.stringify(payload), signal: controller.signal }); + if (!response.ok) process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`); + } catch (error) { + process.stderr.write(`${JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, timedOut: error instanceof Error && error.name === "AbortError", errorType: error instanceof Error ? error.name : "Error", timeoutMs, traceId: context[1]?.toLowerCase(), valuesRedacted: true })}\n`); + } finally { + clearTimeout(timer); + } +} + +function exporterTimeoutMs(): number { + const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS ?? "300"); + return Number.isInteger(value) && value >= 50 && value <= 2_000 ? value : 300; +} + +function attributes(values: Record): Array> { + return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ key, value: attributeValue(value) })); +} + +function attributeValue(value: unknown): Record { + if (typeof value === "boolean") return { boolValue: value }; + if (typeof value === "number" && Number.isFinite(value)) return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value }; + return { stringValue: typeof value === "string" ? value : JSON.stringify(value) }; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function fail(message: string): never { + throw new Error(message); +} diff --git a/scripts/native/cicd/submit-pipelinerun.mjs b/scripts/native/cicd/submit-pipelinerun.mjs index 1387a964..5fc9151d 100644 --- a/scripts/native/cicd/submit-pipelinerun.mjs +++ b/scripts/native/cicd/submit-pipelinerun.mjs @@ -1,4 +1,5 @@ import { readFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import https from "node:https"; const namespace = process.env.NAMESPACE || ""; @@ -14,10 +15,12 @@ const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token" const ca = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"); const manifest = JSON.parse(Buffer.from(readFileSync(0, "utf8").replace(/\s+/g, ""), "base64").toString("utf8")); const startedAt = Date.now(); +const traceContext = resolveTraceContext(); function request(method, path, body, contentType = "application/json") { return new Promise((resolve, reject) => { - const headers = { authorization: `Bearer ${token}` }; + const headers = { authorization: `Bearer ${token}`, traceparent: traceContext.traceparent }; + if (traceContext.tracestate !== null) headers.tracestate = traceContext.tracestate; const payload = body === undefined ? null : typeof body === "string" ? body : JSON.stringify(body); if (payload !== null) { headers["content-type"] = contentType; @@ -86,7 +89,33 @@ if (latest.found) { } else if (result.status >= 200 && result.status < 300) { created = true; } else { - process.stderr.write(result.text || `kube api POST pipelinerun status ${result.status}`); + const error = kubernetesApiError(result, "pipelinerun-create"); + const firstBreak = { code: "kubernetes-api-request-failed", phase: "pipelinerun-create", reason: error.message, valuesRedacted: true }; + process.stdout.write(JSON.stringify({ + ok: false, + submitted: false, + created: false, + reused: false, + wait: shouldWait, + polls: 0, + completed: false, + failed: true, + terminal: true, + stillRunning: false, + timedOutWait: false, + elapsedMs: Date.now() - startedAt, + pipelineRun: compact(null), + traceId: traceContext.traceId, + traceparent: traceContext.traceparent, + firstBreak, + error, + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + warning: false, + blocking: true, + valuesRedacted: true, + })); + await emitSpan("cicd.tekton.pipelinerun.create", false, { error, firstBreak, httpStatus: error.httpStatus, exitCode: 1 }); process.exit(1); } latest = await getPipelineRun(); @@ -122,6 +151,8 @@ const output = { stillRunning: !terminal, timedOutWait: shouldWait && !terminal, elapsedMs: Date.now() - startedAt, + traceId: traceContext.traceId, + traceparent: traceContext.traceparent, pipelineRun: compact(latest.object), logsTail, statusAuthority: "kubernetes-api-serviceaccount", @@ -129,6 +160,12 @@ const output = { valuesRedacted: true, }; process.stdout.write(JSON.stringify(output)); +await emitSpan("cicd.tekton.pipelinerun.wait", !failed, { + phase: failed ? "pipelinerun-terminal-failed" : completed ? "pipelinerun-terminal-succeeded" : "pipelinerun-wait", + status: condition?.status || null, + durationMs: Date.now() - startedAt, + exitCode: failed ? 1 : 0, +}); if (failed) process.exit(1); function requiredPositiveNumber(name) { @@ -137,6 +174,128 @@ function requiredPositiveNumber(name) { return value; } +function resolveTraceContext() { + const incoming = process.env.TRACEPARENT || process.env.OTEL_TRACEPARENT || ""; + const matched = incoming.match(/^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/i); + const traceId = matched?.[1]?.toLowerCase() || randomBytes(16).toString("hex"); + const parentSpanId = matched?.[2]?.toLowerCase() || null; + const spanId = randomBytes(8).toString("hex"); + return { + traceId, + parentSpanId, + spanId, + traceparent: `00-${traceId}-${spanId}-${matched?.[3]?.toLowerCase() || "01"}`, + tracestate: boundedHeader(process.env.TRACESTATE), + }; +} + +function boundedHeader(value) { + return typeof value === "string" && value.length > 0 && value.length <= 512 ? value : null; +} + +function kubernetesApiError(result, phase) { + const body = parseBody(result); + const code = safeToken(body?.reason) || `HTTP_${result.status || 0}`; + const message = redactSummary(body?.message || result.text || `Kubernetes API request failed with HTTP ${result.status || 0}`); + return { + type: "KubernetesApiError", + code, + phase, + httpStatus: result.status || null, + exitCode: 1, + stderrSummary: message, + message, + valuesRedacted: true, + }; +} + +function safeToken(value) { + return typeof value === "string" && /^[A-Za-z0-9_.-]{1,80}$/.test(value) ? value : null; +} + +function redactSummary(value) { + return String(value || "") + .replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/gi, "$1[REDACTED]") + .replace(/(token|password|secret)(\s*[:=]\s*)\S+/gi, "$1$2[REDACTED]") + .replace(/https?:\/\/[^\s/]+@/gi, "https://[REDACTED]@") + .replace(/([?&][^=\s]+)=([^&\s]+)/g, "$1=[REDACTED]") + .replace(/\s+/g, " ") + .trim() + .slice(0, 480); +} + +async function emitSpan(name, ok, attributes) { + const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + if (!endpoint) return; + const started = BigInt(startedAt) * 1_000_000n; + const ended = BigInt(Date.now()) * 1_000_000n; + const payload = { + resourceSpans: [{ + resource: { attributes: otelAttributes({ + "service.name": process.env.OTEL_SERVICE_NAME || "unidesk-cicd", + "deployment.environment": process.env.UNIDESK_LANE || null, + "unidesk.node": process.env.UNIDESK_NODE || null, + "unidesk.values_redacted": true, + }) }, + scopeSpans: [{ + scope: { name: "unidesk.cicd.submit_pipelinerun", version: "1" }, + spans: [{ + traceId: traceContext.traceId, + spanId: traceContext.spanId, + ...(traceContext.parentSpanId === null ? {} : { parentSpanId: traceContext.parentSpanId }), + name, + kind: 3, + startTimeUnixNano: started.toString(), + endTimeUnixNano: ended.toString(), + attributes: otelAttributes({ + "cicd.consumer": process.env.UNIDESK_PAC_CONSUMER || null, + "cicd.pipeline_run": pipelineRun, + "k8s.namespace.name": namespace, + "cicd.source_commit": process.env.SOURCE_COMMIT || null, + "cicd.phase": attributes.phase || null, + "cicd.status": attributes.status || null, + "cicd.duration_ms": attributes.durationMs || null, + "process.exit.code": attributes.exitCode ?? null, + "http.response.status_code": attributes.httpStatus ?? null, + "error.type": attributes.error?.type || null, + "error.code": attributes.error?.code || null, + "cicd.first_break": attributes.firstBreak?.code || null, + "warning": false, + "blocking": attributes.exitCode === 1, + "valuesRedacted": true, + }), + status: { code: ok ? 1 : 2 }, + }], + }], + }], + }; + try { + const controller = new AbortController(); + const timeoutMs = exporterTimeoutMs(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const response = await fetch(endpoint, { method: "POST", headers: { "content-type": "application/json", traceparent: traceContext.traceparent }, body: JSON.stringify(payload), signal: controller.signal }); + clearTimeout(timeout); + if (!response.ok) process.stderr.write(JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, httpStatus: response.status, traceId: traceContext.traceId, valuesRedacted: true }) + "\n"); + } catch (error) { + process.stderr.write(JSON.stringify({ event: "cicd.otel.export", warning: true, blocking: false, errorType: error instanceof Error ? error.name : "Error", timedOut: error instanceof Error && error.name === "AbortError", timeoutMs: exporterTimeoutMs(), traceId: traceContext.traceId, valuesRedacted: true }) + "\n"); + } +} + +function exporterTimeoutMs() { + const value = Number(process.env.OTEL_EXPORTER_TIMEOUT_MS || "300"); + return Number.isInteger(value) && value >= 50 && value <= 2000 ? value : 300; +} + +function otelAttributes(values) { + return Object.entries(values).filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => ({ key, value: otelValue(value) })); +} + +function otelValue(value) { + if (typeof value === "boolean") return { boolValue: value }; + if (typeof value === "number" && Number.isFinite(value)) return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value }; + return { stringValue: typeof value === "string" ? value : JSON.stringify(value) }; +} + async function pipelineRunLogsTail() { const selector = encodeURIComponent(`tekton.dev/pipelineRun=${pipelineRun}`); const podsResult = await request("GET", `/api/v1/namespaces/${encodeURIComponent(namespace)}/pods?labelSelector=${selector}`); diff --git a/scripts/src/cicd-delivery-authority.test.ts b/scripts/src/cicd-delivery-authority.test.ts index fc9a06c8..a74b5c7d 100644 --- a/scripts/src/cicd-delivery-authority.test.ts +++ b/scripts/src/cicd-delivery-authority.test.ts @@ -262,7 +262,7 @@ describe("migrated CLI 执行 guard 与提示清理", () => { }); }); - test("Web sentinel 保持旧自动入口，不把 default SA、Pod env 或可写 metadata 当 creator proof", () => { + test("Web sentinel 自动入口保持唯一 outer PaC，不把 default SA、Pod env 或可写 metadata 当 creator proof", () => { const pacYaml = readFileSync("config/platform-infra/pipelines-as-code.yaml", "utf8"); expect(pacYaml).toContain("sentinelInternalPublish:"); expect(pacYaml).toContain("enabled: false"); @@ -271,7 +271,8 @@ describe("migrated CLI 执行 guard 与提示清理", () => { for (const node of ["jd01", "nc01"] as const) { const source = readFileSync(`.tekton/web-probe-sentinel-${node}-pac.yaml`, "utf8"); expect(source).toContain("serviceAccountName: default"); - expect(source).toContain("web-probe sentinel publish-current \\"); + expect(source).toContain("render-sentinel-publish-task.ts"); + expect(source).not.toContain("web-probe sentinel publish-current"); expect(source).not.toContain("pac-publish-current"); expect(source).not.toContain("UNIDESK_PAC_INTERNAL_ENTRYPOINT"); expect(source).not.toContain("UNIDESK_PAC_POD_UID"); @@ -532,6 +533,24 @@ describe("migrated CLI 执行 guard 与提示清理", () => { expect(() => resolvePacHistoryConsumerSelection(consumers, "NC01", "hwlab-nc01-v03", "hwlab-web-probe-sentinel-nc01-fixture")).toThrow("belongs to sentinel-nc01-v03"); }); + test("Web sentinel PaC 使用唯一 outer PipelineRun 并贯穿有界 OTel 合同", () => { + for (const node of ["nc01", "jd01"]) { + const pipeline = readFileSync(`.tekton/web-probe-sentinel-${node}-pac.yaml`, "utf8"); + expect(pipeline).toContain("render-sentinel-publish-task.ts"); + expect(pipeline).toContain("OTEL_EXPORTER_TIMEOUT_MS"); + expect(pipeline).toContain("TRACEPARENT"); + expect(pipeline).not.toContain("publish-current"); + expect(pipeline).not.toContain("submit-pipelinerun.mjs"); + } + const submitHelper = readFileSync("scripts/native/cicd/submit-pipelinerun.mjs", "utf8"); + expect(submitHelper).toContain("parentSpanId"); + expect(submitHelper).toContain("AbortController"); + expect(submitHelper).toContain("warning: true, blocking: false"); + const historyCollector = readFileSync("scripts/src/platform-infra-pipelines-as-code-remote.sh", "utf8"); + expect(historyCollector).toContain("state.firstBreak"); + expect(historyCollector).toContain("state.traceId"); + }); + test("PaC、HWLAB scoped help 的真实 CLI 路由可达且默认入口不泄漏 delivery mutation", () => { const pacDefault = runCliJson(["platform-infra", "pipelines-as-code", "--help"]); const pacBootstrap = runCliJson(["platform-infra", "pipelines-as-code", "help", "platform-bootstrap"]); diff --git a/scripts/src/cicd-node-status.ts b/scripts/src/cicd-node-status.ts index fa25ac1e..42f7e5fa 100644 --- a/scripts/src/cicd-node-status.ts +++ b/scripts/src/cicd-node-status.ts @@ -90,6 +90,9 @@ function renderNodeStatus(result: Record, full: boolean): Rende `${stringValue(record(row.argo).sync)}/${stringValue(record(row.argo).health)}`, stringValue(record(record(row.argo).revisionRelation).relation), runtimeText(record(row.runtime)), + short(stringValue(row.traceId), 10), + stringValue(record(row.firstBreak).code), + stringValue(record(row.error).code), stringValue(row.reason), ]); const detailRows = consumers.map((row) => [ @@ -111,7 +114,7 @@ function renderNodeStatus(result: Record, full: boolean): Rende ]]), "", "CONSUMERS", - ...(rows.length === 0 ? ["-"] : table(["CONSUMER", "READY", "PIPELINE", "DUR", "SOURCE", "GITOPS", "ARGO", "RELATION", "RUNTIME", "REASON"], rows)), + ...(rows.length === 0 ? ["-"] : table(["CONSUMER", "READY", "PIPELINE", "DUR", "SOURCE", "GITOPS", "ARGO", "RELATION", "RUNTIME", "TRACE", "FIRST_BREAK", "ERROR", "REASON"], rows)), "", ...(full ? [ "DETAIL", diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts index 06d2ed56..b1b706c6 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -403,6 +403,10 @@ function runSentinelPublishPipelineRunInCluster(state: SentinelCicdState, manife payload: Object.keys(payload).length === 0 ? { ok: false, status: completed ? "result-missing" : failed ? "failed" : "timeout", valuesRedacted: true } : payload, polls: parsed.polls ?? null, elapsedMs: parsed.elapsedMs ?? result.durationMs ?? null, + traceId: parsed.traceId ?? null, + traceparent: parsed.traceparent ?? null, + firstBreak: parsed.firstBreak ?? null, + error: parsed.error ?? null, probe: { nativeSubmit: parsed, capture: compactCommand(result), @@ -560,7 +564,7 @@ function requireSentinelBuildkitImage(state: SentinelCicdState): string { return image; } -function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> { +export function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: string; value: string }> { const proxy = state.spec.networkProfile.imageBuildProxy; const noProxy = proxy.noProxy.join(","); return [ @@ -575,7 +579,7 @@ function sentinelImageBuildProxyEnv(state: SentinelCicdState): Array<{ name: str ]; } -function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string { +export function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): string { const monitorWeb = record(state.image.monitorWeb); const checkoutPathsB64 = Buffer.from(JSON.stringify(arrayAt(state.cicd, "source.checkoutPaths").map((item) => { if (typeof item !== "string" || item.length === 0 || item.startsWith("/") || item.includes("..")) throw new Error("source.checkoutPaths must contain safe relative paths"); @@ -604,8 +608,8 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): ": > \"$event_log\"", "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; current_phase=$stage; export current_phase; finished=$(now_ms); elapsed=$((finished - started)); trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"traceId\":\"%s\",\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" \"$trace_id\" | tee -a \"$event_log\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"child-process-exit-nonzero\",\"phase\":\"%s\",\"reason\":\"publish step command exited non-zero\",\"valuesRedacted\":true},\"error\":{\"type\":\"ChildProcessError\",\"code\":\"EXIT_NONZERO\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"${current_phase:-source}\" \"${current_phase:-source}\" \"$code\" \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", "started_ms=$(now_ms)", "write_meta started_ms \"$started_ms\"", @@ -703,7 +707,7 @@ function sentinelPublishSourceShell(state: SentinelCicdState, jobName: string): ].join("\n"); } -function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string { +export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: string): string { const monitorWeb = record(state.image.monitorWeb); const imageBuildPackageMode = stringAt(monitorWeb, "imageBuildPackageMode"); const imageBuildNetworkMode = stringAt(monitorWeb, "imageBuildNetworkMode"); @@ -727,8 +731,8 @@ function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: strin "build_log=/workspace/image-build.log", "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", "write_meta() { printf '%s' \"$2\" > \"$meta_dir/$1\"; }", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" | tee -a \"$event_log\"; }", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; current_phase=$stage; export current_phase; finished=$(now_ms); elapsed=$((finished - started)); trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"traceId\":\"%s\",\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" \"$trace_id\" | tee -a \"$event_log\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"child-process-exit-nonzero\",\"phase\":\"%s\",\"reason\":\"image build command exited non-zero\",\"valuesRedacted\":true},\"error\":{\"type\":\"ChildProcessError\",\"code\":\"EXIT_NONZERO\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"${current_phase:-image-build}\" \"${current_phase:-image-build}\" \"$code\" \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", "cd /workspace/source", "image_build_http_proxy_present=false; if [ -n \"$image_build_http_proxy\" ]; then image_build_http_proxy_present=true; fi", @@ -767,7 +771,7 @@ function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName: strin ].join("\n"); } -function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { +export function sentinelPublishShell(state: SentinelCicdState, jobName: string, publishGitops: boolean): string { const gitopsFiles = publishGitops ? sentinelGitopsFiles(state) : []; const filesB64 = Buffer.from(JSON.stringify(gitopsFiles.map((file) => ({ path: file.path, @@ -786,8 +790,8 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "build_log=/workspace/image-build.log", "now_ms() { seconds=$(date +%s); printf '%s\\n' $((seconds * 1000)); }", "read_meta() { cat \"$meta_dir/$1\"; }", - "emit_stage() { stage=$1; status=$2; started=$3; finished=$(now_ms); elapsed=$((finished - started)); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\"; }", - "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then printf '{\"ok\":false,\"status\":\"failed\",\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$code\" \"$job_name\"; fi; exit \"$code\"; }", + "emit_stage() { stage=$1; status=$2; started=$3; current_phase=$stage; export current_phase; finished=$(now_ms); elapsed=$((finished - started)); trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"event\":\"sentinel-publish-stage\",\"stage\":\"%s\",\"status\":\"%s\",\"elapsedMs\":%s,\"traceId\":\"%s\",\"valuesRedacted\":true}\\n' \"$stage\" \"$status\" \"$elapsed\" \"$trace_id\"; }", + "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"child-process-exit-nonzero\",\"phase\":\"%s\",\"reason\":\"GitOps publish command exited non-zero\",\"valuesRedacted\":true},\"error\":{\"type\":\"ChildProcessError\",\"code\":\"EXIT_NONZERO\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"${current_phase:-gitops-publish}\" \"${current_phase:-gitops-publish}\" \"$code\" \"$code\" \"$job_name\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", "if [ -f \"$event_log\" ]; then cat \"$event_log\"; fi", "mkdir -p /root/.ssh", diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts b/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts index a4a33d1c..2d58f86b 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-shared.ts @@ -515,6 +515,8 @@ export function renderPublishCurrentResult(result: Record): str const stageBudgets = record(result.stageBudgets); const validationPlan = record(result.validationPlan); const blocker = record(result.blocker); + const firstBreak = record(result.firstBreak); + const typedError = record(result.error); const recoveryNext = record(controlPlane.recoveryNext); const next = record(result.next); const drillDown = record(result.drillDown); @@ -609,6 +611,15 @@ export function renderPublishCurrentResult(result: Record): str "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", + table(["TRACE_ID", "FIRST_BREAK", "ERROR_TYPE", "ERROR_CODE", "HTTP", "EXIT"], [[ + result.traceId ?? "-", + firstBreak.code ?? "-", + typedError.type ?? "-", + typedError.code ?? "-", + typedError.httpStatus ?? "-", + typedError.exitCode ?? "-", + ]]), + "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : ["BLOCKER", table(["CODE", "REASON"], [[blocker.code, blocker.reason]])].join("\n"), "", renderSentinelDrillDown(drillDown), @@ -685,6 +696,9 @@ export function renderControlPlaneResult(result: Record): strin const argoApply = record(result.argoApply); const blocker = record(result.blocker); const statusDiagnosis = record(result.statusDiagnosis); + const traceId = result.traceId ?? deliveryStatus.traceId ?? record(publish).traceId ?? "-"; + const firstBreak = record(result.firstBreak); + const typedError = record(result.error); const drillDown = record(result.drillDown); const targetValidation = record(result.targetValidation); const targetValidationBusiness = record(targetValidation.businessStatus); @@ -742,6 +756,8 @@ export function renderControlPlaneResult(result: Record): strin warnings.length, ]]), "", + table(["TRACE_ID", "FIRST_BREAK", "ERROR_TYPE", "ERROR_CODE"], [[traceId, firstBreak.code ?? "-", typedError.type ?? "-", typedError.code ?? "-"]]), + "", flush.mode === "async-job" ? `ASYNC_FLUSH_STATUS\n ${flushNext.status ?? "-"}` : "ASYNC_FLUSH_STATUS\n-", "", renderSentinelNext({ ...next, pipelineRunStatus: drillDown.pipelineRunStatus }), @@ -786,6 +802,8 @@ export function renderControlPlaneResult(result: Record): strin ]]), ].join("\n"), "", + table(["TRACE_ID", "FIRST_BREAK", "ERROR_TYPE", "ERROR_CODE"], [[traceId, firstBreak.code ?? statusDiagnosis.code ?? "-", typedError.type ?? "-", typedError.code ?? "-"]]), + "", renderSentinelDrillDown(drillDown), "", Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "STAGE_REF", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), short(record(sourceMirrorSync.payload).stageRef), sourceMirrorSync.elapsedMs ?? "-"]]), diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index e8146a9b..aa5d3bb8 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -578,6 +578,21 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio const publish = runSentinelPublishJob(state, true, Math.max(1, remainingBudgetSeconds()), options.rerun); const elapsedMs = Date.now() - startedAt; const ok = record(publish).ok === true; + const publishFirstBreak = record(record(publish).firstBreak); + const publishError = record(record(publish).error); + if (!ok) { + sentinelProgressEvent("sentinel.publish.failure", { + phase: publishFirstBreak.phase ?? record(publish).phase ?? "sentinel-publish", + status: "failed", + traceId: record(publish).traceId ?? null, + firstBreak: Object.keys(publishFirstBreak).length === 0 ? null : publishFirstBreak, + error: Object.keys(publishError).length === 0 ? null : publishError, + pipelineRun: record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun), + sourceCommit: state.sourceHead.commit, + node: state.spec.nodeId, + lane: state.spec.lane, + }); + } const result = { ok, command, @@ -590,6 +605,9 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio source: state.sourceHead, image: state.image, pipelineRun: record(publish).jobName ?? sentinelPipelineRunName(state, options.rerun), + traceId: record(publish).traceId ?? null, + firstBreak: Object.keys(publishFirstBreak).length === 0 ? null : publishFirstBreak, + error: Object.keys(publishError).length === 0 ? null : publishError, controlPlane: { ok, mode: "pac-publish-only", @@ -613,7 +631,10 @@ function runSentinelPublishCurrentConfirmedInner(state: SentinelCicdState, optio ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", controlPlaneWaitWarningSeconds(state)), ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ], - blocker: ok ? null : { code: "sentinel-pac-publish-not-ready", reason: "sentinel publish PipelineRun did not complete successfully" }, + blocker: ok ? null : { + code: publishFirstBreak.code ?? publishError.code ?? "sentinel-pac-publish-not-ready", + reason: publishFirstBreak.reason ?? publishError.message ?? "sentinel publish PipelineRun did not complete successfully", + }, recoveryNext: controlPlaneRecoveryNext(state, ok, publish, null, false), next: publishCurrentNext(state), valuesRedacted: true, diff --git a/scripts/src/platform-infra-pipelines-as-code-remote.sh b/scripts/src/platform-infra-pipelines-as-code-remote.sh index 6c82d3af..3328940d 100644 --- a/scripts/src/platform-infra-pipelines-as-code-remote.sh +++ b/scripts/src/platform-infra-pipelines-as-code-remote.sh @@ -712,11 +712,37 @@ function recursiveEnvReuse(value, state) { if (typeof env.serviceReusedCount === 'number') state.serviceReusedCount = env.serviceReusedCount; } if (typeof value.status === 'string' && /reus|skip|built|publish/u.test(value.status)) state.status = value.status; + if (state.traceId === null && typeof value.traceId === 'string' && /^[0-9a-f]{32}$/u.test(value.traceId)) state.traceId = value.traceId; + if (state.firstBreak === null && value.firstBreak && typeof value.firstBreak === 'object') state.firstBreak = boundedFailureRecord(value.firstBreak); + if (state.error === null && value.error && typeof value.error === 'object') state.error = boundedFailureRecord(value.error); for (const item of Object.values(value)) recursiveEnvReuse(item, state); } +function boundedFailureRecord(value) { + const input = value && typeof value === 'object' && !Array.isArray(value) ? value : {}; + const token = (item) => typeof item === 'string' && /^[A-Za-z0-9_.-]{1,100}$/u.test(item) ? item : null; + const summary = (item) => typeof item === 'string' + ? item.replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/giu, '$1[REDACTED]') + .replace(/(token|password|secret)(\s*[:=]\s*)\S+/giu, '$1$2[REDACTED]') + .replace(/https?:\/\/[^\s/]+@/giu, 'https://[REDACTED]@') + .replace(/([?&][^=\s]+)=([^&\s]+)/gu, '$1=[REDACTED]') + .replace(/\s+/gu, ' ').trim().slice(0, 480) + : null; + return { + type: token(input.type), + code: token(input.code), + phase: token(input.phase), + httpStatus: Number.isInteger(input.httpStatus) ? input.httpStatus : null, + exitCode: Number.isInteger(input.exitCode) ? input.exitCode : null, + reason: summary(input.reason), + message: summary(input.message), + stderrSummary: summary(input.stderrSummary), + valuesPrinted: false, + }; +} + function envReuseForPipelineRun(namespace, name, taskRuns) { - const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null }; + const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null }; const matching = (taskRuns.items || []).filter((item) => item.metadata?.labels?.['tekton.dev/pipelineRun'] === name); const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null); const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean); @@ -870,6 +896,9 @@ function rowFor(consumer, item, taskRuns) { sourceObservation, artifact: evidence.artifact, collector: evidence.collector, + traceId: evidence.traceId, + firstBreak: evidence.firstBreak, + error: evidence.error, taskRuns: tasks, source: classification.deliveryClass === 'outer-pac-event' ? 'gitea-pac-event-observed' @@ -1007,6 +1036,32 @@ if (contractTaskRuns.length === 0) { } const records = [...parsePacLogRecords(logText), ...terminalRecords]; const out = extractPacArtifactEvidence(records, logText); +const diagnostic = { traceId: null, firstBreak: null, error: null }; +function visit(value) { + if (value === null || value === undefined) return; + if (Array.isArray(value)) { for (const item of value) visit(item); return; } + if (typeof value !== 'object') return; + if (diagnostic.traceId === null && typeof value.traceId === 'string' && /^[0-9a-f]{32}$/u.test(value.traceId)) diagnostic.traceId = value.traceId; + if (diagnostic.firstBreak === null && value.firstBreak && typeof value.firstBreak === 'object') diagnostic.firstBreak = boundedFailureRecord(value.firstBreak); + if (diagnostic.error === null && value.error && typeof value.error === 'object') diagnostic.error = boundedFailureRecord(value.error); + for (const item of Object.values(value)) visit(item); +} +function boundedFailureRecord(value) { + const input = value && typeof value === 'object' && !Array.isArray(value) ? value : {}; + const token = (item) => typeof item === 'string' && /^[A-Za-z0-9_.-]{1,100}$/u.test(item) ? item : null; + const summary = (item) => typeof item === 'string' + ? item.replace(/(authorization\s*[:=]\s*)(?:bearer\s+)?\S+/giu, '$1[REDACTED]') + .replace(/(token|password|secret)(\s*[:=]\s*)\S+/giu, '$1$2[REDACTED]') + .replace(/https?:\/\/[^\s/]+@/giu, 'https://[REDACTED]@') + .replace(/([?&][^=\s]+)=([^&\s]+)/gu, '$1=[REDACTED]') + .replace(/\s+/gu, ' ').trim().slice(0, 480) + : null; + return { type: token(input.type), code: token(input.code), phase: token(input.phase), httpStatus: Number.isInteger(input.httpStatus) ? input.httpStatus : null, exitCode: Number.isInteger(input.exitCode) ? input.exitCode : null, reason: summary(input.reason), message: summary(input.message), stderrSummary: summary(input.stderrSummary), valuesPrinted: false }; +} +for (const record of records) visit(record); +out.traceId = diagnostic.traceId; +out.firstBreak = diagnostic.firstBreak; +out.error = diagnostic.error; out.collector = { mode, contractTaskCount: contractTaskRuns.length, diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index be5e7a9d..c16e5aae 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -73,6 +73,15 @@ interface PacConfig { display: { timeZone: string; }; + observability: { + configRef: string; + tracesEndpoint: string; + serviceName: string; + propagation: string[]; + samplingMode: "parentbased_traceidratio"; + samplingRatio: number; + exporterFailure: { warning: true; blocking: false; timeoutMs: number }; + }; release: { version: string; manifestUrl: string; @@ -414,6 +423,9 @@ function readPacConfig(): PacConfig { const gitOpsMirrorFlush = y.objectField(closeout, "gitOpsMirrorFlush", "closeout"); const gitea = y.objectField(root, "gitea", ""); const display = y.objectField(root, "display", ""); + const observability = y.objectField(root, "observability", ""); + const observabilitySampling = y.objectField(observability, "sampling", "observability"); + const exporterFailure = y.objectField(observability, "exporterFailure", "observability"); const admin = y.objectField(gitea, "admin", "gitea"); const webhook = y.objectField(gitea, "webhook", "gitea"); const repositories = root.repositories === undefined @@ -439,6 +451,19 @@ function readPacConfig(): PacConfig { display: { timeZone: y.stringField(display, "timeZone", "display"), }, + observability: { + configRef: y.stringField(observability, "configRef", "observability"), + tracesEndpoint: urlField(observability, "tracesEndpoint", "observability"), + serviceName: y.stringField(observability, "serviceName", "observability"), + propagation: y.stringArrayField(observability, "propagation", "observability"), + samplingMode: y.enumField(observabilitySampling, "mode", "observability.sampling", ["parentbased_traceidratio"] as const), + samplingRatio: numberInRange(observabilitySampling.ratio, 0, 1, "observability.sampling.ratio"), + exporterFailure: { + warning: literalBoolean(exporterFailure, "warning", true, "observability.exporterFailure"), + blocking: literalBoolean(exporterFailure, "blocking", false, "observability.exporterFailure"), + timeoutMs: integerInRange(exporterFailure.timeoutMs, 50, 2_000, "observability.exporterFailure.timeoutMs"), + }, + }, release: { version: y.stringField(release, "version", "release"), manifestUrl: urlField(release, "manifestUrl", "release"), @@ -802,6 +827,17 @@ function validateConfig(config: PacConfig): void { if (consumerIds.has(consumer.id)) throw new Error(`${configLabel}.consumers id must be unique: ${consumer.id}`); consumerIds.add(consumer.id); const repository = resolveRepository(config, consumer.repositoryRef); + if (consumer.id.startsWith("sentinel-")) { + if (repository.params.otel_traces_endpoint !== config.observability.tracesEndpoint) { + throw new Error(`${configLabel}.repositories.${repository.id}.params.otel_traces_endpoint must match observability.tracesEndpoint`); + } + if (repository.params.otel_sampling_ratio !== String(config.observability.samplingRatio)) { + throw new Error(`${configLabel}.repositories.${repository.id}.params.otel_sampling_ratio must match observability.sampling.ratio`); + } + if (repository.params.otel_exporter_timeout_ms !== String(config.observability.exporterFailure.timeoutMs)) { + throw new Error(`${configLabel}.repositories.${repository.id}.params.otel_exporter_timeout_ms must match observability.exporterFailure.timeoutMs`); + } + } if (repository.namespace !== consumer.namespace) throw new Error(`${configLabel}.consumers.${consumer.id}.namespace must match repository namespace`); if (consumer.deliveryProvenance !== null) { if (consumer.sourceArtifact === null) throw new Error(`${configLabel}.consumers.${consumer.id}.deliveryProvenance requires sourceArtifact ownership`); @@ -845,6 +881,24 @@ function validateConfig(config: PacConfig): void { validateTimeZone(config.display.timeZone, `${configLabel}.display.timeZone`); } +function numberInRange(value: unknown, minimum: number, maximum: number, path: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < minimum || value > maximum) { + throw new Error(`${configLabel}.${path} must be a number from ${minimum} to ${maximum}`); + } + return value; +} + +function integerInRange(value: unknown, minimum: number, maximum: number, path: string): number { + const parsed = numberInRange(value, minimum, maximum, path); + if (!Number.isInteger(parsed)) throw new Error(`${configLabel}.${path} must be an integer`); + return parsed; +} + +function literalBoolean(value: Record, key: string, expected: T, path: string): T { + if (value[key] !== expected) throw new Error(`${configLabel}.${path}.${key} must be ${String(expected)}`); + return expected; +} + function resolveTarget(config: PacConfig, targetId: string | null): PacTarget { const id = targetId ?? config.defaults.targetId; const target = config.targets.find((item) => item.id.toLowerCase() === id.toLowerCase()); @@ -1281,7 +1335,11 @@ export function resolvePacHistoryConsumerSelection( async function debugStep(config: UniDeskConfig, options: HistoryOptions): Promise> { const pac = readPacConfig(); const target = resolveTarget(pac, options.targetId); - const consumer = resolveConsumer(pac, options.consumerId); + const selection = resolvePacHistoryConsumerSelection(pac.consumers, target.id, options.consumerId, options.detailId); + if (selection.selectedConsumerIds.length !== 1) { + throw new Error("debug-step requires --consumer or --id resolving exactly one Pipelines-as-Code consumer"); + } + const consumer = resolveConsumer(pac, selection.selectedConsumerIds[0] ?? null); const repository = resolveRepository(pac, consumer.repositoryRef); const result = await capture(config, target.route, ["sh"], remoteScript("debug-step", pac, target, repository, consumer, { ...options, confirm: false, dryRun: true, wait: false }, emptySecretMaterial(), "")); const parsed = parseJsonOutput(result.stdout); @@ -1681,6 +1739,9 @@ function nodeStatusRow(targetId: string, consumer: PacConsumer, repository: PacR phase: stringValue(diagnostics.phase), hint: compactLine(stringValue(diagnostics.hint)), }, + traceId: stringValue(artifact.traceId), + firstBreak: record(artifact.firstBreak), + error: record(artifact.error), reason: ready ? "ready" : nodeStatusReason({ ciReady, argoReady, runtimeReady, diagnosticsReady, pipelineStatus: statusText(latest), diagnostics }), statusCommand: `bun scripts/cli.ts platform-infra pipelines-as-code status --target ${targetId} --consumer ${consumer.id}`, historyCommand: `bun scripts/cli.ts platform-infra pipelines-as-code history --target ${targetId} --consumer ${consumer.id} --limit 10`, @@ -2803,6 +2864,8 @@ function renderHistoryDetail(row: Record): string[] { const longest = record(taskRuns.longest); const failed = record(taskRuns.failed); const failedStep = record(failed.failedStep); + const firstBreak = record(row.firstBreak); + const typedError = record(row.error); return [ "", "DETAIL", @@ -2826,6 +2889,14 @@ function renderHistoryDetail(row: Record): string[] { ["failedStepExit", stringValue(failedStep.exitCode)], ["failedStepTermination", stringValue(failedStep.terminationReason ?? failedStep.reason)], ["failure", compactLine(stringValue(failed.message))], + ["traceId", stringValue(row.traceId)], + ["firstBreak", stringValue(firstBreak.code)], + ["firstBreakPhase", stringValue(firstBreak.phase)], + ["errorType", stringValue(typedError.type)], + ["errorCode", stringValue(typedError.code)], + ["errorHttpStatus", stringValue(typedError.httpStatus)], + ["errorExitCode", stringValue(typedError.exitCode)], + ["errorSummary", compactLine(stringValue(typedError.stderrSummary ?? typedError.message))], ["envReuseSource", stringValue(envReuse.source)], ["deliveryMode", stringValue(sourceObservation.mode)], ["deliveryValid", stringValue(sourceObservation.valid)],