diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts index 04718d8c..e27f8a91 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -737,6 +737,7 @@ export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName "failure_error_type=ChildProcessError", "failure_error_code=EXIT_NONZERO", "buildkitd_pid=''", + "isolate_buildkit_otel() { unset TRACEPARENT TRACESTATE OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_TRACES_ENDPOINT OTEL_EXPORTER_OTLP_PROTOCOL OTEL_EXPORTER_OTLP_TRACES_PROTOCOL OTEL_EXPORTER_OTLP_HEADERS OTEL_EXPORTER_OTLP_TRACES_HEADERS; export OTEL_SDK_DISABLED=true OTEL_TRACES_EXPORTER=none; }", "cleanup_buildkitd() { if [ -n \"$buildkitd_pid\" ] && kill -0 \"$buildkitd_pid\" 2>/dev/null; then kill -TERM \"$buildkitd_pid\" 2>/dev/null || true; cleanup_attempt=0; while [ \"$cleanup_attempt\" -lt 5 ] && kill -0 \"$buildkitd_pid\" 2>/dev/null; do cleanup_attempt=$((cleanup_attempt + 1)); sleep 1; done; if kill -0 \"$buildkitd_pid\" 2>/dev/null; then kill -KILL \"$buildkitd_pid\" 2>/dev/null || true; fi; wait \"$buildkitd_pid\" 2>/dev/null || true; fi; }", "emit_failed() { code=$?; cleanup_buildkitd; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"%s\",\"phase\":\"%s\",\"reason\":\"%s\",\"valuesRedacted\":true},\"error\":{\"type\":\"%s\",\"code\":\"%s\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"$failure_code\" \"${current_phase:-image-build}\" \"$failure_reason\" \"$failure_error_type\" \"$failure_error_code\" \"${current_phase:-image-build}\" \"$code\" \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", @@ -761,14 +762,14 @@ export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName "buildkit_log=/workspace/buildkitd.log", "mkdir -p \"$(dirname \"$buildkit_socket\")\"", "rm -f \"$buildkit_socket\"", - "rootlesskit buildkitd --addr \"$buildkit_address\" ${BUILDKITD_FLAGS:-} > \"$buildkit_log\" 2>&1 &", + "( isolate_buildkit_otel; exec rootlesskit buildkitd --addr \"$buildkit_address\" ${BUILDKITD_FLAGS:-} ) > \"$buildkit_log\" 2>&1 &", "buildkitd_pid=$!", "buildkit_ready=false", "buildkit_probe_attempt=0", - "while [ \"$buildkit_probe_attempt\" -lt 30 ]; do buildkit_probe_attempt=$((buildkit_probe_attempt + 1)); if [ -S \"$buildkit_socket\" ] && buildctl --addr \"$buildkit_address\" debug workers >/dev/null 2>&1; then buildkit_ready=true; break; fi; if ! kill -0 \"$buildkitd_pid\" 2>/dev/null; then break; fi; sleep 1; done", + "while [ \"$buildkit_probe_attempt\" -lt 30 ]; do buildkit_probe_attempt=$((buildkit_probe_attempt + 1)); if [ -S \"$buildkit_socket\" ] && ( isolate_buildkit_otel; exec buildctl --addr \"$buildkit_address\" debug workers ) >/dev/null 2>&1; then buildkit_ready=true; break; fi; if ! kill -0 \"$buildkitd_pid\" 2>/dev/null; then break; fi; sleep 1; done", "if [ \"$buildkit_ready\" != true ]; then cat \"$buildkit_log\"; failure_code=buildkit-not-ready; failure_reason=readiness-timeout; failure_error_type=BuildKitReadinessError; failure_error_code=BUILDKIT_NOT_READY; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi", "build_status=0", - "env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl --addr \"$buildkit_address\" build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1 || build_status=$?", + "( isolate_buildkit_otel; exec env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl --addr \"$buildkit_address\" build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" ) > \"$build_log\" 2>&1 || build_status=$?", "cat \"$build_log\"", "image_build_finished_ms=$(now_ms)", "write_meta image_build_finished_ms \"$image_build_finished_ms\"", diff --git a/scripts/src/platform-infra-pac-failure-evidence.test.ts b/scripts/src/platform-infra-pac-failure-evidence.test.ts index 5b9c182b..a1ececa9 100644 --- a/scripts/src/platform-infra-pac-failure-evidence.test.ts +++ b/scripts/src/platform-infra-pac-failure-evidence.test.ts @@ -78,9 +78,14 @@ describe("PaC 失败证据合同", () => { }, } as unknown as SentinelCicdState, "sentinel-test"); expect(shell).toContain("mkdir -p \"$(dirname \"$buildkit_socket\")\""); - expect(shell).toContain("rootlesskit buildkitd --addr"); + expect(shell).toContain("emit_stage() {"); + expect(shell).toContain('${TRACEPARENT:-}'); + expect(shell).toContain("unset TRACEPARENT TRACESTATE OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_TRACES_ENDPOINT OTEL_EXPORTER_OTLP_PROTOCOL OTEL_EXPORTER_OTLP_TRACES_PROTOCOL OTEL_EXPORTER_OTLP_HEADERS OTEL_EXPORTER_OTLP_TRACES_HEADERS"); + expect(shell).toContain("export OTEL_SDK_DISABLED=true OTEL_TRACES_EXPORTER=none"); + expect(shell).toContain("exec rootlesskit buildkitd --addr"); expect(shell).not.toMatch(/(^|\n)buildkitd --addr/); - expect(shell).toContain("debug workers"); + expect(shell).toContain("exec buildctl --addr \"$buildkit_address\" debug workers"); + expect(shell).toContain("exec env HTTP_PROXY=\"$image_build_http_proxy\""); expect(shell).toContain("failure_error_type=BuildKitReadinessError"); expect(shell).toContain("failure_error_code=BUILDKIT_NOT_READY"); expect(shell).toContain("cleanup_buildkitd");