From ea7faeb8786955086d3f2a8e42474702d9495ede Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 17:30:01 +0200 Subject: [PATCH 1/8] fix: isolate BuildKit child OTel exporters --- scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts | 7 ++++--- scripts/src/platform-infra-pac-failure-evidence.test.ts | 9 +++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts index 04718d8c..e27f8a91 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd-jobs.ts @@ -737,6 +737,7 @@ export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName "failure_error_type=ChildProcessError", "failure_error_code=EXIT_NONZERO", "buildkitd_pid=''", + "isolate_buildkit_otel() { unset TRACEPARENT TRACESTATE OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_TRACES_ENDPOINT OTEL_EXPORTER_OTLP_PROTOCOL OTEL_EXPORTER_OTLP_TRACES_PROTOCOL OTEL_EXPORTER_OTLP_HEADERS OTEL_EXPORTER_OTLP_TRACES_HEADERS; export OTEL_SDK_DISABLED=true OTEL_TRACES_EXPORTER=none; }", "cleanup_buildkitd() { if [ -n \"$buildkitd_pid\" ] && kill -0 \"$buildkitd_pid\" 2>/dev/null; then kill -TERM \"$buildkitd_pid\" 2>/dev/null || true; cleanup_attempt=0; while [ \"$cleanup_attempt\" -lt 5 ] && kill -0 \"$buildkitd_pid\" 2>/dev/null; do cleanup_attempt=$((cleanup_attempt + 1)); sleep 1; done; if kill -0 \"$buildkitd_pid\" 2>/dev/null; then kill -KILL \"$buildkitd_pid\" 2>/dev/null || true; fi; wait \"$buildkitd_pid\" 2>/dev/null || true; fi; }", "emit_failed() { code=$?; cleanup_buildkitd; if [ \"$code\" -ne 0 ]; then trace_id=$(printf '%s' \"${TRACEPARENT:-}\" | cut -d- -f2); printf '{\"ok\":false,\"status\":\"failed\",\"traceId\":\"%s\",\"firstBreak\":{\"code\":\"%s\",\"phase\":\"%s\",\"reason\":\"%s\",\"valuesRedacted\":true},\"error\":{\"type\":\"%s\",\"code\":\"%s\",\"phase\":\"%s\",\"exitCode\":%s,\"stderrSummary\":\"bounded step logs contain the command failure\",\"valuesRedacted\":true},\"exitCode\":%s,\"jobName\":\"%s\",\"valuesRedacted\":true}\\n' \"$trace_id\" \"$failure_code\" \"${current_phase:-image-build}\" \"$failure_reason\" \"$failure_error_type\" \"$failure_error_code\" \"${current_phase:-image-build}\" \"$code\" \"$code\" \"$job_name\" | tee -a \"$event_log\"; fi; exit \"$code\"; }", "trap emit_failed EXIT", @@ -761,14 +762,14 @@ export function sentinelPublishImageBuildShell(state: SentinelCicdState, jobName "buildkit_log=/workspace/buildkitd.log", "mkdir -p \"$(dirname \"$buildkit_socket\")\"", "rm -f \"$buildkit_socket\"", - "rootlesskit buildkitd --addr \"$buildkit_address\" ${BUILDKITD_FLAGS:-} > \"$buildkit_log\" 2>&1 &", + "( isolate_buildkit_otel; exec rootlesskit buildkitd --addr \"$buildkit_address\" ${BUILDKITD_FLAGS:-} ) > \"$buildkit_log\" 2>&1 &", "buildkitd_pid=$!", "buildkit_ready=false", "buildkit_probe_attempt=0", - "while [ \"$buildkit_probe_attempt\" -lt 30 ]; do buildkit_probe_attempt=$((buildkit_probe_attempt + 1)); if [ -S \"$buildkit_socket\" ] && buildctl --addr \"$buildkit_address\" debug workers >/dev/null 2>&1; then buildkit_ready=true; break; fi; if ! kill -0 \"$buildkitd_pid\" 2>/dev/null; then break; fi; sleep 1; done", + "while [ \"$buildkit_probe_attempt\" -lt 30 ]; do buildkit_probe_attempt=$((buildkit_probe_attempt + 1)); if [ -S \"$buildkit_socket\" ] && ( isolate_buildkit_otel; exec buildctl --addr \"$buildkit_address\" debug workers ) >/dev/null 2>&1; then buildkit_ready=true; break; fi; if ! kill -0 \"$buildkitd_pid\" 2>/dev/null; then break; fi; sleep 1; done", "if [ \"$buildkit_ready\" != true ]; then cat \"$buildkit_log\"; failure_code=buildkit-not-ready; failure_reason=readiness-timeout; failure_error_type=BuildKitReadinessError; failure_error_code=BUILDKIT_NOT_READY; emit_stage image-build failed \"$image_build_started_ms\"; exit 1; fi", "build_status=0", - "env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl --addr \"$buildkit_address\" build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" > \"$build_log\" 2>&1 || build_status=$?", + "( isolate_buildkit_otel; exec env HTTP_PROXY=\"$image_build_http_proxy\" HTTPS_PROXY=\"$image_build_https_proxy\" ALL_PROXY=\"$image_build_all_proxy\" NO_PROXY=\"$image_build_no_proxy\" http_proxy=\"$image_build_http_proxy\" https_proxy=\"$image_build_https_proxy\" all_proxy=\"$image_build_all_proxy\" no_proxy=\"$image_build_no_proxy\" buildctl --addr \"$buildkit_address\" build --allow network.host --frontend dockerfile.v0 --local context=/workspace/source --local dockerfile=/workspace/source --opt filename=Containerfile.web-probe-sentinel --opt \"network=$image_build_network_mode\" --opt \"build-arg:HTTP_PROXY=$image_build_http_proxy\" --opt \"build-arg:HTTPS_PROXY=$image_build_https_proxy\" --opt \"build-arg:ALL_PROXY=$image_build_all_proxy\" --opt \"build-arg:NO_PROXY=$image_build_no_proxy\" --opt \"build-arg:http_proxy=$image_build_http_proxy\" --opt \"build-arg:https_proxy=$image_build_https_proxy\" --opt \"build-arg:all_proxy=$image_build_all_proxy\" --opt \"build-arg:no_proxy=$image_build_no_proxy\" --metadata-file /workspace/build-metadata.json --output \"type=image,name=$image_ref,push=true,registry.insecure=true\" ) > \"$build_log\" 2>&1 || build_status=$?", "cat \"$build_log\"", "image_build_finished_ms=$(now_ms)", "write_meta image_build_finished_ms \"$image_build_finished_ms\"", diff --git a/scripts/src/platform-infra-pac-failure-evidence.test.ts b/scripts/src/platform-infra-pac-failure-evidence.test.ts index 5b9c182b..a1ececa9 100644 --- a/scripts/src/platform-infra-pac-failure-evidence.test.ts +++ b/scripts/src/platform-infra-pac-failure-evidence.test.ts @@ -78,9 +78,14 @@ describe("PaC 失败证据合同", () => { }, } as unknown as SentinelCicdState, "sentinel-test"); expect(shell).toContain("mkdir -p \"$(dirname \"$buildkit_socket\")\""); - expect(shell).toContain("rootlesskit buildkitd --addr"); + expect(shell).toContain("emit_stage() {"); + expect(shell).toContain('${TRACEPARENT:-}'); + expect(shell).toContain("unset TRACEPARENT TRACESTATE OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_TRACES_ENDPOINT OTEL_EXPORTER_OTLP_PROTOCOL OTEL_EXPORTER_OTLP_TRACES_PROTOCOL OTEL_EXPORTER_OTLP_HEADERS OTEL_EXPORTER_OTLP_TRACES_HEADERS"); + expect(shell).toContain("export OTEL_SDK_DISABLED=true OTEL_TRACES_EXPORTER=none"); + expect(shell).toContain("exec rootlesskit buildkitd --addr"); expect(shell).not.toMatch(/(^|\n)buildkitd --addr/); - expect(shell).toContain("debug workers"); + expect(shell).toContain("exec buildctl --addr \"$buildkit_address\" debug workers"); + expect(shell).toContain("exec env HTTP_PROXY=\"$image_build_http_proxy\""); expect(shell).toContain("failure_error_type=BuildKitReadinessError"); expect(shell).toContain("failure_error_code=BUILDKIT_NOT_READY"); expect(shell).toContain("cleanup_buildkitd"); From a566bfb9cb097698a3e3b2668a763ab5fa3a434d Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 17:48:27 +0200 Subject: [PATCH 2/8] =?UTF-8?q?docs:=20=E8=B7=9F=E8=B8=AA=20AgentRun=20ses?= =?UTF-8?q?sion=20timeout=20=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/MDTODO/artificer-runtime-capabilities.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/MDTODO/artificer-runtime-capabilities.md b/docs/MDTODO/artificer-runtime-capabilities.md index c2b60016..9739ab6c 100644 --- a/docs/MDTODO/artificer-runtime-capabilities.md +++ b/docs/MDTODO/artificer-runtime-capabilities.md @@ -112,3 +112,7 @@ ### R7.3 [completed] 自动上线后以新 Artificer task input 与超过 30 分钟的真实或受控 canary 验证 2 小时 budget 生效，不修改当前 Monitor run 的既有 task policy，完成任务后将详细报告写入[任务报告](./details/artificer-runtime-capabilities/R7.3_Task_Report.md)。 + +## R8 [in_progress] + +解决 [UniDesk #1944](https://github.com/pikasTech/unidesk/issues/1944)：修复 Artificer Aipod 已为 2 小时但 UniDesk session follow-up 仍使用 15 分钟 hard-timeout 的配置分裂；将 `config/agentrun.yaml#client.sessionPolicy.executionPolicy.timeoutMs` 调整为 7200000 ms，在配置解析/校验中拒绝任何低于 3600000 ms 的 hard-timeout，并更新中文长期参考与最小测试；属于 AgentRun 自身运行时，必须由原生子代理执行，禁止 Artificer 递归修复，不新增第二 timeout authority、续跑补丁、锁、租约或状态机，完成任务后将详细报告写入[任务报告](./details/artificer-runtime-capabilities/R8_Task_Report.md)。 From 38efb1605c7296c37d97695ab0d91c2ec85c13bc Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 17:57:29 +0200 Subject: [PATCH 3/8] =?UTF-8?q?fix:=20=E7=BB=9F=E4=B8=80=20AgentRun=20sess?= =?UTF-8?q?ion=20hard-timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/agentrun.yaml | 2 +- docs/reference/agentrun.md | 12 +++++++++++- scripts/src/agentrun-events.test.ts | 2 +- scripts/src/agentrun.test.ts | 16 ++++++++++++++-- scripts/src/agentrun/config.ts | 5 ++++- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 40714f1b..d52c08c9 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -44,7 +44,7 @@ client: executionPolicy: sandbox: workspace-write approval: never - timeoutMs: 900000 + timeoutMs: 7200000 network: enabled secretScope: allowCredentialEcho: false diff --git a/docs/reference/agentrun.md b/docs/reference/agentrun.md index 367dd159..d32467b5 100644 --- a/docs/reference/agentrun.md +++ b/docs/reference/agentrun.md @@ -162,7 +162,17 @@ AgentRun Kafka closeout 需要同时验证 manager 和 runner 两条生产链路 - 终态命令仍被历史 claim、heartbeat、Pod 或 `runner-dispatch-retry` 保护时，修正 retention 分类； - 禁止强杀 Job/Pod 或手工补 CI/CD。 -AgentRun resource/session client policy 也由 `config/agentrun.yaml` 声明。`client.sessionPolicy` 是未显式选择 node/lane 时 `agentrun send session/...` 和相关 session payload 生成的默认 `tenantId`、`projectId`、`providerId`、`backendProfile`、`workspaceRef` 和 execution policy 来源；显式 `--node --lane ` 后，`explain session-policy`、`send session`、resource primitives 和 AipodSpec render 都必须改用目标 lane 的 YAML 事实。lane `secrets[].providerCredential.profile` 声明 provider credential Secret 归属，UniDesk CLI 只按 YAML 聚合 Secret name/key，不再用代码拼接 provider Secret 名称。只读入口 `bun scripts/cli.ts agentrun explain session-policy` 用于查看选中目标 lane、policy 来源、实际 executionPolicy payload 和 provider credential binding 来源；输出只能包含 Secret metadata、key 名和 `valuesPrinted=false`，不得打印 Secret value。 +- AgentRun resource/session client policy： + - 由 `config/agentrun.yaml` 声明； + - `client.sessionPolicy` 是未显式选择 node/lane 时 `agentrun send session/...` 和相关 session payload 生成的默认 `tenantId`、`projectId`、`providerId`、`backendProfile`、`workspaceRef` 和 execution policy 来源； + - `client.sessionPolicy.executionPolicy.timeoutMs` 是 session follow-up hard-timeout 的唯一事实来源； + - Artificer task 与 session follow-up 统一使用 2 小时 hard-timeout； + - 低于 1 小时的 hard-timeout 配置非法，必须在配置解析阶段明确拒绝； + - 显式 `--node --lane ` 后，`explain session-policy`、`send session`、resource primitives 和 AipodSpec render 都必须改用目标 lane 的 YAML 事实； + - lane `secrets[].providerCredential.profile` 声明 provider credential Secret 归属； + - UniDesk CLI 只按 YAML 聚合 Secret name/key，不再用代码拼接 provider Secret 名称； + - 只读入口 `bun scripts/cli.ts agentrun explain session-policy` 用于查看选中目标 lane、policy 来源、实际 executionPolicy payload 和 provider credential binding 来源； + - 输出只能包含 Secret metadata、key 名和 `valuesPrinted=false`，不得打印 Secret value。 非默认 lane 的 session follow-up 必须证明 `send session` 使用的是选中 node/lane 的 run policy。使用短命令形态前，先用 `agentrun explain session-policy --node --lane [--backend-profile ]` 或等价 dry-run/describe 路径确认 `backendProfile`、`providerId`、`workspaceRef`、execution policy 和 provider credential SecretRef 都来自目标 lane；`--prompt-stdin` 短命令形态和 `--json-stdin -o json` 显式 JSON 形态应披露同一份 `sessionPolicy` 摘要。渲染结果回退到全局默认 lane、显示错误的 default lane，或短命令与 JSON body 使用不同 policy，都是 lane policy 缺陷，应修复 YAML 目标解析或 CLI 渲染；不得通过手工创建默认 lane Secret、复制凭据、改写 JSON body 或修改 runtime namespace 来掩盖 policy 选错的问题。 diff --git a/scripts/src/agentrun-events.test.ts b/scripts/src/agentrun-events.test.ts index 5d2dfec9..f4e55f70 100644 --- a/scripts/src/agentrun-events.test.ts +++ b/scripts/src/agentrun-events.test.ts @@ -42,7 +42,7 @@ const agentRunClientYaml = [ " executionPolicy:", " sandbox: workspace-write", " approval: never", - " timeoutMs: 900000", + " timeoutMs: 7200000", " network: enabled", " secretScope:", " allowCredentialEcho: false", diff --git a/scripts/src/agentrun.test.ts b/scripts/src/agentrun.test.ts index 4b94923f..85cba155 100644 --- a/scripts/src/agentrun.test.ts +++ b/scripts/src/agentrun.test.ts @@ -76,7 +76,7 @@ const agentRunClientYaml = [ " executionPolicy:", " sandbox: workspace-write", " approval: never", - " timeoutMs: 900000", + " timeoutMs: 7200000", " network: enabled", " secretScope:", " allowCredentialEcho: false", @@ -108,7 +108,7 @@ const agentRunMinimalClientYaml = [ " executionPolicy:", " sandbox: workspace-write", " approval: never", - " timeoutMs: 900000", + " timeoutMs: 7200000", " network: enabled", " secretScope:", " allowCredentialEcho: false", @@ -166,6 +166,18 @@ describe("AgentRun render-only REST client config", () => { expect(config.client.transport).toBe("direct-http"); }); + test("uses the owning YAML two-hour session hard-timeout", () => { + const config = parseAgentRunClientConfigYaml(readFileSync("config/agentrun.yaml", "utf8"), "config/agentrun.yaml"); + expect(config.client.sessionPolicy.executionPolicy.timeoutMs).toBe(7_200_000); + }); + + test("rejects a session hard-timeout below one hour", () => { + const invalid = agentRunClientYaml.replace(" timeoutMs: 7200000", " timeoutMs: 3599999"); + expect(() => parseAgentRunClientConfigYaml(invalid, "config/agentrun.yaml")).toThrow( + "client.sessionPolicy.executionPolicy.timeoutMs hard-timeout must be at least 3600000 ms", + ); + }); + test("uses env auth before configured file auth without exposing the key", () => { const config = parseAgentRunClientConfigYaml(agentRunClientYaml, "config/agentrun.yaml"); const auth = resolveAgentRunAuth(config, { HWLAB_API_KEY: "secret-value" }); diff --git a/scripts/src/agentrun/config.ts b/scripts/src/agentrun/config.ts index ea2fd6cc..8ca1c052 100644 --- a/scripts/src/agentrun/config.ts +++ b/scripts/src/agentrun/config.ts @@ -124,7 +124,10 @@ export function readAgentRunSessionPolicyConfig(client: Record, stringFieldFromRecord(workspaceRef, "kind", `${pathValue}.workspaceRef`); stringFieldFromRecord(executionPolicy, "sandbox", `${pathValue}.executionPolicy`); stringFieldFromRecord(executionPolicy, "approval", `${pathValue}.executionPolicy`); - positiveIntegerFieldFromRecord(executionPolicy, "timeoutMs", `${pathValue}.executionPolicy`); + const timeoutMs = positiveIntegerFieldFromRecord(executionPolicy, "timeoutMs", `${pathValue}.executionPolicy`); + if (timeoutMs < 3_600_000) { + throw new Error(`${sourcePath}: ${pathValue}.executionPolicy.timeoutMs hard-timeout must be at least 3600000 ms`); + } stringFieldFromRecord(executionPolicy, "network", `${pathValue}.executionPolicy`); booleanFieldFromRecord(secretScope, "allowCredentialEcho", `${pathValue}.executionPolicy.secretScope`); return { From 8fe0837fae4041d32e45743fdcf8757deaf5acdb Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 18:00:17 +0200 Subject: [PATCH 4/8] =?UTF-8?q?docs:=20=E8=AE=B0=E5=BD=95=20R8=20timeout?= =?UTF-8?q?=20=E4=BF=AE=E5=A4=8D=E6=8A=A5=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../R8_Task_Report.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/MDTODO/details/artificer-runtime-capabilities/R8_Task_Report.md diff --git a/docs/MDTODO/details/artificer-runtime-capabilities/R8_Task_Report.md b/docs/MDTODO/details/artificer-runtime-capabilities/R8_Task_Report.md new file mode 100644 index 00000000..237b77e8 --- /dev/null +++ b/docs/MDTODO/details/artificer-runtime-capabilities/R8_Task_Report.md @@ -0,0 +1,33 @@ +# R8 任务报告 + +## 结论 + +已解决 [UniDesk #1944](https://github.com/pikasTech/unidesk/issues/1944) 的 AgentRun session hard-timeout 配置分裂，并创建普通 [PR #1945](https://github.com/pikasTech/unidesk/pull/1945) 等待主代理审核、合并。 + +- 唯一事实来源保持为 `config/agentrun.yaml#client.sessionPolicy.executionPolicy.timeoutMs`。 +- 默认 hard-timeout 已由 15 分钟调整为 `7200000` ms，即 2 小时。 +- 既有通用 session policy 解析入口现在拒绝任何低于 `3600000` ms 的 hard-timeout。 +- 未新增第二 timeout authority、续跑补丁、锁、租约、状态机或交付控制入口。 +- 未执行部署、人工 PipelineRun、mirror sync、rollout 或 PR merge。 + +## 实现 + +- `config/agentrun.yaml`：将 session execution policy 的 `timeoutMs` 调整为 `7200000`。 +- `scripts/src/agentrun/config.ts`：在 `readAgentRunSessionPolicyConfig` 中保留正整数校验，并增加 1 小时通用下限；非法值在解析阶段明确报错。 +- `scripts/src/agentrun.test.ts`：覆盖真实 owning YAML 的 2 小时值和 `3599999` ms 拒绝。 +- `scripts/src/agentrun-events.test.ts`：同步既有合法 fixture，避免保留旧 15 分钟事实。 +- `docs/reference/agentrun.md`：将 session hard-timeout 的唯一 authority、2 小时口径和 1 小时配置下限固化为中文长期参考。 + +## 验证 + +- `bun test scripts/src/agentrun.test.ts scripts/src/agentrun-events.test.ts`：25 pass，0 fail，557 assertions。 +- `bun --check scripts/src/agentrun/config.ts`：通过。 +- `UNIDESK_AGENTRUN_TARGET_EXECUTED=NC01 bun scripts/cli.ts agentrun explain session-policy -o json`：选中 `NC01/nc01-v02`，`POLICY SOURCE=selected-lane`，effective `timeoutMs=7200000`，Secret 仅披露 metadata 且 `valuesPrinted=false`。 +- `git diff --check`：通过。 + +## 工件 + +- 实现 commit：`38efb160`。 +- PR：[pikasTech/unidesk#1945](https://github.com/pikasTech/unidesk/pull/1945)。 +- Issue：[pikasTech/unidesk#1944](https://github.com/pikasTech/unidesk/issues/1944)。 +- MDTODO：`docs/MDTODO/artificer-runtime-capabilities.md` R8。 From f5b401cff1e1684207147c30cacb06f611f4c399 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 18:01:02 +0200 Subject: [PATCH 5/8] =?UTF-8?q?docs:=20=E6=94=B6=E5=8F=A3=20CI/CD=20OTel?= =?UTF-8?q?=20=E4=B8=8E=20BuildKit=20=E5=9B=9E=E5=BD=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../R3.1_Task_Report.md | 6 ++++++ .../R3.2_Task_Report.md | 6 ++++++ .../R3.3_Task_Report.md | 7 +++++++ .../R3.4_Task_Report.md | 7 +++++++ .../R3_Task_Report.md | 11 +++++++++++ docs/MDTODO/observability-trace-reliability.md | 12 ++++++++---- 6 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 docs/MDTODO/details/observability-trace-reliability/R3.1_Task_Report.md create mode 100644 docs/MDTODO/details/observability-trace-reliability/R3.2_Task_Report.md create mode 100644 docs/MDTODO/details/observability-trace-reliability/R3.3_Task_Report.md create mode 100644 docs/MDTODO/details/observability-trace-reliability/R3.4_Task_Report.md create mode 100644 docs/MDTODO/details/observability-trace-reliability/R3_Task_Report.md diff --git a/docs/MDTODO/details/observability-trace-reliability/R3.1_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R3.1_Task_Report.md new file mode 100644 index 00000000..d4bfd71c --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R3.1_Task_Report.md @@ -0,0 +1,6 @@ +# R3.1 任务报告 + +- PR #1925 与 #1937 完成 CI/CD OTel 配置、传播和 renderer 回退修复。 +- owning YAML 继续作为 endpoint、service、sampling 与 exporter timeout 唯一事实来源；无效运行参数回退 YAML，exporter 失败保持 `warning=true, blocking=false`。 +- 自动事件 `hwlab-web-probe-sentinel-nc01-bbjv2` 的 trace `120a9da4e04fc977c19dd594e0d7bb32` 已可由 Tempo 查询，证明 collector 接收生效。 +- 未人工触发 PipelineRun 或修改运行面。 diff --git a/docs/MDTODO/details/observability-trace-reliability/R3.2_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R3.2_Task_Report.md new file mode 100644 index 00000000..0d9ed075 --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R3.2_Task_Report.md @@ -0,0 +1,6 @@ +# R3.2 任务报告 + +- PR #1939 让 `history`、`debug-step`、`status` 默认显示 traceId、firstBreak、typed phase、error type/code/exit，并保留 compact JSON 同构。 +- 真实事件 `hwlab-web-probe-sentinel-nc01-czhg5` 直接暴露 `BuildKitReadinessError / BUILDKIT_NOT_READY` 及 rootless mapped-root 原始错误，无需再猜测 registry 状态。 +- 56 项定点测试、生成 shell 语法与差异检查通过。 +- PR #1939 的裸 `buildkitd` 新回归由 R3.3 继续修复，未以人工 PipelineRun 补跑。 diff --git a/docs/MDTODO/details/observability-trace-reliability/R3.3_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R3.3_Task_Report.md new file mode 100644 index 00000000..fe0aae35 --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R3.3_Task_Report.md @@ -0,0 +1,7 @@ +# R3.3 任务报告 + +- PR #1940 以 `rootlesskit buildkitd` 恢复 rootless mapped-root 合同，保留 30x1s readiness 与 TERM/KILL/wait 有界清理。 +- PR #1940 自动事件证明 BuildKit 一秒内 ready，但 wrapper OTel 环境泄漏使第三方 `buildctl` exporter 每次阻塞约 15 秒；PR #1943 仅在 BuildKit daemon、readiness 和 build 子进程内禁用 exporter，wrapper trace 保持不变。 +- PR #1943 merge commit `11d0e238162d1275d4f859bd275e0b8bc23bfa04` 自动产生 PipelineRun `hwlab-web-probe-sentinel-nc01-m5mkx`，26 秒成功。 +- 最终状态：镜像 `sha256:4c7b119a951...`、GitOps `301252718cc3`、Argo `Synced/Healthy`、runtime `1/1` 且 digest 对齐；Tempo 可查询 trace `11fba99ec4cc3cd2073be449232ee9c6`。 +- 两轮均为普通 PR merge 自动事件，未人工触发 PipelineRun。 diff --git a/docs/MDTODO/details/observability-trace-reliability/R3.4_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R3.4_Task_Report.md new file mode 100644 index 00000000..3ee66453 --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R3.4_Task_Report.md @@ -0,0 +1,7 @@ +# R3.4 任务报告 + +- 潜伏缺陷最早由 commit `6889f6a13303406aeec2e19c1a99987e151b62bf`、PR #1294 引入；作者为 Codex，PR 账号为 pikasTech。动机是移除 host Docker socket并迁移到 rootless BuildKit。 +- 该实现采用官方 `buildctl-daemonless.sh` 默认 10 次短连接探测。成功时 daemon 启动足够快；启动稍慢时 wrapper 先失败，而 worker/socket 随后才 ready。 +- PR #1937 只是首个观测失败 source commit，不是 BuildKit 根因；PR #1938 也与 BuildKit 无关。镜像 digest、节点、securityContext 与 state 路径在成功/失败样本间一致，不能把瞬时节点负载写成已证实因果。 +- PR #1939 的 commit `ca312ae7142a35ccb2c5a897c54871dacc101b20` 又直接执行裸 `buildkitd`，确定性违反 rootless mapped-root 合同，属于后续新回归。 +- 调查仅使用既有 Git、PR、PipelineRun、TaskRun、日志和 trace，未修改运行面或触发 PipelineRun。 diff --git a/docs/MDTODO/details/observability-trace-reliability/R3_Task_Report.md b/docs/MDTODO/details/observability-trace-reliability/R3_Task_Report.md new file mode 100644 index 00000000..45bda499 --- /dev/null +++ b/docs/MDTODO/details/observability-trace-reliability/R3_Task_Report.md @@ -0,0 +1,11 @@ +# R3 任务报告 + +CI/CD 自动链 OTel 与错误投影已完成，并以真实自动事件闭环： + +- PR #1925/#1937 建立 YAML-first OTel 配置、trace context 和非阻塞 exporter。 +- PR #1939 让 PaC 默认输出直接显示 trace、首断点和 typed child-process/BuildKit 错误。 +- PR #1940/#1943 修复 rootless BuildKit 启动与第三方 OTel 环境泄漏。 +- 最终 merge commit `11d0e238162d1275d4f859bd275e0b8bc23bfa04` 的自动 PipelineRun 成功，镜像、GitOps、Argo、runtime、health 与 Tempo trace 全部对齐。 +- 全过程未增加人工交付路径、第二 authority、锁、租约或证据状态机，也未人工触发 PipelineRun。 + +详细事实见 R3.1-R3.4 报告与 UniDesk #1923。 diff --git a/docs/MDTODO/observability-trace-reliability.md b/docs/MDTODO/observability-trace-reliability.md index 5f868a1e..09715af9 100644 --- a/docs/MDTODO/observability-trace-reliability.md +++ b/docs/MDTODO/observability-trace-reliability.md @@ -23,18 +23,22 @@ 调查第二轮业务 trace `trc_9352107b5e8a4125` / session `ses_d4473d6c-4d7a-4612-a102-254977698bc2` 的文件编辑、`apply_patch` 与 tool event 可见性：先解析对应 OTel trace/run/command，再对照 codex stdio notification、AgentRun durable events、`agentrun.event.v1`、`hwlab.event.v1` 和 HWLAB trace/read model，并使用受控的集群内或应用侧 CLI 执行 Kafka replay 校对，判定事件是源头未产生、AgentRun 映射遗漏、Kafka 转换丢失还是 HWLAB 投影丢失；不得手写 Kafka API、不得用重放追加产品 topic event、不得把投影滞后改写成业务失败，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R2_Task_Report.md)。 -## R3 [in_progress] +## R3 [completed] 解决 [UniDesk #1923](https://github.com/pikasTech/unidesk/issues/1923)：提高 CI/CD 自动链错误暴露能力并接入 OTel，覆盖 GitHub webhook→Gitea snapshot→PaC→Tekton PipelineRun/TaskRun/step→artifact/GitOps→Argo/runtime/health 的 trace context、阶段 span、首断点与脱敏错误投影；让 cicd status、PaC status/history/debug-step 和 sentinel control-plane status 默认有界显示 traceId/error code/phase/exit/下钻。exporter、版本或 commit 漂移只能非阻塞 warning；禁止新增人工交付路径、第二 authority、锁/租约/证据链/状态机，以真实失败 hwlab-web-probe-sentinel-nc01-6gjzl 和新 PR merge 自动事件验收，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R3_Task_Report.md)。 -### R3.1 [in_progress] +### R3.1 [completed] 基于 PR #1925 合并后的真实自动事件继续修复 OTel 运行面：禁止 PaC Repository 参数未就绪时把花括号占位符作为 endpoint；从 owning YAML 在 source snapshot 内解析 OTel 配置并保留 300ms 非阻塞策略，增加安全的原始 fetch cause 投影；恢复 Tempo endpoint 后，用新的普通 PR merge 自动事件证明 collector 接收且 Tempo 可按 traceId 查询，禁止人工触发 PipelineRun，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R3.1_Task_Report.md)。 -### R3.2 [in_progress] +### R3.2 [completed] 基于真实自动事件 `hwlab-web-probe-sentinel-nc01-bbjv2` 修复 BuildKit daemon 已启动但 readiness 重试提前耗尽的竞争，并让 PaC `history/debug-step/status` 从失败 TaskRun 日志默认有界投影 traceId、首断点、阶段、error type/code/exit 与精确下钻；不得以人工 PipelineRun、锁/租约或吞掉失败缓解，使用下一次普通 PR merge 自动事件证明业务成功且 Tempo 可按同一 traceId 查询，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R3.2_Task_Report.md)。 -### R3.3 [in_progress] +### R3.3 [completed] 基于真实自动事件 `hwlab-web-probe-sentinel-nc01-czhg5` 修复 `moby/buildkit:rootless` 启动合同：显式通过 RootlessKit 启动 buildkitd，保留 socket + worker readiness、有界 TERM/KILL/wait 与 typed 失败投影；禁止退回短连接重试、人工 PipelineRun 或第二 authority，使用下一次普通 PR merge 自动事件证明 image build、GitOps/Argo/runtime 成功且 Tempo 可按同一 traceId 查询，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R3.3_Task_Report.md)。 + +### R3.4 [completed] + +独立溯源 Web Probe Sentinel CI 从历史可成功构建到连续 BuildKit readiness 失败的首个回归：对齐成功/失败 PipelineRun、镜像 digest、TaskSpec、节点环境与 Git 提交/PR/作者，区分环境启动变慢、官方 daemonless 默认重试窗口不足和 PR #1939 裸 buildkitd 新回归，输出可复核的时间线与责任变更，不修改运行面或人工触发 PipelineRun，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R3.4_Task_Report.md)。 From fcba3ddb16a8f49730a1f48ff6139ecfbe4b3431 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 18:16:46 +0200 Subject: [PATCH 6/8] =?UTF-8?q?docs:=20=E5=AE=8C=E6=88=90=20AgentRun=20ses?= =?UTF-8?q?sion=20timeout=20=E6=94=B6=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/MDTODO/artificer-runtime-capabilities.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/MDTODO/artificer-runtime-capabilities.md b/docs/MDTODO/artificer-runtime-capabilities.md index 9739ab6c..da590765 100644 --- a/docs/MDTODO/artificer-runtime-capabilities.md +++ b/docs/MDTODO/artificer-runtime-capabilities.md @@ -113,6 +113,6 @@ 自动上线后以新 Artificer task input 与超过 30 分钟的真实或受控 canary 验证 2 小时 budget 生效，不修改当前 Monitor run 的既有 task policy，完成任务后将详细报告写入[任务报告](./details/artificer-runtime-capabilities/R7.3_Task_Report.md)。 -## R8 [in_progress] +## R8 [completed] 解决 [UniDesk #1944](https://github.com/pikasTech/unidesk/issues/1944)：修复 Artificer Aipod 已为 2 小时但 UniDesk session follow-up 仍使用 15 分钟 hard-timeout 的配置分裂；将 `config/agentrun.yaml#client.sessionPolicy.executionPolicy.timeoutMs` 调整为 7200000 ms，在配置解析/校验中拒绝任何低于 3600000 ms 的 hard-timeout，并更新中文长期参考与最小测试；属于 AgentRun 自身运行时，必须由原生子代理执行，禁止 Artificer 递归修复，不新增第二 timeout authority、续跑补丁、锁、租约或状态机，完成任务后将详细报告写入[任务报告](./details/artificer-runtime-capabilities/R8_Task_Report.md)。 From bdab46b3dd020114865bddbb1c6900ae810cc076 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 19:35:14 +0200 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20Sub2API=20?= =?UTF-8?q?=E8=B4=A6=E5=8F=B7=E8=BF=90=E8=A1=8C=E6=80=81=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../unidesk-sub2api/references/codex-pool.md | 9 +- .../platform-infra-sub2api-codex/accounts.ts | 4 + .../platform-infra-sub2api-codex/actions.ts | 33 +++- .../platform-infra-sub2api-codex/options.ts | 3 +- .../platform-infra-sub2api-codex/redaction.ts | 37 +++++ .../remote-python-sync.ts | 150 +++++++++++++++++- .../remote-scripts.ts | 4 + .../platform-infra-sub2api-codex/remote.ts | 3 +- .../src/platform-infra-sub2api-codex/types.ts | 3 +- 9 files changed, 238 insertions(+), 8 deletions(-) diff --git a/.agents/skills/unidesk-sub2api/references/codex-pool.md b/.agents/skills/unidesk-sub2api/references/codex-pool.md index bd414a78..1a4a2666 100644 --- a/.agents/skills/unidesk-sub2api/references/codex-pool.md +++ b/.agents/skills/unidesk-sub2api/references/codex-pool.md @@ -1,6 +1,12 @@ ## Codex Pool -`codex-pool plan|sync|validate` 同时覆盖 k3s target 和 PK01 host-Docker target。PK01 host-Docker 的 `sync --confirm` 通过 Sub2API admin API 对齐 group、YAML-managed accounts、统一消费 API key、capacity/loadFactor、WebSocket v2 标记和内置 `temp_unschedulable` 规则；统一消费 key 写入 YAML 声明的 `targets[PK01].hostDocker.envPath`，不创建 k8s Secret，不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启。`sentinel-report`、`sentinel-probe`、`sentinel-image` 和部分 `trace` 能力仍以 k3s target 为主；需要这些能力时显式选择对应 k3s target。 +- `codex-pool plan|sync|validate|runtime-state` 同时覆盖 k3s target 和 PK01 host-Docker target： + - PK01 host-Docker 的 `sync --confirm` 通过 Sub2API admin API 对齐 group、YAML-managed accounts、统一消费 API key、capacity/loadFactor、WebSocket v2 标记和内置 `temp_unschedulable` 规则； + - 统一消费 key 写入 YAML 声明的 `targets[PK01].hostDocker.envPath`； + - PK01 路径不创建 k8s Secret、不部署 sentinel 资源，也不触发 `sub2api apply`、Docker compose、Caddy reload 或容器重启； + - `runtime-state` 只读列出 pool group 内全部账号，包括手工账号，并披露临时不可调度规则、当前冻结状态、`pool_mode`、代理绑定和相对 YAML 基线的差异； + - `runtime-state` 不输出 API key 或 credentials 正文，默认返回紧凑 JSON，使用 `--full` 或 `--raw` 下钻； + - `sentinel-report`、`sentinel-probe`、`sentinel-image` 和部分 `trace` 能力仍以 k3s target 为主，需要这些能力时显式选择对应 k3s target。 k3s 账号池操作示例： @@ -8,6 +14,7 @@ k3s 账号池操作示例： bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sync --target D601 --confirm bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 +bun scripts/cli.ts platform-infra sub2api codex-pool runtime-state --target PK01 bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --target D601 --confirm diff --git a/scripts/src/platform-infra-sub2api-codex/accounts.ts b/scripts/src/platform-infra-sub2api-codex/accounts.ts index 409e5267..029b844d 100644 --- a/scripts/src/platform-infra-sub2api-codex/accounts.ts +++ b/scripts/src/platform-infra-sub2api-codex/accounts.ts @@ -79,3 +79,7 @@ export function desiredAccountTempUnschedulableMap(pool: CodexPoolConfig): Recor } return policies; } + +export function desiredDefaultAccountTempUnschedulable(pool: CodexPoolConfig): Record { + return renderSub2ApiTempUnschedulableCredentials(pool.defaultTempUnschedulable); +} diff --git a/scripts/src/platform-infra-sub2api-codex/actions.ts b/scripts/src/platform-infra-sub2api-codex/actions.ts index d97e781d..9afab1bd 100644 --- a/scripts/src/platform-infra-sub2api-codex/actions.ts +++ b/scripts/src/platform-infra-sub2api-codex/actions.ts @@ -27,9 +27,9 @@ import { collectCodexProfiles, readCodexPoolConfig } from "./config"; import { codexPoolSentinelProbeConfigFingerprint, fingerprint } from "./config-utils"; import { apiKeyPreview, codexConsumerBaseUrl, fetchPoolApiKey, probePublicModels, validatePublicGatewayWithKey, writeLocalCodexConfig } from "./local-codex"; import { manualBindingSourcePlan, poolTarget, prepareTargetPublicExposureSecret, protectedManualAccountNamesForTarget, resolvedManualAccountProtections, secretMaterialSummary, sentinelProfileSecrets, targetFrpPublicExposure, targetPublicExposureApplyScript, targetPublicExposureSummary } from "./public-exposure"; -import { codexPoolConfigSummary, compactProfile, compactSentinelProbeResult, redactProfile, renderSub2ApiTempUnschedulableCredentials } from "./redaction"; +import { codexPoolConfigSummary, compactProfile, compactRuntimeAccountState, compactSentinelProbeResult, redactProfile, renderSub2ApiTempUnschedulableCredentials } from "./redaction"; import { boolField, capture, compactCapture, parseJsonOutput, runRemoteCodexPoolScript } from "./remote"; -import { cleanupProbesScript, sentinelImageBuildScript, sentinelImageStatusScript, sentinelProbeScript, sentinelReportScript, syncScript, traceScript, validateScript } from "./remote-scripts"; +import { cleanupProbesScript, runtimeStateScript, sentinelImageBuildScript, sentinelImageStatusScript, sentinelProbeScript, sentinelReportScript, syncScript, traceScript, validateScript } from "./remote-scripts"; import { codexPoolSyncSummary, codexPoolValidationSummary, renderCodexPoolPlan, renderCodexPoolSentinelProbeResult, renderCodexPoolSyncResult, renderCodexPoolValidateResult, renderSentinelReport, renderTraceReport, renderedCliResult } from "./render"; import { codexPoolRuntimeTarget, defaultCodexPoolRuntimeTargetId, targetFlag } from "./runtime-target"; import { codexPoolConfigPath, serviceName, sub2apiConfigPath } from "./types"; @@ -317,6 +317,35 @@ export async function codexPoolValidate(config: UniDeskConfig, options: Disclosu return options.full ? response : renderCodexPoolValidateResult(response); } +export async function codexPoolRuntimeState(config: UniDeskConfig, options: DisclosureOptions): Promise> { + const pool = readCodexPoolConfig(); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); + const result = await runRemoteCodexPoolScript(config, "runtime-state", runtimeStateScript(pool, runtimeTarget), runtimeTarget); + const parsed = parseJsonOutput(result.stdout); + if (options.raw) { + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + action: "platform-infra-sub2api-codex-pool-runtime-state", + remote: compactCapture(result, { full: true }), + parsed, + valuesPrinted: false, + }; + } + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + action: "platform-infra-sub2api-codex-pool-runtime-state", + target: { + id: runtimeTarget.id, + route: runtimeTarget.route, + namespace: runtimeTarget.namespace, + runtimeMode: runtimeTarget.runtimeMode, + }, + state: options.full ? parsed : compactRuntimeAccountState(parsed), + remote: compactCapture(result, { full: result.exitCode !== 0 || parsed === null }), + valuesPrinted: false, + }; +} + export async function codexPoolTrace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(options.targetId); diff --git a/scripts/src/platform-infra-sub2api-codex/options.ts b/scripts/src/platform-infra-sub2api-codex/options.ts index d2120e16..5c445d50 100644 --- a/scripts/src/platform-infra-sub2api-codex/options.ts +++ b/scripts/src/platform-infra-sub2api-codex/options.ts @@ -22,7 +22,7 @@ import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from ".. import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; import type { ConfirmOptions, DisclosureOptions, SentinelImageOptions, SentinelProbeOptions, SentinelReportOptions, SyncOptions, TraceOptions } from "./types"; -import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions"; +import { codexPoolCleanupProbes, codexPoolConfigureLocal, codexPoolExpose, codexPoolPlan, codexPoolRuntimeState, codexPoolSentinelImage, codexPoolSentinelProbe, codexPoolSentinelReport, codexPoolSync, codexPoolTrace, codexPoolValidate } from "./actions"; import { renderCodexPoolPlan } from "./render"; import { defaultCodexPoolRuntimeTargetId } from "./runtime-target"; import { codexPoolHelp } from "./types"; @@ -36,6 +36,7 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]) } if (action === "sync") return await codexPoolSync(config, parseSyncOptions(args.slice(1))); if (action === "validate") return await codexPoolValidate(config, parseDisclosureOptions(args.slice(1))); + if (action === "runtime-state") return await codexPoolRuntimeState(config, parseDisclosureOptions(args.slice(1))); if (action === "trace") return await codexPoolTrace(config, parseTraceOptions(args.slice(1))); if (action === "sentinel-image") return await codexPoolSentinelImage(config, parseSentinelImageOptions(args.slice(1))); if (action === "sentinel-probe") return await codexPoolSentinelProbe(config, parseSentinelProbeOptions(args.slice(1))); diff --git a/scripts/src/platform-infra-sub2api-codex/redaction.ts b/scripts/src/platform-infra-sub2api-codex/redaction.ts index e2f22c63..34dcc5f9 100644 --- a/scripts/src/platform-infra-sub2api-codex/redaction.ts +++ b/scripts/src/platform-infra-sub2api-codex/redaction.ts @@ -188,6 +188,43 @@ export function pickSummaryFields(item: Record, keys: string[]) return result; } +export function compactRuntimeAccountState(parsed: Record | null): Record | null { + if (parsed === null) return null; + const state = isRecord(parsed.state) ? parsed.state : {}; + const accounts = recordArray(state.accounts).map((item) => { + const temp = isRecord(item.tempUnschedulable) ? item.tempUnschedulable : {}; + const poolMode = isRecord(item.poolMode) ? item.poolMode : {}; + const optimization = isRecord(item.optimization) ? item.optimization : {}; + return { + ...pickSummaryFields(item, ["accountName", "accountId", "type", "status", "schedulable", "proxyId"]), + tempEnabled: temp.enabled, + tempStatusCodes: temp.statusCodes, + tempActive: temp.active, + poolModeEnabled: poolMode.enabled, + tempUnschedulableNeedsAlignment: optimization.tempUnschedulableNeedsAlignment, + credentialsPrinted: false, + }; + }); + const runtimeImage = isRecord(parsed.runtimeImage) ? parsed.runtimeImage : {}; + return { + ok: parsed.ok, + mode: parsed.mode, + namespace: parsed.namespace, + serviceDns: parsed.serviceDns, + appPod: parsed.appPod, + runtimeImage: pickSummaryFields(runtimeImage, ["container", "image", "ready", "restartCount", "startedAt", "health"]), + group: state.group, + yamlBaseline: state.yamlBaseline, + summary: state.summary, + accounts, + disclosure: { + full: "bun scripts/cli.ts platform-infra sub2api codex-pool runtime-state --full", + raw: "bun scripts/cli.ts platform-infra sub2api codex-pool runtime-state --raw", + }, + valuesPrinted: false, + }; +} + export function compactStatusBlock(block: unknown, keys: string[]): Record | null { if (!isRecord(block)) return null; const items = recordArray(block.items); diff --git a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts index 2ee7a07a..48abe73c 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-python-sync.ts @@ -22,7 +22,7 @@ import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from ".. import { runSshCommandCapture, type SshCaptureResult } from "../ssh"; import type { CodexPoolConfig, CodexPoolRuntimeTarget } from "./types"; -import { desiredAccountCapacityMap, desiredAccountLoadFactorMap, desiredAccountTempUnschedulableMap, desiredAccountWebSocketsV2ModeMap } from "./accounts"; +import { desiredAccountCapacityMap, desiredAccountLoadFactorMap, desiredAccountTempUnschedulableMap, desiredAccountWebSocketsV2ModeMap, desiredDefaultAccountTempUnschedulable } from "./accounts"; import { resolvedManualAccountProtections } from "./public-exposure"; import { fieldManager } from "./types"; @@ -30,7 +30,7 @@ function pyJson(value: unknown): string { return `json.loads(${JSON.stringify(JSON.stringify(value))})`; } -export function remotePythonScript(mode: "sync" | "validate" | "trace" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { +export function remotePythonScript(mode: "sync" | "validate" | "runtime-state" | "trace" | "cleanup-probes" | "sentinel-probe", encodedPayload: string, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { const hostDockerEnvPath = target.runtimeMode === "host-docker" ? target.hostDockerEnvPath : null; return ` set -u @@ -75,6 +75,7 @@ EXPECTED_ACCOUNT_CAPACITIES = ${pyJson(desiredAccountCapacityMap(pool))} EXPECTED_ACCOUNT_LOAD_FACTORS = ${pyJson(desiredAccountLoadFactorMap(pool))} EXPECTED_ACCOUNT_WS_MODES = json.loads(${JSON.stringify(JSON.stringify(desiredAccountWebSocketsV2ModeMap(pool)))}) EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredAccountTempUnschedulableMap(pool)))}) +EXPECTED_DEFAULT_TEMP_UNSCHEDULABLE = json.loads(${JSON.stringify(JSON.stringify(desiredDefaultAccountTempUnschedulable(pool)))}) MANUAL_ACCOUNT_PROTECTIONS = json.loads(${JSON.stringify(JSON.stringify(resolvedManualAccountProtections(pool, target)))}) SENTINEL_CONFIG = json.loads(${JSON.stringify(JSON.stringify(pool.sentinel))}) TARGET_EGRESS_PROXY = json.loads(${JSON.stringify(JSON.stringify(target.egressProxy))}) @@ -2436,6 +2437,134 @@ def account_temp_unschedulable_status(token): "valuesPrinted": False, } +def normalized_int(value): + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, str): + try: + return int(value.strip()) + except ValueError: + return None + return None + +def normalized_status_codes(value): + if not isinstance(value, list): + return None + codes = [] + for item in value: + code = normalized_int(item) + if code is not None and 100 <= code <= 599 and code not in codes: + codes.append(code) + return sorted(codes) + +def temp_unschedulable_active(until): + if not isinstance(until, str) or not until.strip(): + return False + try: + parsed = datetime.fromisoformat(until.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed > datetime.now(timezone.utc) + except ValueError: + return False + +def account_runtime_state(token): + group = next((item for item in list_groups(token) if item.get("name") == POOL_GROUP_NAME), None) + if not isinstance(group, dict) or group.get("id") is None: + return { + "ok": False, + "group": {"name": POOL_GROUP_NAME, "id": None, "found": False}, + "accounts": [], + "error": "pool-group-not-found", + "valuesPrinted": False, + } + baseline = normalize_temp_unschedulable_credentials(EXPECTED_DEFAULT_TEMP_UNSCHEDULABLE) + baseline_codes = sorted(set(rule.get("error_code") for rule in baseline["rules"] if isinstance(rule.get("error_code"), int))) + protected_names = set(item.get("accountName") for item in MANUAL_ACCOUNT_PROTECTIONS if isinstance(item, dict) and isinstance(item.get("accountName"), str)) + accounts = list_accounts_for_group(token, group["id"]) + items = [] + for account in sorted(accounts, key=lambda item: str(item.get("name") or "")): + detail = get_account_detail(token, account) + credentials = runtime_account_credentials(detail) + runtime_policy = normalize_temp_unschedulable_credentials(credentials) + runtime_codes = sorted(set(rule.get("error_code") for rule in runtime_policy["rules"] if isinstance(rule.get("error_code"), int))) + name = detail.get("name") or account.get("name") + pool_mode = bool_value(credentials.get("pool_mode")) + temp_until = detail.get("temp_unschedulable_until") or detail.get("tempUnschedulableUntil") + temp_reason = detail.get("temp_unschedulable_reason") or detail.get("tempUnschedulableReason") or "" + origin = "yaml-managed" if name in EXPECTED_ACCOUNT_TEMP_UNSCHEDULABLE else ("yaml-protected-manual" if name in protected_names else "runtime-manual") + supports_policy = (detail.get("type") or account.get("type")) == "apikey" + policy_matches = runtime_policy == baseline if supports_policy else None + missing_codes = [code for code in baseline_codes if code not in runtime_codes] if supports_policy else [] + items.append({ + "accountName": name, + "accountId": detail.get("id") or account.get("id"), + "origin": origin, + "platform": detail.get("platform") or account.get("platform"), + "type": detail.get("type") or account.get("type"), + "status": detail.get("status") or account.get("status"), + "schedulable": detail.get("schedulable") if detail.get("schedulable") is not None else account.get("schedulable"), + "proxyId": detail.get("proxy_id") if detail.get("proxy_id") is not None else account.get("proxy_id"), + "priority": detail.get("priority") if detail.get("priority") is not None else account.get("priority"), + "concurrency": detail.get("concurrency") if detail.get("concurrency") is not None else account.get("concurrency"), + "currentConcurrency": account.get("current_concurrency"), + "baseUrl": normalize_runtime_base_url(credentials.get("base_url")), + "tempUnschedulable": { + "enabled": runtime_policy["enabled"], + "ruleCount": len(runtime_policy["rules"]), + "statusCodes": runtime_codes, + "rules": summarize_temp_unschedulable_rules(runtime_policy["rules"]), + "until": temp_until, + "reasonPreview": text(str(temp_reason), 500) if temp_reason else "", + "stateRecorded": temp_until is not None or bool(temp_reason), + "active": temp_unschedulable_active(temp_until), + "matchesYamlBaseline": policy_matches, + "missingYamlBaselineStatusCodes": missing_codes, + }, + "poolMode": { + "enabled": pool_mode, + "retryCountConfigured": normalized_int(credentials.get("pool_mode_retry_count")) if "pool_mode_retry_count" in credentials else None, + "retryStatusCodesConfigured": normalized_status_codes(credentials.get("pool_mode_retry_status_codes")) if "pool_mode_retry_status_codes" in credentials else None, + }, + "optimization": { + "tempUnschedulableNeedsAlignment": supports_policy and runtime_policy != baseline, + "poolModeShouldBeDisabled": pool_mode, + }, + "credentialsPrinted": False, + }) + temp_alignment = [item["accountName"] for item in items if item["optimization"]["tempUnschedulableNeedsAlignment"]] + pool_mode_enabled = [item["accountName"] for item in items if item["poolMode"]["enabled"]] + return { + "ok": True, + "group": { + "name": POOL_GROUP_NAME, + "id": group.get("id"), + "found": True, + "platform": group.get("platform"), + "status": group.get("status"), + }, + "yamlBaseline": { + "enabled": baseline["enabled"], + "ruleCount": len(baseline["rules"]), + "statusCodes": baseline_codes, + }, + "summary": { + "accountCount": len(items), + "tempUnschedulableEnabledCount": sum(1 for item in items if item["tempUnschedulable"]["enabled"]), + "tempUnschedulableNeedsAlignmentCount": len(temp_alignment), + "tempUnschedulableNeedsAlignment": temp_alignment, + "poolModeEnabledCount": len(pool_mode_enabled), + "poolModeEnabled": pool_mode_enabled, + "optimizationAvailable": bool(temp_alignment or pool_mode_enabled), + }, + "accounts": items, + "valuesPrinted": False, + } + def account_capacity_status(token): accounts = list_accounts(token) by_name = {item.get("name"): item for item in accounts if isinstance(item.get("name"), str)} @@ -2776,6 +2905,21 @@ def run_validate(): "validation": {"gatewayModels": gateway, "gatewayResponses": responses_smoke, "gatewayResponsesRecent": responses_evidence, "gatewayCompactRecent": compact_evidence}, } +def run_runtime_state(): + admin_email, token, admin_compliance = login() + state = account_runtime_state(token) + return { + "ok": state.get("ok") is True, + "mode": "runtime-state", + "namespace": NAMESPACE, + "serviceDns": SERVICE_DNS, + "appPod": APP_POD, + "admin": {"email": admin_email, "tokenPrinted": False, "compliance": admin_compliance}, + "runtimeImage": app_pod_runtime_image(), + "state": state, + "valuesPrinted": False, + } + def parse_log_line(line): json_start = line.find("{") if json_start < 0: @@ -3286,6 +3430,8 @@ def run_cleanup_probes(): try: if MODE == "sync": result = run_sync() + elif MODE == "runtime-state": + result = run_runtime_state() elif MODE == "trace": result = run_trace() elif MODE == "cleanup-probes": diff --git a/scripts/src/platform-infra-sub2api-codex/remote-scripts.ts b/scripts/src/platform-infra-sub2api-codex/remote-scripts.ts index 68e3630c..01a24bf7 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote-scripts.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote-scripts.ts @@ -35,6 +35,10 @@ export function validateScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTa return remotePythonScript("validate", "", pool, target); } +export function runtimeStateScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { + return remotePythonScript("runtime-state", "", pool, target); +} + export function traceScript(pool: CodexPoolConfig, options: TraceOptions, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify({ requestId: options.requestId, diff --git a/scripts/src/platform-infra-sub2api-codex/remote.ts b/scripts/src/platform-infra-sub2api-codex/remote.ts index cf89d506..c3d2f251 100644 --- a/scripts/src/platform-infra-sub2api-codex/remote.ts +++ b/scripts/src/platform-infra-sub2api-codex/remote.ts @@ -28,7 +28,7 @@ export async function capture(config: UniDeskConfig, target: string, args: strin return await runSshCommandCapture(config, target, args, input); } -export type RemoteCodexPoolMode = "sync" | "validate" | "sentinel-probe" | "sentinel-image-status" | "sentinel-image-build"; +export type RemoteCodexPoolMode = "sync" | "validate" | "runtime-state" | "sentinel-probe" | "sentinel-image-status" | "sentinel-image-build"; export async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: RemoteCodexPoolMode, script: string, target = codexPoolRuntimeTarget()): Promise { const jobName = `codex-pool-${mode}-${Date.now().toString(36)}`.slice(0, 63); @@ -83,6 +83,7 @@ export async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: Remo export function codexPoolModeCommand(mode: RemoteCodexPoolMode): string { if (mode === "sync") return "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm"; + if (mode === "runtime-state") return "bun scripts/cli.ts platform-infra sub2api codex-pool runtime-state"; if (mode === "sentinel-probe") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account --confirm"; if (mode === "sentinel-image-status") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status"; if (mode === "sentinel-image-build") return "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm"; diff --git a/scripts/src/platform-infra-sub2api-codex/types.ts b/scripts/src/platform-infra-sub2api-codex/types.ts index 9f401e03..c1b3aaa3 100644 --- a/scripts/src/platform-infra-sub2api-codex/types.ts +++ b/scripts/src/platform-infra-sub2api-codex/types.ts @@ -325,13 +325,14 @@ export function codexPoolHelp(): unknown { const pool = readCodexPoolConfig(); const runtimeTarget = codexPoolRuntimeTarget(); return { - command: "platform-infra sub2api codex-pool plan|sync|validate|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", + command: "platform-infra sub2api codex-pool plan|sync|validate|runtime-state|trace|sentinel-image|sentinel-probe|sentinel-report|cleanup-probes|expose|configure-local", output: "json, except trace and sentinel-report default to low-noise text tables", usage: [ "bun scripts/cli.ts platform-infra sub2api codex-pool plan", "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", "bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]", "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool runtime-state [--target PK01] [--full|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm", From 6074a9c2c08f16c497a6a8e9cfd113ce6a5ceb8b Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 13 Jul 2026 19:52:38 +0200 Subject: [PATCH 8/8] =?UTF-8?q?docs:=20=E8=B7=9F=E8=B8=AA=20Kafka=20SSE=20?= =?UTF-8?q?=E7=BB=88=E6=80=81=E5=BB=B6=E8=BF=9F=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../MDTODO/observability-trace-reliability.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/MDTODO/observability-trace-reliability.md b/docs/MDTODO/observability-trace-reliability.md index 09715af9..b1cb50be 100644 --- a/docs/MDTODO/observability-trace-reliability.md +++ b/docs/MDTODO/observability-trace-reliability.md @@ -42,3 +42,23 @@ ### R3.4 [completed] 独立溯源 Web Probe Sentinel CI 从历史可成功构建到连续 BuildKit readiness 失败的首个回归：对齐成功/失败 PipelineRun、镜像 digest、TaskSpec、节点环境与 Git 提交/PR/作者，区分环境启动变慢、官方 daemonless 默认重试窗口不足和 PR #1939 裸 buildkitd 新回归，输出可复核的时间线与责任变更，不修改运行面或人工触发 PipelineRun，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R3.4_Task_Report.md)。 + +## R4 [in_progress] + +解决 [AgentRun #352](https://github.com/pikasTech/agentrun/issues/352)、[HWLAB #2525](https://github.com/pikasTech/HWLAB/issues/2525) 与 [UniDesk #1949](https://github.com/pikasTech/unidesk/issues/1949)：消除 trace trc_f389d0493c0c489e 的 Kafka outbox 队头阻塞与 17 分钟伪耗时，恢复 agentrun.event.v1→transactional projector→hwlab.event.v1→实时/回放 SSE 单一权威；禁止 HTTP 补链、页面轮询、读侧修复、进程内 finalizer、双 authority、额外锁/租约/状态库或降低 Kafka/SSE 能力，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R4_Task_Report.md)。 + +### R4.1 [in_progress] + +执行 [AgentRun #352](https://github.com/pikasTech/agentrun/issues/352)：按 key 保序、跨 key 有界并行排空 transactional outbox，保留 durable at-least-once 并补 backlog/oldest/rate/typed error 可见性，以 NC01/nc01-v02 真实积压归零和新事件实时发布验收，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R4.1_Task_Report.md)。 + +### R4.2 [in_progress] + +执行 [HWLAB #2525](https://github.com/pikasTech/HWLAB/issues/2525)：恢复固定 group transactional projector、原子 facts/checkpoint/browser outbox、hwlab.event.v1 relay 与同 cursor 实时/回放 SSE；耗时取 canonical 时间，禁止 direct/live 双写、HTTP 补链和页面轮询，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R4.2_Task_Report.md)。 + +### R4.3 [in_progress] + +执行 [UniDesk #1949](https://github.com/pikasTech/unidesk/issues/1949)：由 config/hwlab-node-lanes.yaml 选择 transactionalProjector、projectionOutboxRelay、projectionRealtime 单一产品 authority，关闭 live-only direct authority，保持 renderer/status/preflight YAML-first，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R4.3_Task_Report.md)。 + +### R4.4 + +依赖 R4.1-R4.3，主代理完成 PR 架构审核、顺序合并、自动 CI/CD 和 NC01/v03 原 Workbench 新 hi turn 验收；核对 command/run 实际耗时、Kafka backlog/lag、运行中增量、terminal/final、刷新 replay、公开 result 与 OTel 一致后收口，完成任务后将详细报告写入[任务报告](./details/observability-trace-reliability/R4.4_Task_Report.md)。