From 04cd3e5f12c1e1479a5ccf0bf72bde201de16003 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 11 Jul 2026 04:16:58 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20AgentRun=20provider?= =?UTF-8?q?=20=E5=81=87=E6=B4=BB=E6=80=A7=E4=B8=8E=E5=AE=B9=E9=87=8F?= =?UTF-8?q?=E5=8F=AF=E8=A7=81=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../unidesk-cicd/references/agentrun.md | 68 +- config/agentrun.yaml | 103 +++ config/hwlab-node-lanes.yaml | 3 + docs/reference/agentrun.md | 67 +- scripts/src/agentrun-lanes.ts | 70 +- scripts/src/agentrun-manifests.ts | 2 + scripts/src/agentrun/control-plane.ts | 3 +- scripts/src/agentrun/options.ts | 1 - scripts/src/agentrun/provider-activation.ts | 181 +++++ scripts/src/agentrun/provider-profile.ts | 593 ++++++++++++++-- .../provider-secret-activation-preflight.mjs | 367 ++++++++++ scripts/src/agentrun/public-exposure.ts | 41 +- scripts/src/agentrun/render.ts | 109 ++- scripts/src/agentrun/resource-actions.ts | 119 +++- scripts/src/agentrun/runner-observation.ts | 311 +++++++++ scripts/src/agentrun/secrets.ts | 645 +++++++++++++++--- scripts/src/agentrun/trigger.ts | 160 ++++- scripts/src/agentrun/yaml-lane.ts | 188 ++++- scripts/src/hwlab-node-lanes.ts | 9 + .../src/hwlab-node-web-sentinel-p5-observe.ts | 142 ++++ scripts/src/hwlab-node-web-sentinel-p5.ts | 95 ++- 21 files changed, 3063 insertions(+), 214 deletions(-) create mode 100644 scripts/src/agentrun/provider-activation.ts create mode 100644 scripts/src/agentrun/provider-secret-activation-preflight.mjs create mode 100644 scripts/src/agentrun/runner-observation.ts diff --git a/.agents/skills/unidesk-cicd/references/agentrun.md b/.agents/skills/unidesk-cicd/references/agentrun.md index b603b699..694a332d 100644 --- a/.agents/skills/unidesk-cicd/references/agentrun.md +++ b/.agents/skills/unidesk-cicd/references/agentrun.md @@ -1,8 +1,8 @@ -# AgentRun Control Plane +# AgentRun 控制面 AgentRun YAML-only lane 以 `config/agentrun.yaml` 为部署真相；node/lane、source workspace/branch、image build、GitOps branch/path、runtime namespace、Secret、外置数据库、manager env、git-mirror 和 edge 暴露都从 YAML 进入 CLI。 -## YAML-Only Lane +## YAML-only lane ```bash bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02 @@ -10,7 +10,8 @@ bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 [--dry-ru bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 [--dry-run|--confirm] bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 [--dry-run|--confirm] bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 [--dry-run|--confirm] -bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 [--force-active] [--dry-run|--confirm] +bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 [--dry-run|--confirm] +bun scripts/cli.ts agentrun describe runnerjob/ --run --node D601 --lane v02 -o json bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--pipeline-run |--source-commit ] [--full|--raw] bun scripts/cli.ts platform-infra gitea mirror status --target JD01 [--full|--raw] bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 [--full|--raw] @@ -21,16 +22,67 @@ bun scripts/cli.ts platform-infra pipelines-as-code status --target JD01 [--full - `secret-sync`: 按 YAML 的 Secret sourceRef/keyMapping 同步 runtime Secret 和外置 DB Secret，只输出 fingerprint。 - `restart`: patch manager Deployment restart annotation 并等待 rollout，用于 Secret export/DB 连接串变化后让 workload 读取新 Secret；不要手工删除 Pod。 - `trigger-current`: legacy v0.2 lane source authority 只读 k8s git-mirror snapshot。confirmed 运行先触发受控 `git-mirror sync`，为 source branch tip 创建 `refs/unidesk/snapshots/agentrun-yaml-lane//`，再从该 snapshot 构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，flush git-mirror 并创建 provenance PipelineRun。迁移到 PaC/Gitea 的 lane 不再通过该自维护触发器提交 CI。 -- `cleanup-runners`: 只清 YAML 选中 lane runtime namespace 中匹配 `deployment.runner.retention.selectors` 的 runner Job/Pod；runner 上限、最后活跃排序、active heartbeat 窗口、age-based cleanup 开关和 selector 都以 YAML 为准。 +- `cleanup-runners`: + - 只读分类 YAML 选中 lane 中匹配 `deployment.runner.retention.selectors` 的 runner Job/Pod； + - 默认有界显示 observation、capacity、最近活跃、heartbeat、业务进展、Pod waiting reason 和 `stale-pending`； + - capacity 摘要必须显示 `/`、`status`、`availableSlots`、typed blocker 和 `degradedReason`； + - `runnerJobCount === maxRunners` 时必须返回 at-capacity degraded，不能在下一 runner 已依赖 manager retention 时继续报告健康； + - `--confirm` 在 UniDesk 侧固定 fail-closed，不按旧 Job 名称删除； + - 真正删除由 AgentRun manager 的 DB fence 和 Kubernetes CAS 原子入口负责： + - 分类与 pre-create retention 跟踪 [AgentRun #280](https://github.com/pikasTech/agentrun/issues/280)； + - 原子实现跟踪 [PR #281](https://github.com/pikasTech/agentrun/pull/281)； + - 无 incoming create 时的批量陈旧对象终态化和回收由同一 manager authority 的 [AgentRun #282](https://github.com/pikasTech/agentrun/issues/282) 跟踪。 +- `describe runnerjob/`: + - 先通过 manager 精确读取 runner job identity； + - 再按精确 Job name、`job-name` Pod label 和 Pod name Warning Event 查询运行面； + - 不复用全 namespace retention scan，也不计算 UniDesk 删除结论。 - `status`: 默认返回 compact commander JSON，关键结论在 `.data.summary` 和 `.data.alignment`；完整 YAML target、原始 source/runtime/gitMirror payload 和 probe tail 只在 `--full|--raw` 展开。 -只改 UniDesk YAML 的 manager env、Kafka 开关、provider config 或 runner egress 时，优先复用当前 image 发布 GitOps：`bun scripts/cli.ts agentrun provider-profile apply --node --lane --profile --confirm`。该路径同步选中 profile Secret/config、渲染当前 YAML runtime、刷新 Argo 并重启 manager，不创建 PipelineRun；完成后仍要用窄查询检查 Deployment env 和新建 runner Job env。 +Provider profile 的 YAML-first 入口是： + +```bash +bun scripts/cli.ts agentrun provider-profile plan --node --lane --profile +bun scripts/cli.ts agentrun provider-profile runtime-status --node --lane --profile +bun scripts/cli.ts agentrun provider-profile status --node --lane --profile +bun scripts/cli.ts agentrun provider-profile apply --node --lane --profile --confirm +bun scripts/cli.ts job status --tail-bytes 12000 +bun scripts/cli.ts agentrun provider-profile validate --node --lane --profile --wait +``` + +- `plan`: + - 只读检查 YAML 声明及 owner source 是否具备装配条件； + - 不打印 Secret value。 +- `runtime-status`: + - 只检查目标 runtime Secret/key 和 YAML endpoint 对应的 Kubernetes Service/port 是否存在； + - 不得读取 owner source、计算或比较 fingerprint； + - 不得解码 runtime Secret value。 +- `status`: + - 读取 owner source； + - 用编码 fingerprint 对齐 YAML 期望与 runtime； + - 不得解码 runtime Secret value。 +- `apply`: + - lane parser 必须拒绝重复 Secret `id` 或重复完整 `targetRef`； + - 执行期必须按 `namespace/name/key` 完整 `targetRef` 分类 provider Secret，不能按 `id` 绕过 activation gate； + - Pod、Job 和 manager DB 观察只用于诊断，不能作为 Provider Secret 写授权； + - Secret create、key add 或 key change 在异步 job 创建前和 worker sync 前均 fail-closed； + - Kubernetes 编码态 fingerprint 完全一致时跳过 provider Secret patch； + - 先由 AgentRun manager 完成 admission fence、DB fence、既有 runner 收敛和 Secret resourceVersion CAS，再解除非 no-op 阻断； + - retention #280/#281/#282 不等于 Provider Secret 激活 fence，后者由 [AgentRun #284](https://github.com/pikasTech/agentrun/issues/284) 跟踪； + - 只改 manager env、Kafka 开关、provider config 或 runner egress 时，复用当前 image； + - no-op gate 通过后跳过 profile Secret 写入，发布 GitOps、刷新 Argo 并重启 manager； + - 不创建 PipelineRun； + - `--confirm` 默认立即返回异步 `jobId`； + - 后续只通过返回的 `job status` 命令查看有界进展。 +- `validate`: + - 只读验证 YAML 与 live profile 装配； + - 给出 HWLAB web-probe 原入口验收指引； + - 不替代真实 Workbench 用户路径验证。 YAML-only lane 的长步骤必须由 CLI 拆成短提交和状态轮询：k8s git-mirror snapshot sync、image build、GitOps publish、git-mirror flush 和 PipelineRun 创建不得塞进一个顶层 `trans` 长连接。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace。 AgentRun YAML-only lane closeout 必须同时看当前 k8s git-mirror source snapshot、目标 PipelineRun、GitOps revision、Argo revision 和 manager source commit。发布过程中如果 source branch 被并行 PR 推进，`status --pipeline-run ` 会通过 `summary.branchDrift` / `alignment.branchDrift` 标记目标 PipelineRun 是否已被当前 snapshot tip supersede。最终只用最新 PipelineRun 的 `status` 中 `aligned=true`、`blockers=[]`、`argoSyncedToGitops=true` 和 `managerSourceMatchesExpected=true` 收口。 -## JD01 v0.2 Gitea / Pipelines-as-Code Lane +## JD01 v0.2 Gitea / Pipelines-as-Code lane JD01 `agentrun-jd01-v02` 的 CI 触发路径已经从 branch-follower、act_runner 和自维护 trigger-current 迁移为单一路径：Gitea webhook -> Pipelines-as-Code -> Tekton -> GitOps/Argo -> k8s runtime。不要保留 Gitea Actions、act_runner、legacy branch-follower 或第二套 trigger fallback。Gitea mirror 的 source authority 来自 `config/platform-infra/gitea.yaml`；PaC controller、Repository、webhook 和 Tekton 参数来自 `config/platform-infra/pipelines-as-code.yaml`。 @@ -52,7 +104,7 @@ Kafka event stream 验收也必须创建真实 runtime facts：manager event 用 Provider credential 的 `config.toml` 变更同样走 YAML `sourceRef`、`secret-sync` 和 `restart`；lane config 只声明该 lane 需要的 Codex CLI runtime options。模型、provider 和 endpoint 需要由 lane YAML 拥有时，使用 `sourceMode: codex-config` 加 `codexConfig` 渲染 Secret 中的 `config.toml`；不要修改或复制指挥机全局 `~/.codex/config.toml` 作为长期事实。 -## AgentRun v0.1 Compatibility +## AgentRun v0.1 兼容入口 ```bash bun scripts/cli.ts agentrun control-plane status [--dry-run|--confirm] @@ -70,7 +122,7 @@ bun scripts/cli.ts agentrun control-plane cleanup-released-pvs [--limit N] [--dr AgentRun compact JSON 关键字段在 `.data.summary.sourceCommit`、`.data.summary.expectedPipelineRun`、`.data.summary.runtimeAlignment`、`.data.summary.gitMirror`、`.data.summary.ci.pipelineRun`、`.data.summary.argo` 和 `.data.alignment`，不要假设存在 `.data.status`。 -## AgentRun Git Mirror +## AgentRun Git mirror ```bash bun scripts/cli.ts agentrun git-mirror status [--full|--raw] diff --git a/config/agentrun.yaml b/config/agentrun.yaml index 02f3ab2b..7ead6dec 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -178,6 +178,7 @@ controlPlane: maxRunners: 20 cleanupOrder: oldest-inactive-last-active-first activeHeartbeatMaxAgeMs: 900000 + stalePendingMaxAgeMs: 900000 selectors: matchLabels: app.kubernetes.io/part-of: agentrun @@ -331,6 +332,108 @@ controlPlane: key: config.toml providerCredential: profile: gpt.pika + - id: provider-dsflash-go-auth-json + sourceMode: env + sourceRef: hwlab/nc01-v03-code-agent-provider.env + sourceKey: OPENCODE_API_KEY + transform: codex-auth-json-openai-api-key + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-provider-dsflash-go + key: auth.json + providerCredential: + profile: dsflash-go + - id: provider-dsflash-go-config + sourceMode: codex-config + sourceRef: config/agentrun.yaml#controlPlane.lanes.nc01-v02.secrets.provider-dsflash-go-config.codexConfig + codexConfig: + modelProvider: opencode + model: deepseek-v4-flash + reviewModel: deepseek-v4-flash + modelReasoningEffort: xhigh + disableResponseStorage: true + networkAccess: enabled + modelContextWindow: 1000000 + modelAutoCompactTokenLimit: 900000 + modelCatalogJson: /home/agentrun/.codex-dsflash-go/model-catalog.json + approvalsReviewer: user + modelProviders: + opencode: + name: Moon Bridge OpenCode Zen Go Flash + baseUrl: http://hwlab-deepseek-proxy.hwlab-v03.svc.cluster.local:4000/v1 + wireApi: responses + requiresOpenaiAuth: true + projects: + /root: + trustLevel: trusted + /home/agentrun/workspaces: + trustLevel: trusted + tuiModelAvailabilityNux: + deepseek-v4-flash: 4 + noticeHideFullAccessWarning: true + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-provider-dsflash-go + key: config.toml + providerCredential: + profile: dsflash-go + - id: provider-dsflash-go-model-catalog + sourceMode: json + sourceRef: config/agentrun.yaml#controlPlane.lanes.nc01-v02.secrets.provider-dsflash-go-model-catalog.jsonValue + jsonValue: + models: + - slug: deepseek-v4-flash + display_name: DeepSeek V4 Flash via OpenCode Zen Go + description: >- + DeepSeek V4 Flash exposed to Codex through OpenCode Zen Go and a + Responses-compatible Moon Bridge profile. + default_reasoning_level: xhigh + supported_reasoning_levels: + - effort: low + description: Fast responses with lighter reasoning + - effort: medium + description: Balanced reasoning + - effort: high + description: Deep reasoning + - effort: xhigh + description: Maximum reasoning + shell_type: shell_command + visibility: list + supported_in_api: true + priority: 0 + additional_speed_tiers: [] + service_tiers: [] + availability_nux: null + upgrade: null + base_instructions: >- + You are Codex, a coding agent based on DeepSeek V4 Flash. You and + the user share one workspace, and your job is to collaborate with + them until their goal is genuinely handled. + model_messages: {} + supports_reasoning_summaries: true + default_reasoning_summary: auto + support_verbosity: false + apply_patch_tool_type: freeform + web_search_tool_type: text_and_image + truncation_policy: + mode: tokens + limit: 10000 + supports_parallel_tool_calls: true + supports_image_detail_original: false + context_window: 1000000 + max_context_window: 1000000 + auto_compact_token_limit: 900000 + effective_context_window_percent: 95 + experimental_supported_tools: [] + input_modalities: + - text + supports_search_tool: false + targetRef: + namespace: agentrun-v02 + name: agentrun-v01-provider-dsflash-go + key: model-catalog.json + providerCredential: + profile: dsflash-go - extends: controlPlane.templates.secrets.githubPrRawToken - extends: controlPlane.templates.secrets.unideskSshToken extends: controlPlane.templates.agentrunV02Lane diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 629757f8..6171deef 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -189,6 +189,9 @@ lanes: codeAgentRuntime: enabled: true adapter: agentrun-v02 + controlPlaneTarget: + node: NC01 + lane: nc01-v02 managerUrl: http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080 apiKeySecretName: hwlab-v03-master-server-admin-api-key apiKeySecretKey: api-key diff --git a/docs/reference/agentrun.md b/docs/reference/agentrun.md index bde7d277..320abf9f 100644 --- a/docs/reference/agentrun.md +++ b/docs/reference/agentrun.md @@ -109,7 +109,52 @@ YAML-only lane 的 `trigger-current` 会先确保目标 branch 已同步到 k8s Provider credential Secret 的 `auth.json` 和 `config.toml` 也必须按 NC01 lane 的 YAML `sourceRef` 下发，不能把指挥机全局 Codex 配置当成运行真相。lane 的模型、provider、endpoint 和 Codex runtime options 需要由 UniDesk YAML 拥有时，使用 `sourceMode: codex-config` 和同一 Secret spec 下的 `codexConfig` 渲染 `config.toml`；不要通过修改 `/root/.codex/config.toml` 或 `~/.codex/config.toml` 来改变 HWLAB/AgentRun 运行面模型。 -AgentRun runtime provider profile 的标准在线配置入口是 `bun scripts/cli.ts agentrun provider-profile plan|status|apply|validate --node NC01 --lane nc01-v02 --profile `。`plan/status` 必须输出 YAML 声明的 profile、backendProfile、SecretRef/key、fingerprint、runner egress/NO_PROXY 和 `valuesPrinted=false`；`apply --confirm` 只同步选中 profile 的 Secret/config、复用当前 manager image 渲染 GitOps、刷新 Argo 并重启 manager，不触发 image rebuild 或 PipelineRun；`validate` 只能作为配置和运行面可见性预检，不能替代 HWLAB Web/CLI 原入口真实 turn。需要确认某个 profile 已上线时，先跑 `provider-profile status` 看 live Secret fingerprint 和 manager env，再用 HWLAB Web dispatcher 等价入口创建新 session 并发送一轮真实 prompt。Web 入口验证优先使用 `web-probe observe start` + `observe command --type newSession` + `observe command --type sendPrompt --provider `，证据应包含 providerSelection、`/v1/agent/chat` 202、traceId、terminal turn 和 Final Response。 +AgentRun runtime provider profile 的标准在线配置入口是： + +- `provider-profile plan`：检查 YAML 声明与 owner source； +- `provider-profile runtime-status`：只检查 runtime Secret/key 与 YAML endpoint 对应的 Kubernetes Service/port，不读取 owner source，也不解码 Secret； +- `provider-profile status`：读取 owner source，并对 YAML 期望与 runtime 的 Kubernetes Secret 编码态 fingerprint； +- `provider-profile apply --confirm`：provider Secret 编码态完全一致时跳过 Secret 写入，再复用当前 manager image 发布 GitOps、刷新 Argo 并重启 manager，不创建 image build 或 PipelineRun； +- `provider-profile validate`：只做配置与运行面预检，不能替代 HWLAB Web/CLI 原入口真实 turn。 + +### Provider Secret 激活 + +- lane parser 必须拒绝重复 Secret `id` 或重复完整 `targetRef`； +- 执行期必须按 `namespace/name/key` 完整 `targetRef` 分类 provider Secret： + - 不得用 Secret `id` 作为 activation gate 的写入排除键； + - provider `targetRef` 命中的写入必须先经过同一个 activation gate； +- Pod、Job 和 manager DB 观察只用于诊断，不能作为 Provider Secret 写授权； +- Provider Secret create、key add 或 key change 固定 fail-closed： + - 异步 job 创建前阻断； + - worker Secret sync 前再次阻断； + - Kubernetes 编码态 fingerprint 完全一致时只允许 no-op，并跳过 provider Secret patch； +- 继续激活前必须由 AgentRun manager 在同一个 authority 中完成： + - 阻断相关新 runner admission； + - DB fence 与既有 runner 收敛； + - Secret UID/resourceVersion CAS 写入； + - 写后编码态 fingerprint 验证； +- manager fence 上线并经 NC01 验收后才能解除非 no-op 阻断； +- 依赖入口： + - pre-create retention 分类由 [AgentRun #280](https://github.com/pikasTech/agentrun/issues/280) 跟踪； + - 原子实现由 [PR #281](https://github.com/pikasTech/agentrun/pull/281) 跟踪； + - 无 incoming create 时的周期回收由 [AgentRun #282](https://github.com/pikasTech/agentrun/issues/282) 跟踪； + - Provider Secret admission fence 与 CAS 由 [AgentRun #284](https://github.com/pikasTech/agentrun/issues/284) 跟踪； + - #280/#281/#282 只解决 retention，不等于 Provider Secret 激活 fence。 + +### Provider profile 验收 + +- 先运行 `provider-profile status`； +- 再通过 HWLAB Web dispatcher 等价入口创建 session 并发送真实 prompt； +- Web 验收入口： + - `web-probe observe start`； + - `observe command --type newSession`； + - `observe command --type sendPrompt --provider `； +- 证据必须包含： + - `providerSelection`； + - `/v1/agent/chat` 202； + - `traceId`； + - terminal turn； + - Final Response。 仅修改 UniDesk YAML 中的 manager env、Kafka 开关、runner egress 或 provider config 时，不一定需要重建 AgentRun image。若 AgentRun source commit 已经是目标版本，应优先使用 `agentrun provider-profile apply --node NC01 --lane nc01-v02 --profile --confirm` 复用当前 image 发布最新 YAML-rendered GitOps、刷新 Argo 并 rollout manager，再用窄查询确认 Deployment env 和新建 runner Job env；不要只看 `control-plane status` 的 manager source commit 对齐就认定 YAML env 已进入 runtime。 @@ -117,15 +162,29 @@ HWLAB 通过 NC01 `agentrun-v02` 使用 Codex profile 时，`config.toml` 应只 AgentRun Kafka closeout 需要同时验证 manager 和 runner 两条生产链路。manager 侧至少创建一个短 run，并在 manager pod 内执行 `./scripts/agentrun kafka tail agentrun --run-id ` 查到 `agentrun.event.v1` 的 `agentrun.run.created` 或后续 durable event；runner 侧必须创建真实 runner Job，并确认 Job env 中存在 `AGENTRUN_KAFKA_ENABLED`、`AGENTRUN_KAFKA_BOOTSTRAP_SERVERS`、`AGENTRUN_KAFKA_STDIO_TOPIC`、`AGENTRUN_KAFKA_STDIO_CLIENT_ID` 和 `AGENTRUN_KAFKA_STDIO_PRODUCE_ENABLED`，再用 `./scripts/agentrun kafka tail stdio --run-id ` 查到 `codex-stdio.raw.v1` 的 stdin/stdout/stderr frame。默认查询不得展开 raw value；需要排查裸帧时才显式加 `--values`，并避免把 Secret value 写入 issue 或日志。 -如果 smoke runner 被 retention 挡住，先运行 `agentrun control-plane cleanup-runners --node NC01 --lane nc01-v02 --dry-run`。只有 operator 明确接受中断旧 active runner 的风险时，才可按同参 `--force-active --dry-run` 确认 selected runner，再 `--force-active --confirm` 清理；不得为了验证 Kafka raw frame 直接裸删 Kubernetes Job/Pod。 +如果 smoke runner 被 retention 挡住，先运行 `agentrun control-plane cleanup-runners --node NC01 --lane nc01-v02 --dry-run`。该入口只做有界分类，不拥有删除 authority；必须由 AgentRun manager 在创建新 runner 前通过 DB fence 与 Kubernetes CAS 原子回收，UniDesk 不提供强杀参数，也不得为了验证 Kafka raw frame 直接裸删 Kubernetes Job/Pod。 AgentRun resource/session client policy 也由 `config/agentrun.yaml` 声明。`client.sessionPolicy` 是未显式选择 node/lane 时 `agentrun send session/...` 和相关 session payload 生成的默认 `tenantId`、`projectId`、`providerId`、`backendProfile`、`workspaceRef` 和 execution policy 来源；显式 `--node --lane ` 后，`explain session-policy`、`send session`、resource primitives 和 AipodSpec render 都必须改用目标 lane 的 YAML 事实。lane `secrets[].providerCredential.profile` 声明 provider credential Secret 归属，UniDesk CLI 只按 YAML 聚合 Secret name/key，不再用代码拼接 provider Secret 名称。只读入口 `bun scripts/cli.ts agentrun explain session-policy` 用于查看选中目标 lane、policy 来源、实际 executionPolicy payload 和 provider credential binding 来源；输出只能包含 Secret metadata、key 名和 `valuesPrinted=false`，不得打印 Secret value。 非默认 lane 的 session follow-up 必须证明 `send session` 使用的是选中 node/lane 的 run policy。使用短命令形态前，先用 `agentrun explain session-policy --node --lane [--backend-profile ]` 或等价 dry-run/describe 路径确认 `backendProfile`、`providerId`、`workspaceRef`、execution policy 和 provider credential SecretRef 都来自目标 lane；`--prompt-stdin` 短命令形态和 `--json-stdin -o json` 显式 JSON 形态应披露同一份 `sessionPolicy` 摘要。渲染结果回退到全局默认 lane、显示错误的 default lane，或短命令与 JSON body 使用不同 policy，都是 lane policy 缺陷，应修复 YAML 目标解析或 CLI 渲染；不得通过手工创建默认 lane Secret、复制凭据、改写 JSON body 或修改 runtime namespace 来掩盖 policy 选错的问题。 -`cleanup-runners` 是 AgentRun runtime runner retention 入口，只清理 YAML 选中 lane 的 runtime namespace 中匹配 `deployment.runner.retention.selectors` 的 runner Job/Pod。runner 上限、最后活跃排序策略、active heartbeat 窗口、Job name prefix 和是否启用 age-based cleanup 都以 `config/agentrun.yaml` 为唯一真相；命令行不得覆盖这些数值。dry-run 必须披露清理前 runner Job 数、runner 非终态 Pod 数、按最后活跃时间排序的 inactive 候选、selected runner Job、manager facts 可用性和 active run 风险；confirm 默认只删除 selected runner Job，并重新统计清理后 runner Job/Pod 数。manager facts 不可用时，只允许清理终态或无活动 Pod 的安全候选，并保留风险字段，不能把 Kubernetes 创建时间冒充为完整最后活跃事实。 +`cleanup-runners` 是 AgentRun runtime runner retention 的只读观察入口： -`cleanup-runners --force-active` 只用于 operator 已明确决定“强杀 runner pod”的资源恢复场景，例如 runner Job 顶满单节点 pod 配额并阻塞 git-mirror、CI/CD 或其他控制面调度。使用前仍必须先执行同参 `--dry-run`，确认 `criteria.forceActive=true`、命中的 namespace/selector、`selectedRunnerJobs` 和预期一致；`--confirm` 会删除所有匹配 runner Job，包括 protected active runner，并会中断对应 run、command 或 session。该开关不得作为日常 retention、静默自愈或 over-limit 默认策略；需要强杀时也必须走这个受控入口，禁止回退到裸 `kubectl delete pod/job`。 +- 只对 YAML 选中 lane 的 runtime namespace 中匹配 `deployment.runner.retention.selectors` 的 runner Job/Pod 做有界分类； +- runner 上限、最后活跃排序策略、active heartbeat 窗口、stale Pending 窗口、Job name prefix 和 age-based cleanup 都以 `config/agentrun.yaml` 为唯一真相； +- 命令行不得覆盖这些数值； +- dry-run 必须分开披露 observation 与 capacity； +- 默认 capacity 摘要必须显示： + - `/`； + - `status`； + - `availableSlots`； + - typed blocker； + - `degradedReason`； +- manager facts 或 Warning Event 查询不完整时返回 observation degraded； +- `runnerJobCount === maxRunners` 时返回 at-capacity degraded，不能在下一次创建已经依赖尚未验收的 manager retention 时继续显示健康； +- UniDesk 的 `--confirm` 固定 fail-closed，不按名称删除 Job/Pod； +- 真正回收只能由 AgentRun manager 在 incoming runner 创建前通过 DB fence 与 Kubernetes CAS 原子完成； +- manager facts 不可用时继续保护，不能把 Kubernetes 创建时间冒充为完整最后活跃事实。 `cleanup-runs` 是 AgentRun `v0.2` 完成态 CI workspace retention 入口，只清理 `agentrun-ci` namespace 中超过 `--min-age-minutes` 的 `agentrun-v02-ci-*` PipelineRun，通过 Tekton ownerRef 释放临时 workspace PVC。dry-run 必须披露候选 PipelineRun、owned PVC、active mount 保护、local-path 实际估算 bytes 和 confirm 命令。默认保护最新完成的 PipelineRun，保留当前 CI/CD 状态证据。`cleanup-released-pvs` 是二次回收入口，只处理 `agentrun-ci`、`local-path`、`Delete` reclaim policy 的 `Released` PV；它不触碰 AgentRun runtime namespace、业务 PVC、Secret、registry storage 或 GitOps desired state。磁盘治理规则见 `docs/reference/gc.md`。 diff --git a/scripts/src/agentrun-lanes.ts b/scripts/src/agentrun-lanes.ts index 1fc29043..682ee63c 100644 --- a/scripts/src/agentrun-lanes.ts +++ b/scripts/src/agentrun-lanes.ts @@ -47,10 +47,12 @@ export interface AgentRunSecretRef { export interface AgentRunLaneSecretSpec { readonly id: string; readonly sourceRef: string; - readonly sourceMode: "env" | "file" | "codex-config"; + readonly sourceMode: "env" | "file" | "codex-config" | "json"; readonly sourceKey: string | null; readonly sourceFormat: "env" | "raw-token" | null; + readonly transform: "codex-auth-json-openai-api-key" | null; readonly codexConfig: AgentRunCodexConfigSpec | null; + readonly jsonValue: unknown; readonly targetRef: AgentRunSecretRef; readonly providerCredentialProfile: string | null; } @@ -64,6 +66,11 @@ export interface AgentRunCodexConfigSpec { readonly networkAccess: string | null; readonly windowsWslSetupAcknowledged: boolean | null; readonly serviceTier: string | null; + readonly modelContextWindow: number | null; + readonly modelAutoCompactTokenLimit: number | null; + readonly modelCatalogJson: string | null; + readonly approvalsReviewer: string | null; + readonly noticeHideFullAccessWarning: boolean | null; readonly modelProviders: readonly AgentRunCodexModelProviderSpec[]; readonly features: Readonly>; readonly projects: readonly AgentRunCodexProjectSpec[]; @@ -247,6 +254,7 @@ export interface AgentRunRunnerRetentionSpec { readonly maxRunners: number; readonly cleanupOrder: "oldest-inactive-last-active-first"; readonly activeHeartbeatMaxAgeMs: number; + readonly stalePendingMaxAgeMs: number; readonly selectors: { readonly matchLabels: Readonly>; readonly jobNamePrefixes: readonly string[]; @@ -437,10 +445,11 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record ({ id: secret.id, - sourceRef: secret.sourceMode === "codex-config" ? secret.sourceRef : secret.sourceRef.startsWith("/") ? secret.sourceRef : rootPath(".state", "secrets", secret.sourceRef), + sourceRef: secret.sourceMode === "codex-config" || secret.sourceMode === "json" ? secret.sourceRef : secret.sourceRef.startsWith("/") ? secret.sourceRef : rootPath(".state", "secrets", secret.sourceRef), sourceMode: secret.sourceMode, sourceKey: secret.sourceKey, sourceFormat: secret.sourceFormat, + transform: secret.transform, targetRef: secret.targetRef, providerCredential: secret.providerCredentialProfile === null ? null : { profile: secret.providerCredentialProfile }, codexConfig: secret.codexConfig === null ? null : { @@ -450,6 +459,11 @@ export function agentRunLaneSummary(spec: AgentRunLaneSpec): Record parseGitMirrorRepository(repo, `${path}.gitMirror.repositories[${index}]`)), }, database: parseDatabase(database, `${path}.database`), - secrets: arrayField(input, "secrets", path).map((secret, index) => parseLaneSecret(secret, `${path}.secrets[${index}]`)), + secrets: parseLaneSecrets(input, path), }; validateAgentRunLaneSourceAuthority(spec, path); return spec; @@ -763,6 +777,7 @@ function parseRunnerRetention(input: Record, path: string): Age maxRunners: positiveIntegerField(input, "maxRunners", path), cleanupOrder: enumField(input, "cleanupOrder", path, ["oldest-inactive-last-active-first"]), activeHeartbeatMaxAgeMs: positiveIntegerField(input, "activeHeartbeatMaxAgeMs", path), + stalePendingMaxAgeMs: positiveIntegerField(input, "stalePendingMaxAgeMs", path), selectors: { matchLabels: kubernetesLabelRecordField(recordField(selectors, "matchLabels", `${path}.selectors`), `${path}.selectors.matchLabels`), jobNamePrefixes: stringArrayField(selectors, "jobNamePrefixes", `${path}.selectors`).map((prefix, index) => { @@ -873,14 +888,18 @@ function parseImageBuild(input: Record, path: string): AgentRun function parseLaneSecret(input: Record, path: string): AgentRunLaneSecretSpec { const sourceMode = optionalStringField(input, "sourceMode", path) ?? "env"; - if (sourceMode !== "env" && sourceMode !== "file" && sourceMode !== "codex-config") throw new Error(`${path}.sourceMode must be env, file, or codex-config`); + if (sourceMode !== "env" && sourceMode !== "file" && sourceMode !== "codex-config" && sourceMode !== "json") throw new Error(`${path}.sourceMode must be env, file, codex-config, or json`); const sourceKey = optionalStringField(input, "sourceKey", path) ?? null; if (sourceMode === "env" && sourceKey === null) throw new Error(`${path}.sourceKey is required when sourceMode is env`); + if (sourceMode !== "env" && sourceKey !== null) throw new Error(`${path}.sourceKey is only supported when sourceMode is env`); const sourceFormat = optionalStringField(input, "sourceFormat", path) ?? null; if (sourceFormat !== null && sourceFormat !== "env" && sourceFormat !== "raw-token") throw new Error(`${path}.sourceFormat must be env or raw-token`); if (sourceFormat === "raw-token" && sourceMode !== "env") throw new Error(`${path}.sourceFormat raw-token requires sourceMode env`); - if (sourceMode === "codex-config" && sourceKey !== null) throw new Error(`${path}.sourceKey is not supported when sourceMode is codex-config`); - if (sourceMode === "codex-config" && sourceFormat !== null) throw new Error(`${path}.sourceFormat is not supported when sourceMode is codex-config`); + if (sourceMode !== "env" && sourceFormat !== null) throw new Error(`${path}.sourceFormat is only supported when sourceMode is env`); + const transform = optionalStringField(input, "transform", path) ?? null; + if (transform !== null && transform !== "codex-auth-json-openai-api-key") throw new Error(`${path}.transform must be codex-auth-json-openai-api-key when set`); + if (transform !== null && sourceMode !== "env") throw new Error(`${path}.transform ${transform} requires sourceMode env`); + const jsonValue = sourceMode === "json" ? jsonValueField(input, "jsonValue", path) : null; const providerCredential = parseProviderCredentialBinding(input, `${path}.providerCredential`); return { id: stringField(input, "id", path), @@ -888,12 +907,35 @@ function parseLaneSecret(input: Record, path: string): AgentRun sourceMode, sourceKey, sourceFormat, + transform, codexConfig: sourceMode === "codex-config" ? parseCodexConfig(recordField(input, "codexConfig", path), `${path}.codexConfig`) : null, + jsonValue, targetRef: parseNamespacedSecretRef(recordField(input, "targetRef", path), `${path}.targetRef`), providerCredentialProfile: providerCredential, }; } +function parseLaneSecrets(input: Record, path: string): AgentRunLaneSecretSpec[] { + const secrets = arrayField(input, "secrets", path).map((secret, index) => parseLaneSecret(secret, `${path}.secrets[${index}]`)); + const idPaths = new Map(); + const targetRefPaths = new Map(); + for (const [index, secret] of secrets.entries()) { + const secretPath = `${path}.secrets[${index}]`; + const previousIdPath = idPaths.get(secret.id); + if (previousIdPath !== undefined) { + throw new Error(`${secretPath}.id duplicates ${previousIdPath}.id (${secret.id})`); + } + idPaths.set(secret.id, secretPath); + const targetRef = `${secret.targetRef.namespace}/${secret.targetRef.name}/${secret.targetRef.key}`; + const previousTargetRefPath = targetRefPaths.get(targetRef); + if (previousTargetRefPath !== undefined) { + throw new Error(`${secretPath}.targetRef duplicates ${previousTargetRefPath}.targetRef (${targetRef})`); + } + targetRefPaths.set(targetRef, secretPath); + } + return secrets; +} + function parseCodexConfig(input: Record, path: string): AgentRunCodexConfigSpec { const providersRaw = recordField(input, "modelProviders", path); const modelProviders = Object.entries(providersRaw).map(([id, raw]) => { @@ -924,6 +966,11 @@ function parseCodexConfig(input: Record, path: string): AgentRu networkAccess: optionalStringField(input, "networkAccess", path) ?? null, windowsWslSetupAcknowledged: optionalBooleanField(input, "windowsWslSetupAcknowledged", path), serviceTier: optionalStringField(input, "serviceTier", path) ?? null, + modelContextWindow: optionalIntegerField(input, "modelContextWindow", path) ?? null, + modelAutoCompactTokenLimit: optionalIntegerField(input, "modelAutoCompactTokenLimit", path) ?? null, + modelCatalogJson: optionalStringField(input, "modelCatalogJson", path) ?? null, + approvalsReviewer: optionalStringField(input, "approvalsReviewer", path) ?? null, + noticeHideFullAccessWarning: optionalBooleanField(input, "noticeHideFullAccessWarning", path), modelProviders, features, projects, @@ -931,6 +978,17 @@ function parseCodexConfig(input: Record, path: string): AgentRu }; } +function jsonValueField(obj: Record, key: string, path: string): unknown { + if (!(key in obj)) throw new Error(`${path}.${key} is required when sourceMode is json`); + const value = obj[key]; + try { + if (JSON.stringify(value) === undefined) throw new Error("not-json"); + } catch { + throw new Error(`${path}.${key} must be JSON-serializable`); + } + return value; +} + function optionalBooleanField(obj: Record, key: string, path: string): boolean | null { const value = obj[key]; if (value === undefined || value === null) return null; diff --git a/scripts/src/agentrun-manifests.ts b/scripts/src/agentrun-manifests.ts index 8a60ddb9..37d1770b 100644 --- a/scripts/src/agentrun-manifests.ts +++ b/scripts/src/agentrun-manifests.ts @@ -666,6 +666,7 @@ function agentRunManagerManifests(spec: AgentRunLaneSpec, sourceCommit: string, rules: [ { apiGroups: ["batch"], resources: ["jobs"], verbs: ["create", "delete", "get", "list", "watch"] }, { apiGroups: [""], resources: ["pods"], verbs: ["delete", "get", "list", "watch"] }, + { apiGroups: [""], resources: ["events"], verbs: ["get", "list", "watch"] }, { apiGroups: [""], resources: ["persistentvolumeclaims"], verbs: ["create", "get", "list", "watch", "delete"] }, ], }, @@ -716,6 +717,7 @@ function managerEnv(spec: AgentRunLaneSpec, sourceCommit: string, imageRef: stri { name: "AGENTRUN_RUNNER_RETENTION_MAX_RUNNERS", value: String(spec.deployment.runner.retention.maxRunners) }, { name: "AGENTRUN_RUNNER_RETENTION_CLEANUP_ORDER", value: spec.deployment.runner.retention.cleanupOrder }, { name: "AGENTRUN_RUNNER_RETENTION_ACTIVE_HEARTBEAT_MAX_AGE_MS", value: String(spec.deployment.runner.retention.activeHeartbeatMaxAgeMs) }, + { name: "AGENTRUN_RUNNER_RETENTION_STALE_PENDING_MAX_AGE_MS", value: String(spec.deployment.runner.retention.stalePendingMaxAgeMs) }, { name: "AGENTRUN_RUNNER_RETENTION_MATCH_LABELS_JSON", value: JSON.stringify(spec.deployment.runner.retention.selectors.matchLabels) }, { name: "AGENTRUN_RUNNER_RETENTION_JOB_NAME_PREFIXES", value: spec.deployment.runner.retention.selectors.jobNamePrefixes.join(",") }, { name: "AGENTRUN_RUNNER_RETENTION_AGE_BASED_CLEANUP_ENABLED", value: String(spec.deployment.runner.retention.ageBasedCleanup.enabled) }, diff --git a/scripts/src/agentrun/control-plane.ts b/scripts/src/agentrun/control-plane.ts index 2122d2ed..965d95d0 100644 --- a/scripts/src/agentrun/control-plane.ts +++ b/scripts/src/agentrun/control-plane.ts @@ -175,7 +175,7 @@ export function parseGitMirrorOptions(args: string[]): GitMirrorOptions { } export function parseCleanupRunnersOptions(args: string[]): CleanupRunnersOptions { - validateOptions(args, new Set(["--confirm", "--dry-run", "--force-active", "--full", "--raw"]), new Set(["--timeout-seconds", "--node", "--lane"])); + validateOptions(args, new Set(["--confirm", "--dry-run", "--full", "--raw"]), new Set(["--timeout-seconds", "--node", "--lane"])); const base = parseConfirmOptions(args); const raw = args.includes("--raw"); return { @@ -183,7 +183,6 @@ export function parseCleanupRunnersOptions(args: string[]): CleanupRunnersOption node: optionValue(args, "--node") ?? null, lane: optionValue(args, "--lane") ?? null, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600), - forceActive: args.includes("--force-active"), full: raw || args.includes("--full"), raw, }; diff --git a/scripts/src/agentrun/options.ts b/scripts/src/agentrun/options.ts index 1f9f26c5..f49d9ebe 100644 --- a/scripts/src/agentrun/options.ts +++ b/scripts/src/agentrun/options.ts @@ -248,7 +248,6 @@ export interface CleanupRunnersOptions extends ConfirmOptions, DisclosureOptions node: string | null; lane: string | null; timeoutSeconds: number; - forceActive: boolean; } export interface CleanupRunsOptions extends ConfirmOptions { diff --git a/scripts/src/agentrun/provider-activation.ts b/scripts/src/agentrun/provider-activation.ts new file mode 100644 index 00000000..6ebad68a --- /dev/null +++ b/scripts/src/agentrun/provider-activation.ts @@ -0,0 +1,181 @@ +import { readFileSync } from "node:fs"; + +import type { AgentRunLaneSecretSpec, AgentRunLaneSpec } from "../agentrun-lanes"; +import type { UniDeskConfig } from "../config"; +import { rootPath } from "../config"; +import { sha256Fingerprint } from "../platform-infra-ops-library"; + +import { capture, captureJsonPayload, compactCapture, record, shQuote } from "./utils"; + +export const AGENTRUN_MANAGER_RETENTION_ISSUE = "https://github.com/pikasTech/agentrun/issues/280"; +export const AGENTRUN_MANAGER_RETENTION_PULL_REQUEST = "https://github.com/pikasTech/agentrun/pull/281"; +export const AGENTRUN_MANAGER_RECONCILER_ISSUE = "https://github.com/pikasTech/agentrun/issues/282"; +export const AGENTRUN_PROVIDER_ACTIVATION_FENCE_ISSUE = "https://github.com/pikasTech/agentrun/issues/284"; + +export interface ProviderSecretActivationTarget { + readonly id: string; + readonly targetRef: AgentRunLaneSecretSpec["targetRef"]; + readonly desiredEncodedFingerprint: string; +} + +export function providerSecretDesiredEncodedFingerprint(value: string): string { + return sha256Fingerprint(Buffer.from(value, "utf8").toString("base64")); +} + +export function providerSecretActivationPreflightScript(spec: AgentRunLaneSpec, targets: readonly ProviderSecretActivationTarget[]): string { + const script = readFileSync(rootPath("scripts/src/agentrun/provider-secret-activation-preflight.mjs"), "utf8"); + return [ + "set -eu", + `namespace=${shQuote(spec.runtime.namespace)}`, + `targets_json_b64=${shQuote(Buffer.from(JSON.stringify(targets), "utf8").toString("base64"))}`, + `match_labels_json_b64=${shQuote(Buffer.from(JSON.stringify(spec.deployment.runner.retention.selectors.matchLabels), "utf8").toString("base64"))}`, + `job_name_prefixes_json_b64=${shQuote(Buffer.from(JSON.stringify(spec.deployment.runner.retention.selectors.jobNamePrefixes), "utf8").toString("base64"))}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "cat > \"$tmp_dir/provider-secret-activation-preflight.mjs\" <<'NODE'", + script, + "NODE", + "env NAMESPACE=\"$namespace\" TARGETS_JSON_B64=\"$targets_json_b64\" MATCH_LABELS_JSON_B64=\"$match_labels_json_b64\" JOB_NAME_PREFIXES_JSON_B64=\"$job_name_prefixes_json_b64\" node \"$tmp_dir/provider-secret-activation-preflight.mjs\"", + ].join("\n"); +} + +export async function providerSecretActivationPreflight( + config: UniDeskConfig, + spec: AgentRunLaneSpec, + targets: readonly ProviderSecretActivationTarget[], + options: { readonly full?: boolean } = {}, +): Promise & { ok: boolean }> { + const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", providerSecretActivationPreflightScript(spec, targets)]); + const payload = captureJsonPayload(result); + const ok = result.exitCode === 0 + && payload.ok === true + && payload.targetSecretObservationComplete === true + && payload.providerSecretDataNoOp === true + && Number(payload.providerSecretMutationCount ?? -1) === 0; + const visiblePayload = options.full === true ? payload : compactProviderSecretActivationPayload(payload); + return { + ...visiblePayload, + ok, + status: ok ? "ready" : "blocked", + reason: ok ? null : payload.reason ?? (result.exitCode === 0 ? "provider-secret-activation-preflight-blocked" : "provider-secret-activation-preflight-failed"), + capture: compactCapture(result, { + full: options.full === true || result.exitCode !== 0, + stdoutTailChars: 2400, + stderrTailChars: 2400, + }), + managerAuthority: { + owner: "agentrun-manager-provider-activation-fence", + requiredBeforeActivation: record(payload).requiresManagerFence === true, + atomicFenceAvailable: false, + mutationAuthorized: false, + noOpOnly: true, + trackingIssue: AGENTRUN_PROVIDER_ACTIVATION_FENCE_ISSUE, + retentionDependencies: { + issue: AGENTRUN_MANAGER_RETENTION_ISSUE, + pullRequest: AGENTRUN_MANAGER_RETENTION_PULL_REQUEST, + stalePendingReconcilerIssue: AGENTRUN_MANAGER_RECONCILER_ISSUE, + sufficientForProviderActivation: false, + valuesPrinted: false, + }, + valuesPrinted: false, + }, + mutation: false, + runtimeSecretValuesDecoded: false, + valuesPrinted: false, + }; +} + +function compactProviderSecretActivationPayload(payload: Record): Record { + const targetSecrets = Array.isArray(payload.targetSecrets) ? payload.targetSecrets.map(record) : []; + const risks = Array.isArray(payload.activationRisks) ? payload.activationRisks.map(record) : []; + const visibleRisks = risks.slice(0, 1).map((risk) => ({ + pod: risk.pod ?? null, + jobName: risk.jobName ?? null, + phase: risk.phase ?? null, + createdAt: risk.createdAt ?? null, + runId: risk.runId ?? null, + commandId: risk.commandId ?? null, + runnerJobId: risk.runnerJobId ?? null, + runnerId: risk.runnerId ?? null, + waitingReasons: Array.isArray(risk.waitingReasons) ? risk.waitingReasons.slice(0, 4) : [], + dependencies: Array.isArray(risk.dependencies) + ? risk.dependencies.map(record).slice(0, 2).map((dependency) => ({ + source: dependency.source ?? null, + name: dependency.name ?? null, + affectedDesiredKeys: Array.isArray(dependency.affectedDesiredKeys) ? dependency.affectedDesiredKeys.slice(0, 8) : [], + mutationKinds: Array.isArray(dependency.mutationKinds) ? dependency.mutationKinds.slice(0, 4) : [], + reason: dependency.reason ?? null, + valuesPrinted: false, + })) + : [], + classification: risk.classification ?? null, + valuesPrinted: false, + })); + return { + ok: payload.ok === true, + status: payload.status ?? null, + reason: payload.reason ?? null, + observedAt: payload.observedAt ?? null, + target: payload.target ?? null, + targetSecretCount: payload.targetSecretCount ?? targetSecrets.length, + targetSecrets: targetSecrets.map((target) => ({ + namespace: target.namespace ?? null, + name: target.name ?? null, + desiredKeys: Array.isArray(target.desiredKeys) ? target.desiredKeys.slice(0, 8) : [], + queryOk: target.queryOk === true, + present: typeof target.present === "boolean" ? target.present : null, + mutationCount: target.mutationCount ?? null, + noOp: target.noOp === true, + mutations: Array.isArray(target.mutations) + ? target.mutations.map(record).slice(0, 8).map((mutation) => ({ + id: mutation.id ?? null, + key: mutation.key ?? null, + desiredFingerprintSuffix: typeof mutation.desiredEncodedFingerprint === "string" ? mutation.desiredEncodedFingerprint.slice(-12) : null, + runtimeFingerprintSuffix: typeof mutation.runtimeEncodedFingerprint === "string" ? mutation.runtimeEncodedFingerprint.slice(-12) : null, + changeKind: mutation.changeKind ?? null, + valuesPrinted: false, + })) + : [], + valuesPrinted: false, + })), + podObservation: payload.podObservation ?? null, + targetSecretObservationComplete: payload.targetSecretObservationComplete === true, + runnerObservationComplete: payload.runnerObservationComplete === true, + allObservationsComplete: payload.allObservationsComplete === true, + providerSecretMutationCount: payload.providerSecretMutationCount ?? null, + providerSecretDataNoOp: payload.providerSecretDataNoOp === true, + providerSecretMutationAuthorized: payload.providerSecretMutationAuthorized === true, + activationAuthority: payload.activationAuthority ?? null, + activationRiskCount: payload.activationRiskCount ?? risks.length, + activationRisks: visibleRisks, + activationRiskOmittedCount: Math.max(0, Number(payload.activationRiskCount ?? risks.length) - visibleRisks.length), + requiresManagerFence: payload.requiresManagerFence === true, + queryComplete: payload.queryComplete === true, + mutation: false, + runtimeSecretValuesDecoded: false, + valuesPrinted: false, + }; +} + +export function providerSecretStagedActivation( + spec: AgentRunLaneSpec, + profiles: readonly string[], +): Record { + return { + status: "blocked-before-secret-activation", + targetProfiles: [...new Set(profiles)], + secretMutationStarted: false, + requiredSequence: [ + "agentrun-manager-reconcile-terminalize-and-fence-affected-runners", + "agentrun-manager-acquire-provider-activation-admission-fence-and-secret-cas", + "repeat-provider-secret-activation-preflight", + "sync-yaml-declared-provider-secret", + ], + managerTrackingIssue: AGENTRUN_MANAGER_RETENTION_ISSUE, + managerTrackingPullRequest: AGENTRUN_MANAGER_RETENTION_PULL_REQUEST, + managerReconcilerIssue: AGENTRUN_MANAGER_RECONCILER_ISSUE, + providerActivationFenceIssue: AGENTRUN_PROVIDER_ACTIVATION_FENCE_ISSUE, + retry: `bun scripts/cli.ts agentrun provider-profile apply --node ${spec.nodeId} --lane ${spec.lane} --profile --confirm`, + valuesPrinted: false, + }; +} diff --git a/scripts/src/agentrun/provider-profile.ts b/scripts/src/agentrun/provider-profile.ts index 973baea4..839ab8b8 100644 --- a/scripts/src/agentrun/provider-profile.ts +++ b/scripts/src/agentrun/provider-profile.ts @@ -1,6 +1,8 @@ // SPEC: issue-1616 YAML-first AgentRun provider profile online configuration. // Provides a thin CLI over YAML-declared provider credential Secrets, runtime GitOps env and bounded validation. import type { UniDeskConfig } from "../config"; +import { startJob } from "../jobs"; +import { sha256Fingerprint } from "../platform-infra-ops-library"; import { agentRunImageArtifact, renderedFilesDigest, @@ -15,13 +17,21 @@ import { } from "../agentrun-lanes"; import { refreshYamlLaneScript } from "./git-mirror"; import type { ConfirmOptions, DisclosureOptions } from "./options"; +import { + AGENTRUN_PROVIDER_ACTIVATION_FENCE_ISSUE, + providerSecretActivationPreflight, + providerSecretDesiredEncodedFingerprint, + providerSecretStagedActivation, + type ProviderSecretActivationTarget, +} from "./provider-activation"; import { compactAgentRunLaneStatusTarget } from "./trigger"; import { collectLaneSecretSources, createYamlLaneJobScript, - readSecretSourceValue, + inspectSecretSourceValue, restartYamlLaneScript, - secretSyncScript, + type LaneSecretSource, + type LaneSecretSourceInspection, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript, @@ -62,7 +72,6 @@ export function parseProviderProfileOptions(args: string[]): ProviderProfileOpti } if (arg === "--raw") { raw = true; - full = true; continue; } if (arg === "--wait") { @@ -87,16 +96,18 @@ export function parseProviderProfileOptions(args: string[]): ProviderProfileOpti export function providerProfileHelp(): string { return [ - "Usage: bun scripts/cli.ts agentrun provider-profile --node --lane --profile [options]", + "Usage: bun scripts/cli.ts agentrun provider-profile --node --lane --profile [options]", "", "Examples:", " bun scripts/cli.ts agentrun provider-profile plan --node NC01 --lane nc01-v02 --profile gpt.pika", " bun scripts/cli.ts agentrun provider-profile status --node NC01 --lane nc01-v02 --profile gpt.pika", + " bun scripts/cli.ts agentrun provider-profile runtime-status --node NC01 --lane nc01-v02 --profile gpt.pika", " bun scripts/cli.ts agentrun provider-profile apply --node NC01 --lane nc01-v02 --profile gpt.pika --confirm", + " bun scripts/cli.ts job status --tail-bytes 12000", " bun scripts/cli.ts agentrun provider-profile validate --node NC01 --lane nc01-v02 --profile gpt.pika --wait", "", "Boundary:", - " apply syncs YAML-declared Secret/config, publishes runtime GitOps using the current manager image, refreshes Argo and restarts the manager; it does not rebuild images or create a PipelineRun.", + " confirmed apply returns an async job only after the provider Secret gate reports an exact data no-op; the worker skips the provider Secret write, publishes runtime GitOps using the current manager image, refreshes Argo and restarts the manager without rebuilding images or creating a PipelineRun.", ].join("\n"); } @@ -104,11 +115,89 @@ export async function runProviderProfileCommand(config: UniDeskConfig, action: s const options = parseProviderProfileOptions(args); if (action === "plan") return providerProfilePlan(options); if (action === "status") return providerProfileStatus(config, options); - if (action === "apply") return providerProfileApply(config, options); + if (action === "runtime-status") return providerProfileRuntimeStatus(config, options); + if (action === "apply") { + if (options.confirm && !options.wait) return providerProfileApplyAsync(config, options); + return providerProfileApply(config, options); + } if (action === "validate") return providerProfileValidate(config, options); throw new Error(`unsupported provider-profile action: ${action ?? "(empty)"}`); } +async function providerProfileApplyAsync(config: UniDeskConfig, options: ProviderProfileOptions): Promise> { + const plan = providerProfilePlan({ ...options, confirm: false, dryRun: true }); + if (plan.ok !== true) return plan; + const { spec, profile } = resolveProviderProfile(options); + const secrets = spec.secrets.filter((secret) => secret.providerCredentialProfile === profile); + const activationTargets = providerProfileActivationTargets(inspectProviderProfileSources(spec, secrets)); + const activationPreflight = await providerSecretActivationPreflight(config, spec, activationTargets, { full: options.full || options.raw }); + if (activationPreflight.ok !== true) { + return providerProfileActivationBlockedResult(spec, profile, secrets, activationPreflight, "async-submit"); + } + const args = [ + "bun", + "scripts/cli.ts", + "agentrun", + "provider-profile", + "apply", + "--node", + spec.nodeId, + "--lane", + spec.lane, + "--profile", + profile, + "--confirm", + "--wait", + ]; + const jobName = `agentrun_${spec.lane}_provider_profile_${profile.replace(/[^A-Za-z0-9_-]/gu, "-")}_apply`; + const job = startJob(jobName, args, `Apply YAML-declared AgentRun provider profile ${profile} on ${spec.nodeId}/${spec.lane}`); + const statusCommand = `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`; + return { + ok: true, + command: "agentrun provider-profile apply", + mode: "async-job", + mutation: true, + node: spec.nodeId, + lane: spec.lane, + profile, + job, + jobCreated: true, + activationPreflight, + statusCommand, + next: { + status: statusCommand, + full: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000 --full`, + }, + valuesPrinted: false, + }; +} + +async function providerProfileRuntimeStatus(config: UniDeskConfig, options: ProviderProfileOptions): Promise> { + const { configPath, spec, profile, secrets } = resolveProviderProfile(options); + const live = await capture(config, spec.nodeKubeRoute, ["sh", "--", providerProfileRuntimeStatusScript(spec, secrets)]); + const payload = captureJsonPayload(live); + const ready = live.exitCode === 0 + && payload.ok === true + && record(payload.secrets).ready === true + && record(payload.providerServices).ready === true; + return { + ok: ready, + command: "agentrun provider-profile runtime-status", + mode: "runtime-only-profile-preflight", + status: ready ? "ready" : "blocked", + degradedReason: ready ? null : "runtime-provider-profile-unavailable", + mutation: false, + ownerSourceInspected: false, + runtimeSecretValuesDecoded: false, + configPath, + target: options.full ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), + profile: providerProfileSummary(spec, profile, secrets), + runtime: payload, + capture: compactCapture(live, { full: options.full || options.raw || live.exitCode !== 0, stdoutTailChars: 2000, stderrTailChars: 2000 }), + valuesPrinted: false, + }; +} + function resolveProviderProfile(options: ProviderProfileOptions): { configPath: string; spec: AgentRunLaneSpec; profile: string; secrets: AgentRunLaneSecretSpec[] } { if (options.profile === null) throw new Error("provider-profile requires --profile"); const target = resolveAgentRunLaneTarget(options); @@ -122,40 +211,52 @@ function resolveProviderProfile(options: ProviderProfileOptions): { configPath: function providerProfilePlan(options: ProviderProfileOptions): Record { const { configPath, spec, profile, secrets } = resolveProviderProfile(options); - const values = secrets.map((secret) => { - const source = collectLaneSecretSources(spec).find((item) => item.id === secret.id); - const sourceValue = source === undefined ? null : readSecretSourceValue(spec, source); + const inspected = inspectProviderProfileSources(spec, secrets); + const source = providerProfileSourceSummary(inspected); + const discloseFull = options.full || options.raw; + const values = inspected.map(({ secret, inspection }) => { return { id: secret.id, sourceRef: secret.sourceRef, sourceMode: secret.sourceMode, + transform: secret.transform, targetRef: secret.targetRef, key: secret.targetRef.key, codexConfig: secret.codexConfig === null ? null : codexConfigSummary(secret.codexConfig), - sourcePath: sourceValue?.redactedPath ?? null, - fingerprint: sourceValue?.fingerprint ?? null, - valueBytes: sourceValue?.valueBytes ?? null, + ...inspection.summary, valuesPrinted: false, }; }); return { - ok: true, + ok: source.ready, command: "agentrun provider-profile plan", mode: "yaml-first-online-profile", mutation: false, + status: source.ready ? "ready" : "blocked", + degradedReason: source.ready ? null : "lane-secret-source-unavailable", + blockedBeforeMutation: !source.ready, configPath, - target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), + target: discloseFull ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), profile: providerProfileSummary(spec, profile, secrets), + source: discloseFull ? source : compactProviderProfileSource(source), plan: { secretCount: values.length, - secrets: values, + secrets: discloseFull ? values : compactProviderProfilePlanSecrets(values), runtime: { managerDeployment: spec.runtime.managerDeployment, runnerEgressProxyUrl: spec.deployment.runner.egressProxyUrl, runnerNoProxyExtra: spec.deployment.runner.noProxyExtra, + providerServices: providerRuntimeServiceTargets(secrets), + valuesPrinted: false, + }, + applySteps: ["activation-preflight", "provider-secret-no-op-skip", "gitops-publish-current-image", "argocd-refresh-sync", "manager-rollout-restart"], + activationPreflightRequired: true, + providerSecretMutationPolicy: { + mode: "fail-closed-until-manager-activation-fence", + exactNoOpWriteSkipped: true, + trackingIssue: AGENTRUN_PROVIDER_ACTIVATION_FENCE_ISSUE, valuesPrinted: false, }, - applySteps: ["secret-sync", "gitops-publish-current-image", "argocd-refresh-sync", "manager-rollout-restart"], rebuildImages: false, pipelineRun: false, valuesPrinted: false, @@ -168,31 +269,55 @@ function providerProfilePlan(options: ProviderProfileOptions): Record[]): Record[] { + return values.map((item) => ({ + id: item.id ?? null, + targetRef: item.targetRef ?? null, + key: item.key ?? null, + present: item.present === true, + fingerprint: item.fingerprint ?? null, + valueBytes: item.valueBytes ?? null, + degradedReason: item.degradedReason ?? null, + valuesPrinted: false, + })); +} + async function providerProfileStatus(config: UniDeskConfig, options: ProviderProfileOptions): Promise> { const { configPath, spec, profile, secrets } = resolveProviderProfile(options); + const inspected = inspectProviderProfileSources(spec, secrets); + const source = providerProfileSourceSummary(inspected); + if (!source.ready) return providerProfileSourceUnavailable("status", configPath, spec, profile, secrets, source); const live = await capture(config, spec.nodeKubeRoute, ["sh", "--", providerProfileLiveStatusScript(spec, secrets)]); const payload = captureJsonPayload(live); + const comparison = providerProfileLiveComparison(inspected, payload); + const discloseFull = options.full || options.raw; + const yamlDesired = { + secrets: inspected.map(({ secret, inspection }) => ({ + id: secret.id, + targetRef: secret.targetRef, + key: secret.targetRef.key, + fingerprint: inspection.summary.fingerprint, + valueBytes: inspection.summary.valueBytes, + codexConfig: secret.codexConfig === null ? null : codexConfigSummary(secret.codexConfig), + valuesPrinted: false, + })), + runnerNoProxyExtra: spec.deployment.runner.noProxyExtra, + runnerEgressProxyUrl: spec.deployment.runner.egressProxyUrl, + valuesPrinted: false, + }; return { - ok: live.exitCode === 0 && payload.ok !== false, + ok: live.exitCode === 0 && payload.ok !== false && payload.runtimeSecretValuesDecoded === false && comparison.ready, command: "agentrun provider-profile status", mode: "yaml-first-online-profile", mutation: false, configPath, - target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), + target: options.full ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), profile: providerProfileSummary(spec, profile, secrets), - yamlDesired: { - secrets: secrets.map((secret) => ({ - id: secret.id, - targetRef: secret.targetRef, - key: secret.targetRef.key, - codexConfig: secret.codexConfig === null ? null : codexConfigSummary(secret.codexConfig), - valuesPrinted: false, - })), - runnerNoProxyExtra: spec.deployment.runner.noProxyExtra, - runnerEgressProxyUrl: spec.deployment.runner.egressProxyUrl, - valuesPrinted: false, - }, - live: payload, + source: discloseFull ? source : compactProviderProfileSource(source), + yamlDesired: discloseFull ? yamlDesired : compactProviderProfileDesired(yamlDesired), + live: discloseFull + ? { ...payload, desiredComparison: comparison, valuesPrinted: false } + : compactProviderProfileLive(payload, comparison), capture: compactCapture(live, { full: options.full || options.raw || live.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { apply: `bun scripts/cli.ts agentrun provider-profile apply --node ${spec.nodeId} --lane ${spec.lane} --profile ${profile} --confirm`, @@ -202,44 +327,151 @@ async function providerProfileStatus(config: UniDeskConfig, options: ProviderPro }; } +function compactProviderProfileSource(source: ReturnType): Record { + return { + ready: source.ready, + count: source.count, + presentCount: source.presentCount, + missingCount: source.missingCount, + missing: source.items + .filter((item) => item.present !== true) + .map((item) => ({ id: item.id ?? null, targetRef: item.targetRef ?? null, degradedReason: item.degradedReason ?? null, valuesPrinted: false })), + valuesPrinted: false, + }; +} + +function compactProviderProfileDesired(desired: Record): Record { + const secrets = Array.isArray(desired.secrets) ? desired.secrets.map(record) : []; + return { + secretCount: secrets.length, + secrets: secrets.map((item) => ({ + id: item.id ?? null, + targetRef: item.targetRef ?? null, + fingerprint: item.fingerprint ?? null, + valueBytes: item.valueBytes ?? null, + valuesPrinted: false, + })), + runnerEgressProxyUrl: desired.runnerEgressProxyUrl ?? null, + runnerNoProxyExtraCount: Array.isArray(desired.runnerNoProxyExtra) ? desired.runnerNoProxyExtra.length : 0, + valuesPrinted: false, + }; +} + +function compactProviderProfileLive(payload: Record, comparison: ReturnType): Record { + const manager = record(payload.manager); + const secrets = record(payload.secrets); + return { + ok: payload.ok === true, + manager: { + deploymentExists: manager.deploymentExists === true, + readyReplicas: manager.readyReplicas ?? null, + replicas: manager.replicas ?? null, + sourceCommit: manager.sourceCommit ?? null, + envIdentity: manager.envIdentity ?? null, + runnerEgressProxyUrl: manager.runnerEgressProxyUrl ?? null, + noProxyContainsHyue: manager.noProxyContainsHyue === true, + noProxyContainsPikapython: manager.noProxyContainsPikapython === true, + valuesPrinted: false, + }, + secrets: { + count: secrets.count ?? null, + ready: secrets.ready === true, + items: Array.isArray(secrets.items) ? secrets.items.map(record).map((item) => ({ + namespace: item.namespace ?? null, + name: item.name ?? null, + key: item.key ?? null, + present: item.present === true, + keyPresent: item.keyPresent === true, + encodedBytes: item.encodedBytes ?? null, + encodedFingerprint: item.encodedFingerprint ?? null, + valuesPrinted: false, + })) : [], + valuesPrinted: false, + }, + providerServices: { + count: record(payload.providerServices).count ?? null, + ready: record(payload.providerServices).ready === true, + items: Array.isArray(record(payload.providerServices).items) + ? (record(payload.providerServices).items as unknown[]).map(record).map((item) => ({ + providerId: item.providerId ?? null, + baseUrl: item.baseUrl ?? null, + namespace: item.namespace ?? null, + name: item.name ?? null, + port: item.port ?? null, + present: item.present === true, + portPresent: item.portPresent === true, + valuesPrinted: false, + })) + : [], + valuesPrinted: false, + }, + desiredComparison: comparison, + runtimeSecretValuesDecoded: payload.runtimeSecretValuesDecoded === true, + valuesPrinted: false, + }; +} + async function providerProfileApply(config: UniDeskConfig, options: ProviderProfileOptions): Promise> { const { configPath, spec, profile, secrets } = resolveProviderProfile(options); if (options.dryRun || !options.confirm) return providerProfilePlan({ ...options, dryRun: true, confirm: false }); - const secretIds = secrets.map((secret) => secret.id); - const allSources = collectLaneSecretSources(spec); - const selectedSources = allSources.filter((source) => secretIds.includes(source.id)); - const secretValues = selectedSources.map((source) => ({ spec: source, value: readSecretSourceValue(spec, source) })); - const secretSync = await capture(config, spec.nodeKubeRoute, ["sh", "--", secretSyncScript(spec, secretValues.map(({ spec: item, value }) => ({ targetRef: item.targetRef, value: value.value })))]); - const secretPayload = captureJsonPayload(secretSync); - if (secretSync.exitCode !== 0 || secretPayload.ok === false) { - return applyFailure(configPath, spec, profile, "secret-sync", secretPayload, secretSync); + const inspected = inspectProviderProfileSources(spec, secrets); + const source = providerProfileSourceSummary(inspected); + if (!source.ready) return providerProfileSourceUnavailable("apply", configPath, spec, profile, secrets, source); + providerProfileApplyProgress(spec, profile, "activation-preflight", "running"); + const activationPreflight = await providerSecretActivationPreflight(config, spec, providerProfileActivationTargets(inspected), { full: options.full || options.raw }); + providerProfileApplyProgress(spec, profile, "activation-preflight", activationPreflight.ok ? "succeeded" : "failed"); + if (activationPreflight.ok !== true) { + return providerProfileActivationBlockedResult(spec, profile, secrets, activationPreflight, "worker-before-secret-sync"); } + const secretPayload = { + ok: true, + status: "skipped-exact-provider-secret-no-op", + secretSyncStarted: false, + providerSecretWriteSkipped: true, + skippedProviderSecretCount: inspected.length, + reason: "provider-secret-writes-require-agentrun-manager-activation-fence", + activationFenceIssue: AGENTRUN_PROVIDER_ACTIVATION_FENCE_ISSUE, + runtimeSecretValuesDecoded: false, + valuesPrinted: false, + }; + providerProfileApplyProgress(spec, profile, "secret-sync", "succeeded", { status: secretPayload.status }); + providerProfileApplyProgress(spec, profile, "current-image-resolve", "running"); const liveBefore = await capture(config, spec.nodeKubeRoute, ["sh", "--", currentManagerImageScript(spec)]); const liveBeforePayload = captureJsonPayload(liveBefore); const sourceCommit = stringOrNull(liveBeforePayload.sourceCommit); const envIdentity = stringOrNull(liveBeforePayload.envIdentity); const digest = stringOrNull(liveBeforePayload.digest); if (liveBefore.exitCode !== 0 || sourceCommit === null || envIdentity === null || digest === null) { + providerProfileApplyProgress(spec, profile, "current-image-resolve", "failed"); return applyFailure(configPath, spec, profile, "current-image-resolve", liveBeforePayload, liveBefore); } + providerProfileApplyProgress(spec, profile, "current-image-resolve", "succeeded", { sourceCommit }); const image = agentRunImageArtifact(spec, { sourceCommit, envIdentity, digest, status: "reused" }); const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit, image }); + providerProfileApplyProgress(spec, profile, "gitops-publish", "running", { sourceCommit }); const publish = await publishProfileGitops(config, spec, sourceCommit, renderedFiles); + providerProfileApplyProgress(spec, profile, "gitops-publish", publish.ok ? "succeeded" : "failed", { sourceCommit }); if (publish.ok !== true) { return { ok: false, command: "agentrun provider-profile apply", mode: "confirmed-online-profile", mutation: true, configPath, target: compactAgentRunLaneStatusTarget(spec), profile, phase: "gitops-publish", result: publish, valuesPrinted: false }; } + providerProfileApplyProgress(spec, profile, "argocd-refresh-sync", "running", { sourceCommit }); const argo = await capture(config, spec.nodeKubeRoute, ["sh", "--", providerProfileArgoRefreshSyncScript(spec)]); const argoPayload = captureJsonPayload(argo); + providerProfileApplyProgress(spec, profile, "argocd-refresh-sync", argo.exitCode === 0 && argoPayload.ok !== false ? "succeeded" : "failed", { sourceCommit }); if (argo.exitCode !== 0 || argoPayload.ok === false) { return applyFailure(configPath, spec, profile, "argocd-refresh-sync", argoPayload, argo); } + providerProfileApplyProgress(spec, profile, "manager-restart", "running", { sourceCommit }); const restart = await capture(config, spec.nodeKubeRoute, ["sh", "--", restartYamlLaneScript(spec)]); const restartPayload = captureJsonPayload(restart); + providerProfileApplyProgress(spec, profile, "manager-restart", restart.exitCode === 0 && restartPayload.ok !== false ? "succeeded" : "failed", { sourceCommit }); if (restart.exitCode !== 0 || restartPayload.ok === false) { return applyFailure(configPath, spec, profile, "manager-restart", restartPayload, restart); } + providerProfileApplyProgress(spec, profile, "runtime-status", "running", { sourceCommit }); const liveAfter = await capture(config, spec.nodeKubeRoute, ["sh", "--", providerProfileLiveStatusScript(spec, secrets)]); const liveAfterPayload = captureJsonPayload(liveAfter); + providerProfileApplyProgress(spec, profile, "runtime-status", liveAfter.exitCode === 0 && liveAfterPayload.ok !== false ? "succeeded" : "failed", { sourceCommit }); return { ok: liveAfter.exitCode === 0 && liveAfterPayload.ok !== false, command: "agentrun provider-profile apply", @@ -249,6 +481,7 @@ async function providerProfileApply(config: UniDeskConfig, options: ProviderProf target: compactAgentRunLaneStatusTarget(spec), profile: providerProfileSummary(spec, profile, secrets), steps: { + activationPreflight, secretSync: secretPayload, gitops: publish.payload, argo: argoPayload, @@ -265,7 +498,6 @@ async function providerProfileApply(config: UniDeskConfig, options: ProviderProf rebuildImages: false, pipelineRun: false, captures: options.full || options.raw || liveAfter.exitCode !== 0 ? { - secretSync: compactCapture(secretSync, { full: options.full || options.raw || secretSync.exitCode !== 0 }), argo: compactCapture(argo, { full: options.full || options.raw || argo.exitCode !== 0 }), restart: compactCapture(restart, { full: options.full || options.raw || restart.exitCode !== 0 }), live: compactCapture(liveAfter, { full: options.full || options.raw || liveAfter.exitCode !== 0 }), @@ -278,8 +510,61 @@ async function providerProfileApply(config: UniDeskConfig, options: ProviderProf }; } +function providerProfileActivationBlockedResult( + spec: AgentRunLaneSpec, + profile: string, + secrets: readonly AgentRunLaneSecretSpec[], + activationPreflight: Record, + stage: "async-submit" | "worker-before-secret-sync", +): Record { + return { + ok: false, + command: "agentrun provider-profile apply", + mode: stage === "async-submit" ? "activation-preflight-blocked-before-job" : "activation-preflight-blocked-before-secret-sync", + status: "blocked", + degradedReason: activationPreflight.reason ?? "existing-runner-provider-secret-activation-risk", + phase: "activation-preflight", + stage, + mutation: false, + blockedBeforeMutation: true, + jobCreated: stage === "worker-before-secret-sync", + jobAlreadyRunning: stage === "worker-before-secret-sync", + secretSyncStarted: false, + target: { node: spec.nodeId, lane: spec.lane, namespace: spec.runtime.namespace, valuesPrinted: false }, + profile: providerProfileSummary(spec, profile, secrets), + activationPreflight, + stagedActivation: providerSecretStagedActivation(spec, [profile]), + next: { + observeRunners: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane} --dry-run`, + retryAfterManagerFence: `bun scripts/cli.ts agentrun provider-profile apply --node ${spec.nodeId} --lane ${spec.lane} --profile ${profile} --confirm`, + }, + valuesPrinted: false, + }; +} + +function providerProfileApplyProgress( + spec: AgentRunLaneSpec, + profile: string, + phase: string, + status: "running" | "succeeded" | "failed", + detail: Record = {}, +): void { + progressEvent("agentrun.provider-profile.apply.progress", { + node: spec.nodeId, + lane: spec.lane, + profile, + phase, + status, + ...detail, + valuesPrinted: false, + }); +} + async function providerProfileValidate(config: UniDeskConfig, options: ProviderProfileOptions): Promise> { const { configPath, spec, profile, secrets } = resolveProviderProfile(options); + const inspected = inspectProviderProfileSources(spec, secrets); + const source = providerProfileSourceSummary(inspected); + if (!source.ready) return providerProfileSourceUnavailable("validate", configPath, spec, profile, secrets, source); const prompt = options.prompt ?? `请只回复：${profile} upstream OK`; const live = await capture(config, spec.nodeKubeRoute, ["sh", "--", providerProfileLiveStatusScript(spec, secrets)]); const livePayload = captureJsonPayload(live); @@ -292,6 +577,7 @@ async function providerProfileValidate(config: UniDeskConfig, options: ProviderP configPath, target: compactAgentRunLaneStatusTarget(spec), profile: providerProfileSummary(spec, profile, secrets), + source, promptBytes: Buffer.byteLength(prompt, "utf8"), live: livePayload, note: "This command validates YAML/live profile wiring. User-entry Web validation still runs through HWLAB web-probe so it uses the same Workbench path.", @@ -300,6 +586,140 @@ async function providerProfileValidate(config: UniDeskConfig, options: ProviderP }; } +interface ProviderProfileInspectedSecret { + readonly secret: AgentRunLaneSecretSpec; + readonly source: LaneSecretSource | null; + readonly inspection: LaneSecretSourceInspection; +} + +function inspectProviderProfileSources(spec: AgentRunLaneSpec, secrets: readonly AgentRunLaneSecretSpec[]): ProviderProfileInspectedSecret[] { + const sources = new Map(collectLaneSecretSources(spec).map((source) => [source.id, source])); + return secrets.map((secret) => { + const source = sources.get(secret.id) ?? null; + if (source !== null) return { secret, source, inspection: inspectSecretSourceValue(spec, source) }; + return { + secret, + source, + inspection: { + ok: false, + sourceValue: null, + summary: { + present: false, + sourceFilePresent: false, + sourcePath: secret.sourceRef, + fingerprint: null, + valueBytes: null, + degradedReason: "lane-secret-source-declaration-missing", + message: `secret ${secret.id} has no resolved lane source`, + valuesPrinted: false, + }, + }, + }; + }); +} + +function providerProfileActivationTargets(items: readonly ProviderProfileInspectedSecret[]): ProviderSecretActivationTarget[] { + return items.map(({ secret, inspection }) => { + if (inspection.sourceValue === null) throw new Error(`provider profile source ${secret.id} is unavailable before activation preflight`); + return { + id: secret.id, + targetRef: secret.targetRef, + desiredEncodedFingerprint: providerSecretDesiredEncodedFingerprint(inspection.sourceValue.value), + }; + }); +} + +function providerProfileSourceSummary(items: readonly ProviderProfileInspectedSecret[]): { + ready: boolean; + count: number; + presentCount: number; + missingCount: number; + items: Record[]; + valuesPrinted: false; +} { + const summaries = items.map(({ secret, inspection }) => ({ + id: secret.id, + sourceRef: secret.sourceRef, + sourceMode: secret.sourceMode, + transform: secret.transform, + targetRef: secret.targetRef, + ...inspection.summary, + valuesPrinted: false, + })); + const presentCount = summaries.filter((item) => item.present === true).length; + return { + ready: items.length > 0 && items.every((item) => item.inspection.ok), + count: items.length, + presentCount, + missingCount: items.length - presentCount, + items: summaries, + valuesPrinted: false, + }; +} + +function providerProfileSourceUnavailable( + action: "status" | "apply" | "validate", + configPath: string, + spec: AgentRunLaneSpec, + profile: string, + secrets: readonly AgentRunLaneSecretSpec[], + source: ReturnType, +): Record { + return { + ok: false, + command: `agentrun provider-profile ${action}`, + mode: "profile-source-preflight", + status: "blocked", + degradedReason: "lane-secret-source-unavailable", + mutation: false, + blockedBeforeMutation: true, + configPath, + target: compactAgentRunLaneStatusTarget(spec), + profile: providerProfileSummary(spec, profile, secrets), + source, + next: { + recheck: `bun scripts/cli.ts agentrun provider-profile status --node ${spec.nodeId} --lane ${spec.lane} --profile ${profile} --raw`, + applyAfterOwnerProvision: `bun scripts/cli.ts agentrun provider-profile apply --node ${spec.nodeId} --lane ${spec.lane} --profile ${profile} --confirm`, + }, + valuesPrinted: false, + }; +} + +function providerProfileLiveComparison(items: readonly ProviderProfileInspectedSecret[], payload: Record): Record & { ready: boolean } { + const liveItems = Array.isArray(record(payload.secrets).items) + ? (record(payload.secrets).items as unknown[]).map(record) + : []; + const byTarget = new Map(liveItems.map((item) => [ + `${stringOrNull(item.namespace) ?? ""}/${stringOrNull(item.name) ?? ""}/${stringOrNull(item.key) ?? ""}`, + item, + ])); + const comparisons = items.map(({ secret, inspection }) => { + const target = secret.targetRef; + const live = byTarget.get(`${target.namespace}/${target.name}/${target.key}`) ?? {}; + const desiredEncodedFingerprint = inspection.sourceValue === null + ? null + : sha256Fingerprint(Buffer.from(inspection.sourceValue.value, "utf8").toString("base64")); + const liveEncodedFingerprint = stringOrNull(live.encodedFingerprint); + return { + id: secret.id, + targetRef: target, + desiredEncodedFingerprint, + liveEncodedFingerprint, + present: live.present === true, + keyPresent: live.keyPresent === true, + encodedFingerprintMatches: desiredEncodedFingerprint !== null && liveEncodedFingerprint === desiredEncodedFingerprint, + valuesPrinted: false, + }; + }); + return { + ready: comparisons.length > 0 && comparisons.every((item) => item.keyPresent && item.encodedFingerprintMatches), + count: comparisons.length, + matchingCount: comparisons.filter((item) => item.encodedFingerprintMatches).length, + items: comparisons, + valuesPrinted: false, + }; +} + function applyFailure(configPath: string, spec: AgentRunLaneSpec, profile: string, phase: string, payload: Record, captureResult: Awaited>): Record { return { ok: false, @@ -356,6 +776,10 @@ function codexConfigSummary(config: NonNullable ({ id: provider.id, @@ -367,6 +791,32 @@ function codexConfigSummary(config: NonNullable[] { + const targets: Record[] = []; + const seen = new Set(); + for (const secret of secrets) { + for (const provider of secret.codexConfig?.modelProviders ?? []) { + const parsed = new URL(provider.baseUrl); + const hostParts = parsed.hostname.split("."); + if (hostParts.length < 4 || hostParts[2] !== "svc" || hostParts[3] !== "cluster") continue; + const port = parsed.port.length > 0 ? Number(parsed.port) : parsed.protocol === "https:" ? 443 : 80; + const key = `${provider.id}/${hostParts[1]}/${hostParts[0]}/${port}`; + if (seen.has(key)) continue; + seen.add(key); + targets.push({ + providerId: provider.id, + baseUrl: provider.baseUrl, + host: parsed.hostname, + namespace: hostParts[1], + name: hostParts[0], + port, + valuesPrinted: false, + }); + } + } + return targets; +} + function currentManagerImageScript(spec: AgentRunLaneSpec): string { return [ "set -eu", @@ -386,7 +836,10 @@ function currentManagerImageScript(spec: AgentRunLaneSpec): string { } function providerProfileLiveStatusScript(spec: AgentRunLaneSpec, secrets: readonly AgentRunLaneSecretSpec[]): string { - const refs = Buffer.from(JSON.stringify(secrets.map((secret) => secret.targetRef)), "utf8").toString("base64"); + const refs = Buffer.from(JSON.stringify({ + secrets: secrets.map((secret) => secret.targetRef), + providerServices: providerRuntimeServiceTargets(secrets), + }), "utf8").toString("base64"); return [ "set -eu", `namespace=${shQuote(spec.runtime.namespace)}`, @@ -401,16 +854,64 @@ function providerProfileLiveStatusScript(spec: AgentRunLaneSpec, secrets: readon "const container = deployment.spec?.template?.spec?.containers?.[0] || {};", "const env = new Map((container.env || []).map((e) => [e.name, e.value || null]));", "const noProxy = env.get('AGENTRUN_RUNNER_NO_PROXY_EXTRA') || '';", - "const secretItems = refs.map((ref) => {", + "const secretItems = (refs.secrets || []).map((ref) => {", " const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });", " if (out.status !== 0) return { namespace: ref.namespace, name: ref.name, key: ref.key, present: false, keyPresent: false, valuesPrinted: false };", " const secret = JSON.parse(out.stdout);", " const raw = secret.data?.[ref.key] || '';", - " const bytes = raw ? Buffer.from(raw, 'base64').length : 0;", - " const fingerprint = raw ? 'sha256:' + require('node:crypto').createHash('sha256').update(Buffer.from(raw, 'base64')).digest('hex') : null;", - " return { namespace: ref.namespace, name: ref.name, key: ref.key, present: true, keyPresent: Boolean(raw), valueBytes: bytes, fingerprint, valuesPrinted: false };", + " const encodedBytes = raw ? Buffer.byteLength(raw, 'utf8') : 0;", + " const encodedFingerprint = raw ? 'sha256:' + require('node:crypto').createHash('sha256').update(raw).digest('hex') : null;", + " return { namespace: ref.namespace, name: ref.name, key: ref.key, present: true, keyPresent: Boolean(raw), encodedBytes, encodedFingerprint, valuesPrinted: false };", "});", - "console.log(JSON.stringify({ ok: secretItems.every((item) => item.keyPresent), manager: { deploymentExists: Boolean(deployment.metadata?.name), readyReplicas: deployment.status?.readyReplicas ?? null, replicas: deployment.status?.replicas ?? null, sourceCommit: env.get('AGENTRUN_SOURCE_COMMIT'), envIdentity: env.get('AGENTRUN_ENV_IDENTITY'), runnerEgressProxyUrl: env.get('AGENTRUN_RUNNER_EGRESS_PROXY_URL'), runnerNoProxyExtra: noProxy, noProxyContainsHyue: noProxy.includes('hyueapi.com'), noProxyContainsPikapython: noProxy.includes('api.pikapython.com') || noProxy.includes('.pikapython.com'), valuesPrinted: false }, secrets: { count: secretItems.length, ready: secretItems.every((item) => item.keyPresent), items: secretItems, valuesPrinted: false }, valuesPrinted: false }));", + "const serviceItems = (refs.providerServices || []).map((target) => {", + " const out = cp.spawnSync('kubectl', ['-n', target.namespace, 'get', 'service', target.name, '-o', 'json'], { encoding: 'utf8' });", + " if (out.status !== 0) return { ...target, present: false, portPresent: false, valuesPrinted: false };", + " const service = JSON.parse(out.stdout);", + " const ports = Array.isArray(service.spec?.ports) ? service.spec.ports : [];", + " return { ...target, present: true, portPresent: ports.some((item) => Number(item.port) === Number(target.port)), valuesPrinted: false };", + "});", + "const secretsReady = secretItems.every((item) => item.keyPresent);", + "const servicesReady = serviceItems.every((item) => item.present && item.portPresent);", + "console.log(JSON.stringify({ ok: secretsReady && servicesReady, manager: { deploymentExists: Boolean(deployment.metadata?.name), readyReplicas: deployment.status?.readyReplicas ?? null, replicas: deployment.status?.replicas ?? null, sourceCommit: env.get('AGENTRUN_SOURCE_COMMIT'), envIdentity: env.get('AGENTRUN_ENV_IDENTITY'), runnerEgressProxyUrl: env.get('AGENTRUN_RUNNER_EGRESS_PROXY_URL'), runnerNoProxyExtra: noProxy, noProxyContainsHyue: noProxy.includes('hyueapi.com'), noProxyContainsPikapython: noProxy.includes('api.pikapython.com') || noProxy.includes('.pikapython.com'), valuesPrinted: false }, secrets: { count: secretItems.length, ready: secretsReady, items: secretItems, valuesPrinted: false }, providerServices: { count: serviceItems.length, ready: servicesReady, items: serviceItems, valuesPrinted: false }, runtimeSecretValuesDecoded: false, valuesPrinted: false }));", + "NODE", + ].join("\n"); +} + +function providerProfileRuntimeStatusScript(spec: AgentRunLaneSpec, secrets: readonly AgentRunLaneSecretSpec[]): string { + const refs = Buffer.from(JSON.stringify({ + secrets: secrets.map((secret) => ({ id: secret.id, targetRef: secret.targetRef })), + providerServices: providerRuntimeServiceTargets(secrets), + }), "utf8").toString("base64"); + return [ + "set -eu", + `namespace=${shQuote(spec.runtime.namespace)}`, + `deployment=${shQuote(spec.runtime.managerDeployment)}`, + `refs_b64=${shQuote(refs)}`, + "kubectl -n \"$namespace\" get deployment \"$deployment\" -o json >/tmp/agentrun-provider-profile-runtime-deploy.json", + "REFS_B64=\"$refs_b64\" node <<'NODE'", + "const fs = require('node:fs');", + "const cp = require('node:child_process');", + "const refs = JSON.parse(Buffer.from(process.env.REFS_B64 || '', 'base64').toString('utf8'));", + "const deployment = JSON.parse(fs.readFileSync('/tmp/agentrun-provider-profile-runtime-deploy.json', 'utf8'));", + "const secretItems = (refs.secrets || []).map((entry) => {", + " const ref = entry.targetRef || {};", + " const out = cp.spawnSync('kubectl', ['-n', ref.namespace, 'get', 'secret', ref.name, '-o', 'json'], { encoding: 'utf8' });", + " if (out.status !== 0) return { id: entry.id || null, namespace: ref.namespace || null, name: ref.name || null, key: ref.key || null, present: false, keyPresent: false, valuesPrinted: false };", + " const secret = JSON.parse(out.stdout);", + " const data = secret.data && typeof secret.data === 'object' ? secret.data : {};", + " const keyPresent = Object.prototype.hasOwnProperty.call(data, ref.key) && typeof data[ref.key] === 'string' && data[ref.key].length > 0;", + " return { id: entry.id || null, namespace: ref.namespace || null, name: ref.name || null, key: ref.key || null, present: true, keyPresent, valuesPrinted: false };", + "});", + "const serviceItems = (refs.providerServices || []).map((target) => {", + " const out = cp.spawnSync('kubectl', ['-n', target.namespace, 'get', 'service', target.name, '-o', 'json'], { encoding: 'utf8' });", + " if (out.status !== 0) return { ...target, present: false, portPresent: false, valuesPrinted: false };", + " const service = JSON.parse(out.stdout);", + " const ports = Array.isArray(service.spec?.ports) ? service.spec.ports : [];", + " return { ...target, present: true, portPresent: ports.some((item) => Number(item.port) === Number(target.port)), valuesPrinted: false };", + "});", + "const secretsReady = secretItems.length > 0 && secretItems.every((item) => item.keyPresent);", + "const servicesReady = serviceItems.every((item) => item.present && item.portPresent);", + "console.log(JSON.stringify({ ok: secretsReady && servicesReady, manager: { deploymentExists: Boolean(deployment.metadata?.name), readyReplicas: deployment.status?.readyReplicas ?? null, replicas: deployment.status?.replicas ?? null, valuesPrinted: false }, secrets: { count: secretItems.length, ready: secretsReady, presentCount: secretItems.filter((item) => item.present).length, keyPresentCount: secretItems.filter((item) => item.keyPresent).length, items: secretItems, valuesPrinted: false }, providerServices: { count: serviceItems.length, ready: servicesReady, items: serviceItems, valuesPrinted: false }, ownerSourceInspected: false, runtimeSecretValuesDecoded: false, valuesPrinted: false }));", "NODE", ].join("\n"); } diff --git a/scripts/src/agentrun/provider-secret-activation-preflight.mjs b/scripts/src/agentrun/provider-secret-activation-preflight.mjs new file mode 100644 index 00000000..71c04821 --- /dev/null +++ b/scripts/src/agentrun/provider-secret-activation-preflight.mjs @@ -0,0 +1,367 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; + +function decodeJson(name) { + const raw = process.env[name] || ""; + return JSON.parse(Buffer.from(raw, "base64").toString("utf8")); +} + +function bounded(value, max = 320) { + if (typeof value !== "string" || value.length === 0) return null; + const oneLine = value.replace(/\s+/gu, " ").trim(); + return oneLine.length <= max ? oneLine : oneLine.slice(0, max); +} + +function runKubectl(args) { + return spawnSync("kubectl", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }); +} + +function labelsOf(value) { + return value?.metadata?.labels && typeof value.metadata.labels === "object" ? value.metadata.labels : {}; +} + +function annotationsOf(value) { + return value?.metadata?.annotations && typeof value.metadata.annotations === "object" ? value.metadata.annotations : {}; +} + +function jobNameOf(pod) { + const labels = labelsOf(pod); + const labelled = labels["batch.kubernetes.io/job-name"] || labels["job-name"]; + if (typeof labelled === "string" && labelled.length > 0) return labelled; + const owners = Array.isArray(pod?.metadata?.ownerReferences) ? pod.metadata.ownerReferences : []; + return owners.find((owner) => owner?.kind === "Job" && typeof owner?.name === "string")?.name || null; +} + +function matchesPrefix(name, prefixes) { + return typeof name === "string" && prefixes.some((prefix) => name === prefix || name.startsWith(`${prefix}-`)); +} + +function uniqueStrings(values) { + return [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))]; +} + +function podWaiting(pod) { + const statuses = [ + ...(Array.isArray(pod?.status?.initContainerStatuses) ? pod.status.initContainerStatuses : []), + ...(Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []), + ]; + const waiting = statuses.flatMap((status) => { + const state = status?.state?.waiting; + if (!state || typeof state.reason !== "string") return []; + return [{ source: "container", name: status?.name || null, reason: state.reason, message: bounded(state.message), valuesPrinted: false }]; + }); + const conditions = Array.isArray(pod?.status?.conditions) ? pod.status.conditions : []; + for (const condition of conditions) { + if (condition?.status !== "False" || typeof condition.reason !== "string") continue; + waiting.push({ source: "condition", name: condition.type || null, reason: condition.reason, message: bounded(condition.message), valuesPrinted: false }); + } + return waiting; +} + +function warningEvents(namespace, podName) { + const fieldSelector = `involvedObject.kind=Pod,involvedObject.name=${podName},type=Warning`; + const result = runKubectl(["-n", namespace, "get", "events", "--field-selector", fieldSelector, "-o", "json"]); + if (result.status !== 0) { + return { ok: false, items: [], error: bounded(result.stderr || result.stdout), valuesPrinted: false }; + } + try { + const payload = JSON.parse(result.stdout || "{}"); + const items = (Array.isArray(payload.items) ? payload.items : []).map((event) => ({ + source: "event", + name: event?.metadata?.name || null, + reason: event?.reason || null, + message: bounded(event?.message), + count: Number.isFinite(event?.count) ? event.count : null, + valuesPrinted: false, + })); + return { ok: true, items: items.slice(0, 8), omittedCount: Math.max(0, items.length - 8), valuesPrinted: false }; + } catch (error) { + return { ok: false, items: [], error: bounded(error instanceof Error ? error.message : String(error)), valuesPrinted: false }; + } +} + +function targetKey(namespace, name) { + return `${namespace}/${name}`; +} + +function inspectTargetSecrets(targets) { + const grouped = new Map(); + for (const target of targets) { + const ref = target?.targetRef || {}; + const key = targetKey(ref.namespace, ref.name); + const current = grouped.get(key) || { namespace: ref.namespace, name: ref.name, entries: [] }; + current.entries.push({ id: target.id, key: ref.key, desiredEncodedFingerprint: target.desiredEncodedFingerprint }); + grouped.set(key, current); + } + const results = []; + for (const group of grouped.values()) { + const desiredKeys = uniqueStrings(group.entries.map((entry) => entry.key)); + const conflictingKeys = desiredKeys.filter((key) => new Set(group.entries.filter((entry) => entry.key === key).map((entry) => entry.desiredEncodedFingerprint)).size !== 1); + const query = runKubectl(["-n", group.namespace, "get", "secret", group.name, "--ignore-not-found=true", "-o", "json"]); + if (query.status !== 0) { + results.push({ + namespace: group.namespace, + name: group.name, + ids: uniqueStrings(group.entries.map((entry) => entry.id)), + desiredKeys, + queryOk: false, + present: null, + presentDesiredKeys: [], + absentDesiredKeys: desiredKeys, + mutations: [], + error: bounded(query.stderr || query.stdout), + valuesPrinted: false, + }); + continue; + } + if (conflictingKeys.length > 0) { + results.push({ + namespace: group.namespace, + name: group.name, + ids: uniqueStrings(group.entries.map((entry) => entry.id)), + desiredKeys, + queryOk: false, + present: null, + presentDesiredKeys: [], + absentDesiredKeys: [], + mutations: [], + error: `conflicting desired encoded fingerprints for keys: ${conflictingKeys.join(",")}`, + valuesPrinted: false, + }); + continue; + } + const raw = (query.stdout || "").trim(); + let secret = null; + try { + secret = raw.length > 0 ? JSON.parse(raw) : null; + } catch (error) { + results.push({ + namespace: group.namespace, + name: group.name, + ids: uniqueStrings(group.entries.map((entry) => entry.id)), + desiredKeys, + queryOk: false, + present: null, + presentDesiredKeys: [], + absentDesiredKeys: desiredKeys, + mutations: [], + error: bounded(error instanceof Error ? error.message : String(error)), + valuesPrinted: false, + }); + continue; + } + const data = secret?.data && typeof secret.data === "object" ? secret.data : {}; + const presentDesiredKeys = desiredKeys.filter((key) => Object.prototype.hasOwnProperty.call(data, key) && typeof data[key] === "string" && data[key].length > 0); + const absentDesiredKeys = desiredKeys.filter((key) => !presentDesiredKeys.includes(key)); + const items = group.entries.map((entry) => { + const raw = typeof data[entry.key] === "string" && data[entry.key].length > 0 ? data[entry.key] : null; + const runtimeEncodedFingerprint = raw === null ? null : `sha256:${createHash("sha256").update(raw).digest("hex")}`; + const changeKind = secret === null + ? "create-secret" + : raw === null + ? "add-key" + : runtimeEncodedFingerprint === entry.desiredEncodedFingerprint + ? "no-op" + : "change-key"; + return { + id: entry.id, + key: entry.key, + desiredEncodedFingerprint: entry.desiredEncodedFingerprint, + runtimeEncodedFingerprint, + keyPresent: raw !== null, + changeKind, + valuesPrinted: false, + }; + }); + const mutations = items.filter((item) => item.changeKind !== "no-op"); + results.push({ + namespace: group.namespace, + name: group.name, + ids: uniqueStrings(group.entries.map((entry) => entry.id)), + desiredKeys, + queryOk: true, + present: secret !== null, + presentDesiredKeys, + absentDesiredKeys, + items, + mutations, + mutationCount: mutations.length, + noOp: mutations.length === 0, + mutationWouldCreateSecret: secret === null, + mutationWouldAddKeys: uniqueStrings(mutations.filter((item) => item.changeKind === "add-key").map((item) => item.key)), + mutationWouldChangeKeys: uniqueStrings(mutations.filter((item) => item.changeKind === "change-key").map((item) => item.key)), + valuesPrinted: false, + }); + } + return results; +} + +function selectedTarget(targetsBySecret, namespace, name) { + return targetsBySecret.get(targetKey(namespace, name)) || null; +} + +function affectedKeys(target, requestedKeys) { + if (target === null || target.queryOk !== true) return []; + const mutationKeys = uniqueStrings(target.mutations.map((item) => item.key)); + if (requestedKeys.length === 0) return mutationKeys; + return requestedKeys.filter((key) => mutationKeys.includes(key)); +} + +function podSecretDependencies(pod, targetsBySecret) { + const namespace = pod?.metadata?.namespace || process.env.NAMESPACE; + const dependencies = []; + const add = (source, name, optional, requestedKeys) => { + if (typeof name !== "string" || name.length === 0) return; + const target = selectedTarget(targetsBySecret, namespace, name); + const keys = affectedKeys(target, requestedKeys); + if (target === null || keys.length === 0) return; + const mutations = target.mutations.filter((item) => keys.includes(item.key)); + dependencies.push({ + source, + namespace, + name, + optional: optional === true, + requiredKeys: requestedKeys, + affectedDesiredKeys: keys, + mutationKinds: uniqueStrings(mutations.map((item) => item.changeKind)), + currentSecretPresent: target.present, + reason: "nonterminal-runner-provider-secret-mutation", + valuesPrinted: false, + }); + }; + for (const volume of Array.isArray(pod?.spec?.volumes) ? pod.spec.volumes : []) { + if (volume?.secret) { + const items = Array.isArray(volume.secret.items) ? volume.secret.items.map((item) => item?.key) : []; + add("volume-secret", volume.secret.secretName, volume.secret.optional, uniqueStrings(items)); + } + const sources = Array.isArray(volume?.projected?.sources) ? volume.projected.sources : []; + for (const source of sources) { + if (!source?.secret) continue; + const items = Array.isArray(source.secret.items) ? source.secret.items.map((item) => item?.key) : []; + add("projected-secret", source.secret.name, source.secret.optional, uniqueStrings(items)); + } + } + const containers = [ + ...(Array.isArray(pod?.spec?.initContainers) ? pod.spec.initContainers : []), + ...(Array.isArray(pod?.spec?.containers) ? pod.spec.containers : []), + ...(Array.isArray(pod?.spec?.ephemeralContainers) ? pod.spec.ephemeralContainers : []), + ]; + for (const container of containers) { + for (const env of Array.isArray(container?.env) ? container.env : []) { + const ref = env?.valueFrom?.secretKeyRef; + if (ref) add("env-secret-key-ref", ref.name, ref.optional, uniqueStrings([ref.key])); + } + for (const envFrom of Array.isArray(container?.envFrom) ? container.envFrom : []) { + const ref = envFrom?.secretRef; + if (ref) add("env-from-secret-ref", ref.name, ref.optional, []); + } + } + return dependencies; +} + +const namespace = process.env.NAMESPACE || ""; +const targets = decodeJson("TARGETS_JSON_B64"); +const matchLabels = decodeJson("MATCH_LABELS_JSON_B64"); +const jobNamePrefixes = decodeJson("JOB_NAME_PREFIXES_JSON_B64"); +const selector = Object.entries(matchLabels).map(([key, value]) => `${key}=${value}`).join(","); +const podArgs = ["-n", namespace, "get", "pods"]; +if (selector.length > 0) podArgs.push("-l", selector); +podArgs.push("-o", "json"); +const podQuery = runKubectl(podArgs); +let podPayload = { items: [] }; +let podQueryError = null; +if (podQuery.status === 0) { + try { + podPayload = JSON.parse(podQuery.stdout || "{}"); + } catch (error) { + podQueryError = bounded(error instanceof Error ? error.message : String(error)); + } +} else { + podQueryError = bounded(podQuery.stderr || podQuery.stdout); +} + +const targetSecrets = inspectTargetSecrets(targets); +const targetsBySecret = new Map(targetSecrets.map((target) => [targetKey(target.namespace, target.name), target])); +const allPods = Array.isArray(podPayload.items) ? podPayload.items : []; +const runnerPods = allPods.filter((pod) => { + const jobName = jobNameOf(pod); + return matchesPrefix(jobName, jobNamePrefixes); +}); +const nonTerminalPods = runnerPods.filter((pod) => !["Succeeded", "Failed"].includes(pod?.status?.phase) && typeof pod?.metadata?.deletionTimestamp !== "string"); +const activationRisks = []; +for (const pod of nonTerminalPods) { + const dependencies = podSecretDependencies(pod, targetsBySecret); + if (dependencies.length === 0) continue; + const events = warningEvents(namespace, pod?.metadata?.name || ""); + const waiting = [...podWaiting(pod), ...events.items]; + const annotations = annotationsOf(pod); + activationRisks.push({ + pod: pod?.metadata?.name || null, + jobName: jobNameOf(pod), + phase: pod?.status?.phase || null, + createdAt: pod?.metadata?.creationTimestamp || null, + runId: annotations["agentrun.pikastech.local/run-id"] || null, + commandId: annotations["agentrun.pikastech.local/command-id"] || null, + runnerJobId: annotations["agentrun.pikastech.local/runner-job-id"] || null, + runnerId: annotations["agentrun.pikastech.local/runner-id"] || null, + dependencies, + waiting: waiting.slice(0, 8), + waitingOmittedCount: Math.max(0, waiting.length - 8), + waitingReasons: uniqueStrings(waiting.map((item) => item.reason)).slice(0, 8), + warningEventsAvailable: events.ok, + classification: "nonterminal-runner-required-provider-secret-missing", + valuesPrinted: false, + }); +} + +const targetSecretObservationComplete = targets.length > 0 && targetSecrets.every((target) => target.queryOk === true); +const runnerObservationComplete = podQuery.status === 0 && podQueryError === null; +const allObservationsComplete = targetSecretObservationComplete && runnerObservationComplete; +const providerSecretMutationCount = targetSecrets.reduce((total, target) => total + Number(target.mutationCount || 0), 0); +const providerSecretDataNoOp = targetSecretObservationComplete && providerSecretMutationCount === 0; +const providerSecretMutationAuthorized = false; +const ok = providerSecretDataNoOp; +console.log(JSON.stringify({ + ok, + status: ok ? "ready-no-op" : "blocked", + reason: !targetSecretObservationComplete + ? "provider-secret-target-observation-incomplete" + : providerSecretMutationCount > 0 + ? "provider-secret-activation-fence-unavailable" + : null, + observedAt: new Date().toISOString(), + target: { namespace, labelSelector: selector || null, jobNamePrefixes, valuesPrinted: false }, + targetSecretCount: targetSecrets.length, + targetSecrets, + podObservation: { + queryOk: runnerObservationComplete, + error: podQueryError, + selectedPodCount: allPods.length, + runnerPodCount: runnerPods.length, + nonTerminalRunnerPodCount: nonTerminalPods.length, + valuesPrinted: false, + }, + activationRiskCount: activationRisks.length, + activationRisks: activationRisks.slice(0, 8), + activationRiskOmittedCount: Math.max(0, activationRisks.length - 8), + requiresManagerFence: providerSecretMutationCount > 0, + targetSecretObservationComplete, + runnerObservationComplete, + allObservationsComplete, + providerSecretMutationCount, + providerSecretDataNoOp, + providerSecretMutationAuthorized, + activationAuthority: { + owner: "agentrun-manager-provider-activation-fence", + atomicFenceAvailable: false, + admissionFenceAvailable: false, + secretCasAvailable: false, + noOpOnly: true, + diagnosticRunnerObservationOnly: true, + valuesPrinted: false, + }, + queryComplete: targetSecretObservationComplete, + mutation: false, + runtimeSecretValuesDecoded: false, + valuesPrinted: false, +})); diff --git a/scripts/src/agentrun/public-exposure.ts b/scripts/src/agentrun/public-exposure.ts index ab8072b6..7c3feb81 100644 --- a/scripts/src/agentrun/public-exposure.ts +++ b/scripts/src/agentrun/public-exposure.ts @@ -218,6 +218,7 @@ export function renderAgentRunControlPlaneActionSummary(result: Record row[1] !== "-"); const countKeys = [ "runnerJobCount", + "overLimitCount", "inactiveCandidateCount", + "managerRetentionCandidateCount", + "stalePendingObservationCount", "selectedRunnerJobCount", "protectedActiveRunnerCount", "remainingAfterSelection", "deletedRunnerJobCount", + "nonTerminalRunnerPodCount", "nonTerminalPodCount", - "forceActive", + "nextRunnerRequiresManagerRetention", ]; const countRows = countKeys .filter((key) => result[key] !== undefined) .map((key) => [key, displayValue(result[key])]); + const capacity = record(result.capacity); + const capacityBlocker = record(capacity.blocker); + const capacityCurrent = capacity.runnerJobCount ?? result.runnerJobCount; + const capacityLimit = capacity.maxRunners; + const capacityRows = [ + ["usage", capacityCurrent === undefined || capacityLimit === undefined ? "-" : `${String(capacityCurrent)}/${String(capacityLimit)}`], + ["status", displayValue(capacity.status ?? "-")], + ["availableSlots", displayValue(capacity.availableSlots ?? "-")], + ["blockerType", displayValue(capacityBlocker.type ?? "-")], + ["blockerCode", displayValue(capacityBlocker.code ?? "-")], + ].filter((row) => row[1] !== "-"); const lines = [ title, renderTable( @@ -271,7 +287,30 @@ export function renderAgentRunControlPlaneActionSummary(result: Record 0) lines.push("", "CAPACITY", renderTable(["FIELD", "VALUE"], capacityRows)); if (countRows.length > 0) lines.push("", renderTable(["COUNT", "VALUE"], countRows)); + const cleanupRows = (value: unknown): string[][] => (Array.isArray(value) ? value : []) + .slice(0, 4) + .map(record) + .map((item) => { + const drillDown = record(item.drillDown); + return [ + displayValue(item.runnerJobId ?? item.name ?? "-"), + Array.isArray(item.podPhases) ? item.podPhases.map(String).join(",") || "-" : "-", + Array.isArray(item.waitingReasons) ? item.waitingReasons.map(String).join(",") || "-" : "-", + displayValue(item.lastActiveAgeMs ?? "-"), + displayValue(item.classification ?? item.protectedReason ?? "-"), + displayValue(drillDown.runnerJob ?? "-"), + ]; + }); + const candidates = cleanupRows(result.candidates); + const selected = cleanupRows(result.selected); + if (candidates.length > 0) { + lines.push("", `CANDIDATES (showing ${String(candidates.length)}, omitted=${displayValue(result.candidateOmittedCount ?? 0)})`, renderTable(["RUNNERJOB", "POD PHASE", "WAITING", "AGE MS", "CLASSIFICATION", "DRILL-DOWN"], candidates)); + } + if (selected.length > 0) { + lines.push("", `SELECTED (showing ${String(selected.length)}, omitted=${displayValue(result.selectedOmittedCount ?? 0)})`, renderTable(["RUNNERJOB", "POD PHASE", "WAITING", "AGE MS", "CLASSIFICATION", "DRILL-DOWN"], selected)); + } const nextLines = renderNextObjectLines(record(result.next ?? result.followUp)); if (nextLines.length > 0) lines.push("", "NEXT", ...nextLines); lines.push("", "DETAIL", " use --full for capture/probe details; valuesPrinted=false"); diff --git a/scripts/src/agentrun/render.ts b/scripts/src/agentrun/render.ts index 231f06a7..976e2e20 100644 --- a/scripts/src/agentrun/render.ts +++ b/scripts/src/agentrun/render.ts @@ -148,10 +148,90 @@ export function renderResourceResult(command: string, raw: Record, options: AgentRunResourceOptions, ref: AgentRunResourceRef, text: string): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); - if (options.output === "json" || options.output === "yaml") return renderMachine(command, { kind: ref.kind, name: ref.name, resource: innerData(raw) }, options.output, raw.ok !== false); + if (options.output === "json" || options.output === "yaml") { + const resource = record(innerData(raw)); + const visibleResource = ref.kind === "runnerjob" && !options.full ? compactRunnerJobDescriptionPayload(resource) : resource; + return renderMachine(command, { kind: ref.kind, name: ref.name, resource: visibleResource }, options.output, raw.ok !== false); + } return renderedCliResult(raw.ok !== false, command, text); } +export function compactRunnerJobDescriptionPayload(value: Record): Record { + const observation = record(value.runtimeObservation); + const runner = record(observation.runner); + const target = record(observation.target); + const runId = value.runId ?? runner.runId ?? null; + const commandId = value.commandId ?? runner.commandId ?? null; + const runnerJobId = value.id ?? runner.runnerJobId ?? null; + const targetArgs = typeof target.node === "string" && typeof target.lane === "string" + ? ` --node ${target.node} --lane ${target.lane}` + : ""; + return { + id: runnerJobId, + runId, + commandId, + attemptId: value.attemptId ?? null, + runnerId: value.runnerId ?? runner.runnerId ?? null, + namespace: value.namespace ?? runner.namespace ?? target.namespace ?? null, + jobName: value.jobName ?? runner.name ?? null, + state: value.state ?? null, + managerPhase: value.managerPhase ?? value.phase ?? null, + terminalStatus: value.terminalStatus ?? null, + failureKind: value.failureKind ?? null, + image: value.image ?? null, + createdAt: value.createdAt ?? null, + startedAt: value.startedAt ?? null, + finishedAt: value.finishedAt ?? null, + runtimeObservation: { + ok: observation.ok ?? false, + reason: observation.reason ?? null, + observedAt: observation.observedAt ?? null, + target, + managerFacts: observation.managerFacts ?? null, + kubernetesFacts: observation.kubernetesFacts ?? null, + runner: Object.keys(runner).length === 0 ? null : { + name: runner.name ?? null, + runId: runner.runId ?? null, + commandId: runner.commandId ?? null, + runnerJobId: runner.runnerJobId ?? null, + runnerId: runner.runnerId ?? null, + createdAt: runner.createdAt ?? null, + lastActiveAt: runner.lastActiveAt ?? null, + lastActiveAgeMs: runner.lastActiveAgeMs ?? null, + runStatus: runner.runStatus ?? null, + runClaimedBy: runner.runClaimedBy ?? null, + commandState: runner.commandState ?? null, + heartbeatAt: runner.heartbeatAt ?? null, + heartbeatAgeMs: runner.heartbeatAgeMs ?? null, + state: runner.state ?? null, + jobStatus: runner.jobStatus ?? null, + podPhases: Array.isArray(runner.podPhases) ? runner.podPhases.slice(0, 4) : [], + waitingReasons: Array.isArray(runner.waitingReasons) ? runner.waitingReasons.slice(0, 8) : [], + podObservations: Array.isArray(runner.podObservations) ? runner.podObservations.slice(0, 2) : [], + identityComplete: runner.identityComplete === true, + managerFactsComplete: runner.managerFactsComplete === true, + podFactsComplete: runner.podFactsComplete === true, + cleanupAuthority: runner.cleanupAuthority ?? null, + cleanupDecisionComputed: runner.cleanupDecisionComputed === true, + safeCleanupCandidate: typeof runner.safeCleanupCandidate === "boolean" ? runner.safeCleanupCandidate : null, + protectedActive: typeof runner.protectedActive === "boolean" ? runner.protectedActive : null, + protectedReason: runner.protectedReason ?? null, + candidateKind: runner.candidateKind ?? null, + classification: runner.classification ?? null, + valuesPrinted: false, + }, + mutation: false, + valuesPrinted: false, + }, + next: { + events: typeof runId === "string" ? `bun scripts/cli.ts agentrun events run/${runId} --after-seq 0 --limit 100${targetArgs}` : null, + command: typeof runId === "string" && typeof commandId === "string" ? `bun scripts/cli.ts agentrun describe command/${commandId} --run ${runId}${targetArgs}` : null, + full: typeof runId === "string" && typeof runnerJobId === "string" ? `bun scripts/cli.ts agentrun describe runnerjob/${runnerJobId} --run ${runId}${targetArgs} --full -o json` : null, + }, + valuesPrinted: false, + }; +} + export function renderEventLike(command: string, raw: Record, options: AgentRunResourceOptions, kind: string, items: Record[], sourceName: string): RenderedCliResult { if (options.raw) return renderMachine(command, raw, "json", raw.ok !== false); const data = record(innerData(raw)); @@ -925,6 +1005,8 @@ export function renderGenericDescription(ref: AgentRunResourceRef, data: unknown ]; const identity = renderResourceIdentityLines(ref, value); if (identity.length > 0) lines.push("", ...identity); + const runnerObservation = renderRunnerJobObservationLines(ref, value); + if (runnerObservation.length > 0) lines.push("", "Runtime observation:", ...runnerObservation.map((line) => ` ${line}`)); const supervisor = renderSupervisorLines(ref, value); if (supervisor.length > 0) lines.push("", "Supervisor:", ...supervisor.map((line) => ` ${line}`)); const failure = renderFailureLines(value); @@ -941,6 +1023,31 @@ export function renderGenericDescription(ref: AgentRunResourceRef, data: unknown return lines.join("\n"); } +export function renderRunnerJobObservationLines(ref: AgentRunResourceRef, value: Record): string[] { + if (ref.kind !== "runnerjob") return []; + const observation = record(value.runtimeObservation); + const runner = record(observation.runner); + if (Object.keys(observation).length === 0) return []; + if (Object.keys(runner).length === 0) { + return [ + `Available: false`, + `Reason: ${displayValue(observation.reason ?? "runtime-observation-unavailable")}`, + ]; + } + const podPhases = Array.isArray(runner.podPhases) ? runner.podPhases.map(String).join(",") : "-"; + const waitingReasons = Array.isArray(runner.waitingReasons) ? runner.waitingReasons.map(String).join(",") : "-"; + return [ + `Manager phase: ${displayValue(value.managerPhase ?? "-")}`, + `Pod phase: ${podPhases || "-"}`, + `Waiting: ${waitingReasons || "-"}`, + `AgeMs: ${displayValue(runner.lastActiveAgeMs ?? "-")}`, + `Run: ${displayValue(runner.runStatus ?? "-")} claimedBy=${displayValue(runner.runClaimedBy ?? "-")}`, + `Command: ${displayValue(runner.commandState ?? "-")}`, + `Heartbeat: ${displayValue(runner.heartbeatAt ?? "-")} ageMs=${displayValue(runner.heartbeatAgeMs ?? "-")}`, + `Classification: ${displayValue(runner.classification ?? "-")} protected=${displayValue(runner.protectedActive ?? "-")}`, + ]; +} + export function renderResourceIdentityLines(ref: AgentRunResourceRef, value: Record): string[] { const supervisor = record(value.supervisor); const fields: [string, unknown][] = [ diff --git a/scripts/src/agentrun/resource-actions.ts b/scripts/src/agentrun/resource-actions.ts index f863fdfb..17783399 100644 --- a/scripts/src/agentrun/resource-actions.ts +++ b/scripts/src/agentrun/resource-actions.ts @@ -39,7 +39,8 @@ import { isHelpArg, renderAgentRunHelp } from "./entry"; import { agentRunExplain, arrayRecords, shortId } from "./options"; import { agentRunCancelAuthorityDisclosure, agentRunCancelCascadeScope, agentRunCancelFencingDisclosure, agentRunCancelRunnerAbortDisclosure, compactTaskDescriptionPayload, innerData, normalizeAipodSpecItems, normalizeAttemptResources, normalizeEventItems, normalizeLogItems, normalizeRunnerJobItems, normalizeSessionItems, normalizeSingleCommand, normalizeTaskItems, parseResourceKind, parseResourceRef, renderDescribe, renderEventLike, renderGenericDescription, renderMachine, renderMutationSummary, renderResourceResult, renderResultSummary, renderTaskDescription, renderedCliResult, requiredContext, rerunWithoutDryRun, resolveAgentRunCancelPolicyTarget, resourceApply, resourceCreate, resourceDispatch, resourceSessionPromptCommand, taskListState, unwrapTaskDetail } from "./render"; import { AgentRunRestError, renderAgentRunRestError, resolveAgentRunRestTarget, runAgentRunRestCommand, withAgentRunRestTarget } from "./rest-bridge"; -import { nonNegativeIntegerOrNull, record } from "./utils"; +import { runnerJobObservationScript, type RunnerJobObservationIdentity } from "./runner-observation"; +import { capture, captureJsonPayload, compactCapture, nonNegativeIntegerOrNull, record, stringOrNull } from "./utils"; export function agentRunGetKindHelp(kindRaw: string): string { const kind = parseResourceKind(kindRaw); @@ -264,7 +265,8 @@ export async function resourceDescribe(config: UniDeskConfig | null, command: st if (ref.kind === "runnerjob") { const runId = options.runId ?? requiredContext("runnerjob describe", "--run "); const result = await runAgentRunRestCommand(config, "runner", ["job-status", ref.name, "--run-id", runId]); - return renderDescribe(command, result, options, ref, renderGenericDescription(ref, innerData(result))); + const observed = await observeRunnerJob(config, options, ref.name, result); + return renderDescribe(command, observed, options, ref, renderGenericDescription(ref, innerData(observed))); } if (ref.kind === "session") { const result = await runAgentRunRestCommand(config, "sessions", ["show", ref.name, ...(options.full ? ["--full"] : [])]); @@ -277,6 +279,119 @@ export async function resourceDescribe(config: UniDeskConfig | null, command: st throw new Error(`unsupported describe kind: ${ref.kind}`); } +async function observeRunnerJob( + config: UniDeskConfig | null, + options: AgentRunResourceOptions, + runnerJobId: string, + managerResult: Record, +): Promise> { + const managerResource = record(innerData(managerResult)); + const target = resolveAgentRunRestTarget(config, options); + if (target === null) { + return { + ...managerResult, + data: { + ...managerResource, + managerPhase: managerResource.phase ?? null, + runtimeObservation: { + ok: false, + reason: "yaml-lane-target-unavailable", + mutation: false, + valuesPrinted: false, + }, + }, + }; + } + const identity = runnerJobObservationIdentity(managerResource, runnerJobId, target.spec.runtime.namespace); + if (identity === null) { + return { + ...managerResult, + ok: false, + data: { + ...managerResource, + managerPhase: managerResource.phase ?? null, + runtimeObservation: { + ok: false, + reason: "manager-runner-identity-incomplete-or-mismatched", + target: { + node: target.spec.nodeId, + lane: target.spec.lane, + namespace: target.spec.runtime.namespace, + }, + mutation: false, + valuesPrinted: false, + }, + }, + }; + } + const observationResult = await capture(target.config, target.spec.nodeKubeRoute, [ + "sh", + "--", + runnerJobObservationScript(target.spec, identity), + ]); + const observation = captureJsonPayload(observationResult); + const runner = record(observation.runner); + const podPhases = Array.isArray(runner.podPhases) ? runner.podPhases.map(String) : []; + const observedState = stringOrNull(runner.state) ?? (podPhases.includes("Pending") + ? "Pending" + : podPhases.includes("Running") + ? "Running" + : podPhases.includes("Failed") + ? "Failed" + : podPhases.includes("Succeeded") + ? "Succeeded" + : stringOrNull(runner.classification) ?? stringOrNull(managerResource.phase) ?? "Unknown"); + const observationFound = Object.keys(runner).length > 0; + const runtimeObservation = { + ok: observationResult.exitCode === 0 && observation.ok === true && observationFound, + reason: observationResult.exitCode === 0 ? observation.reason ?? (observationFound ? null : "runner-job-runtime-observation-not-found") : "runner-job-runtime-observation-failed", + observedAt: observation.observedAt ?? null, + target: { + node: target.spec.nodeId, + lane: target.spec.lane, + namespace: target.spec.runtime.namespace, + }, + managerFacts: observation.managerFacts ?? null, + kubernetesFacts: observation.kubernetesFacts ?? null, + runner: observationFound ? runner : null, + mutation: false, + valuesPrinted: false, + }; + return { + ...managerResult, + ok: managerResult.ok !== false && runtimeObservation.ok, + data: { + ...managerResource, + state: observedState, + managerPhase: managerResource.phase ?? null, + runtimeObservation, + valuesPrinted: false, + }, + runnerObservationBridge: { + mode: "yaml-lane-exact-runner-job-read-only", + configPath: target.configPath, + node: target.spec.nodeId, + lane: target.spec.lane, + capture: compactCapture(observationResult, { full: observationResult.exitCode !== 0, stdoutTailChars: 1200, stderrTailChars: 2000 }), + valuesPrinted: false, + }, + }; +} + +function runnerJobObservationIdentity(managerResource: Record, runnerJobId: string, namespace: string): RunnerJobObservationIdentity | null { + const identity = { + runnerJobId: stringOrNull(managerResource.id), + runId: stringOrNull(managerResource.runId), + commandId: stringOrNull(managerResource.commandId), + runnerId: stringOrNull(managerResource.runnerId), + namespace: stringOrNull(managerResource.namespace), + jobName: stringOrNull(managerResource.jobName), + }; + if (identity.runnerJobId !== runnerJobId || identity.namespace !== namespace) return null; + if (identity.runId === null || identity.commandId === null || identity.runnerId === null || identity.jobName === null) return null; + return identity as RunnerJobObservationIdentity; +} + export async function resourceEvents(config: UniDeskConfig | null, command: string, action: string | undefined, args: string[], options: AgentRunResourceOptions): Promise { const ref = parseResourceRef(action, args, "run"); const runId = ref.kind === "run" ? ref.name : options.runId ?? requiredContext("events", "--run "); diff --git a/scripts/src/agentrun/runner-observation.ts b/scripts/src/agentrun/runner-observation.ts new file mode 100644 index 00000000..c3828145 --- /dev/null +++ b/scripts/src/agentrun/runner-observation.ts @@ -0,0 +1,311 @@ +import type { AgentRunLaneSpec } from "../agentrun-lanes"; + +import { cleanupRunnersFactsNodeScript } from "./secrets"; +import { shQuote } from "./utils"; + +export interface RunnerJobObservationIdentity { + readonly runnerJobId: string; + readonly runId: string; + readonly commandId: string; + readonly runnerId: string; + readonly namespace: string; + readonly jobName: string; +} + +export function runnerJobObservationScript(spec: AgentRunLaneSpec, identity: RunnerJobObservationIdentity): string { + const identityJsonB64 = Buffer.from(JSON.stringify(identity), "utf8").toString("base64"); + return [ + "set -eu", + `namespace=${shQuote(spec.runtime.namespace)}`, + `manager_deployment=${shQuote(spec.runtime.managerDeployment)}`, + `runner_job_id=${shQuote(identity.runnerJobId)}`, + `job_name=${shQuote(identity.jobName)}`, + `identity_json_b64=${shQuote(identityJsonB64)}`, + "tmp_dir=$(mktemp -d)", + "trap 'rm -rf \"$tmp_dir\"' EXIT", + "facts_exit=0", + "set +e", + "kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" RETENTION_RUNNER_JOB_ID=\"$runner_job_id\" sh -lc 'cat >/tmp/agentrun-runner-observation.mjs && bun /tmp/agentrun-runner-observation.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'", + cleanupRunnersFactsNodeScript(), + "NODE", + "facts_exit=$?", + "set -e", + "if [ \"$facts_exit\" -ne 0 ]; then printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"; fi", + "job_exit=0", + "set +e", + "kubectl -n \"$namespace\" get job \"$job_name\" -o json > \"$tmp_dir/job.json\" 2> \"$tmp_dir/job.err\"", + "job_exit=$?", + "set -e", + "if [ \"$job_exit\" -ne 0 ]; then printf '%s\\n' '{}' > \"$tmp_dir/job.json\"; fi", + "pods_exit=0", + "set +e", + "kubectl -n \"$namespace\" get pods -l \"job-name=$job_name\" -o json > \"$tmp_dir/pods.json\" 2> \"$tmp_dir/pods.err\"", + "pods_exit=$?", + "set -e", + "if [ \"$pods_exit\" -ne 0 ]; then printf '%s\\n' '{\"items\":[]}' > \"$tmp_dir/pods.json\"; fi", + "node -e 'const p=JSON.parse(require(\"node:fs\").readFileSync(process.argv[1],\"utf8\")); for(const x of Array.isArray(p.items)?p.items.slice(0,8):[]){const n=x?.metadata?.name;if(typeof n===\"string\"&&n)console.log(n)}' \"$tmp_dir/pods.json\" > \"$tmp_dir/pod-names.txt\"", + "events_failed=0", + "event_index=0", + "while IFS= read -r pod_name; do", + " [ -n \"$pod_name\" ] || continue", + " event_file=\"$tmp_dir/event-$event_index.json\"", + " set +e", + " kubectl -n \"$namespace\" get events --field-selector \"involvedObject.kind=Pod,involvedObject.name=$pod_name,type=Warning\" -o json > \"$event_file\" 2> \"$tmp_dir/event-$event_index.err\"", + " event_exit=$?", + " set -e", + " if [ \"$event_exit\" -ne 0 ]; then events_failed=$((events_failed + 1)); printf '%s\\n' '{\"items\":[]}' > \"$event_file\"; fi", + " event_index=$((event_index + 1))", + "done < \"$tmp_dir/pod-names.txt\"", + "IDENTITY_JSON_B64=\"$identity_json_b64\" FACTS_EXIT=\"$facts_exit\" JOB_EXIT=\"$job_exit\" PODS_EXIT=\"$pods_exit\" EVENTS_FAILED=\"$events_failed\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE'", + runnerJobObservationNodeScript(), + "NODE", + ].join("\n"); +} + +function runnerJobObservationNodeScript(): string { + return String.raw` +const fs = require("node:fs"); +const path = require("node:path"); +const tmp = process.env.TMP_DIR; +const expected = JSON.parse(Buffer.from(process.env.IDENTITY_JSON_B64, "base64").toString("utf8")); +const factsExit = Number(process.env.FACTS_EXIT); +const jobExit = Number(process.env.JOB_EXIT); +const podsExit = Number(process.env.PODS_EXIT); +const eventsFailed = Number(process.env.EVENTS_FAILED); + +function readJson(name) { + return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8")); +} + +function iso(value) { + if (typeof value !== "string" || value.length === 0) return null; + const ms = Date.parse(value); + return Number.isFinite(ms) ? new Date(ms).toISOString() : null; +} + +function ageMs(value) { + const ms = Date.parse(value || ""); + return Number.isFinite(ms) ? Math.max(0, Date.now() - ms) : null; +} + +function bounded(value, max = 320) { + if (typeof value !== "string" || value.length === 0) return null; + const oneLine = value.replace(/\s+/g, " ").trim(); + return oneLine.length <= max ? oneLine : oneLine.slice(0, max); +} + +function annotationsOf(value) { + return value?.metadata?.annotations && typeof value.metadata.annotations === "object" ? value.metadata.annotations : {}; +} + +function stateKind(state) { + if (state?.running) return "running"; + if (state?.terminated) return "terminated"; + if (state?.waiting) return "waiting"; + return "unknown"; +} + +function waitingFromStatus(status, source) { + const waiting = status?.state?.waiting; + if (!waiting || typeof waiting.reason !== "string" || waiting.reason.length === 0) return []; + return [{ + source, + name: typeof status.name === "string" ? status.name : null, + reason: waiting.reason, + message: bounded(waiting.message), + valuesPrinted: false, + }]; +} + +function warningFromEvent(event) { + if (event?.type !== "Warning" || typeof event?.reason !== "string" || event.reason.length === 0) return null; + return { + source: "event", + name: typeof event?.metadata?.name === "string" ? event.metadata.name : null, + reason: event.reason, + message: bounded(event.message), + count: Number.isFinite(event.count) ? event.count : null, + lastObservedAt: iso(event?.series?.lastObservedTime || event?.eventTime || event?.lastTimestamp || event?.firstTimestamp), + valuesPrinted: false, + }; +} + +function identityMatches(value) { + const annotations = annotationsOf(value); + const optionalRunnerJobId = annotations["agentrun.pikastech.local/runner-job-id"]; + return annotations["agentrun.pikastech.local/run-id"] === expected.runId + && annotations["agentrun.pikastech.local/command-id"] === expected.commandId + && annotations["agentrun.pikastech.local/runner-id"] === expected.runnerId + && (optionalRunnerJobId === undefined || optionalRunnerJobId === expected.runnerJobId); +} + +const facts = readJson("runner-facts.json"); +const factItems = Array.isArray(facts.items) ? facts.items : []; +const fact = factItems.find((item) => item?.id === expected.runnerJobId) || null; +const job = readJson("job.json"); +const podsPayload = readJson("pods.json"); +const pods = Array.isArray(podsPayload.items) ? podsPayload.items.slice(0, 8) : []; +const eventDocuments = fs.readdirSync(tmp) + .filter((name) => /^event-\d+\.json$/u.test(name)) + .sort() + .map(readJson); +const warningEvents = eventDocuments.flatMap((value) => Array.isArray(value.items) ? value.items : []); +const warningsByPod = new Map(); +for (const event of warningEvents) { + const podName = event?.involvedObject?.kind === "Pod" && typeof event?.involvedObject?.name === "string" + ? event.involvedObject.name + : null; + if (podName === null) continue; + const items = warningsByPod.get(podName) || []; + items.push(event); + warningsByPod.set(podName, items); +} + +const podObservations = pods.map((pod) => { + const statuses = Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []; + const initStatuses = Array.isArray(pod?.status?.initContainerStatuses) ? pod.status.initContainerStatuses : []; + const runner = statuses.find((status) => status?.name === "runner") || null; + const conditions = Array.isArray(pod?.status?.conditions) ? pod.status.conditions : []; + const eventWarnings = (warningsByPod.get(pod?.metadata?.name || "") || []).map(warningFromEvent).filter(Boolean); + const waiting = [ + ...statuses.flatMap((status) => waitingFromStatus(status, "container")), + ...initStatuses.flatMap((status) => waitingFromStatus(status, "init-container")), + ...conditions.flatMap((condition) => condition?.status === "False" && typeof condition?.reason === "string" && condition.reason.length > 0 + ? [{ source: "condition", name: condition.type || null, reason: condition.reason, message: bounded(condition.message), valuesPrinted: false }] + : []), + ...eventWarnings, + ]; + const ready = conditions.some((condition) => condition?.type === "Ready" && condition?.status === "True"); + return { + name: pod?.metadata?.name || null, + uid: pod?.metadata?.uid || null, + resourceVersion: pod?.metadata?.resourceVersion || null, + phase: pod?.status?.phase || null, + createdAt: iso(pod?.metadata?.creationTimestamp), + ageMs: ageMs(pod?.metadata?.creationTimestamp), + ready, + identityMatches: identityMatches(pod), + runnerContainerState: stateKind(runner?.state || {}), + runnerContainerReady: runner?.ready === true, + runnerContainerRestartCount: Number(runner?.restartCount || 0), + waiting: waiting.slice(0, 8), + waitingOmittedCount: Math.max(0, waiting.length - 8), + eventFactsAvailable: eventsFailed === 0, + valuesPrinted: false, + }; +}); + +const managerIdentityMatches = fact !== null + && fact.runId === expected.runId + && fact.commandId === expected.commandId + && fact.runnerId === expected.runnerId + && fact.namespace === expected.namespace + && fact.jobName === expected.jobName; +const jobPresent = jobExit === 0 && job?.metadata?.name === expected.jobName; +const jobIdentityMatches = jobPresent && identityMatches(job); +const podIdentityMatches = podObservations.length > 0 && podObservations.every((pod) => pod.identityMatches === true); +const podPhases = podObservations.map((pod) => pod.phase).filter((value) => typeof value === "string"); +const waitingReasons = [...new Set(podObservations.flatMap((pod) => pod.waiting.map((item) => item.reason)).filter(Boolean))].slice(0, 8); +const state = podPhases.includes("Pending") + ? "Pending" + : podPhases.includes("Running") + ? "Running" + : podPhases.includes("Failed") + ? "Failed" + : podPhases.includes("Succeeded") + ? "Succeeded" + : jobPresent + ? "Unknown" + : "NotFound"; +const classification = state === "Pending" && waitingReasons.includes("FailedMount") + ? "pending-failed-mount" + : state.toLowerCase(); +const warningEventsAvailable = podObservations.length > 0 && eventsFailed === 0; +const ok = factsExit === 0 + && facts.ok === true + && factItems.length === 1 + && managerIdentityMatches + && jobPresent + && jobIdentityMatches + && podsExit === 0 + && podIdentityMatches + && warningEventsAvailable; +const reason = ok + ? null + : factsExit !== 0 || facts.ok !== true + ? "manager-runner-facts-query-failed" + : factItems.length !== 1 || !managerIdentityMatches + ? "manager-runner-identity-mismatch" + : !jobPresent + ? "runner-job-not-found" + : !jobIdentityMatches + ? "runner-job-identity-mismatch" + : podsExit !== 0 || !podIdentityMatches + ? "runner-pod-observation-incomplete" + : "runner-warning-event-observation-incomplete"; + +console.log(JSON.stringify({ + ok, + reason, + observedAt: new Date().toISOString(), + target: { namespace: process.env.NAMESPACE }, + managerFacts: { + ok: factsExit === 0 && facts.ok === true && factItems.length === 1, + queryMode: facts.queryMode || null, + runnerJobId: expected.runnerJobId, + itemCount: factItems.length, + identityMatches: managerIdentityMatches, + valuesPrinted: false, + }, + kubernetesFacts: { + jobQuery: "exact-name", + podQuery: "exact-job-name-label", + warningEventQuery: "exact-pod-name", + jobPresent, + podCount: podObservations.length, + warningEventCount: warningEvents.length, + warningEventsAvailable, + valuesPrinted: false, + }, + runner: { + name: expected.jobName, + namespace: expected.namespace, + runId: expected.runId, + commandId: expected.commandId, + runnerJobId: expected.runnerJobId, + runnerId: expected.runnerId, + resourceUid: job?.metadata?.uid || null, + resourceVersion: job?.metadata?.resourceVersion || null, + createdAt: iso(job?.metadata?.creationTimestamp), + lastActiveAt: fact?.runnerHeartbeatAt || fact?.runUpdatedAt || fact?.runnerJobUpdatedAt || null, + lastActiveAgeMs: ageMs(fact?.runnerHeartbeatAt || fact?.runUpdatedAt || fact?.runnerJobUpdatedAt), + runStatus: fact?.runStatus || null, + runClaimedBy: fact?.runClaimedBy || null, + commandState: fact?.commandState || null, + heartbeatAt: fact?.runnerHeartbeatAt || null, + heartbeatAgeMs: ageMs(fact?.runnerHeartbeatAt), + jobStatus: { + active: Number(job?.status?.active || 0), + succeeded: Number(job?.status?.succeeded || 0), + failed: Number(job?.status?.failed || 0), + ready: Number(job?.status?.ready || 0), + }, + podPhases, + waitingReasons, + podObservations: podObservations.slice(0, 4), + podObservationOmittedCount: Math.max(0, podObservations.length - 4), + identityComplete: managerIdentityMatches && jobIdentityMatches && podIdentityMatches, + managerFactsComplete: factsExit === 0 && facts.ok === true && factItems.length === 1, + podFactsComplete: podsExit === 0 && podIdentityMatches && warningEventsAvailable, + state, + classification, + cleanupAuthority: "agentrun-manager-pre-create-retention", + cleanupDecisionComputed: false, + mutation: false, + valuesPrinted: false, + }, + mutation: false, + valuesPrinted: false, +})); +`; +} diff --git a/scripts/src/agentrun/secrets.ts b/scripts/src/agentrun/secrets.ts index dab4ef07..524bb453 100644 --- a/scripts/src/agentrun/secrets.ts +++ b/scripts/src/agentrun/secrets.ts @@ -706,25 +706,84 @@ export function yamlLaneGitMirrorStatusScript(spec: AgentRunLaneSpec): string { export type LaneSecretSource = { id: string; sourceRef: string; - sourceMode: "env" | "file" | "codex-config"; + sourceMode: "env" | "file" | "codex-config" | "json"; sourceKey: string | null; sourceFormat?: "env" | "raw-token" | null; codexConfig?: AgentRunCodexConfigSpec | null; + jsonValue?: unknown; targetRef: { namespace: string; name: string; key: string }; - transform?: "local-postgres-database" | "local-postgres-user" | "local-postgres-database-url"; + transform?: "codex-auth-json-openai-api-key" | "local-postgres-database" | "local-postgres-user" | "local-postgres-database-url" | null; }; +export interface LaneSecretSourceInspection { + readonly ok: boolean; + readonly sourceValue: ReturnType | null; + readonly summary: { + readonly present: boolean; + readonly sourceFilePresent: boolean; + readonly sourcePath: string; + readonly fingerprint: string | null; + readonly valueBytes: number | null; + readonly degradedReason: string | null; + readonly message: string | null; + readonly valuesPrinted: false; + }; +} + +export function inspectSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource): LaneSecretSourceInspection { + let sourcePath: string | null = null; + let sourceFilePresent = isRenderedLaneSecretSource(source); + try { + sourcePath = resolveLaneSecretSourcePath(spec, source); + sourceFilePresent = isRenderedLaneSecretSource(source) || sourcePath !== null && existsSync(sourcePath); + const sourceValue = readSecretSourceValue(spec, source); + return { + ok: true, + sourceValue, + summary: { + present: true, + sourceFilePresent, + sourcePath: sourceValue.redactedPath, + fingerprint: sourceValue.fingerprint, + valueBytes: sourceValue.valueBytes, + degradedReason: null, + message: null, + valuesPrinted: false, + }, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + ok: false, + sourceValue: null, + summary: { + present: false, + sourceFilePresent, + sourcePath: isRenderedLaneSecretSource(source) + ? source.sourceRef + : redactAbsoluteSecretPath(sourcePath ?? source.sourceRef), + fingerprint: null, + valueBytes: null, + degradedReason: classifySecretSourceFailure(message), + message: message.slice(0, 500), + valuesPrinted: false, + }, + }; + } +} + export function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource): { redactedPath: string; value: string; valueBytes: number; fingerprint: string } { const { sourceRef } = source; if (sourceRef.includes("..")) throw new Error(`secret sourceRef must not contain ..: ${sourceRef}`); - const sourcePath = source.sourceMode === "codex-config" ? null : sourceRef.startsWith("/") - ? sourceRef - : join(resolveSecretSourceRoot(spec), ...sourceRef.split("/")); + const sourcePath = resolveLaneSecretSourcePath(spec, source); if (sourcePath !== null && !existsSync(sourcePath)) throw new Error(`secret source ${sourceRef} is missing`); let value: string; if (source.sourceMode === "codex-config") { if (source.codexConfig === undefined || source.codexConfig === null) throw new Error(`secret source ${sourceRef} is missing codexConfig`); value = renderCodexConfigToml(source.codexConfig); + } else if (source.sourceMode === "json") { + if (source.jsonValue === undefined) throw new Error(`secret source ${sourceRef} is missing jsonValue`); + value = `${JSON.stringify(source.jsonValue, null, 2)}\n`; } else if (source.sourceMode === "file") { if (sourcePath === null) throw new Error(`secret source ${sourceRef} has no source path`); value = readFileSync(sourcePath, "utf8"); @@ -739,13 +798,33 @@ export function readSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecret if (value.length === 0) throw new Error(`secret source ${sourceRef} is empty`); value = transformSecretSourceValue(spec, source, value); return { - redactedPath: source.sourceMode === "codex-config" ? sourceRef : redactAbsoluteSecretPath(sourcePath ?? sourceRef), + redactedPath: isRenderedLaneSecretSource(source) ? sourceRef : redactAbsoluteSecretPath(sourcePath ?? sourceRef), value, valueBytes: Buffer.byteLength(value, "utf8"), fingerprint: sha256Fingerprint(value), }; } +function resolveLaneSecretSourcePath(spec: AgentRunLaneSpec, source: LaneSecretSource): string | null { + if (source.sourceRef.includes("..")) throw new Error(`secret sourceRef must not contain ..: ${source.sourceRef}`); + if (isRenderedLaneSecretSource(source)) return null; + return source.sourceRef.startsWith("/") + ? source.sourceRef + : join(resolveSecretSourceRoot(spec), ...source.sourceRef.split("/")); +} + +function isRenderedLaneSecretSource(source: LaneSecretSource): boolean { + return source.sourceMode === "codex-config" || source.sourceMode === "json"; +} + +function classifySecretSourceFailure(message: string): string { + if (message.includes("missing required key")) return "secret-source-key-missing"; + if (message.includes(" is missing")) return "secret-source-file-missing"; + if (message.includes(" is empty")) return "secret-source-empty"; + if (message.includes("must not contain ..")) return "secret-source-ref-invalid"; + return "secret-source-read-failed"; +} + function renderCodexConfigToml(config: AgentRunCodexConfigSpec): string { const lines = [ `model_provider = ${tomlString(config.modelProvider)}`, @@ -756,6 +835,10 @@ function renderCodexConfigToml(config: AgentRunCodexConfigSpec): string { ...(config.networkAccess === null ? [] : [`network_access = ${tomlString(config.networkAccess)}`]), ...(config.windowsWslSetupAcknowledged === null ? [] : [`windows_wsl_setup_acknowledged = ${tomlBoolean(config.windowsWslSetupAcknowledged)}`]), ...(config.serviceTier === null ? [] : [`service_tier = ${tomlString(config.serviceTier)}`]), + ...(config.modelContextWindow === null ? [] : [`model_context_window = ${config.modelContextWindow}`]), + ...(config.modelAutoCompactTokenLimit === null ? [] : [`model_auto_compact_token_limit = ${config.modelAutoCompactTokenLimit}`]), + ...(config.modelCatalogJson === null ? [] : [`model_catalog_json = ${tomlString(config.modelCatalogJson)}`]), + ...(config.approvalsReviewer === null ? [] : [`approvals_reviewer = ${tomlString(config.approvalsReviewer)}`]), "", ]; for (const provider of config.modelProviders) { @@ -781,6 +864,9 @@ function renderCodexConfigToml(config: AgentRunCodexConfigSpec): string { for (const [key, value] of Object.entries(config.tuiModelAvailabilityNux)) lines.push(`${tomlKeySegment(key)} = ${value}`); lines.push(""); } + if (config.noticeHideFullAccessWarning !== null) { + lines.push("[notice]", `hide_full_access_warning = ${tomlBoolean(config.noticeHideFullAccessWarning)}`, ""); + } return `${lines.join("\n").trimEnd()}\n`; } @@ -797,7 +883,10 @@ function tomlKeySegment(value: string): string { } function transformSecretSourceValue(spec: AgentRunLaneSpec, source: LaneSecretSource, rawValue: string): string { - if (source.transform === undefined) return rawValue; + if (source.transform === undefined || source.transform === null) return rawValue; + if (source.transform === "codex-auth-json-openai-api-key") { + return `${JSON.stringify({ OPENAI_API_KEY: rawValue }, null, 2)}\n`; + } const localPostgres = spec.deployment.localPostgres; if (!localPostgres.enabled || localPostgres.serviceName === null || localPostgres.database === null || localPostgres.user === null || localPostgres.port === null) { throw new Error(`secret source ${source.id} requires enabled localPostgres with database/user/port`); @@ -900,24 +989,36 @@ export function collectLaneSecretSources(spec: AgentRunLaneSpec): LaneSecretSour sourceMode: secret.sourceMode, sourceKey: secret.sourceKey, sourceFormat: secret.sourceFormat, + transform: secret.transform, codexConfig: secret.codexConfig, + jsonValue: secret.jsonValue, targetRef: secret.targetRef, }); } - const seen = new Set(); - return result.filter((item) => { - const key = `${item.targetRef.namespace}/${item.targetRef.name}/${item.targetRef.key}`; - if (seen.has(key)) return false; - seen.add(key); - return true; - }); + const seenIds = new Set(); + const seenTargetRefs = new Set(); + for (const item of result) { + if (seenIds.has(item.id)) throw new Error(`duplicate rendered lane Secret id: ${item.id}`); + seenIds.add(item.id); + const targetRef = `${item.targetRef.namespace}/${item.targetRef.name}/${item.targetRef.key}`; + if (seenTargetRefs.has(targetRef)) throw new Error(`duplicate rendered lane Secret targetRef: ${targetRef}`); + seenTargetRefs.add(targetRef); + } + return result; } export function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ targetRef: { namespace: string; name: string; key: string }; value: string }>): string { - const encoded = Buffer.from(JSON.stringify(values.map((item) => ({ - targetRef: item.targetRef, - valueBase64: Buffer.from(item.value, "utf8").toString("base64"), - }))), "utf8").toString("base64"); + const encoded = Buffer.from(JSON.stringify(values.map((item) => { + const valueBase64 = Buffer.from(item.value, "utf8").toString("base64"); + return { + targetRef: item.targetRef, + valueBase64, + sourceValueBytes: Buffer.byteLength(item.value, "utf8"), + sourceFingerprint: sha256Fingerprint(item.value), + encodedBytes: Buffer.byteLength(valueBase64, "utf8"), + encodedFingerprint: sha256Fingerprint(valueBase64), + }; + })), "utf8").toString("base64"); return [ "set -eu", `payload_b64=${shQuote(encoded)}`, @@ -926,7 +1027,7 @@ export function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ target "payload_json=\"$tmp_dir/payload.json\"", "printf '%s' \"$payload_b64\" | base64 -d > \"$payload_json\"", "PAYLOAD_JSON=\"$payload_json\" TMP_DIR=\"$tmp_dir\" python3 - <<'PY'", - "import base64, hashlib, json, os, subprocess", + "import hashlib, json, os, subprocess", "def run(argv, input_text=None):", " out = subprocess.run(argv, input=input_text, text=True, capture_output=True)", " if out.returncode != 0:", @@ -940,7 +1041,7 @@ export function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ target "for item in items:", " ref = item.get('targetRef') or {}", " key = (ref.get('namespace'), ref.get('name'))", - " groups.setdefault(key, {'namespace': ref.get('namespace'), 'name': ref.get('name'), 'items': []})['items'].append({'key': ref.get('key'), 'valueBase64': item.get('valueBase64') or ''})", + " groups.setdefault(key, {'namespace': ref.get('namespace'), 'name': ref.get('name'), 'items': []})['items'].append({'key': ref.get('key'), 'valueBase64': item.get('valueBase64') or '', 'sourceValueBytes': item.get('sourceValueBytes'), 'sourceFingerprint': item.get('sourceFingerprint'), 'desiredEncodedBytes': item.get('encodedBytes'), 'desiredEncodedFingerprint': item.get('encodedFingerprint')})", "results = []", "for group in groups.values():", " namespace = group['namespace']", @@ -959,9 +1060,11 @@ export function secretSyncScript(_spec: AgentRunLaneSpec, values: Array<{ target " data = fetched.get('data') or {}", " for entry in group['items']:", " raw = data.get(entry['key']) or ''", - " decoded = base64.b64decode(raw.encode('utf-8')) if raw else b''", - " results.append({'namespace': namespace, 'secret': name, 'key': entry['key'], 'ok': bool(raw), 'valueBytes': len(decoded), 'fingerprint': ('sha256:' + hashlib.sha256(decoded).hexdigest()) if raw else None, 'valuesPrinted': False})", - "print(json.dumps({'ok': all(item['ok'] for item in results), 'secretCount': len(results), 'items': results, 'valuesPrinted': False}, ensure_ascii=False))", + " encoded_bytes = len(raw.encode('utf-8')) if raw else 0", + " encoded_fingerprint = ('sha256:' + hashlib.sha256(raw.encode('utf-8')).hexdigest()) if raw else None", + " encoded_matches = bool(raw) and encoded_fingerprint == entry['desiredEncodedFingerprint'] and encoded_bytes == entry['desiredEncodedBytes']", + " results.append({'namespace': namespace, 'secret': name, 'key': entry['key'], 'present': bool(raw), 'ok': encoded_matches, 'sourceValueBytes': entry['sourceValueBytes'], 'sourceFingerprint': entry['sourceFingerprint'], 'encodedBytes': encoded_bytes, 'encodedFingerprint': encoded_fingerprint, 'encodedFingerprintMatches': encoded_matches, 'valuesPrinted': False})", + "print(json.dumps({'ok': all(item['ok'] for item in results), 'secretCount': len(results), 'items': results, 'runtimeFingerprintMode': 'kubernetes-secret-data-base64-sha256', 'runtimeSecretValuesDecoded': False, 'valuesPrinted': False}, ensure_ascii=False))", "PY", ].join("\n"); } @@ -1071,22 +1174,38 @@ function iso(value) { try { const { Pool } = await import("pg"); const namespace = process.env.RETENTION_NAMESPACE || process.env.AGENTRUN_RUNTIME_NAMESPACE || ""; + const runnerJobId = process.env.RETENTION_RUNNER_JOB_ID || null; const pool = new Pool({ connectionString: process.env.DATABASE_URL }); const sql = [ "SELECT", " j.id, j.run_id, j.command_id, j.attempt_id, j.runner_id, j.namespace, j.job_name, j.created_at, j.updated_at,", " r.registered_at AS runner_registered_at, r.heartbeat_at AS runner_heartbeat_at,", - " rr.status AS run_status, rr.terminal_status AS run_terminal_status, rr.failure_kind AS run_failure_kind, rr.updated_at AS run_updated_at", + " rr.status AS run_status, rr.terminal_status AS run_terminal_status, rr.failure_kind AS run_failure_kind,", + " rr.created_at AS run_created_at, rr.updated_at AS run_updated_at, rr.claimed_by AS run_claimed_by, rr.lease_expires_at AS run_lease_expires_at,", + " c.state AS command_state, c.created_at AS command_created_at, c.updated_at AS command_updated_at, c.acknowledged_at AS command_acknowledged_at,", + " command_facts.command_count, event_facts.event_count, event_facts.events", "FROM agentrun_runner_jobs j", "LEFT JOIN agentrun_runners r ON r.id = j.runner_id", "LEFT JOIN agentrun_runs rr ON rr.id = j.run_id", - "WHERE j.namespace = $1", + "LEFT JOIN agentrun_commands c ON c.id = j.command_id", + "LEFT JOIN LATERAL (", + " SELECT COUNT(*)::int AS command_count FROM agentrun_commands cc WHERE cc.run_id = j.run_id", + ") command_facts ON TRUE", + "LEFT JOIN LATERAL (", + " SELECT COUNT(*)::int AS event_count, COALESCE(jsonb_agg(jsonb_build_object(", + " 'seq', e.seq, 'type', e.type, 'phase', e.payload->>'phase', 'commandId', e.payload->>'commandId', 'createdAt', e.created_at", + " ) ORDER BY e.seq), '[]'::jsonb) AS events", + " FROM (SELECT seq, type, payload, created_at FROM agentrun_events WHERE run_id = j.run_id ORDER BY seq LIMIT 501) e", + ") event_facts ON TRUE", + runnerJobId === null ? "WHERE j.namespace = $1" : "WHERE j.namespace = $1 AND j.id = $2", "ORDER BY COALESCE(r.heartbeat_at, j.updated_at, j.created_at) ASC", ].join("\n"); - const result = await pool.query(sql, [namespace]); + const result = await pool.query(sql, runnerJobId === null ? [namespace] : [namespace, runnerJobId]); await pool.end(); console.log(JSON.stringify({ ok: true, + queryMode: runnerJobId === null ? "namespace" : "runner-job-id", + runnerJobId, itemCount: result.rows.length, items: result.rows.map((row) => ({ id: row.id, @@ -1103,7 +1222,17 @@ try { runStatus: row.run_status || null, runTerminalStatus: row.run_terminal_status || null, runFailureKind: row.run_failure_kind || null, + runCreatedAt: iso(row.run_created_at), runUpdatedAt: iso(row.run_updated_at), + runClaimedBy: row.run_claimed_by || null, + runLeaseExpiresAt: iso(row.run_lease_expires_at), + commandState: row.command_state || null, + commandCreatedAt: iso(row.command_created_at), + commandUpdatedAt: iso(row.command_updated_at), + commandAcknowledgedAt: iso(row.command_acknowledged_at), + commandCount: Number.isInteger(row.command_count) ? row.command_count : null, + eventCount: Number.isInteger(row.event_count) ? row.event_count : null, + events: Array.isArray(row.events) ? row.events.slice(0, 501) : [], valuesPrinted: false, })), valuesPrinted: false, @@ -1125,17 +1254,47 @@ export function cleanupRunnersPlanNodeScript(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); -const tmp = process.env.TMP_DIR; -const namespace = process.env.NAMESPACE; -const maxRunners = Number(process.env.MAX_RUNNERS || 0); -const cleanupOrder = process.env.CLEANUP_ORDER || ""; -const activeHeartbeatMaxAgeMs = Number(process.env.ACTIVE_HEARTBEAT_MAX_AGE_MS || 0); -const ageBasedCleanupEnabled = process.env.AGE_BASED_CLEANUP_ENABLED === "true"; +const crypto = require("node:crypto"); + +function requiredEnv(name) { + const value = process.env[name]; + if (typeof value !== "string" || value.length === 0) throw new Error(name + " must be supplied from the owning runner retention YAML"); + return value; +} + +function requiredPositiveIntegerEnv(name) { + const value = Number(requiredEnv(name)); + if (!Number.isInteger(value) || value <= 0) throw new Error(name + " must be a positive integer from the owning runner retention YAML"); + return value; +} + +function requiredNonNegativeIntegerEnv(name) { + const value = Number(requiredEnv(name)); + if (!Number.isInteger(value) || value < 0) throw new Error(name + " must be a non-negative integer supplied by the runner observation wrapper"); + return value; +} + +function requiredBooleanEnv(name) { + const value = requiredEnv(name); + if (value !== "true" && value !== "false") throw new Error(name + " must be true or false"); + return value === "true"; +} + +const tmp = requiredEnv("TMP_DIR"); +const namespace = requiredEnv("NAMESPACE"); +const maxRunners = requiredPositiveIntegerEnv("MAX_RUNNERS"); +const cleanupOrder = requiredEnv("CLEANUP_ORDER"); +const activeHeartbeatMaxAgeMs = requiredPositiveIntegerEnv("ACTIVE_HEARTBEAT_MAX_AGE_MS"); +const stalePendingMaxAgeMs = requiredPositiveIntegerEnv("STALE_PENDING_MAX_AGE_MS"); +const ageBasedCleanupEnabled = requiredBooleanEnv("AGE_BASED_CLEANUP_ENABLED"); const ageBasedMaxAgeHours = process.env.AGE_BASED_MAX_AGE_HOURS ? Number(process.env.AGE_BASED_MAX_AGE_HOURS) : null; -const forceActive = process.env.FORCE_ACTIVE === "true"; -const matchLabels = JSON.parse(process.env.MATCH_LABELS_JSON || "{}"); -const jobNamePrefixes = JSON.parse(process.env.JOB_NAME_PREFIXES_JSON || "[]"); +const outputFull = requiredBooleanEnv("OUTPUT_FULL"); +const factsExit = requiredNonNegativeIntegerEnv("FACTS_EXIT"); +const eventsExit = requiredNonNegativeIntegerEnv("EVENTS_EXIT"); +const matchLabels = JSON.parse(requiredEnv("MATCH_LABELS_JSON")); +const jobNamePrefixes = JSON.parse(requiredEnv("JOB_NAME_PREFIXES_JSON")); const now = Date.now(); +const allowedNoProgressPhases = new Set(["run-created", "command-created", "runner-job-created", "runner-dispatch-completed"]); function readJson(name) { return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8")); @@ -1159,6 +1318,22 @@ function annotationsOf(item) { return item?.metadata?.annotations && typeof item.metadata.annotations === "object" ? item.metadata.annotations : {}; } +function identityOf(item) { + const annotations = annotationsOf(item); + return { + runId: typeof annotations["agentrun.pikastech.local/run-id"] === "string" ? annotations["agentrun.pikastech.local/run-id"] : null, + commandId: typeof annotations["agentrun.pikastech.local/command-id"] === "string" ? annotations["agentrun.pikastech.local/command-id"] : null, + runnerId: typeof annotations["agentrun.pikastech.local/runner-id"] === "string" ? annotations["agentrun.pikastech.local/runner-id"] : null, + }; +} + +function sameIdentity(left, right) { + return Boolean(left.runId && left.commandId && left.runnerId + && left.runId === right.runId + && left.commandId === right.commandId + && left.runnerId === right.runnerId); +} + function matchesLabels(labels) { return Object.entries(matchLabels).every(([key, value]) => labels?.[key] === value); } @@ -1185,6 +1360,12 @@ function isTerminalPod(pod) { return phase === "Succeeded" || phase === "Failed"; } +function bounded(value, max = 320) { + if (typeof value !== "string" || value.length === 0) return null; + const oneLine = value.replace(/\s+/g, " ").trim(); + return oneLine.length <= max ? oneLine : oneLine.slice(0, max); +} + function jobNameForPod(pod) { const labels = labelsOf(pod); if (typeof labels["job-name"] === "string") return labels["job-name"]; @@ -1202,6 +1383,135 @@ function preferredLastActiveAt(fact, job) { || null; } +function conditionReady(pod) { + return (Array.isArray(pod?.status?.conditions) ? pod.status.conditions : []) + .some((entry) => entry?.type === "Ready" && entry?.status === "True"); +} + +function stateKind(state) { + if (state?.running) return "running"; + if (state?.terminated) return "terminated"; + if (state?.waiting) return "waiting"; + return "unknown"; +} + +function statusStarted(status) { + const lastState = status?.lastState || {}; + const kind = stateKind(status?.state || {}); + return status?.started === true + || status?.ready === true + || Number(status?.restartCount || 0) > 0 + || kind === "running" + || kind === "terminated" + || Boolean(lastState.running) + || Boolean(lastState.terminated); +} + +function waitingFromStatus(status, source) { + const waiting = status?.state?.waiting; + if (!waiting || typeof waiting.reason !== "string" || waiting.reason.length === 0) return []; + return [{ + source, + name: typeof status.name === "string" ? status.name : null, + reason: waiting.reason, + message: bounded(waiting.message), + deterministic: waiting.reason !== "ContainerCreating" && waiting.reason !== "PodInitializing", + valuesPrinted: false, + }]; +} + +function warningFromEvent(event) { + if (event?.type !== "Warning" || typeof event?.reason !== "string" || event.reason.length === 0) return null; + return { + source: "event", + name: typeof event?.metadata?.name === "string" ? event.metadata.name : null, + reason: event.reason, + message: bounded(event.message), + deterministic: true, + count: Number.isFinite(event.count) ? event.count : null, + lastObservedAt: bounded(event?.series?.lastObservedTime || event?.eventTime || event?.lastTimestamp || event?.firstTimestamp), + valuesPrinted: false, + }; +} + +function podObservation(pod, warningEvents) { + const statuses = Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []; + const initStatuses = Array.isArray(pod?.status?.initContainerStatuses) ? pod.status.initContainerStatuses : []; + const runner = statuses.find((status) => status?.name === "runner") || null; + const conditions = Array.isArray(pod?.status?.conditions) ? pod.status.conditions : []; + const warnings = warningEvents.map(warningFromEvent).filter(Boolean); + const waiting = [ + ...statuses.flatMap((status) => waitingFromStatus(status, "container")), + ...initStatuses.flatMap((status) => waitingFromStatus(status, "init-container")), + ...conditions.flatMap((condition) => condition?.status === "False" && typeof condition?.reason === "string" && condition.reason.length > 0 + ? [{ source: "condition", name: condition.type || null, reason: condition.reason, message: bounded(condition.message), deterministic: condition.reason === "Unschedulable", valuesPrinted: false }] + : []), + ...warnings, + ]; + const createdAt = pod?.metadata?.creationTimestamp || null; + const runnerLastState = runner?.lastState || {}; + const allStatuses = [...statuses, ...initStatuses]; + return { + name: pod?.metadata?.name || null, + uid: pod?.metadata?.uid || null, + resourceVersion: pod?.metadata?.resourceVersion || null, + identity: identityOf(pod), + phase: pod?.status?.phase || null, + createdAt, + ageMs: ageMs(createdAt), + podStartTime: pod?.status?.startTime || null, + ready: conditionReady(pod), + runnerContainerState: stateKind(runner?.state || {}), + runnerContainerLastState: runnerLastState.running ? "running" : runnerLastState.terminated ? "terminated" : "none", + runnerContainerStarted: runner ? statusStarted(runner) : false, + anyContainerStarted: allStatuses.some(statusStarted), + runnerContainerReady: runner?.ready === true, + runnerContainerRestartCount: Number(runner?.restartCount || 0), + waiting: waiting.slice(0, 8), + waitingOmittedCount: Math.max(0, waiting.length - 8), + waitingEvidenceSources: [...new Set(waiting.map((entry) => entry.source))], + eventFactsAvailable: eventsExit === 0, + factsAvailable: typeof pod?.status?.phase === "string" && dateMs(createdAt) !== null, + valuesPrinted: false, + }; +} + +function businessFactsComplete(fact) { + return Boolean(fact + && typeof fact.runId === "string" + && typeof fact.commandId === "string" + && typeof fact.runnerId === "string" + && typeof fact.runStatus === "string" + && typeof fact.commandState === "string" + && Number.isInteger(fact.commandCount) + && Number.isInteger(fact.eventCount) + && Array.isArray(fact.events) + && fact.eventCount === fact.events.length + && fact.eventCount > 0 + && fact.eventCount < 500); +} + +function hasNoBusinessProgress(fact) { + if (!businessFactsComplete(fact)) return false; + if (fact.runStatus !== "pending" || fact.runClaimedBy !== null || fact.runLeaseExpiresAt !== null) return false; + if (fact.commandState !== "pending" || fact.commandAcknowledgedAt !== null) return false; + const runCreated = dateMs(fact.runCreatedAt); + const runUpdated = dateMs(fact.runUpdatedAt); + const commandCreated = dateMs(fact.commandCreatedAt); + const commandUpdated = dateMs(fact.commandUpdatedAt); + if (runCreated === null || runCreated !== runUpdated || commandCreated === null || commandCreated !== commandUpdated) return false; + if (now - runCreated < stalePendingMaxAgeMs || now - commandCreated < stalePendingMaxAgeMs) return false; + if (fact.commandCount !== 1) return false; + const phases = fact.events.map((event) => event?.phase || null); + if (fact.events.some((event) => event?.type !== "backend_status" || typeof event?.phase !== "string" || !allowedNoProgressPhases.has(event.phase))) return false; + if (!phases.includes("run-created") || !phases.includes("command-created") || !phases.includes("runner-job-created")) return false; + return fact.events.every((event) => event?.commandId === null || event?.commandId === undefined || event.commandId === fact.commandId); +} + +function safetyFingerprint(value) { + return "sha256:" + crypto.createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + function sortOldestLastActive(left, right) { const leftMs = dateMs(left.lastActiveAt) ?? dateMs(left.createdAt) ?? Number.MAX_SAFE_INTEGER; const rightMs = dateMs(right.lastActiveAt) ?? dateMs(right.createdAt) ?? Number.MAX_SAFE_INTEGER; @@ -1215,11 +1525,23 @@ function countNonTerminalPods(pods) { const jobs = readJson("jobs.json"); const pods = readJson("pods.json"); +const events = readJson("events.json"); const facts = readJson("runner-facts.json"); const factItems = Array.isArray(facts.items) ? facts.items : []; const factByJob = new Map(factItems.map((item) => [item.jobName, item])); const allPods = Array.isArray(pods.items) ? pods.items : []; +const allEvents = Array.isArray(events.items) ? events.items : []; const podsByJob = new Map(); +const warningsByPod = new Map(); +for (const event of allEvents) { + const podName = event?.involvedObject?.kind === "Pod" && typeof event?.involvedObject?.name === "string" + ? event.involvedObject.name + : null; + if (!podName) continue; + const entry = warningsByPod.get(podName) || []; + entry.push(event); + warningsByPod.set(podName, entry); +} for (const pod of allPods) { const jobName = jobNameForPod(pod); if (!jobName) continue; @@ -1238,38 +1560,52 @@ const matchedJobs = (Array.isArray(jobs.items) ? jobs.items : []) const runnerJobs = matchedJobs.map(({ job, name, labels, annotations }) => { const fact = factByJob.get(name) || null; const jobPods = podsByJob.get(name) || []; + const podObservations = jobPods.map((pod) => podObservation(pod, warningsByPod.get(pod?.metadata?.name || "") || [])); const nonTerminalPods = jobPods.filter((pod) => !isTerminalPod(pod)); const terminatingPods = nonTerminalPods.filter((pod) => typeof pod?.metadata?.deletionTimestamp === "string"); const terminal = isTerminalJob(job) || terminalRunStatus(fact?.runStatus) || terminalRunStatus(fact?.runTerminalStatus); const heartbeatAgeMs = ageMs(fact?.runnerHeartbeatAt); const heartbeatFresh = heartbeatAgeMs !== null && heartbeatAgeMs <= activeHeartbeatMaxAgeMs; - const hasActivePod = Number(job?.status?.active || 0) > 0 || nonTerminalPods.length > 0; + const jobIdentity = identityOf(job); + const factIdentity = { runId: fact?.runId || null, commandId: fact?.commandId || null, runnerId: fact?.runnerId || null }; + const identityComplete = sameIdentity(jobIdentity, factIdentity) + && podObservations.length > 0 + && podObservations.every((pod) => sameIdentity(jobIdentity, pod.identity)); + const managerFactsComplete = factsExit === 0 && facts.ok === true && businessFactsComplete(fact); + const podFactsComplete = podObservations.length > 0 && podObservations.every((pod) => pod.factsAvailable && pod.eventFactsAvailable); + const activePodReason = podObservations.some((pod) => pod.phase === "Running" + || pod.ready + || pod.runnerContainerReady + || pod.runnerContainerStarted + || pod.anyContainerStarted + || pod.runnerContainerState === "running" + || pod.runnerContainerState === "terminated") + ? "runner-pod-started-running-or-ready" + : null; const lastActiveAt = preferredLastActiveAt(fact, job); - let inactive = false; - let protectedActive = false; - let classification = "unknown"; - if (terminal) { - inactive = true; - classification = "terminal"; - } else if (heartbeatFresh) { - protectedActive = true; - classification = "active-fresh-heartbeat"; - } else if (heartbeatAgeMs !== null) { - inactive = true; - classification = "inactive-stale-heartbeat"; - } else if (terminatingPods.length > 0 && nonTerminalPods.length === terminatingPods.length) { - inactive = true; - classification = "inactive-terminating"; - } else if (hasActivePod) { - protectedActive = true; - classification = facts.ok === true ? "active-no-heartbeat-row" : "active-unverified-manager-facts"; - } else { - inactive = true; - classification = "inactive-no-active-pod"; - } - return { + const allPending = podObservations.length > 0 && podObservations.every((pod) => pod.phase === "Pending"); + const allStale = podObservations.length > 0 && podObservations.every((pod) => pod.ageMs !== null && pod.ageMs >= stalePendingMaxAgeMs); + const deterministicWaiting = podObservations.length > 0 && podObservations.every((pod) => pod.waiting.length > 0 && pod.waiting.some((item) => item.deterministic === true)); + let protectedReason = null; + let candidateKind = null; + if (!facts.ok || factsExit !== 0) protectedReason = "manager-facts-unavailable"; + else if (!identityComplete) protectedReason = "identity-facts-unavailable-or-mismatch"; + else if (!managerFactsComplete) protectedReason = "business-progress-facts-unavailable"; + else if (!podFactsComplete) protectedReason = eventsExit === 0 ? "pod-facts-unavailable" : "pod-warning-event-facts-unavailable"; + else if (fact?.runClaimedBy) protectedReason = "run-claimed"; + else if (fact?.runnerRegisteredAt || fact?.runnerHeartbeatAt) protectedReason = heartbeatFresh ? "runner-heartbeat-fresh" : "runner-registration-or-heartbeat-observed"; + else if (activePodReason) protectedReason = activePodReason; + else if (terminal) candidateKind = "terminal"; + else if (!allPending) protectedReason = "runner-pod-not-pending"; + else if (!allStale) protectedReason = "stale-pending-window-active"; + else if (!deterministicWaiting) protectedReason = "pod-deterministic-waiting-reason-unavailable"; + else if (!hasNoBusinessProgress(fact)) protectedReason = "business-progress-observed-or-unverifiable"; + else candidateKind = "stale-pending"; + const item = { name, namespace, + resourceUid: job?.metadata?.uid || null, + resourceVersion: job?.metadata?.resourceVersion || null, createdAt: job?.metadata?.creationTimestamp || null, lastActiveAt, lastActiveAgeMs: ageMs(lastActiveAt), @@ -1280,6 +1616,16 @@ const runnerJobs = matchedJobs.map(({ job, name, labels, annotations }) => { runnerId: fact?.runnerId || null, runStatus: fact?.runStatus || null, runTerminalStatus: fact?.runTerminalStatus || null, + runClaimedBy: fact?.runClaimedBy || null, + runLeaseExpiresAt: fact?.runLeaseExpiresAt || null, + commandState: fact?.commandState || null, + commandAcknowledgedAt: fact?.commandAcknowledgedAt || null, + commandCount: fact?.commandCount ?? null, + eventCount: fact?.eventCount ?? null, + eventPhases: Array.isArray(fact?.events) ? fact.events.slice(0, 12).map((event) => event?.phase || event?.type || null) : [], + heartbeatAt: fact?.runnerHeartbeatAt || null, + heartbeatAgeMs, + heartbeatFresh, jobStatus: { active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), @@ -1290,52 +1636,99 @@ const runnerJobs = matchedJobs.map(({ job, name, labels, annotations }) => { podCount: jobPods.length, nonTerminalPodCount: nonTerminalPods.length, terminatingPodCount: terminatingPods.length, - inactive, - protectedActive, - classification, - labels, + podObservations: podObservations.slice(0, 4), + podObservationOmittedCount: Math.max(0, podObservations.length - 4), + podPhases: podObservations.map((pod) => pod.phase), + waitingReasons: [...new Set(podObservations.flatMap((pod) => pod.waiting.map((entry) => entry.reason)))].slice(0, 8), + identityComplete, + managerFactsComplete, + podFactsComplete, + managerRetentionCandidate: candidateKind !== null, + safeCleanupCandidate: false, + protectedActive: candidateKind === null, + protectedReason, + candidateKind, + classification: candidateKind === "stale-pending" ? "stale-pending-no-business-progress" : candidateKind === "terminal" ? "terminal-complete-facts" : protectedReason, valuesPrinted: false, }; + return { + ...item, + safetyFingerprint: safetyFingerprint({ + resourceUid: item.resourceUid, + resourceVersion: item.resourceVersion, + runId: item.runId, + commandId: item.commandId, + runnerId: item.runnerId, + runStatus: item.runStatus, + runTerminalStatus: item.runTerminalStatus, + runClaimedBy: item.runClaimedBy, + runLeaseExpiresAt: item.runLeaseExpiresAt, + commandState: item.commandState, + commandAcknowledgedAt: item.commandAcknowledgedAt, + commandCount: item.commandCount, + eventCount: item.eventCount, + eventPhases: item.eventPhases, + heartbeatAt: item.heartbeatAt, + podObservations: item.podObservations, + candidateKind: item.candidateKind, + classification: item.classification, + }), + }; }); const runnerJobCount = runnerJobs.length; -const inactiveCandidates = runnerJobs.filter((item) => item.inactive).sort(sortOldestLastActive); +const managerRetentionCandidates = runnerJobs.filter((item) => item.managerRetentionCandidate).sort(sortOldestLastActive); +const stalePendingObservations = managerRetentionCandidates.filter((item) => item.candidateKind === "stale-pending"); +const terminalObservations = managerRetentionCandidates.filter((item) => item.candidateKind === "terminal"); +const protectedRunners = runnerJobs.filter((item) => item.protectedActive).sort(sortOldestLastActive); const overLimitCount = Math.max(0, runnerJobCount - maxRunners); -const selectedByName = new Map(); -const selectionReasons = new Map(); -for (const item of inactiveCandidates.slice(0, overLimitCount)) { - selectedByName.set(item.name, item); - selectionReasons.set(item.name, "over-max-runners"); -} -if (ageBasedCleanupEnabled && ageBasedMaxAgeHours !== null) { - const maxAgeMs = ageBasedMaxAgeHours * 3600 * 1000; - for (const item of inactiveCandidates) { - const itemAgeMs = item.lastActiveAgeMs; - if (itemAgeMs !== null && itemAgeMs >= maxAgeMs) { - selectedByName.set(item.name, item); - if (!selectionReasons.has(item.name)) selectionReasons.set(item.name, "age-based-cleanup"); - } - } -} -if (forceActive) { - for (const item of runnerJobs) { - selectedByName.set(item.name, item); - selectionReasons.set(item.name, item.protectedActive ? "force-active-runner-cleanup" : "force-runner-cleanup"); - } -} - -const selectionPool = forceActive ? runnerJobs : inactiveCandidates; -const selected = selectionPool - .filter((item) => selectedByName.has(item.name)) - .map((item) => ({ ...item, selectionReason: selectionReasons.get(item.name) || "selected" })); +const selected = []; const matchedPodNames = new Set(runnerJobs.map((item) => item.name)); const matchedPods = allPods.filter((pod) => { const jobName = jobNameForPod(pod); return jobName !== null && matchedPodNames.has(jobName); }); const remainingAfterSelection = runnerJobCount - selected.length; +const observationOk = factsExit === 0 && facts.ok === true && eventsExit === 0; +const managerRetentionCapabilityValidated = false; +const atCapacity = runnerJobCount === maxRunners; +const overCapacity = runnerJobCount > maxRunners; +const capacityReady = runnerJobCount < maxRunners || managerRetentionCapabilityValidated; +const capacityStatus = overCapacity + ? "over-capacity-manager-retention-required" + : atCapacity + ? "at-capacity-manager-retention-required" + : "available"; +const availableSlots = Math.max(0, maxRunners - runnerJobCount); +const capacityBlockerCode = overCapacity + ? "runner-capacity-over-limit-manager-retention-not-validated" + : atCapacity + ? "runner-capacity-at-limit-manager-retention-not-validated" + : null; +const capacityBlocker = capacityBlockerCode === null + ? null + : { + type: "runner-capacity", + code: capacityBlockerCode, + status: capacityStatus, + current: runnerJobCount, + limit: maxRunners, + availableSlots, + managerRetentionCapabilityValidated, + valuesPrinted: false, + }; +const degradedReason = !observationOk + ? "runner-retention-observation-incomplete" + : overCapacity + ? "runner-capacity-over-limit-manager-retention-not-validated" + : atCapacity + ? "runner-capacity-at-limit-manager-retention-not-validated" + : null; +const ok = observationOk && capacityReady; +const visible = (items) => outputFull ? items : items.slice(0, 8); console.log(JSON.stringify({ - ok: true, + ok, + degradedReason, planKind: "agentrun-runtime-runner-retention", generatedAt: new Date(now).toISOString(), namespace, @@ -1343,30 +1736,84 @@ console.log(JSON.stringify({ maxRunners, cleanupOrder, activeHeartbeatMaxAgeMs, - forceActive, + stalePendingMaxAgeMs, + selectionBoundary: "read-only-observation", + incomingRunnerAuthority: "agentrun-pre-create-retention", + unideskDeletionAuthority: "none", + managerTrackingIssue: "https://github.com/pikasTech/agentrun/issues/280", + managerTrackingPullRequest: "https://github.com/pikasTech/agentrun/pull/281", selectors: { matchLabels, jobNamePrefixes }, ageBasedCleanup: { enabled: ageBasedCleanupEnabled, maxAgeHours: ageBasedMaxAgeHours }, }, managerFacts: { ok: facts.ok === true, factCount: factItems.length, - factsExit: Number(process.env.FACTS_EXIT || 0), + factsExit, failureKind: facts.failureKind || null, message: facts.ok === true ? null : facts.message || null, valuesPrinted: false, }, + kubernetesFacts: { + podListAvailable: true, + warningEventsAvailable: eventsExit === 0, + eventsExit, + valuesPrinted: false, + }, + observation: { + ok: observationOk, + managerFactsAvailable: factsExit === 0 && facts.ok === true, + jobListAvailable: true, + podListAvailable: true, + warningEventsAvailable: eventsExit === 0, + valuesPrinted: false, + }, + capacity: { + ok: capacityReady, + status: capacityStatus, + runnerJobCount, + maxRunners, + availableSlots, + atCapacity, + overCapacity, + nextRunnerRequiresManagerRetention: runnerJobCount >= maxRunners, + managerRetentionCapabilityValidated, + blocker: capacityBlocker, + valuesPrinted: false, + }, runnerJobCount, nonTerminalRunnerPodCount: countNonTerminalPods(matchedPods), overLimitCount, - inactiveCandidateCount: inactiveCandidates.length, - protectedActiveRunnerCount: runnerJobs.filter((item) => item.protectedActive).length, - activeRunRisk: remainingAfterSelection > maxRunners, + inactiveCandidateCount: managerRetentionCandidates.length, + managerRetentionCandidateCount: managerRetentionCandidates.length, + stalePendingObservationCount: stalePendingObservations.length, + terminalObservationCount: terminalObservations.length, + protectedActiveRunnerCount: protectedRunners.length, + activeRunRisk: remainingAfterSelection > maxRunners || protectedRunners.some((item) => item.runClaimedBy !== null || item.heartbeatAt !== null), + nextRunnerRequiresManagerRetention: runnerJobCount >= maxRunners, + managerRetentionCapabilityValidated, remainingRunnerJobCountAfterSelection: remainingAfterSelection, - runnerJobs: runnerJobs.sort(sortOldestLastActive), - candidates: inactiveCandidates, + remainingAfterSelection, + runnerJobs: visible(runnerJobs.sort(sortOldestLastActive)), + runnerJobOmittedCount: outputFull ? 0 : Math.max(0, runnerJobs.length - 8), + candidates: visible(managerRetentionCandidates), + candidateOmittedCount: outputFull ? 0 : Math.max(0, managerRetentionCandidates.length - 8), + stalePendingObservations: visible(stalePendingObservations), + stalePendingObservationOmittedCount: outputFull ? 0 : Math.max(0, stalePendingObservations.length - 8), + terminalObservations: visible(terminalObservations), + terminalObservationOmittedCount: outputFull ? 0 : Math.max(0, terminalObservations.length - 8), + protectedRunners: visible(protectedRunners), + protectedRunnerOmittedCount: outputFull ? 0 : Math.max(0, protectedRunners.length - 8), selected, selectedRunnerJobs: selected.map((item) => item.name), selectedRunnerJobCount: selected.length, + selectedRunnerJobOmittedCount: 0, + deletionAuthority: { + owner: "agentrun-manager-pre-create-retention", + unideskConfirmSupported: false, + trackingIssue: "https://github.com/pikasTech/agentrun/issues/280", + trackingPullRequest: "https://github.com/pikasTech/agentrun/pull/281", + valuesPrinted: false, + }, valuesPrinted: false, })); `; diff --git a/scripts/src/agentrun/trigger.ts b/scripts/src/agentrun/trigger.ts index 96618881..e450f2a5 100644 --- a/scripts/src/agentrun/trigger.ts +++ b/scripts/src/agentrun/trigger.ts @@ -39,8 +39,9 @@ import { triggerCurrentYamlLaneConfirmed } from "./cleanup-scripts"; import { readAgentRunClientConfig } from "./config"; import { displayValue } from "./options"; import { pathValue } from "./render"; +import { providerSecretActivationPreflight, providerSecretDesiredEncodedFingerprint, providerSecretStagedActivation } from "./provider-activation"; import { startAsyncAgentRunJob } from "./rest-bridge"; -import { collectLaneSecretSources, readSecretSourceValue, restartYamlLaneScript, secretSyncScript } from "./secrets"; +import { collectLaneSecretSources, inspectSecretSourceValue, restartYamlLaneScript, secretSyncScript } from "./secrets"; import { capture, captureJsonPayload, compactCapture, isGitSha, stringOrNull } from "./utils"; import { yamlLaneK3sSourceStatusScript, yamlLaneSourceBootstrapProbeScript } from "./yaml-lane"; @@ -217,8 +218,8 @@ export async function secretSync(config: UniDeskConfig, options: SecretSyncOptio valuesPrinted: false, }; } - const values = sources.map((source) => ({ spec: source, value: readSecretSourceValue(spec, source) })); - const plan = values.map(({ spec: item, value }) => ({ + const inspected = sources.map((source) => ({ spec: source, inspection: inspectSecretSourceValue(spec, source) })); + const plan = inspected.map(({ spec: item, inspection }) => ({ id: item.id, namespace: item.targetRef.namespace, secret: item.targetRef.name, @@ -226,11 +227,105 @@ export async function secretSync(config: UniDeskConfig, options: SecretSyncOptio sourceRef: item.sourceRef, sourceMode: item.sourceMode, sourceKey: item.sourceKey, - sourcePath: value.redactedPath, - fingerprint: value.fingerprint, - valueBytes: value.valueBytes, + transform: item.transform ?? null, + ...inspection.summary, valuesPrinted: false, })); + const unavailable = inspected.filter(({ inspection }) => !inspection.ok); + if (unavailable.length > 0) { + return { + ok: false, + command: "agentrun control-plane secret-sync", + mode: options.dryRun || !options.confirm ? "dry-run-blocked" : "confirm-blocked-before-mutation", + mutation: false, + configPath, + target: compactAgentRunLaneStatusTarget(spec), + degradedReason: "lane-secret-source-unavailable", + blockedBeforeMutation: true, + filter: options.secretIds.length === 0 ? null : { secretIds: options.secretIds, selected: sources.map((source) => source.id) }, + plan: { + secretCount: plan.length, + readyCount: plan.length - unavailable.length, + unavailableCount: unavailable.length, + items: plan, + valuesPrinted: false, + }, + next: { + retryDryRun: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane}${options.secretIds.map((id) => ` --secret-id ${id}`).join("")} --dry-run`, + }, + valuesPrinted: false, + }; + } + const values = inspected.map(({ spec: item, inspection }) => ({ spec: item, value: inspection.sourceValue! })); + const providerSecretsByTargetRef = new Map(); + for (const secret of spec.secrets) { + if (secret.providerCredentialProfile === null) continue; + const targetRef = laneSecretTargetRefKey(secret.targetRef); + const existing = providerSecretsByTargetRef.get(targetRef); + if (existing !== undefined) { + throw new Error(`provider Secret targetRef ${targetRef} is declared by both ${existing.id} and ${secret.id}`); + } + providerSecretsByTargetRef.set(targetRef, secret); + } + const providerValues = values.filter(({ spec: item }) => providerSecretsByTargetRef.has(laneSecretTargetRefKey(item.targetRef))); + const providerSecrets = providerValues.map(({ spec: item }) => providerSecretsByTargetRef.get(laneSecretTargetRefKey(item.targetRef))!); + const providerActivationTargets = providerValues.map(({ spec: item, value }) => { + return { + id: item.id, + targetRef: item.targetRef, + desiredEncodedFingerprint: providerSecretDesiredEncodedFingerprint(value.value), + }; + }); + const activationPreflight = providerValues.length > 0 + ? await providerSecretActivationPreflight(config, spec, providerActivationTargets, { full: options.full || options.raw }) + : null; + const syncValues = values.filter(({ spec: item }) => !providerSecretsByTargetRef.has(laneSecretTargetRefKey(item.targetRef))); + if (activationPreflight !== null && activationPreflight.ok !== true) { + const profiles = providerSecrets + .map((secret) => secret.providerCredentialProfile) + .filter((profile): profile is string => profile !== null); + return { + ok: false, + command: "agentrun control-plane secret-sync", + mode: options.dryRun || !options.confirm ? "dry-run-blocked-before-mutation" : "confirm-blocked-before-mutation", + status: "blocked", + mutation: false, + blockedBeforeMutation: true, + secretSyncStarted: false, + configPath, + target: { node: spec.nodeId, lane: spec.lane, namespace: spec.runtime.namespace, valuesPrinted: false }, + degradedReason: activationPreflight.reason ?? "existing-runner-provider-secret-activation-risk", + filter: options.secretIds.length === 0 ? null : { secretIds: options.secretIds, selected: sources.map((source) => source.id) }, + plan: { + secretCount: plan.length, + items: plan.slice(0, 8).map((item) => ({ + id: item.id, + namespace: item.namespace, + secret: item.secret, + key: item.key, + present: item.present, + fingerprint: item.fingerprint, + valueBytes: item.valueBytes, + valuesPrinted: false, + })), + omittedCount: Math.max(0, plan.length - 8), + valuesPrinted: false, + }, + activationPreflight, + execution: { + providerSecretClassification: "exact-target-ref", + providerSecretWriteBlockedCount: providerValues.length, + secretSyncStarted: false, + valuesPrinted: false, + }, + stagedActivation: providerSecretStagedActivation(spec, profiles), + next: { + observeRunners: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane} --dry-run`, + retryAfterManagerFence: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane}${options.secretIds.map((id) => ` --secret-id ${id}`).join("")} --confirm`, + }, + valuesPrinted: false, + }; + } if (options.dryRun || !options.confirm) { return { ok: true, @@ -240,7 +335,14 @@ export async function secretSync(config: UniDeskConfig, options: SecretSyncOptio configPath, target: compactAgentRunLaneStatusTarget(spec), filter: options.secretIds.length === 0 ? null : { secretIds: options.secretIds, selected: sources.map((source) => source.id) }, - plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, + plan: { secretCount: plan.length, items: plan, activationPreflight, valuesPrinted: false }, + execution: { + providerSecretClassification: "exact-target-ref", + providerSecretWritePolicy: "skip-exact-no-op-until-manager-activation-fence", + providerSecretWriteSkippedCount: providerValues.length, + writableNonProviderSecretCount: syncValues.length, + valuesPrinted: false, + }, next: { confirm: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane}${options.secretIds.map((id) => ` --secret-id ${id}`).join("")} --confirm`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, @@ -248,7 +350,35 @@ export async function secretSync(config: UniDeskConfig, options: SecretSyncOptio valuesPrinted: false, }; } - const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", secretSyncScript(spec, values.map(({ spec: item, value }) => ({ targetRef: item.targetRef, value: value.value })))]); + if (syncValues.length === 0) { + return { + ok: true, + command: "agentrun control-plane secret-sync", + mode: "confirmed-provider-secret-no-op-skip", + mutation: false, + secretSyncStarted: false, + providerSecretWriteSkipped: providerValues.length > 0, + configPath, + target: compactAgentRunLaneStatusTarget(spec), + filter: options.secretIds.length === 0 ? null : { secretIds: options.secretIds, selected: sources.map((source) => source.id) }, + plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, + activationPreflight, + result: { + ok: true, + status: "skipped-exact-provider-secret-no-op", + skippedProviderSecretCount: providerValues.length, + providerSecretClassification: "exact-target-ref", + reason: "provider-secret-writes-require-agentrun-manager-activation-fence", + runtimeSecretValuesDecoded: false, + valuesPrinted: false, + }, + next: { + status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; + } + const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", secretSyncScript(spec, syncValues.map(({ spec: item, value }) => ({ targetRef: item.targetRef, value: value.value })))]); const payload = captureJsonPayload(result); return { ok: result.exitCode === 0 && payload.ok !== false, @@ -258,7 +388,15 @@ export async function secretSync(config: UniDeskConfig, options: SecretSyncOptio configPath, target: compactAgentRunLaneStatusTarget(spec), filter: options.secretIds.length === 0 ? null : { secretIds: options.secretIds, selected: sources.map((source) => source.id) }, - plan: { secretCount: plan.length, items: plan, valuesPrinted: false }, + plan: { + secretCount: plan.length, + items: plan, + providerSecretWriteSkippedCount: providerValues.length, + providerSecretClassification: "exact-target-ref", + writableNonProviderSecretCount: syncValues.length, + valuesPrinted: false, + }, + activationPreflight, result: payload, capture: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), next: { @@ -268,6 +406,10 @@ export async function secretSync(config: UniDeskConfig, options: SecretSyncOptio }; } +function laneSecretTargetRefKey(targetRef: { namespace: string; name: string; key: string }): string { + return `${targetRef.namespace}/${targetRef.name}/${targetRef.key}`; +} + export async function exposeAgentRun(_config: UniDeskConfig, options: ConfirmOptions): Promise> { const clientConfig = readAgentRunClientConfig(); const exposure = clientConfig.publicExposure; diff --git a/scripts/src/agentrun/yaml-lane.ts b/scripts/src/agentrun/yaml-lane.ts index b82b07b1..06d3e744 100644 --- a/scripts/src/agentrun/yaml-lane.ts +++ b/scripts/src/agentrun/yaml-lane.ts @@ -36,8 +36,9 @@ import { import { sha256Fingerprint } from "../platform-infra-ops-library"; import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, LaneConfirmOptions, RefreshOptions } from "./options"; -import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror"; +import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror"; import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets"; +import { compactAgentRunLaneStatusTarget } from "./trigger"; import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils"; export async function refresh(config: UniDeskConfig, options: RefreshOptions): Promise> { @@ -95,14 +96,16 @@ export async function cleanupRunners(config: UniDeskConfig, options: CleanupRunn const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunnersScript(options, spec)]); const payload = captureJsonPayload(result); - const ok = result.exitCode === 0 && payload.ok !== false; + const planOk = result.exitCode === 0 && payload.ok !== false; + const visiblePayload = compactRunnerCleanupPayload(payload, spec, options.full || options.raw); + const selectedRunnerJobCount = Number(visiblePayload.selectedRunnerJobCount ?? 0); const base = { - ...payload, - ok, + ...visiblePayload, + ok: planOk, command: "agentrun control-plane cleanup-runners", configPath, - target: agentRunLaneSummary(spec), - mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup", + target: compactAgentRunLaneStatusTarget(spec), + mode: options.dryRun || !options.confirm ? "dry-run" : "blocked-manager-authority", namespace: spec.runtime.namespace, retention: spec.deployment.runner.retention, probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }), @@ -112,22 +115,159 @@ export async function cleanupRunners(config: UniDeskConfig, options: CleanupRunn ...base, dryRun: true, mutation: false, + selectedRunnerJobCount, + deletionAuthority: { + owner: "agentrun-manager-pre-create-retention", + unideskConfirmSupported: false, + trackingIssue: "https://github.com/pikasTech/agentrun/issues/280", + trackingPullRequest: "https://github.com/pikasTech/agentrun/pull/281", + valuesPrinted: false, + }, next: { - confirm: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --confirm`, + recheck: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane} --dry-run`, + managerAuthority: "https://github.com/pikasTech/agentrun/issues/280", }, }; } return { ...base, + ok: false, + degradedReason: "runner-retention-confirm-owned-by-agentrun-manager", dryRun: false, - mutation: true, + mutation: false, + selectedRunnerJobCount, + deletionAuthority: { + owner: "agentrun-manager-pre-create-retention", + unideskConfirmSupported: false, + trackingIssue: "https://github.com/pikasTech/agentrun/issues/280", + trackingPullRequest: "https://github.com/pikasTech/agentrun/pull/281", + reason: "db-fence-and-kubernetes-cas-required", + valuesPrinted: false, + }, + blocker: { + code: "runner-retention-confirm-owned-by-agentrun-manager", + mutationAttempted: false, + valuesPrinted: false, + }, followUp: { - dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane}${options.forceActive ? " --force-active" : ""} --dry-run`, + dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-runners --node ${spec.nodeId} --lane ${spec.lane} --dry-run`, status: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane}`, }, }; } +function compactRunnerCleanupPayload(payload: Record, spec: AgentRunLaneSpec, full: boolean): Record { + const compactItems = (value: unknown, limit: number, includePod: boolean): Record[] => { + if (!Array.isArray(value)) return []; + return (full ? value : value.slice(0, limit)).map((raw) => { + const item = typeof raw === "object" && raw !== null && !Array.isArray(raw) ? raw as Record : {}; + const pod = Array.isArray(item.podObservations) && item.podObservations.length > 0 + ? item.podObservations[0] as Record + : null; + const runId = stringOrNull(item.runId); + const commandId = stringOrNull(item.commandId); + const runnerJobId = stringOrNull(item.runnerJobId); + return { + name: item.name ?? null, + namespace: item.namespace ?? null, + createdAt: item.createdAt ?? null, + lastActiveAt: item.lastActiveAt ?? null, + lastActiveAgeMs: item.lastActiveAgeMs ?? null, + runId, + commandId, + runnerJobId, + runnerId: item.runnerId ?? null, + runStatus: item.runStatus ?? null, + runClaimedBy: item.runClaimedBy ?? null, + commandState: item.commandState ?? null, + heartbeatAt: item.heartbeatAt ?? null, + heartbeatAgeMs: item.heartbeatAgeMs ?? null, + podPhases: Array.isArray(item.podPhases) ? item.podPhases.slice(0, 4) : [], + waitingReasons: Array.isArray(item.waitingReasons) ? item.waitingReasons.slice(0, 6) : [], + identityComplete: item.identityComplete === true, + managerFactsComplete: item.managerFactsComplete === true, + podFactsComplete: item.podFactsComplete === true, + safeCleanupCandidate: item.safeCleanupCandidate === true, + protectedActive: item.protectedActive === true, + protectedReason: item.protectedReason ?? null, + candidateKind: item.candidateKind ?? null, + classification: item.classification ?? null, + safetyFingerprint: item.safetyFingerprint ?? null, + ...(includePod && pod !== null ? { + pod: { + name: pod.name ?? null, + phase: pod.phase ?? null, + ageMs: pod.ageMs ?? null, + ready: pod.ready === true, + runnerContainerState: pod.runnerContainerState ?? null, + runnerContainerLastState: pod.runnerContainerLastState ?? null, + runnerContainerStarted: pod.runnerContainerStarted === true, + runnerContainerReady: pod.runnerContainerReady === true, + runnerContainerRestartCount: pod.runnerContainerRestartCount ?? null, + waiting: Array.isArray(pod.waiting) ? pod.waiting.slice(0, 4) : [], + eventFactsAvailable: pod.eventFactsAvailable === true, + valuesPrinted: false, + }, + } : {}), + drillDown: { + runnerJob: runId === null || runnerJobId === null + ? null + : `bun scripts/cli.ts agentrun describe runnerjob/${runnerJobId} --run ${runId} --node ${spec.nodeId} --lane ${spec.lane} -o json`, + command: runId === null || commandId === null + ? null + : `bun scripts/cli.ts agentrun describe command/${commandId} --run ${runId} --node ${spec.nodeId} --lane ${spec.lane}`, + events: runId === null + ? null + : `bun scripts/cli.ts agentrun events run/${runId} --after-seq 0 --limit 100 --node ${spec.nodeId} --lane ${spec.lane}`, + }, + valuesPrinted: false, + }; + }); + }; + const runnerJobs = compactItems(payload.runnerJobs, 4, true); + const candidates = compactItems(payload.candidates, 4, false); + const stalePendingObservations = compactItems(payload.stalePendingObservations, 4, true); + const terminalObservations = compactItems(payload.terminalObservations, 4, true); + const protectedRunners = compactItems(payload.protectedRunners, 4, false); + const selected = compactItems(payload.selected, 4, true); + const compactNames = (value: unknown, limit: number): string[] => { + if (!Array.isArray(value)) return []; + const names = value.filter((item): item is string => typeof item === "string"); + return full ? names : names.slice(0, limit); + }; + const selectedRunnerJobs = compactNames(payload.selectedRunnerJobs, 8); + const deletedRunnerJobs = compactNames(payload.deletedRunnerJobs, 8); + const remainingSelectedRunnerJobs = compactNames(payload.remainingSelectedRunnerJobs, 8); + const selectedRunnerJobCount = Number(payload.selectedRunnerJobCount ?? (Array.isArray(payload.selectedRunnerJobs) ? payload.selectedRunnerJobs.length : 0)); + const deletedRunnerJobCount = Number(payload.deletedRunnerJobCount ?? (Array.isArray(payload.deletedRunnerJobs) ? payload.deletedRunnerJobs.length : 0)); + const remainingSelectedRunnerJobCount = Number(payload.remainingSelectedRunnerJobCount ?? (Array.isArray(payload.remainingSelectedRunnerJobs) ? payload.remainingSelectedRunnerJobs.length : 0)); + return { + ...payload, + runnerJobs, + runnerJobOmittedCount: Number(payload.runnerJobOmittedCount ?? 0) + Math.max(0, (Array.isArray(payload.runnerJobs) ? payload.runnerJobs.length : 0) - runnerJobs.length), + candidates, + candidateOmittedCount: Number(payload.candidateOmittedCount ?? 0) + Math.max(0, (Array.isArray(payload.candidates) ? payload.candidates.length : 0) - candidates.length), + stalePendingObservations, + stalePendingObservationOmittedCount: Number(payload.stalePendingObservationOmittedCount ?? 0) + Math.max(0, (Array.isArray(payload.stalePendingObservations) ? payload.stalePendingObservations.length : 0) - stalePendingObservations.length), + terminalObservations, + terminalObservationOmittedCount: Number(payload.terminalObservationOmittedCount ?? 0) + Math.max(0, (Array.isArray(payload.terminalObservations) ? payload.terminalObservations.length : 0) - terminalObservations.length), + protectedRunners, + protectedRunnerOmittedCount: Number(payload.protectedRunnerOmittedCount ?? 0) + Math.max(0, (Array.isArray(payload.protectedRunners) ? payload.protectedRunners.length : 0) - protectedRunners.length), + selected, + selectedOmittedCount: Math.max(0, (Array.isArray(payload.selected) ? payload.selected.length : 0) - selected.length), + selectedRunnerJobs, + selectedRunnerJobCount, + selectedRunnerJobOmittedCount: Math.max(0, selectedRunnerJobCount - selectedRunnerJobs.length), + deletedRunnerJobs, + deletedRunnerJobCount, + deletedRunnerJobOmittedCount: Math.max(0, deletedRunnerJobCount - deletedRunnerJobs.length), + remainingSelectedRunnerJobs, + remainingSelectedRunnerJobCount, + remainingSelectedRunnerJobOmittedCount: Math.max(0, remainingSelectedRunnerJobCount - remainingSelectedRunnerJobs.length), + valuesPrinted: false, + }; +} + export async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupRunsScript(options, spec.ci.namespace, spec.ci.pipelineRunPrefix)]); @@ -363,10 +503,10 @@ export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: Agent `max_runners=${String(retention.maxRunners)}`, `cleanup_order=${shQuote(retention.cleanupOrder)}`, `active_heartbeat_max_age_ms=${String(retention.activeHeartbeatMaxAgeMs)}`, + `stale_pending_max_age_ms=${String(retention.stalePendingMaxAgeMs)}`, `age_based_cleanup_enabled=${retention.ageBasedCleanup.enabled ? "true" : "false"}`, `age_based_max_age_hours=${retention.ageBasedCleanup.maxAgeHours === null ? "" : String(retention.ageBasedCleanup.maxAgeHours)}`, - `timeout_seconds=${String(options.timeoutSeconds)}`, - `force_active=${options.forceActive ? "true" : "false"}`, + `output_full=${options.full || options.raw ? "true" : "false"}`, `match_labels_json_b64=${shQuote(matchLabelsB64)}`, `job_name_prefixes_json_b64=${shQuote(jobNamePrefixesB64)}`, "match_labels_json=$(printf '%s' \"$match_labels_json_b64\" | base64 -d)", @@ -375,6 +515,12 @@ export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: Agent "trap 'rm -rf \"$tmp_dir\"' EXIT", "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs.json\"", "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"", + "events_exit=0", + "set +e", + "kubectl -n \"$namespace\" get events --field-selector involvedObject.kind=Pod,type=Warning -o json > \"$tmp_dir/events.json\" 2> \"$tmp_dir/events.err\"", + "events_exit=$?", + "set -e", + "if [ \"$events_exit\" -ne 0 ]; then printf '%s\\n' '{\"items\":[]}' > \"$tmp_dir/events.json\"; fi", "facts_exit=0", "set +e", "kubectl -n \"$namespace\" exec -i deploy/\"$manager_deployment\" -- env RETENTION_NAMESPACE=\"$namespace\" sh -lc 'cat >/tmp/agentrun-runner-retention.mjs && bun /tmp/agentrun-runner-retention.mjs' > \"$tmp_dir/runner-facts.json\" 2> \"$tmp_dir/runner-facts.err\" <<'NODE'", @@ -385,26 +531,10 @@ export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: Agent "if [ \"$facts_exit\" -ne 0 ]; then", " printf '%s\\n' '{\"ok\":false,\"items\":[],\"failureKind\":\"manager-facts-unavailable\",\"valuesPrinted\":false}' > \"$tmp_dir/runner-facts.json\"", "fi", - "MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" FORCE_ACTIVE=\"$force_active\" FACTS_EXIT=\"$facts_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"", + "MATCH_LABELS_JSON=\"$match_labels_json\" JOB_NAME_PREFIXES_JSON=\"$job_name_prefixes_json\" MAX_RUNNERS=\"$max_runners\" CLEANUP_ORDER=\"$cleanup_order\" ACTIVE_HEARTBEAT_MAX_AGE_MS=\"$active_heartbeat_max_age_ms\" STALE_PENDING_MAX_AGE_MS=\"$stale_pending_max_age_ms\" AGE_BASED_CLEANUP_ENABLED=\"$age_based_cleanup_enabled\" AGE_BASED_MAX_AGE_HOURS=\"$age_based_max_age_hours\" OUTPUT_FULL=\"$output_full\" FACTS_EXIT=\"$facts_exit\" EVENTS_EXIT=\"$events_exit\" TMP_DIR=\"$tmp_dir\" NAMESPACE=\"$namespace\" node <<'NODE' > \"$tmp_dir/plan.json\"", cleanupRunnersPlanNodeScript(), "NODE", - "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then", - " cat \"$tmp_dir/plan.json\"", - " exit 0", - "fi", - "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedRunnerJobs)?plan.selectedRunnerJobs:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-jobs.txt\"", - "delete_exit=0", - "if [ -s \"$tmp_dir/selected-jobs.txt\" ]; then", - " xargs -r kubectl -n \"$namespace\" delete job --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-jobs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?", - "else", - " : > \"$tmp_dir/delete.out\"", - " : > \"$tmp_dir/delete.err\"", - "fi", - "kubectl -n \"$namespace\" get job -o json > \"$tmp_dir/jobs-after.json\"", - "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods-after.json\"", - "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'", - cleanupRunnersFinalizeNodeScript(), - "NODE", + "cat \"$tmp_dir/plan.json\"", ].join("\n"); } diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 17d0090e..eed49267 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -492,6 +492,10 @@ export interface HwlabRuntimeKafkaEventBridgeSpec { export interface HwlabRuntimeCodeAgentRuntimeSpec { readonly enabled: boolean; readonly adapter: string; + readonly controlPlaneTarget: { + readonly node: string; + readonly lane: string; + }; readonly managerUrl: string; readonly apiKeySecretName: string; readonly apiKeySecretKey: string; @@ -1047,9 +1051,14 @@ function codeAgentRuntimeConfig(value: unknown, path: string): HwlabRuntimeCodeA } catch { throw new Error(`${path}.managerUrl must be an http(s) URL`); } + const controlPlaneTarget = asRecord(raw.controlPlaneTarget, `${path}.controlPlaneTarget`); return { enabled: booleanField(raw, "enabled", path), adapter: stringField(raw, "adapter", path), + controlPlaneTarget: { + node: stringField(controlPlaneTarget, "node", `${path}.controlPlaneTarget`), + lane: stringField(controlPlaneTarget, "lane", `${path}.controlPlaneTarget`), + }, managerUrl, apiKeySecretName: stringField(raw, "apiKeySecretName", path), apiKeySecretKey: secretKeyField(raw, "apiKeySecretKey", path), diff --git a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts index 59b6ea85..1bdc7b58 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5-observe.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5-observe.ts @@ -64,6 +64,22 @@ export function runSentinelQuickVerify(state: SentinelCicdState, reason: string, const maxSeconds = numberAt(state.cicd, "targetValidation.maxSeconds"); const scenario = findScenario(state, scenarioId); if (scenario === null) return { ok: false, status: "blocked", reason: "scenario-not-found", scenarioId, valuesRedacted: true }; + const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, timeoutSeconds, scenario); + if (providerProfilePreflight.ok !== true) { + return { + ok: false, + status: "blocked", + reason: "exact-provider-profile-unavailable", + scenarioId, + providerProfile: stringAtNullable(scenario, "providerProfile"), + providerProfileMode: stringAtNullable(scenario, "providerProfileMode"), + providerProfilePreflight, + observerCreated: false, + runCreated: false, + mutation: false, + valuesRedacted: true, + }; + } const runId = `sentinel-run-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; emitWebProbeSentinelSpan(sentinelOtelContext(state), "web_probe_sentinel.quick_verify.job_start", { scenarioId, @@ -1952,6 +1968,132 @@ function findScenario(state: SentinelCicdState, scenarioId: string): Record item.id === scenarioId) ?? null; } +export function runSentinelExactProviderProfilePreflight( + state: SentinelCicdState, + timeoutSeconds: number, + selectedScenario?: Record, +): Record & { ok: boolean } { + const scenarioId = stringAt(state.cicd, "targetValidation.scenarioId"); + const scenario = selectedScenario ?? findScenario(state, scenarioId); + if (scenario === null) { + return { + ok: false, + status: "blocked", + reason: "scenario-not-found", + scenarioId, + mutation: false, + valuesRedacted: true, + }; + } + const profile = stringAtNullable(scenario, "providerProfile"); + const mode = stringAtNullable(scenario, "providerProfileMode"); + if (mode !== "exact") { + return { + ok: true, + status: "not-required", + profile, + mode, + mutation: false, + valuesRedacted: true, + }; + } + if (profile === null) { + return { + ok: false, + status: "blocked", + reason: "exact-provider-profile-missing", + profile, + mode, + mutation: false, + valuesRedacted: true, + }; + } + const runtime = state.spec.codeAgentRuntime; + if (runtime === undefined) { + return { + ok: false, + status: "blocked", + reason: "code-agent-runtime-target-unavailable", + profile, + mode, + mutation: false, + valuesRedacted: true, + }; + } + const target = runtime.controlPlaneTarget; + const result = runChildCli([ + "agentrun", + "provider-profile", + "runtime-status", + "--node", + target.node, + "--lane", + target.lane, + "--profile", + profile, + "--raw", + ], Math.max(5, Math.min(timeoutSeconds, 55))); + const payload = cliDataPayload(result.parsed); + const runtimePayload = record(payload.runtime); + const secrets = record(runtimePayload.secrets); + const providerServices = record(runtimePayload.providerServices); + const ready = result.ok + && payload.ok === true + && payload.ownerSourceInspected === false + && payload.runtimeSecretValuesDecoded === false + && secrets.ready === true; + return { + ok: ready, + status: ready ? "ready" : "blocked", + reason: ready ? null : "exact-provider-profile-unavailable", + profile, + mode, + target: { + node: target.node, + lane: target.lane, + }, + runtime: { + ready: payload.ok === true, + secretsReady: secrets.ready === true, + providerServicesReady: providerServices.ready === true, + count: numberAtNullable(secrets, "count"), + presentCount: numberAtNullable(secrets, "presentCount"), + keyPresentCount: numberAtNullable(secrets, "keyPresentCount"), + missing: arrayAt(secrets, "items") + .map(record) + .filter((item) => item.present !== true || item.keyPresent !== true) + .slice(0, 8) + .map((item) => ({ + id: stringAtNullable(item, "id"), + namespace: stringAtNullable(item, "namespace"), + name: stringAtNullable(item, "name"), + key: stringAtNullable(item, "key"), + present: item.present === true, + keyPresent: item.keyPresent === true, + valuesRedacted: true, + })), + providerServices: arrayAt(providerServices, "items") + .map(record) + .slice(0, 8) + .map((item) => ({ + providerId: stringAtNullable(item, "providerId"), + namespace: stringAtNullable(item, "namespace"), + name: stringAtNullable(item, "name"), + port: numberAtNullable(item, "port"), + present: item.present === true, + portPresent: item.portPresent === true, + valuesRedacted: true, + })), + valuesRedacted: true, + }, + ownerSourceInspected: payload.ownerSourceInspected === true, + runtimeSecretValuesDecoded: payload.runtimeSecretValuesDecoded === true, + child: result.result, + mutation: false, + valuesRedacted: true, + }; +} + function readPromptSetForScenario(state: SentinelCicdState, scenario: Record): { ok: true; prompts: string[]; summary: Record } | { ok: false; error: string; summary: Record } { const promptSetRef = stringAt(scenario, "promptSetRef"); const promptSet = recordTarget(readWebProbeSentinelConfigRefTarget(state.spec, promptSetRef), promptSetRef); diff --git a/scripts/src/hwlab-node-web-sentinel-p5.ts b/scripts/src/hwlab-node-web-sentinel-p5.ts index 572d9b9a..346b0153 100644 --- a/scripts/src/hwlab-node-web-sentinel-p5.ts +++ b/scripts/src/hwlab-node-web-sentinel-p5.ts @@ -33,7 +33,7 @@ import { text, withWarnings, } from "./hwlab-node-web-sentinel-cicd"; -import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe"; +import { metricNames, reclassifyQuickVerifyControlFindings, runSentinelExactProviderProfilePreflight, runSentinelQuickVerify, sentinelP5Next, serviceUnavailableBlocker, validationBlocker } from "./hwlab-node-web-sentinel-p5-observe"; const SENTINEL_REPORT_ARTIFACT_READ_TIMEOUT_SECONDS = 55; @@ -54,27 +54,39 @@ export function runSentinelMaintenance(state: SentinelCicdState, options: Extrac }; return rendered(result.ok, command, renderMaintenanceResult(result)); } + const providerProfilePreflight = options.action === "stop" + ? runSentinelExactProviderProfilePreflight(state, options.timeoutSeconds) + : null; if (!options.confirm) { + const preflightOk = providerProfilePreflight === null || providerProfilePreflight.ok === true; const result = { - ok: serviceHealth.ok, + ok: serviceHealth.ok && preflightOk, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "dry-run", serviceHealth, + providerProfilePreflight, mutation: false, + jobCreated: false, + observerCreated: false, + runCreated: false, planned: { action: options.action, releaseId: options.releaseId, reason: options.reason, quickVerify: options.action === "stop" && options.quickVerify, }, + blocker: preflightOk ? null : exactProviderProfileBlocker(), next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderMaintenanceResult(result)); } - if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); + if (providerProfilePreflight !== null && providerProfilePreflight.ok !== true) { + const result = exactProviderProfileBlockedResult(state, command, options.wait ? "confirm-wait" : "confirm-async", serviceHealth, providerProfilePreflight); + return rendered(false, command, renderMaintenanceResult(result)); + } if (!serviceHealth.ok) { const result = { ok: false, @@ -90,6 +102,7 @@ export function runSentinelMaintenance(state: SentinelCicdState, options: Extrac }; return rendered(false, command, renderMaintenanceResult(result)); } + if (!options.wait) return renderAsyncP5Job(state, ["maintenance", options.action], options.timeoutSeconds, options.releaseId, options.reason, options.quickVerify); const body = { releaseId: options.releaseId, reason: options.reason, source: "unidesk-cli", valuesRedacted: true }; const mutation = callSentinelService(state, "POST", `/api/maintenance/${options.action}`, body, options.timeoutSeconds); const quickVerify = options.action === "stop" && options.quickVerify && mutation.ok @@ -103,6 +116,7 @@ export function runSentinelMaintenance(state: SentinelCicdState, options: Extrac mode: "confirm-wait", mutation: true, serviceHealth, + providerProfilePreflight, maintenance: mutation, quickVerify, blocker: mutation.ok ? null : serviceUnavailableBlocker(state), @@ -118,21 +132,43 @@ export function runSentinelValidate(state: SentinelCicdState, options: Extract | null = null; if (options.quickVerify) { + const providerProfilePreflight = runSentinelExactProviderProfilePreflight(state, options.timeoutSeconds); if (!options.confirm) { + quickVerify = { + ok: providerProfilePreflight.ok, + status: providerProfilePreflight.ok ? "planned" : "blocked", + reason: providerProfilePreflight.ok ? null : "exact-provider-profile-unavailable", + providerProfile: providerProfilePreflight.profile ?? null, + providerProfilePreflight, + observerCreated: false, + runCreated: false, + mutation: false, + valuesRedacted: true, + }; const result = { - ok: initialHealth.ok, + ok: initialHealth.ok && providerProfilePreflight.ok, command, node: state.spec.nodeId, lane: state.spec.lane, mode: "dry-run", serviceHealth: initialHealth, + quickVerify, planned: { quickVerify: true, waitRequired: true }, + blocker: providerProfilePreflight.ok ? null : { + code: "exact-provider-profile-unavailable", + blockers: ["exact-provider-profile-unavailable"], + reason: "exact-provider-profile-unavailable", + valuesRedacted: true, + }, next: sentinelP5Next(state), valuesRedacted: true, }; return rendered(result.ok, command, renderValidateResult(result)); } - if (!options.wait) return renderAsyncP5Job(state, ["validate"], options.timeoutSeconds, null, "manual-validate-quick-verify", true); + if (providerProfilePreflight.ok !== true) { + const result = exactProviderProfileBlockedResult(state, command, options.wait ? "confirm-wait" : "confirm-async", initialHealth, providerProfilePreflight); + return rendered(false, command, renderValidateResult(result)); + } if (!initialHealth.ok) { const result = { ok: false, @@ -147,6 +183,7 @@ export function runSentinelValidate(state: SentinelCicdState, options: Extract { + return { + code: "exact-provider-profile-unavailable", + blockers: ["exact-provider-profile-unavailable"], + reason: "exact-provider-profile-unavailable", + valuesRedacted: true, + }; +} + +function exactProviderProfileBlockedResult( + state: SentinelCicdState, + command: string, + mode: "confirm-wait" | "confirm-async", + serviceHealth: Record, + providerProfilePreflight: Record, +): Record & { ok: false } { + const quickVerify = { + ok: false, + status: "blocked", + reason: "exact-provider-profile-unavailable", + providerProfile: providerProfilePreflight.profile ?? null, + providerProfilePreflight, + observerCreated: false, + runCreated: false, + mutation: false, + valuesRedacted: true, + }; + return { + ok: false, + command, + node: state.spec.nodeId, + lane: state.spec.lane, + mode, + serviceHealth, + providerProfilePreflight, + mutation: false, + jobCreated: false, + observerCreated: false, + runCreated: false, + quickVerify, + blocker: exactProviderProfileBlocker(), + next: sentinelP5Next(state), + valuesRedacted: true, + }; +} + export function probeSentinelRuntimeHealthEndpoint(state: SentinelCicdState, timeoutSeconds: number): Record { const endpoint = stringAt(state.runtime, "healthPath"); const serviceProbeTimeoutSeconds = Math.min(timeoutSeconds, 20); @@ -2206,7 +2289,7 @@ function renderValidateResult(result: Record): string { ["recent-report", report.ok, `${record(record(report.bodyJson).run).id ?? "-"} ${short(record(record(report.bodyJson).run).report_json_sha256)}`], ["public-exposure", publicExposure.ok, `${record(publicExposure.dns).expectedA ?? "-"} http=${record(publicExposure.https).httpStatus ?? "-"}`], ["public-dashboard", publicDashboard.ok, `${record(publicDashboard.root).url ?? "-"} root=${record(publicDashboard.root).httpStatus ?? "-"} css=${record(publicDashboard.css).httpStatus ?? "-"} js=${record(publicDashboard.js).httpStatus ?? "-"}`], - ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? "-"} ${short(quickVerify.reportJsonSha256)}`], + ["quick-verify", Object.keys(quickVerify).length === 0 ? "skipped" : quickVerify.ok, `${quickVerify.runId ?? quickVerify.providerProfile ?? "-"} ${short(quickVerify.reportJsonSha256 ?? quickVerify.reason)}`], ]), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"),