diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 1a71e691..05fad3bb 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -236,15 +236,15 @@ bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 [--dry-ru bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 [--dry-run|--confirm] bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 [--dry-run|--confirm] bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 [--dry-run|--confirm] -bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--full] +bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--pipeline-run |--source-commit ] [--full|--raw] ``` - `plan`: 只读解析 YAML，输出控制面、source、image build、GitOps、runtime 和 Secret plan，不打印 Secret value - `apply`: 按 YAML 渲染并 apply Tekton RBAC/Pipeline、Argo AppProject/Application 和 runtime namespace - `secret-sync`: 按 YAML 的 Secret sourceRef/keyMapping 同步 runtime Secret 和外置 DB Secret，只输出 fingerprint - `restart`: patch manager Deployment 的 restart annotation 并等待 rollout，用于 Secret export/DB 连接串变化后让 workload 读取新 Secret；不要手工删除 Pod -- `trigger-current`: 确保 source branch/workspace，删除新 lane source branch 的 `deploy/deploy.json`，构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，触发 git-mirror sync 和 provenance PipelineRun；confirmed 运行可返回异步 job，必须用 `job status --full` 和 `status --pipeline-run ` 轮询收口 -- `status`: 汇总 node/lane 控制面、runtime、Argo、Secret、source workspace 和 GitOps 对齐状态 +- `trigger-current`: 确保 source branch/workspace，删除新 lane source branch 的 `deploy/deploy.json`，构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，触发 git-mirror sync 和 provenance PipelineRun；confirmed 运行可返回异步 job，必须用 `job status --tail-bytes 12000` 看 `agentrun-yaml-lane-trigger` progress，再用 `status --pipeline-run ` 轮询收口 +- `status`: 默认返回 compact commander JSON，关键结论在 `.data.summary` 和 `.data.alignment`，完整 YAML target、原始 source/runtime/gitMirror payload 和成功 probe tail 只在 `--full|--raw` 展开 YAML-only lane 的长步骤必须由 CLI 拆成短提交和状态轮询：source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建不得塞进一个顶层 `trans` 长连接。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败发布留下 dirty/detached/GitOps branch 状态，只清理已知发布残留并恢复到 lane source branch 后再重试。后台步骤的 `status` 和 `ok` 要共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不继续轮询到超时。 @@ -275,7 +275,7 @@ bun scripts/cli.ts agentrun control-plane cleanup-released-pvs \ - `cleanup-runs`: 只清理 `agentrun-ci` 中已完成 PipelineRun + 临时 PVC；不清理 `agentrun-v01` runtime runner Job/Pod/Secret - `cleanup-released-pvs`: 回收 Released PV -AgentRun `control-plane status` 的 compact JSON 关键字段在 `.data.sourceCommit`、`.data.expectedPipelineRun`、`.data.runtimeAlignment`、`.data.gitMirror.summary` 等位置，不要假设存在 `.data.status`。触发部署后如果 GitOps 已 promotion 但 git mirror `pendingFlush=true`，先执行 `bun scripts/cli.ts agentrun git-mirror flush --confirm --wait`，再 `control-plane refresh --confirm`，最后用 `control-plane status --full` 证明 `runtimeAlignment.localHeadMatchesOrigin=true`、`syncedToGitopsLatest=true`、`managerSourceMatchesExpected=true`。 +AgentRun `control-plane status` 的 compact JSON 关键字段在 `.data.summary.sourceCommit`、`.data.summary.expectedPipelineRun`、`.data.summary.runtimeAlignment`、`.data.summary.gitMirror`、`.data.summary.ci.pipelineRun`、`.data.summary.argo` 和 `.data.alignment`，不要假设存在 `.data.status`。触发部署后如果 GitOps 已 promotion 但 git mirror `pendingFlush=true`，先执行 `bun scripts/cli.ts agentrun git-mirror flush --confirm --wait`，再 `control-plane refresh --confirm`，最后用 `control-plane status --full` 证明 `.data.summary.runtimeAlignment.argoSyncedToGitops=true`、`.data.summary.runtimeAlignment.managerSourceMatchesExpected=true` 且 `.data.summary.ci.pipelineRun.status=True`。 ## AgentRun v0.1 Git Mirror diff --git a/docs/reference/agentrun.md b/docs/reference/agentrun.md index acdb049a..94af1c3f 100644 --- a/docs/reference/agentrun.md +++ b/docs/reference/agentrun.md @@ -92,9 +92,9 @@ bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 --full ``` -`status` 只读观察 YAML 选中 lane 的 source workspace 当前 commit、对应 PipelineRun、GitOps latest、Argo Application、runtime workload、manager source commit 和 git mirror 摘要，并报告 Argo revision 是否对齐该 lane 的 GitOps latest。默认输出是 compact commander 视图，只保留 `summary`、阶段耗时、对齐状态和 drill-down 命令；需要远端 stdout/stderr tail 时显式加 `--full`，需要原始 git mirror cache 输出时显式加 `--raw`。`status` 额外支持 `--pipeline-run ` 与 `--source-commit ` 定点查询，返回 `target`、`targetValidation` 和 `next.*` drill-down，便于直接判断某次 run 是成功、历史成功、运行中、缺失还是 source mismatch。`status` 会向 stderr 输出 `agentrun.control-plane.status.progress` 阶段事件，覆盖 `source`、`runtime` 和 `git-mirror`，避免长时间聚合时无可见进展。`trigger-current` 会先把 YAML 声明的 source worktree 快进到 lane source branch，再以当前 commit 创建 commit-pinned PipelineRun；同名 PipelineRun 正在运行或已经成功时必须拒绝重复触发，只允许在失败态或不存在时创建。该命令只提交 CI/CD 工作，不等待完整 PipelineRun 或 rollout 完成，后续用 `status` 轮询。`refresh` 只对 YAML 声明的 Argo Application 执行 hard refresh，用于 GitOps promotion 已完成但 Argo 仍停留旧 revision 时的受控同步入口；它不直接 patch runtime workload。 +`status` 只读观察 YAML 选中 lane 的 source workspace 当前 commit、对应 PipelineRun、GitOps latest、Argo Application、runtime workload、manager source commit 和 git mirror 摘要，并报告 Argo revision 是否对齐该 lane 的 GitOps latest。默认输出是 compact commander 视图：`target` 只保留 node/lane/source/runtime/CI/GitOps/git-mirror/database 摘要，关键结论在 `summary` 和 `alignment`，成功 probe 的 stdout/stderr tail、完整 YAML target、原始 `source`、`runtime`、`gitMirror` payload 默认省略；需要完整展开时使用返回的 `disclosure.fullCommand` 或显式加 `--full`，需要原始调试视图时加 `--raw`。`status` 额外支持 `--pipeline-run ` 与 `--source-commit ` 定点查询，并在 `next.*` 中返回后续 drill-down 命令，便于直接判断某次 run 的 PipelineRun、GitOps、Argo 和 manager source 是否对齐。`status` 会向 stderr 输出 `agentrun.control-plane.status.progress` 阶段事件，覆盖 `source`、`runtime` 和 `git-mirror`，避免长时间聚合时无可见进展。`trigger-current` 会先把 YAML 声明的 source worktree 快进到 lane source branch，再以当前 commit 创建 commit-pinned PipelineRun；同名 PipelineRun 正在运行或已经成功时必须拒绝重复触发，只允许在失败态或不存在时创建。该命令只提交 CI/CD 工作，不等待完整 PipelineRun 或 rollout 完成，后续用 `job status` 和 `status --pipeline-run ` 轮询。`refresh` 只对 YAML 声明的 Argo Application 执行 hard refresh，用于 GitOps promotion 已完成但 Argo 仍停留旧 revision 时的受控同步入口；它不直接 patch runtime workload。 -YAML-only lane 的 `trigger-current --confirm` 是受控长流程入口；source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建必须拆成短提交与状态轮询，不得把 clone、build、push 或长时间 polling 放进一个顶层 `trans` 长连接。`trigger-current` 返回异步 job 时，先用 `bun scripts/cli.ts job status --full` 观察 job 事件，再用 `agentrun control-plane status --node --lane --pipeline-run ` 观察 Tekton、GitOps 和 Argo 对齐。后台步骤的 `status` 与 `ok` 必须共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不能继续轮询到超时。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败 publish 已让固定 workspace dirty、detached 或停在 GitOps 分支，只清理已知生成产物/失败发布残留并恢复到 lane source branch 后再重试。 +YAML-only lane 的 `trigger-current --confirm` 是受控长流程入口；source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建必须拆成短提交与状态轮询，不得把 clone、build、push 或长时间 polling 放进一个顶层 `trans` 长连接。`trigger-current` 返回异步 job 时，先用 `bun scripts/cli.ts job status --tail-bytes 12000` 观察 `agentrun-yaml-lane-trigger` progress，再用 `agentrun control-plane status --node --lane --pipeline-run ` 观察 Tekton、GitOps 和 Argo 对齐。后台步骤的 `status` 与 `ok` 必须共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不能继续轮询到超时。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败 publish 已让固定 workspace dirty、detached 或停在 GitOps 分支，只清理已知生成产物/失败发布残留并恢复到 lane source branch 后再重试。 YAML-only lane 的 `trigger-current` 会先确保目标 source workspace/branch 存在，再从 UniDesk YAML 声明的 image build、GitOps branch/path、runtime namespace、Secret、数据库和 manager env 渲染 artifact catalog 与 GitOps desired state。该路径会删除新 lane source branch 中的 `deploy/deploy.json`，因为部署真相已经迁入 UniDesk YAML；旧 `v0.1` branch 中历史文件只作为迁移前遗留产物存在，不能作为新 lane 的事实来源。Secret export 格式或外部数据库连接参数变化时，先用 `platform-db postgres export-secrets --confirm` 物化本地 Secret source，再用 `agentrun control-plane secret-sync --node --lane --confirm` 下发，最后用 `agentrun control-plane restart --node --lane --confirm` 让 manager Deployment 通过 rollout 读取新 Secret；不要手工删除 Pod 或直接 patch Secret。 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 40fe3fed..099b9923 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -126,7 +126,7 @@ G14/D601 v03 的 bootstrap admin password 是 HWLAB runtime Secret 生命周期 - `codex interrupt|cancel ` 通过 Code Queue 私有代理请求中断；running/judging 任务会请求 D601 当前 agent run 停止，queued/retry_wait 任务的取消也必须保持与 WebUI 相同代理路径，返回有界 task 摘要和后续查询命令。任何需要接触 active run 的动作仍属于 D601 执行面。 - 旧 Code Queue 多队列 lane 现在是归档视图：`codex queues [--full|--all] [--limit N] [--page N|--offset N]` 只读展示历史 queue 摘要、activity、commanderConcurrency、counts 和 execution diagnostics。`queue create`、`queue merge`、`move` 等旧队列写入口冻结并返回 `legacy-code-queue-frozen`；AgentRun 新任务的排队、派发和取消必须使用 `agentrun create|apply|get|cancel`。 - 所有旧 `codex` 历史查询、已读和残留 interrupt/cancel 命令必须走与 WebUI 相同的 backend-core 私有代理路径 `/api/microservices/code-queue/proxy/...`。旧 submit/steer/resume/queue mutation/move/workdir mutation 不得绕过冻结；若需要新任务或新 session 控制，使用 AgentRun 资源原语。 -- `job list [--limit N] [--include-command]` 与 `job status [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态，是异步命令的可观测入口。`job list` 默认只返回最新 50 条摘要，并为已知异步工作流返回轻量 `progress.summary` 与后续查询命令；`job status` 默认返回结构化 `progress`、stdout/stderr 末尾 12000 字节、`tailPolicy` 与完整日志路径。已知工作流应从有界日志尾部抽取阶段、关键对象名和下一步命令，避免为了判断当前阶段而手工打开完整 stdout/stderr。`hwlab_g14_v02_trigger_current` 的 progress 必须暴露 trigger 阶段、source commit 和 PipelineRun；`hwlab_g14_v02_pr_monitor` 的 progress 必须暴露 preflight、merge、source-head、cd-trigger、cd-status、git-mirror-flush 和 pr-comment 阶段，以及 PR、source commit、PipelineRun、targetValidation/pendingFlush 摘要；`hwlab_g14_git_mirror_sync|flush` 与 `agentrun_v01_git_mirror_sync|flush` 的 progress 必须暴露 sync/flush 状态、Job 名、pendingFlush 与 fetch/push/total/SSH timing，并给出对应 repo 的 mirror status 命令。 +- `job list [--limit N] [--include-command]` 与 `job status [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态，是异步命令的可观测入口。`job list` 默认只返回最新 50 条摘要，并为已知异步工作流返回轻量 `progress.summary` 与后续查询命令；`job status` 默认返回结构化 `progress`、stdout/stderr 末尾 12000 字节、`tailPolicy` 与完整日志路径。已知工作流应从有界日志尾部抽取阶段、关键对象名和下一步命令，避免为了判断当前阶段而手工打开完整 stdout/stderr。`hwlab_g14_v02_trigger_current` 的 progress 必须暴露 trigger 阶段、source commit 和 PipelineRun；`hwlab_g14_v02_pr_monitor` 的 progress 必须暴露 preflight、merge、source-head、cd-trigger、cd-status、git-mirror-flush 和 pr-comment 阶段，以及 PR、source commit、PipelineRun、targetValidation/pendingFlush 摘要；`agentrun_vNN_trigger_current` 的 progress 必须识别 YAML lane 的 source-bootstrap、image-build、gitops-publish、git-mirror 阶段，暴露 source commit、PipelineRun、stage diagnostics、timing 和 `agentrun control-plane status --node --lane --pipeline-run `；`hwlab_g14_git_mirror_sync|flush` 与 `agentrun_v01_git_mirror_sync|flush` 的 progress 必须暴露 sync/flush 状态、Job 名、pendingFlush 与 fetch/push/total/SSH timing，并给出对应 repo 的 mirror status 命令。 - `debug health`、`debug ssh-pool `、`debug dispatch` 与 `debug task` 走真实内部 core、WebSocket、数据库、provider、系统指标、Docker 状态和 Host SSH 维护桥流程，只用于开发调试，不写入 `TEST.md` 的正式验收步骤；`debug ssh-pool` 只裁剪单个 provider 的 `providerGatewaySshData*` labels，用于低噪声判断 tcp-pool 是否 ready、claimed、exhausted 或有 lastError。 - `e2e run [--only pattern[,pattern...]] [--skip pattern[,pattern...]]` 使用 publicHost 派生的公开 production frontend/dev frontend/provider ingress URL，并通过 Docker 内网验证 core API、PostgreSQL、provider self-connection、系统指标曲线、Docker 状态快照、provider.upgrade 预检和 Playwright 前端页面，是交付前的自动化 E2E 门禁；CLI 默认输出 check 状态摘要，完整诊断写入 `resultPath`，日常迭代应优先用 `--only` / `--skip` 跑最小必要集合。 diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index bfbd1136..46ab8396 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -2117,52 +2117,74 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar ...(secrets.ready === true ? [] : ["lane-secret-missing"]), ...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []), ]; - return { + const aligned = blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected; + const runtimeAlignment = { + pipelineSucceeded, + argoRevision: stringOrNull(argo.revision), + argoSyncedToGitops, + managerSourceCommit: stringOrNull(manager.sourceCommit), + managerSourceMatchesExpected, + }; + const summary = { + aligned, + blockers, + sourceCommit, + expectedPipelineRun: pipelineRunName, + expectedGitopsRevision, + runtimeAlignment, + source: { + workspaceExists: sourcePayload.workspaceExists ?? false, + workspaceClean: sourcePayload.workspaceClean ?? null, + localHead: sourcePayload.localHead ?? null, + remoteBranchExists: sourcePayload.remoteBranchExists ?? false, + remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, + }, + gitMirror: { + readReady: mirrorPayload.readReady ?? false, + writeReady: mirrorPayload.writeReady ?? false, + cachePvcExists: mirrorPayload.cachePvcExists ?? false, + repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null, + sourceCommit: stringOrNull(mirrorPayload.sourceCommit), + gitopsCommit: expectedGitopsRevision, + }, + ci: { + namespaceExists: runtimePayload.ciNamespaceExists ?? false, + serviceAccountExists: runtimePayload.serviceAccountExists ?? false, + pipeline, + pipelineRun: pipelineRunStatus, + }, + argo, + runtime: { + namespaceExists: runtimePayload.runtimeNamespaceExists ?? false, + manager: { + deploymentExists: manager.deploymentExists ?? false, + serviceExists: manager.serviceExists ?? false, + deployment: manager.deployment ?? null, + service: manager.service ?? null, + image: manager.image ?? null, + sourceCommit: stringOrNull(manager.sourceCommit), + sourceMatchesExpected: managerSourceMatchesExpected, + }, + database, + secrets: compactLaneSecretsStatus(secrets), + localPostgres, + }, + }; + const statusFullCommand = agentRunControlPlaneStatusCommand(spec, options, true); + const result: Record = { ok: sourceProbe.value.exitCode === 0 && runtimeProbe.value.exitCode === 0 && mirrorProbe.value.exitCode === 0 && blockers.length === 0, command: "agentrun control-plane status", mode: "yaml-declared-node-lane", configPath: target.configPath, - target: agentRunLaneSummary(spec), - summary: { - aligned: blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected, + target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec), + summary, + alignment: { + aligned, blockers, sourceCommit, expectedPipelineRun: pipelineRunName, expectedGitopsRevision, - runtimeAlignment: { - pipelineSucceeded, - argoRevision: stringOrNull(argo.revision), - argoSyncedToGitops, - managerSourceCommit: stringOrNull(manager.sourceCommit), - managerSourceMatchesExpected, - }, - source: { - workspaceExists: sourcePayload.workspaceExists ?? false, - workspaceClean: sourcePayload.workspaceClean ?? null, - localHead: sourcePayload.localHead ?? null, - remoteBranchExists: sourcePayload.remoteBranchExists ?? false, - remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null, - }, - gitMirror: { - readReady: mirrorPayload.readReady ?? false, - writeReady: mirrorPayload.writeReady ?? false, - cachePvcExists: mirrorPayload.cachePvcExists ?? false, - repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null, - }, - ci: { - namespaceExists: runtimePayload.ciNamespaceExists ?? false, - serviceAccountExists: runtimePayload.serviceAccountExists ?? false, - pipeline, - pipelineRun: pipelineRunStatus, - }, - argo, - runtime: { - namespaceExists: runtimePayload.runtimeNamespaceExists ?? false, - manager, - database, - secrets, - localPostgres, - }, + runtimeAlignment, }, timings: { sourceMs: sourceProbe.elapsedMs, @@ -2170,24 +2192,118 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar gitMirrorMs: mirrorProbe.elapsedMs, totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs), }, - source: sourcePayload, - runtime: runtimePayload, - gitMirror: mirrorPayload, captures: { source: compactCapture(sourceProbe.value, { full: options.full || options.raw || sourceProbe.value.exitCode !== 0 }), runtime: compactCapture(runtimeProbe.value, { full: options.full || options.raw || runtimeProbe.value.exitCode !== 0 }), gitMirror: compactCapture(mirrorProbe.value, { full: options.full || options.raw || mirrorProbe.value.exitCode !== 0 }), }, + disclosure: { + output: options.full || options.raw ? "full" : "compact-summary", + full: options.full, + raw: options.raw, + omittedWhenCompact: options.full || options.raw ? [] : ["full target YAML summary", "source", "runtime", "gitMirror", "capture stdout/stderr tails for successful probes"], + fullCommand: statusFullCommand, + }, next: { plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`, postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null, postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null, secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`, - statusFull: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --full`, + statusFull: statusFullCommand, }, valuesPrinted: false, }; + if (options.full || options.raw) { + result.source = sourcePayload; + result.runtime = runtimePayload; + result.gitMirror = mirrorPayload; + } + return result; +} + +function agentRunControlPlaneStatusCommand(spec: AgentRunLaneSpec, options: Pick, full: boolean): string { + return [ + "bun scripts/cli.ts agentrun control-plane status", + `--node ${spec.nodeId}`, + `--lane ${spec.lane}`, + options.pipelineRun === null ? null : `--pipeline-run ${options.pipelineRun}`, + options.sourceCommit === null ? null : `--source-commit ${options.sourceCommit}`, + full ? "--full" : null, + ].filter(Boolean).join(" "); +} + +function compactLaneSecretsStatus(secrets: Record): Record { + const items = Array.isArray(secrets.items) ? secrets.items.filter((item): item is Record => typeof item === "object" && item !== null && !Array.isArray(item)) : []; + const missing = items + .filter((item) => item.present !== true || item.keyPresent !== true) + .map((item) => ({ + namespace: item.namespace ?? null, + name: item.name ?? null, + key: item.key ?? null, + present: item.present ?? false, + keyPresent: item.keyPresent ?? false, + })); + return { + ready: secrets.ready ?? false, + count: secrets.count ?? items.length, + missingCount: missing.length, + missing, + valuesPrinted: false, + }; +} + +function compactAgentRunLaneStatusTarget(spec: AgentRunLaneSpec): Record { + return { + node: { + id: spec.nodeId, + route: spec.nodeRoute, + kubeRoute: spec.nodeKubeRoute, + }, + lane: spec.lane, + version: spec.version, + source: { + repository: spec.source.repository, + branch: spec.source.branch, + workspace: spec.source.workspace, + }, + runtime: { + namespace: spec.runtime.namespace, + managerDeployment: spec.runtime.managerDeployment, + managerService: spec.runtime.managerService, + internalBaseUrl: spec.runtime.internalBaseUrl, + }, + ci: { + namespace: spec.ci.namespace, + pipeline: spec.ci.pipeline, + pipelineRunPrefix: spec.ci.pipelineRunPrefix, + registryPrefix: spec.ci.registryPrefix, + }, + gitops: { + branch: spec.gitops.branch, + path: spec.gitops.path, + argoNamespace: spec.gitops.argoNamespace, + argoApplication: spec.gitops.argoApplication, + }, + gitMirror: { + namespace: spec.gitMirror.namespace, + readService: spec.gitMirror.readService, + writeService: spec.gitMirror.writeService, + repositoryCount: spec.gitMirror.repositories.length, + }, + database: { + mode: spec.database.mode, + provider: spec.database.provider, + configRef: spec.database.configRef, + database: spec.database.database, + user: spec.database.user, + secretRef: spec.database.secretRef, + localPostgresExpectedAbsent: spec.database.localPostgresExpectedAbsent, + valuesPrinted: false, + }, + secretCount: spec.secrets.length, + valuesPrinted: false, + }; } async function restartYamlLane(config: UniDeskConfig, options: LaneConfirmOptions): Promise> { diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index b8d885ed..44e83794 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -26,7 +26,7 @@ export interface JobRecord { } export interface JobProgressSummary { - kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "git-mirror" | "generic"; + kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic"; stage: string | null; stageStatus: string | null; sourceCommit: string | null; @@ -229,15 +229,17 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current"; const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor"; const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name); + const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name); const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush"; const ciInstallWorkflow = job.name === "ci_install"; if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs); - if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail); + if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail); const progressTailBytes = Math.max(4096, Math.floor(maxBytes)); const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes); const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes); if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs); if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs); + if (agentRunYamlLaneTriggerWorkflow) return summarizeAgentRunYamlLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs); const events = v02PrMonitorWorkflow ? [ ...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"), @@ -479,6 +481,123 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri }; } +const agentRunYamlLaneProgressEvents = [ + { event: "agentrun.yaml-lane.source-bootstrap.progress", stage: "source-bootstrap" }, + { event: "agentrun.yaml-lane.image-build.progress", stage: "image-build" }, + { event: "agentrun.yaml-lane.gitops-publish.progress", stage: "gitops-publish" }, + { event: "agentrun.yaml-lane.git-mirror.progress", stage: "git-mirror" }, +] as const; + +function summarizeAgentRunYamlLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary { + const events = agentRunYamlLaneProgressEvents + .flatMap(({ event, stage }) => parseJsonLineEvents(stderrTail, event).map((item) => ({ ...item, stage }))) + .sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? ""))); + const lastEvent = events.at(-1) ?? {}; + const stage = stringField(lastEvent.stage); + const stageStatus = agentRunYamlLaneStageStatus(lastEvent); + const sourceCommit = lastStringField(events, "sourceCommit") ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu); + const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u); + const pipelineCreatedValue = firstMatch(stdoutTail, /"created"\s*:\s*\{[\s\S]{0,400}?"created"\s*:\s*(true|false)/u); + const pipelineCreated = pipelineCreatedValue === "true" ? true : pipelineCreatedValue === "false" ? false : null; + const lastEventAt = stringField(lastEvent.at); + const elapsedSeconds = jobElapsedSeconds(job, nowMs); + const eventElapsedMs = numberField(lastEvent.elapsedMs); + const stageElapsedSeconds = eventElapsedMs === null ? null : Math.round(eventElapsedMs / 1000); + const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs); + const timings = agentRunYamlLaneTimings(events); + const warnings = jobProgressWarnings({ + job, + eventsObserved: events.length, + elapsedSeconds, + stage, + stageStatus, + stageElapsedSeconds, + lastEventAgeSeconds, + }); + const slow = warnings.length > 0; + const eventNode = stringField(lastEvent.node); + const eventLane = stringField(lastEvent.lane) ?? firstMatch(job.name, /^agentrun_(v[0-9]{2})_trigger_current$/u); + const nextCommand = pipelineRun + ? [ + "bun scripts/cli.ts agentrun control-plane status", + eventNode ? `--node ${eventNode}` : null, + eventLane ? `--lane ${eventLane}` : null, + `--pipeline-run ${pipelineRun}`, + ].filter(Boolean).join(" ") + : job.status === "running" + ? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000` + : null; + return { + kind: "agentrun-yaml-lane-trigger", + stage, + stageStatus, + sourceCommit, + pipelineRun, + pipelineCreated, + elapsedSeconds, + stageElapsedSeconds, + lastEventAt, + lastEventAgeSeconds, + eventsObserved: events.length, + slow, + warnings, + diagnostics: { + stages: agentRunYamlLaneStageDiagnostics(events), + }, + timings, + summary: [ + job.status, + stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown", + sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null, + pipelineRun ? `pipelineRun=${pipelineRun}` : null, + pipelineCreated === true ? "created" : pipelineCreated === false ? "create-reused" : null, + elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null, + stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null, + lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null, + `events=${events.length}`, + slow ? "visibility-warning" : null, + ].filter(Boolean).join(" "), + nextCommand, + }; +} + +function agentRunYamlLaneStageStatus(event: Record): string | null { + const explicitStatus = stringField(event.status); + if (explicitStatus !== null) return explicitStatus; + if (event.succeeded === true) return "succeeded"; + if (event.failed === true) return "failed"; + if (event.stage !== undefined) return "running"; + return null; +} + +function agentRunYamlLaneTimings(events: Record[]): Record { + const timings: Record = {}; + for (const event of events) { + const stage = stringField(event.stage); + const elapsedMs = numberField(event.elapsedMs); + if (stage !== null && elapsedMs !== null) timings[`agentrunYamlLane.${stage}.elapsedMs`] = Math.round(elapsedMs); + } + return timings; +} + +function agentRunYamlLaneStageDiagnostics(events: Record[]): Array> { + return agentRunYamlLaneProgressEvents + .map(({ stage }) => { + const stageEvents = events.filter((event) => stringField(event.stage) === stage); + const lastEvent = stageEvents.at(-1) ?? {}; + return { + stage, + eventsObserved: stageEvents.length, + status: agentRunYamlLaneStageStatus(lastEvent), + jobId: stringField(lastEvent.jobId), + jobName: stringField(lastEvent.jobName), + polls: numberField(lastEvent.polls), + elapsedMs: numberField(lastEvent.elapsedMs), + }; + }) + .filter((item) => item.eventsObserved > 0); +} + function hwlabRuntimeLaneStatusNextCommand(pipelineRun: string | null, lastEvent: Record): { command: string | null; warning: string | null } { if (pipelineRun === null) return { command: null, warning: null }; const eventNode = stringField(lastEvent.node); @@ -738,6 +857,20 @@ function stringField(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } +function numberField(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value); + if (Number.isFinite(parsed)) return parsed; + } + return null; +} + +function lastStringField(events: Record[], field: string): string | null { + const event = events.findLast((item) => stringField(item[field]) !== null); + return stringField(event?.[field]); +} + function firstMatch(text: string, pattern: RegExp): string | null { const match = pattern.exec(text); return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null;