Merge pull request #452 from pikasTech/fix/451-agentrun-status-summary
fix: 改进 AgentRun 状态可见性
This commit is contained in:
Lyon
@@ -236,15 +236,15 @@ bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 [--dry-ru
|
||||
bun scripts/cli.ts agentrun control-plane secret-sync --node D601 --lane v02 [--dry-run|--confirm]
|
||||
bun scripts/cli.ts agentrun control-plane restart --node D601 --lane v02 [--dry-run|--confirm]
|
||||
bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02 [--dry-run|--confirm]
|
||||
bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--full]
|
||||
bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 [--pipeline-run <name>|--source-commit <sha>] [--full|--raw]
|
||||
```
|
||||
|
||||
- `plan`: 只读解析 YAML，输出控制面、source、image build、GitOps、runtime 和 Secret plan，不打印 Secret value
|
||||
- `apply`: 按 YAML 渲染并 apply Tekton RBAC/Pipeline、Argo AppProject/Application 和 runtime namespace
|
||||
- `secret-sync`: 按 YAML 的 Secret sourceRef/keyMapping 同步 runtime Secret 和外置 DB Secret，只输出 fingerprint
|
||||
- `restart`: patch manager Deployment 的 restart annotation 并等待 rollout，用于 Secret export/DB 连接串变化后让 workload 读取新 Secret；不要手工删除 Pod
|
||||
- `trigger-current`: 确保 source branch/workspace，删除新 lane source branch 的 `deploy/deploy.json`，构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，触发 git-mirror sync 和 provenance PipelineRun；confirmed 运行可返回异步 job，必须用 `job status <jobId> --full` 和 `status --pipeline-run <name>` 轮询收口
|
||||
- `status`: 汇总 node/lane 控制面、runtime、Argo、Secret、source workspace 和 GitOps 对齐状态
|
||||
- `trigger-current`: 确保 source branch/workspace，删除新 lane source branch 的 `deploy/deploy.json`，构建并推送 YAML 声明的 image，渲染 GitOps/artifact catalog，触发 git-mirror sync 和 provenance PipelineRun；confirmed 运行可返回异步 job，必须用 `job status <jobId> --tail-bytes 12000` 看 `agentrun-yaml-lane-trigger` progress，再用 `status --pipeline-run <name>` 轮询收口
|
||||
- `status`: 默认返回 compact commander JSON，关键结论在 `.data.summary` 和 `.data.alignment`，完整 YAML target、原始 source/runtime/gitMirror payload 和成功 probe tail 只在 `--full|--raw` 展开
|
||||
|
||||
YAML-only lane 的长步骤必须由 CLI 拆成短提交和状态轮询：source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建不得塞进一个顶层 `trans` 长连接。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败发布留下 dirty/detached/GitOps branch 状态，只清理已知发布残留并恢复到 lane source branch 后再重试。后台步骤的 `status` 和 `ok` 要共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不继续轮询到超时。
|
||||
|
||||
@@ -275,7 +275,7 @@ bun scripts/cli.ts agentrun control-plane cleanup-released-pvs \
|
||||
- `cleanup-runs`: 只清理 `agentrun-ci` 中已完成 PipelineRun + 临时 PVC；不清理 `agentrun-v01` runtime runner Job/Pod/Secret
|
||||
- `cleanup-released-pvs`: 回收 Released PV
|
||||
|
||||
AgentRun `control-plane status` 的 compact JSON 关键字段在 `.data.sourceCommit`、`.data.expectedPipelineRun`、`.data.runtimeAlignment`、`.data.gitMirror.summary` 等位置，不要假设存在 `.data.status`。触发部署后如果 GitOps 已 promotion 但 git mirror `pendingFlush=true`，先执行 `bun scripts/cli.ts agentrun git-mirror flush --confirm --wait`，再 `control-plane refresh --confirm`，最后用 `control-plane status --full` 证明 `runtimeAlignment.localHeadMatchesOrigin=true`、`syncedToGitopsLatest=true`、`managerSourceMatchesExpected=true`。
|
||||
AgentRun `control-plane status` 的 compact JSON 关键字段在 `.data.summary.sourceCommit`、`.data.summary.expectedPipelineRun`、`.data.summary.runtimeAlignment`、`.data.summary.gitMirror`、`.data.summary.ci.pipelineRun`、`.data.summary.argo` 和 `.data.alignment`，不要假设存在 `.data.status`。触发部署后如果 GitOps 已 promotion 但 git mirror `pendingFlush=true`，先执行 `bun scripts/cli.ts agentrun git-mirror flush --confirm --wait`，再 `control-plane refresh --confirm`，最后用 `control-plane status --full` 证明 `.data.summary.runtimeAlignment.argoSyncedToGitops=true`、`.data.summary.runtimeAlignment.managerSourceMatchesExpected=true` 且 `.data.summary.ci.pipelineRun.status=True`。
|
||||
|
||||
## AgentRun v0.1 Git Mirror
|
||||
|
||||
|
||||
@@ -92,9 +92,9 @@ bun scripts/cli.ts agentrun control-plane trigger-current --node D601 --lane v02
|
||||
bun scripts/cli.ts agentrun control-plane status --node D601 --lane v02 --full
|
||||
```
|
||||
|
||||
`status` 只读观察 YAML 选中 lane 的 source workspace 当前 commit、对应 PipelineRun、GitOps latest、Argo Application、runtime workload、manager source commit 和 git mirror 摘要，并报告 Argo revision 是否对齐该 lane 的 GitOps latest。默认输出是 compact commander 视图，只保留 `summary`、阶段耗时、对齐状态和 drill-down 命令；需要远端 stdout/stderr tail 时显式加 `--full`，需要原始 git mirror cache 输出时显式加 `--raw`。`status` 额外支持 `--pipeline-run <name>` 与 `--source-commit <sha>` 定点查询，返回 `target`、`targetValidation` 和 `next.*` drill-down，便于直接判断某次 run 是成功、历史成功、运行中、缺失还是 source mismatch。`status` 会向 stderr 输出 `agentrun.control-plane.status.progress` 阶段事件，覆盖 `source`、`runtime` 和 `git-mirror`，避免长时间聚合时无可见进展。`trigger-current` 会先把 YAML 声明的 source worktree 快进到 lane source branch，再以当前 commit 创建 commit-pinned PipelineRun；同名 PipelineRun 正在运行或已经成功时必须拒绝重复触发，只允许在失败态或不存在时创建。该命令只提交 CI/CD 工作，不等待完整 PipelineRun 或 rollout 完成，后续用 `status` 轮询。`refresh` 只对 YAML 声明的 Argo Application 执行 hard refresh，用于 GitOps promotion 已完成但 Argo 仍停留旧 revision 时的受控同步入口；它不直接 patch runtime workload。
|
||||
`status` 只读观察 YAML 选中 lane 的 source workspace 当前 commit、对应 PipelineRun、GitOps latest、Argo Application、runtime workload、manager source commit 和 git mirror 摘要，并报告 Argo revision 是否对齐该 lane 的 GitOps latest。默认输出是 compact commander 视图：`target` 只保留 node/lane/source/runtime/CI/GitOps/git-mirror/database 摘要，关键结论在 `summary` 和 `alignment`，成功 probe 的 stdout/stderr tail、完整 YAML target、原始 `source`、`runtime`、`gitMirror` payload 默认省略；需要完整展开时使用返回的 `disclosure.fullCommand` 或显式加 `--full`，需要原始调试视图时加 `--raw`。`status` 额外支持 `--pipeline-run <name>` 与 `--source-commit <sha>` 定点查询，并在 `next.*` 中返回后续 drill-down 命令，便于直接判断某次 run 的 PipelineRun、GitOps、Argo 和 manager source 是否对齐。`status` 会向 stderr 输出 `agentrun.control-plane.status.progress` 阶段事件，覆盖 `source`、`runtime` 和 `git-mirror`，避免长时间聚合时无可见进展。`trigger-current` 会先把 YAML 声明的 source worktree 快进到 lane source branch，再以当前 commit 创建 commit-pinned PipelineRun；同名 PipelineRun 正在运行或已经成功时必须拒绝重复触发，只允许在失败态或不存在时创建。该命令只提交 CI/CD 工作，不等待完整 PipelineRun 或 rollout 完成，后续用 `job status` 和 `status --pipeline-run <name>` 轮询。`refresh` 只对 YAML 声明的 Argo Application 执行 hard refresh，用于 GitOps promotion 已完成但 Argo 仍停留旧 revision 时的受控同步入口；它不直接 patch runtime workload。
|
||||
|
||||
YAML-only lane 的 `trigger-current --confirm` 是受控长流程入口；source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建必须拆成短提交与状态轮询，不得把 clone、build、push 或长时间 polling 放进一个顶层 `trans` 长连接。`trigger-current` 返回异步 job 时，先用 `bun scripts/cli.ts job status <jobId> --full` 观察 job 事件，再用 `agentrun control-plane status --node <node> --lane <lane> --pipeline-run <name>` 观察 Tekton、GitOps 和 Argo 对齐。后台步骤的 `status` 与 `ok` 必须共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不能继续轮询到超时。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败 publish 已让固定 workspace dirty、detached 或停在 GitOps 分支，只清理已知生成产物/失败发布残留并恢复到 lane source branch 后再重试。
|
||||
YAML-only lane 的 `trigger-current --confirm` 是受控长流程入口；source bootstrap、image build、GitOps publish、git-mirror sync 和 PipelineRun 创建必须拆成短提交与状态轮询，不得把 clone、build、push 或长时间 polling 放进一个顶层 `trans` 长连接。`trigger-current` 返回异步 job 时，先用 `bun scripts/cli.ts job status <jobId> --tail-bytes 12000` 观察 `agentrun-yaml-lane-trigger` progress，再用 `agentrun control-plane status --node <node> --lane <lane> --pipeline-run <name>` 观察 Tekton、GitOps 和 Argo 对齐。后台步骤的 `status` 与 `ok` 必须共同判定，`status=succeeded` 但 `ok=false` 是终态失败，不能继续轮询到超时。GitOps publish 必须使用隔离临时 clone/worktree，不能切换或污染 YAML 声明的固定 source workspace；如果历史失败 publish 已让固定 workspace dirty、detached 或停在 GitOps 分支，只清理已知生成产物/失败发布残留并恢复到 lane source branch 后再重试。
|
||||
|
||||
YAML-only lane 的 `trigger-current` 会先确保目标 source workspace/branch 存在，再从 UniDesk YAML 声明的 image build、GitOps branch/path、runtime namespace、Secret、数据库和 manager env 渲染 artifact catalog 与 GitOps desired state。该路径会删除新 lane source branch 中的 `deploy/deploy.json`，因为部署真相已经迁入 UniDesk YAML；旧 `v0.1` branch 中历史文件只作为迁移前遗留产物存在，不能作为新 lane 的事实来源。Secret export 格式或外部数据库连接参数变化时，先用 `platform-db postgres export-secrets --confirm` 物化本地 Secret source，再用 `agentrun control-plane secret-sync --node <node> --lane <lane> --confirm` 下发，最后用 `agentrun control-plane restart --node <node> --lane <lane> --confirm` 让 manager Deployment 通过 rollout 读取新 Secret；不要手工删除 Pod 或直接 patch Secret。
|
||||
|
||||
|
||||
@@ -126,7 +126,7 @@ G14/D601 v03 的 bootstrap admin password 是 HWLAB runtime Secret 生命周期
|
||||
- `codex interrupt|cancel <taskId>` 通过 Code Queue 私有代理请求中断；running/judging 任务会请求 D601 当前 agent run 停止，queued/retry_wait 任务的取消也必须保持与 WebUI 相同代理路径，返回有界 task 摘要和后续查询命令。任何需要接触 active run 的动作仍属于 D601 执行面。
|
||||
- 旧 Code Queue 多队列 lane 现在是归档视图：`codex queues [--full|--all] [--limit N] [--page N|--offset N]` 只读展示历史 queue 摘要、activity、commanderConcurrency、counts 和 execution diagnostics。`queue create`、`queue merge`、`move` 等旧队列写入口冻结并返回 `legacy-code-queue-frozen`；AgentRun 新任务的排队、派发和取消必须使用 `agentrun create|apply|get|cancel`。
|
||||
- 所有旧 `codex` 历史查询、已读和残留 interrupt/cancel 命令必须走与 WebUI 相同的 backend-core 私有代理路径 `/api/microservices/code-queue/proxy/...`。旧 submit/steer/resume/queue mutation/move/workdir mutation 不得绕过冻结；若需要新任务或新 session 控制，使用 AgentRun 资源原语。
|
||||
- `job list [--limit N] [--include-command]` 与 `job status <jobId|latest> [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态，是异步命令的可观测入口。`job list` 默认只返回最新 50 条摘要，并为已知异步工作流返回轻量 `progress.summary` 与后续查询命令；`job status` 默认返回结构化 `progress`、stdout/stderr 末尾 12000 字节、`tailPolicy` 与完整日志路径。已知工作流应从有界日志尾部抽取阶段、关键对象名和下一步命令，避免为了判断当前阶段而手工打开完整 stdout/stderr。`hwlab_g14_v02_trigger_current` 的 progress 必须暴露 trigger 阶段、source commit 和 PipelineRun；`hwlab_g14_v02_pr_monitor` 的 progress 必须暴露 preflight、merge、source-head、cd-trigger、cd-status、git-mirror-flush 和 pr-comment 阶段，以及 PR、source commit、PipelineRun、targetValidation/pendingFlush 摘要；`hwlab_g14_git_mirror_sync|flush` 与 `agentrun_v01_git_mirror_sync|flush` 的 progress 必须暴露 sync/flush 状态、Job 名、pendingFlush 与 fetch/push/total/SSH timing，并给出对应 repo 的 mirror status 命令。
|
||||
- `job list [--limit N] [--include-command]` 与 `job status <jobId|latest> [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态，是异步命令的可观测入口。`job list` 默认只返回最新 50 条摘要，并为已知异步工作流返回轻量 `progress.summary` 与后续查询命令；`job status` 默认返回结构化 `progress`、stdout/stderr 末尾 12000 字节、`tailPolicy` 与完整日志路径。已知工作流应从有界日志尾部抽取阶段、关键对象名和下一步命令，避免为了判断当前阶段而手工打开完整 stdout/stderr。`hwlab_g14_v02_trigger_current` 的 progress 必须暴露 trigger 阶段、source commit 和 PipelineRun；`hwlab_g14_v02_pr_monitor` 的 progress 必须暴露 preflight、merge、source-head、cd-trigger、cd-status、git-mirror-flush 和 pr-comment 阶段，以及 PR、source commit、PipelineRun、targetValidation/pendingFlush 摘要；`agentrun_vNN_trigger_current` 的 progress 必须识别 YAML lane 的 source-bootstrap、image-build、gitops-publish、git-mirror 阶段，暴露 source commit、PipelineRun、stage diagnostics、timing 和 `agentrun control-plane status --node <node> --lane <lane> --pipeline-run <name>`；`hwlab_g14_git_mirror_sync|flush` 与 `agentrun_v01_git_mirror_sync|flush` 的 progress 必须暴露 sync/flush 状态、Job 名、pendingFlush 与 fetch/push/total/SSH timing，并给出对应 repo 的 mirror status 命令。
|
||||
- `debug health`、`debug ssh-pool <providerId>`、`debug dispatch` 与 `debug task` 走真实内部 core、WebSocket、数据库、provider、系统指标、Docker 状态和 Host SSH 维护桥流程，只用于开发调试，不写入 `TEST.md` 的正式验收步骤；`debug ssh-pool` 只裁剪单个 provider 的 `providerGatewaySshData*` labels，用于低噪声判断 tcp-pool 是否 ready、claimed、exhausted 或有 lastError。
|
||||
- `e2e run [--only pattern[,pattern...]] [--skip pattern[,pattern...]]` 使用 publicHost 派生的公开 production frontend/dev frontend/provider ingress URL，并通过 Docker 内网验证 core API、PostgreSQL、provider self-connection、系统指标曲线、Docker 状态快照、provider.upgrade 预检和 Playwright 前端页面，是交付前的自动化 E2E 门禁；CLI 默认输出 check 状态摘要，完整诊断写入 `resultPath`，日常迭代应优先用 `--only` / `--skip` 跑最小必要集合。
|
||||
|
||||
|
||||
+158
-42
@@ -2117,52 +2117,74 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
...(secrets.ready === true ? [] : ["lane-secret-missing"]),
|
||||
...(spec.database.localPostgresExpectedAbsent && localPostgres.absent !== true ? ["local-postgres-present"] : []),
|
||||
];
|
||||
return {
|
||||
const aligned = blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected;
|
||||
const runtimeAlignment = {
|
||||
pipelineSucceeded,
|
||||
argoRevision: stringOrNull(argo.revision),
|
||||
argoSyncedToGitops,
|
||||
managerSourceCommit: stringOrNull(manager.sourceCommit),
|
||||
managerSourceMatchesExpected,
|
||||
};
|
||||
const summary = {
|
||||
aligned,
|
||||
blockers,
|
||||
sourceCommit,
|
||||
expectedPipelineRun: pipelineRunName,
|
||||
expectedGitopsRevision,
|
||||
runtimeAlignment,
|
||||
source: {
|
||||
workspaceExists: sourcePayload.workspaceExists ?? false,
|
||||
workspaceClean: sourcePayload.workspaceClean ?? null,
|
||||
localHead: sourcePayload.localHead ?? null,
|
||||
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
|
||||
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
|
||||
},
|
||||
gitMirror: {
|
||||
readReady: mirrorPayload.readReady ?? false,
|
||||
writeReady: mirrorPayload.writeReady ?? false,
|
||||
cachePvcExists: mirrorPayload.cachePvcExists ?? false,
|
||||
repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null,
|
||||
sourceCommit: stringOrNull(mirrorPayload.sourceCommit),
|
||||
gitopsCommit: expectedGitopsRevision,
|
||||
},
|
||||
ci: {
|
||||
namespaceExists: runtimePayload.ciNamespaceExists ?? false,
|
||||
serviceAccountExists: runtimePayload.serviceAccountExists ?? false,
|
||||
pipeline,
|
||||
pipelineRun: pipelineRunStatus,
|
||||
},
|
||||
argo,
|
||||
runtime: {
|
||||
namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
|
||||
manager: {
|
||||
deploymentExists: manager.deploymentExists ?? false,
|
||||
serviceExists: manager.serviceExists ?? false,
|
||||
deployment: manager.deployment ?? null,
|
||||
service: manager.service ?? null,
|
||||
image: manager.image ?? null,
|
||||
sourceCommit: stringOrNull(manager.sourceCommit),
|
||||
sourceMatchesExpected: managerSourceMatchesExpected,
|
||||
},
|
||||
database,
|
||||
secrets: compactLaneSecretsStatus(secrets),
|
||||
localPostgres,
|
||||
},
|
||||
};
|
||||
const statusFullCommand = agentRunControlPlaneStatusCommand(spec, options, true);
|
||||
const result: Record<string, unknown> = {
|
||||
ok: sourceProbe.value.exitCode === 0 && runtimeProbe.value.exitCode === 0 && mirrorProbe.value.exitCode === 0 && blockers.length === 0,
|
||||
command: "agentrun control-plane status",
|
||||
mode: "yaml-declared-node-lane",
|
||||
configPath: target.configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
summary: {
|
||||
aligned: blockers.length === 0 && pipelineSucceeded && argoSyncedToGitops && managerSourceMatchesExpected,
|
||||
target: options.full || options.raw ? agentRunLaneSummary(spec) : compactAgentRunLaneStatusTarget(spec),
|
||||
summary,
|
||||
alignment: {
|
||||
aligned,
|
||||
blockers,
|
||||
sourceCommit,
|
||||
expectedPipelineRun: pipelineRunName,
|
||||
expectedGitopsRevision,
|
||||
runtimeAlignment: {
|
||||
pipelineSucceeded,
|
||||
argoRevision: stringOrNull(argo.revision),
|
||||
argoSyncedToGitops,
|
||||
managerSourceCommit: stringOrNull(manager.sourceCommit),
|
||||
managerSourceMatchesExpected,
|
||||
},
|
||||
source: {
|
||||
workspaceExists: sourcePayload.workspaceExists ?? false,
|
||||
workspaceClean: sourcePayload.workspaceClean ?? null,
|
||||
localHead: sourcePayload.localHead ?? null,
|
||||
remoteBranchExists: sourcePayload.remoteBranchExists ?? false,
|
||||
remoteBranchCommit: sourcePayload.remoteBranchCommit ?? null,
|
||||
},
|
||||
gitMirror: {
|
||||
readReady: mirrorPayload.readReady ?? false,
|
||||
writeReady: mirrorPayload.writeReady ?? false,
|
||||
cachePvcExists: mirrorPayload.cachePvcExists ?? false,
|
||||
repositoryCount: Array.isArray(mirrorPayload.repositories) ? mirrorPayload.repositories.length : null,
|
||||
},
|
||||
ci: {
|
||||
namespaceExists: runtimePayload.ciNamespaceExists ?? false,
|
||||
serviceAccountExists: runtimePayload.serviceAccountExists ?? false,
|
||||
pipeline,
|
||||
pipelineRun: pipelineRunStatus,
|
||||
},
|
||||
argo,
|
||||
runtime: {
|
||||
namespaceExists: runtimePayload.runtimeNamespaceExists ?? false,
|
||||
manager,
|
||||
database,
|
||||
secrets,
|
||||
localPostgres,
|
||||
},
|
||||
runtimeAlignment,
|
||||
},
|
||||
timings: {
|
||||
sourceMs: sourceProbe.elapsedMs,
|
||||
@@ -2170,24 +2192,118 @@ async function statusYamlLane(config: UniDeskConfig, options: StatusOptions, tar
|
||||
gitMirrorMs: mirrorProbe.elapsedMs,
|
||||
totalMs: sourceProbe.elapsedMs + Math.max(runtimeProbe.elapsedMs, mirrorProbe.elapsedMs),
|
||||
},
|
||||
source: sourcePayload,
|
||||
runtime: runtimePayload,
|
||||
gitMirror: mirrorPayload,
|
||||
captures: {
|
||||
source: compactCapture(sourceProbe.value, { full: options.full || options.raw || sourceProbe.value.exitCode !== 0 }),
|
||||
runtime: compactCapture(runtimeProbe.value, { full: options.full || options.raw || runtimeProbe.value.exitCode !== 0 }),
|
||||
gitMirror: compactCapture(mirrorProbe.value, { full: options.full || options.raw || mirrorProbe.value.exitCode !== 0 }),
|
||||
},
|
||||
disclosure: {
|
||||
output: options.full || options.raw ? "full" : "compact-summary",
|
||||
full: options.full,
|
||||
raw: options.raw,
|
||||
omittedWhenCompact: options.full || options.raw ? [] : ["full target YAML summary", "source", "runtime", "gitMirror", "capture stdout/stderr tails for successful probes"],
|
||||
fullCommand: statusFullCommand,
|
||||
},
|
||||
next: {
|
||||
plan: `bun scripts/cli.ts agentrun control-plane plan --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
postgresStatus: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres status --config ${spec.database.configRef}` : null,
|
||||
postgresApply: spec.database.configRef ? `bun scripts/cli.ts platform-db postgres apply --config ${spec.database.configRef} --confirm` : null,
|
||||
secretSync: `bun scripts/cli.ts agentrun control-plane secret-sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
||||
restart: `bun scripts/cli.ts agentrun control-plane restart --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
||||
statusFull: `bun scripts/cli.ts agentrun control-plane status --node ${spec.nodeId} --lane ${spec.lane} --full`,
|
||||
statusFull: statusFullCommand,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
if (options.full || options.raw) {
|
||||
result.source = sourcePayload;
|
||||
result.runtime = runtimePayload;
|
||||
result.gitMirror = mirrorPayload;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function agentRunControlPlaneStatusCommand(spec: AgentRunLaneSpec, options: Pick<StatusOptions, "sourceCommit" | "pipelineRun">, full: boolean): string {
|
||||
return [
|
||||
"bun scripts/cli.ts agentrun control-plane status",
|
||||
`--node ${spec.nodeId}`,
|
||||
`--lane ${spec.lane}`,
|
||||
options.pipelineRun === null ? null : `--pipeline-run ${options.pipelineRun}`,
|
||||
options.sourceCommit === null ? null : `--source-commit ${options.sourceCommit}`,
|
||||
full ? "--full" : null,
|
||||
].filter(Boolean).join(" ");
|
||||
}
|
||||
|
||||
function compactLaneSecretsStatus(secrets: Record<string, unknown>): Record<string, unknown> {
|
||||
const items = Array.isArray(secrets.items) ? secrets.items.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
||||
const missing = items
|
||||
.filter((item) => item.present !== true || item.keyPresent !== true)
|
||||
.map((item) => ({
|
||||
namespace: item.namespace ?? null,
|
||||
name: item.name ?? null,
|
||||
key: item.key ?? null,
|
||||
present: item.present ?? false,
|
||||
keyPresent: item.keyPresent ?? false,
|
||||
}));
|
||||
return {
|
||||
ready: secrets.ready ?? false,
|
||||
count: secrets.count ?? items.length,
|
||||
missingCount: missing.length,
|
||||
missing,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactAgentRunLaneStatusTarget(spec: AgentRunLaneSpec): Record<string, unknown> {
|
||||
return {
|
||||
node: {
|
||||
id: spec.nodeId,
|
||||
route: spec.nodeRoute,
|
||||
kubeRoute: spec.nodeKubeRoute,
|
||||
},
|
||||
lane: spec.lane,
|
||||
version: spec.version,
|
||||
source: {
|
||||
repository: spec.source.repository,
|
||||
branch: spec.source.branch,
|
||||
workspace: spec.source.workspace,
|
||||
},
|
||||
runtime: {
|
||||
namespace: spec.runtime.namespace,
|
||||
managerDeployment: spec.runtime.managerDeployment,
|
||||
managerService: spec.runtime.managerService,
|
||||
internalBaseUrl: spec.runtime.internalBaseUrl,
|
||||
},
|
||||
ci: {
|
||||
namespace: spec.ci.namespace,
|
||||
pipeline: spec.ci.pipeline,
|
||||
pipelineRunPrefix: spec.ci.pipelineRunPrefix,
|
||||
registryPrefix: spec.ci.registryPrefix,
|
||||
},
|
||||
gitops: {
|
||||
branch: spec.gitops.branch,
|
||||
path: spec.gitops.path,
|
||||
argoNamespace: spec.gitops.argoNamespace,
|
||||
argoApplication: spec.gitops.argoApplication,
|
||||
},
|
||||
gitMirror: {
|
||||
namespace: spec.gitMirror.namespace,
|
||||
readService: spec.gitMirror.readService,
|
||||
writeService: spec.gitMirror.writeService,
|
||||
repositoryCount: spec.gitMirror.repositories.length,
|
||||
},
|
||||
database: {
|
||||
mode: spec.database.mode,
|
||||
provider: spec.database.provider,
|
||||
configRef: spec.database.configRef,
|
||||
database: spec.database.database,
|
||||
user: spec.database.user,
|
||||
secretRef: spec.database.secretRef,
|
||||
localPostgresExpectedAbsent: spec.database.localPostgresExpectedAbsent,
|
||||
valuesPrinted: false,
|
||||
},
|
||||
secretCount: spec.secrets.length,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function restartYamlLane(config: UniDeskConfig, options: LaneConfirmOptions): Promise<Record<string, unknown>> {
|
||||
|
||||
+135
-2
@@ -26,7 +26,7 @@ export interface JobRecord {
|
||||
}
|
||||
|
||||
export interface JobProgressSummary {
|
||||
kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "git-mirror" | "generic";
|
||||
kind: "hwlab-v02-trigger" | "hwlab-runtime-lane-trigger" | "agentrun-yaml-lane-trigger" | "git-mirror" | "generic";
|
||||
stage: string | null;
|
||||
stageStatus: string | null;
|
||||
sourceCommit: string | null;
|
||||
@@ -229,15 +229,17 @@ function summarizeJobProgress(job: JobRecord, maxBytes = 96_000, tails?: { stdou
|
||||
const knownWorkflow = job.name === "hwlab_g14_v02_trigger_current";
|
||||
const v02PrMonitorWorkflow = job.name === "hwlab_g14_v02_pr_monitor";
|
||||
const runtimeLaneTriggerWorkflow = /^hwlab_nodes_v[0-9]{2}_control-plane_trigger-current$/u.test(job.name);
|
||||
const agentRunYamlLaneTriggerWorkflow = /^agentrun_v[0-9]{2}_trigger_current$/u.test(job.name);
|
||||
const gitMirrorWorkflow = job.name === "hwlab_g14_git_mirror_sync" || job.name === "hwlab_g14_git_mirror_flush" || job.name === "agentrun_v01_git_mirror_sync" || job.name === "agentrun_v01_git_mirror_flush";
|
||||
const ciInstallWorkflow = job.name === "ci_install";
|
||||
if (ciInstallWorkflow) return summarizeCiInstallJobProgress(job, tails?.stderrTail, nowMs);
|
||||
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
|
||||
if (!knownWorkflow && !v02PrMonitorWorkflow && !runtimeLaneTriggerWorkflow && !agentRunYamlLaneTriggerWorkflow && !gitMirrorWorkflow) return genericJobProgress(job, tails?.stderrTail);
|
||||
const progressTailBytes = Math.max(4096, Math.floor(maxBytes));
|
||||
const stderrTail = tails?.stderrTail ?? tailFile(job.stderrFile, progressTailBytes);
|
||||
const stdoutTail = tails?.stdoutTail ?? tailFile(job.stdoutFile, progressTailBytes);
|
||||
if (gitMirrorWorkflow) return summarizeGitMirrorJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
if (runtimeLaneTriggerWorkflow) return summarizeRuntimeLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
if (agentRunYamlLaneTriggerWorkflow) return summarizeAgentRunYamlLaneTriggerJobProgress(job, stdoutTail, stderrTail, nowMs);
|
||||
const events = v02PrMonitorWorkflow
|
||||
? [
|
||||
...parseJsonLineEvents(stderrTail, "hwlab.v02.pr-monitor.progress"),
|
||||
@@ -479,6 +481,123 @@ function summarizeRuntimeLaneTriggerJobProgress(job: JobRecord, stdoutTail: stri
|
||||
};
|
||||
}
|
||||
|
||||
const agentRunYamlLaneProgressEvents = [
|
||||
{ event: "agentrun.yaml-lane.source-bootstrap.progress", stage: "source-bootstrap" },
|
||||
{ event: "agentrun.yaml-lane.image-build.progress", stage: "image-build" },
|
||||
{ event: "agentrun.yaml-lane.gitops-publish.progress", stage: "gitops-publish" },
|
||||
{ event: "agentrun.yaml-lane.git-mirror.progress", stage: "git-mirror" },
|
||||
] as const;
|
||||
|
||||
function summarizeAgentRunYamlLaneTriggerJobProgress(job: JobRecord, stdoutTail: string, stderrTail: string, nowMs = Date.now()): JobProgressSummary {
|
||||
const events = agentRunYamlLaneProgressEvents
|
||||
.flatMap(({ event, stage }) => parseJsonLineEvents(stderrTail, event).map((item) => ({ ...item, stage })))
|
||||
.sort((left, right) => String(left.at ?? "").localeCompare(String(right.at ?? "")));
|
||||
const lastEvent = events.at(-1) ?? {};
|
||||
const stage = stringField(lastEvent.stage);
|
||||
const stageStatus = agentRunYamlLaneStageStatus(lastEvent);
|
||||
const sourceCommit = lastStringField(events, "sourceCommit") ?? firstMatch(stdoutTail, /"sourceCommit"\s*:\s*"([0-9a-f]{40})"/iu);
|
||||
const pipelineRun = stringField(lastEvent.pipelineRun) ?? firstMatch(stdoutTail, /"pipelineRun"\s*:\s*"([^"]+)"/u);
|
||||
const pipelineCreatedValue = firstMatch(stdoutTail, /"created"\s*:\s*\{[\s\S]{0,400}?"created"\s*:\s*(true|false)/u);
|
||||
const pipelineCreated = pipelineCreatedValue === "true" ? true : pipelineCreatedValue === "false" ? false : null;
|
||||
const lastEventAt = stringField(lastEvent.at);
|
||||
const elapsedSeconds = jobElapsedSeconds(job, nowMs);
|
||||
const eventElapsedMs = numberField(lastEvent.elapsedMs);
|
||||
const stageElapsedSeconds = eventElapsedMs === null ? null : Math.round(eventElapsedMs / 1000);
|
||||
const lastEventAgeSeconds = lastEventAt === null ? null : secondsSince(lastEventAt, job.finishedAt ?? nowMs);
|
||||
const timings = agentRunYamlLaneTimings(events);
|
||||
const warnings = jobProgressWarnings({
|
||||
job,
|
||||
eventsObserved: events.length,
|
||||
elapsedSeconds,
|
||||
stage,
|
||||
stageStatus,
|
||||
stageElapsedSeconds,
|
||||
lastEventAgeSeconds,
|
||||
});
|
||||
const slow = warnings.length > 0;
|
||||
const eventNode = stringField(lastEvent.node);
|
||||
const eventLane = stringField(lastEvent.lane) ?? firstMatch(job.name, /^agentrun_(v[0-9]{2})_trigger_current$/u);
|
||||
const nextCommand = pipelineRun
|
||||
? [
|
||||
"bun scripts/cli.ts agentrun control-plane status",
|
||||
eventNode ? `--node ${eventNode}` : null,
|
||||
eventLane ? `--lane ${eventLane}` : null,
|
||||
`--pipeline-run ${pipelineRun}`,
|
||||
].filter(Boolean).join(" ")
|
||||
: job.status === "running"
|
||||
? `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`
|
||||
: null;
|
||||
return {
|
||||
kind: "agentrun-yaml-lane-trigger",
|
||||
stage,
|
||||
stageStatus,
|
||||
sourceCommit,
|
||||
pipelineRun,
|
||||
pipelineCreated,
|
||||
elapsedSeconds,
|
||||
stageElapsedSeconds,
|
||||
lastEventAt,
|
||||
lastEventAgeSeconds,
|
||||
eventsObserved: events.length,
|
||||
slow,
|
||||
warnings,
|
||||
diagnostics: {
|
||||
stages: agentRunYamlLaneStageDiagnostics(events),
|
||||
},
|
||||
timings,
|
||||
summary: [
|
||||
job.status,
|
||||
stage ? `${stage}${stageStatus ? `:${stageStatus}` : ""}` : "stage:unknown",
|
||||
sourceCommit ? `source=${sourceCommit.slice(0, 12)}` : null,
|
||||
pipelineRun ? `pipelineRun=${pipelineRun}` : null,
|
||||
pipelineCreated === true ? "created" : pipelineCreated === false ? "create-reused" : null,
|
||||
elapsedSeconds !== null ? `elapsed=${elapsedSeconds}s` : null,
|
||||
stageElapsedSeconds !== null && job.status === "running" ? `stageElapsed=${stageElapsedSeconds}s` : null,
|
||||
lastEventAgeSeconds !== null && job.status === "running" ? `lastEventAge=${lastEventAgeSeconds}s` : null,
|
||||
`events=${events.length}`,
|
||||
slow ? "visibility-warning" : null,
|
||||
].filter(Boolean).join(" "),
|
||||
nextCommand,
|
||||
};
|
||||
}
|
||||
|
||||
function agentRunYamlLaneStageStatus(event: Record<string, unknown>): string | null {
|
||||
const explicitStatus = stringField(event.status);
|
||||
if (explicitStatus !== null) return explicitStatus;
|
||||
if (event.succeeded === true) return "succeeded";
|
||||
if (event.failed === true) return "failed";
|
||||
if (event.stage !== undefined) return "running";
|
||||
return null;
|
||||
}
|
||||
|
||||
function agentRunYamlLaneTimings(events: Record<string, unknown>[]): Record<string, number> {
|
||||
const timings: Record<string, number> = {};
|
||||
for (const event of events) {
|
||||
const stage = stringField(event.stage);
|
||||
const elapsedMs = numberField(event.elapsedMs);
|
||||
if (stage !== null && elapsedMs !== null) timings[`agentrunYamlLane.${stage}.elapsedMs`] = Math.round(elapsedMs);
|
||||
}
|
||||
return timings;
|
||||
}
|
||||
|
||||
function agentRunYamlLaneStageDiagnostics(events: Record<string, unknown>[]): Array<Record<string, unknown>> {
|
||||
return agentRunYamlLaneProgressEvents
|
||||
.map(({ stage }) => {
|
||||
const stageEvents = events.filter((event) => stringField(event.stage) === stage);
|
||||
const lastEvent = stageEvents.at(-1) ?? {};
|
||||
return {
|
||||
stage,
|
||||
eventsObserved: stageEvents.length,
|
||||
status: agentRunYamlLaneStageStatus(lastEvent),
|
||||
jobId: stringField(lastEvent.jobId),
|
||||
jobName: stringField(lastEvent.jobName),
|
||||
polls: numberField(lastEvent.polls),
|
||||
elapsedMs: numberField(lastEvent.elapsedMs),
|
||||
};
|
||||
})
|
||||
.filter((item) => item.eventsObserved > 0);
|
||||
}
|
||||
|
||||
function hwlabRuntimeLaneStatusNextCommand(pipelineRun: string | null, lastEvent: Record<string, unknown>): { command: string | null; warning: string | null } {
|
||||
if (pipelineRun === null) return { command: null, warning: null };
|
||||
const eventNode = stringField(lastEvent.node);
|
||||
@@ -738,6 +857,20 @@ function stringField(value: unknown): string | null {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function numberField(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isFinite(value)) return value;
|
||||
if (typeof value === "string" && value.trim().length > 0) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed)) return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function lastStringField(events: Record<string, unknown>[], field: string): string | null {
|
||||
const event = events.findLast((item) => stringField(item[field]) !== null);
|
||||
return stringField(event?.[field]);
|
||||
}
|
||||
|
||||
function firstMatch(text: string, pattern: RegExp): string | null {
|
||||
const match = pattern.exec(text);
|
||||
return typeof match?.[1] === "string" && match[1].length > 0 ? match[1] : null;
|
||||
|
||||
Reference in New Issue
Block a user