diff --git a/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md b/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md index 29b5cf02..58f8e6da 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060508-web-probe-sentinel.md @@ -122,6 +122,7 @@ Web哨兵必须遵循 UniDesk YAML-first ops。目标 node/lane、public origin | PJ2026-0106050810 | 多实例与账号切换 | 本规格 6.10 | sentinel registry、实例隔离、账号切换 command、route prefix 和 report label | YAML配置、Wrapper边界、安全隔离 | 多哨兵巡检、账号链路值守 | | PJ2026-0106050811 | Monitor Web 聚合 | 本规格 6.11 | runner/web 职责拆分、单 monitor-web 聚合、Kubernetes discovery、Vue+TS 前端和 public exposure 收敛 | 多实例与账号切换、Dashboard工作台、发布集成 | `monitor.pikapython.com` 统一值守入口 | | PJ2026-0106050812 | Monitor Web 观察面板治理 | 本规格 6.12 | 趋势曲线、运行时间线、固定视口三栏、cadence freshness、Vue CI/CD/env reuse/git mirror | Monitor Web 聚合、Dashboard工作台、发布集成、源码同步 | 可滚动上线和值守的统一观察面板 | +| PJ2026-0106050814 | 哨兵 CI/CD 可见性 | 本规格 6.14 | publish 阶段耗时、env reuse、docker cache、超时诊断、git mirror/Argo/runtime 收敛下一步 | 发布集成、源码同步、Monitor Web 观察面板治理 | 小改动滚动上线可诊断、可续跑、可验收 | ### 5.1 目标架构图 @@ -669,6 +670,22 @@ dashboard verify/screenshot 必须断言 selected sentinel 的 route 和 API 返 OTel 根因契约必须与应用内 report/index 分工清楚。sentinel report/index 负责 latest、history、finding、artifact、timeline、availability 和 runner heartbeat；OTel/Tempo 负责跨服务根因链路。D518 `diagnose-code-agent` 的 AgentRun namespace/lane 必须从 `config/hwlab-node-lanes.yaml` 解析到实际 `agentrun-v02`，不得硬编码 `agentrun-v01`。HWLAB cloud-api、AgentRun manager 和 AgentRun runner 的 span 必须能通过 trace context 串联；缺失任一段时标为 instrumentation blocker 或 instrumentation gap，不得静默跳过。 +### 6.14 OPS-SENTINEL-REQ-014 Web 哨兵 CI/CD 可见性与续跑边界 + +| 编号 | 短名 | 主责模块 | 关联模块 | +| --- | --- | --- | --- | +| OPS-SENTINEL-REQ-014 | 哨兵 CI/CD 可见性 | PJ2026-0106050814 哨兵 CI/CD 可见性 | 发布集成、源码同步、Monitor Web 观察面板治理、YAML运维 | + +本阶段执行 issue 为 [#1285](https://github.com/pikasTech/unidesk/issues/1285)。`web-probe sentinel control-plane trigger-current --confirm --wait` 的默认输出必须把 source mirror、publish、git mirror flush、Argo apply 和 runtime observed 明确分阶段展示。`confirmWait.maxSeconds` 仍是 YAML 声明的交互等待预算；超过预算时 CLI 必须停止盲等并返回结构化阶段归因，不能只输出 `job-timeout`。 + +publish 结果必须输出 env reuse 摘要、依赖复用路径、docker build cache 摘要、镜像 digest、GitOps commit、各阶段耗时和 bounded 日志摘要。env reuse 摘要至少包括 reuse mode、node deps path、path 是否存在、依赖项数量或等价命中信号；docker cache 摘要至少包括 build log 中的 cache hit 行数、构建步骤行数和 layer cache 口径。上述字段只作为可见性证据，不新增业务门禁，也不得打印 Secret、token、cookie、provider payload 或完整无界构建日志。 + +publish Job 未在等待预算内结束时，CLI 必须输出 job 名称、pod、pod phase、当前阶段、已完成阶段、最近日志摘要和可安全继续的 drill-down 命令。drill-down 命令必须优先指向受控 UniDesk CLI；只读 k3s 日志或 describe 可以通过 `trans :k3s kubectl ...` 作为诊断入口，但不得把手工 `kubectl apply/delete/patch` 变成正式控制面。 + +当 publish 已产出镜像 digest，但 GitOps、git mirror、Argo 或 runtime observed 未收敛时，CLI 必须直接给出下一步：先查看 `web-probe sentinel control-plane status` 和 `hwlab nodes git-mirror status`，若 GitOps pending flush 则走 `hwlab nodes git-mirror flush --confirm --wait`，若 Argo/runtime stale 则走 `web-probe sentinel control-plane apply --confirm --wait`。操作员不得靠猜测在裸 `kubectl/argo` 和多条旧路径之间切换。 + +JD01/v03 `jd01-web-probe-sentinel` 的小改动滚动上线是本阶段验收入口。closeout 必须记录 SPEC P14 引用、source commit、publish job、digest、GitOps revision、git mirror pending/inSync、Argo/runtime alignment、`validate`、远程 dashboard screenshot 和 latest report 证据。若总等待仍超过 120s，closeout 必须记录阶段归因、env reuse/cache 摘要和下一步优化方向；不得通过单纯放宽 120s 预算收口。 + ## 7. 过程控制 Web哨兵架构执行 issue 为 [#883](https://github.com/pikasTech/unidesk/issues/883)。阶段跟踪 issue 为 P0 [#885](https://github.com/pikasTech/unidesk/issues/885)、P1 [#886](https://github.com/pikasTech/unidesk/issues/886)、P2 [#887](https://github.com/pikasTech/unidesk/issues/887)、P3 [#888](https://github.com/pikasTech/unidesk/issues/888)、P4 [#889](https://github.com/pikasTech/unidesk/issues/889)、P5 [#890](https://github.com/pikasTech/unidesk/issues/890) 和 P6 [#891](https://github.com/pikasTech/unidesk/issues/891)。 @@ -692,3 +709,5 @@ P11 monitor-web 观察面板治理执行 issue 为 [#1112](https://github.com/pi P12 cadence 调度和 monitor-web 交互修复执行 issue 为 [#1123](https://github.com/pikasTech/unidesk/issues/1123)。P12 closeout 必须回写：SPEC P12 引用、两个 10m cadence sentinel 的 stale 证据、k3s CronJob/GitOps 调度器 due 判断和触发记录、auth sentinel Argo/source alignment、趋势曲线 hover 数值和时间截图/DOM 证据、三栏 sticky header 遮盖复测、远程 PNG localPath/SHA、k3s CronJob 状态、以及两个目标 sentinel 最新 run 已刷新到当前窗口的证据。 P13 D518 多 runner 强边界与 OTel 根因收敛执行 issue 为 [#1206](https://github.com/pikasTech/unidesk/issues/1206)。P13 closeout 必须回写：SPEC P13 引用、[#1208](https://github.com/pikasTech/unidesk/issues/1208)-[#1216](https://github.com/pikasTech/unidesk/issues/1216) 阶段状态、D518 双 sentinel 独立 Deployment/Service/PVC/CronJob/GitOps/Argo/public route 证据、route/API sentinelId 强断言、report/index 不串线证据、dashboard verify/screenshot localPath/SHA、k3s CronJob 调度证据、latest selected run 与 historical trend 状态分层证据、以及 OTel AgentRun namespace/trace gap 是否已解除或拆入后续 issue。 + +P14 Web 哨兵 CI/CD 可见性执行 issue 为 [#1285](https://github.com/pikasTech/unidesk/issues/1285)。P14 closeout 必须回写：SPEC P14 引用、source commit、PR/merge commit、JD01/v03 `jd01-web-probe-sentinel` publish job、digest、GitOps revision、git mirror flush 状态、Argo/runtime observed alignment、`validate`、dashboard screenshot、latest report，以及超过 120s 时的结构化阶段归因和可续跑命令。 diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 106824e1..d21ed7d7 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -5,6 +5,7 @@ // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries. +// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility. // Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel. import { createHash, randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; @@ -179,6 +180,7 @@ interface SentinelRemoteJobResult { readonly elapsedMs?: number; readonly create?: Record; readonly probe?: Record; + readonly diagnostics?: Record; readonly valuesRedacted: true; } @@ -197,7 +199,7 @@ export interface ChildCliResult { readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string }; } -const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard"; +const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility"; export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult { if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId)); @@ -368,7 +370,8 @@ function sentinelImagePlan(spec: HwlabRuntimeLaneSpec, cicd: Record /usr/local/bin/trans && chmod 0755 /usr/local/bin/trans", "RUN bun scripts/verify-web-probe-sentinel-monitor-web.ts", "ENV NODE_ENV=production", @@ -895,6 +899,8 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), + ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), + ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ], blocker: ok @@ -1002,6 +1008,8 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel control-plane confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), + ...sentinelRemoteJobTimeoutWarnings(sourceMirrorSync, "sentinel source mirror sync"), + ...sentinelRemoteJobTimeoutWarnings(publish, "sentinel publish"), ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds), ...asyncGitMirrorFlushWarnings(flush), ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), @@ -1011,6 +1019,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []), ])), blocker, + recoveryNext: controlPlaneRecoveryNext(state, ok, publish, flush, observed), next: controlPlaneNext(state, options.action), valuesRedacted: true, }; @@ -1281,7 +1290,7 @@ function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); if (created.exitCode !== 0) { sentinelProgressEvent("sentinel.source-mirror.progress", { phase: "create-job", status: "failed", jobName, node: state.spec.nodeId, lane: state.spec.lane }); - return { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "source-mirror"); } const startedAt = Date.now(); const timeoutMs = Math.max(5_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); @@ -1308,10 +1317,10 @@ function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds }); if (probe.succeeded === true) { const ok = payload.ok === true; - return { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); } if (probe.failed === true) { - return { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); } if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { slowWarningSent = true; @@ -1319,7 +1328,7 @@ function runSentinelSourceMirrorSyncJob(state: SentinelCicdState, timeoutSeconds } runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } - return { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "source-mirror"); } function sentinelBlockedRemoteResult(phase: string, reason: string): SentinelRemoteJobResult { @@ -1551,7 +1560,7 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, const created = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", createK8sJobScript(namespace, manifest)], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 }); if (created.exitCode !== 0) { sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "failed", jobName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); - return { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "create-job", jobName, payload: { ok: false, status: "create-failed", valuesRedacted: true }, create: compactCommand(created), valuesRedacted: true }, "publish"); } sentinelProgressEvent("sentinel.publish.progress", { phase: "create-job", status: "succeeded", jobName, publishGitops, node: state.spec.nodeId, lane: state.spec.lane }); const startedAt = Date.now(); @@ -1580,10 +1589,10 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, }); if (probe.succeeded === true) { const ok = payload.ok === true; - return { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok, phase: "job-succeeded", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "result-missing", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); } if (probe.failed === true) { - return { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-failed", jobName, payload: Object.keys(payload).length === 0 ? { ok: false, status: "failed", valuesRedacted: true } : payload, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); } if (!slowWarningSent && Date.now() - startedAt > warningBudgetMs) { slowWarningSent = true; @@ -1591,7 +1600,7 @@ function runSentinelPublishJob(state: SentinelCicdState, publishGitops: boolean, } runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); } - return { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }; + return withSentinelRemoteJobDiagnostics(state, { ok: false, phase: "job-timeout", jobName, payload: { ok: false, status: "timeout", valuesRedacted: true }, polls, elapsedMs: Date.now() - startedAt, probe: lastProbe, valuesRedacted: true }, "publish"); } function sentinelPublishJobManifest(state: SentinelCicdState, jobName: string, publishGitops: boolean): Record { @@ -1645,6 +1654,7 @@ function sentinelGitMirrorCacheVolume(state: SentinelCicdState): Record ({ path: file.path, contentBase64: Buffer.from(file.content, "utf8").toString("base64"), @@ -1654,6 +1664,8 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish return item; })), "utf8").toString("base64"); const dockerfileB64 = Buffer.from(state.image.dockerfilePreview, "utf8").toString("base64"); + const envReuseMode = stringAt(monitorWeb, "envReuseMode"); + const envReuseNodeDepsPath = stringAt(monitorWeb, "envReuseNodeDepsPath"); return [ "set -eu", `job_name=${shellQuote(jobName)}`, @@ -1668,7 +1680,10 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish `gitops_repository=${shellQuote(stringAt(state.controlPlaneTarget, "source.repository"))}`, `gitops_branch=${shellQuote(stringAt(state.cicd, "argo.targetRevision"))}`, `files_b64=${shellQuote(filesB64)}`, + `env_reuse_mode=${shellQuote(envReuseMode)}`, + `env_reuse_node_deps_path=${shellQuote(envReuseNodeDepsPath)}`, "started_ms=$(node -e 'console.log(Date.now())')", + "emit_stage() { stage=$1; status=$2; started=$3; finished=$(node -e 'console.log(Date.now())'); node - \"$stage\" \"$status\" \"$started\" \"$finished\" <<'NODE'\nconst [stage, status, started, finished] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-stage', stage, status, elapsedMs:Number(finished)-Number(started), valuesRedacted:true }));\nNODE\n}", "emit_failed() { code=$?; if [ \"$code\" -ne 0 ]; then node - \"$code\" \"$job_name\" <<'NODE'\nconst [code, jobName] = process.argv.slice(2); console.log(JSON.stringify({ ok:false, status:'failed', exitCode:Number(code), jobName, valuesRedacted:true }));\nNODE\nfi; exit \"$code\"; }", "trap emit_failed EXIT", "mkdir -p /root/.ssh", @@ -1690,17 +1705,39 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "fs.mkdirSync('.git/info', { recursive: true });", "fs.writeFileSync('.git/info/sparse-checkout', paths.map((item) => item.endsWith('/') ? item : item + (item.includes('.') ? '' : '/')).join('\\n') + '\\n');", "NODE", + "source_fetch_started_ms=$(node -e 'console.log(Date.now())')", + "emit_stage source-fetch running \"$source_fetch_started_ms\"", "git fetch --depth=1 --filter=blob:none origin \"+refs/heads/$source_branch:refs/remotes/origin/$source_branch\"", "git checkout --detach \"$source_commit\"", "mirror_commit=$(git rev-parse HEAD)", "test \"$mirror_commit\" = \"$source_commit\"", + "source_fetch_finished_ms=$(node -e 'console.log(Date.now())')", + "emit_stage source-fetch succeeded \"$source_fetch_started_ms\"", + "env_reuse_node_deps_present=false", + "env_reuse_node_deps_entries=0", + "if [ -d \"$env_reuse_node_deps_path\" ]; then env_reuse_node_deps_present=true; env_reuse_node_deps_entries=$(find \"$env_reuse_node_deps_path\" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l | tr -d ' '); fi", + "node - \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" <<'NODE'", + "const [mode, nodeDepsPath, nodeDepsPresent, nodeDepsEntries] = process.argv.slice(2); console.log(JSON.stringify({ event:'sentinel-publish-env-reuse', mode, nodeDepsPath, nodeDepsPresent: nodeDepsPresent === 'true', nodeDepsEntries: Number(nodeDepsEntries || 0), dependencyReuse: nodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }));", + "NODE", "DOCKERFILE_B64=\"$dockerfile_b64\" node <<'NODE'", "const fs = require('node:fs');", "fs.writeFileSync('Dockerfile.web-probe-sentinel', Buffer.from(process.env.DOCKERFILE_B64 || '', 'base64'));", "NODE", - "docker build -f Dockerfile.web-probe-sentinel -t \"$image_ref\" .", - "docker push \"$image_ref\" > /tmp/web-probe-sentinel-docker-push.log 2>&1", + "docker_build_started_ms=$(node -e 'console.log(Date.now())')", + "emit_stage docker-build running \"$docker_build_started_ms\"", + "if ! docker build -f Dockerfile.web-probe-sentinel -t \"$image_ref\" . > /tmp/web-probe-sentinel-docker-build.log 2>&1; then cat /tmp/web-probe-sentinel-docker-build.log; emit_stage docker-build failed \"$docker_build_started_ms\"; exit 1; fi", + "cat /tmp/web-probe-sentinel-docker-build.log", + "docker_build_finished_ms=$(node -e 'console.log(Date.now())')", + "emit_stage docker-build succeeded \"$docker_build_started_ms\"", + "docker_build_cache_hits=$(grep -Eci '(^|[[:space:]])CACHED([[:space:]]|$)|Using cache|cache hit' /tmp/web-probe-sentinel-docker-build.log 2>/dev/null || true)", + "docker_build_step_lines=$(grep -Eci '^(#|STEP|[[:space:]]*=>)' /tmp/web-probe-sentinel-docker-build.log 2>/dev/null || true)", + "docker_build_log_tail_b64=$(tail -n 30 /tmp/web-probe-sentinel-docker-build.log 2>/dev/null | tail -c 4000 | base64 | tr -d '\\n')", + "docker_push_started_ms=$(node -e 'console.log(Date.now())')", + "emit_stage docker-push running \"$docker_push_started_ms\"", + "if ! docker push \"$image_ref\" > /tmp/web-probe-sentinel-docker-push.log 2>&1; then cat /tmp/web-probe-sentinel-docker-push.log; emit_stage docker-push failed \"$docker_push_started_ms\"; exit 1; fi", "cat /tmp/web-probe-sentinel-docker-push.log", + "docker_push_finished_ms=$(node -e 'console.log(Date.now())')", + "emit_stage docker-push succeeded \"$docker_push_started_ms\"", "tag=${image_ref##*:}", "repo_no_tag=${image_ref%:*}", "registry_path=${repo_no_tag#127.0.0.1:5000/}", @@ -1711,7 +1748,9 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish "gitops_commit=''", "changed=false", "file_count=0", + "gitops_started_ms=$(node -e 'console.log(Date.now())')", "if [ \"$files_b64\" != \"W10=\" ]; then", + " emit_stage gitops running \"$gitops_started_ms\"", " gitops_cache=\"/cache/${gitops_repository}.git\"", " gitops_worktree=\"/tmp/$job_name/gitops\"", " git clone --no-checkout \"$gitops_cache\" \"$gitops_worktree\"", @@ -1736,11 +1775,17 @@ function sentinelPublishShell(state: SentinelCicdState, jobName: string, publish " if git diff --quiet --cached; then changed=false; else changed=true; git -c user.email=web-probe-sentinel@unidesk.local -c user.name='UniDesk Web Probe Sentinel' commit -m \"deploy: render web-probe sentinel ${source_commit}\"; fi", " git push origin \"HEAD:refs/heads/$gitops_branch\"", " gitops_commit=$(git rev-parse HEAD)", + " emit_stage gitops succeeded \"$gitops_started_ms\"", + "else", + " emit_stage gitops skipped \"$gitops_started_ms\"", "fi", + "gitops_finished_ms=$(node -e 'console.log(Date.now())')", "finished_ms=$(node -e 'console.log(Date.now())')", - "node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" <<'NODE'", - "const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs] = process.argv.slice(2);", - "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: Number(finishedMs) - Number(startedMs), valuesRedacted:true }));", + "node - \"$job_name\" \"$source_commit\" \"$mirror_commit\" \"$image_ref\" \"$digest_ref\" \"$gitops_commit\" \"$changed\" \"$file_count\" \"$started_ms\" \"$finished_ms\" \"$source_fetch_started_ms\" \"$source_fetch_finished_ms\" \"$docker_build_started_ms\" \"$docker_build_finished_ms\" \"$docker_push_started_ms\" \"$docker_push_finished_ms\" \"$gitops_started_ms\" \"$gitops_finished_ms\" \"$env_reuse_mode\" \"$env_reuse_node_deps_path\" \"$env_reuse_node_deps_present\" \"$env_reuse_node_deps_entries\" \"$docker_build_cache_hits\" \"$docker_build_step_lines\" \"$docker_build_log_tail_b64\" <<'NODE'", + "const [jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit, changed, fileCount, startedMs, finishedMs, sourceFetchStartedMs, sourceFetchFinishedMs, dockerBuildStartedMs, dockerBuildFinishedMs, dockerPushStartedMs, dockerPushFinishedMs, gitopsStartedMs, gitopsFinishedMs, envReuseMode, envReuseNodeDepsPath, envReuseNodeDepsPresent, envReuseNodeDepsEntries, dockerBuildCacheHits, dockerBuildStepLines, dockerBuildLogTailB64] = process.argv.slice(2);", + "const elapsed = (start, finish) => Number(finish) - Number(start);", + "const cacheHits = Number(dockerBuildCacheHits || 0);", + "console.log(JSON.stringify({ ok:true, status:'succeeded', jobName, sourceCommit, mirrorCommit, imageRef, digestRef, gitopsCommit: gitopsCommit || null, changed: changed === 'true', fileCount: Number(fileCount || 0), elapsedMs: elapsed(startedMs, finishedMs), stageTimings: { sourceFetchMs: elapsed(sourceFetchStartedMs, sourceFetchFinishedMs), dockerBuildMs: elapsed(dockerBuildStartedMs, dockerBuildFinishedMs), dockerPushMs: elapsed(dockerPushStartedMs, dockerPushFinishedMs), gitopsMs: elapsed(gitopsStartedMs, gitopsFinishedMs), totalMs: elapsed(startedMs, finishedMs), valuesRedacted:true }, envReuse: { mode: envReuseMode, nodeDepsPath: envReuseNodeDepsPath, nodeDepsPresent: envReuseNodeDepsPresent === 'true', nodeDepsEntries: Number(envReuseNodeDepsEntries || 0), dependencyReuse: envReuseNodeDepsPresent === 'true' ? 'hit' : 'miss', valuesRedacted:true }, dockerBuild: { cacheHitLines: cacheHits, stepLines: Number(dockerBuildStepLines || 0), layerCache: cacheHits > 0 ? 'hit' : 'unknown-or-miss', logTail: Buffer.from(dockerBuildLogTailB64 || '', 'base64').toString('utf8'), valuesRedacted:true }, completedStages: ['source-fetch', 'docker-build', 'docker-push', gitopsCommit ? 'gitops' : 'gitops-skipped'], valuesRedacted:true }));", "NODE", "trap - EXIT", ].join("\n"); @@ -1789,12 +1834,15 @@ function probeK8sJobScript(namespace: string, jobName: string): string { `job=${shellQuote(jobName)}`, "succeeded=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.succeeded}' 2>/dev/null)", "failed=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.failed}' 2>/dev/null)", + "active=$(kubectl -n \"$namespace\" get job \"$job\" -o jsonpath='{.status.active}' 2>/dev/null)", "pod=$(kubectl -n \"$namespace\" get pod -l job-name=\"$job\" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)", + "pod_phase=''", + "if [ -n \"$pod\" ]; then pod_phase=$(kubectl -n \"$namespace\" get pod \"$pod\" -o jsonpath='{.status.phase}' 2>/dev/null); fi", "logs_tail=''", "if [ -n \"$pod\" ]; then logs_tail=$(kubectl -n \"$namespace\" logs \"$pod\" --tail=120 2>/dev/null | tail -c 12000 | base64 | tr -d '\\n'); fi", - "node - \"$succeeded\" \"$failed\" \"$pod\" \"$logs_tail\" <<'NODE'", - "const [succeeded, failed, pod, logsB64] = process.argv.slice(2);", - "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, pod: pod || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", + "node - \"$succeeded\" \"$failed\" \"$active\" \"$pod\" \"$pod_phase\" \"$logs_tail\" <<'NODE'", + "const [succeeded, failed, active, pod, podPhase, logsB64] = process.argv.slice(2);", + "console.log(JSON.stringify({ succeeded: Number(succeeded || 0) > 0, failed: Number(failed || 0) > 0, active: Number(active || 0) > 0, pod: pod || null, podPhase: podPhase || null, logsTail: Buffer.from(logsB64 || '', 'base64').toString('utf8'), valuesRedacted: true }));", "NODE", ].join("\n"); } @@ -1810,6 +1858,104 @@ function sentinelPayloadFromLogs(logsTail: string): Record { return {}; } +function withSentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): SentinelRemoteJobResult { + return { ...result, diagnostics: sentinelRemoteJobDiagnostics(state, result, domain), valuesRedacted: true }; +} + +function sentinelRemoteJobDiagnostics(state: SentinelCicdState, result: SentinelRemoteJobResult, domain: "source-mirror" | "publish"): Record { + const namespace = stringAt(state.cicd, "builder.namespace"); + const probe = record(result.probe); + const logsTail = typeof probe.logsTail === "string" ? probe.logsTail : ""; + const events = sentinelStageEventsFromLogs(logsTail, domain); + const envReuse = sentinelEnvReuseFromLogs(logsTail); + const completedStages = sentinelCompletedStages(events, record(result.payload)); + const currentPhase = sentinelCurrentRemotePhase(result, events, domain); + const commands = { + cliStatus: domain === "publish" + ? `bun scripts/cli.ts web-probe sentinel control-plane status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}` + : `bun scripts/cli.ts web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)}`, + logs: result.jobName === "-" + ? "-" + : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} logs job/${result.jobName} --tail=120`, + describe: result.jobName === "-" + ? "-" + : `trans ${stringAt(state.controlPlaneNode, "kubeRoute")} kubectl -n ${namespace} describe job/${result.jobName}`, + gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, + gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${state.spec.nodeId} --lane ${state.spec.lane} --confirm --wait`, + controlPlaneApply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${state.spec.nodeId} --lane ${state.spec.lane}${sentinelCliSuffix(state)} --confirm --wait`, + valuesRedacted: true, + }; + return { + domain, + currentPhase, + completedStages, + envReuse, + pod: probe.pod ?? null, + podPhase: probe.podPhase ?? null, + active: probe.active ?? null, + recentLogSummary: sentinelRecentLogSummary(logsTail), + commands, + valuesRedacted: true, + }; +} + +function sentinelEnvReuseFromLogs(logsTail: string): Record | null { + const lines = logsTail.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const parsed = parseJsonObject(lines[index]); + if (parsed !== null && parsed.event === "sentinel-publish-env-reuse") return { ...parsed, valuesRedacted: true }; + } + return null; +} + +function sentinelStageEventsFromLogs(logsTail: string, domain: "source-mirror" | "publish"): Record[] { + const expectedEvent = domain === "publish" ? "sentinel-publish-stage" : "sentinel-source-mirror-stage"; + return logsTail + .split(/\r?\n/u) + .map((line) => parseJsonObject(line.trim())) + .filter((item): item is Record => item !== null && item.event === expectedEvent); +} + +function sentinelCompletedStages(events: readonly Record[], payload: Record): string[] { + const completed = events + .filter((event) => event.status === "succeeded" || event.status === "skipped") + .map((event) => `${text(event.stage)}:${text(event.status)}`); + const payloadStages = Array.isArray(payload.completedStages) ? payload.completedStages.map(text) : []; + return Array.from(new Set([...completed, ...payloadStages])).filter((item) => item !== "-"); +} + +function sentinelCurrentRemotePhase(result: SentinelRemoteJobResult, events: readonly Record[], domain: "source-mirror" | "publish"): string { + if (result.phase === "job-succeeded") return "completed"; + if (result.phase === "create-job") return "create-job"; + const reversed = [...events].reverse(); + const failed = reversed.find((event) => event.status === "failed"); + if (failed !== undefined) return text(failed.stage); + const running = reversed.find((event) => event.status === "running"); + if (running !== undefined) return text(running.stage); + const completed = new Set(events.filter((event) => event.status === "succeeded" || event.status === "skipped").map((event) => text(event.stage))); + const order = domain === "publish" ? ["source-fetch", "docker-build", "docker-push", "gitops"] : ["source-mirror-fetch"]; + const next = order.find((stage) => !completed.has(stage)); + return next ?? result.phase; +} + +function sentinelRecentLogSummary(logsTail: string): string { + const lines = logsTail + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !line.startsWith("{")) + .slice(-5) + .map((line) => short(line)); + return lines.length === 0 ? "-" : lines.join(" | "); +} + +function sentinelRemoteJobTimeoutWarnings(job: unknown, subject: string): string[] { + const remote = record(job); + if (remote.phase !== "job-timeout") return []; + const diagnostics = record(remote.diagnostics); + const commands = record(diagnostics.commands); + return [`${subject} reached wait budget at phase=${text(diagnostics.currentPhase)} completed=${text(Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "")}; inspect logs with ${text(commands.logs)} and continue via ${text(commands.cliStatus)}.`]; +} + function sentinelElapsedWarnings(value: unknown, subject = "sentinel confirmed operation", budgetSeconds = 120): string[] { const elapsedMs = typeof value === "number" && Number.isFinite(value) ? value : null; const budgetMs = Math.max(1, Math.trunc(budgetSeconds)) * 1000; @@ -1913,13 +2059,36 @@ function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelCont status: `bun scripts/cli.ts web-probe sentinel control-plane status --node ${node} --lane ${lane}${suffix}`, image: `bun scripts/cli.ts web-probe sentinel image status --node ${node} --lane ${lane}${suffix}`, triggerCurrent: `bun scripts/cli.ts web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane}${suffix} --dry-run`, + apply: `bun scripts/cli.ts web-probe sentinel control-plane apply --node ${node} --lane ${lane}${suffix} --confirm --wait`, validate: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix}`, quickVerify: `bun scripts/cli.ts web-probe sentinel validate --node ${node} --lane ${lane}${suffix} --quick-verify --confirm --wait`, - issue: "https://github.com/pikasTech/unidesk/issues/889", + gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${node} --lane ${lane}`, + gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${node} --lane ${lane} --confirm --wait`, + issue: "https://github.com/pikasTech/unidesk/issues/1285", currentAction: action, }; } +function controlPlaneRecoveryNext(state: SentinelCicdState, ok: boolean, publish: unknown, flush: unknown, observed: unknown): Record | null { + const payload = record(record(publish).payload); + if (ok || nonEmptyString(payload.digestRef) === null) return null; + const next = controlPlaneNext(state, "apply"); + const flushRecord = record(flush); + const observedRecord = record(observed); + return { + reason: "publish produced an image digest, but GitOps/git-mirror/Argo/runtime alignment is not complete yet", + digestRef: payload.digestRef, + gitopsCommit: payload.gitopsCommit ?? null, + flushMode: flushRecord.mode ?? null, + observedReady: sentinelObservedReady(observedRecord), + nextStatus: next.status, + gitMirrorStatus: next.gitMirrorStatus, + gitMirrorFlush: next.gitMirrorFlush, + controlPlaneApply: next.apply, + valuesRedacted: true, + }; +} + function applySentinelRuntimeSecrets(state: SentinelCicdState, timeoutSeconds: number): Record { const sourcesByPurpose = new Map>(); for (const source of arrayAt(state.secrets, "sources").map(record)) { @@ -2405,6 +2574,83 @@ function safeKubernetesSegment(value: string, maxLength: number): string { return (normalized || "sentinel").slice(0, Math.max(1, maxLength)).replace(/-+$/u, "") || "sentinel"; } +function renderPublishResult(publish: Record): string { + const payload = record(publish.payload); + const diagnostics = record(publish.diagnostics); + const diagnosticEnvReuse = record(diagnostics.envReuse); + const envReuse = Object.keys(record(payload.envReuse)).length > 0 ? record(payload.envReuse) : diagnosticEnvReuse; + const dockerBuild = record(payload.dockerBuild); + const timings = record(payload.stageTimings); + const commands = record(diagnostics.commands); + const lines = [ + "PUBLISH", + table(["OK", "PHASE", "JOB", "ELAPSED", "POD", "CURRENT", "DIGEST", "GITOPS"], [[ + publish.ok, + publish.phase, + publish.jobName, + publish.elapsedMs ?? "-", + diagnostics.pod ?? "-", + diagnostics.currentPhase ?? "-", + short(payload.digestRef), + short(payload.gitopsCommit), + ]]), + ]; + if (Object.keys(envReuse).length > 0) { + lines.push( + "", + "PUBLISH_ENV_REUSE", + table(["MODE", "NODE_DEPS", "PRESENT", "ENTRIES", "DEPENDENCY"], [[ + envReuse.mode, + envReuse.nodeDepsPath, + envReuse.nodeDepsPresent, + envReuse.nodeDepsEntries, + envReuse.dependencyReuse, + ]]), + ); + } + if (Object.keys(dockerBuild).length > 0 || Object.keys(timings).length > 0) { + lines.push( + "", + "PUBLISH_BUILD", + table(["CACHE", "CACHE_LINES", "STEP_LINES", "SOURCE_MS", "BUILD_MS", "PUSH_MS", "GITOPS_MS", "TOTAL_MS"], [[ + dockerBuild.layerCache ?? "-", + dockerBuild.cacheHitLines ?? "-", + dockerBuild.stepLines ?? "-", + timings.sourceFetchMs ?? "-", + timings.dockerBuildMs ?? "-", + timings.dockerPushMs ?? "-", + timings.gitopsMs ?? "-", + timings.totalMs ?? payload.elapsedMs ?? "-", + ]]), + ); + } + if (Object.keys(diagnostics).length > 0) { + lines.push( + "", + "PUBLISH_DIAGNOSTICS", + table(["POD_PHASE", "ACTIVE", "COMPLETED", "RECENT_LOG"], [[ + diagnostics.podPhase ?? "-", + diagnostics.active ?? "-", + Array.isArray(diagnostics.completedStages) ? diagnostics.completedStages.join(",") : "-", + diagnostics.recentLogSummary ?? "-", + ]]), + ); + } + if (publish.ok !== true && Object.keys(commands).length > 0) { + lines.push( + "", + "PUBLISH_DRILLDOWN", + ` status: ${commands.cliStatus ?? "-"}`, + ` logs: ${commands.logs ?? "-"}`, + ` describe: ${commands.describe ?? "-"}`, + ` git-mirror: ${commands.gitMirrorStatus ?? "-"}`, + ` flush: ${commands.gitMirrorFlush ?? "-"}`, + ` apply: ${commands.controlPlaneApply ?? "-"}`, + ); + } + return lines.join("\n"); +} + function renderImageResult(result: Record): string { const source = record(result.source); const sourceMirror = record(result.sourceMirror); @@ -2433,7 +2679,7 @@ function renderImageResult(result: Record): string { "", Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]), "", - Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]), + Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), "", warnings.length === 0 ? "WARNINGS\n-" : ["WARNINGS", ...warnings.map((item) => `- ${text(item)}`)].join("\n"), "", @@ -2468,6 +2714,7 @@ function renderControlPlaneResult(result: Record): string { const blocker = record(result.blocker); const targetValidation = record(result.targetValidation); const targetValidationBusiness = record(targetValidation.businessStatus); + const recoveryNext = record(result.recoveryNext); const next = record(result.next); const warnings = Array.isArray(result.warnings) ? result.warnings : []; return [ @@ -2497,7 +2744,7 @@ function renderControlPlaneResult(result: Record): string { targetValidation.artifactCount, ]]), "", - Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]), + Object.keys(publish).length === 0 ? "PUBLISH\n-" : renderPublishResult(publish), "", Object.keys(flush).length === 0 ? "FLUSH\n-" @@ -2519,13 +2766,25 @@ function renderControlPlaneResult(result: Record): string { "", Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]), "", + Object.keys(recoveryNext).length === 0 ? "RECOVERY_NEXT\n-" : [ + "RECOVERY_NEXT", + table(["REASON", "DIGEST", "GITOPS"], [[recoveryNext.reason, short(recoveryNext.digestRef), short(recoveryNext.gitopsCommit)]]), + ` status: ${recoveryNext.nextStatus ?? "-"}`, + ` git-mirror: ${recoveryNext.gitMirrorStatus ?? "-"}`, + ` flush: ${recoveryNext.gitMirrorFlush ?? "-"}`, + ` apply: ${recoveryNext.controlPlaneApply ?? "-"}`, + ].join("\n"), + "", "NEXT", ` plan: ${next.plan ?? "-"}`, ` status: ${next.status ?? "-"}`, ` image: ${next.image ?? "-"}`, ` trigger-current: ${next.triggerCurrent ?? "-"}`, + ` apply: ${next.apply ?? "-"}`, ` validate: ${next.validate ?? "-"}`, ` quick-verify: ${next.quickVerify ?? "-"}`, + ` git-mirror: ${next.gitMirrorStatus ?? "-"}`, + ` flush: ${next.gitMirrorFlush ?? "-"}`, "", "DISCLOSURE", " default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.",