feat: add web probe sentinel cicd visibility (#897)
Co-authored-by: Codex <codex@noreply.local>
This commit is contained in:
@@ -98,6 +98,11 @@ bun scripts/cli.ts hwlab nodes web-probe observe stop webobs-xxxx
|
||||
bun scripts/cli.ts hwlab nodes web-probe observe analyze webobs-xxxx
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel plan --node D601 --lane v03 --dry-run
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel status --node D601 --lane v03
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel image status --node D601 --lane v03
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel image build --node D601 --lane v03 --dry-run
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node D601 --lane v03 --dry-run
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane status --node D601 --lane v03
|
||||
bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane trigger-current --node D601 --lane v03 --dry-run
|
||||
bun scripts/web-probe-sentinel-service.ts --node D601 --lane v03 --state-root .state/web-probe-sentinel-smoke --scheduler-disabled --once
|
||||
```
|
||||
|
||||
@@ -125,7 +130,7 @@ bun scripts/cli.ts hwlab nodes web-probe observe analyze webobs-xxxx
|
||||
- `observe command --type steer` 和 `--type cancel` 是显式用户/control action:steer 复用当前 Workbench composer 的运行中 turn 引导路径,cancel 复用同一 composer 主按钮的取消路径。二者必须进入 `control.jsonl`,不能用后端私有 API、AgentRun direct cancel 或测试后门替代。`selectProjectSource`、`selectMdtodoFile`、`selectMdtodoTask` 和 `launchWorkbenchFromTask` 也是显式用户/control action,只能使用页面公开 `data-*` id、正式按钮和 YAML 允许的自然 API;它们通过 opaque public id 与 Workbench 关联,不能读取内部 store、私有后端或把 mdtodo 页面包含进 Workbench。
|
||||
- `observe collect --view turn-summary` 是第一层 CLI 阅读视图:只从 `samples.jsonl`、`control.jsonl` 和已有 `analysis/report.json` 按需渲染同一 session 的多 turn 摘要,包含用户消息 preview/hash、traceId、状态、耗时/最近更新时间、steer/cancel 标记和 Final Response 摘要。`observe collect --view trace-frame --trace-id <id> --sample-seq <n>` 是第二层 CLI 阅读视图:从同一采样帧渲染单帧 trace 文字截图,并固定输出 `Final Response` 区块。`observe collect --view project-summary` 从同一 artifact 渲染项目管理 / mdtodo DOM 采样、Workbench launch command、捕获到的 `x-hwlab-otel-trace-id` 和 Tempo drill-down 命令。collect 视图不是采样器新增保存物,不构成第二事实源。
|
||||
- `observe start/status/command/collect/analyze` 默认输出包含 `Wrapper contract` 区块;该区块证明 Web 哨兵只能 wrap 现有 observe CLI verb、现有 runner/analyzer 和既有 artifact contract,不新增第二套 Playwright runner、analyzer、状态机或私有 web-probe API。
|
||||
- `web-probe sentinel plan|status` 只读取 `observability.webProbe.sentinel.enabled/configRefs` 和 owning YAML,渲染 redacted 配置引用图、文件 hash、缺失字段和跨 ref 冲突;它不启动浏览器、不读取 Secret 值、不保存采样结果,也不是第二套 runner/analyzer。真正的采样和判定仍以 `observe start|command|collect|analyze` artifacts 为准。
|
||||
- `web-probe sentinel plan|status` 只读取 `observability.webProbe.sentinel.enabled/configRefs` 和 owning YAML,渲染 redacted 配置引用图、文件 hash、缺失字段和跨 ref 冲突;`web-probe sentinel image|control-plane` 继续从 owning YAML 渲染 image、GitOps、Argo 和 manifest 计划,并在远端 publish job 接通前拒绝报告部署 mutation。它不启动浏览器、不读取 Secret 值、不保存采样结果,也不是第二套 runner/analyzer。真正的采样和判定仍以 `observe start|command|collect|analyze` artifacts 为准。
|
||||
- `scripts/web-probe-sentinel-service.ts` 是 Web 哨兵 Pod entrypoint;`--once` 只做 config/PVC/SQLite/scheduler/analyzer-command health 快照,`--scheduler-disabled` 仅用于本地服务健康冒烟,不能作为生产运行参数。HTTP 服务只提供 `/api/health`、`/api/status`、`/api/runs`、`/api/maintenance`、`/metrics` 和 redacted dashboard 外壳,底层采样仍只能经 observe CLI adapter。
|
||||
- `trace-frame` 出现 `(无 trace rows;这是 blocker...)` 时,必须先看同一输出中的 `TRACE DIAGNOSTIC`:记录 pageRole/pageId、traceRows/turns/messages 数量、sampleTraceIds、尾部 traceRow/turn/message 归属。若目标 trace 的 turn/message/final 存在但 traceRows 全部属于旧 trace,应按 Workbench read model authority 分裂登记到架构/业务 issue(例:HWLAB #2124),不得把旧 traceRows 当作新 turn 通过证据,也不得让 analyzer 的聚合计数压过 CLI trace 视图。
|
||||
- analyzer finding 不得压过 CLI `trace-frame` 人工视图。尤其 `trace-assistant-message-duplicates-final-response` 只有在 `trace-frame` 中同一 completed turn 可见多条相同 assistant final rows 时才按业务 bug 处理;如果 `trace-frame` 只有一条 assistant final row、后面固定 `Final Response` 区块正确且 API messages/turns 对齐,该 amber 归类为 analyzer 精度问题,应登记/修工具,不得阻止业务 closeout。
|
||||
|
||||
@@ -6,11 +6,24 @@ metadata:
|
||||
specRef: PJ2026-01060508
|
||||
sentinel:
|
||||
cicd:
|
||||
controlPlaneConfigRef: config/hwlab-node-control-plane.yaml#targets.D601.lanes.v03
|
||||
controlPlaneConfigRef: config/hwlab-node-control-plane.yaml#targets[0]
|
||||
source:
|
||||
repository: pikasTech/unidesk
|
||||
branch: master
|
||||
gitMirrorReadUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/unidesk.git
|
||||
buildContext: .
|
||||
entrypoint: scripts/web-probe-sentinel-service.ts
|
||||
gitopsPath: deploy/gitops/node/d601/web-probe-sentinel
|
||||
argo:
|
||||
namespace: argocd
|
||||
projectName: hwlab-d601
|
||||
applicationName: hwlab-web-probe-sentinel
|
||||
repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git
|
||||
targetRevision: v0.3-gitops
|
||||
image:
|
||||
repository: 127.0.0.1:5000/hwlab/web-probe-sentinel
|
||||
tagSource: source-commit
|
||||
baseImageRef: config/hwlab-node-control-plane.yaml#targets[0].tekton.toolsImage.output
|
||||
envRecipeRef: config/hwlab-web-probe-sentinel/runtime.d601-v03.yaml#sentinel.runtime
|
||||
maintenance:
|
||||
startCommand: sentinel maintenance start
|
||||
@@ -18,4 +31,4 @@ sentinel:
|
||||
targetValidation:
|
||||
scenarioId: workbench-dsflash-go-tool-call-10x
|
||||
maxSeconds: 120
|
||||
serviceUnavailableVerifyMode: pure-observe-cli
|
||||
serviceUnavailablePolicy: structured-failure
|
||||
|
||||
@@ -18,6 +18,7 @@ sentinel:
|
||||
listenHost: 0.0.0.0
|
||||
servicePort: 8080
|
||||
pvcName: hwlab-web-probe-sentinel-state
|
||||
pvcStorage: 10Gi
|
||||
stateRoot: /var/lib/web-probe-sentinel
|
||||
imageRef: 127.0.0.1:5000/hwlab/web-probe-sentinel:p3-service
|
||||
replicas: 1
|
||||
|
||||
@@ -96,7 +96,7 @@ Web哨兵必须遵循 UniDesk YAML-first ops。目标 node/lane、public origin
|
||||
| PJ2026-0106050803 | 常驻服务 | 本规格 6.3 | scheduler、scenario runner、PVC/SQLite、health、metrics、maintenance API | Wrapper边界、YAML配置 | dashboard、CI/CD |
|
||||
| PJ2026-0106050804 | 报告视图 | 本规格 6.4 | report/dashboard 渐进读取、分页、redaction、trace-frame | observe collect/analyze、Workbench唯一投影 | issue evidence、值守页面 |
|
||||
| PJ2026-0106050805 | 发布集成 | 本规格 6.5 | CI/CD、GitOps、Argo、maintenance、targetValidation、publicExposure | 发布流水、源码同步、公开入口 | 发布恢复判定 |
|
||||
| PJ2026-0106050806 | Canary验收 | 本规格 6.6 | dsflash-go 十轮工具调用、24 小时 dry-run 和 profile/fallback 边界 | Agent编排、Workbench、web-probe | 生产巡检收口 |
|
||||
| PJ2026-0106050806 | Canary验收 | 本规格 6.6 | dsflash-go 十轮工具调用、24 小时 dry-run 和 profile 结构化失败边界 | Agent编排、Workbench、web-probe | 生产巡检收口 |
|
||||
| PJ2026-0106050807 | 安全隔离 | 本规格 6.7 | Secret/prompt/provider redaction、NetworkPolicy、public dashboard auth | 用户管理、平台运维 | 安全 closeout |
|
||||
| PJ2026-0106050808 | 代码引用 | 本规格 6.8 | SPEC 头部标注和生成/配置追溯 | 规格治理 | 后续 PR 审计 |
|
||||
|
||||
@@ -270,23 +270,21 @@ sequenceDiagram
|
||||
|
||||
哨兵自身 rollout 只验证哨兵服务健康、配置装载、PVC/SQLite 可写、metrics、dashboard 和调度循环;它不能把另一个哨兵当作 HWLAB Web 业务观察对象。
|
||||
|
||||
### 5.7 纯 CLI fallback 时序图
|
||||
### 5.7 哨兵不可用结构化失败时序图
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant CI as CI/CD targetValidation
|
||||
participant Sen as Sentinel service
|
||||
participant CLI as web-probe observe CLI
|
||||
participant Ana as observe analyze
|
||||
participant Log as Structured closeout evidence
|
||||
|
||||
CI->>Sen: validate or maintenance/stop
|
||||
Sen-->>CI: unavailable or first-install
|
||||
CI->>CLI: observe start/status/command quick verify
|
||||
CLI->>Ana: observe analyze
|
||||
Ana-->>CI: report SHA, findings, fallback=true
|
||||
CI->>Log: missing service/config/health detail
|
||||
Log-->>CI: failed targetValidation and retry command
|
||||
```
|
||||
|
||||
Fallback 只能退回原纯客户端 `web-probe observe` CLI,不得退回裸 Playwright、私有 API、reload repair、读侧补洞或手工浏览器判断。
|
||||
哨兵不可用、首次安装未完成或配置未就绪时,CI/CD 必须结构化失败并输出缺失项、恢复建议和可重试命令;不得自动切换到第二执行通道。人工排障仍可显式运行原 `web-probe observe` CLI,但该人工动作不属于 CI/CD targetValidation 的自动通过路径。
|
||||
|
||||
## 6. 原子需求
|
||||
|
||||
@@ -352,7 +350,7 @@ Web哨兵自身必须纳入受控 CI/CD:Tekton 构建,GitOps 发布,ArgoCD
|
||||
|
||||
HWLAB runtime 发布 Pipeline 应在 Argo sync 前调用当前哨兵 `maintenance/start`,进入观察不告警模式;sync 完成且业务 Deployment Ready 后调用 `maintenance/stop`,触发同一 observe CLI quick verify 和 analyze。targetValidation 不能只因 Argo `Synced/Healthy` 通过而绿;还必须包含 quick verify 结果、analysis report SHA、finding 摘要、public origin、scenario id 和 observer/run id。
|
||||
|
||||
哨兵服务不可用或首次安装时,CI/CD 必须退回原纯客户端 `web-probe observe start/status/command/collect/analyze` quick verify,并把 fallback 事实写入 closeout。Fallback 不得使用裸 Playwright、私有 API、read-side repair、reload 循环或 session repair。
|
||||
哨兵服务不可用、首次安装未完成或配置未就绪时,CI/CD 必须结构化失败并输出缺失项、恢复建议和可重试命令;不得自动回退到原纯客户端 CLI、裸 Playwright、私有 API、read-side repair、reload 循环或 session repair 形成第二执行路径。人工排障可以显式运行原 `web-probe observe start/status/command/collect/analyze`,但不能被 targetValidation 当作自动通过证据。
|
||||
|
||||
### 6.6 OPS-SENTINEL-REQ-006 dsflash-go 十轮 canary
|
||||
|
||||
|
||||
@@ -53,6 +53,11 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab nodes web-probe run --node D601 --lane v03 --wait-messages-ms 1000",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel plan --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel image status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel image build --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane trigger-current --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes observability plan --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes observability status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes observability apply --node D601 --lane v03 --dry-run",
|
||||
@@ -64,6 +69,7 @@ export function hwlabNodeHelp(): Record<string, unknown> {
|
||||
"`trigger-current --confirm --wait` is the one-command CICD path: git-mirror pre-sync/pre-flush, control-plane refresh, PipelineRun create/reuse, bounded wait, and post-flush when terminal.",
|
||||
"`control-plane sync --confirm` syncs the YAML-declared local-k3s postgres bootstrap Secret, terminates a stale running Argo operation, deletes failed Argo hook Jobs, and recreates stale non-ready StatefulSet pods that are still pinned to an old controller revision with pull/backoff errors.",
|
||||
"`--wait` defaults to 120 seconds. If the PipelineRun is still active after 120 seconds, the CLI returns a warning plus env-reuse and git-mirror inspection commands instead of blocking.",
|
||||
"`web-probe sentinel image/control-plane` renders the YAML-first image, GitOps and Argo plan from the owning configRefs; unavailable service validation is a structured failure, not an automatic second execution path.",
|
||||
"Use `--rerun` for a deliberate YAML-first config-only publish when UniDesk node/lane render inputs changed but the HWLAB source commit did not."
|
||||
],
|
||||
};
|
||||
@@ -96,13 +102,18 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab nodes web-probe observe analyze webobs-xxxx",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel plan --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel image status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel image build --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane status --node D601 --lane v03",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane trigger-current --node D601 --lane v03 --dry-run",
|
||||
"bun scripts/cli.ts hwlab nodes web-probe script --node D601 --lane v03 <<'JS'\nexport default async ({ waitWorkbenchReady, fetchJson, fetchApiMatrix, recordStep, collectText, safeEvaluate, screenshot }) => {\n const ready = await waitWorkbenchReady();\n const workspace = await fetchJson('/v1/workbench/workspace?projectId=prj_hwpod_workbench');\n const apiMatrix = await fetchApiMatrix(['/v1/workbench/workspace?projectId=prj_hwpod_workbench', '/auth/session']);\n const workspaceText = await collectText('#workspace');\n const evaluated = await safeEvaluate(({ a, b }) => ({ sum: a + b }), { a: 1, b: 2 });\n await screenshot('workbench.png');\n recordStep('workbench-summary', { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, apiMatrixOk: apiMatrix.ok });\n return { finalUrl: ready.finalUrl, workspaceOk: workspace.ok, workspaceText, evaluated };\n};\nJS",
|
||||
],
|
||||
actions: {
|
||||
run: "Run the repo-owned scripts/web-live-dom-probe.mjs helper.",
|
||||
script: "Run caller-provided Playwright JS after CLI-managed /auth/login; the script receives authenticated browser/context/page plus gotoStable/reloadStable/gotoCurrentStable/safeReload/fetchJson/safeFetchJson/fetchApiMatrix/recordStep/collectText/safeEvaluate/waitWorkbenchReady/screenshotOnError/summarizeWorkspace/summarizeConversation helpers and must not handle secrets itself.",
|
||||
observe: "Start, inspect, control, stop, collect, and analyze a pure-client long-running Workbench observer on the target host. The observer runs a control page plus a passive observer page in a shared-auth browser context, receives commands through stateDir/commands files, writes JSONL artifacts, and does not expose any inbound service API.",
|
||||
sentinel: "Render the YAML-first service wrapper configRef graph for the production web-probe sentinel. This reads observability.webProbe.sentinel.enabled/configRefs and validates owning YAML presence, shape, redacted hashes, and cross-ref consistency without starting a browser or reading secret values.",
|
||||
sentinel: "Render the YAML-first service wrapper configRef graph plus image/GitOps/Argo control-plane plan for the production web-probe sentinel. This reads observability.webProbe.sentinel.enabled/configRefs and validates owning YAML presence, shape, redacted hashes, and cross-ref consistency without starting a browser or reading secret values.",
|
||||
},
|
||||
notes: [
|
||||
"The default probe URL, browser proxy mode, observe/analyze alert thresholds, and project-management observe behavior come from config/hwlab-node-lanes.yaml webProbe; pass --url only when intentionally overriding the YAML-selected origin.",
|
||||
@@ -118,7 +129,7 @@ export function hwlabNodeWebProbeHelp(): Record<string, unknown> {
|
||||
"observe analyze scans every sampled DOM point, extracts Workbench timing text such as 总耗时/total and 最近 N 秒/分前, and writes a sample point vs turn timing report: each Markdown table row starts with the timestamp, followed by each turn's 总耗时(s) and 最近更新(s). Timing series are reported for post-processing/manual analysis instead of auto-judged from status tail output.",
|
||||
"observe analyze also reports visible “加载中” count, owner attribution, concurrent loading owners, and continuous visible segments; fixes must reduce real loading latency, not reveal incomplete content early to make this metric disappear.",
|
||||
"script/observe support --browser-proxy-mode auto|direct; use direct for A/B evidence when frontend RUM is slow but OTel server spans are absent or fast, instead of falling back to raw Playwright.",
|
||||
"sentinel plan/status is a configuration visibility command for the service wrapper; observe start/status/command/collect/analyze remain the sampling and analysis truth.",
|
||||
"sentinel plan/status is a configuration visibility command for the service wrapper; sentinel image/control-plane renders image, GitOps and Argo control-plane state from owning YAML and refuses to report deployment mutation until the remote publish job is wired. observe start/status/command/collect/analyze remain the sampling and analysis truth.",
|
||||
"Use recordStep(name, data) or fetchApiMatrix(paths) to keep structured partial evidence when a later step fails.",
|
||||
"Use reloadStable(), gotoCurrentStable(), or safeReload() for bounded retries around page reload/current-URL navigation jitter such as ERR_NETWORK_CHANGED.",
|
||||
"Playwright page.evaluate accepts one serializable argument; use page.evaluate(({ a, b }) => ..., { a, b }) or safeEvaluate(fn, { a, b }).",
|
||||
|
||||
@@ -17,7 +17,7 @@ import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectVie
|
||||
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "./hwlab-node-web-observe-render";
|
||||
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "./hwlab-node-web-observe-wrapper";
|
||||
import { renderWebObserveWrapperContract } from "./hwlab-node-web-observe-wrapper-render";
|
||||
import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered, type WebProbeSentinelConfigAction } from "./hwlab-node-web-sentinel-config";
|
||||
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "./hwlab-node-web-sentinel-cicd";
|
||||
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "./hwlab-node-help";
|
||||
import { compactWebProbeResult, compactWebProbeScriptResult } from "./hwlab-node-web-probe-summary";
|
||||
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "./hwlab-node-observability-promql";
|
||||
@@ -115,10 +115,9 @@ interface NodeWebProbeObserveOptions {
|
||||
|
||||
interface NodeWebProbeSentinelOptions {
|
||||
action: "sentinel";
|
||||
sentinelAction: WebProbeSentinelConfigAction;
|
||||
sentinel: WebProbeSentinelOptions;
|
||||
node: string;
|
||||
lane: string;
|
||||
dryRun: boolean;
|
||||
}
|
||||
|
||||
type NodeWebProbeOptions = NodeWebProbeRunOptions | NodeWebProbeScriptOptions | NodeWebProbeObserveOptions | NodeWebProbeSentinelOptions;
|
||||
@@ -7439,21 +7438,38 @@ function parseNodeWebProbeOptions(args: string[]): NodeWebProbeOptions {
|
||||
|
||||
function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSentinelOptions {
|
||||
const [sentinelActionRaw] = args;
|
||||
if (sentinelActionRaw !== "plan" && sentinelActionRaw !== "status") {
|
||||
throw new Error("web-probe sentinel usage: sentinel plan|status --node NODE --lane vNN [--dry-run]");
|
||||
if (sentinelActionRaw !== "plan" && sentinelActionRaw !== "status" && sentinelActionRaw !== "image" && sentinelActionRaw !== "control-plane") {
|
||||
throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
}
|
||||
assertKnownOptions(args, new Set(["--node", "--lane"]), new Set(["--dry-run"]));
|
||||
assertKnownOptions(args, new Set(["--node", "--lane", "--timeout-seconds"]), new Set(["--dry-run", "--confirm", "--wait"]));
|
||||
const node = requiredOption(args, "--node");
|
||||
assertNodeId(node);
|
||||
const lane = requiredOption(args, "--lane");
|
||||
assertLane(lane);
|
||||
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
||||
const confirm = args.includes("--confirm");
|
||||
const dryRun = args.includes("--dry-run");
|
||||
if (confirm && dryRun) throw new Error("web-probe sentinel accepts only one of --confirm or --dry-run");
|
||||
const timeoutSeconds = positiveIntegerOption(args, "--timeout-seconds", 60, 3600);
|
||||
let sentinel: WebProbeSentinelOptions;
|
||||
if (sentinelActionRaw === "plan" || sentinelActionRaw === "status") {
|
||||
sentinel = { kind: "config", action: sentinelActionRaw, node, lane, dryRun };
|
||||
} else if (sentinelActionRaw === "image") {
|
||||
const imageAction = args[1];
|
||||
if (imageAction !== "status" && imageAction !== "build") throw new Error("web-probe sentinel image usage: image status|build --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
sentinel = { kind: "image", action: imageAction, node, lane, dryRun: imageAction === "build" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds };
|
||||
} else {
|
||||
const controlPlaneAction = args[1];
|
||||
if (controlPlaneAction !== "plan" && controlPlaneAction !== "apply" && controlPlaneAction !== "status" && controlPlaneAction !== "trigger-current") {
|
||||
throw new Error("web-probe sentinel control-plane usage: control-plane plan|apply|status|trigger-current --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
}
|
||||
sentinel = { kind: "control-plane", action: controlPlaneAction, node, lane, dryRun: controlPlaneAction === "apply" || controlPlaneAction === "trigger-current" ? dryRun || !confirm : dryRun, confirm, wait: args.includes("--wait"), timeoutSeconds };
|
||||
}
|
||||
return {
|
||||
action: "sentinel",
|
||||
sentinelAction: sentinelActionRaw,
|
||||
sentinel,
|
||||
node,
|
||||
lane,
|
||||
dryRun: args.includes("--dry-run"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7687,7 +7703,7 @@ function runNodeWebProbe(options: NodeWebProbeOptions): Record<string, unknown>
|
||||
const lane = options.lane;
|
||||
if (!isHwlabRuntimeLane(lane)) throw new Error(`web-probe only supports HWLAB runtime lanes, got ${lane}`);
|
||||
const spec = hwlabRuntimeLaneSpecForNode(lane, options.node);
|
||||
if (options.action === "sentinel") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.sentinelAction));
|
||||
if (options.action === "sentinel") return runWebProbeSentinelCommand(spec, options.sentinel);
|
||||
if (options.action === "observe" && options.observeAction !== "start") return runNodeWebProbeObserve(options, spec, null, null, null);
|
||||
const secretSpec = runtimeSecretSpec({ node: options.node, lane });
|
||||
const material = readBootstrapAdminPasswordMaterial(secretSpec);
|
||||
|
||||
@@ -0,0 +1,665 @@
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
||||
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { repoRoot, rootPath } from "./config";
|
||||
import { runCommand, type CommandResult } from "./command";
|
||||
import { webProbeSentinelConfigPlan, withWebProbeSentinelConfigRendered } from "./hwlab-node-web-sentinel-config";
|
||||
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
|
||||
export type WebProbeSentinelConfigAction = "plan" | "status";
|
||||
export type WebProbeSentinelImageAction = "status" | "build";
|
||||
export type WebProbeSentinelControlPlaneAction = "plan" | "apply" | "status" | "trigger-current";
|
||||
|
||||
export type WebProbeSentinelOptions =
|
||||
| {
|
||||
readonly kind: "config";
|
||||
readonly action: WebProbeSentinelConfigAction;
|
||||
readonly node: string;
|
||||
readonly lane: string;
|
||||
readonly dryRun: boolean;
|
||||
}
|
||||
| {
|
||||
readonly kind: "image";
|
||||
readonly action: WebProbeSentinelImageAction;
|
||||
readonly node: string;
|
||||
readonly lane: string;
|
||||
readonly dryRun: boolean;
|
||||
readonly confirm: boolean;
|
||||
readonly wait: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
}
|
||||
| {
|
||||
readonly kind: "control-plane";
|
||||
readonly action: WebProbeSentinelControlPlaneAction;
|
||||
readonly node: string;
|
||||
readonly lane: string;
|
||||
readonly dryRun: boolean;
|
||||
readonly confirm: boolean;
|
||||
readonly wait: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
};
|
||||
|
||||
interface SentinelCicdState {
|
||||
readonly spec: HwlabRuntimeLaneSpec;
|
||||
readonly configReady: boolean;
|
||||
readonly runtime: Record<string, unknown>;
|
||||
readonly cicd: Record<string, unknown>;
|
||||
readonly publicExposure: Record<string, unknown>;
|
||||
readonly controlPlaneTarget: Record<string, unknown>;
|
||||
readonly controlPlaneNode: Record<string, unknown>;
|
||||
readonly sourceHead: SourceHead;
|
||||
readonly image: SentinelImagePlan;
|
||||
readonly manifests: readonly Record<string, unknown>[];
|
||||
readonly manifestSha256: string;
|
||||
readonly valuesRedacted: true;
|
||||
}
|
||||
|
||||
interface SourceHead {
|
||||
readonly ok: boolean;
|
||||
readonly repository: string;
|
||||
readonly branch: string;
|
||||
readonly commit: string | null;
|
||||
readonly localHead: string | null;
|
||||
readonly result: CompactCommandResult;
|
||||
}
|
||||
|
||||
interface SentinelImagePlan {
|
||||
readonly repository: string;
|
||||
readonly tag: string;
|
||||
readonly ref: string;
|
||||
readonly digestRef: string | null;
|
||||
readonly baseImage: string;
|
||||
readonly buildContext: string;
|
||||
readonly entrypoint: string;
|
||||
readonly dockerfileSha256: string;
|
||||
readonly dockerfilePreview: string;
|
||||
}
|
||||
|
||||
interface CompactCommandResult {
|
||||
readonly exitCode: number | null;
|
||||
readonly timedOut: boolean;
|
||||
readonly stdoutBytes: number;
|
||||
readonly stderrBytes: number;
|
||||
readonly stdoutPreview: string;
|
||||
readonly stderrPreview: string;
|
||||
}
|
||||
|
||||
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel";
|
||||
|
||||
export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
|
||||
if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action));
|
||||
const state = loadSentinelCicdState(spec, options.timeoutSeconds);
|
||||
if (options.kind === "image") return runSentinelImage(state, options);
|
||||
return runSentinelControlPlane(state, options);
|
||||
}
|
||||
|
||||
function runSentinelImage(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "image" }>): RenderedCliResult {
|
||||
const command = `hwlab nodes web-probe sentinel image ${options.action}`;
|
||||
const registry = options.action === "status" ? probeImageRegistry(state, options.timeoutSeconds) : null;
|
||||
const mutationBlocked = options.confirm ? confirmBlocked("image build", state) : null;
|
||||
const registryReady = options.action !== "status" || record(registry?.probe).present === true;
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && registryReady && mutationBlocked === null,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
|
||||
mutation: false,
|
||||
specRef: SPEC_REF,
|
||||
source: state.sourceHead,
|
||||
image: state.image,
|
||||
registry,
|
||||
blocker: mutationBlocked,
|
||||
next: {
|
||||
status: `bun scripts/cli.ts hwlab nodes web-probe sentinel image status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
|
||||
dryRun: `bun scripts/cli.ts hwlab nodes web-probe sentinel image build --node ${state.spec.nodeId} --lane ${state.spec.lane} --dry-run`,
|
||||
controlPlanePlan: `bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node ${state.spec.nodeId} --lane ${state.spec.lane} --dry-run`,
|
||||
},
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok, command, renderImageResult(result));
|
||||
}
|
||||
|
||||
function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "control-plane" }>): RenderedCliResult {
|
||||
const command = `hwlab nodes web-probe sentinel control-plane ${options.action}`;
|
||||
const mutationAction = options.action === "apply" || options.action === "trigger-current";
|
||||
const mutationBlocked = options.confirm && mutationAction ? confirmBlocked(options.action, state) : null;
|
||||
const gitMirrorStatus = options.action === "status" ? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], options.timeoutSeconds) : null;
|
||||
const nodeControlPlaneStatus = options.action === "status" ? runChildCli(["hwlab", "nodes", "control-plane", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], options.timeoutSeconds) : null;
|
||||
const observedReady = options.action !== "status" || (record(gitMirrorStatus).ok === true && record(nodeControlPlaneStatus).ok === true);
|
||||
const pipelineRun = sentinelPipelineRunName(state);
|
||||
const result = {
|
||||
ok: state.configReady && state.sourceHead.ok && observedReady && mutationBlocked === null,
|
||||
command,
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
mode: options.action === "status" ? "status" : options.confirm ? "confirm" : "dry-run",
|
||||
mutation: false,
|
||||
specRef: SPEC_REF,
|
||||
source: state.sourceHead,
|
||||
image: state.image,
|
||||
pipelineRun,
|
||||
gitops: {
|
||||
path: stringField(state.cicd, "gitopsPath"),
|
||||
targetRevision: stringAt(state.cicd, "argo.targetRevision"),
|
||||
manifestObjects: state.manifests.length,
|
||||
manifestSha256: state.manifestSha256,
|
||||
},
|
||||
argo: {
|
||||
namespace: stringAt(state.cicd, "argo.namespace"),
|
||||
projectName: stringAt(state.cicd, "argo.projectName"),
|
||||
applicationName: stringAt(state.cicd, "argo.applicationName"),
|
||||
},
|
||||
maintenance: {
|
||||
startCommand: stringAt(state.cicd, "maintenance.startCommand"),
|
||||
stopCommand: stringAt(state.cicd, "maintenance.stopCommand"),
|
||||
serviceUnavailablePolicy: stringAt(state.cicd, "targetValidation.serviceUnavailablePolicy"),
|
||||
},
|
||||
validation: {
|
||||
scenarioId: stringAt(state.cicd, "targetValidation.scenarioId"),
|
||||
maxSeconds: numberAt(state.cicd, "targetValidation.maxSeconds"),
|
||||
automaticSecondPath: false,
|
||||
},
|
||||
manifests: {
|
||||
objects: manifestObjectSummary(state.manifests),
|
||||
sha256: state.manifestSha256,
|
||||
},
|
||||
observed: {
|
||||
gitMirror: gitMirrorStatus,
|
||||
nodeControlPlane: nodeControlPlaneStatus,
|
||||
},
|
||||
blocker: mutationBlocked,
|
||||
next: controlPlaneNext(state, options.action),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
return rendered(result.ok, command, renderControlPlaneResult(result));
|
||||
}
|
||||
|
||||
function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): SentinelCicdState {
|
||||
const sentinel = spec.observability.webProbe?.sentinel;
|
||||
if (sentinel === undefined) throw new Error(`config/hwlab-node-lanes.yaml#lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.sentinel is missing`);
|
||||
const configPlan = webProbeSentinelConfigPlan(spec, "status");
|
||||
const runtime = recordTarget(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime);
|
||||
const cicd = recordTarget(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd);
|
||||
const publicExposure = recordTarget(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
|
||||
const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef");
|
||||
const controlPlaneTarget = recordTarget(readConfigRefTarget(controlPlaneRef), controlPlaneRef);
|
||||
const controlPlaneConfig = recordTarget(readConfigFile(configRefFile(controlPlaneRef)), configRefFile(controlPlaneRef));
|
||||
const nodeId = stringField(controlPlaneTarget, "node");
|
||||
const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
|
||||
const sourceHead = resolveSourceHead(cicd, timeoutSeconds);
|
||||
const image = sentinelImagePlan(cicd, sourceHead);
|
||||
const manifests = renderSentinelManifests(spec, runtime, cicd, publicExposure, image);
|
||||
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
|
||||
return {
|
||||
spec,
|
||||
configReady: configPlan.ok,
|
||||
runtime,
|
||||
cicd,
|
||||
publicExposure,
|
||||
controlPlaneTarget,
|
||||
controlPlaneNode,
|
||||
sourceHead,
|
||||
image,
|
||||
manifests,
|
||||
manifestSha256: sha256(manifestYaml),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveSourceHead(cicd: Record<string, unknown>, timeoutSeconds: number): SourceHead {
|
||||
const repository = stringAt(cicd, "source.repository");
|
||||
const branch = stringAt(cicd, "source.branch");
|
||||
const remote = runCommand(["git", "ls-remote", "origin", `refs/heads/${branch}`], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const local = runCommand(["git", "rev-parse", "HEAD"], repoRoot, { timeoutMs: 10_000 });
|
||||
const commit = /^[0-9a-f]{40}\b/iu.exec(remote.stdout.trim())?.[0].toLowerCase() ?? null;
|
||||
const localHead = /^[0-9a-f]{40}$/iu.test(local.stdout.trim()) ? local.stdout.trim().toLowerCase() : null;
|
||||
return {
|
||||
ok: remote.exitCode === 0 && commit !== null,
|
||||
repository,
|
||||
branch,
|
||||
commit,
|
||||
localHead,
|
||||
result: compactCommand(remote),
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelImagePlan(cicd: Record<string, unknown>, sourceHead: SourceHead): SentinelImagePlan {
|
||||
const repository = stringAt(cicd, "image.repository");
|
||||
const tag = sourceHead.commit === null ? "source-unresolved" : sourceHead.commit.slice(0, 12);
|
||||
const baseImageRef = stringAt(cicd, "image.baseImageRef");
|
||||
const baseImage = stringTarget(readConfigRefTarget(baseImageRef), baseImageRef);
|
||||
const entrypoint = stringAt(cicd, "source.entrypoint");
|
||||
const dockerfile = sentinelDockerfile(baseImage, entrypoint);
|
||||
return {
|
||||
repository,
|
||||
tag,
|
||||
ref: `${repository}:${tag}`,
|
||||
digestRef: null,
|
||||
baseImage,
|
||||
buildContext: stringAt(cicd, "source.buildContext"),
|
||||
entrypoint,
|
||||
dockerfileSha256: sha256(dockerfile),
|
||||
dockerfilePreview: dockerfile,
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelDockerfile(baseImage: string, entrypoint: string): string {
|
||||
return [
|
||||
`FROM ${baseImage}`,
|
||||
"WORKDIR /app",
|
||||
"COPY . /app",
|
||||
"ENV NODE_ENV=production",
|
||||
`ENTRYPOINT ["bun", "${entrypoint}"]`,
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function renderSentinelManifests(
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
runtime: Record<string, unknown>,
|
||||
cicd: Record<string, unknown>,
|
||||
publicExposure: Record<string, unknown>,
|
||||
image: SentinelImagePlan,
|
||||
): readonly Record<string, unknown>[] {
|
||||
const namespace = stringAt(runtime, "namespace");
|
||||
const labels = {
|
||||
"app.kubernetes.io/name": stringAt(runtime, "deploymentName"),
|
||||
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
||||
"app.kubernetes.io/managed-by": "unidesk",
|
||||
"unidesk.ai/spec-ref": "PJ2026-01060508",
|
||||
"unidesk.ai/node": spec.nodeId,
|
||||
"unidesk.ai/lane": spec.lane,
|
||||
};
|
||||
const deploymentName = stringAt(runtime, "deploymentName");
|
||||
const serviceName = stringAt(runtime, "serviceName");
|
||||
const servicePort = numberAt(runtime, "servicePort");
|
||||
const pvcStorage = stringAt(runtime, "pvcStorage");
|
||||
const stateRoot = stringAt(runtime, "stateRoot");
|
||||
return [
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "ServiceAccount",
|
||||
metadata: { name: stringAt(runtime, "serviceAccountName"), namespace, labels },
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "PersistentVolumeClaim",
|
||||
metadata: { name: stringAt(runtime, "pvcName"), namespace, labels },
|
||||
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: pvcStorage } } },
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: { name: `${deploymentName}-config`, namespace, labels },
|
||||
data: {
|
||||
"config-summary.json": JSON.stringify({
|
||||
specRef: SPEC_REF,
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
publicBaseUrl: stringAt(publicExposure, "publicBaseUrl"),
|
||||
gitopsPath: stringAt(cicd, "gitopsPath"),
|
||||
valuesRedacted: true,
|
||||
}, null, 2),
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
metadata: { name: deploymentName, namespace, labels },
|
||||
spec: {
|
||||
replicas: numberAt(runtime, "replicas"),
|
||||
selector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
serviceAccountName: stringAt(runtime, "serviceAccountName"),
|
||||
containers: [{
|
||||
name: "sentinel",
|
||||
image: image.ref,
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
args: [
|
||||
"--node",
|
||||
spec.nodeId,
|
||||
"--lane",
|
||||
spec.lane,
|
||||
"--state-root",
|
||||
stateRoot,
|
||||
"--host",
|
||||
stringAt(runtime, "listenHost"),
|
||||
"--port",
|
||||
String(servicePort),
|
||||
],
|
||||
ports: [{ name: "http", containerPort: servicePort }],
|
||||
readinessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
|
||||
livenessProbe: { httpGet: { path: stringAt(runtime, "healthPath"), port: "http" } },
|
||||
volumeMounts: [{ name: "state", mountPath: stateRoot }],
|
||||
}],
|
||||
volumes: [{ name: "state", persistentVolumeClaim: { claimName: stringAt(runtime, "pvcName") } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: "v1",
|
||||
kind: "Service",
|
||||
metadata: { name: serviceName, namespace, labels },
|
||||
spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
|
||||
},
|
||||
{
|
||||
apiVersion: "networking.k8s.io/v1",
|
||||
kind: "NetworkPolicy",
|
||||
metadata: { name: `${deploymentName}-egress`, namespace, labels },
|
||||
spec: {
|
||||
podSelector: { matchLabels: { "app.kubernetes.io/name": deploymentName } },
|
||||
policyTypes: ["Ingress", "Egress"],
|
||||
ingress: [{ from: [{ namespaceSelector: {} }], ports: [{ protocol: "TCP", port: servicePort }] }],
|
||||
egress: [{ to: [{ namespaceSelector: {} }] }],
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: "argoproj.io/v1alpha1",
|
||||
kind: "Application",
|
||||
metadata: { name: stringAt(cicd, "argo.applicationName"), namespace: stringAt(cicd, "argo.namespace"), labels },
|
||||
spec: {
|
||||
project: stringAt(cicd, "argo.projectName"),
|
||||
source: {
|
||||
repoURL: stringAt(cicd, "argo.repoURL"),
|
||||
targetRevision: stringAt(cicd, "argo.targetRevision"),
|
||||
path: stringAt(cicd, "gitopsPath"),
|
||||
},
|
||||
destination: { server: "https://kubernetes.default.svc", namespace },
|
||||
syncPolicy: { automated: { prune: true, selfHeal: true } },
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
const endpoint = stringAt(state.controlPlaneNode, "registry.endpoint");
|
||||
const repoTag = state.image.ref.replace(`${endpoint}/`, "");
|
||||
const repo = repoTag.slice(0, repoTag.lastIndexOf(":"));
|
||||
const tag = repoTag.slice(repoTag.lastIndexOf(":") + 1);
|
||||
const route = stringAt(state.controlPlaneNode, "route");
|
||||
const script = [
|
||||
"set +e",
|
||||
`url=${shellQuote(`http://${endpoint}/v2/${repo}/manifests/${tag}`)}`,
|
||||
"headers=$(mktemp)",
|
||||
"if command -v curl >/dev/null 2>&1 && curl -fsSI --max-time 5 \"$url\" >\"$headers\" 2>/tmp/web-probe-sentinel-image.err; then present=true; else present=false; fi",
|
||||
"digest=$(awk 'BEGIN{IGNORECASE=1} /^docker-content-digest:/ {gsub(/\\r/,\"\",$2); print $2; exit}' \"$headers\" 2>/dev/null)",
|
||||
"python3 - \"$present\" \"$digest\" \"$url\" <<'PY'",
|
||||
"import json, sys",
|
||||
"print(json.dumps({'present': sys.argv[1] == 'true', 'digest': sys.argv[2] or None, 'url': sys.argv[3], 'valuesRedacted': True}))",
|
||||
"PY",
|
||||
].join("\n");
|
||||
const result = runCommand(["trans", route, "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
return {
|
||||
ok: result.exitCode === 0,
|
||||
probe: parseJsonObject(result.stdout),
|
||||
result: compactCommand(result),
|
||||
};
|
||||
}
|
||||
|
||||
function confirmBlocked(action: string, state: SentinelCicdState): Record<string, unknown> {
|
||||
return {
|
||||
code: "sentinel-cicd-confirm-requires-remote-publish-job",
|
||||
action,
|
||||
reason: "P4 currently provides YAML-first render/status/trigger dry-run and refuses to report a deployment mutation before the remote publish job is wired to the node-local git mirror.",
|
||||
sourceGitMirrorReadUrl: stringAt(state.cicd, "source.gitMirrorReadUrl"),
|
||||
requiredNextImplementation: [
|
||||
"clone source from source.gitMirrorReadUrl at selected commit",
|
||||
"build and push digest-pinned image on the selected node",
|
||||
"publish manifests to the HWLAB gitops branch/path through git-mirror",
|
||||
"flush/recheck git-mirror and let Argo reconcile the Application",
|
||||
],
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function controlPlaneNext(state: SentinelCicdState, action: WebProbeSentinelControlPlaneAction): Record<string, string> {
|
||||
const node = state.spec.nodeId;
|
||||
const lane = state.spec.lane;
|
||||
return {
|
||||
plan: `bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane plan --node ${node} --lane ${lane} --dry-run`,
|
||||
status: `bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane status --node ${node} --lane ${lane}`,
|
||||
image: `bun scripts/cli.ts hwlab nodes web-probe sentinel image status --node ${node} --lane ${lane}`,
|
||||
triggerCurrent: `bun scripts/cli.ts hwlab nodes web-probe sentinel control-plane trigger-current --node ${node} --lane ${lane} --dry-run`,
|
||||
issue: "https://github.com/pikasTech/unidesk/issues/889",
|
||||
currentAction: action,
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelPipelineRunName(state: SentinelCicdState): string {
|
||||
const commit = state.sourceHead.commit ?? "source";
|
||||
return `hwlab-web-probe-sentinel-${commit.slice(0, 12)}`;
|
||||
}
|
||||
|
||||
function runChildCli(args: string[], timeoutSeconds: number): Record<string, unknown> {
|
||||
const result = runCommand(["bun", "scripts/cli.ts", ...args], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 120) * 1000 });
|
||||
return {
|
||||
ok: result.exitCode === 0,
|
||||
parsed: parseJsonObject(result.stdout),
|
||||
result: compactCommand(result),
|
||||
};
|
||||
}
|
||||
|
||||
function renderImageResult(result: Record<string, unknown>): string {
|
||||
const source = record(result.source);
|
||||
const image = record(result.image);
|
||||
const registry = record(result.registry);
|
||||
const blocker = record(result.blocker);
|
||||
const next = record(result.next);
|
||||
return [
|
||||
String(result.command),
|
||||
"",
|
||||
table(["NODE", "LANE", "STATUS", "MODE", "MUTATION"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.mutation]]),
|
||||
"",
|
||||
table(["SOURCE_REPO", "BRANCH", "COMMIT", "LOCAL_HEAD"], [[source.repository, source.branch, short(source.commit), short(source.localHead)]]),
|
||||
"",
|
||||
table(["IMAGE", "BASE", "ENTRYPOINT", "DOCKERFILE"], [[image.ref, image.baseImage, image.entrypoint, short(image.dockerfileSha256)]]),
|
||||
"",
|
||||
Object.keys(registry).length === 0 ? "REGISTRY\n-" : table(["PROBED", "PRESENT", "DIGEST"], [[record(registry.probe).url ?? "-", record(registry.probe).present ?? "-", short(record(registry.probe).digest)]]),
|
||||
"",
|
||||
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
||||
"",
|
||||
"NEXT",
|
||||
` status: ${next.status ?? "-"}`,
|
||||
` dry-run: ${next.dryRun ?? "-"}`,
|
||||
` control-plane: ${next.controlPlanePlan ?? "-"}`,
|
||||
"",
|
||||
"DISCLOSURE",
|
||||
" valuesRedacted=true; image status shows refs, hashes and object names only.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
const source = record(result.source);
|
||||
const image = record(result.image);
|
||||
const gitops = record(result.gitops);
|
||||
const argo = record(result.argo);
|
||||
const validation = record(result.validation);
|
||||
const observed = record(result.observed);
|
||||
const blocker = record(result.blocker);
|
||||
const next = record(result.next);
|
||||
return [
|
||||
String(result.command),
|
||||
"",
|
||||
table(["NODE", "LANE", "STATUS", "MODE", "PIPELINERUN"], [[result.node, result.lane, result.ok === true ? "ok" : "blocked", result.mode, result.pipelineRun]]),
|
||||
"",
|
||||
table(["SOURCE", "COMMIT", "IMAGE", "MANIFEST"], [[`${source.repository}@${source.branch}`, short(source.commit), image.ref, short(gitops.manifestSha256)]]),
|
||||
"",
|
||||
table(["GITOPS_PATH", "ARGO_APP", "TARGET_REV", "OBJECTS"], [[gitops.path, argo.applicationName, gitops.targetRevision, gitops.manifestObjects]]),
|
||||
"",
|
||||
table(["SCENARIO", "MAX_SECONDS", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.automaticSecondPath]]),
|
||||
"",
|
||||
renderObservedStatus(observed),
|
||||
"",
|
||||
Object.keys(blocker).length === 0 ? "BLOCKER\n-" : table(["CODE", "REASON"], [[blocker.code, blocker.reason]]),
|
||||
"",
|
||||
"NEXT",
|
||||
` plan: ${next.plan ?? "-"}`,
|
||||
` status: ${next.status ?? "-"}`,
|
||||
` image: ${next.image ?? "-"}`,
|
||||
` trigger-current: ${next.triggerCurrent ?? "-"}`,
|
||||
"",
|
||||
"DISCLOSURE",
|
||||
" default view is a bounded CI/CD summary; full manifest content is represented by object counts and sha256.",
|
||||
" sentinel unavailable policy is structured-failure; no automatic second execution path is rendered.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function renderObservedStatus(observed: Record<string, unknown>): string {
|
||||
const rows = [
|
||||
observedStatusRow("git-mirror", observed.gitMirror),
|
||||
observedStatusRow("control-plane", observed.nodeControlPlane),
|
||||
].filter((row) => row !== null);
|
||||
if (rows.length === 0) return "OBSERVED\n-";
|
||||
return table(["CHECK", "OK", "EXIT", "TIMED_OUT", "STDOUT_BYTES", "PREVIEW"], rows);
|
||||
}
|
||||
|
||||
function observedStatusRow(name: string, value: unknown): unknown[] | null {
|
||||
const item = record(value);
|
||||
if (Object.keys(item).length === 0) return null;
|
||||
const result = record(item.result);
|
||||
return [name, item.ok, result.exitCode, result.timedOut, result.stdoutBytes, result.stdoutPreview];
|
||||
}
|
||||
|
||||
function rendered(ok: boolean, command: string, text: string): RenderedCliResult {
|
||||
return { ok, command, renderedText: `${text.trimEnd()}\n`, contentType: "text/plain" };
|
||||
}
|
||||
|
||||
function readConfigRefTarget(ref: string): unknown {
|
||||
const file = configRefFile(ref);
|
||||
const path = configRefPath(ref);
|
||||
return valueAtPath(readConfigFile(file), path);
|
||||
}
|
||||
|
||||
function readConfigFile(file: string): unknown {
|
||||
if (file.startsWith("/") || file.includes("..") || !file.startsWith("config/")) throw new Error(`unsafe configRef file: ${file}`);
|
||||
const abs = rootPath(file);
|
||||
if (!existsSync(abs)) throw new Error(`${file} does not exist`);
|
||||
return Bun.YAML.parse(readFileSync(abs, "utf8")) as unknown;
|
||||
}
|
||||
|
||||
function configRefFile(ref: string): string {
|
||||
const [file, path, extra] = ref.split("#");
|
||||
if (extra !== undefined || file === undefined || path === undefined || file.length === 0 || path.length === 0) throw new Error(`invalid configRef: ${ref}`);
|
||||
return file;
|
||||
}
|
||||
|
||||
function configRefPath(ref: string): string {
|
||||
const [, path] = ref.split("#");
|
||||
if (path === undefined || path.length === 0) throw new Error(`invalid configRef: ${ref}`);
|
||||
return path;
|
||||
}
|
||||
|
||||
function valueAtPath(value: unknown, path: string): unknown {
|
||||
let current: unknown = value;
|
||||
for (const segment of path.split(".")) {
|
||||
const match = /^([A-Za-z0-9_-]+)?(?:\[(\d+)\])?$/u.exec(segment);
|
||||
if (match === null) return undefined;
|
||||
if (match[1] !== undefined) {
|
||||
if (!isRecord(current)) return undefined;
|
||||
current = current[match[1]];
|
||||
}
|
||||
if (match[2] !== undefined) {
|
||||
if (!Array.isArray(current)) return undefined;
|
||||
current = current[Number(match[2])];
|
||||
}
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function stringAt(value: unknown, path: string): string {
|
||||
const found = valueAtPath(value, path);
|
||||
if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function stringField(value: Record<string, unknown>, path: string): string {
|
||||
return stringAt(value, path);
|
||||
}
|
||||
|
||||
function stringTarget(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || value.length === 0) throw new Error(`${label} must resolve to a non-empty string`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function numberAt(value: unknown, path: string): number {
|
||||
const found = valueAtPath(value, path);
|
||||
if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`);
|
||||
return found;
|
||||
}
|
||||
|
||||
function recordTarget(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!isRecord(value)) throw new Error(`${label} must resolve to an object`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return isRecord(value) ? value : {};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function manifestObjectSummary(items: readonly Record<string, unknown>[]): readonly Record<string, unknown>[] {
|
||||
return items.map((item) => ({
|
||||
kind: item.kind ?? null,
|
||||
name: record(item.metadata).name ?? null,
|
||||
namespace: record(item.metadata).namespace ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function compactCommand(result: CommandResult): CompactCommandResult {
|
||||
return {
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
stdoutBytes: Buffer.byteLength(result.stdout),
|
||||
stderrBytes: Buffer.byteLength(result.stderr),
|
||||
stdoutPreview: result.stdout.trim().slice(0, 500),
|
||||
stderrPreview: result.stderr.trim().slice(0, 500),
|
||||
};
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): Record<string, unknown> | null {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length === 0) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
return isRecord(parsed) ? parsed : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function table(headers: string[], rows: unknown[][]): string {
|
||||
const normalized = [headers, ...rows.map((row) => row.map(text))];
|
||||
const widths = headers.map((_, index) => Math.max(...normalized.map((row) => text(row[index] ?? "").length)));
|
||||
return normalized.map((row) => row.map((cell, index) => text(cell).padEnd(widths[index])).join(" ").trimEnd()).join("\n");
|
||||
}
|
||||
|
||||
function text(value: unknown): string {
|
||||
if (value === undefined || value === null || value === "") return "-";
|
||||
if (typeof value === "boolean") return value ? "true" : "false";
|
||||
return String(value).replace(/\s+/gu, " ").trim();
|
||||
}
|
||||
|
||||
function short(value: unknown): string {
|
||||
const raw = text(value);
|
||||
if (raw === "-") return raw;
|
||||
if (/^sha256:[0-9a-f]{64}$/iu.test(raw)) return `${raw.slice(0, 19)}...`;
|
||||
if (/^[0-9a-f]{40}$/iu.test(raw)) return raw.slice(0, 12);
|
||||
return raw.length > 42 ? `${raw.slice(0, 39)}...` : raw;
|
||||
}
|
||||
|
||||
function sha256(textValue: string): string {
|
||||
return `sha256:${createHash("sha256").update(textValue).digest("hex")}`;
|
||||
}
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replace(/'/gu, "'\\''")}'`;
|
||||
}
|
||||
@@ -62,6 +62,7 @@ const REQUIRED_TARGET_SHAPES: Record<HwlabRuntimeWebProbeSentinelConfigRefKey, R
|
||||
"listenHost",
|
||||
"servicePort",
|
||||
"pvcName",
|
||||
"pvcStorage",
|
||||
"stateRoot",
|
||||
"imageRef",
|
||||
"replicas",
|
||||
@@ -126,15 +127,26 @@ const REQUIRED_TARGET_SHAPES: Record<HwlabRuntimeWebProbeSentinelConfigRefKey, R
|
||||
kind: "object",
|
||||
requiredPaths: [
|
||||
"controlPlaneConfigRef",
|
||||
"source.repository",
|
||||
"source.branch",
|
||||
"source.gitMirrorReadUrl",
|
||||
"source.buildContext",
|
||||
"source.entrypoint",
|
||||
"gitopsPath",
|
||||
"argo.namespace",
|
||||
"argo.projectName",
|
||||
"argo.applicationName",
|
||||
"argo.repoURL",
|
||||
"argo.targetRevision",
|
||||
"image.repository",
|
||||
"image.tagSource",
|
||||
"image.baseImageRef",
|
||||
"image.envRecipeRef",
|
||||
"maintenance.startCommand",
|
||||
"maintenance.stopCommand",
|
||||
"targetValidation.scenarioId",
|
||||
"targetValidation.maxSeconds",
|
||||
"targetValidation.serviceUnavailableVerifyMode",
|
||||
"targetValidation.serviceUnavailablePolicy",
|
||||
],
|
||||
},
|
||||
secrets: {
|
||||
|
||||
Reference in New Issue
Block a user