diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 5829e2f1..811ec534 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -91,10 +91,13 @@ bun scripts/cli.ts hwlab nodes control-plane infra runtime-image preload --node bun scripts/cli.ts hwlab nodes control-plane infra runtime-image logs --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm +bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run |--source-commit ] [--full|--raw] ``` 从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap，不触发 runtime rollout，不创建 PK01 DB，也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact；输入 base/pull image 必须是 YAML 中声明的公开 registry 来源，缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明：官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML，不能使用手工 kubectl/argo CLI 作为正式安装路径。 +`hwlab nodes control-plane status` 默认返回 compact commander summary，只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action；完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。 + ### G14 v0.3 runtime base image ```bash diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 3b014502..131b3452 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -218,6 +218,7 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane plan --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 --full", "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", @@ -1582,14 +1583,21 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType line.trim()).filter(Boolean); + const workloadReadinessProbe = namespaceExists + ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60) + : null; + const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? ""); const bridge = externalPostgresBridgeStatus(spec, namespaceExists); const secrets = externalPostgresSecretStatus(spec, namespaceExists); + const publicProbes = nodeRuntimePublicProbeStatus(spec); const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0; - const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready)); + const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready); + const runtimeReady = namespaceExists && localPostgresObjects.length === 0 && workloadsReady && (spec.externalPostgres === undefined || (bridge.ready && secrets.ready)); const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy"; const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True"; - return { - ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady, + const publicReady = publicProbes.ready === true; + const fullStatus = { + ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady, command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, mode: "node-scoped-runtime-status", mutation: false, @@ -1620,16 +1628,21 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType> { + return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { + const [ref = "", counts = ""] = line.split(/\t/u); + const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/"); + const readyReplicas = nullableInteger(readyRaw); + const currentReplicas = nullableInteger(currentRaw); + const desiredReplicas = nullableInteger(desiredRaw); + const ready = desiredReplicas !== null + ? (readyReplicas ?? 0) >= desiredReplicas + : readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas; + return { + ref, + readyReplicas, + currentReplicas, + desiredReplicas, + ready, + }; + }); +} + +function nullableInteger(value: string): number | null { + if (!/^[0-9]+$/u.test(value)) return null; + return Number(value); +} + +function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record { + const web = publicHttpProbe("web", spec.publicWebUrl); + const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live")); + return { + ready: web.ok === true && apiHealth.ok === true, + web, + apiHealth, + }; +} + +function publicHttpProbe(kind: string, url: string): Record { + const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 }); + const httpStatus = nullableInteger(result.stdout.trim().slice(-3)); + return { + kind, + url, + ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400, + httpStatus, + result: compactRuntimeCommand(result), + }; +} + +function joinUrlPath(baseUrl: string, suffix: string): string { + return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`; +} + +function summarizeNodeRuntimeControlPlaneStatus(status: Record, scoped: ReturnType): Record { + const pipelineRun = record(status.pipelineRun); + const argo = record(status.argo); + const runtime = record(status.runtime); + const publicProbes = record(status.publicProbes); + const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : []; + const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length; + const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length; + const webProbe = record(publicProbes.web); + const apiProbe = record(publicProbes.apiHealth); + return { + ok: status.ok === true, + command: status.command, + mode: "node-scoped-runtime-status-summary", + mutation: false, + node: status.node, + lane: status.lane, + sourceCommit: status.sourceCommit, + degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null, + pipelineRun: { + name: pipelineRun.name ?? null, + exists: pipelineRun.exists === true, + status: pipelineRun.status ?? null, + reason: pipelineRun.reason ?? null, + message: pipelineRun.message ?? null, + createdAt: pipelineRun.createdAt ?? null, + ready: pipelineRun.status === "True", + }, + argo: { + application: argo.application ?? null, + ready: argo.ready === true, + syncRevision: argo.syncRevision ?? null, + syncStatus: argo.syncStatus ?? null, + health: argo.health ?? null, + }, + runtime: { + namespace: runtime.namespace ?? null, + namespaceExists: runtime.namespaceExists === true, + ready: runtime.ready === true, + workloadCount, + workloadReady: `${readyWorkloads}/${workloadReadiness.length}`, + localPostgresAbsent: runtime.localPostgresAbsent === true, + externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined + ? null + : record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true, + }, + publicProbe: { + ready: publicProbes.ready === true, + web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null }, + apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null }, + }, + nextAction: nodeRuntimeStatusNextAction(status, scoped), + next: { + full: `${nodeRuntimeStatusCommand(scoped)} --full`, + plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, + triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, + webProbe: `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`, + }, + }; +} + +function nodeRuntimeStatusNextAction(status: Record, scoped: ReturnType): string { + const reason = typeof status.degradedReason === "string" ? status.degradedReason : null; + if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`; + if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") { + return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`; + } + if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`; + if (reason === "argo-not-synced-healthy") { + return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`; + } + if (reason === "pipelinerun-not-succeeded") { + return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`; + } + if (reason === "public-probe-not-ready") { + return `bun scripts/cli.ts hwlab nodes web-probe run --node ${scoped.node} --lane ${scoped.lane}`; + } + return `${nodeRuntimeStatusCommand(scoped)} --full`; +} + +function nodeRuntimeStatusCommand(scoped: ReturnType): string { + const sourceCommit = optionValue(scoped.originalArgs, "--source-commit"); + const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run"); + return [ + `bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`, + sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`, + pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`, + ].filter(Boolean).join(" "); } function nodeRuntimeRenderToken(): string { diff --git a/scripts/src/jobs.ts b/scripts/src/jobs.ts index 9a053de1..a5c7cf79 100644 --- a/scripts/src/jobs.ts +++ b/scripts/src/jobs.ts @@ -208,18 +208,22 @@ export function jobWithTail(job: JobRecord, maxBytes = 12000): JobRecord & { const progressTailBytes = Math.max(maxBytes, 96_000); const stdoutProgressTail = tailFile(job.stdoutFile, progressTailBytes); const stderrProgressTail = tailFile(job.stderrFile, progressTailBytes); + const progress = summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail }); + const rawStdoutTail = tailTextByBytes(stdoutProgressTail, maxBytes); + const stdoutTail = compactJobStdoutTail(job, progress, rawStdoutTail, maxBytes); return { ...job, - progress: summarizeJobProgress(job, progressTailBytes, { stdoutTail: stdoutProgressTail, stderrTail: stderrProgressTail }), + progress, tailPolicy: { requestedTailBytes: maxBytes, stdoutBytes, stderrBytes, stdoutTruncated: stdoutBytes > maxBytes, stderrTruncated: stderrBytes > maxBytes, + stdoutCompacted: stdoutTail !== rawStdoutTail, fullLogPaths: { stdoutFile: job.stdoutFile, stderrFile: job.stderrFile }, }, - stdoutTail: tailTextByBytes(stdoutProgressTail, maxBytes), + stdoutTail, stderrTail: tailTextByBytes(stderrProgressTail, maxBytes), }; } @@ -791,6 +795,22 @@ function tailTextByBytes(text: string, maxBytes: number): string { return buffer.subarray(buffer.length - safeMaxBytes).toString("utf8"); } +function compactJobStdoutTail(job: JobRecord, progress: JobProgressSummary, rawTail: string, requestedTailBytes: number): string { + if (progress.kind !== "hwlab-runtime-lane-trigger") return rawTail; + if (Buffer.byteLength(rawTail, "utf8") <= 4096 && !rawTail.includes("\"expected\"") && !rawTail.includes("stdoutTail")) return rawTail; + const interestingLines = rawTail.split(/\r?\n/u) + .map((line) => line.trimEnd()) + .filter((line) => !/"(stdoutTail|stderrTail|expected|result)"\s*:/u.test(line)) + .filter((line) => /hwlab\.runtime-lane\.trigger\.progress|sourceCommit|pipelineRun|pipelinerun|degradedReason|next(Command|Action)|statusCommand|created|Completed|failed|succeeded/iu.test(line)) + .slice(-80); + const header = [ + `[stdout compacted: ${progress.kind}; requestedTailBytes=${requestedTailBytes}]`, + `full stdout: ${job.stdoutFile}`, + ].join("\n"); + if (interestingLines.length === 0) return `${header}\n${tailTextByBytes(rawTail, Math.min(requestedTailBytes, 3000))}`; + return `${header}\n${interestingLines.join("\n")}\n`; +} + function parseJsonLineEvents(text: string, eventName: string): Record[] { const events: Record[] = []; for (const line of text.split(/\r?\n/u)) {