const { spawn } = require("node:child_process"); const crypto = require("node:crypto"); const fs = require("node:fs"); const fsp = require("node:fs/promises"); const os = require("node:os"); const path = require("node:path"); const namespace = "hwlab-dev"; const lockName = "hwlab-dev-cd-lock"; const nativeKubeconfig = "/etc/rancher/k3s/k3s.yaml"; const legacyDefaultRepo = "/home/ubuntu/hwlab_cd"; const rejectedRepo = "/home/ubuntu/hwlab"; const defaultRemoteCandidates = ["git@github.com:pikasTech/HWLAB.git"]; const dedicatedCacheRoot = path.join(os.homedir(), ".cache", "unidesk", "hwlab-cd"); const dedicatedCacheRepo = path.join(os.homedir(), ".cache", "unidesk", "hwlab-cd", "git-cache", "HWLAB.git"); const defaultEgressProxy = "http://127.0.0.1:18789"; const defaultNoProxy = "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local"; const seedCacheRepos = ["/home/ubuntu/hwlab", "/home/ubuntu/workspace/hwlab", "/tmp/hwlab-dev-cd-35bbbee-host"]; const requiredNodeName = "d601"; const tailChars = 1400; const parseCaptureLimitBytes = 1024 * 1024; const requiredSecretRefs = [ { secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url", consumers: ["hwlab-cloud-api"] }, { secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url", consumers: ["runtime provisioning", "runtime migration"] }, { secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key", consumers: ["hwlab-cloud-api", "code agent provider"] }, ]; function decodeOptions() { const raw = process.env.UNIDESK_HWLAB_CD_OPTIONS_B64 || ""; if (!raw) throw new Error("UNIDESK_HWLAB_CD_OPTIONS_B64 is required"); return JSON.parse(Buffer.from(raw, "base64").toString("utf8")); } function redact(value) { return String(value || "") .replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1") .replace(/\b(token|password|api[_-]?key|secret|client-key-data|client-certificate-data|certificate-authority-data)\b(\s*[:=]\s*)(["']?)([^"'\s,}]+)/giu, "$1$2$3") .replace(/\b(postgres(?:ql)?:\/\/[^:\s/@]+:)([^@\s]+)(@)/giu, "$1$3"); } function boundedTail(value) { const text = redact(value); return text.slice(Math.max(0, text.length - tailChars)); } function readFileTail(filePath) { try { return boundedTail(fs.readFileSync(filePath, "utf8")); } catch { return ""; } } function commandView(result) { return { id: result.id, command: result.command, cwd: result.cwd, ok: result.ok, exitCode: result.exitCode, signal: result.signal, timedOut: result.timedOut, durationMs: result.durationMs, dump: { stdout: result.dump.stdout, stderr: result.dump.stderr, stdoutBytes: result.dump.stdoutBytes, stderrBytes: result.dump.stderrBytes, }, }; } async function runCaptured(command, cwd, dumpDir, id, options = {}) { await fsp.mkdir(dumpDir, { recursive: true, mode: 0o700 }); const stdoutPath = path.join(dumpDir, `${id}.stdout.txt`); const stderrPath = path.join(dumpDir, `${id}.stderr.txt`); fs.writeFileSync(stdoutPath, "", { mode: 0o600 }); fs.writeFileSync(stderrPath, "", { mode: 0o600 }); const startedAt = Date.now(); let timedOut = false; let spawnErrorMessage = ""; let stdoutBytes = 0; let stderrBytes = 0; const stdoutChunks = []; const stderrChunks = []; const stdout = fs.createWriteStream(stdoutPath, { flags: "a" }); const stderr = fs.createWriteStream(stderrPath, { flags: "a" }); const child = spawn(command[0], command.slice(1), { cwd, env: options.env || process.env, stdio: ["ignore", "pipe", "pipe"], }); const timeout = options.timeoutMs ? setTimeout(() => { timedOut = true; child.kill("SIGTERM"); setTimeout(() => child.kill("SIGKILL"), 1000).unref(); }, options.timeoutMs) : null; child.stdout.on("data", (chunk) => { stdoutBytes += chunk.byteLength; stdout.write(chunk); if (Buffer.concat(stdoutChunks).byteLength < parseCaptureLimitBytes) stdoutChunks.push(Buffer.from(chunk)); }); child.stderr.on("data", (chunk) => { stderrBytes += chunk.byteLength; stderr.write(chunk); if (Buffer.concat(stderrChunks).byteLength < parseCaptureLimitBytes) stderrChunks.push(Buffer.from(chunk)); }); const closed = await new Promise((resolve) => { child.on("error", (error) => { spawnErrorMessage = error.message; }); child.on("close", (exitCode, signal) => resolve({ exitCode, signal })); }); if (timeout !== null) clearTimeout(timeout); await new Promise((resolve) => stdout.end(resolve)); await new Promise((resolve) => stderr.end(resolve)); const stdoutText = stdoutBytes <= parseCaptureLimitBytes ? Buffer.concat(stdoutChunks).toString("utf8") : ""; const stderrText = stderrBytes <= parseCaptureLimitBytes ? Buffer.concat(stderrChunks).toString("utf8") : spawnErrorMessage; const exitCode = spawnErrorMessage && closed.exitCode === null ? 127 : closed.exitCode; return { id, command, cwd, ok: exitCode === 0 && !timedOut, exitCode, signal: closed.signal, timedOut, durationMs: Date.now() - startedAt, stdoutText, stderrText, dump: { stdout: stdoutPath, stderr: stderrPath, stdoutBytes, stderrBytes, stdoutTail: boundedTail(stdoutText || readFileTail(stdoutPath)), stderrTail: boundedTail(stderrText || readFileTail(stderrPath)), }, }; } function parseJson(text) { if (!String(text || "").trim()) return null; try { return JSON.parse(text); } catch { return null; } } function asRecord(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : null; } function stringValue(value) { return typeof value === "string" && value.length > 0 ? value : null; } function sha256(text) { return `sha256:${crypto.createHash("sha256").update(text).digest("hex")}`; } function accessCheck(targetPath, mode) { try { fs.accessSync(targetPath, mode); return { ok: true, path: targetPath }; } catch (error) { return { ok: false, path: targetPath, error: error instanceof Error ? error.message : String(error) }; } } function repoRecord(rawPath, source, extra = {}) { const absolutePath = path.resolve(rawPath); const rejected = absolutePath === rejectedRepo; const devCdApply = path.join(absolutePath, "scripts/dev-cd-apply.mjs"); const deployJson = path.join(absolutePath, "deploy/deploy.json"); return { status: !rejected && fs.existsSync(absolutePath) && fs.existsSync(devCdApply) && fs.existsSync(deployJson) ? "selected" : "blocked", source, path: absolutePath, defaultPath: legacyDefaultRepo, defaultMode: "ephemeral-clone", rejected, rejectionReason: rejected ? "runner-history-directory-is-not-hwlab-cd-release-truth" : null, exists: fs.existsSync(absolutePath), hasDevCdApply: fs.existsSync(devCdApply), hasDeployJson: fs.existsSync(deployJson), ...extra, }; } function remoteCandidates() { const fromEnv = process.env.UNIDESK_HWLAB_REMOTE_URL; if (typeof fromEnv === "string" && fromEnv.trim().length > 0) return [fromEnv.trim()]; return defaultRemoteCandidates; } function seedRepoCandidates() { const fromEnv = process.env.UNIDESK_HWLAB_CACHE_SEED_REPOS; const configured = typeof fromEnv === "string" && fromEnv.trim().length > 0 ? fromEnv.split(":").map((item) => item.trim()).filter(Boolean) : seedCacheRepos; return [...new Set(configured.map((item) => path.resolve(item)))]; } function egressEnv() { const proxy = process.env.UNIDESK_HWLAB_EGRESS_PROXY || defaultEgressProxy; const noProxy = process.env.NO_PROXY || process.env.no_proxy || defaultNoProxy; const proxyHostPort = proxy.replace(/^https?:\/\//u, ""); const gitSshCommand = process.env.GIT_SSH_COMMAND || [ "ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", `"ProxyCommand=nc -X connect -x ${proxyHostPort} %h %p"`, ].join(" "); return { ...process.env, HTTP_PROXY: proxy, HTTPS_PROXY: proxy, ALL_PROXY: proxy, http_proxy: proxy, https_proxy: proxy, all_proxy: proxy, NO_PROXY: noProxy, no_proxy: noProxy, GIT_SSH_COMMAND: gitSshCommand, }; } async function egressProxyHealth(dumpDir, timeoutMs) { const proxy = process.env.UNIDESK_HWLAB_EGRESS_PROXY || defaultEgressProxy; const healthUrl = `${proxy.replace(/\/$/u, "")}/__unidesk/egress-proxy/health`; const command = await runCaptured(["curl", "-fsS", "--max-time", "3", healthUrl], os.homedir(), dumpDir, "egress-proxy-health", { env: egressEnv(), timeoutMs: Math.min(timeoutMs, 5000) }); const body = parseJson(command.stdoutText); return { status: command.ok && asRecord(body)?.connected === true ? "pass" : "blocked", proxy, healthUrl, connected: asRecord(body)?.connected ?? null, command: commandView(command), }; } function pathOwner(targetPath) { try { const stat = fs.statSync(targetPath); return { uid: stat.uid, gid: stat.gid, mode: (stat.mode & 0o777).toString(8), isDirectory: stat.isDirectory() }; } catch (error) { return { uid: null, gid: null, mode: null, isDirectory: false, error: error instanceof Error ? error.message : String(error) }; } } function cacheRepoRecord(cachePath) { const absolutePath = path.resolve(cachePath); const gitPath = path.join(absolutePath, ".git"); const bareHead = path.join(absolutePath, "HEAD"); const bareObjects = path.join(absolutePath, "objects"); const shallowPath = path.join(absolutePath, "shallow"); const exists = fs.existsSync(gitPath) || (fs.existsSync(bareHead) && fs.existsSync(bareObjects)); const readable = accessCheck(absolutePath, fs.constants.R_OK | fs.constants.X_OK); const writable = accessCheck(absolutePath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK); const owner = pathOwner(absolutePath); return { path: absolutePath, exists, readable: readable.ok, writable: writable.ok, owner, shallow: fs.existsSync(shallowPath), error: readable.error || writable.error || null }; } async function deployJsonCommitFromCache(dumpDir, timeoutMs) { const command = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "show", "refs/heads/main:deploy/deploy.json"], os.homedir(), dumpDir, "git-dedicated-cache-deploy-json", { timeoutMs: Math.min(timeoutMs, 5000) }); if (!command.ok) return { commitId: null, command: commandView(command) }; const parsed = parseJson(command.stdoutText); return { commitId: stringValue(asRecord(parsed)?.commitId), command: commandView(command) }; } async function cacheCommitExists(commitId, dumpDir, timeoutMs, idPrefix = "git-dedicated-cache-commit") { if (!commitId) return { exists: false, command: null }; const command = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "--verify", `${commitId}^{commit}`], os.homedir(), dumpDir, idPrefix, { timeoutMs: Math.min(timeoutMs, 5000) }); return { exists: command.ok, resolvedCommit: command.stdoutText.trim() || null, command: commandView(command) }; } async function refreshDedicatedCache(remoteUrl, dumpDir, timeoutMs) { const env = egressEnv(); const parent = path.dirname(dedicatedCacheRepo); await fsp.mkdir(parent, { recursive: true, mode: 0o700 }); await fsp.chmod(dedicatedCacheRoot, 0o700).catch(() => undefined); await fsp.chmod(path.dirname(dedicatedCacheRoot), 0o700).catch(() => undefined); const attempts = []; const existing = cacheRepoRecord(dedicatedCacheRepo); if (existing.exists && (!existing.writable || existing.owner.uid !== process.getuid?.())) { attempts.push({ stage: "dedicated-cache-owner", ok: false, cache: existing }); return { ok: false, cache: existing, attempts, stage: "dedicated-cache-owner" }; } if (existing.exists) { const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions", { timeoutMs: Math.min(timeoutMs, 5000) }); attempts.push({ stage: "cache-permissions", ok: permissions.ok, command: commandView(permissions) }); if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions" }; } if (!existing.exists) { for (const seed of seedRepoCandidates().map(cacheRepoRecord)) { if (!seed.exists || !seed.readable) { attempts.push({ stage: "seed-unavailable", ok: false, seed }); continue; } await fsp.rm(dedicatedCacheRepo, { recursive: true, force: true }); const cloneSeed = await runCaptured(["git", "clone", "--bare", seed.path, dedicatedCacheRepo], parent, dumpDir, "git-dedicated-cache-seed-clone", { timeoutMs: Math.min(timeoutMs, 15000) }); attempts.push({ stage: "seed-clone", ok: cloneSeed.ok, seed, command: commandView(cloneSeed) }); if (cloneSeed.ok) break; } if (!fs.existsSync(dedicatedCacheRepo)) { const init = await runCaptured(["git", "init", "--bare", dedicatedCacheRepo], parent, dumpDir, "git-dedicated-cache-init", { timeoutMs: Math.min(timeoutMs, 5000) }); attempts.push({ stage: "bare-init", ok: init.ok, command: commandView(init) }); if (!init.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "bare-init" }; } const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions", { timeoutMs: Math.min(timeoutMs, 5000) }); attempts.push({ stage: "cache-permissions", ok: permissions.ok, command: commandView(permissions) }); if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions" }; } const remoteSet = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "remote", "set-url", "origin", remoteUrl], os.homedir(), dumpDir, "git-dedicated-cache-remote-set-url", { timeoutMs: Math.min(timeoutMs, 5000) }); const remoteAdd = remoteSet.ok ? null : await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "remote", "add", "origin", remoteUrl], os.homedir(), dumpDir, "git-dedicated-cache-remote-add", { timeoutMs: Math.min(timeoutMs, 5000) }); attempts.push({ stage: "remote-set-url", ok: remoteSet.ok || remoteAdd?.ok === true, command: commandView(remoteSet), fallback: remoteAdd ? commandView(remoteAdd) : null }); if (!remoteSet.ok && remoteAdd?.ok !== true) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "remote-set-url" }; const remoteHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "ls-remote", "--heads", "origin", "main"], os.homedir(), dumpDir, "git-dedicated-cache-ls-remote-main", { env, timeoutMs: Math.min(timeoutMs, 15000) }); const remoteMainCommit = remoteHead.stdoutText.trim().split(/\s+/u)[0] || null; attempts.push({ stage: "ls-remote-main", ok: remoteHead.ok && Boolean(remoteMainCommit), remoteMainCommit, command: commandView(remoteHead) }); const localHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "refs/heads/main"], os.homedir(), dumpDir, "git-dedicated-cache-local-main", { timeoutMs: Math.min(timeoutMs, 5000) }); const localMainCommit = localHead.stdoutText.trim() || null; attempts.push({ stage: "local-main", ok: localHead.ok && Boolean(localMainCommit), localMainCommit, command: commandView(localHead) }); const deployJson = localHead.ok ? await deployJsonCommitFromCache(dumpDir, timeoutMs) : { commitId: null, command: null }; attempts.push({ stage: "deploy-json-commit", ok: Boolean(deployJson.commitId), commitId: deployJson.commitId, command: deployJson.command }); const deployCommit = await cacheCommitExists(deployJson.commitId, dumpDir, timeoutMs, "git-dedicated-cache-deploy-commit"); attempts.push({ stage: "deploy-commit-present", ok: deployCommit.exists, commitId: deployJson.commitId, resolvedCommit: deployCommit.resolvedCommit, command: deployCommit.command }); const cacheRecordBeforeFetch = cacheRepoRecord(dedicatedCacheRepo); if (remoteHead.ok && remoteMainCommit && localHead.ok && localMainCommit === remoteMainCommit && deployCommit.exists && cacheRecordBeforeFetch.shallow !== true) { return { ok: true, cache: cacheRecordBeforeFetch, attempts, stage: "ready", refreshStatus: "current-full", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId }; } const fetchArgs = ["git", "--git-dir", dedicatedCacheRepo, "fetch", "--prune", "--tags", "origin", "+refs/heads/*:refs/heads/*"]; if (cacheRecordBeforeFetch.shallow === true) fetchArgs.splice(4, 0, "--unshallow"); const fetch = await runCaptured(fetchArgs, os.homedir(), dumpDir, "git-dedicated-cache-fetch-full", { env, timeoutMs: Math.min(timeoutMs, 55000) }); attempts.push({ stage: "fetch-main", ok: fetch.ok, command: commandView(fetch) }); if (!fetch.ok) { if (localHead.ok && localMainCommit && deployCommit.exists) { return { ok: true, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "stale-cache", refreshStatus: "blocked", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId }; } return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "fetch-full", refreshStatus: "blocked", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId }; } const fetchedLocalHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "refs/heads/main"], os.homedir(), dumpDir, "git-dedicated-cache-local-main-after-fetch", { timeoutMs: Math.min(timeoutMs, 5000) }); const fetchedLocalMainCommit = fetchedLocalHead.stdoutText.trim() || localMainCommit; attempts.push({ stage: "local-main-after-fetch", ok: fetchedLocalHead.ok && Boolean(fetchedLocalMainCommit), localMainCommit: fetchedLocalMainCommit, command: commandView(fetchedLocalHead) }); const fetchedDeployJson = await deployJsonCommitFromCache(dumpDir, timeoutMs); attempts.push({ stage: "deploy-json-commit-after-fetch", ok: Boolean(fetchedDeployJson.commitId), commitId: fetchedDeployJson.commitId, command: fetchedDeployJson.command }); const fetchedDeployCommit = await cacheCommitExists(fetchedDeployJson.commitId, dumpDir, timeoutMs, "git-dedicated-cache-deploy-commit-after-fetch"); attempts.push({ stage: "deploy-commit-present-after-fetch", ok: fetchedDeployCommit.exists, commitId: fetchedDeployJson.commitId, resolvedCommit: fetchedDeployCommit.resolvedCommit, command: fetchedDeployCommit.command }); const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions-after-fetch", { timeoutMs: Math.min(timeoutMs, 5000) }); attempts.push({ stage: "cache-permissions-after-fetch", ok: permissions.ok, command: commandView(permissions) }); if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions-after-fetch", refreshStatus: "blocked", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId }; if (!fetchedDeployCommit.exists) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "deploy-commit-unresolved", refreshStatus: "blocked", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId }; return { ok: true, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "ready", refreshStatus: "refreshed-full", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId }; } async function materializeFromCache(cachePath, repoPath, remoteUrl, dumpDir, timeoutMs) { await fsp.rm(repoPath, { recursive: true, force: true }); const clone = await runCaptured(["git", "clone", "--no-hardlinks", "--no-checkout", cachePath, repoPath], path.dirname(repoPath), dumpDir, "git-cache-clone", { timeoutMs: Math.min(timeoutMs, 15000) }); if (!clone.ok) return { ok: false, stage: "cache-clone", commands: [commandView(clone)] }; const remote = await runCaptured(["git", "remote", "set-url", "origin", remoteUrl], repoPath, dumpDir, "git-cache-remote-set-url", { timeoutMs: Math.min(timeoutMs, 5000) }); if (!remote.ok) return { ok: false, stage: "remote-set-url", commands: [commandView(clone), commandView(remote)] }; const checkout = await runCaptured(["git", "checkout", "-B", "main", "origin/main"], repoPath, dumpDir, "git-cache-checkout-main", { timeoutMs: Math.min(timeoutMs, 5000) }); const reset = checkout.ok ? await runCaptured(["git", "reset", "--hard", "origin/main"], repoPath, dumpDir, "git-cache-reset-hard", { timeoutMs: Math.min(timeoutMs, 5000) }) : null; const clean = checkout.ok && reset?.ok ? await runCaptured(["git", "clean", "-ffdqx"], repoPath, dumpDir, "git-cache-clean", { timeoutMs: Math.min(timeoutMs, 5000) }) : null; const upstream = checkout.ok && reset?.ok ? await runCaptured(["git", "branch", "--set-upstream-to=origin/main", "main"], repoPath, dumpDir, "git-cache-upstream", { timeoutMs: Math.min(timeoutMs, 5000) }) : null; const fetchHead = checkout.ok && reset?.ok && clean?.ok ? await runCaptured(["bash", "-lc", "p=\"$(git rev-parse --path-format=absolute --git-common-dir)/FETCH_HEAD\"; : > \"$p\"; chmod 600 \"$p\""], repoPath, dumpDir, "git-cache-fetch-head", { timeoutMs: Math.min(timeoutMs, 5000) }) : null; const commands = [clone, remote, checkout, reset, clean, upstream, fetchHead].filter(Boolean).map(commandView); return { ok: checkout.ok && reset?.ok === true && clean?.ok === true && fetchHead?.ok === true, stage: "ready", commands }; } async function cloneEphemeralRepo(dumpDir, timeoutMs) { const root = path.join(dumpDir, "ephemeral-repo"); const attempts = []; await fsp.mkdir(root, { recursive: true, mode: 0o700 }); const egress = await egressProxyHealth(dumpDir, Math.min(timeoutMs, 5000)); if (egress.status !== "pass") { return repoRecord(path.join(root, "HWLAB"), "ephemeral:git-cache", { ephemeral: true, remoteUrl: null, egressProxy: egress, dedicatedCache: cacheRepoRecord(dedicatedCacheRepo), seedRepos: seedRepoCandidates().map(cacheRepoRecord), cloneAttempts: [], cloneFailed: true, cloneFailureStage: "egress-proxy", retainedForDiagnostics: true, }); } for (const remoteUrl of remoteCandidates()) { const repoPath = path.join(root, "HWLAB"); const refreshed = await refreshDedicatedCache(remoteUrl, dumpDir, Math.min(timeoutMs, 45000)); attempts.push({ remoteUrl, dedicatedCache: refreshed.cache, ok: refreshed.ok, stage: refreshed.stage, refreshAttempts: refreshed.attempts }); if (!refreshed.ok) continue; const materialized = await materializeFromCache(dedicatedCacheRepo, repoPath, remoteUrl, dumpDir, Math.min(timeoutMs, 9000)); attempts.push({ remoteUrl, dedicatedCache: refreshed.cache, ok: materialized.ok, stage: materialized.stage, commands: materialized.commands }); if (materialized.ok) { return repoRecord(repoPath, "ephemeral:dedicated-git-cache", { ephemeral: true, remoteUrl, egressProxy: egress, dedicatedCache: refreshed.cache, cacheRefreshStatus: refreshed.refreshStatus || "current", remoteMainCommit: refreshed.remoteMainCommit || null, cacheMainCommit: refreshed.localMainCommit || null, deployCommitId: refreshed.deployCommitId || null, seedRepos: seedRepoCandidates().map(cacheRepoRecord), cloneAttempts: attempts, retainedForDiagnostics: true, }); } } return repoRecord(path.join(root, "HWLAB"), "ephemeral:dedicated-git-cache", { ephemeral: true, remoteUrl: null, egressProxy: egress, dedicatedCache: cacheRepoRecord(dedicatedCacheRepo), seedRepos: seedRepoCandidates().map(cacheRepoRecord), cloneAttempts: attempts, cloneFailed: true, cloneFailureStage: attempts.findLast((attempt) => attempt.stage)?.stage || "cache-unavailable", retainedForDiagnostics: true, }); } async function resolveRepo(provided, dumpDir, timeoutMs) { if (provided) return repoRecord(provided, "option", { ephemeral: false }); if (process.env.UNIDESK_HWLAB_REPO) return repoRecord(process.env.UNIDESK_HWLAB_REPO, "env:UNIDESK_HWLAB_REPO", { ephemeral: false }); return cloneEphemeralRepo(dumpDir, Math.min(timeoutMs, 55000)); } async function gitSummary(repoPath, dumpDir, timeoutMs) { const [branch, head, originMain, remote, gitDir, gitCommonDir, statusShort, statusPorcelain] = await Promise.all([ runCaptured(["git", "rev-parse", "--abbrev-ref", "HEAD"], repoPath, dumpDir, "git-branch", { timeoutMs }), runCaptured(["git", "rev-parse", "HEAD"], repoPath, dumpDir, "git-head", { timeoutMs }), runCaptured(["git", "rev-parse", "--verify", "origin/main^{commit}"], repoPath, dumpDir, "git-origin-main", { timeoutMs }), runCaptured(["git", "remote", "get-url", "origin"], repoPath, dumpDir, "git-remote-origin", { timeoutMs }), runCaptured(["git", "rev-parse", "--path-format=absolute", "--git-dir"], repoPath, dumpDir, "git-dir", { timeoutMs }), runCaptured(["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], repoPath, dumpDir, "git-common-dir", { timeoutMs }), runCaptured(["git", "status", "--short", "--branch"], repoPath, dumpDir, "git-status-short", { timeoutMs }), runCaptured(["git", "status", "--porcelain=v1"], repoPath, dumpDir, "git-status-porcelain", { timeoutMs }), ]); const branchName = branch.stdoutText.trim(); const headCommit = head.stdoutText.trim(); const originMainCommit = originMain.stdoutText.trim(); const remoteUrl = remote.stdoutText.trim(); const gitDirPath = gitDir.stdoutText.trim(); const gitCommonDirPath = gitCommonDir.stdoutText.trim() || gitDirPath; const fetchHeadPath = gitCommonDirPath ? path.join(gitCommonDirPath, "FETCH_HEAD") : path.join(repoPath, ".git", "FETCH_HEAD"); const worktreesPath = gitCommonDirPath ? path.join(gitCommonDirPath, "worktrees") : path.join(repoPath, ".git", "worktrees"); const porcelain = statusPorcelain.stdoutText.trim(); const dirtyLines = porcelain ? porcelain.split("\n").filter(Boolean) : []; const worktreeAccess = accessCheck(repoPath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK); const fetchHeadAccess = accessCheck(fetchHeadPath, fs.constants.R_OK | fs.constants.W_OK); const worktreesAccess = fs.existsSync(worktreesPath) ? accessCheck(worktreesPath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK) : { ok: true, path: worktreesPath, exists: false }; const remoteMatches = /github\.com[:/]pikasTech\/HWLAB(?:\.git)?$/u.test(remoteUrl); const blockers = []; if (!branch.ok || !head.ok || !remote.ok || !gitDir.ok || !gitCommonDir.ok || !statusShort.ok || !statusPorcelain.ok) blockers.push({ scope: "hwlab-git-command", summary: "One or more HWLAB git guard commands failed." }); if (dirtyLines.length > 0) blockers.push({ scope: "hwlab-git-clean", summary: "HWLAB CD worktree has local modifications; deploy/deploy.json cannot be treated as release truth." }); if (branchName !== "main") blockers.push({ scope: "hwlab-git-main", summary: `HWLAB CD worktree branch is ${branchName || ""}, expected main.` }); if (!remoteMatches) blockers.push({ scope: "hwlab-git-remote", summary: "HWLAB CD worktree origin is not pikasTech/HWLAB." }); if (originMain.ok && headCommit && originMainCommit && headCommit !== originMainCommit) blockers.push({ scope: "hwlab-git-origin-main", summary: "HWLAB CD worktree HEAD does not match local origin/main." }); if (!worktreeAccess.ok) blockers.push({ scope: "hwlab-worktree-permission", summary: "HWLAB CD worktree is not readable/writable/executable by the wrapper." }); if (!fetchHeadAccess.ok) blockers.push({ scope: "hwlab-fetch-head-permission", summary: "HWLAB CD FETCH_HEAD is missing or not writable by the wrapper." }); if (!worktreesAccess.ok) blockers.push({ scope: "hwlab-worktrees-permission", summary: "HWLAB .git/worktrees permission guard failed." }); return { status: blockers.length === 0 ? "pass" : "blocked", clean: dirtyLines.length === 0, branch: branchName || null, onMain: branchName === "main", remote: remoteUrl || null, remoteMatches, expectedRemote: "git@github.com:pikasTech/HWLAB.git or https://github.com/pikasTech/HWLAB.git", headCommit: headCommit || null, originMainCommit: originMain.ok ? originMainCommit || null : null, headMatchesOriginMain: originMain.ok && headCommit.length > 0 && headCommit === originMainCommit, worktreeAccess, fetchHead: { path: fetchHeadPath, exists: fs.existsSync(fetchHeadPath), writable: fetchHeadAccess.ok, error: fetchHeadAccess.error || null }, worktrees: worktreesAccess, statusShort: statusShort.stdoutText.trim().split("\n").filter(Boolean).slice(0, 30), dirtyCount: dirtyLines.length, dirtyPreview: dirtyLines.slice(0, 20), blockers, commands: [branch, head, originMain, remote, gitDir, gitCommonDir, statusShort, statusPorcelain].map(commandView), }; } function readJsonSummary(repoPath, relativePath, fallback = {}) { const absolutePath = path.join(repoPath, relativePath); try { const raw = fs.readFileSync(absolutePath, "utf8"); const parsed = JSON.parse(raw); return { path: relativePath, exists: true, hash: sha256(raw), commitId: parsed.commitId || parsed.sourceCommitId || null, environment: parsed.environment || null, namespace: parsed.namespace || namespace, endpoint: parsed.endpoint || null, artifactState: parsed.artifactState || parsed.publish?.artifactState || null, ciPublished: parsed.publish?.ciPublished ?? parsed.artifactPublish?.ciPublished ?? null, registryVerified: parsed.publish?.registryVerified ?? parsed.artifactPublish?.registryVerified ?? null, serviceCount: Array.isArray(parsed.services) ? parsed.services.length : parsed.artifactPublish?.serviceCount ?? null, ...fallback, }; } catch (error) { return { path: relativePath, exists: false, hash: null, error: error instanceof Error ? error.message : String(error), ...fallback, }; } } function readJsonFile(repoPath, relativePath) { const absolutePath = path.join(repoPath, relativePath); try { const raw = fs.readFileSync(absolutePath, "utf8"); return { ok: true, path: relativePath, exists: true, hash: sha256(raw), json: JSON.parse(raw), error: null }; } catch (error) { return { ok: false, path: relativePath, exists: fs.existsSync(absolutePath), hash: null, json: null, error: error instanceof Error ? error.message : String(error), }; } } function commitMatches(actual, expected) { if (typeof actual !== "string" || typeof expected !== "string" || actual.length === 0 || expected.length === 0) return false; return actual === expected || actual.startsWith(expected) || expected.startsWith(actual); } function imageTag(image) { const value = String(image || ""); const slash = value.lastIndexOf("/"); const colon = value.lastIndexOf(":"); return colon > slash ? value.slice(colon + 1) : null; } function servicesFromManifest(manifest) { const services = Array.isArray(manifest?.services) ? manifest.services : manifest?.services && typeof manifest.services === "object" ? Object.values(manifest.services) : []; return services .filter((service) => service && typeof service === "object") .map((service) => ({ serviceId: stringValue(service.serviceId) || stringValue(service.id) || stringValue(service.name), name: stringValue(service.name) || stringValue(service.serviceId) || stringValue(service.id), image: stringValue(service.image), imageTag: stringValue(service.imageTag) || imageTag(service.image), commitId: stringValue(service.commitId) || stringValue(service.imageTag) || imageTag(service.image), namespace: stringValue(service.namespace) || namespace, healthPath: stringValue(service.healthPath) || "/health/live", replicas: Number.isInteger(service.replicas) ? service.replicas : null, env: asRecord(service.env) || {}, digest: stringValue(service.digest), publishState: stringValue(service.publishState) || stringValue(service.status), artifactRequired: service.artifactRequired !== false, })) .filter((service) => service.serviceId); } function summarizeDeployFile(file) { const manifest = asRecord(file.json); const endpoints = asRecord(manifest?.publicEndpoints) || {}; return { path: file.path, exists: file.exists, hash: file.hash, commitId: stringValue(manifest?.commitId), environment: stringValue(manifest?.environment), namespace: stringValue(manifest?.namespace) || namespace, endpoint: stringValue(manifest?.endpoint), serviceCount: servicesFromManifest(manifest).length, publicEndpoints: { frontend: stringValue(asRecord(endpoints.frontend)?.url), api: stringValue(asRecord(endpoints.api)?.url), }, unavailableReason: file.ok ? null : file.error, }; } function summarizeArtifactFile(file) { const manifest = asRecord(file.json); const services = servicesFromManifest(manifest); const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 }; for (const service of services) { if (/^sha256:[a-f0-9]{64}$/u.test(service.digest || "")) digestCounts.sha256 += 1; else if (service.digest === "not_published") digestCounts.notPublished += 1; else digestCounts.invalid += 1; } return { path: file.path, exists: file.exists, hash: file.hash, commitId: stringValue(manifest?.commitId), artifactState: stringValue(manifest?.artifactState), ciPublished: manifest?.publish?.ciPublished ?? null, registryVerified: manifest?.publish?.registryVerified ?? null, serviceCount: services.length, digestCounts, unavailableReason: file.ok ? null : file.error, }; } function buildDesiredStateAudit(repoPath, target) { const deployFile = readJsonFile(repoPath, "deploy/deploy.json"); const catalogFile = readJsonFile(repoPath, "deploy/artifact-catalog.dev.json"); const reportFile = readJsonFile(repoPath, "reports/dev-gate/dev-artifacts.json"); const deployServices = servicesFromManifest(deployFile.json); const catalogServices = servicesFromManifest(catalogFile.json); const catalogById = new Map(catalogServices.map((service) => [service.serviceId, service])); const mismatches = []; const missing = []; for (const service of deployServices) { if (!service.image) continue; const catalog = catalogById.get(service.serviceId); if (!catalog && service.artifactRequired) { missing.push({ serviceId: service.serviceId, reason: "missing-from-artifact-catalog" }); continue; } if (catalog?.image && catalog.image !== service.image) { mismatches.push({ serviceId: service.serviceId, deployImage: service.image, catalogImage: catalog.image }); } if (catalog && catalog.artifactRequired && catalog.publishState && catalog.publishState !== "published") { missing.push({ serviceId: service.serviceId, reason: `artifact-${catalog.publishState}` }); } } const deploySummary = summarizeDeployFile(deployFile); const catalogSummary = summarizeArtifactFile(catalogFile); const reportSummary = summarizeArtifactFile(reportFile); const targetCommit = stringValue(target?.shortCommitId) || stringValue(target?.promotionCommit) || deploySummary.commitId; const commitConverged = Boolean( targetCommit && commitMatches(deploySummary.commitId, targetCommit) && (!catalogSummary.commitId || commitMatches(catalogSummary.commitId, targetCommit)) ); const status = !deployFile.ok || !catalogFile.ok || missing.length > 0 ? "missing" : mismatches.length > 0 || !commitConverged ? "mismatch" : "pass"; return { status, targetCommit, commitConverged, deployJson: deploySummary, artifactCatalog: catalogSummary, artifactReport: reportSummary, imageConvergence: { status: missing.length === 0 && mismatches.length === 0 ? "pass" : missing.length > 0 ? "missing" : "mismatch", serviceCount: deployServices.length, catalogServiceCount: catalogServices.length, missing: missing.slice(0, 12), mismatches: mismatches.slice(0, 12), }, services: deployServices.map((service) => ({ serviceId: service.serviceId, image: service.image, imageTag: service.imageTag, catalogImage: catalogById.get(service.serviceId)?.image || null, catalogDigest: catalogById.get(service.serviceId)?.digest || null, replicas: service.replicas, })).slice(0, 30), serviceEnv: { cloudApi: asRecord(deployServices.find((service) => service.serviceId === "hwlab-cloud-api")?.env) || {}, }, }; } function forbiddenKubeSignals(text) { const signals = []; if (/docker-desktop/iu.test(text)) signals.push("docker-desktop"); if (/desktop-control-plane/iu.test(text)) signals.push("desktop-control-plane"); if (/127\.0\.0\.1:11700/u.test(text)) signals.push("127.0.0.1:11700"); return [...new Set(signals)]; } async function observeKube(kubeconfig, dumpDir, timeoutMs, withExplicitEnv) { const env = withExplicitEnv ? { ...process.env, KUBECONFIG: kubeconfig } : { ...process.env }; if (!withExplicitEnv) { delete env.KUBECONFIG; delete env.HWLAB_DEV_KUBECONFIG; } const [context, server, nodes, namespaceCheck] = await Promise.all([ runCaptured(["kubectl", "config", "current-context"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-current-context" : "default-current-context", { env, timeoutMs }), runCaptured(["kubectl", "config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-server" : "default-server", { env, timeoutMs }), runCaptured(["kubectl", "get", "nodes", "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-nodes" : "default-nodes", { env, timeoutMs }), runCaptured(["kubectl", "get", "namespace", namespace, "-o", "name"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-namespace" : "default-namespace", { env, timeoutMs }), ]); const combinedText = [context.stdoutText, context.stderrText, server.stdoutText, server.stderrText, nodes.stdoutText, nodes.stderrText, namespaceCheck.stdoutText, namespaceCheck.stderrText].join("\n"); return { checked: true, explicit: withExplicitEnv, currentContext: context.stdoutText.trim() || null, apiServer: server.stdoutText.trim() || null, nodeNames: nodes.stdoutText.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), namespaceObserved: namespaceCheck.ok, commandsOk: context.ok && server.ok && nodes.ok, refusalSignals: forbiddenKubeSignals(combinedText), commands: [context, server, nodes, namespaceCheck].map(commandView), }; } async function nodeGuard(kubeconfig, dumpDir, timeoutMs) { const [explicit, bare] = await Promise.all([ observeKube(kubeconfig, dumpDir, timeoutMs, true), observeKube(kubeconfig, dumpDir, timeoutMs, false), ]); const wrongKubeconfig = kubeconfig !== nativeKubeconfig; const requiredNodePresent = explicit.nodeNames.includes(requiredNodeName); const explicitRefusal = explicit.refusalSignals.length > 0; const secondControlPlaneRisk = Boolean( bare.namespaceObserved && explicit.apiServer && bare.apiServer && bare.apiServer !== explicit.apiServer ); const status = explicitRefusal ? "refused" : wrongKubeconfig || !explicit.commandsOk || !requiredNodePresent ? "blocked" : "pass"; return { status, refusal: explicitRefusal, refusalSignals: explicit.refusalSignals, kubeconfig, expectedKubeconfig: nativeKubeconfig, currentContext: explicit.currentContext, apiServer: explicit.apiServer, nodeNames: explicit.nodeNames, nodeCount: explicit.nodeNames.length, requiredNodeName, requiredNodePresent, commandsOk: explicit.commandsOk, secondControlPlaneRisk, defaultKubectlDiagnostic: { checked: true, currentContext: bare.currentContext, apiServer: bare.apiServer, nodeNames: bare.nodeNames, namespaceObserved: bare.namespaceObserved, refusalSignals: bare.refusalSignals, status: bare.refusalSignals.length > 0 ? "stale-forbidden-default" : bare.commandsOk ? "clean" : "unavailable", secondControlPlaneRisk, summary: secondControlPlaneRisk ? "Bare kubectl can observe hwlab-dev through a different control plane; write paths must stop." : bare.refusalSignals.length > 0 ? "Bare kubectl shows a forbidden local control-plane signal; explicit D601 KUBECONFIG remains the target." : "Bare kubectl did not prove a second HWLAB DEV control plane.", }, summary: explicitRefusal ? "Refusing because explicit D601 kubeconfig resolved to a forbidden Docker Desktop control-plane signal." : wrongKubeconfig ? `Expected KUBECONFIG=${nativeKubeconfig}.` : !explicit.commandsOk ? "Explicit D601 kubeconfig could not read context, server, and nodes." : !requiredNodePresent ? "Explicit D601 kubeconfig did not report node d601." : "D601 native k3s guard passed with explicit KUBECONFIG.", commands: [...explicit.commands, ...bare.commands], }; } async function lockStatus(kubeconfig, guard, dumpDir, timeoutMs) { if (guard.status !== "pass") { return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, lockName, present: null }; } const env = { ...process.env, KUBECONFIG: kubeconfig }; const result = await runCaptured(["kubectl", "-n", namespace, "get", "lease", lockName, "-o", "json"], process.cwd(), dumpDir, "cd-lock", { env, timeoutMs }); const text = `${result.stdoutText}\n${result.stderrText}`; if (!result.ok && /not\s*found|notfound/iu.test(text)) { return { status: "absent", namespace, lockName, present: false, held: false, stale: false, command: commandView(result) }; } const parsed = asRecord(parseJson(result.stdoutText)); const metadata = asRecord(parsed?.metadata) || {}; const spec = asRecord(parsed?.spec) || {}; const annotations = asRecord(metadata.annotations) || {}; const annotation = (name) => stringValue(annotations[`hwlab.pikastech.local/${name}`]); const lock = parsed === null ? null : { lockBackend: "Lease", lockName: stringValue(metadata.name) || lockName, namespace: stringValue(metadata.namespace) || namespace, holderIdentity: stringValue(spec.holderIdentity), resourceVersion: stringValue(metadata.resourceVersion), ownerTaskId: annotation("ownerTaskId"), transactionId: annotation("transactionId"), phase: annotation("phase") || "unknown", promotionCommit: annotation("promotionCommit"), deployJsonHash: annotation("deployJsonHash"), targetRef: annotation("targetRef"), targetNamespace: annotation("targetNamespace"), updatedAt: annotation("updatedAt") || stringValue(spec.renewTime), ttlSeconds: Number(annotation("ttlSeconds") || spec.leaseDurationSeconds || 3600), releaseStatus: annotation("releaseStatus"), releasedAt: annotation("releasedAt"), }; const updatedAt = stringValue(lock?.updatedAt) || ""; const ttlSeconds = Number(lock?.ttlSeconds || 3600); const expiresAtMs = Number.isFinite(Date.parse(updatedAt)) ? Date.parse(updatedAt) + ttlSeconds * 1000 : 0; const retryAfterSeconds = Math.max(0, Math.ceil((expiresAtMs - Date.now()) / 1000)); return { status: result.ok && lock !== null ? "observed" : "blocked", present: result.ok && lock !== null, ...(lock || {}), held: lock !== null && lock.phase !== "released" && retryAfterSeconds > 0, stale: lock !== null && lock.phase !== "released" && retryAfterSeconds <= 0, retryAfterSeconds, expiresAt: expiresAtMs > 0 ? new Date(expiresAtMs).toISOString() : null, command: commandView(result), }; } function parseSecretDescribeKeys(stdout) { const keys = new Set(); let inData = false; for (const rawLine of String(stdout || "").split(/\r?\n/u)) { const line = rawLine.trim(); if (/^Data\b/iu.test(line)) { inData = true; continue; } if (!inData || line === "" || /^=+$/u.test(line)) continue; const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu); if (match) keys.add(match[1]); } return [...keys].sort(); } async function secretPreflight(kubeconfig, guard, dumpDir, timeoutMs) { if (guard.status !== "pass") { return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, mutation: false, safety: { secretValuesRead: false, secretValuesPrinted: false, secretKeyNamesOnly: true }, secretRefs: requiredSecretRefs.map((ref) => ({ ...ref, status: "not_checked" })), blockers: [], }; } const env = { ...process.env, KUBECONFIG: kubeconfig }; const observations = []; for (const ref of requiredSecretRefs) { const idBase = `secretref-${ref.secretName}-${ref.secretKey}`.replace(/[^A-Za-z0-9_.-]/gu, "-"); const exists = await runCaptured(["kubectl", "-n", namespace, "get", "secret", ref.secretName, "-o", "name"], process.cwd(), dumpDir, `${idBase}-exists`, { env, timeoutMs }); if (!exists.ok) { observations.push({ ...ref, status: "missing-secret", exists: false, keyPresent: false, command: commandView(exists) }); continue; } const describe = await runCaptured(["kubectl", "-n", namespace, "describe", "secret", ref.secretName], process.cwd(), dumpDir, `${idBase}-describe`, { env, timeoutMs }); const keysObserved = parseSecretDescribeKeys(describe.stdoutText); const keyPresent = keysObserved.includes(ref.secretKey); observations.push({ ...ref, status: keyPresent ? "present" : describe.ok ? "missing-key" : "key-observation-blocked", exists: true, keyPresent, keysObserved, command: commandView(describe), }); } const blockers = observations .filter((observation) => observation.status !== "present") .map((observation) => ({ scope: `secretref:${observation.secretName}/${observation.secretKey}`, summary: observation.status === "missing-secret" ? `Required Secret ${observation.secretName} is missing in ${namespace}.` : `Required SecretRef ${observation.secretName}/${observation.secretKey} is not present as key metadata in ${namespace}.`, impact: `${observation.consumers.join(", ")} would fail after a DEV CD apply or runtime preflight Job.`, runbook: `Create or repair Secret ${observation.secretName} with key ${observation.secretKey} in namespace ${namespace}; verify key presence only and do not print the Secret value.`, })); return { status: blockers.length === 0 ? "pass" : "blocked", namespace, mutation: false, safety: { secretValuesRead: false, secretValuesPrinted: false, secretKeyNamesOnly: true }, requiredSecretRefs: requiredSecretRefs.map((ref) => `${ref.secretName}/${ref.secretKey}`), secretRefs: observations, blockers, }; } async function registryAudit(kubeconfig, guard, dumpDir, timeoutMs) { const result = await runCaptured(["curl", "-fsS", "--max-time", "5", "http://127.0.0.1:5000/v2/"], process.cwd(), dumpDir, "registry-v2", { timeoutMs: Math.min(timeoutMs, 8000) }); const reachable = result.ok || /401|unauthorized/iu.test(`${result.stdoutText}\n${result.stderrText}`); let k3sPullAccess = { status: "skipped", reason: "d601-native-k3s-guard-not-pass", pullFailureCount: null, pulledRegistryImageCount: null }; if (guard.status === "pass") { const env = { ...process.env, KUBECONFIG: kubeconfig }; const pods = await runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "registry-k3s-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }); const parsed = asRecord(parseJson(pods.stdoutText)); const items = Array.isArray(parsed?.items) ? parsed.items : []; let pullFailureCount = 0; let pulledRegistryImageCount = 0; for (const item of items) { const statuses = Array.isArray(item?.status?.containerStatuses) ? item.status.containerStatuses : []; for (const status of statuses) { const image = String(status?.image || ""); const imageID = String(status?.imageID || ""); const waiting = String(status?.state?.waiting?.reason || ""); if (/ErrImagePull|ImagePullBackOff|InvalidImageName/u.test(waiting)) pullFailureCount += 1; if (image.includes("127.0.0.1:5000/") && imageID.includes("sha256:")) pulledRegistryImageCount += 1; } } k3sPullAccess = { status: !pods.ok ? "unavailable" : pullFailureCount > 0 ? "blocked" : pulledRegistryImageCount > 0 ? "pass" : "degraded", pullFailureCount, pulledRegistryImageCount, command: commandView(pods), }; } return { status: reachable && !["blocked", "unavailable"].includes(k3sPullAccess.status) ? "pass" : reachable ? "degraded" : "blocked", endpoint: "http://127.0.0.1:5000/v2/", processHttpAccess: { status: reachable ? "pass" : "blocked", reachable, exitCode: result.exitCode, command: commandView(result), }, k3sPullAccess, }; } function deploymentCondition(conditions, type) { return (Array.isArray(conditions) ? conditions : []).find((condition) => condition?.type === type) || null; } function envValue(container, name) { const item = (Array.isArray(container?.env) ? container.env : []).find((entry) => entry?.name === name); return item?.value ?? null; } async function workloadAudit(kubeconfig, guard, desired, dumpDir, timeoutMs) { if (guard.status !== "pass") { return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, deployments: [], currentImageConvergence: { status: "unavailable", unavailableReason: "d601-native-k3s-guard-not-pass" } }; } const env = { ...process.env, KUBECONFIG: kubeconfig }; const [deployments, pods, jobs] = await Promise.all([ runCaptured(["kubectl", "-n", namespace, "get", "deployments", "-o", "json"], process.cwd(), dumpDir, "workload-deployments", { env, timeoutMs: Math.min(timeoutMs, 10000) }), runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "workload-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }), runCaptured(["kubectl", "-n", namespace, "get", "jobs", "-o", "json"], process.cwd(), dumpDir, "runtime-jobs", { env, timeoutMs: Math.min(timeoutMs, 10000) }), ]); const desiredById = new Map((desired.services || []).map((service) => [service.serviceId, service])); const deploymentItems = Array.isArray(parseJson(deployments.stdoutText)?.items) ? parseJson(deployments.stdoutText).items : []; const summaries = deploymentItems.map((deployment) => { const metadata = asRecord(deployment.metadata) || {}; const spec = asRecord(deployment.spec) || {}; const status = asRecord(deployment.status) || {}; const template = asRecord(spec.template) || {}; const podSpec = asRecord(template.spec) || {}; const containers = Array.isArray(podSpec.containers) ? podSpec.containers : []; const container = containers[0] || {}; const labels = asRecord(metadata.labels) || {}; const serviceId = stringValue(labels["hwlab.pikastech.local/service-id"]) || stringValue(labels["app.kubernetes.io/name"]) || stringValue(metadata.name); const desiredService = desiredById.get(serviceId); const image = stringValue(container.image); const desiredImage = desiredService?.image || null; const available = Number(status.availableReplicas || 0); const desiredReplicas = Number(spec.replicas || 0); const updated = Number(status.updatedReplicas || 0); const unavailable = Number(status.unavailableReplicas || 0); const progressing = deploymentCondition(status.conditions, "Progressing"); const availableCondition = deploymentCondition(status.conditions, "Available"); return { name: stringValue(metadata.name), serviceId, image, imageTag: imageTag(image), desiredImage, imageMatchesDesired: desiredImage ? image === desiredImage : null, revision: stringValue(asRecord(metadata.annotations)?.["deployment.kubernetes.io/revision"]), replicas: desiredReplicas, availableReplicas: available, updatedReplicas: updated, unavailableReplicas: unavailable, rolloutStatus: desiredReplicas === 0 || (available >= desiredReplicas && updated >= desiredReplicas && unavailable === 0) ? "healthy" : "unhealthy", conditions: { available: availableCondition ? { status: availableCondition.status ?? null, reason: availableCondition.reason ?? null } : null, progressing: progressing ? { status: progressing.status ?? null, reason: progressing.reason ?? null } : null, }, commitEnv: envValue(container, "HWLAB_COMMIT_ID"), }; }); const podItems = Array.isArray(parseJson(pods.stdoutText)?.items) ? parseJson(pods.stdoutText).items : []; const waiting = []; for (const pod of podItems) { const podName = stringValue(pod?.metadata?.name); for (const status of Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []) { const reason = stringValue(status?.state?.waiting?.reason); if (reason) waiting.push({ pod: podName, container: status.name ?? null, reason, image: status.image ?? null }); } } const jobItems = Array.isArray(parseJson(jobs.stdoutText)?.items) ? parseJson(jobs.stdoutText).items : []; const runtimeJobs = jobItems .filter((job) => /hwlab-runtime|runtime-db|runtime/i.test(String(job?.metadata?.name || ""))) .map((job) => ({ name: stringValue(job?.metadata?.name), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), active: Number(job?.status?.active || 0), completionTime: stringValue(job?.status?.completionTime), status: Number(job?.status?.failed || 0) > 0 ? "blocked" : Number(job?.status?.active || 0) > 0 ? "running" : Number(job?.status?.succeeded || 0) > 0 ? "pass" : "unknown", })) .slice(0, 12); const imageMismatches = summaries.filter((deployment) => deployment.imageMatchesDesired === false); const unhealthy = summaries.filter((deployment) => deployment.rolloutStatus === "unhealthy"); return { status: !deployments.ok ? "unavailable" : unhealthy.length > 0 ? "unhealthy" : imageMismatches.length > 0 ? "mismatch" : "healthy", namespace, deploymentCount: summaries.length, deployments: summaries.slice(0, 30), currentImageConvergence: { status: imageMismatches.length === 0 ? "pass" : "mismatch", mismatches: imageMismatches.map((deployment) => ({ serviceId: deployment.serviceId, image: deployment.image, desiredImage: deployment.desiredImage })).slice(0, 12), }, podWaiting: waiting.slice(0, 12), runtimeJobs, commands: [deployments, pods, jobs].map(commandView), }; } function compactHealth(body, expectedServiceId, expectedCommit) { const json = asRecord(body); const commit = stringValue(asRecord(json?.commit)?.id) || stringValue(json?.commitId) || stringValue(json?.revision) || stringValue(asRecord(json?.image)?.tag); const runtime = asRecord(json?.runtime); const db = asRecord(json?.db); const blockerCodes = Array.isArray(json?.blockerCodes) ? json.blockerCodes.map(String).slice(0, 12) : []; return { serviceId: stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id), environment: stringValue(json?.environment), status: stringValue(json?.status), ready: typeof json?.ready === "boolean" ? json.ready : null, observedCommit: commit, commitMatches: expectedCommit ? commitMatches(commit, expectedCommit) : null, serviceMatches: expectedServiceId ? (stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id)) === expectedServiceId : null, image: typeof json?.image === "string" ? json.image : stringValue(asRecord(json?.image)?.reference), blockerCodes, db: db === null ? null : { ready: db.ready ?? null, connected: db.connected ?? null, liveDbEvidence: db.liveDbEvidence ?? null, runtimeReadiness: asRecord(db.runtimeReadiness) ? { status: stringValue(db.runtimeReadiness.status), ready: db.runtimeReadiness.ready ?? null, blocker: stringValue(db.runtimeReadiness.blocker), } : null, }, runtime: runtime === null ? null : { status: stringValue(runtime.status), ready: runtime.ready ?? null, durable: runtime.durable ?? null, durableRequested: runtime.durableRequested ?? null, liveRuntimeEvidence: runtime.liveRuntimeEvidence ?? null, blocker: stringValue(runtime.blocker), adapter: stringValue(runtime.adapter), }, }; } async function httpHealthAudit(desired, dumpDir, timeoutMs) { const endpoints = [ { id: "cloud-web-16666", url: desired.deployJson.publicEndpoints.frontend || "http://74.48.78.17:16666", expectedServiceId: "hwlab-cloud-web" }, { id: "cloud-api-16667", url: desired.deployJson.publicEndpoints.api || desired.deployJson.endpoint || "http://74.48.78.17:16667", expectedServiceId: "hwlab-cloud-api" }, ]; const expectedCommit = desired.targetCommit || desired.deployJson.commitId || null; const observations = []; for (const endpoint of endpoints) { const result = await runCaptured(["curl", "-fsS", "--max-time", "8", `${endpoint.url.replace(/\/+$/u, "")}/health/live`], process.cwd(), dumpDir, `public-health-${endpoint.id}`, { timeoutMs: Math.min(timeoutMs, 10000) }); const parsed = parseJson(result.stdoutText); const health = compactHealth(parsed, endpoint.expectedServiceId, expectedCommit); const reachable = result.ok && parsed !== null; observations.push({ id: endpoint.id, url: endpoint.url, status: reachable && health.serviceMatches !== false && health.commitMatches !== false ? "pass" : "blocked", reachable, expectedServiceId: endpoint.expectedServiceId, expectedCommit, health, command: commandView(result), }); } return { status: observations.every((item) => item.status === "pass") ? "pass" : "blocked", expectedCommit, summary: { checked: observations.length, reachable: observations.filter((item) => item.reachable).length, commitMatches: observations.filter((item) => item.health.commitMatches === true).length, readinessPass: observations.filter((item) => item.health.ready !== false && !["blocked", "failed"].includes(String(item.health.status || ""))).length, ports: [16666, 16667], }, endpoints: observations, }; } function runtimeDurabilityAudit(desired, publicHealth, secretRefs, workload) { const api = publicHealth?.endpoints?.find((endpoint) => endpoint.id === "cloud-api-16667")?.health || null; const cloudApiEnv = asRecord(desired.serviceEnv?.cloudApi) || {}; const configuredDurable = cloudApiEnv.HWLAB_CLOUD_RUNTIME_DURABLE === "true"; const configuredAdapter = stringValue(cloudApiEnv.HWLAB_CLOUD_RUNTIME_ADAPTER); const dbSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db" && ref.secretKey === "database-url"); const adminSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db-admin" && ref.secretKey === "admin-url"); const runtime = asRecord(api?.runtime); const dbRuntimeReadiness = asRecord(api?.db?.runtimeReadiness); const liveReady = runtime?.durable === true && runtime?.ready !== false && runtime?.liveRuntimeEvidence === true && !["blocked", "degraded", "failed"].includes(String(runtime?.status || "")); const status = liveReady ? "pass" : api ? "risk" : configuredDurable && configuredAdapter === "postgres" && dbSecret?.keyPresent === true && adminSecret?.keyPresent === true ? "unavailable" : "risk"; return { status, configured: { adapter: configuredAdapter, durable: configuredDurable, dbSecretPresent: dbSecret?.keyPresent === true, adminSecretPresent: adminSecret?.keyPresent === true, }, live: api ? { db: api.db, runtime: api.runtime, dbRuntimeReadiness, } : null, workloadEvidence: { cloudApiDeployment: (workload?.deployments || []).find((deployment) => deployment.serviceId === "hwlab-cloud-api") || null, }, unavailableReason: api ? null : "public-health-cloud-api-unavailable", risk: status === "pass" ? null : "db-runtime-durability-risk", }; } function blocker(scope, summary, details = {}) { return { scope, summary, ...details }; } function classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }) { const blockers = []; if (guard?.status !== "pass") blockers.push(blocker("control-plane-unavailable", guard?.summary || "D601 native k3s guard did not pass.")); if (guard?.refusalSignals?.length > 0) blockers.push(blocker("docker-desktop-context-risk", "Explicit D601 kubeconfig resolved to forbidden Docker Desktop control-plane signal.", { refusalSignals: guard.refusalSignals })); if (guard?.secondControlPlaneRisk) blockers.push(blocker("second-control-plane-risk", "Bare kubectl can observe hwlab-dev through a different control plane; audit is read-only and will not release or mutate locks.")); if (repo.status !== "selected") blockers.push(blocker("workspace-unavailable", "No eligible HWLAB CD repo was selected.")); for (const item of git?.blockers || []) { const scope = item.scope === "hwlab-git-clean" ? "dirty-worktree" : item.scope?.startsWith("hwlab-git") || item.scope?.includes("permission") ? "workspace-unavailable" : item.scope; blockers.push(blocker(scope, item.summary)); } for (const item of secretRefs?.blockers || []) blockers.push(blocker("secret-missing", item.summary, { secretRef: item.scope })); if (registry?.status === "blocked" || registry?.status === "degraded") blockers.push(blocker("registry-unavailable", "Registry reachability or k3s pull evidence is not healthy.", { status: registry.status })); if (lock?.held === true) blockers.push(blocker("lease-held", `HWLAB DEV CD Lease is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.`, { phase: lock.phase, retryAfterSeconds: lock.retryAfterSeconds })); if (lock?.stale === true) blockers.push(blocker("lease-stale-candidate", "HWLAB DEV CD Lease appears stale; audit is read-only and did not release or break it.", { phase: lock.phase })); if (desired?.status === "missing") blockers.push(blocker("artifact-missing", "deploy/deploy.json or artifact catalog/report is missing or incomplete.", { imageConvergence: desired.imageConvergence })); if (desired?.status === "mismatch" || workload?.currentImageConvergence?.status === "mismatch") blockers.push(blocker("artifact-mismatch", "deploy/deploy.json, artifact catalog, or current workload images are not converged.", { desired: desired?.status, currentImageConvergence: workload?.currentImageConvergence })); if ((workload?.runtimeJobs || []).some((job) => job.status === "blocked" || job.status === "running")) blockers.push(blocker("runtime-job-blocked", "Runtime DB maintenance Job is blocked or still running.", { runtimeJobs: workload.runtimeJobs })); if (workload?.status === "unhealthy") blockers.push(blocker("rollout-unhealthy", "One or more hwlab-dev Deployments are not fully available.", { deployments: (workload.deployments || []).filter((item) => item.rolloutStatus === "unhealthy").slice(0, 8) })); if (publicHealth?.status === "blocked") blockers.push(blocker("public-tunnel-unhealthy", "16666/16667 public health did not pass read-only commit/readiness checks.", { summary: publicHealth.summary })); if (durability?.status === "risk") blockers.push(blocker("db-runtime-durability-risk", "DB/runtime durability is not proven by live cloud-api health.", { live: durability.live, configured: durability.configured })); if (controlled?.commandOk === false || controlled?.status === "blocked") blockers.push(blocker("runtime-job-blocked", "HWLAB repo-owned dev-cd-apply status reported blocked or failed.", { status: controlled?.status })); return blockers; } function auditSummary({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled, dumpDir }) { const blockers = classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }); const blockerTypes = [...new Set(blockers.map((item) => item.scope))]; return { ok: blockers.length === 0, status: blockers.length === 0 ? "pass" : guard?.refusal ? "refused" : "blocked", env: "dev", namespace, nodeGuard: { status: guard?.status ?? "unavailable", nodeName: guard?.requiredNodeName ?? requiredNodeName, nodeNames: guard?.nodeNames ?? [], requiredNodePresent: guard?.requiredNodePresent ?? false, currentContext: guard?.currentContext ?? null, apiServer: guard?.apiServer ?? null, kubeconfig: guard?.kubeconfig ?? nativeKubeconfig, refusalSignals: guard?.refusalSignals ?? [], secondControlPlaneRisk: guard?.secondControlPlaneRisk ?? false, defaultKubectlDiagnostic: guard?.defaultKubectlDiagnostic ?? null, }, secrets: { status: secretRefs?.status ?? "unavailable", valuesRead: false, valuesPrinted: false, secretRefs: (secretRefs?.secretRefs || []).map((ref) => ({ secretName: ref.secretName, secretKey: ref.secretKey, exists: ref.exists === true, keyPresent: ref.keyPresent === true, status: ref.status, })), unavailableReason: secretRefs?.reason ?? null, }, registry: { status: registry?.status ?? "unavailable", endpoint: registry?.endpoint ?? "http://127.0.0.1:5000/v2/", processHttpAccess: registry?.processHttpAccess ? { status: registry.processHttpAccess.status, reachable: registry.processHttpAccess.reachable } : null, k3sPullAccess: registry?.k3sPullAccess ? { status: registry.k3sPullAccess.status, pullFailureCount: registry.k3sPullAccess.pullFailureCount, pulledRegistryImageCount: registry.k3sPullAccess.pulledRegistryImageCount, } : null, }, lease: { status: lock?.status ?? "unavailable", phase: lock?.phase ?? null, holder: lock?.ownerTaskId || lock?.holderIdentity || null, ageSeconds: lock?.updatedAt && Number.isFinite(Date.parse(lock.updatedAt)) ? Math.max(0, Math.floor((Date.now() - Date.parse(lock.updatedAt)) / 1000)) : null, retryAfterSeconds: lock?.retryAfterSeconds ?? null, staleClassification: lock?.stale === true ? "lease-stale-candidate" : lock?.held === true ? "lease-held" : "not-held", lockName: lock?.lockName ?? lockName, }, desiredState: { status: desired?.status ?? "unavailable", targetCommit: desired?.targetCommit ?? null, deployJson: desired?.deployJson ?? null, artifactCatalog: desired?.artifactCatalog ?? null, artifactReport: desired?.artifactReport ?? null, imageConvergence: desired?.imageConvergence ?? null, }, workload: { status: workload?.status ?? "unavailable", currentImageConvergence: workload?.currentImageConvergence ?? null, deploymentCount: workload?.deploymentCount ?? 0, deployments: (workload?.deployments || []).map((deployment) => ({ name: deployment.name, serviceId: deployment.serviceId, image: deployment.image, imageTag: deployment.imageTag, desiredImage: deployment.desiredImage, revision: deployment.revision, replicas: deployment.replicas, availableReplicas: deployment.availableReplicas, rolloutStatus: deployment.rolloutStatus, commitEnv: deployment.commitEnv, })), podWaiting: workload?.podWaiting ?? [], runtimeJobs: workload?.runtimeJobs ?? [], }, publicHealth: { status: publicHealth?.status ?? "unavailable", expectedCommit: publicHealth?.expectedCommit ?? null, summary: publicHealth?.summary ?? { checked: 0 }, endpoints: (publicHealth?.endpoints || []).map((endpoint) => ({ id: endpoint.id, url: endpoint.url, status: endpoint.status, reachable: endpoint.reachable, expectedServiceId: endpoint.expectedServiceId, expectedCommit: endpoint.expectedCommit, health: endpoint.health, })), }, durability: { status: durability?.status ?? "unavailable", configured: durability?.configured ?? null, live: durability?.live ?? null, unavailableReason: durability?.unavailableReason ?? null, }, controlledDevCd: { status: controlled?.status ?? "unavailable", commandOk: controlled?.commandOk ?? null, controlledEntrypoint: "scripts/dev-cd-apply.mjs", target: controlled?.target ?? null, blockers: controlled?.blockers ?? [], }, blockers, blockerTypes, dumpPath: dumpDir, safety: { mutation: false, kubectlApplyExecuted: false, rolloutExecuted: false, liveVerifyExecuted: false, cdLockMutated: false, secretValuesRead: false, secretValuesPrinted: false, reportsWritten: false, repoReportsWritten: false, }, }; } function summarizeControlled(parsed, command) { const record = asRecord(parsed); const target = asRecord(record?.target); const desiredStateCheck = asRecord(target?.desiredStateCheck); const artifactBoundary = asRecord(target?.artifactBoundary); return { status: stringValue(record?.status) || (command.ok ? "unknown" : "blocked"), commandOk: command.ok, mode: stringValue(record?.mode), mutationAttempted: record?.mutationAttempted ?? false, prodTouched: record?.prodTouched ?? false, controlledEntrypoint: "scripts/dev-cd-apply.mjs", target: target === null ? null : { ref: stringValue(target.ref), promotionCommit: stringValue(target.promotionCommit), shortCommitId: stringValue(target.shortCommitId), promotionSource: stringValue(target.promotionSource), publishRequired: target.publishRequired ?? null, headCommitId: stringValue(target.headCommitId), headMatchesTarget: target.headMatchesTarget ?? null, desiredStateCheck: desiredStateCheck === null ? null : { status: stringValue(desiredStateCheck.status), summary: desiredStateCheck.summary ?? null, diagnostics: Array.isArray(desiredStateCheck.diagnostics) ? desiredStateCheck.diagnostics.slice(0, 5) : [], }, artifactBoundary: artifactBoundary === null ? null : { status: stringValue(artifactBoundary.status), desiredState: artifactBoundary.desiredState ?? null, }, namespace: stringValue(target.namespace) || namespace, }, deployJson: asRecord(record?.deployJson), artifactCatalog: asRecord(record?.artifactCatalog), artifactReport: asRecord(record?.artifactReport), lock: asRecord(record?.lock), liveDelta: asRecord(record?.liveDelta), blockers: Array.isArray(record?.blockers) ? record.blockers.slice(0, 12) : [], nextActions: Array.isArray(record?.nextActions) ? record.nextActions.slice(0, 12) : [], parsed: record !== null, command: commandView(command), }; } async function runDevCdApply(repoPath, kubeconfig, action, dumpDir, timeoutMs) { const modeArg = action === "status" ? "--status" : "--dry-run"; const command = ["node", "scripts/dev-cd-apply.mjs", modeArg, "--kubeconfig", kubeconfig, "--skip-live-verify"]; const result = await runCaptured(command, repoPath, dumpDir, `controlled-dev-cd-apply-${action === "status" ? "status" : "dry-run"}`, { env: { ...process.env, KUBECONFIG: kubeconfig }, timeoutMs, }); return summarizeControlled(parseJson(result.stdoutText), result); } function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action }) { const blockers = []; if (repo.status !== "selected") { blockers.push({ scope: "hwlab-repo", summary: repo.rejected ? "Rejected runner history HWLAB directory." : "No eligible HWLAB CD repo with scripts/dev-cd-apply.mjs and deploy/deploy.json was found." }); } if (repo.cacheRefreshStatus === "blocked") { blockers.push({ scope: "hwlab-cache-refresh", summary: "Dedicated HWLAB git cache could not refresh from GitHub through the provider egress proxy; using the last local cache state is not a release-truth PASS.", remoteMainCommit: repo.remoteMainCommit || null, cacheMainCommit: repo.cacheMainCommit || null, }); } if (git?.blockers) blockers.push(...git.blockers); if (guard) { if (guard.refusal) blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary, refusal: true }); else if (guard.status !== "pass") blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary }); if ((action === "preflight" || action === "audit" || action === "apply") && guard.secondControlPlaneRisk) { blockers.push({ scope: "second-hwlab-dev-control-plane", summary: "Bare kubectl can observe hwlab-dev through a different control plane; refusing write-path planning." }); } } if ((action === "preflight" || action === "apply") && lock?.held === true) { blockers.push({ scope: "cd-lock", summary: `HWLAB DEV CD lock is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.` }); } if (secretRefs?.blockers) blockers.push(...secretRefs.blockers); if (controlled) { if (controlled.commandOk === false) blockers.push({ scope: "controlled-dev-cd", summary: "HWLAB scripts/dev-cd-apply.mjs did not complete successfully." }); if (controlled.status === "blocked") blockers.push({ scope: "controlled-dev-cd-status", summary: "HWLAB scripts/dev-cd-apply.mjs reported blocked status." }); } return blockers; } function nextSafeCommand(action, blockers) { if (blockers.length > 0) return action === "audit" ? "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd audit --env dev" : "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev"; if (action === "audit") return "audit is read-only; host commander may compare blockerTypes before deciding whether the unique CD runner should continue"; if (action === "status") return "bun scripts/cli.ts hwlab cd apply --env dev --dry-run"; return "host commander or the unique CD runner may decide whether to run node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report on D601; this wrapper did not apply or rollout"; } function compactBlockers(blockers, limit = 12) { return (Array.isArray(blockers) ? blockers : []).slice(0, limit).map((item) => ({ scope: item?.scope ?? item?.blockerScope ?? null, type: item?.type ?? null, status: item?.status ?? null, summary: item?.summary ?? item?.reason ?? null, })); } function compactGit(git) { if (!git) return null; return { status: git.status, clean: git.clean, branch: git.branch, onMain: git.onMain, remoteMatches: git.remoteMatches, headMatchesOriginMain: git.headMatchesOriginMain, dirtyCount: git.dirtyCount, blockers: compactBlockers(git.blockers, 8), }; } function compactNodeGuard(guard) { if (!guard) return null; return { status: guard.status, refusal: guard.refusal, kubeconfig: guard.kubeconfig, currentContext: guard.currentContext, apiServer: guard.apiServer, nodeNames: guard.nodeNames, requiredNodePresent: guard.requiredNodePresent, refusalSignals: guard.refusalSignals, secondControlPlaneRisk: guard.secondControlPlaneRisk, defaultKubectlStatus: guard.defaultKubectlDiagnostic?.status ?? null, summary: guard.summary, }; } function compactSecretPreflight(secretRefs) { if (!secretRefs) return null; return { status: secretRefs.status, valuesRead: secretRefs.safety?.secretValuesRead === true, valuesPrinted: secretRefs.safety?.secretValuesPrinted === true, secretRefs: (secretRefs.secretRefs || []).map((ref) => ({ secretName: ref.secretName, secretKey: ref.secretKey, exists: ref.exists === true, keyPresent: ref.keyPresent === true, status: ref.status, })).slice(0, 8), blockers: compactBlockers(secretRefs.blockers, 8), }; } function compactDesiredState(desired) { if (!desired) return null; return { status: desired.status, targetCommit: desired.targetCommit, deployJson: desired.deployJson ? { hash: desired.deployJson.hash, commitId: desired.deployJson.commitId, environment: desired.deployJson.environment, namespace: desired.deployJson.namespace, serviceCount: desired.deployJson.serviceCount, } : null, artifactCatalog: desired.artifactCatalog ? { hash: desired.artifactCatalog.hash, commitId: desired.artifactCatalog.commitId, artifactState: desired.artifactCatalog.artifactState, ciPublished: desired.artifactCatalog.ciPublished, registryVerified: desired.artifactCatalog.registryVerified, digestCounts: desired.artifactCatalog.digestCounts, } : null, imageConvergence: desired.imageConvergence ? { status: desired.imageConvergence.status, missing: (desired.imageConvergence.missing || []).slice(0, 5), mismatches: (desired.imageConvergence.mismatches || []).slice(0, 5), } : null, }; } function compactControlled(controlled) { if (!controlled) return null; return { status: controlled.status, commandOk: controlled.commandOk, mode: controlled.mode, mutationAttempted: controlled.mutationAttempted, prodTouched: controlled.prodTouched, target: controlled.target ? { ref: controlled.target.ref, promotionCommit: controlled.target.promotionCommit, shortCommitId: controlled.target.shortCommitId, promotionSource: controlled.target.promotionSource, publishRequired: controlled.target.publishRequired, headMatchesTarget: controlled.target.headMatchesTarget, desiredStateStatus: controlled.target.desiredStateCheck?.status ?? null, artifactBoundaryStatus: controlled.target.artifactBoundary?.status ?? null, namespace: controlled.target.namespace, } : null, blockers: compactBlockers(controlled.blockers, 8), }; } function compactAudit(audit) { if (!audit) return null; return { ok: audit.ok, status: audit.status, env: audit.env, namespace: audit.namespace, nodeGuard: audit.nodeGuard, secrets: audit.secrets ? { status: audit.secrets.status, valuesRead: audit.secrets.valuesRead, valuesPrinted: audit.secrets.valuesPrinted, } : null, registry: audit.registry ? { status: audit.registry.status, processHttpAccess: audit.registry.processHttpAccess, k3sPullAccess: audit.registry.k3sPullAccess, } : null, lease: audit.lease, desiredState: audit.desiredState ? { status: audit.desiredState.status, targetCommit: audit.desiredState.targetCommit, imageConvergence: audit.desiredState.imageConvergence, } : null, workload: audit.workload ? { status: audit.workload.status, deploymentCount: audit.workload.deploymentCount, currentImageConvergence: audit.workload.currentImageConvergence, podWaiting: (audit.workload.podWaiting || []).slice(0, 5), runtimeJobs: (audit.workload.runtimeJobs || []).slice(0, 5), } : null, publicHealth: audit.publicHealth ? { status: audit.publicHealth.status, expectedCommit: audit.publicHealth.expectedCommit, summary: audit.publicHealth.summary, } : null, durability: audit.durability ? { status: audit.durability.status, unavailableReason: audit.durability.unavailableReason, } : null, controlledDevCd: compactControlled(audit.controlledDevCd), blockerTypes: audit.blockerTypes || [], blockers: compactBlockers(audit.blockers, 12), safety: audit.safety, }; } function compactForTransport(payload, fullResultPath) { return { ok: payload.ok, env: payload.env, environment: payload.environment, status: payload.status, error: payload.error ?? null, action: payload.action ?? null, dryRun: payload.dryRun, mutation: payload.mutation, remoteHost: payload.remoteHost, workspace: payload.workspace, kubeconfig: payload.kubeconfig, dumpPath: payload.dumpPath, remoteFullResultPath: fullResultPath, repo: payload.repo ? { status: payload.repo.status, path: payload.repo.path, source: payload.repo.source, rejected: payload.repo.rejected, ephemeral: payload.repo.ephemeral === true, cacheRefreshStatus: payload.repo.cacheRefreshStatus || null, } : null, worktreeGuard: compactGit(payload.worktreeGuard), nodeGuard: compactNodeGuard(payload.nodeGuard), secretPreflight: compactSecretPreflight(payload.secretPreflight), lockState: payload.lockState ? { status: payload.lockState.status, phase: payload.lockState.phase, holder: payload.lockState.ownerTaskId || payload.lockState.holderIdentity || null, held: payload.lockState.held, stale: payload.lockState.stale, retryAfterSeconds: payload.lockState.retryAfterSeconds, lockName: payload.lockState.lockName, } : null, desiredState: payload.desiredState ? { deployJson: payload.desiredState.deployJson, artifactCatalog: payload.desiredState.artifactCatalog, artifactReport: payload.desiredState.artifactReport, } : null, target: payload.target, promotion: payload.promotion ? { source: payload.promotion.source, deployJson: payload.promotion.deployJson, artifactCatalog: payload.promotion.artifactCatalog, artifactReport: payload.promotion.artifactReport, } : null, controlledDevCd: compactControlled(payload.controlledDevCd), audit: compactAudit(payload.audit), blockers: compactBlockers(payload.blockers, 12), blockerTypes: payload.blockerTypes || [...new Set(compactBlockers(payload.blockers, 12).map((item) => item.scope).filter(Boolean))], nextSafeCommand: payload.nextSafeCommand, safety: payload.safety, }; } function emitResult(payload, dumpDir, compactStdout = false) { const fullResultPath = path.join(dumpDir, "result.full.json"); const compactResultPath = path.join(dumpDir, "result.compact.json"); fs.writeFileSync(fullResultPath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); if (!compactStdout) { fs.writeFileSync(compactResultPath, `${JSON.stringify(compactForTransport(payload, fullResultPath), null, 2)}\n`, { mode: 0o600 }); console.log(JSON.stringify(payload, null, 2)); return; } let compact = compactForTransport(payload, fullResultPath); let text = JSON.stringify(compact, null, 2); if (Buffer.byteLength(text, "utf8") > 3600) { compact = { ok: compact.ok, env: compact.env, status: compact.status, error: compact.error, action: compact.action, dryRun: compact.dryRun, mutation: compact.mutation, workspace: compact.workspace, dumpPath: compact.dumpPath, remoteFullResultPath: compact.remoteFullResultPath, repo: compact.repo, worktreeGuard: compact.worktreeGuard, nodeGuard: compact.nodeGuard, lockState: compact.lockState, target: compact.target ? { ref: compact.target.ref, promotionCommit: compact.target.promotionCommit, shortCommitId: compact.target.shortCommitId, promotionSource: compact.target.promotionSource, publishRequired: compact.target.publishRequired, } : null, controlledDevCd: compact.controlledDevCd, audit: compact.audit ? { status: compact.audit.status, blockerTypes: compact.audit.blockerTypes, blockers: compact.audit.blockers, publicHealth: compact.audit.publicHealth, workload: compact.audit.workload, durability: compact.audit.durability, } : null, blockers: compact.blockers, blockerTypes: compact.blockerTypes, nextSafeCommand: compact.nextSafeCommand, safety: compact.safety, outputCompacted: "provider-host-ssh-stdout-limit", }; text = JSON.stringify(compact, null, 2); } if (Buffer.byteLength(text, "utf8") > 2400) { compact = { ok: payload.ok, env: payload.env, status: payload.status, error: payload.error ?? null, action: payload.action ?? null, dryRun: payload.dryRun, mutation: payload.mutation, workspace: payload.workspace, dumpPath: payload.dumpPath, remoteFullResultPath: fullResultPath, repo: payload.repo ? { status: payload.repo.status, path: payload.repo.path, source: payload.repo.source, rejected: payload.repo.rejected, ephemeral: payload.repo.ephemeral === true, cacheRefreshStatus: payload.repo.cacheRefreshStatus || null, } : null, worktreeStatus: payload.worktreeGuard?.status ?? null, nodeGuardStatus: payload.nodeGuard?.status ?? payload.audit?.nodeGuard?.status ?? null, lockState: payload.lockState ? { status: payload.lockState.status, phase: payload.lockState.phase, held: payload.lockState.held, holder: payload.lockState.ownerTaskId || payload.lockState.holderIdentity || null, stale: payload.lockState.stale, } : null, target: payload.target ? { ref: payload.target.ref, promotionCommit: payload.target.promotionCommit, shortCommitId: payload.target.shortCommitId, promotionSource: payload.target.promotionSource, } : payload.audit?.desiredState ? { targetCommit: payload.audit.desiredState.targetCommit, } : null, controlledDevCdStatus: payload.controlledDevCd?.status ?? payload.audit?.controlledDevCd?.status ?? null, auditStatus: payload.audit?.status ?? null, blockerTypes: payload.blockerTypes || payload.audit?.blockerTypes || [...new Set(compactBlockers(payload.blockers, 8).map((item) => item.scope).filter(Boolean))], blockers: compactBlockers(payload.blockers || payload.audit?.blockers, 6), nextSafeCommand: payload.nextSafeCommand, outputCompacted: "provider-host-ssh-stdout-limit", }; text = JSON.stringify(compact); } if (Buffer.byteLength(text, "utf8") > 520) { const blockerTypes = payload.blockerTypes || payload.audit?.blockerTypes || [...new Set(compactBlockers(payload.blockers || payload.audit?.blockers, 8).map((item) => item.scope).filter(Boolean))]; compact = { ok: payload.ok, env: payload.env, status: payload.status, error: payload.error ?? null, action: payload.action ?? null, remoteFullResultPath: fullResultPath, repoPath: payload.repo?.path ?? payload.workspace?.path ?? null, repoEphemeral: payload.repo?.ephemeral === true, cacheRefreshStatus: payload.repo?.cacheRefreshStatus ?? null, worktreeStatus: payload.worktreeGuard?.status ?? null, nodeGuardStatus: payload.nodeGuard?.status ?? payload.audit?.nodeGuard?.status ?? null, lockStatus: payload.lockState ? `${payload.lockState.status || "unknown"}/${payload.lockState.phase || "unknown"}/${payload.lockState.held === true ? "held" : "free"}` : null, targetCommit: payload.target?.promotionCommit ?? payload.audit?.desiredState?.targetCommit ?? null, blockerTypes: Array.isArray(blockerTypes) ? blockerTypes.slice(0, 4) : [], outputCompacted: true, }; text = JSON.stringify(compact); } fs.writeFileSync(compactResultPath, `${text}\n`, { mode: 0o600 }); console.log(text); } async function main() { const options = decodeOptions(); const timeoutMs = Math.min(Number(options.timeoutMs || 45000), 60000); const runId = options.runId || `${new Date().toISOString().replace(/[:.]/g, "-")}-${process.pid}`; const dumpDir = path.join(os.homedir(), ".state", "unidesk-hwlab-cd", runId); await fsp.mkdir(dumpDir, { recursive: true, mode: 0o700 }); const action = options.action; const repo = await resolveRepo(options.repoPath || null, dumpDir, Math.min(timeoutMs, 55000)); const base = { ok: false, env: "dev", environment: "dev", action, dryRun: action === "apply" ? Boolean(options.dryRun) : action !== "status", mutation: false, remoteHost: "D601", workspace: repo.status === "selected" ? { path: repo.path, source: repo.source } : null, kubeconfig: { source: "explicit", path: options.kubeconfig || nativeKubeconfig, required: nativeKubeconfig }, dumpPath: dumpDir, safety: { wrapperCallsOnlyRepoOwnedDevCdApply: true, deployJsonAuthoritative: "deploy/deploy.json", kubectlApplyExecuted: false, rolloutExecuted: false, liveVerifyExecuted: false, secretValuesRead: false, secretValuesPrinted: false, cdLockMutated: false, prodTouched: false, reportsWritten: false, repoReportsWritten: false, }, }; if (repo.status !== "selected") { const blockers = collectBlockers({ repo, action }); emitResult({ ...base, status: "blocked", error: "hwlab-repo-not-found", repo, blockers, nextSafeCommand: nextSafeCommand(action, blockers) }, dumpDir, options.compactStdout === true); process.exitCode = 1; return; } const repoPath = repo.path; process.chdir(repoPath); const [git, guard] = await Promise.all([ gitSummary(repoPath, dumpDir, Math.min(timeoutMs, 15000)), nodeGuard(options.kubeconfig || nativeKubeconfig, dumpDir, Math.min(timeoutMs, 15000)), ]); const desiredState = { deployJson: readJsonSummary(repoPath, "deploy/deploy.json"), artifactCatalog: readJsonSummary(repoPath, "deploy/artifact-catalog.dev.json"), artifactReport: readJsonSummary(repoPath, "reports/dev-gate/dev-artifacts.json"), }; const lock = await lockStatus(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)); const needsSecretPreflight = action === "preflight" || action === "audit" || action === "apply"; const secretRefs = needsSecretPreflight ? await secretPreflight(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)) : { status: "skipped", reason: "status-does-not-run-secret-preflight", namespace, mutation: false, blockers: [] }; const preCommandBlockers = collectBlockers({ repo, git, guard, lock, secretRefs, action }); const canRunControlled = preCommandBlockers.length === 0 || action === "status" || action === "audit"; const controlled = canRunControlled && git.status === "pass" && guard.status === "pass" ? await runDevCdApply(repoPath, options.kubeconfig || nativeKubeconfig, action === "status" ? "status" : "apply", dumpDir, Math.max(1000, timeoutMs - 5000)) : { status: "skipped", commandOk: true, reason: "preflight-blockers-before-controlled-dev-cd-apply" }; const target = controlled.target || { ref: "deploy/deploy.json", promotionCommit: desiredState.deployJson.commitId || git.headCommit, shortCommitId: desiredState.deployJson.commitId || null, promotionSource: "deploy/deploy.json", namespace, }; if (action === "audit") { const auditDesiredState = buildDesiredStateAudit(repoPath, target); const [registry, workload, publicHealth] = await Promise.all([ registryAudit(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)), workloadAudit(options.kubeconfig || nativeKubeconfig, guard, auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)), httpHealthAudit(auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)), ]); const durability = runtimeDurabilityAudit(auditDesiredState, publicHealth, secretRefs, workload); const summary = auditSummary({ repo, git, guard, secretRefs, registry, lock, desired: auditDesiredState, workload, publicHealth, durability, controlled, dumpDir }); emitResult({ ...base, ok: summary.ok, status: summary.status, audit: summary, repo: { status: repo.status, path: repo.path, source: repo.source, rejected: repo.rejected, }, workspace: { path: repoPath, source: repo.source }, remoteDumpPath: dumpDir, reportDumpPath: controlled.command?.dump?.stdout || dumpDir, blockers: summary.blockers, blockerTypes: summary.blockerTypes, nextSafeCommand: nextSafeCommand(action, summary.blockers), }, dumpDir, options.compactStdout === true); process.exitCode = summary.ok ? 0 : 1; return; } const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action }); const ok = blockers.length === 0; emitResult({ ...base, ok, status: ok ? action === "status" ? "ready" : "prepared" : guard.refusal ? "refused" : "blocked", repo, workspace: { path: repoPath, source: repo.source }, worktreeGuard: git, nodeGuard: guard, secretPreflight: secretRefs, lockState: lock, desiredState, target, promotion: { source: "deploy/deploy.json", deployJson: desiredState.deployJson, artifactCatalog: desiredState.artifactCatalog, artifactReport: desiredState.artifactReport, }, controlledDevCd: controlled, reportDumpPath: controlled.command?.dump?.stdout || dumpDir, blockers, nextSafeCommand: nextSafeCommand(action, blockers), }, dumpDir, options.compactStdout === true); process.exitCode = ok ? 0 : 1; } main().catch((error) => { const message = error instanceof Error ? error.message : String(error); console.log(JSON.stringify({ ok: false, env: "dev", status: "blocked", error: "remote-wrapper-exception", message: redact(message) }, null, 2)); process.exitCode = 1; });