Codex
1878 lines
92 KiB
JavaScript
1878 lines
92 KiB
JavaScript
const { spawn } = require("node:child_process");
|
|
const crypto = require("node:crypto");
|
|
const fs = require("node:fs");
|
|
const fsp = require("node:fs/promises");
|
|
const os = require("node:os");
|
|
const path = require("node:path");
|
|
|
|
const namespace = "hwlab-dev";
|
|
const lockName = "hwlab-dev-cd-lock";
|
|
const nativeKubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
|
const legacyDefaultRepo = "/home/ubuntu/hwlab_cd";
|
|
const rejectedRepo = "/home/ubuntu/hwlab";
|
|
const defaultRemoteCandidates = ["git@github.com:pikasTech/HWLAB.git"];
|
|
const dedicatedCacheRoot = path.join(os.homedir(), ".cache", "unidesk", "hwlab-cd");
|
|
const dedicatedCacheRepo = path.join(os.homedir(), ".cache", "unidesk", "hwlab-cd", "git-cache", "HWLAB.git");
|
|
const defaultEgressProxy = "http://127.0.0.1:18789";
|
|
const defaultNoProxy = "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local";
|
|
const seedCacheRepos = ["/home/ubuntu/hwlab", "/home/ubuntu/workspace/hwlab", "/tmp/hwlab-dev-cd-35bbbee-host"];
|
|
const requiredNodeName = "d601";
|
|
const tailChars = 1400;
|
|
const parseCaptureLimitBytes = 1024 * 1024;
|
|
const requiredSecretRefs = [
|
|
{ secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url", consumers: ["hwlab-cloud-api"] },
|
|
{ secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url", consumers: ["runtime provisioning", "runtime migration"] },
|
|
{ secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key", consumers: ["hwlab-cloud-api", "code agent provider"] },
|
|
];
|
|
|
|
function decodeOptions() {
|
|
const raw = process.env.UNIDESK_HWLAB_CD_OPTIONS_B64 || "";
|
|
if (!raw) throw new Error("UNIDESK_HWLAB_CD_OPTIONS_B64 is required");
|
|
return JSON.parse(Buffer.from(raw, "base64").toString("utf8"));
|
|
}
|
|
|
|
function redact(value) {
|
|
return String(value || "")
|
|
.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1<redacted>")
|
|
.replace(/\b(token|password|api[_-]?key|secret|client-key-data|client-certificate-data|certificate-authority-data)\b(\s*[:=]\s*)(["']?)([^"'\s,}]+)/giu, "$1$2$3<redacted>")
|
|
.replace(/\b(postgres(?:ql)?:\/\/[^:\s/@]+:)([^@\s]+)(@)/giu, "$1<redacted>$3");
|
|
}
|
|
|
|
function boundedTail(value) {
|
|
const text = redact(value);
|
|
return text.slice(Math.max(0, text.length - tailChars));
|
|
}
|
|
|
|
function readFileTail(filePath) {
|
|
try {
|
|
return boundedTail(fs.readFileSync(filePath, "utf8"));
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function commandView(result) {
|
|
return {
|
|
id: result.id,
|
|
command: result.command,
|
|
cwd: result.cwd,
|
|
ok: result.ok,
|
|
exitCode: result.exitCode,
|
|
signal: result.signal,
|
|
timedOut: result.timedOut,
|
|
durationMs: result.durationMs,
|
|
dump: {
|
|
stdout: result.dump.stdout,
|
|
stderr: result.dump.stderr,
|
|
stdoutBytes: result.dump.stdoutBytes,
|
|
stderrBytes: result.dump.stderrBytes,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function runCaptured(command, cwd, dumpDir, id, options = {}) {
|
|
await fsp.mkdir(dumpDir, { recursive: true, mode: 0o700 });
|
|
const stdoutPath = path.join(dumpDir, `${id}.stdout.txt`);
|
|
const stderrPath = path.join(dumpDir, `${id}.stderr.txt`);
|
|
fs.writeFileSync(stdoutPath, "", { mode: 0o600 });
|
|
fs.writeFileSync(stderrPath, "", { mode: 0o600 });
|
|
|
|
const startedAt = Date.now();
|
|
let timedOut = false;
|
|
let spawnErrorMessage = "";
|
|
let stdoutBytes = 0;
|
|
let stderrBytes = 0;
|
|
const stdoutChunks = [];
|
|
const stderrChunks = [];
|
|
const stdout = fs.createWriteStream(stdoutPath, { flags: "a" });
|
|
const stderr = fs.createWriteStream(stderrPath, { flags: "a" });
|
|
const child = spawn(command[0], command.slice(1), {
|
|
cwd,
|
|
env: options.env || process.env,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
const timeout = options.timeoutMs
|
|
? setTimeout(() => {
|
|
timedOut = true;
|
|
child.kill("SIGTERM");
|
|
setTimeout(() => child.kill("SIGKILL"), 1000).unref();
|
|
}, options.timeoutMs)
|
|
: null;
|
|
|
|
child.stdout.on("data", (chunk) => {
|
|
stdoutBytes += chunk.byteLength;
|
|
stdout.write(chunk);
|
|
if (Buffer.concat(stdoutChunks).byteLength < parseCaptureLimitBytes) stdoutChunks.push(Buffer.from(chunk));
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderrBytes += chunk.byteLength;
|
|
stderr.write(chunk);
|
|
if (Buffer.concat(stderrChunks).byteLength < parseCaptureLimitBytes) stderrChunks.push(Buffer.from(chunk));
|
|
});
|
|
|
|
const closed = await new Promise((resolve) => {
|
|
child.on("error", (error) => {
|
|
spawnErrorMessage = error.message;
|
|
});
|
|
child.on("close", (exitCode, signal) => resolve({ exitCode, signal }));
|
|
});
|
|
if (timeout !== null) clearTimeout(timeout);
|
|
await new Promise((resolve) => stdout.end(resolve));
|
|
await new Promise((resolve) => stderr.end(resolve));
|
|
|
|
const stdoutText = stdoutBytes <= parseCaptureLimitBytes ? Buffer.concat(stdoutChunks).toString("utf8") : "";
|
|
const stderrText = stderrBytes <= parseCaptureLimitBytes ? Buffer.concat(stderrChunks).toString("utf8") : spawnErrorMessage;
|
|
const exitCode = spawnErrorMessage && closed.exitCode === null ? 127 : closed.exitCode;
|
|
return {
|
|
id,
|
|
command,
|
|
cwd,
|
|
ok: exitCode === 0 && !timedOut,
|
|
exitCode,
|
|
signal: closed.signal,
|
|
timedOut,
|
|
durationMs: Date.now() - startedAt,
|
|
stdoutText,
|
|
stderrText,
|
|
dump: {
|
|
stdout: stdoutPath,
|
|
stderr: stderrPath,
|
|
stdoutBytes,
|
|
stderrBytes,
|
|
stdoutTail: boundedTail(stdoutText || readFileTail(stdoutPath)),
|
|
stderrTail: boundedTail(stderrText || readFileTail(stderrPath)),
|
|
},
|
|
};
|
|
}
|
|
|
|
function parseJson(text) {
|
|
if (!String(text || "").trim()) return null;
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function asRecord(value) {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
}
|
|
|
|
function stringValue(value) {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function sha256(text) {
|
|
return `sha256:${crypto.createHash("sha256").update(text).digest("hex")}`;
|
|
}
|
|
|
|
function accessCheck(targetPath, mode) {
|
|
try {
|
|
fs.accessSync(targetPath, mode);
|
|
return { ok: true, path: targetPath };
|
|
} catch (error) {
|
|
return { ok: false, path: targetPath, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function repoRecord(rawPath, source, extra = {}) {
|
|
const absolutePath = path.resolve(rawPath);
|
|
const rejected = absolutePath === rejectedRepo;
|
|
const devCdApply = path.join(absolutePath, "scripts/dev-cd-apply.mjs");
|
|
const deployJson = path.join(absolutePath, "deploy/deploy.json");
|
|
return {
|
|
status: !rejected && fs.existsSync(absolutePath) && fs.existsSync(devCdApply) && fs.existsSync(deployJson) ? "selected" : "blocked",
|
|
source,
|
|
path: absolutePath,
|
|
defaultPath: legacyDefaultRepo,
|
|
defaultMode: "ephemeral-clone",
|
|
rejected,
|
|
rejectionReason: rejected ? "runner-history-directory-is-not-hwlab-cd-release-truth" : null,
|
|
exists: fs.existsSync(absolutePath),
|
|
hasDevCdApply: fs.existsSync(devCdApply),
|
|
hasDeployJson: fs.existsSync(deployJson),
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
function remoteCandidates() {
|
|
const fromEnv = process.env.UNIDESK_HWLAB_REMOTE_URL;
|
|
if (typeof fromEnv === "string" && fromEnv.trim().length > 0) return [fromEnv.trim()];
|
|
return defaultRemoteCandidates;
|
|
}
|
|
|
|
function seedRepoCandidates() {
|
|
const fromEnv = process.env.UNIDESK_HWLAB_CACHE_SEED_REPOS;
|
|
const configured = typeof fromEnv === "string" && fromEnv.trim().length > 0
|
|
? fromEnv.split(":").map((item) => item.trim()).filter(Boolean)
|
|
: seedCacheRepos;
|
|
return [...new Set(configured.map((item) => path.resolve(item)))];
|
|
}
|
|
|
|
function egressEnv() {
|
|
const proxy = process.env.UNIDESK_HWLAB_EGRESS_PROXY || defaultEgressProxy;
|
|
const noProxy = process.env.NO_PROXY || process.env.no_proxy || defaultNoProxy;
|
|
const proxyHostPort = proxy.replace(/^https?:\/\//u, "");
|
|
const gitSshCommand = process.env.GIT_SSH_COMMAND || [
|
|
"ssh",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
"-o", `"ProxyCommand=nc -X connect -x ${proxyHostPort} %h %p"`,
|
|
].join(" ");
|
|
return {
|
|
...process.env,
|
|
HTTP_PROXY: proxy,
|
|
HTTPS_PROXY: proxy,
|
|
ALL_PROXY: proxy,
|
|
http_proxy: proxy,
|
|
https_proxy: proxy,
|
|
all_proxy: proxy,
|
|
NO_PROXY: noProxy,
|
|
no_proxy: noProxy,
|
|
GIT_SSH_COMMAND: gitSshCommand,
|
|
};
|
|
}
|
|
|
|
async function egressProxyHealth(dumpDir, timeoutMs) {
|
|
const proxy = process.env.UNIDESK_HWLAB_EGRESS_PROXY || defaultEgressProxy;
|
|
const healthUrl = `${proxy.replace(/\/$/u, "")}/__unidesk/egress-proxy/health`;
|
|
const command = await runCaptured(["curl", "-fsS", "--max-time", "3", healthUrl], os.homedir(), dumpDir, "egress-proxy-health", { env: egressEnv(), timeoutMs: Math.min(timeoutMs, 5000) });
|
|
const body = parseJson(command.stdoutText);
|
|
return {
|
|
status: command.ok && asRecord(body)?.connected === true ? "pass" : "blocked",
|
|
proxy,
|
|
healthUrl,
|
|
connected: asRecord(body)?.connected ?? null,
|
|
command: commandView(command),
|
|
};
|
|
}
|
|
|
|
function pathOwner(targetPath) {
|
|
try {
|
|
const stat = fs.statSync(targetPath);
|
|
return { uid: stat.uid, gid: stat.gid, mode: (stat.mode & 0o777).toString(8), isDirectory: stat.isDirectory() };
|
|
} catch (error) {
|
|
return { uid: null, gid: null, mode: null, isDirectory: false, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function cacheRepoRecord(cachePath) {
|
|
const absolutePath = path.resolve(cachePath);
|
|
const gitPath = path.join(absolutePath, ".git");
|
|
const bareHead = path.join(absolutePath, "HEAD");
|
|
const bareObjects = path.join(absolutePath, "objects");
|
|
const shallowPath = path.join(absolutePath, "shallow");
|
|
const exists = fs.existsSync(gitPath) || (fs.existsSync(bareHead) && fs.existsSync(bareObjects));
|
|
const readable = accessCheck(absolutePath, fs.constants.R_OK | fs.constants.X_OK);
|
|
const writable = accessCheck(absolutePath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
|
|
const owner = pathOwner(absolutePath);
|
|
return { path: absolutePath, exists, readable: readable.ok, writable: writable.ok, owner, shallow: fs.existsSync(shallowPath), error: readable.error || writable.error || null };
|
|
}
|
|
|
|
async function deployJsonCommitFromCache(dumpDir, timeoutMs) {
|
|
const command = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "show", "refs/heads/main:deploy/deploy.json"], os.homedir(), dumpDir, "git-dedicated-cache-deploy-json", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
if (!command.ok) return { commitId: null, command: commandView(command) };
|
|
const parsed = parseJson(command.stdoutText);
|
|
return { commitId: stringValue(asRecord(parsed)?.commitId), command: commandView(command) };
|
|
}
|
|
|
|
async function cacheCommitExists(commitId, dumpDir, timeoutMs, idPrefix = "git-dedicated-cache-commit") {
|
|
if (!commitId) return { exists: false, command: null };
|
|
const command = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "--verify", `${commitId}^{commit}`], os.homedir(), dumpDir, idPrefix, { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
return { exists: command.ok, resolvedCommit: command.stdoutText.trim() || null, command: commandView(command) };
|
|
}
|
|
|
|
async function refreshDedicatedCache(remoteUrl, dumpDir, timeoutMs) {
|
|
const env = egressEnv();
|
|
const parent = path.dirname(dedicatedCacheRepo);
|
|
await fsp.mkdir(parent, { recursive: true, mode: 0o700 });
|
|
await fsp.chmod(dedicatedCacheRoot, 0o700).catch(() => undefined);
|
|
await fsp.chmod(path.dirname(dedicatedCacheRoot), 0o700).catch(() => undefined);
|
|
const attempts = [];
|
|
const existing = cacheRepoRecord(dedicatedCacheRepo);
|
|
if (existing.exists && (!existing.writable || existing.owner.uid !== process.getuid?.())) {
|
|
attempts.push({ stage: "dedicated-cache-owner", ok: false, cache: existing });
|
|
return { ok: false, cache: existing, attempts, stage: "dedicated-cache-owner" };
|
|
}
|
|
if (existing.exists) {
|
|
const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
attempts.push({ stage: "cache-permissions", ok: permissions.ok, command: commandView(permissions) });
|
|
if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions" };
|
|
}
|
|
|
|
if (!existing.exists) {
|
|
for (const seed of seedRepoCandidates().map(cacheRepoRecord)) {
|
|
if (!seed.exists || !seed.readable) {
|
|
attempts.push({ stage: "seed-unavailable", ok: false, seed });
|
|
continue;
|
|
}
|
|
await fsp.rm(dedicatedCacheRepo, { recursive: true, force: true });
|
|
const cloneSeed = await runCaptured(["git", "clone", "--bare", seed.path, dedicatedCacheRepo], parent, dumpDir, "git-dedicated-cache-seed-clone", { timeoutMs: Math.min(timeoutMs, 15000) });
|
|
attempts.push({ stage: "seed-clone", ok: cloneSeed.ok, seed, command: commandView(cloneSeed) });
|
|
if (cloneSeed.ok) break;
|
|
}
|
|
if (!fs.existsSync(dedicatedCacheRepo)) {
|
|
const init = await runCaptured(["git", "init", "--bare", dedicatedCacheRepo], parent, dumpDir, "git-dedicated-cache-init", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
attempts.push({ stage: "bare-init", ok: init.ok, command: commandView(init) });
|
|
if (!init.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "bare-init" };
|
|
}
|
|
const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
attempts.push({ stage: "cache-permissions", ok: permissions.ok, command: commandView(permissions) });
|
|
if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions" };
|
|
}
|
|
|
|
const remoteSet = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "remote", "set-url", "origin", remoteUrl], os.homedir(), dumpDir, "git-dedicated-cache-remote-set-url", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
const remoteAdd = remoteSet.ok ? null : await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "remote", "add", "origin", remoteUrl], os.homedir(), dumpDir, "git-dedicated-cache-remote-add", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
attempts.push({ stage: "remote-set-url", ok: remoteSet.ok || remoteAdd?.ok === true, command: commandView(remoteSet), fallback: remoteAdd ? commandView(remoteAdd) : null });
|
|
if (!remoteSet.ok && remoteAdd?.ok !== true) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "remote-set-url" };
|
|
|
|
const remoteHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "ls-remote", "--heads", "origin", "main"], os.homedir(), dumpDir, "git-dedicated-cache-ls-remote-main", { env, timeoutMs: Math.min(timeoutMs, 15000) });
|
|
const remoteMainCommit = remoteHead.stdoutText.trim().split(/\s+/u)[0] || null;
|
|
attempts.push({ stage: "ls-remote-main", ok: remoteHead.ok && Boolean(remoteMainCommit), remoteMainCommit, command: commandView(remoteHead) });
|
|
const localHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "refs/heads/main"], os.homedir(), dumpDir, "git-dedicated-cache-local-main", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
const localMainCommit = localHead.stdoutText.trim() || null;
|
|
attempts.push({ stage: "local-main", ok: localHead.ok && Boolean(localMainCommit), localMainCommit, command: commandView(localHead) });
|
|
const deployJson = localHead.ok ? await deployJsonCommitFromCache(dumpDir, timeoutMs) : { commitId: null, command: null };
|
|
attempts.push({ stage: "deploy-json-commit", ok: Boolean(deployJson.commitId), commitId: deployJson.commitId, command: deployJson.command });
|
|
const deployCommit = await cacheCommitExists(deployJson.commitId, dumpDir, timeoutMs, "git-dedicated-cache-deploy-commit");
|
|
attempts.push({ stage: "deploy-commit-present", ok: deployCommit.exists, commitId: deployJson.commitId, resolvedCommit: deployCommit.resolvedCommit, command: deployCommit.command });
|
|
const cacheRecordBeforeFetch = cacheRepoRecord(dedicatedCacheRepo);
|
|
if (remoteHead.ok && remoteMainCommit && localHead.ok && localMainCommit === remoteMainCommit && deployCommit.exists && cacheRecordBeforeFetch.shallow !== true) {
|
|
return { ok: true, cache: cacheRecordBeforeFetch, attempts, stage: "ready", refreshStatus: "current-full", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId };
|
|
}
|
|
|
|
const fetchArgs = ["git", "--git-dir", dedicatedCacheRepo, "fetch", "--prune", "--tags", "origin", "+refs/heads/*:refs/heads/*"];
|
|
if (cacheRecordBeforeFetch.shallow === true) fetchArgs.splice(4, 0, "--unshallow");
|
|
const fetch = await runCaptured(fetchArgs, os.homedir(), dumpDir, "git-dedicated-cache-fetch-full", { env, timeoutMs: Math.min(timeoutMs, 55000) });
|
|
attempts.push({ stage: "fetch-main", ok: fetch.ok, command: commandView(fetch) });
|
|
if (!fetch.ok) {
|
|
if (localHead.ok && localMainCommit && deployCommit.exists) {
|
|
return { ok: true, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "stale-cache", refreshStatus: "blocked", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId };
|
|
}
|
|
return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "fetch-full", refreshStatus: "blocked", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId };
|
|
}
|
|
const fetchedLocalHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "refs/heads/main"], os.homedir(), dumpDir, "git-dedicated-cache-local-main-after-fetch", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
const fetchedLocalMainCommit = fetchedLocalHead.stdoutText.trim() || localMainCommit;
|
|
attempts.push({ stage: "local-main-after-fetch", ok: fetchedLocalHead.ok && Boolean(fetchedLocalMainCommit), localMainCommit: fetchedLocalMainCommit, command: commandView(fetchedLocalHead) });
|
|
const fetchedDeployJson = await deployJsonCommitFromCache(dumpDir, timeoutMs);
|
|
attempts.push({ stage: "deploy-json-commit-after-fetch", ok: Boolean(fetchedDeployJson.commitId), commitId: fetchedDeployJson.commitId, command: fetchedDeployJson.command });
|
|
const fetchedDeployCommit = await cacheCommitExists(fetchedDeployJson.commitId, dumpDir, timeoutMs, "git-dedicated-cache-deploy-commit-after-fetch");
|
|
attempts.push({ stage: "deploy-commit-present-after-fetch", ok: fetchedDeployCommit.exists, commitId: fetchedDeployJson.commitId, resolvedCommit: fetchedDeployCommit.resolvedCommit, command: fetchedDeployCommit.command });
|
|
const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions-after-fetch", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
attempts.push({ stage: "cache-permissions-after-fetch", ok: permissions.ok, command: commandView(permissions) });
|
|
if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions-after-fetch", refreshStatus: "blocked", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId };
|
|
if (!fetchedDeployCommit.exists) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "deploy-commit-unresolved", refreshStatus: "blocked", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId };
|
|
return { ok: true, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "ready", refreshStatus: "refreshed-full", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId };
|
|
}
|
|
|
|
async function materializeFromCache(cachePath, repoPath, remoteUrl, dumpDir, timeoutMs) {
|
|
await fsp.rm(repoPath, { recursive: true, force: true });
|
|
const clone = await runCaptured(["git", "clone", "--no-hardlinks", "--no-checkout", cachePath, repoPath], path.dirname(repoPath), dumpDir, "git-cache-clone", { timeoutMs: Math.min(timeoutMs, 15000) });
|
|
if (!clone.ok) return { ok: false, stage: "cache-clone", commands: [commandView(clone)] };
|
|
const remote = await runCaptured(["git", "remote", "set-url", "origin", remoteUrl], repoPath, dumpDir, "git-cache-remote-set-url", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
if (!remote.ok) return { ok: false, stage: "remote-set-url", commands: [commandView(clone), commandView(remote)] };
|
|
const checkout = await runCaptured(["git", "checkout", "-B", "main", "origin/main"], repoPath, dumpDir, "git-cache-checkout-main", { timeoutMs: Math.min(timeoutMs, 5000) });
|
|
const reset = checkout.ok
|
|
? await runCaptured(["git", "reset", "--hard", "origin/main"], repoPath, dumpDir, "git-cache-reset-hard", { timeoutMs: Math.min(timeoutMs, 5000) })
|
|
: null;
|
|
const clean = checkout.ok && reset?.ok
|
|
? await runCaptured(["git", "clean", "-ffdqx"], repoPath, dumpDir, "git-cache-clean", { timeoutMs: Math.min(timeoutMs, 5000) })
|
|
: null;
|
|
const upstream = checkout.ok && reset?.ok
|
|
? await runCaptured(["git", "branch", "--set-upstream-to=origin/main", "main"], repoPath, dumpDir, "git-cache-upstream", { timeoutMs: Math.min(timeoutMs, 5000) })
|
|
: null;
|
|
const fetchHead = checkout.ok && reset?.ok && clean?.ok
|
|
? await runCaptured(["bash", "-lc", "p=\"$(git rev-parse --path-format=absolute --git-common-dir)/FETCH_HEAD\"; : > \"$p\"; chmod 600 \"$p\""], repoPath, dumpDir, "git-cache-fetch-head", { timeoutMs: Math.min(timeoutMs, 5000) })
|
|
: null;
|
|
const commands = [clone, remote, checkout, reset, clean, upstream, fetchHead].filter(Boolean).map(commandView);
|
|
return { ok: checkout.ok && reset?.ok === true && clean?.ok === true && fetchHead?.ok === true, stage: "ready", commands };
|
|
}
|
|
|
|
async function cloneEphemeralRepo(dumpDir, timeoutMs) {
|
|
const root = path.join(dumpDir, "ephemeral-repo");
|
|
const attempts = [];
|
|
await fsp.mkdir(root, { recursive: true, mode: 0o700 });
|
|
const egress = await egressProxyHealth(dumpDir, Math.min(timeoutMs, 5000));
|
|
if (egress.status !== "pass") {
|
|
return repoRecord(path.join(root, "HWLAB"), "ephemeral:git-cache", {
|
|
ephemeral: true,
|
|
remoteUrl: null,
|
|
egressProxy: egress,
|
|
dedicatedCache: cacheRepoRecord(dedicatedCacheRepo),
|
|
seedRepos: seedRepoCandidates().map(cacheRepoRecord),
|
|
cloneAttempts: [],
|
|
cloneFailed: true,
|
|
cloneFailureStage: "egress-proxy",
|
|
retainedForDiagnostics: true,
|
|
});
|
|
}
|
|
for (const remoteUrl of remoteCandidates()) {
|
|
const repoPath = path.join(root, "HWLAB");
|
|
const refreshed = await refreshDedicatedCache(remoteUrl, dumpDir, Math.min(timeoutMs, 45000));
|
|
attempts.push({ remoteUrl, dedicatedCache: refreshed.cache, ok: refreshed.ok, stage: refreshed.stage, refreshAttempts: refreshed.attempts });
|
|
if (!refreshed.ok) continue;
|
|
const materialized = await materializeFromCache(dedicatedCacheRepo, repoPath, remoteUrl, dumpDir, Math.min(timeoutMs, 9000));
|
|
attempts.push({ remoteUrl, dedicatedCache: refreshed.cache, ok: materialized.ok, stage: materialized.stage, commands: materialized.commands });
|
|
if (materialized.ok) {
|
|
return repoRecord(repoPath, "ephemeral:dedicated-git-cache", {
|
|
ephemeral: true,
|
|
remoteUrl,
|
|
egressProxy: egress,
|
|
dedicatedCache: refreshed.cache,
|
|
cacheRefreshStatus: refreshed.refreshStatus || "current",
|
|
remoteMainCommit: refreshed.remoteMainCommit || null,
|
|
cacheMainCommit: refreshed.localMainCommit || null,
|
|
deployCommitId: refreshed.deployCommitId || null,
|
|
seedRepos: seedRepoCandidates().map(cacheRepoRecord),
|
|
cloneAttempts: attempts,
|
|
retainedForDiagnostics: true,
|
|
});
|
|
}
|
|
}
|
|
return repoRecord(path.join(root, "HWLAB"), "ephemeral:dedicated-git-cache", {
|
|
ephemeral: true,
|
|
remoteUrl: null,
|
|
egressProxy: egress,
|
|
dedicatedCache: cacheRepoRecord(dedicatedCacheRepo),
|
|
seedRepos: seedRepoCandidates().map(cacheRepoRecord),
|
|
cloneAttempts: attempts,
|
|
cloneFailed: true,
|
|
cloneFailureStage: attempts.findLast((attempt) => attempt.stage)?.stage || "cache-unavailable",
|
|
retainedForDiagnostics: true,
|
|
});
|
|
}
|
|
|
|
async function resolveRepo(provided, dumpDir, timeoutMs) {
|
|
if (provided) return repoRecord(provided, "option", { ephemeral: false });
|
|
if (process.env.UNIDESK_HWLAB_REPO) return repoRecord(process.env.UNIDESK_HWLAB_REPO, "env:UNIDESK_HWLAB_REPO", { ephemeral: false });
|
|
return cloneEphemeralRepo(dumpDir, Math.min(timeoutMs, 55000));
|
|
}
|
|
|
|
async function gitSummary(repoPath, dumpDir, timeoutMs) {
|
|
const [branch, head, originMain, remote, gitDir, gitCommonDir, statusShort, statusPorcelain] = await Promise.all([
|
|
runCaptured(["git", "rev-parse", "--abbrev-ref", "HEAD"], repoPath, dumpDir, "git-branch", { timeoutMs }),
|
|
runCaptured(["git", "rev-parse", "HEAD"], repoPath, dumpDir, "git-head", { timeoutMs }),
|
|
runCaptured(["git", "rev-parse", "--verify", "origin/main^{commit}"], repoPath, dumpDir, "git-origin-main", { timeoutMs }),
|
|
runCaptured(["git", "remote", "get-url", "origin"], repoPath, dumpDir, "git-remote-origin", { timeoutMs }),
|
|
runCaptured(["git", "rev-parse", "--path-format=absolute", "--git-dir"], repoPath, dumpDir, "git-dir", { timeoutMs }),
|
|
runCaptured(["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], repoPath, dumpDir, "git-common-dir", { timeoutMs }),
|
|
runCaptured(["git", "status", "--short", "--branch"], repoPath, dumpDir, "git-status-short", { timeoutMs }),
|
|
runCaptured(["git", "status", "--porcelain=v1"], repoPath, dumpDir, "git-status-porcelain", { timeoutMs }),
|
|
]);
|
|
const branchName = branch.stdoutText.trim();
|
|
const headCommit = head.stdoutText.trim();
|
|
const originMainCommit = originMain.stdoutText.trim();
|
|
const remoteUrl = remote.stdoutText.trim();
|
|
const gitDirPath = gitDir.stdoutText.trim();
|
|
const gitCommonDirPath = gitCommonDir.stdoutText.trim() || gitDirPath;
|
|
const fetchHeadPath = gitCommonDirPath ? path.join(gitCommonDirPath, "FETCH_HEAD") : path.join(repoPath, ".git", "FETCH_HEAD");
|
|
const worktreesPath = gitCommonDirPath ? path.join(gitCommonDirPath, "worktrees") : path.join(repoPath, ".git", "worktrees");
|
|
const porcelain = statusPorcelain.stdoutText.trim();
|
|
const dirtyLines = porcelain ? porcelain.split("\n").filter(Boolean) : [];
|
|
const worktreeAccess = accessCheck(repoPath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
|
|
const fetchHeadAccess = accessCheck(fetchHeadPath, fs.constants.R_OK | fs.constants.W_OK);
|
|
const worktreesAccess = fs.existsSync(worktreesPath)
|
|
? accessCheck(worktreesPath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK)
|
|
: { ok: true, path: worktreesPath, exists: false };
|
|
const remoteMatches = /github\.com[:/]pikasTech\/HWLAB(?:\.git)?$/u.test(remoteUrl);
|
|
const blockers = [];
|
|
if (!branch.ok || !head.ok || !remote.ok || !gitDir.ok || !gitCommonDir.ok || !statusShort.ok || !statusPorcelain.ok) blockers.push({ scope: "hwlab-git-command", summary: "One or more HWLAB git guard commands failed." });
|
|
if (dirtyLines.length > 0) blockers.push({ scope: "hwlab-git-clean", summary: "HWLAB CD worktree has local modifications; deploy/deploy.json cannot be treated as release truth." });
|
|
if (branchName !== "main") blockers.push({ scope: "hwlab-git-main", summary: `HWLAB CD worktree branch is ${branchName || "<unknown>"}, expected main.` });
|
|
if (!remoteMatches) blockers.push({ scope: "hwlab-git-remote", summary: "HWLAB CD worktree origin is not pikasTech/HWLAB." });
|
|
if (originMain.ok && headCommit && originMainCommit && headCommit !== originMainCommit) blockers.push({ scope: "hwlab-git-origin-main", summary: "HWLAB CD worktree HEAD does not match local origin/main." });
|
|
if (!worktreeAccess.ok) blockers.push({ scope: "hwlab-worktree-permission", summary: "HWLAB CD worktree is not readable/writable/executable by the wrapper." });
|
|
if (!fetchHeadAccess.ok) blockers.push({ scope: "hwlab-fetch-head-permission", summary: "HWLAB CD FETCH_HEAD is missing or not writable by the wrapper." });
|
|
if (!worktreesAccess.ok) blockers.push({ scope: "hwlab-worktrees-permission", summary: "HWLAB .git/worktrees permission guard failed." });
|
|
return {
|
|
status: blockers.length === 0 ? "pass" : "blocked",
|
|
clean: dirtyLines.length === 0,
|
|
branch: branchName || null,
|
|
onMain: branchName === "main",
|
|
remote: remoteUrl || null,
|
|
remoteMatches,
|
|
expectedRemote: "git@github.com:pikasTech/HWLAB.git or https://github.com/pikasTech/HWLAB.git",
|
|
headCommit: headCommit || null,
|
|
originMainCommit: originMain.ok ? originMainCommit || null : null,
|
|
headMatchesOriginMain: originMain.ok && headCommit.length > 0 && headCommit === originMainCommit,
|
|
worktreeAccess,
|
|
fetchHead: { path: fetchHeadPath, exists: fs.existsSync(fetchHeadPath), writable: fetchHeadAccess.ok, error: fetchHeadAccess.error || null },
|
|
worktrees: worktreesAccess,
|
|
statusShort: statusShort.stdoutText.trim().split("\n").filter(Boolean).slice(0, 30),
|
|
dirtyCount: dirtyLines.length,
|
|
dirtyPreview: dirtyLines.slice(0, 20),
|
|
blockers,
|
|
commands: [branch, head, originMain, remote, gitDir, gitCommonDir, statusShort, statusPorcelain].map(commandView),
|
|
};
|
|
}
|
|
|
|
function readJsonSummary(repoPath, relativePath, fallback = {}) {
|
|
const absolutePath = path.join(repoPath, relativePath);
|
|
try {
|
|
const raw = fs.readFileSync(absolutePath, "utf8");
|
|
const parsed = JSON.parse(raw);
|
|
return {
|
|
path: relativePath,
|
|
exists: true,
|
|
hash: sha256(raw),
|
|
commitId: parsed.commitId || parsed.sourceCommitId || null,
|
|
environment: parsed.environment || null,
|
|
namespace: parsed.namespace || namespace,
|
|
endpoint: parsed.endpoint || null,
|
|
artifactState: parsed.artifactState || parsed.publish?.artifactState || null,
|
|
ciPublished: parsed.publish?.ciPublished ?? parsed.artifactPublish?.ciPublished ?? null,
|
|
registryVerified: parsed.publish?.registryVerified ?? parsed.artifactPublish?.registryVerified ?? null,
|
|
serviceCount: Array.isArray(parsed.services) ? parsed.services.length : parsed.artifactPublish?.serviceCount ?? null,
|
|
...fallback,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
path: relativePath,
|
|
exists: false,
|
|
hash: null,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
...fallback,
|
|
};
|
|
}
|
|
}
|
|
|
|
function readJsonFile(repoPath, relativePath) {
|
|
const absolutePath = path.join(repoPath, relativePath);
|
|
try {
|
|
const raw = fs.readFileSync(absolutePath, "utf8");
|
|
return { ok: true, path: relativePath, exists: true, hash: sha256(raw), json: JSON.parse(raw), error: null };
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
path: relativePath,
|
|
exists: fs.existsSync(absolutePath),
|
|
hash: null,
|
|
json: null,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
function commitMatches(actual, expected) {
|
|
if (typeof actual !== "string" || typeof expected !== "string" || actual.length === 0 || expected.length === 0) return false;
|
|
return actual === expected || actual.startsWith(expected) || expected.startsWith(actual);
|
|
}
|
|
|
|
function imageTag(image) {
|
|
const value = String(image || "");
|
|
const slash = value.lastIndexOf("/");
|
|
const colon = value.lastIndexOf(":");
|
|
return colon > slash ? value.slice(colon + 1) : null;
|
|
}
|
|
|
|
function servicesFromManifest(manifest) {
|
|
const services = Array.isArray(manifest?.services)
|
|
? manifest.services
|
|
: manifest?.services && typeof manifest.services === "object"
|
|
? Object.values(manifest.services)
|
|
: [];
|
|
return services
|
|
.filter((service) => service && typeof service === "object")
|
|
.map((service) => ({
|
|
serviceId: stringValue(service.serviceId) || stringValue(service.id) || stringValue(service.name),
|
|
name: stringValue(service.name) || stringValue(service.serviceId) || stringValue(service.id),
|
|
image: stringValue(service.image),
|
|
imageTag: stringValue(service.imageTag) || imageTag(service.image),
|
|
commitId: stringValue(service.commitId) || stringValue(service.imageTag) || imageTag(service.image),
|
|
namespace: stringValue(service.namespace) || namespace,
|
|
healthPath: stringValue(service.healthPath) || "/health/live",
|
|
replicas: Number.isInteger(service.replicas) ? service.replicas : null,
|
|
env: asRecord(service.env) || {},
|
|
digest: stringValue(service.digest),
|
|
publishState: stringValue(service.publishState) || stringValue(service.status),
|
|
artifactRequired: service.artifactRequired !== false,
|
|
}))
|
|
.filter((service) => service.serviceId);
|
|
}
|
|
|
|
function summarizeDeployFile(file) {
|
|
const manifest = asRecord(file.json);
|
|
const endpoints = asRecord(manifest?.publicEndpoints) || {};
|
|
return {
|
|
path: file.path,
|
|
exists: file.exists,
|
|
hash: file.hash,
|
|
commitId: stringValue(manifest?.commitId),
|
|
environment: stringValue(manifest?.environment),
|
|
namespace: stringValue(manifest?.namespace) || namespace,
|
|
endpoint: stringValue(manifest?.endpoint),
|
|
serviceCount: servicesFromManifest(manifest).length,
|
|
publicEndpoints: {
|
|
frontend: stringValue(asRecord(endpoints.frontend)?.url),
|
|
api: stringValue(asRecord(endpoints.api)?.url),
|
|
},
|
|
unavailableReason: file.ok ? null : file.error,
|
|
};
|
|
}
|
|
|
|
function summarizeArtifactFile(file) {
|
|
const manifest = asRecord(file.json);
|
|
const services = servicesFromManifest(manifest);
|
|
const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 };
|
|
for (const service of services) {
|
|
if (/^sha256:[a-f0-9]{64}$/u.test(service.digest || "")) digestCounts.sha256 += 1;
|
|
else if (service.digest === "not_published") digestCounts.notPublished += 1;
|
|
else digestCounts.invalid += 1;
|
|
}
|
|
return {
|
|
path: file.path,
|
|
exists: file.exists,
|
|
hash: file.hash,
|
|
commitId: stringValue(manifest?.commitId),
|
|
artifactState: stringValue(manifest?.artifactState),
|
|
ciPublished: manifest?.publish?.ciPublished ?? null,
|
|
registryVerified: manifest?.publish?.registryVerified ?? null,
|
|
serviceCount: services.length,
|
|
digestCounts,
|
|
unavailableReason: file.ok ? null : file.error,
|
|
};
|
|
}
|
|
|
|
function buildDesiredStateAudit(repoPath, target) {
|
|
const deployFile = readJsonFile(repoPath, "deploy/deploy.json");
|
|
const catalogFile = readJsonFile(repoPath, "deploy/artifact-catalog.dev.json");
|
|
const reportFile = readJsonFile(repoPath, "reports/dev-gate/dev-artifacts.json");
|
|
const deployServices = servicesFromManifest(deployFile.json);
|
|
const catalogServices = servicesFromManifest(catalogFile.json);
|
|
const catalogById = new Map(catalogServices.map((service) => [service.serviceId, service]));
|
|
const mismatches = [];
|
|
const missing = [];
|
|
for (const service of deployServices) {
|
|
if (!service.image) continue;
|
|
const catalog = catalogById.get(service.serviceId);
|
|
if (!catalog && service.artifactRequired) {
|
|
missing.push({ serviceId: service.serviceId, reason: "missing-from-artifact-catalog" });
|
|
continue;
|
|
}
|
|
if (catalog?.image && catalog.image !== service.image) {
|
|
mismatches.push({ serviceId: service.serviceId, deployImage: service.image, catalogImage: catalog.image });
|
|
}
|
|
if (catalog && catalog.artifactRequired && catalog.publishState && catalog.publishState !== "published") {
|
|
missing.push({ serviceId: service.serviceId, reason: `artifact-${catalog.publishState}` });
|
|
}
|
|
}
|
|
const deploySummary = summarizeDeployFile(deployFile);
|
|
const catalogSummary = summarizeArtifactFile(catalogFile);
|
|
const reportSummary = summarizeArtifactFile(reportFile);
|
|
const targetCommit = stringValue(target?.shortCommitId) || stringValue(target?.promotionCommit) || deploySummary.commitId;
|
|
const commitConverged = Boolean(
|
|
targetCommit &&
|
|
commitMatches(deploySummary.commitId, targetCommit) &&
|
|
(!catalogSummary.commitId || commitMatches(catalogSummary.commitId, targetCommit))
|
|
);
|
|
const status = !deployFile.ok || !catalogFile.ok || missing.length > 0
|
|
? "missing"
|
|
: mismatches.length > 0 || !commitConverged
|
|
? "mismatch"
|
|
: "pass";
|
|
return {
|
|
status,
|
|
targetCommit,
|
|
commitConverged,
|
|
deployJson: deploySummary,
|
|
artifactCatalog: catalogSummary,
|
|
artifactReport: reportSummary,
|
|
imageConvergence: {
|
|
status: missing.length === 0 && mismatches.length === 0 ? "pass" : missing.length > 0 ? "missing" : "mismatch",
|
|
serviceCount: deployServices.length,
|
|
catalogServiceCount: catalogServices.length,
|
|
missing: missing.slice(0, 12),
|
|
mismatches: mismatches.slice(0, 12),
|
|
},
|
|
services: deployServices.map((service) => ({
|
|
serviceId: service.serviceId,
|
|
image: service.image,
|
|
imageTag: service.imageTag,
|
|
catalogImage: catalogById.get(service.serviceId)?.image || null,
|
|
catalogDigest: catalogById.get(service.serviceId)?.digest || null,
|
|
replicas: service.replicas,
|
|
})).slice(0, 30),
|
|
serviceEnv: {
|
|
cloudApi: asRecord(deployServices.find((service) => service.serviceId === "hwlab-cloud-api")?.env) || {},
|
|
},
|
|
};
|
|
}
|
|
|
|
function forbiddenKubeSignals(text) {
|
|
const signals = [];
|
|
if (/docker-desktop/iu.test(text)) signals.push("docker-desktop");
|
|
if (/desktop-control-plane/iu.test(text)) signals.push("desktop-control-plane");
|
|
if (/127\.0\.0\.1:11700/u.test(text)) signals.push("127.0.0.1:11700");
|
|
return [...new Set(signals)];
|
|
}
|
|
|
|
async function observeKube(kubeconfig, dumpDir, timeoutMs, withExplicitEnv) {
|
|
const env = withExplicitEnv ? { ...process.env, KUBECONFIG: kubeconfig } : { ...process.env };
|
|
if (!withExplicitEnv) {
|
|
delete env.KUBECONFIG;
|
|
delete env.HWLAB_DEV_KUBECONFIG;
|
|
}
|
|
const [context, server, nodes, namespaceCheck] = await Promise.all([
|
|
runCaptured(["kubectl", "config", "current-context"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-current-context" : "default-current-context", { env, timeoutMs }),
|
|
runCaptured(["kubectl", "config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-server" : "default-server", { env, timeoutMs }),
|
|
runCaptured(["kubectl", "get", "nodes", "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-nodes" : "default-nodes", { env, timeoutMs }),
|
|
runCaptured(["kubectl", "get", "namespace", namespace, "-o", "name"], process.cwd(), dumpDir, withExplicitEnv ? "k3s-namespace" : "default-namespace", { env, timeoutMs }),
|
|
]);
|
|
const combinedText = [context.stdoutText, context.stderrText, server.stdoutText, server.stderrText, nodes.stdoutText, nodes.stderrText, namespaceCheck.stdoutText, namespaceCheck.stderrText].join("\n");
|
|
return {
|
|
checked: true,
|
|
explicit: withExplicitEnv,
|
|
currentContext: context.stdoutText.trim() || null,
|
|
apiServer: server.stdoutText.trim() || null,
|
|
nodeNames: nodes.stdoutText.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
|
namespaceObserved: namespaceCheck.ok,
|
|
commandsOk: context.ok && server.ok && nodes.ok,
|
|
refusalSignals: forbiddenKubeSignals(combinedText),
|
|
commands: [context, server, nodes, namespaceCheck].map(commandView),
|
|
};
|
|
}
|
|
|
|
async function nodeGuard(kubeconfig, dumpDir, timeoutMs) {
|
|
const [explicit, bare] = await Promise.all([
|
|
observeKube(kubeconfig, dumpDir, timeoutMs, true),
|
|
observeKube(kubeconfig, dumpDir, timeoutMs, false),
|
|
]);
|
|
const wrongKubeconfig = kubeconfig !== nativeKubeconfig;
|
|
const requiredNodePresent = explicit.nodeNames.includes(requiredNodeName);
|
|
const explicitRefusal = explicit.refusalSignals.length > 0;
|
|
const secondControlPlaneRisk = Boolean(
|
|
bare.namespaceObserved &&
|
|
explicit.apiServer &&
|
|
bare.apiServer &&
|
|
bare.apiServer !== explicit.apiServer
|
|
);
|
|
const status = explicitRefusal
|
|
? "refused"
|
|
: wrongKubeconfig || !explicit.commandsOk || !requiredNodePresent
|
|
? "blocked"
|
|
: "pass";
|
|
return {
|
|
status,
|
|
refusal: explicitRefusal,
|
|
refusalSignals: explicit.refusalSignals,
|
|
kubeconfig,
|
|
expectedKubeconfig: nativeKubeconfig,
|
|
currentContext: explicit.currentContext,
|
|
apiServer: explicit.apiServer,
|
|
nodeNames: explicit.nodeNames,
|
|
nodeCount: explicit.nodeNames.length,
|
|
requiredNodeName,
|
|
requiredNodePresent,
|
|
commandsOk: explicit.commandsOk,
|
|
secondControlPlaneRisk,
|
|
defaultKubectlDiagnostic: {
|
|
checked: true,
|
|
currentContext: bare.currentContext,
|
|
apiServer: bare.apiServer,
|
|
nodeNames: bare.nodeNames,
|
|
namespaceObserved: bare.namespaceObserved,
|
|
refusalSignals: bare.refusalSignals,
|
|
status: bare.refusalSignals.length > 0 ? "stale-forbidden-default" : bare.commandsOk ? "clean" : "unavailable",
|
|
secondControlPlaneRisk,
|
|
summary: secondControlPlaneRisk
|
|
? "Bare kubectl can observe hwlab-dev through a different control plane; write paths must stop."
|
|
: bare.refusalSignals.length > 0
|
|
? "Bare kubectl shows a forbidden local control-plane signal; explicit D601 KUBECONFIG remains the target."
|
|
: "Bare kubectl did not prove a second HWLAB DEV control plane.",
|
|
},
|
|
summary: explicitRefusal
|
|
? "Refusing because explicit D601 kubeconfig resolved to a forbidden Docker Desktop control-plane signal."
|
|
: wrongKubeconfig
|
|
? `Expected KUBECONFIG=${nativeKubeconfig}.`
|
|
: !explicit.commandsOk
|
|
? "Explicit D601 kubeconfig could not read context, server, and nodes."
|
|
: !requiredNodePresent
|
|
? "Explicit D601 kubeconfig did not report node d601."
|
|
: "D601 native k3s guard passed with explicit KUBECONFIG.",
|
|
commands: [...explicit.commands, ...bare.commands],
|
|
};
|
|
}
|
|
|
|
async function lockStatus(kubeconfig, guard, dumpDir, timeoutMs) {
|
|
if (guard.status !== "pass") {
|
|
return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, lockName, present: null };
|
|
}
|
|
const env = { ...process.env, KUBECONFIG: kubeconfig };
|
|
const result = await runCaptured(["kubectl", "-n", namespace, "get", "lease", lockName, "-o", "json"], process.cwd(), dumpDir, "cd-lock", { env, timeoutMs });
|
|
const text = `${result.stdoutText}\n${result.stderrText}`;
|
|
if (!result.ok && /not\s*found|notfound/iu.test(text)) {
|
|
return { status: "absent", namespace, lockName, present: false, held: false, stale: false, command: commandView(result) };
|
|
}
|
|
const parsed = asRecord(parseJson(result.stdoutText));
|
|
const metadata = asRecord(parsed?.metadata) || {};
|
|
const spec = asRecord(parsed?.spec) || {};
|
|
const annotations = asRecord(metadata.annotations) || {};
|
|
const annotation = (name) => stringValue(annotations[`hwlab.pikastech.local/${name}`]);
|
|
const lock = parsed === null ? null : {
|
|
lockBackend: "Lease",
|
|
lockName: stringValue(metadata.name) || lockName,
|
|
namespace: stringValue(metadata.namespace) || namespace,
|
|
holderIdentity: stringValue(spec.holderIdentity),
|
|
resourceVersion: stringValue(metadata.resourceVersion),
|
|
ownerTaskId: annotation("ownerTaskId"),
|
|
transactionId: annotation("transactionId"),
|
|
phase: annotation("phase") || "unknown",
|
|
promotionCommit: annotation("promotionCommit"),
|
|
deployJsonHash: annotation("deployJsonHash"),
|
|
targetRef: annotation("targetRef"),
|
|
targetNamespace: annotation("targetNamespace"),
|
|
updatedAt: annotation("updatedAt") || stringValue(spec.renewTime),
|
|
ttlSeconds: Number(annotation("ttlSeconds") || spec.leaseDurationSeconds || 3600),
|
|
releaseStatus: annotation("releaseStatus"),
|
|
releasedAt: annotation("releasedAt"),
|
|
};
|
|
const updatedAt = stringValue(lock?.updatedAt) || "";
|
|
const ttlSeconds = Number(lock?.ttlSeconds || 3600);
|
|
const expiresAtMs = Number.isFinite(Date.parse(updatedAt)) ? Date.parse(updatedAt) + ttlSeconds * 1000 : 0;
|
|
const retryAfterSeconds = Math.max(0, Math.ceil((expiresAtMs - Date.now()) / 1000));
|
|
return {
|
|
status: result.ok && lock !== null ? "observed" : "blocked",
|
|
present: result.ok && lock !== null,
|
|
...(lock || {}),
|
|
held: lock !== null && lock.phase !== "released" && retryAfterSeconds > 0,
|
|
stale: lock !== null && lock.phase !== "released" && retryAfterSeconds <= 0,
|
|
retryAfterSeconds,
|
|
expiresAt: expiresAtMs > 0 ? new Date(expiresAtMs).toISOString() : null,
|
|
command: commandView(result),
|
|
};
|
|
}
|
|
|
|
function parseSecretDescribeKeys(stdout) {
|
|
const keys = new Set();
|
|
let inData = false;
|
|
for (const rawLine of String(stdout || "").split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (/^Data\b/iu.test(line)) {
|
|
inData = true;
|
|
continue;
|
|
}
|
|
if (!inData || line === "" || /^=+$/u.test(line)) continue;
|
|
const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu);
|
|
if (match) keys.add(match[1]);
|
|
}
|
|
return [...keys].sort();
|
|
}
|
|
|
|
async function secretPreflight(kubeconfig, guard, dumpDir, timeoutMs) {
|
|
if (guard.status !== "pass") {
|
|
return {
|
|
status: "skipped",
|
|
reason: "d601-native-k3s-guard-not-pass",
|
|
namespace,
|
|
mutation: false,
|
|
safety: { secretValuesRead: false, secretValuesPrinted: false, secretKeyNamesOnly: true },
|
|
secretRefs: requiredSecretRefs.map((ref) => ({ ...ref, status: "not_checked" })),
|
|
blockers: [],
|
|
};
|
|
}
|
|
const env = { ...process.env, KUBECONFIG: kubeconfig };
|
|
const observations = [];
|
|
for (const ref of requiredSecretRefs) {
|
|
const idBase = `secretref-${ref.secretName}-${ref.secretKey}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
|
const exists = await runCaptured(["kubectl", "-n", namespace, "get", "secret", ref.secretName, "-o", "name"], process.cwd(), dumpDir, `${idBase}-exists`, { env, timeoutMs });
|
|
if (!exists.ok) {
|
|
observations.push({ ...ref, status: "missing-secret", exists: false, keyPresent: false, command: commandView(exists) });
|
|
continue;
|
|
}
|
|
const describe = await runCaptured(["kubectl", "-n", namespace, "describe", "secret", ref.secretName], process.cwd(), dumpDir, `${idBase}-describe`, { env, timeoutMs });
|
|
const keysObserved = parseSecretDescribeKeys(describe.stdoutText);
|
|
const keyPresent = keysObserved.includes(ref.secretKey);
|
|
observations.push({
|
|
...ref,
|
|
status: keyPresent ? "present" : describe.ok ? "missing-key" : "key-observation-blocked",
|
|
exists: true,
|
|
keyPresent,
|
|
keysObserved,
|
|
command: commandView(describe),
|
|
});
|
|
}
|
|
const blockers = observations
|
|
.filter((observation) => observation.status !== "present")
|
|
.map((observation) => ({
|
|
scope: `secretref:${observation.secretName}/${observation.secretKey}`,
|
|
summary: observation.status === "missing-secret"
|
|
? `Required Secret ${observation.secretName} is missing in ${namespace}.`
|
|
: `Required SecretRef ${observation.secretName}/${observation.secretKey} is not present as key metadata in ${namespace}.`,
|
|
impact: `${observation.consumers.join(", ")} would fail after a DEV CD apply or runtime preflight Job.`,
|
|
runbook: `Create or repair Secret ${observation.secretName} with key ${observation.secretKey} in namespace ${namespace}; verify key presence only and do not print the Secret value.`,
|
|
}));
|
|
return {
|
|
status: blockers.length === 0 ? "pass" : "blocked",
|
|
namespace,
|
|
mutation: false,
|
|
safety: { secretValuesRead: false, secretValuesPrinted: false, secretKeyNamesOnly: true },
|
|
requiredSecretRefs: requiredSecretRefs.map((ref) => `${ref.secretName}/${ref.secretKey}`),
|
|
secretRefs: observations,
|
|
blockers,
|
|
};
|
|
}
|
|
|
|
async function registryAudit(kubeconfig, guard, dumpDir, timeoutMs) {
|
|
const result = await runCaptured(["curl", "-fsS", "--max-time", "5", "http://127.0.0.1:5000/v2/"], process.cwd(), dumpDir, "registry-v2", { timeoutMs: Math.min(timeoutMs, 8000) });
|
|
const reachable = result.ok || /401|unauthorized/iu.test(`${result.stdoutText}\n${result.stderrText}`);
|
|
let k3sPullAccess = { status: "skipped", reason: "d601-native-k3s-guard-not-pass", pullFailureCount: null, pulledRegistryImageCount: null };
|
|
if (guard.status === "pass") {
|
|
const env = { ...process.env, KUBECONFIG: kubeconfig };
|
|
const pods = await runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "registry-k3s-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) });
|
|
const parsed = asRecord(parseJson(pods.stdoutText));
|
|
const items = Array.isArray(parsed?.items) ? parsed.items : [];
|
|
let pullFailureCount = 0;
|
|
let pulledRegistryImageCount = 0;
|
|
for (const item of items) {
|
|
const statuses = Array.isArray(item?.status?.containerStatuses) ? item.status.containerStatuses : [];
|
|
for (const status of statuses) {
|
|
const image = String(status?.image || "");
|
|
const imageID = String(status?.imageID || "");
|
|
const waiting = String(status?.state?.waiting?.reason || "");
|
|
if (/ErrImagePull|ImagePullBackOff|InvalidImageName/u.test(waiting)) pullFailureCount += 1;
|
|
if (image.includes("127.0.0.1:5000/") && imageID.includes("sha256:")) pulledRegistryImageCount += 1;
|
|
}
|
|
}
|
|
k3sPullAccess = {
|
|
status: !pods.ok ? "unavailable" : pullFailureCount > 0 ? "blocked" : pulledRegistryImageCount > 0 ? "pass" : "degraded",
|
|
pullFailureCount,
|
|
pulledRegistryImageCount,
|
|
command: commandView(pods),
|
|
};
|
|
}
|
|
return {
|
|
status: reachable && !["blocked", "unavailable"].includes(k3sPullAccess.status) ? "pass" : reachable ? "degraded" : "blocked",
|
|
endpoint: "http://127.0.0.1:5000/v2/",
|
|
processHttpAccess: {
|
|
status: reachable ? "pass" : "blocked",
|
|
reachable,
|
|
exitCode: result.exitCode,
|
|
command: commandView(result),
|
|
},
|
|
k3sPullAccess,
|
|
};
|
|
}
|
|
|
|
function deploymentCondition(conditions, type) {
|
|
return (Array.isArray(conditions) ? conditions : []).find((condition) => condition?.type === type) || null;
|
|
}
|
|
|
|
function envValue(container, name) {
|
|
const item = (Array.isArray(container?.env) ? container.env : []).find((entry) => entry?.name === name);
|
|
return item?.value ?? null;
|
|
}
|
|
|
|
async function workloadAudit(kubeconfig, guard, desired, dumpDir, timeoutMs) {
|
|
if (guard.status !== "pass") {
|
|
return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, deployments: [], currentImageConvergence: { status: "unavailable", unavailableReason: "d601-native-k3s-guard-not-pass" } };
|
|
}
|
|
const env = { ...process.env, KUBECONFIG: kubeconfig };
|
|
const [deployments, pods, jobs] = await Promise.all([
|
|
runCaptured(["kubectl", "-n", namespace, "get", "deployments", "-o", "json"], process.cwd(), dumpDir, "workload-deployments", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
|
|
runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "workload-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
|
|
runCaptured(["kubectl", "-n", namespace, "get", "jobs", "-o", "json"], process.cwd(), dumpDir, "runtime-jobs", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
|
|
]);
|
|
const desiredById = new Map((desired.services || []).map((service) => [service.serviceId, service]));
|
|
const deploymentItems = Array.isArray(parseJson(deployments.stdoutText)?.items) ? parseJson(deployments.stdoutText).items : [];
|
|
const summaries = deploymentItems.map((deployment) => {
|
|
const metadata = asRecord(deployment.metadata) || {};
|
|
const spec = asRecord(deployment.spec) || {};
|
|
const status = asRecord(deployment.status) || {};
|
|
const template = asRecord(spec.template) || {};
|
|
const podSpec = asRecord(template.spec) || {};
|
|
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
|
const container = containers[0] || {};
|
|
const labels = asRecord(metadata.labels) || {};
|
|
const serviceId = stringValue(labels["hwlab.pikastech.local/service-id"]) || stringValue(labels["app.kubernetes.io/name"]) || stringValue(metadata.name);
|
|
const desiredService = desiredById.get(serviceId);
|
|
const image = stringValue(container.image);
|
|
const desiredImage = desiredService?.image || null;
|
|
const available = Number(status.availableReplicas || 0);
|
|
const desiredReplicas = Number(spec.replicas || 0);
|
|
const updated = Number(status.updatedReplicas || 0);
|
|
const unavailable = Number(status.unavailableReplicas || 0);
|
|
const progressing = deploymentCondition(status.conditions, "Progressing");
|
|
const availableCondition = deploymentCondition(status.conditions, "Available");
|
|
return {
|
|
name: stringValue(metadata.name),
|
|
serviceId,
|
|
image,
|
|
imageTag: imageTag(image),
|
|
desiredImage,
|
|
imageMatchesDesired: desiredImage ? image === desiredImage : null,
|
|
revision: stringValue(asRecord(metadata.annotations)?.["deployment.kubernetes.io/revision"]),
|
|
replicas: desiredReplicas,
|
|
availableReplicas: available,
|
|
updatedReplicas: updated,
|
|
unavailableReplicas: unavailable,
|
|
rolloutStatus: desiredReplicas === 0 || (available >= desiredReplicas && updated >= desiredReplicas && unavailable === 0) ? "healthy" : "unhealthy",
|
|
conditions: {
|
|
available: availableCondition ? { status: availableCondition.status ?? null, reason: availableCondition.reason ?? null } : null,
|
|
progressing: progressing ? { status: progressing.status ?? null, reason: progressing.reason ?? null } : null,
|
|
},
|
|
commitEnv: envValue(container, "HWLAB_COMMIT_ID"),
|
|
};
|
|
});
|
|
const podItems = Array.isArray(parseJson(pods.stdoutText)?.items) ? parseJson(pods.stdoutText).items : [];
|
|
const waiting = [];
|
|
for (const pod of podItems) {
|
|
const podName = stringValue(pod?.metadata?.name);
|
|
for (const status of Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []) {
|
|
const reason = stringValue(status?.state?.waiting?.reason);
|
|
if (reason) waiting.push({ pod: podName, container: status.name ?? null, reason, image: status.image ?? null });
|
|
}
|
|
}
|
|
const jobItems = Array.isArray(parseJson(jobs.stdoutText)?.items) ? parseJson(jobs.stdoutText).items : [];
|
|
const runtimeJobs = jobItems
|
|
.filter((job) => /hwlab-runtime|runtime-db|runtime/i.test(String(job?.metadata?.name || "")))
|
|
.map((job) => ({
|
|
name: stringValue(job?.metadata?.name),
|
|
succeeded: Number(job?.status?.succeeded || 0),
|
|
failed: Number(job?.status?.failed || 0),
|
|
active: Number(job?.status?.active || 0),
|
|
completionTime: stringValue(job?.status?.completionTime),
|
|
status: Number(job?.status?.failed || 0) > 0 ? "blocked" : Number(job?.status?.active || 0) > 0 ? "running" : Number(job?.status?.succeeded || 0) > 0 ? "pass" : "unknown",
|
|
}))
|
|
.slice(0, 12);
|
|
const imageMismatches = summaries.filter((deployment) => deployment.imageMatchesDesired === false);
|
|
const unhealthy = summaries.filter((deployment) => deployment.rolloutStatus === "unhealthy");
|
|
return {
|
|
status: !deployments.ok ? "unavailable" : unhealthy.length > 0 ? "unhealthy" : imageMismatches.length > 0 ? "mismatch" : "healthy",
|
|
namespace,
|
|
deploymentCount: summaries.length,
|
|
deployments: summaries.slice(0, 30),
|
|
currentImageConvergence: {
|
|
status: imageMismatches.length === 0 ? "pass" : "mismatch",
|
|
mismatches: imageMismatches.map((deployment) => ({ serviceId: deployment.serviceId, image: deployment.image, desiredImage: deployment.desiredImage })).slice(0, 12),
|
|
},
|
|
podWaiting: waiting.slice(0, 12),
|
|
runtimeJobs,
|
|
commands: [deployments, pods, jobs].map(commandView),
|
|
};
|
|
}
|
|
|
|
function compactHealth(body, expectedServiceId, expectedCommit) {
|
|
const json = asRecord(body);
|
|
const commit = stringValue(asRecord(json?.commit)?.id) || stringValue(json?.commitId) || stringValue(json?.revision) || stringValue(asRecord(json?.image)?.tag);
|
|
const runtime = asRecord(json?.runtime);
|
|
const db = asRecord(json?.db);
|
|
const blockerCodes = Array.isArray(json?.blockerCodes) ? json.blockerCodes.map(String).slice(0, 12) : [];
|
|
return {
|
|
serviceId: stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id),
|
|
environment: stringValue(json?.environment),
|
|
status: stringValue(json?.status),
|
|
ready: typeof json?.ready === "boolean" ? json.ready : null,
|
|
observedCommit: commit,
|
|
commitMatches: expectedCommit ? commitMatches(commit, expectedCommit) : null,
|
|
serviceMatches: expectedServiceId ? (stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id)) === expectedServiceId : null,
|
|
image: typeof json?.image === "string" ? json.image : stringValue(asRecord(json?.image)?.reference),
|
|
blockerCodes,
|
|
db: db === null ? null : {
|
|
ready: db.ready ?? null,
|
|
connected: db.connected ?? null,
|
|
liveDbEvidence: db.liveDbEvidence ?? null,
|
|
runtimeReadiness: asRecord(db.runtimeReadiness) ? {
|
|
status: stringValue(db.runtimeReadiness.status),
|
|
ready: db.runtimeReadiness.ready ?? null,
|
|
blocker: stringValue(db.runtimeReadiness.blocker),
|
|
} : null,
|
|
},
|
|
runtime: runtime === null ? null : {
|
|
status: stringValue(runtime.status),
|
|
ready: runtime.ready ?? null,
|
|
durable: runtime.durable ?? null,
|
|
durableRequested: runtime.durableRequested ?? null,
|
|
liveRuntimeEvidence: runtime.liveRuntimeEvidence ?? null,
|
|
blocker: stringValue(runtime.blocker),
|
|
adapter: stringValue(runtime.adapter),
|
|
},
|
|
};
|
|
}
|
|
|
|
async function httpHealthAudit(desired, dumpDir, timeoutMs) {
|
|
const endpoints = [
|
|
{ id: "cloud-web-16666", url: desired.deployJson.publicEndpoints.frontend || "http://74.48.78.17:16666", expectedServiceId: "hwlab-cloud-web" },
|
|
{ id: "cloud-api-16667", url: desired.deployJson.publicEndpoints.api || desired.deployJson.endpoint || "http://74.48.78.17:16667", expectedServiceId: "hwlab-cloud-api" },
|
|
];
|
|
const expectedCommit = desired.targetCommit || desired.deployJson.commitId || null;
|
|
const observations = [];
|
|
for (const endpoint of endpoints) {
|
|
const result = await runCaptured(["curl", "-fsS", "--max-time", "8", `${endpoint.url.replace(/\/+$/u, "")}/health/live`], process.cwd(), dumpDir, `public-health-${endpoint.id}`, { timeoutMs: Math.min(timeoutMs, 10000) });
|
|
const parsed = parseJson(result.stdoutText);
|
|
const health = compactHealth(parsed, endpoint.expectedServiceId, expectedCommit);
|
|
const reachable = result.ok && parsed !== null;
|
|
observations.push({
|
|
id: endpoint.id,
|
|
url: endpoint.url,
|
|
status: reachable && health.serviceMatches !== false && health.commitMatches !== false ? "pass" : "blocked",
|
|
reachable,
|
|
expectedServiceId: endpoint.expectedServiceId,
|
|
expectedCommit,
|
|
health,
|
|
command: commandView(result),
|
|
});
|
|
}
|
|
return {
|
|
status: observations.every((item) => item.status === "pass") ? "pass" : "blocked",
|
|
expectedCommit,
|
|
summary: {
|
|
checked: observations.length,
|
|
reachable: observations.filter((item) => item.reachable).length,
|
|
commitMatches: observations.filter((item) => item.health.commitMatches === true).length,
|
|
readinessPass: observations.filter((item) => item.health.ready !== false && !["blocked", "failed"].includes(String(item.health.status || ""))).length,
|
|
ports: [16666, 16667],
|
|
},
|
|
endpoints: observations,
|
|
};
|
|
}
|
|
|
|
function runtimeDurabilityAudit(desired, publicHealth, secretRefs, workload) {
|
|
const api = publicHealth?.endpoints?.find((endpoint) => endpoint.id === "cloud-api-16667")?.health || null;
|
|
const cloudApiEnv = asRecord(desired.serviceEnv?.cloudApi) || {};
|
|
const configuredDurable = cloudApiEnv.HWLAB_CLOUD_RUNTIME_DURABLE === "true";
|
|
const configuredAdapter = stringValue(cloudApiEnv.HWLAB_CLOUD_RUNTIME_ADAPTER);
|
|
const dbSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db" && ref.secretKey === "database-url");
|
|
const adminSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db-admin" && ref.secretKey === "admin-url");
|
|
const runtime = asRecord(api?.runtime);
|
|
const dbRuntimeReadiness = asRecord(api?.db?.runtimeReadiness);
|
|
const liveReady = runtime?.durable === true && runtime?.ready !== false && runtime?.liveRuntimeEvidence === true && !["blocked", "degraded", "failed"].includes(String(runtime?.status || ""));
|
|
const status = liveReady
|
|
? "pass"
|
|
: api
|
|
? "risk"
|
|
: configuredDurable && configuredAdapter === "postgres" && dbSecret?.keyPresent === true && adminSecret?.keyPresent === true
|
|
? "unavailable"
|
|
: "risk";
|
|
return {
|
|
status,
|
|
configured: {
|
|
adapter: configuredAdapter,
|
|
durable: configuredDurable,
|
|
dbSecretPresent: dbSecret?.keyPresent === true,
|
|
adminSecretPresent: adminSecret?.keyPresent === true,
|
|
},
|
|
live: api ? {
|
|
db: api.db,
|
|
runtime: api.runtime,
|
|
dbRuntimeReadiness,
|
|
} : null,
|
|
workloadEvidence: {
|
|
cloudApiDeployment: (workload?.deployments || []).find((deployment) => deployment.serviceId === "hwlab-cloud-api") || null,
|
|
},
|
|
unavailableReason: api ? null : "public-health-cloud-api-unavailable",
|
|
risk: status === "pass" ? null : "db-runtime-durability-risk",
|
|
};
|
|
}
|
|
|
|
function blocker(scope, summary, details = {}) {
|
|
return { scope, summary, ...details };
|
|
}
|
|
|
|
function classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }) {
|
|
const blockers = [];
|
|
if (guard?.status !== "pass") blockers.push(blocker("control-plane-unavailable", guard?.summary || "D601 native k3s guard did not pass."));
|
|
if (guard?.refusalSignals?.length > 0) blockers.push(blocker("docker-desktop-context-risk", "Explicit D601 kubeconfig resolved to forbidden Docker Desktop control-plane signal.", { refusalSignals: guard.refusalSignals }));
|
|
if (guard?.secondControlPlaneRisk) blockers.push(blocker("second-control-plane-risk", "Bare kubectl can observe hwlab-dev through a different control plane; audit is read-only and will not release or mutate locks."));
|
|
if (repo.status !== "selected") blockers.push(blocker("workspace-unavailable", "No eligible HWLAB CD repo was selected."));
|
|
for (const item of git?.blockers || []) {
|
|
const scope = item.scope === "hwlab-git-clean" ? "dirty-worktree" : item.scope?.startsWith("hwlab-git") || item.scope?.includes("permission") ? "workspace-unavailable" : item.scope;
|
|
blockers.push(blocker(scope, item.summary));
|
|
}
|
|
for (const item of secretRefs?.blockers || []) blockers.push(blocker("secret-missing", item.summary, { secretRef: item.scope }));
|
|
if (registry?.status === "blocked" || registry?.status === "degraded") blockers.push(blocker("registry-unavailable", "Registry reachability or k3s pull evidence is not healthy.", { status: registry.status }));
|
|
if (lock?.held === true) blockers.push(blocker("lease-held", `HWLAB DEV CD Lease is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.`, { phase: lock.phase, retryAfterSeconds: lock.retryAfterSeconds }));
|
|
if (lock?.stale === true) blockers.push(blocker("lease-stale-candidate", "HWLAB DEV CD Lease appears stale; audit is read-only and did not release or break it.", { phase: lock.phase }));
|
|
if (desired?.status === "missing") blockers.push(blocker("artifact-missing", "deploy/deploy.json or artifact catalog/report is missing or incomplete.", { imageConvergence: desired.imageConvergence }));
|
|
if (desired?.status === "mismatch" || workload?.currentImageConvergence?.status === "mismatch") blockers.push(blocker("artifact-mismatch", "deploy/deploy.json, artifact catalog, or current workload images are not converged.", { desired: desired?.status, currentImageConvergence: workload?.currentImageConvergence }));
|
|
if ((workload?.runtimeJobs || []).some((job) => job.status === "blocked" || job.status === "running")) blockers.push(blocker("runtime-job-blocked", "Runtime DB maintenance Job is blocked or still running.", { runtimeJobs: workload.runtimeJobs }));
|
|
if (workload?.status === "unhealthy") blockers.push(blocker("rollout-unhealthy", "One or more hwlab-dev Deployments are not fully available.", { deployments: (workload.deployments || []).filter((item) => item.rolloutStatus === "unhealthy").slice(0, 8) }));
|
|
if (publicHealth?.status === "blocked") blockers.push(blocker("public-tunnel-unhealthy", "16666/16667 public health did not pass read-only commit/readiness checks.", { summary: publicHealth.summary }));
|
|
if (durability?.status === "risk") blockers.push(blocker("db-runtime-durability-risk", "DB/runtime durability is not proven by live cloud-api health.", { live: durability.live, configured: durability.configured }));
|
|
if (controlled?.commandOk === false || controlled?.status === "blocked") blockers.push(blocker("runtime-job-blocked", "HWLAB repo-owned dev-cd-apply status reported blocked or failed.", { status: controlled?.status }));
|
|
return blockers;
|
|
}
|
|
|
|
function auditSummary({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled, dumpDir }) {
|
|
const blockers = classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled });
|
|
const blockerTypes = [...new Set(blockers.map((item) => item.scope))];
|
|
return {
|
|
ok: blockers.length === 0,
|
|
status: blockers.length === 0 ? "pass" : guard?.refusal ? "refused" : "blocked",
|
|
env: "dev",
|
|
namespace,
|
|
nodeGuard: {
|
|
status: guard?.status ?? "unavailable",
|
|
nodeName: guard?.requiredNodeName ?? requiredNodeName,
|
|
nodeNames: guard?.nodeNames ?? [],
|
|
requiredNodePresent: guard?.requiredNodePresent ?? false,
|
|
currentContext: guard?.currentContext ?? null,
|
|
apiServer: guard?.apiServer ?? null,
|
|
kubeconfig: guard?.kubeconfig ?? nativeKubeconfig,
|
|
refusalSignals: guard?.refusalSignals ?? [],
|
|
secondControlPlaneRisk: guard?.secondControlPlaneRisk ?? false,
|
|
defaultKubectlDiagnostic: guard?.defaultKubectlDiagnostic ?? null,
|
|
},
|
|
secrets: {
|
|
status: secretRefs?.status ?? "unavailable",
|
|
valuesRead: false,
|
|
valuesPrinted: false,
|
|
secretRefs: (secretRefs?.secretRefs || []).map((ref) => ({
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
exists: ref.exists === true,
|
|
keyPresent: ref.keyPresent === true,
|
|
status: ref.status,
|
|
})),
|
|
unavailableReason: secretRefs?.reason ?? null,
|
|
},
|
|
registry: {
|
|
status: registry?.status ?? "unavailable",
|
|
endpoint: registry?.endpoint ?? "http://127.0.0.1:5000/v2/",
|
|
processHttpAccess: registry?.processHttpAccess ? { status: registry.processHttpAccess.status, reachable: registry.processHttpAccess.reachable } : null,
|
|
k3sPullAccess: registry?.k3sPullAccess ? {
|
|
status: registry.k3sPullAccess.status,
|
|
pullFailureCount: registry.k3sPullAccess.pullFailureCount,
|
|
pulledRegistryImageCount: registry.k3sPullAccess.pulledRegistryImageCount,
|
|
} : null,
|
|
},
|
|
lease: {
|
|
status: lock?.status ?? "unavailable",
|
|
phase: lock?.phase ?? null,
|
|
holder: lock?.ownerTaskId || lock?.holderIdentity || null,
|
|
ageSeconds: lock?.updatedAt && Number.isFinite(Date.parse(lock.updatedAt)) ? Math.max(0, Math.floor((Date.now() - Date.parse(lock.updatedAt)) / 1000)) : null,
|
|
retryAfterSeconds: lock?.retryAfterSeconds ?? null,
|
|
staleClassification: lock?.stale === true ? "lease-stale-candidate" : lock?.held === true ? "lease-held" : "not-held",
|
|
lockName: lock?.lockName ?? lockName,
|
|
},
|
|
desiredState: {
|
|
status: desired?.status ?? "unavailable",
|
|
targetCommit: desired?.targetCommit ?? null,
|
|
deployJson: desired?.deployJson ?? null,
|
|
artifactCatalog: desired?.artifactCatalog ?? null,
|
|
artifactReport: desired?.artifactReport ?? null,
|
|
imageConvergence: desired?.imageConvergence ?? null,
|
|
},
|
|
workload: {
|
|
status: workload?.status ?? "unavailable",
|
|
currentImageConvergence: workload?.currentImageConvergence ?? null,
|
|
deploymentCount: workload?.deploymentCount ?? 0,
|
|
deployments: (workload?.deployments || []).map((deployment) => ({
|
|
name: deployment.name,
|
|
serviceId: deployment.serviceId,
|
|
image: deployment.image,
|
|
imageTag: deployment.imageTag,
|
|
desiredImage: deployment.desiredImage,
|
|
revision: deployment.revision,
|
|
replicas: deployment.replicas,
|
|
availableReplicas: deployment.availableReplicas,
|
|
rolloutStatus: deployment.rolloutStatus,
|
|
commitEnv: deployment.commitEnv,
|
|
})),
|
|
podWaiting: workload?.podWaiting ?? [],
|
|
runtimeJobs: workload?.runtimeJobs ?? [],
|
|
},
|
|
publicHealth: {
|
|
status: publicHealth?.status ?? "unavailable",
|
|
expectedCommit: publicHealth?.expectedCommit ?? null,
|
|
summary: publicHealth?.summary ?? { checked: 0 },
|
|
endpoints: (publicHealth?.endpoints || []).map((endpoint) => ({
|
|
id: endpoint.id,
|
|
url: endpoint.url,
|
|
status: endpoint.status,
|
|
reachable: endpoint.reachable,
|
|
expectedServiceId: endpoint.expectedServiceId,
|
|
expectedCommit: endpoint.expectedCommit,
|
|
health: endpoint.health,
|
|
})),
|
|
},
|
|
durability: {
|
|
status: durability?.status ?? "unavailable",
|
|
configured: durability?.configured ?? null,
|
|
live: durability?.live ?? null,
|
|
unavailableReason: durability?.unavailableReason ?? null,
|
|
},
|
|
controlledDevCd: {
|
|
status: controlled?.status ?? "unavailable",
|
|
commandOk: controlled?.commandOk ?? null,
|
|
controlledEntrypoint: "scripts/dev-cd-apply.mjs",
|
|
target: controlled?.target ?? null,
|
|
blockers: controlled?.blockers ?? [],
|
|
},
|
|
blockers,
|
|
blockerTypes,
|
|
dumpPath: dumpDir,
|
|
safety: {
|
|
mutation: false,
|
|
kubectlApplyExecuted: false,
|
|
rolloutExecuted: false,
|
|
liveVerifyExecuted: false,
|
|
cdLockMutated: false,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false,
|
|
reportsWritten: false,
|
|
repoReportsWritten: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function summarizeControlled(parsed, command) {
|
|
const record = asRecord(parsed);
|
|
const target = asRecord(record?.target);
|
|
const desiredStateCheck = asRecord(target?.desiredStateCheck);
|
|
const artifactBoundary = asRecord(target?.artifactBoundary);
|
|
return {
|
|
status: stringValue(record?.status) || (command.ok ? "unknown" : "blocked"),
|
|
commandOk: command.ok,
|
|
mode: stringValue(record?.mode),
|
|
mutationAttempted: record?.mutationAttempted ?? false,
|
|
prodTouched: record?.prodTouched ?? false,
|
|
controlledEntrypoint: "scripts/dev-cd-apply.mjs",
|
|
target: target === null ? null : {
|
|
ref: stringValue(target.ref),
|
|
promotionCommit: stringValue(target.promotionCommit),
|
|
shortCommitId: stringValue(target.shortCommitId),
|
|
promotionSource: stringValue(target.promotionSource),
|
|
publishRequired: target.publishRequired ?? null,
|
|
headCommitId: stringValue(target.headCommitId),
|
|
headMatchesTarget: target.headMatchesTarget ?? null,
|
|
desiredStateCheck: desiredStateCheck === null ? null : {
|
|
status: stringValue(desiredStateCheck.status),
|
|
summary: desiredStateCheck.summary ?? null,
|
|
diagnostics: Array.isArray(desiredStateCheck.diagnostics) ? desiredStateCheck.diagnostics.slice(0, 5) : [],
|
|
},
|
|
artifactBoundary: artifactBoundary === null ? null : {
|
|
status: stringValue(artifactBoundary.status),
|
|
desiredState: artifactBoundary.desiredState ?? null,
|
|
},
|
|
namespace: stringValue(target.namespace) || namespace,
|
|
},
|
|
deployJson: asRecord(record?.deployJson),
|
|
artifactCatalog: asRecord(record?.artifactCatalog),
|
|
artifactReport: asRecord(record?.artifactReport),
|
|
lock: asRecord(record?.lock),
|
|
liveDelta: asRecord(record?.liveDelta),
|
|
blockers: Array.isArray(record?.blockers) ? record.blockers.slice(0, 12) : [],
|
|
nextActions: Array.isArray(record?.nextActions) ? record.nextActions.slice(0, 12) : [],
|
|
parsed: record !== null,
|
|
command: commandView(command),
|
|
};
|
|
}
|
|
|
|
async function runDevCdApply(repoPath, kubeconfig, action, dumpDir, timeoutMs) {
|
|
const modeArg = action === "status" ? "--status" : "--dry-run";
|
|
const command = ["node", "scripts/dev-cd-apply.mjs", modeArg, "--kubeconfig", kubeconfig, "--skip-live-verify"];
|
|
const result = await runCaptured(command, repoPath, dumpDir, `controlled-dev-cd-apply-${action === "status" ? "status" : "dry-run"}`, {
|
|
env: { ...process.env, KUBECONFIG: kubeconfig },
|
|
timeoutMs,
|
|
});
|
|
return summarizeControlled(parseJson(result.stdoutText), result);
|
|
}
|
|
|
|
function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action }) {
|
|
const blockers = [];
|
|
if (repo.status !== "selected") {
|
|
blockers.push({ scope: "hwlab-repo", summary: repo.rejected ? "Rejected runner history HWLAB directory." : "No eligible HWLAB CD repo with scripts/dev-cd-apply.mjs and deploy/deploy.json was found." });
|
|
}
|
|
if (repo.cacheRefreshStatus === "blocked") {
|
|
blockers.push({
|
|
scope: "hwlab-cache-refresh",
|
|
summary: "Dedicated HWLAB git cache could not refresh from GitHub through the provider egress proxy; using the last local cache state is not a release-truth PASS.",
|
|
remoteMainCommit: repo.remoteMainCommit || null,
|
|
cacheMainCommit: repo.cacheMainCommit || null,
|
|
});
|
|
}
|
|
if (git?.blockers) blockers.push(...git.blockers);
|
|
if (guard) {
|
|
if (guard.refusal) blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary, refusal: true });
|
|
else if (guard.status !== "pass") blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary });
|
|
if ((action === "preflight" || action === "audit" || action === "apply") && guard.secondControlPlaneRisk) {
|
|
blockers.push({ scope: "second-hwlab-dev-control-plane", summary: "Bare kubectl can observe hwlab-dev through a different control plane; refusing write-path planning." });
|
|
}
|
|
}
|
|
if ((action === "preflight" || action === "apply") && lock?.held === true) {
|
|
blockers.push({ scope: "cd-lock", summary: `HWLAB DEV CD lock is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.` });
|
|
}
|
|
if (secretRefs?.blockers) blockers.push(...secretRefs.blockers);
|
|
if (controlled) {
|
|
if (controlled.commandOk === false) blockers.push({ scope: "controlled-dev-cd", summary: "HWLAB scripts/dev-cd-apply.mjs did not complete successfully." });
|
|
if (controlled.status === "blocked") blockers.push({ scope: "controlled-dev-cd-status", summary: "HWLAB scripts/dev-cd-apply.mjs reported blocked status." });
|
|
}
|
|
return blockers;
|
|
}
|
|
|
|
function nextSafeCommand(action, blockers) {
|
|
if (blockers.length > 0) return action === "audit" ? "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd audit --env dev" : "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev";
|
|
if (action === "audit") return "audit is read-only; host commander may compare blockerTypes before deciding whether the unique CD runner should continue";
|
|
if (action === "status") return "bun scripts/cli.ts hwlab cd apply --env dev --dry-run";
|
|
return "host commander or the unique CD runner may decide whether to run node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report on D601; this wrapper did not apply or rollout";
|
|
}
|
|
|
|
function compactBlockers(blockers, limit = 12) {
|
|
return (Array.isArray(blockers) ? blockers : []).slice(0, limit).map((item) => ({
|
|
scope: item?.scope ?? item?.blockerScope ?? null,
|
|
type: item?.type ?? null,
|
|
status: item?.status ?? null,
|
|
summary: item?.summary ?? item?.reason ?? null,
|
|
}));
|
|
}
|
|
|
|
function compactGit(git) {
|
|
if (!git) return null;
|
|
return {
|
|
status: git.status,
|
|
clean: git.clean,
|
|
branch: git.branch,
|
|
onMain: git.onMain,
|
|
remoteMatches: git.remoteMatches,
|
|
headMatchesOriginMain: git.headMatchesOriginMain,
|
|
dirtyCount: git.dirtyCount,
|
|
blockers: compactBlockers(git.blockers, 8),
|
|
};
|
|
}
|
|
|
|
function compactNodeGuard(guard) {
|
|
if (!guard) return null;
|
|
return {
|
|
status: guard.status,
|
|
refusal: guard.refusal,
|
|
kubeconfig: guard.kubeconfig,
|
|
currentContext: guard.currentContext,
|
|
apiServer: guard.apiServer,
|
|
nodeNames: guard.nodeNames,
|
|
requiredNodePresent: guard.requiredNodePresent,
|
|
refusalSignals: guard.refusalSignals,
|
|
secondControlPlaneRisk: guard.secondControlPlaneRisk,
|
|
defaultKubectlStatus: guard.defaultKubectlDiagnostic?.status ?? null,
|
|
summary: guard.summary,
|
|
};
|
|
}
|
|
|
|
function compactSecretPreflight(secretRefs) {
|
|
if (!secretRefs) return null;
|
|
return {
|
|
status: secretRefs.status,
|
|
valuesRead: secretRefs.safety?.secretValuesRead === true,
|
|
valuesPrinted: secretRefs.safety?.secretValuesPrinted === true,
|
|
secretRefs: (secretRefs.secretRefs || []).map((ref) => ({
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
exists: ref.exists === true,
|
|
keyPresent: ref.keyPresent === true,
|
|
status: ref.status,
|
|
})).slice(0, 8),
|
|
blockers: compactBlockers(secretRefs.blockers, 8),
|
|
};
|
|
}
|
|
|
|
function compactDesiredState(desired) {
|
|
if (!desired) return null;
|
|
return {
|
|
status: desired.status,
|
|
targetCommit: desired.targetCommit,
|
|
deployJson: desired.deployJson ? {
|
|
hash: desired.deployJson.hash,
|
|
commitId: desired.deployJson.commitId,
|
|
environment: desired.deployJson.environment,
|
|
namespace: desired.deployJson.namespace,
|
|
serviceCount: desired.deployJson.serviceCount,
|
|
} : null,
|
|
artifactCatalog: desired.artifactCatalog ? {
|
|
hash: desired.artifactCatalog.hash,
|
|
commitId: desired.artifactCatalog.commitId,
|
|
artifactState: desired.artifactCatalog.artifactState,
|
|
ciPublished: desired.artifactCatalog.ciPublished,
|
|
registryVerified: desired.artifactCatalog.registryVerified,
|
|
digestCounts: desired.artifactCatalog.digestCounts,
|
|
} : null,
|
|
imageConvergence: desired.imageConvergence ? {
|
|
status: desired.imageConvergence.status,
|
|
missing: (desired.imageConvergence.missing || []).slice(0, 5),
|
|
mismatches: (desired.imageConvergence.mismatches || []).slice(0, 5),
|
|
} : null,
|
|
};
|
|
}
|
|
|
|
function compactControlled(controlled) {
|
|
if (!controlled) return null;
|
|
return {
|
|
status: controlled.status,
|
|
commandOk: controlled.commandOk,
|
|
mode: controlled.mode,
|
|
mutationAttempted: controlled.mutationAttempted,
|
|
prodTouched: controlled.prodTouched,
|
|
target: controlled.target ? {
|
|
ref: controlled.target.ref,
|
|
promotionCommit: controlled.target.promotionCommit,
|
|
shortCommitId: controlled.target.shortCommitId,
|
|
promotionSource: controlled.target.promotionSource,
|
|
publishRequired: controlled.target.publishRequired,
|
|
headMatchesTarget: controlled.target.headMatchesTarget,
|
|
desiredStateStatus: controlled.target.desiredStateCheck?.status ?? null,
|
|
artifactBoundaryStatus: controlled.target.artifactBoundary?.status ?? null,
|
|
namespace: controlled.target.namespace,
|
|
} : null,
|
|
blockers: compactBlockers(controlled.blockers, 8),
|
|
};
|
|
}
|
|
|
|
function compactAudit(audit) {
|
|
if (!audit) return null;
|
|
return {
|
|
ok: audit.ok,
|
|
status: audit.status,
|
|
env: audit.env,
|
|
namespace: audit.namespace,
|
|
nodeGuard: audit.nodeGuard,
|
|
secrets: audit.secrets ? {
|
|
status: audit.secrets.status,
|
|
valuesRead: audit.secrets.valuesRead,
|
|
valuesPrinted: audit.secrets.valuesPrinted,
|
|
} : null,
|
|
registry: audit.registry ? {
|
|
status: audit.registry.status,
|
|
processHttpAccess: audit.registry.processHttpAccess,
|
|
k3sPullAccess: audit.registry.k3sPullAccess,
|
|
} : null,
|
|
lease: audit.lease,
|
|
desiredState: audit.desiredState ? {
|
|
status: audit.desiredState.status,
|
|
targetCommit: audit.desiredState.targetCommit,
|
|
imageConvergence: audit.desiredState.imageConvergence,
|
|
} : null,
|
|
workload: audit.workload ? {
|
|
status: audit.workload.status,
|
|
deploymentCount: audit.workload.deploymentCount,
|
|
currentImageConvergence: audit.workload.currentImageConvergence,
|
|
podWaiting: (audit.workload.podWaiting || []).slice(0, 5),
|
|
runtimeJobs: (audit.workload.runtimeJobs || []).slice(0, 5),
|
|
} : null,
|
|
publicHealth: audit.publicHealth ? {
|
|
status: audit.publicHealth.status,
|
|
expectedCommit: audit.publicHealth.expectedCommit,
|
|
summary: audit.publicHealth.summary,
|
|
} : null,
|
|
durability: audit.durability ? {
|
|
status: audit.durability.status,
|
|
unavailableReason: audit.durability.unavailableReason,
|
|
} : null,
|
|
controlledDevCd: compactControlled(audit.controlledDevCd),
|
|
blockerTypes: audit.blockerTypes || [],
|
|
blockers: compactBlockers(audit.blockers, 12),
|
|
safety: audit.safety,
|
|
};
|
|
}
|
|
|
|
function compactForTransport(payload, fullResultPath) {
|
|
return {
|
|
ok: payload.ok,
|
|
env: payload.env,
|
|
environment: payload.environment,
|
|
status: payload.status,
|
|
error: payload.error ?? null,
|
|
action: payload.action ?? null,
|
|
dryRun: payload.dryRun,
|
|
mutation: payload.mutation,
|
|
remoteHost: payload.remoteHost,
|
|
workspace: payload.workspace,
|
|
kubeconfig: payload.kubeconfig,
|
|
dumpPath: payload.dumpPath,
|
|
remoteFullResultPath: fullResultPath,
|
|
repo: payload.repo ? {
|
|
status: payload.repo.status,
|
|
path: payload.repo.path,
|
|
source: payload.repo.source,
|
|
rejected: payload.repo.rejected,
|
|
ephemeral: payload.repo.ephemeral === true,
|
|
cacheRefreshStatus: payload.repo.cacheRefreshStatus || null,
|
|
} : null,
|
|
worktreeGuard: compactGit(payload.worktreeGuard),
|
|
nodeGuard: compactNodeGuard(payload.nodeGuard),
|
|
secretPreflight: compactSecretPreflight(payload.secretPreflight),
|
|
lockState: payload.lockState ? {
|
|
status: payload.lockState.status,
|
|
phase: payload.lockState.phase,
|
|
holder: payload.lockState.ownerTaskId || payload.lockState.holderIdentity || null,
|
|
held: payload.lockState.held,
|
|
stale: payload.lockState.stale,
|
|
retryAfterSeconds: payload.lockState.retryAfterSeconds,
|
|
lockName: payload.lockState.lockName,
|
|
} : null,
|
|
desiredState: payload.desiredState ? {
|
|
deployJson: payload.desiredState.deployJson,
|
|
artifactCatalog: payload.desiredState.artifactCatalog,
|
|
artifactReport: payload.desiredState.artifactReport,
|
|
} : null,
|
|
target: payload.target,
|
|
promotion: payload.promotion ? {
|
|
source: payload.promotion.source,
|
|
deployJson: payload.promotion.deployJson,
|
|
artifactCatalog: payload.promotion.artifactCatalog,
|
|
artifactReport: payload.promotion.artifactReport,
|
|
} : null,
|
|
controlledDevCd: compactControlled(payload.controlledDevCd),
|
|
audit: compactAudit(payload.audit),
|
|
blockers: compactBlockers(payload.blockers, 12),
|
|
blockerTypes: payload.blockerTypes || [...new Set(compactBlockers(payload.blockers, 12).map((item) => item.scope).filter(Boolean))],
|
|
nextSafeCommand: payload.nextSafeCommand,
|
|
safety: payload.safety,
|
|
};
|
|
}
|
|
|
|
function emitResult(payload, dumpDir, compactStdout = false) {
|
|
const fullResultPath = path.join(dumpDir, "result.full.json");
|
|
const compactResultPath = path.join(dumpDir, "result.compact.json");
|
|
fs.writeFileSync(fullResultPath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 });
|
|
if (!compactStdout) {
|
|
fs.writeFileSync(compactResultPath, `${JSON.stringify(compactForTransport(payload, fullResultPath), null, 2)}\n`, { mode: 0o600 });
|
|
console.log(JSON.stringify(payload, null, 2));
|
|
return;
|
|
}
|
|
let compact = compactForTransport(payload, fullResultPath);
|
|
let text = JSON.stringify(compact, null, 2);
|
|
if (Buffer.byteLength(text, "utf8") > 3600) {
|
|
compact = {
|
|
ok: compact.ok,
|
|
env: compact.env,
|
|
status: compact.status,
|
|
error: compact.error,
|
|
action: compact.action,
|
|
dryRun: compact.dryRun,
|
|
mutation: compact.mutation,
|
|
workspace: compact.workspace,
|
|
dumpPath: compact.dumpPath,
|
|
remoteFullResultPath: compact.remoteFullResultPath,
|
|
repo: compact.repo,
|
|
worktreeGuard: compact.worktreeGuard,
|
|
nodeGuard: compact.nodeGuard,
|
|
lockState: compact.lockState,
|
|
target: compact.target ? {
|
|
ref: compact.target.ref,
|
|
promotionCommit: compact.target.promotionCommit,
|
|
shortCommitId: compact.target.shortCommitId,
|
|
promotionSource: compact.target.promotionSource,
|
|
publishRequired: compact.target.publishRequired,
|
|
} : null,
|
|
controlledDevCd: compact.controlledDevCd,
|
|
audit: compact.audit ? {
|
|
status: compact.audit.status,
|
|
blockerTypes: compact.audit.blockerTypes,
|
|
blockers: compact.audit.blockers,
|
|
publicHealth: compact.audit.publicHealth,
|
|
workload: compact.audit.workload,
|
|
durability: compact.audit.durability,
|
|
} : null,
|
|
blockers: compact.blockers,
|
|
blockerTypes: compact.blockerTypes,
|
|
nextSafeCommand: compact.nextSafeCommand,
|
|
safety: compact.safety,
|
|
outputCompacted: "provider-host-ssh-stdout-limit",
|
|
};
|
|
text = JSON.stringify(compact, null, 2);
|
|
}
|
|
if (Buffer.byteLength(text, "utf8") > 2400) {
|
|
compact = {
|
|
ok: payload.ok,
|
|
env: payload.env,
|
|
status: payload.status,
|
|
error: payload.error ?? null,
|
|
action: payload.action ?? null,
|
|
dryRun: payload.dryRun,
|
|
mutation: payload.mutation,
|
|
workspace: payload.workspace,
|
|
dumpPath: payload.dumpPath,
|
|
remoteFullResultPath: fullResultPath,
|
|
repo: payload.repo ? {
|
|
status: payload.repo.status,
|
|
path: payload.repo.path,
|
|
source: payload.repo.source,
|
|
rejected: payload.repo.rejected,
|
|
ephemeral: payload.repo.ephemeral === true,
|
|
cacheRefreshStatus: payload.repo.cacheRefreshStatus || null,
|
|
} : null,
|
|
worktreeStatus: payload.worktreeGuard?.status ?? null,
|
|
nodeGuardStatus: payload.nodeGuard?.status ?? payload.audit?.nodeGuard?.status ?? null,
|
|
lockState: payload.lockState ? {
|
|
status: payload.lockState.status,
|
|
phase: payload.lockState.phase,
|
|
held: payload.lockState.held,
|
|
holder: payload.lockState.ownerTaskId || payload.lockState.holderIdentity || null,
|
|
stale: payload.lockState.stale,
|
|
} : null,
|
|
target: payload.target ? {
|
|
ref: payload.target.ref,
|
|
promotionCommit: payload.target.promotionCommit,
|
|
shortCommitId: payload.target.shortCommitId,
|
|
promotionSource: payload.target.promotionSource,
|
|
} : payload.audit?.desiredState ? {
|
|
targetCommit: payload.audit.desiredState.targetCommit,
|
|
} : null,
|
|
controlledDevCdStatus: payload.controlledDevCd?.status ?? payload.audit?.controlledDevCd?.status ?? null,
|
|
auditStatus: payload.audit?.status ?? null,
|
|
blockerTypes: payload.blockerTypes || payload.audit?.blockerTypes || [...new Set(compactBlockers(payload.blockers, 8).map((item) => item.scope).filter(Boolean))],
|
|
blockers: compactBlockers(payload.blockers || payload.audit?.blockers, 6),
|
|
nextSafeCommand: payload.nextSafeCommand,
|
|
outputCompacted: "provider-host-ssh-stdout-limit",
|
|
};
|
|
text = JSON.stringify(compact);
|
|
}
|
|
if (Buffer.byteLength(text, "utf8") > 520) {
|
|
const blockerTypes = payload.blockerTypes
|
|
|| payload.audit?.blockerTypes
|
|
|| [...new Set(compactBlockers(payload.blockers || payload.audit?.blockers, 8).map((item) => item.scope).filter(Boolean))];
|
|
compact = {
|
|
ok: payload.ok,
|
|
env: payload.env,
|
|
status: payload.status,
|
|
error: payload.error ?? null,
|
|
action: payload.action ?? null,
|
|
remoteFullResultPath: fullResultPath,
|
|
repoPath: payload.repo?.path ?? payload.workspace?.path ?? null,
|
|
repoEphemeral: payload.repo?.ephemeral === true,
|
|
cacheRefreshStatus: payload.repo?.cacheRefreshStatus ?? null,
|
|
worktreeStatus: payload.worktreeGuard?.status ?? null,
|
|
nodeGuardStatus: payload.nodeGuard?.status ?? payload.audit?.nodeGuard?.status ?? null,
|
|
lockStatus: payload.lockState ? `${payload.lockState.status || "unknown"}/${payload.lockState.phase || "unknown"}/${payload.lockState.held === true ? "held" : "free"}` : null,
|
|
targetCommit: payload.target?.promotionCommit ?? payload.audit?.desiredState?.targetCommit ?? null,
|
|
blockerTypes: Array.isArray(blockerTypes) ? blockerTypes.slice(0, 4) : [],
|
|
outputCompacted: true,
|
|
};
|
|
text = JSON.stringify(compact);
|
|
}
|
|
fs.writeFileSync(compactResultPath, `${text}\n`, { mode: 0o600 });
|
|
console.log(text);
|
|
}
|
|
|
|
async function main() {
|
|
const options = decodeOptions();
|
|
const timeoutMs = Math.min(Number(options.timeoutMs || 45000), 60000);
|
|
const runId = options.runId || `${new Date().toISOString().replace(/[:.]/g, "-")}-${process.pid}`;
|
|
const dumpDir = path.join(os.homedir(), ".state", "unidesk-hwlab-cd", runId);
|
|
await fsp.mkdir(dumpDir, { recursive: true, mode: 0o700 });
|
|
|
|
const action = options.action;
|
|
const repo = await resolveRepo(options.repoPath || null, dumpDir, Math.min(timeoutMs, 55000));
|
|
const base = {
|
|
ok: false,
|
|
env: "dev",
|
|
environment: "dev",
|
|
action,
|
|
dryRun: action === "apply" ? Boolean(options.dryRun) : action !== "status",
|
|
mutation: false,
|
|
remoteHost: "D601",
|
|
workspace: repo.status === "selected" ? { path: repo.path, source: repo.source } : null,
|
|
kubeconfig: { source: "explicit", path: options.kubeconfig || nativeKubeconfig, required: nativeKubeconfig },
|
|
dumpPath: dumpDir,
|
|
safety: {
|
|
wrapperCallsOnlyRepoOwnedDevCdApply: true,
|
|
deployJsonAuthoritative: "deploy/deploy.json",
|
|
kubectlApplyExecuted: false,
|
|
rolloutExecuted: false,
|
|
liveVerifyExecuted: false,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false,
|
|
cdLockMutated: false,
|
|
prodTouched: false,
|
|
reportsWritten: false,
|
|
repoReportsWritten: false,
|
|
},
|
|
};
|
|
|
|
if (repo.status !== "selected") {
|
|
const blockers = collectBlockers({ repo, action });
|
|
emitResult({ ...base, status: "blocked", error: "hwlab-repo-not-found", repo, blockers, nextSafeCommand: nextSafeCommand(action, blockers) }, dumpDir, options.compactStdout === true);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
const repoPath = repo.path;
|
|
process.chdir(repoPath);
|
|
const [git, guard] = await Promise.all([
|
|
gitSummary(repoPath, dumpDir, Math.min(timeoutMs, 15000)),
|
|
nodeGuard(options.kubeconfig || nativeKubeconfig, dumpDir, Math.min(timeoutMs, 15000)),
|
|
]);
|
|
const desiredState = {
|
|
deployJson: readJsonSummary(repoPath, "deploy/deploy.json"),
|
|
artifactCatalog: readJsonSummary(repoPath, "deploy/artifact-catalog.dev.json"),
|
|
artifactReport: readJsonSummary(repoPath, "reports/dev-gate/dev-artifacts.json"),
|
|
};
|
|
const lock = await lockStatus(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000));
|
|
const needsSecretPreflight = action === "preflight" || action === "audit" || action === "apply";
|
|
const secretRefs = needsSecretPreflight
|
|
? await secretPreflight(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000))
|
|
: { status: "skipped", reason: "status-does-not-run-secret-preflight", namespace, mutation: false, blockers: [] };
|
|
const preCommandBlockers = collectBlockers({ repo, git, guard, lock, secretRefs, action });
|
|
const canRunControlled = preCommandBlockers.length === 0 || action === "status" || action === "audit";
|
|
const controlled = canRunControlled && git.status === "pass" && guard.status === "pass"
|
|
? await runDevCdApply(repoPath, options.kubeconfig || nativeKubeconfig, action === "status" ? "status" : "apply", dumpDir, Math.max(1000, timeoutMs - 5000))
|
|
: { status: "skipped", commandOk: true, reason: "preflight-blockers-before-controlled-dev-cd-apply" };
|
|
const target = controlled.target || {
|
|
ref: "deploy/deploy.json",
|
|
promotionCommit: desiredState.deployJson.commitId || git.headCommit,
|
|
shortCommitId: desiredState.deployJson.commitId || null,
|
|
promotionSource: "deploy/deploy.json",
|
|
namespace,
|
|
};
|
|
if (action === "audit") {
|
|
const auditDesiredState = buildDesiredStateAudit(repoPath, target);
|
|
const [registry, workload, publicHealth] = await Promise.all([
|
|
registryAudit(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)),
|
|
workloadAudit(options.kubeconfig || nativeKubeconfig, guard, auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)),
|
|
httpHealthAudit(auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)),
|
|
]);
|
|
const durability = runtimeDurabilityAudit(auditDesiredState, publicHealth, secretRefs, workload);
|
|
const summary = auditSummary({ repo, git, guard, secretRefs, registry, lock, desired: auditDesiredState, workload, publicHealth, durability, controlled, dumpDir });
|
|
emitResult({
|
|
...base,
|
|
ok: summary.ok,
|
|
status: summary.status,
|
|
audit: summary,
|
|
repo: {
|
|
status: repo.status,
|
|
path: repo.path,
|
|
source: repo.source,
|
|
rejected: repo.rejected,
|
|
},
|
|
workspace: { path: repoPath, source: repo.source },
|
|
remoteDumpPath: dumpDir,
|
|
reportDumpPath: controlled.command?.dump?.stdout || dumpDir,
|
|
blockers: summary.blockers,
|
|
blockerTypes: summary.blockerTypes,
|
|
nextSafeCommand: nextSafeCommand(action, summary.blockers),
|
|
}, dumpDir, options.compactStdout === true);
|
|
process.exitCode = summary.ok ? 0 : 1;
|
|
return;
|
|
}
|
|
const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action });
|
|
const ok = blockers.length === 0;
|
|
emitResult({
|
|
...base,
|
|
ok,
|
|
status: ok ? action === "status" ? "ready" : "prepared" : guard.refusal ? "refused" : "blocked",
|
|
repo,
|
|
workspace: { path: repoPath, source: repo.source },
|
|
worktreeGuard: git,
|
|
nodeGuard: guard,
|
|
secretPreflight: secretRefs,
|
|
lockState: lock,
|
|
desiredState,
|
|
target,
|
|
promotion: {
|
|
source: "deploy/deploy.json",
|
|
deployJson: desiredState.deployJson,
|
|
artifactCatalog: desiredState.artifactCatalog,
|
|
artifactReport: desiredState.artifactReport,
|
|
},
|
|
controlledDevCd: controlled,
|
|
reportDumpPath: controlled.command?.dump?.stdout || dumpDir,
|
|
blockers,
|
|
nextSafeCommand: nextSafeCommand(action, blockers),
|
|
}, dumpDir, options.compactStdout === true);
|
|
process.exitCode = ok ? 0 : 1;
|
|
}
|
|
|
|
main().catch((error) => {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
console.log(JSON.stringify({ ok: false, env: "dev", status: "blocked", error: "remote-wrapper-exception", message: redact(message) }, null, 2));
|
|
process.exitCode = 1;
|
|
});
|