fix: harden hwlab cd cache
This commit is contained in:
@@ -8,8 +8,14 @@ const path = require("node:path");
|
||||
const namespace = "hwlab-dev";
|
||||
const lockName = "hwlab-dev-cd-lock";
|
||||
const nativeKubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
const defaultRepo = "/home/ubuntu/hwlab_cd";
|
||||
const legacyDefaultRepo = "/home/ubuntu/hwlab_cd";
|
||||
const rejectedRepo = "/home/ubuntu/hwlab";
|
||||
const defaultRemoteCandidates = ["git@github.com:pikasTech/HWLAB.git"];
|
||||
const dedicatedCacheRoot = path.join(os.homedir(), ".cache", "unidesk", "hwlab-cd");
|
||||
const dedicatedCacheRepo = path.join(os.homedir(), ".cache", "unidesk", "hwlab-cd", "git-cache", "HWLAB.git");
|
||||
const defaultEgressProxy = "http://127.0.0.1:18789";
|
||||
const defaultNoProxy = "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local";
|
||||
const seedCacheRepos = ["/home/ubuntu/hwlab", "/home/ubuntu/workspace/hwlab", "/tmp/hwlab-dev-cd-35bbbee-host"];
|
||||
const requiredNodeName = "d601";
|
||||
const tailChars = 1400;
|
||||
const parseCaptureLimitBytes = 1024 * 1024;
|
||||
@@ -169,25 +175,279 @@ function accessCheck(targetPath, mode) {
|
||||
}
|
||||
}
|
||||
|
||||
function resolveRepo(provided) {
|
||||
const rawPath = provided || process.env.UNIDESK_HWLAB_REPO || defaultRepo;
|
||||
function repoRecord(rawPath, source, extra = {}) {
|
||||
const absolutePath = path.resolve(rawPath);
|
||||
const rejected = absolutePath === rejectedRepo;
|
||||
const devCdApply = path.join(absolutePath, "scripts/dev-cd-apply.mjs");
|
||||
const deployJson = path.join(absolutePath, "deploy/deploy.json");
|
||||
return {
|
||||
status: !rejected && fs.existsSync(absolutePath) && fs.existsSync(devCdApply) && fs.existsSync(deployJson) ? "selected" : "blocked",
|
||||
source: provided ? "option" : process.env.UNIDESK_HWLAB_REPO ? "env:UNIDESK_HWLAB_REPO" : "default:d601-clean-mirror",
|
||||
source,
|
||||
path: absolutePath,
|
||||
defaultPath: defaultRepo,
|
||||
defaultPath: legacyDefaultRepo,
|
||||
defaultMode: "ephemeral-clone",
|
||||
rejected,
|
||||
rejectionReason: rejected ? "runner-history-directory-is-not-hwlab-cd-release-truth" : null,
|
||||
exists: fs.existsSync(absolutePath),
|
||||
hasDevCdApply: fs.existsSync(devCdApply),
|
||||
hasDeployJson: fs.existsSync(deployJson),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function remoteCandidates() {
|
||||
const fromEnv = process.env.UNIDESK_HWLAB_REMOTE_URL;
|
||||
if (typeof fromEnv === "string" && fromEnv.trim().length > 0) return [fromEnv.trim()];
|
||||
return defaultRemoteCandidates;
|
||||
}
|
||||
|
||||
function seedRepoCandidates() {
|
||||
const fromEnv = process.env.UNIDESK_HWLAB_CACHE_SEED_REPOS;
|
||||
const configured = typeof fromEnv === "string" && fromEnv.trim().length > 0
|
||||
? fromEnv.split(":").map((item) => item.trim()).filter(Boolean)
|
||||
: seedCacheRepos;
|
||||
return [...new Set(configured.map((item) => path.resolve(item)))];
|
||||
}
|
||||
|
||||
function egressEnv() {
|
||||
const proxy = process.env.UNIDESK_HWLAB_EGRESS_PROXY || defaultEgressProxy;
|
||||
const noProxy = process.env.NO_PROXY || process.env.no_proxy || defaultNoProxy;
|
||||
const proxyHostPort = proxy.replace(/^https?:\/\//u, "");
|
||||
const gitSshCommand = process.env.GIT_SSH_COMMAND || [
|
||||
"ssh",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", `"ProxyCommand=nc -X connect -x ${proxyHostPort} %h %p"`,
|
||||
].join(" ");
|
||||
return {
|
||||
...process.env,
|
||||
HTTP_PROXY: proxy,
|
||||
HTTPS_PROXY: proxy,
|
||||
ALL_PROXY: proxy,
|
||||
http_proxy: proxy,
|
||||
https_proxy: proxy,
|
||||
all_proxy: proxy,
|
||||
NO_PROXY: noProxy,
|
||||
no_proxy: noProxy,
|
||||
GIT_SSH_COMMAND: gitSshCommand,
|
||||
};
|
||||
}
|
||||
|
||||
async function egressProxyHealth(dumpDir, timeoutMs) {
|
||||
const proxy = process.env.UNIDESK_HWLAB_EGRESS_PROXY || defaultEgressProxy;
|
||||
const healthUrl = `${proxy.replace(/\/$/u, "")}/__unidesk/egress-proxy/health`;
|
||||
const command = await runCaptured(["curl", "-fsS", "--max-time", "3", healthUrl], os.homedir(), dumpDir, "egress-proxy-health", { env: egressEnv(), timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
const body = parseJson(command.stdoutText);
|
||||
return {
|
||||
status: command.ok && asRecord(body)?.connected === true ? "pass" : "blocked",
|
||||
proxy,
|
||||
healthUrl,
|
||||
connected: asRecord(body)?.connected ?? null,
|
||||
command: commandView(command),
|
||||
};
|
||||
}
|
||||
|
||||
function pathOwner(targetPath) {
|
||||
try {
|
||||
const stat = fs.statSync(targetPath);
|
||||
return { uid: stat.uid, gid: stat.gid, mode: (stat.mode & 0o777).toString(8), isDirectory: stat.isDirectory() };
|
||||
} catch (error) {
|
||||
return { uid: null, gid: null, mode: null, isDirectory: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function cacheRepoRecord(cachePath) {
|
||||
const absolutePath = path.resolve(cachePath);
|
||||
const gitPath = path.join(absolutePath, ".git");
|
||||
const bareHead = path.join(absolutePath, "HEAD");
|
||||
const bareObjects = path.join(absolutePath, "objects");
|
||||
const shallowPath = path.join(absolutePath, "shallow");
|
||||
const exists = fs.existsSync(gitPath) || (fs.existsSync(bareHead) && fs.existsSync(bareObjects));
|
||||
const readable = accessCheck(absolutePath, fs.constants.R_OK | fs.constants.X_OK);
|
||||
const writable = accessCheck(absolutePath, fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
|
||||
const owner = pathOwner(absolutePath);
|
||||
return { path: absolutePath, exists, readable: readable.ok, writable: writable.ok, owner, shallow: fs.existsSync(shallowPath), error: readable.error || writable.error || null };
|
||||
}
|
||||
|
||||
async function deployJsonCommitFromCache(dumpDir, timeoutMs) {
|
||||
const command = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "show", "refs/heads/main:deploy/deploy.json"], os.homedir(), dumpDir, "git-dedicated-cache-deploy-json", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
if (!command.ok) return { commitId: null, command: commandView(command) };
|
||||
const parsed = parseJson(command.stdoutText);
|
||||
return { commitId: stringValue(asRecord(parsed)?.commitId), command: commandView(command) };
|
||||
}
|
||||
|
||||
async function cacheCommitExists(commitId, dumpDir, timeoutMs, idPrefix = "git-dedicated-cache-commit") {
|
||||
if (!commitId) return { exists: false, command: null };
|
||||
const command = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "--verify", `${commitId}^{commit}`], os.homedir(), dumpDir, idPrefix, { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
return { exists: command.ok, resolvedCommit: command.stdoutText.trim() || null, command: commandView(command) };
|
||||
}
|
||||
|
||||
async function refreshDedicatedCache(remoteUrl, dumpDir, timeoutMs) {
|
||||
const env = egressEnv();
|
||||
const parent = path.dirname(dedicatedCacheRepo);
|
||||
await fsp.mkdir(parent, { recursive: true, mode: 0o700 });
|
||||
await fsp.chmod(dedicatedCacheRoot, 0o700).catch(() => undefined);
|
||||
await fsp.chmod(path.dirname(dedicatedCacheRoot), 0o700).catch(() => undefined);
|
||||
const attempts = [];
|
||||
const existing = cacheRepoRecord(dedicatedCacheRepo);
|
||||
if (existing.exists && (!existing.writable || existing.owner.uid !== process.getuid?.())) {
|
||||
attempts.push({ stage: "dedicated-cache-owner", ok: false, cache: existing });
|
||||
return { ok: false, cache: existing, attempts, stage: "dedicated-cache-owner" };
|
||||
}
|
||||
if (existing.exists) {
|
||||
const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
attempts.push({ stage: "cache-permissions", ok: permissions.ok, command: commandView(permissions) });
|
||||
if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions" };
|
||||
}
|
||||
|
||||
if (!existing.exists) {
|
||||
for (const seed of seedRepoCandidates().map(cacheRepoRecord)) {
|
||||
if (!seed.exists || !seed.readable) {
|
||||
attempts.push({ stage: "seed-unavailable", ok: false, seed });
|
||||
continue;
|
||||
}
|
||||
await fsp.rm(dedicatedCacheRepo, { recursive: true, force: true });
|
||||
const cloneSeed = await runCaptured(["git", "clone", "--bare", seed.path, dedicatedCacheRepo], parent, dumpDir, "git-dedicated-cache-seed-clone", { timeoutMs: Math.min(timeoutMs, 15000) });
|
||||
attempts.push({ stage: "seed-clone", ok: cloneSeed.ok, seed, command: commandView(cloneSeed) });
|
||||
if (cloneSeed.ok) break;
|
||||
}
|
||||
if (!fs.existsSync(dedicatedCacheRepo)) {
|
||||
const init = await runCaptured(["git", "init", "--bare", dedicatedCacheRepo], parent, dumpDir, "git-dedicated-cache-init", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
attempts.push({ stage: "bare-init", ok: init.ok, command: commandView(init) });
|
||||
if (!init.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "bare-init" };
|
||||
}
|
||||
const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
attempts.push({ stage: "cache-permissions", ok: permissions.ok, command: commandView(permissions) });
|
||||
if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions" };
|
||||
}
|
||||
|
||||
const remoteSet = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "remote", "set-url", "origin", remoteUrl], os.homedir(), dumpDir, "git-dedicated-cache-remote-set-url", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
const remoteAdd = remoteSet.ok ? null : await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "remote", "add", "origin", remoteUrl], os.homedir(), dumpDir, "git-dedicated-cache-remote-add", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
attempts.push({ stage: "remote-set-url", ok: remoteSet.ok || remoteAdd?.ok === true, command: commandView(remoteSet), fallback: remoteAdd ? commandView(remoteAdd) : null });
|
||||
if (!remoteSet.ok && remoteAdd?.ok !== true) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "remote-set-url" };
|
||||
|
||||
const remoteHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "ls-remote", "--heads", "origin", "main"], os.homedir(), dumpDir, "git-dedicated-cache-ls-remote-main", { env, timeoutMs: Math.min(timeoutMs, 15000) });
|
||||
const remoteMainCommit = remoteHead.stdoutText.trim().split(/\s+/u)[0] || null;
|
||||
attempts.push({ stage: "ls-remote-main", ok: remoteHead.ok && Boolean(remoteMainCommit), remoteMainCommit, command: commandView(remoteHead) });
|
||||
const localHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "refs/heads/main"], os.homedir(), dumpDir, "git-dedicated-cache-local-main", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
const localMainCommit = localHead.stdoutText.trim() || null;
|
||||
attempts.push({ stage: "local-main", ok: localHead.ok && Boolean(localMainCommit), localMainCommit, command: commandView(localHead) });
|
||||
const deployJson = localHead.ok ? await deployJsonCommitFromCache(dumpDir, timeoutMs) : { commitId: null, command: null };
|
||||
attempts.push({ stage: "deploy-json-commit", ok: Boolean(deployJson.commitId), commitId: deployJson.commitId, command: deployJson.command });
|
||||
const deployCommit = await cacheCommitExists(deployJson.commitId, dumpDir, timeoutMs, "git-dedicated-cache-deploy-commit");
|
||||
attempts.push({ stage: "deploy-commit-present", ok: deployCommit.exists, commitId: deployJson.commitId, resolvedCommit: deployCommit.resolvedCommit, command: deployCommit.command });
|
||||
const cacheRecordBeforeFetch = cacheRepoRecord(dedicatedCacheRepo);
|
||||
if (remoteHead.ok && remoteMainCommit && localHead.ok && localMainCommit === remoteMainCommit && deployCommit.exists && cacheRecordBeforeFetch.shallow !== true) {
|
||||
return { ok: true, cache: cacheRecordBeforeFetch, attempts, stage: "ready", refreshStatus: "current-full", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId };
|
||||
}
|
||||
|
||||
const fetchArgs = ["git", "--git-dir", dedicatedCacheRepo, "fetch", "--prune", "--tags", "origin", "+refs/heads/*:refs/heads/*"];
|
||||
if (cacheRecordBeforeFetch.shallow === true) fetchArgs.splice(4, 0, "--unshallow");
|
||||
const fetch = await runCaptured(fetchArgs, os.homedir(), dumpDir, "git-dedicated-cache-fetch-full", { env, timeoutMs: Math.min(timeoutMs, 55000) });
|
||||
attempts.push({ stage: "fetch-main", ok: fetch.ok, command: commandView(fetch) });
|
||||
if (!fetch.ok) {
|
||||
if (localHead.ok && localMainCommit && deployCommit.exists) {
|
||||
return { ok: true, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "stale-cache", refreshStatus: "blocked", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId };
|
||||
}
|
||||
return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "fetch-full", refreshStatus: "blocked", remoteMainCommit, localMainCommit, deployCommitId: deployJson.commitId };
|
||||
}
|
||||
const fetchedLocalHead = await runCaptured(["git", "--git-dir", dedicatedCacheRepo, "rev-parse", "refs/heads/main"], os.homedir(), dumpDir, "git-dedicated-cache-local-main-after-fetch", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
const fetchedLocalMainCommit = fetchedLocalHead.stdoutText.trim() || localMainCommit;
|
||||
attempts.push({ stage: "local-main-after-fetch", ok: fetchedLocalHead.ok && Boolean(fetchedLocalMainCommit), localMainCommit: fetchedLocalMainCommit, command: commandView(fetchedLocalHead) });
|
||||
const fetchedDeployJson = await deployJsonCommitFromCache(dumpDir, timeoutMs);
|
||||
attempts.push({ stage: "deploy-json-commit-after-fetch", ok: Boolean(fetchedDeployJson.commitId), commitId: fetchedDeployJson.commitId, command: fetchedDeployJson.command });
|
||||
const fetchedDeployCommit = await cacheCommitExists(fetchedDeployJson.commitId, dumpDir, timeoutMs, "git-dedicated-cache-deploy-commit-after-fetch");
|
||||
attempts.push({ stage: "deploy-commit-present-after-fetch", ok: fetchedDeployCommit.exists, commitId: fetchedDeployJson.commitId, resolvedCommit: fetchedDeployCommit.resolvedCommit, command: fetchedDeployCommit.command });
|
||||
const permissions = await runCaptured(["chmod", "-R", "go-rwx", dedicatedCacheRoot], os.homedir(), dumpDir, "git-dedicated-cache-permissions-after-fetch", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
attempts.push({ stage: "cache-permissions-after-fetch", ok: permissions.ok, command: commandView(permissions) });
|
||||
if (!permissions.ok) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "cache-permissions-after-fetch", refreshStatus: "blocked", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId };
|
||||
if (!fetchedDeployCommit.exists) return { ok: false, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "deploy-commit-unresolved", refreshStatus: "blocked", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId };
|
||||
return { ok: true, cache: cacheRepoRecord(dedicatedCacheRepo), attempts, stage: "ready", refreshStatus: "refreshed-full", remoteMainCommit, localMainCommit: fetchedLocalMainCommit, deployCommitId: fetchedDeployJson.commitId };
|
||||
}
|
||||
|
||||
async function materializeFromCache(cachePath, repoPath, remoteUrl, dumpDir, timeoutMs) {
|
||||
await fsp.rm(repoPath, { recursive: true, force: true });
|
||||
const clone = await runCaptured(["git", "clone", "--no-hardlinks", "--no-checkout", cachePath, repoPath], path.dirname(repoPath), dumpDir, "git-cache-clone", { timeoutMs: Math.min(timeoutMs, 15000) });
|
||||
if (!clone.ok) return { ok: false, stage: "cache-clone", commands: [commandView(clone)] };
|
||||
const remote = await runCaptured(["git", "remote", "set-url", "origin", remoteUrl], repoPath, dumpDir, "git-cache-remote-set-url", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
if (!remote.ok) return { ok: false, stage: "remote-set-url", commands: [commandView(clone), commandView(remote)] };
|
||||
const checkout = await runCaptured(["git", "checkout", "-B", "main", "origin/main"], repoPath, dumpDir, "git-cache-checkout-main", { timeoutMs: Math.min(timeoutMs, 5000) });
|
||||
const reset = checkout.ok
|
||||
? await runCaptured(["git", "reset", "--hard", "origin/main"], repoPath, dumpDir, "git-cache-reset-hard", { timeoutMs: Math.min(timeoutMs, 5000) })
|
||||
: null;
|
||||
const clean = checkout.ok && reset?.ok
|
||||
? await runCaptured(["git", "clean", "-ffdqx"], repoPath, dumpDir, "git-cache-clean", { timeoutMs: Math.min(timeoutMs, 5000) })
|
||||
: null;
|
||||
const upstream = checkout.ok && reset?.ok
|
||||
? await runCaptured(["git", "branch", "--set-upstream-to=origin/main", "main"], repoPath, dumpDir, "git-cache-upstream", { timeoutMs: Math.min(timeoutMs, 5000) })
|
||||
: null;
|
||||
const fetchHead = checkout.ok && reset?.ok && clean?.ok
|
||||
? await runCaptured(["bash", "-lc", "p=\"$(git rev-parse --path-format=absolute --git-common-dir)/FETCH_HEAD\"; : > \"$p\"; chmod 600 \"$p\""], repoPath, dumpDir, "git-cache-fetch-head", { timeoutMs: Math.min(timeoutMs, 5000) })
|
||||
: null;
|
||||
const commands = [clone, remote, checkout, reset, clean, upstream, fetchHead].filter(Boolean).map(commandView);
|
||||
return { ok: checkout.ok && reset?.ok === true && clean?.ok === true && fetchHead?.ok === true, stage: "ready", commands };
|
||||
}
|
||||
|
||||
async function cloneEphemeralRepo(dumpDir, timeoutMs) {
|
||||
const root = path.join(dumpDir, "ephemeral-repo");
|
||||
const attempts = [];
|
||||
await fsp.mkdir(root, { recursive: true, mode: 0o700 });
|
||||
const egress = await egressProxyHealth(dumpDir, Math.min(timeoutMs, 5000));
|
||||
if (egress.status !== "pass") {
|
||||
return repoRecord(path.join(root, "HWLAB"), "ephemeral:git-cache", {
|
||||
ephemeral: true,
|
||||
remoteUrl: null,
|
||||
egressProxy: egress,
|
||||
dedicatedCache: cacheRepoRecord(dedicatedCacheRepo),
|
||||
seedRepos: seedRepoCandidates().map(cacheRepoRecord),
|
||||
cloneAttempts: [],
|
||||
cloneFailed: true,
|
||||
cloneFailureStage: "egress-proxy",
|
||||
retainedForDiagnostics: true,
|
||||
});
|
||||
}
|
||||
for (const remoteUrl of remoteCandidates()) {
|
||||
const repoPath = path.join(root, "HWLAB");
|
||||
const refreshed = await refreshDedicatedCache(remoteUrl, dumpDir, Math.min(timeoutMs, 45000));
|
||||
attempts.push({ remoteUrl, dedicatedCache: refreshed.cache, ok: refreshed.ok, stage: refreshed.stage, refreshAttempts: refreshed.attempts });
|
||||
if (!refreshed.ok) continue;
|
||||
const materialized = await materializeFromCache(dedicatedCacheRepo, repoPath, remoteUrl, dumpDir, Math.min(timeoutMs, 9000));
|
||||
attempts.push({ remoteUrl, dedicatedCache: refreshed.cache, ok: materialized.ok, stage: materialized.stage, commands: materialized.commands });
|
||||
if (materialized.ok) {
|
||||
return repoRecord(repoPath, "ephemeral:dedicated-git-cache", {
|
||||
ephemeral: true,
|
||||
remoteUrl,
|
||||
egressProxy: egress,
|
||||
dedicatedCache: refreshed.cache,
|
||||
cacheRefreshStatus: refreshed.refreshStatus || "current",
|
||||
remoteMainCommit: refreshed.remoteMainCommit || null,
|
||||
cacheMainCommit: refreshed.localMainCommit || null,
|
||||
deployCommitId: refreshed.deployCommitId || null,
|
||||
seedRepos: seedRepoCandidates().map(cacheRepoRecord),
|
||||
cloneAttempts: attempts,
|
||||
retainedForDiagnostics: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
return repoRecord(path.join(root, "HWLAB"), "ephemeral:dedicated-git-cache", {
|
||||
ephemeral: true,
|
||||
remoteUrl: null,
|
||||
egressProxy: egress,
|
||||
dedicatedCache: cacheRepoRecord(dedicatedCacheRepo),
|
||||
seedRepos: seedRepoCandidates().map(cacheRepoRecord),
|
||||
cloneAttempts: attempts,
|
||||
cloneFailed: true,
|
||||
cloneFailureStage: attempts.findLast((attempt) => attempt.stage)?.stage || "cache-unavailable",
|
||||
retainedForDiagnostics: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveRepo(provided, dumpDir, timeoutMs) {
|
||||
if (provided) return repoRecord(provided, "option", { ephemeral: false });
|
||||
if (process.env.UNIDESK_HWLAB_REPO) return repoRecord(process.env.UNIDESK_HWLAB_REPO, "env:UNIDESK_HWLAB_REPO", { ephemeral: false });
|
||||
return cloneEphemeralRepo(dumpDir, Math.min(timeoutMs, 55000));
|
||||
}
|
||||
|
||||
async function gitSummary(repoPath, dumpDir, timeoutMs) {
|
||||
const [branch, head, originMain, remote, gitDir, gitCommonDir, statusShort, statusPorcelain] = await Promise.all([
|
||||
runCaptured(["git", "rev-parse", "--abbrev-ref", "HEAD"], repoPath, dumpDir, "git-branch", { timeoutMs }),
|
||||
@@ -1113,6 +1373,14 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio
|
||||
if (repo.status !== "selected") {
|
||||
blockers.push({ scope: "hwlab-repo", summary: repo.rejected ? "Rejected runner history HWLAB directory." : "No eligible HWLAB CD repo with scripts/dev-cd-apply.mjs and deploy/deploy.json was found." });
|
||||
}
|
||||
if (repo.cacheRefreshStatus === "blocked") {
|
||||
blockers.push({
|
||||
scope: "hwlab-cache-refresh",
|
||||
summary: "Dedicated HWLAB git cache could not refresh from GitHub through the provider egress proxy; using the last local cache state is not a release-truth PASS.",
|
||||
remoteMainCommit: repo.remoteMainCommit || null,
|
||||
cacheMainCommit: repo.cacheMainCommit || null,
|
||||
});
|
||||
}
|
||||
if (git?.blockers) blockers.push(...git.blockers);
|
||||
if (guard) {
|
||||
if (guard.refusal) blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary, refusal: true });
|
||||
@@ -1139,6 +1407,341 @@ function nextSafeCommand(action, blockers) {
|
||||
return "host commander or the unique CD runner may decide whether to run node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report on D601; this wrapper did not apply or rollout";
|
||||
}
|
||||
|
||||
function compactBlockers(blockers, limit = 12) {
|
||||
return (Array.isArray(blockers) ? blockers : []).slice(0, limit).map((item) => ({
|
||||
scope: item?.scope ?? item?.blockerScope ?? null,
|
||||
type: item?.type ?? null,
|
||||
status: item?.status ?? null,
|
||||
summary: item?.summary ?? item?.reason ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function compactGit(git) {
|
||||
if (!git) return null;
|
||||
return {
|
||||
status: git.status,
|
||||
clean: git.clean,
|
||||
branch: git.branch,
|
||||
onMain: git.onMain,
|
||||
remoteMatches: git.remoteMatches,
|
||||
headMatchesOriginMain: git.headMatchesOriginMain,
|
||||
dirtyCount: git.dirtyCount,
|
||||
blockers: compactBlockers(git.blockers, 8),
|
||||
};
|
||||
}
|
||||
|
||||
function compactNodeGuard(guard) {
|
||||
if (!guard) return null;
|
||||
return {
|
||||
status: guard.status,
|
||||
refusal: guard.refusal,
|
||||
kubeconfig: guard.kubeconfig,
|
||||
currentContext: guard.currentContext,
|
||||
apiServer: guard.apiServer,
|
||||
nodeNames: guard.nodeNames,
|
||||
requiredNodePresent: guard.requiredNodePresent,
|
||||
refusalSignals: guard.refusalSignals,
|
||||
secondControlPlaneRisk: guard.secondControlPlaneRisk,
|
||||
defaultKubectlStatus: guard.defaultKubectlDiagnostic?.status ?? null,
|
||||
summary: guard.summary,
|
||||
};
|
||||
}
|
||||
|
||||
function compactSecretPreflight(secretRefs) {
|
||||
if (!secretRefs) return null;
|
||||
return {
|
||||
status: secretRefs.status,
|
||||
valuesRead: secretRefs.safety?.secretValuesRead === true,
|
||||
valuesPrinted: secretRefs.safety?.secretValuesPrinted === true,
|
||||
secretRefs: (secretRefs.secretRefs || []).map((ref) => ({
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
exists: ref.exists === true,
|
||||
keyPresent: ref.keyPresent === true,
|
||||
status: ref.status,
|
||||
})).slice(0, 8),
|
||||
blockers: compactBlockers(secretRefs.blockers, 8),
|
||||
};
|
||||
}
|
||||
|
||||
function compactDesiredState(desired) {
|
||||
if (!desired) return null;
|
||||
return {
|
||||
status: desired.status,
|
||||
targetCommit: desired.targetCommit,
|
||||
deployJson: desired.deployJson ? {
|
||||
hash: desired.deployJson.hash,
|
||||
commitId: desired.deployJson.commitId,
|
||||
environment: desired.deployJson.environment,
|
||||
namespace: desired.deployJson.namespace,
|
||||
serviceCount: desired.deployJson.serviceCount,
|
||||
} : null,
|
||||
artifactCatalog: desired.artifactCatalog ? {
|
||||
hash: desired.artifactCatalog.hash,
|
||||
commitId: desired.artifactCatalog.commitId,
|
||||
artifactState: desired.artifactCatalog.artifactState,
|
||||
ciPublished: desired.artifactCatalog.ciPublished,
|
||||
registryVerified: desired.artifactCatalog.registryVerified,
|
||||
digestCounts: desired.artifactCatalog.digestCounts,
|
||||
} : null,
|
||||
imageConvergence: desired.imageConvergence ? {
|
||||
status: desired.imageConvergence.status,
|
||||
missing: (desired.imageConvergence.missing || []).slice(0, 5),
|
||||
mismatches: (desired.imageConvergence.mismatches || []).slice(0, 5),
|
||||
} : null,
|
||||
};
|
||||
}
|
||||
|
||||
function compactControlled(controlled) {
|
||||
if (!controlled) return null;
|
||||
return {
|
||||
status: controlled.status,
|
||||
commandOk: controlled.commandOk,
|
||||
mode: controlled.mode,
|
||||
mutationAttempted: controlled.mutationAttempted,
|
||||
prodTouched: controlled.prodTouched,
|
||||
target: controlled.target ? {
|
||||
ref: controlled.target.ref,
|
||||
promotionCommit: controlled.target.promotionCommit,
|
||||
shortCommitId: controlled.target.shortCommitId,
|
||||
promotionSource: controlled.target.promotionSource,
|
||||
publishRequired: controlled.target.publishRequired,
|
||||
headMatchesTarget: controlled.target.headMatchesTarget,
|
||||
desiredStateStatus: controlled.target.desiredStateCheck?.status ?? null,
|
||||
artifactBoundaryStatus: controlled.target.artifactBoundary?.status ?? null,
|
||||
namespace: controlled.target.namespace,
|
||||
} : null,
|
||||
blockers: compactBlockers(controlled.blockers, 8),
|
||||
};
|
||||
}
|
||||
|
||||
function compactAudit(audit) {
|
||||
if (!audit) return null;
|
||||
return {
|
||||
ok: audit.ok,
|
||||
status: audit.status,
|
||||
env: audit.env,
|
||||
namespace: audit.namespace,
|
||||
nodeGuard: audit.nodeGuard,
|
||||
secrets: audit.secrets ? {
|
||||
status: audit.secrets.status,
|
||||
valuesRead: audit.secrets.valuesRead,
|
||||
valuesPrinted: audit.secrets.valuesPrinted,
|
||||
} : null,
|
||||
registry: audit.registry ? {
|
||||
status: audit.registry.status,
|
||||
processHttpAccess: audit.registry.processHttpAccess,
|
||||
k3sPullAccess: audit.registry.k3sPullAccess,
|
||||
} : null,
|
||||
lease: audit.lease,
|
||||
desiredState: audit.desiredState ? {
|
||||
status: audit.desiredState.status,
|
||||
targetCommit: audit.desiredState.targetCommit,
|
||||
imageConvergence: audit.desiredState.imageConvergence,
|
||||
} : null,
|
||||
workload: audit.workload ? {
|
||||
status: audit.workload.status,
|
||||
deploymentCount: audit.workload.deploymentCount,
|
||||
currentImageConvergence: audit.workload.currentImageConvergence,
|
||||
podWaiting: (audit.workload.podWaiting || []).slice(0, 5),
|
||||
runtimeJobs: (audit.workload.runtimeJobs || []).slice(0, 5),
|
||||
} : null,
|
||||
publicHealth: audit.publicHealth ? {
|
||||
status: audit.publicHealth.status,
|
||||
expectedCommit: audit.publicHealth.expectedCommit,
|
||||
summary: audit.publicHealth.summary,
|
||||
} : null,
|
||||
durability: audit.durability ? {
|
||||
status: audit.durability.status,
|
||||
unavailableReason: audit.durability.unavailableReason,
|
||||
} : null,
|
||||
controlledDevCd: compactControlled(audit.controlledDevCd),
|
||||
blockerTypes: audit.blockerTypes || [],
|
||||
blockers: compactBlockers(audit.blockers, 12),
|
||||
safety: audit.safety,
|
||||
};
|
||||
}
|
||||
|
||||
function compactForTransport(payload, fullResultPath) {
|
||||
return {
|
||||
ok: payload.ok,
|
||||
env: payload.env,
|
||||
environment: payload.environment,
|
||||
status: payload.status,
|
||||
error: payload.error ?? null,
|
||||
action: payload.action ?? null,
|
||||
dryRun: payload.dryRun,
|
||||
mutation: payload.mutation,
|
||||
remoteHost: payload.remoteHost,
|
||||
workspace: payload.workspace,
|
||||
kubeconfig: payload.kubeconfig,
|
||||
dumpPath: payload.dumpPath,
|
||||
remoteFullResultPath: fullResultPath,
|
||||
repo: payload.repo ? {
|
||||
status: payload.repo.status,
|
||||
path: payload.repo.path,
|
||||
source: payload.repo.source,
|
||||
rejected: payload.repo.rejected,
|
||||
ephemeral: payload.repo.ephemeral === true,
|
||||
cacheRefreshStatus: payload.repo.cacheRefreshStatus || null,
|
||||
} : null,
|
||||
worktreeGuard: compactGit(payload.worktreeGuard),
|
||||
nodeGuard: compactNodeGuard(payload.nodeGuard),
|
||||
secretPreflight: compactSecretPreflight(payload.secretPreflight),
|
||||
lockState: payload.lockState ? {
|
||||
status: payload.lockState.status,
|
||||
phase: payload.lockState.phase,
|
||||
holder: payload.lockState.ownerTaskId || payload.lockState.holderIdentity || null,
|
||||
held: payload.lockState.held,
|
||||
stale: payload.lockState.stale,
|
||||
retryAfterSeconds: payload.lockState.retryAfterSeconds,
|
||||
lockName: payload.lockState.lockName,
|
||||
} : null,
|
||||
desiredState: payload.desiredState ? {
|
||||
deployJson: payload.desiredState.deployJson,
|
||||
artifactCatalog: payload.desiredState.artifactCatalog,
|
||||
artifactReport: payload.desiredState.artifactReport,
|
||||
} : null,
|
||||
target: payload.target,
|
||||
promotion: payload.promotion ? {
|
||||
source: payload.promotion.source,
|
||||
deployJson: payload.promotion.deployJson,
|
||||
artifactCatalog: payload.promotion.artifactCatalog,
|
||||
artifactReport: payload.promotion.artifactReport,
|
||||
} : null,
|
||||
controlledDevCd: compactControlled(payload.controlledDevCd),
|
||||
audit: compactAudit(payload.audit),
|
||||
blockers: compactBlockers(payload.blockers, 12),
|
||||
blockerTypes: payload.blockerTypes || [...new Set(compactBlockers(payload.blockers, 12).map((item) => item.scope).filter(Boolean))],
|
||||
nextSafeCommand: payload.nextSafeCommand,
|
||||
safety: payload.safety,
|
||||
};
|
||||
}
|
||||
|
||||
function emitResult(payload, dumpDir, compactStdout = false) {
|
||||
const fullResultPath = path.join(dumpDir, "result.full.json");
|
||||
const compactResultPath = path.join(dumpDir, "result.compact.json");
|
||||
fs.writeFileSync(fullResultPath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 });
|
||||
if (!compactStdout) {
|
||||
fs.writeFileSync(compactResultPath, `${JSON.stringify(compactForTransport(payload, fullResultPath), null, 2)}\n`, { mode: 0o600 });
|
||||
console.log(JSON.stringify(payload, null, 2));
|
||||
return;
|
||||
}
|
||||
let compact = compactForTransport(payload, fullResultPath);
|
||||
let text = JSON.stringify(compact, null, 2);
|
||||
if (Buffer.byteLength(text, "utf8") > 3600) {
|
||||
compact = {
|
||||
ok: compact.ok,
|
||||
env: compact.env,
|
||||
status: compact.status,
|
||||
error: compact.error,
|
||||
action: compact.action,
|
||||
dryRun: compact.dryRun,
|
||||
mutation: compact.mutation,
|
||||
workspace: compact.workspace,
|
||||
dumpPath: compact.dumpPath,
|
||||
remoteFullResultPath: compact.remoteFullResultPath,
|
||||
repo: compact.repo,
|
||||
worktreeGuard: compact.worktreeGuard,
|
||||
nodeGuard: compact.nodeGuard,
|
||||
lockState: compact.lockState,
|
||||
target: compact.target ? {
|
||||
ref: compact.target.ref,
|
||||
promotionCommit: compact.target.promotionCommit,
|
||||
shortCommitId: compact.target.shortCommitId,
|
||||
promotionSource: compact.target.promotionSource,
|
||||
publishRequired: compact.target.publishRequired,
|
||||
} : null,
|
||||
controlledDevCd: compact.controlledDevCd,
|
||||
audit: compact.audit ? {
|
||||
status: compact.audit.status,
|
||||
blockerTypes: compact.audit.blockerTypes,
|
||||
blockers: compact.audit.blockers,
|
||||
publicHealth: compact.audit.publicHealth,
|
||||
workload: compact.audit.workload,
|
||||
durability: compact.audit.durability,
|
||||
} : null,
|
||||
blockers: compact.blockers,
|
||||
blockerTypes: compact.blockerTypes,
|
||||
nextSafeCommand: compact.nextSafeCommand,
|
||||
safety: compact.safety,
|
||||
outputCompacted: "provider-host-ssh-stdout-limit",
|
||||
};
|
||||
text = JSON.stringify(compact, null, 2);
|
||||
}
|
||||
if (Buffer.byteLength(text, "utf8") > 2400) {
|
||||
compact = {
|
||||
ok: payload.ok,
|
||||
env: payload.env,
|
||||
status: payload.status,
|
||||
error: payload.error ?? null,
|
||||
action: payload.action ?? null,
|
||||
dryRun: payload.dryRun,
|
||||
mutation: payload.mutation,
|
||||
workspace: payload.workspace,
|
||||
dumpPath: payload.dumpPath,
|
||||
remoteFullResultPath: fullResultPath,
|
||||
repo: payload.repo ? {
|
||||
status: payload.repo.status,
|
||||
path: payload.repo.path,
|
||||
source: payload.repo.source,
|
||||
rejected: payload.repo.rejected,
|
||||
ephemeral: payload.repo.ephemeral === true,
|
||||
cacheRefreshStatus: payload.repo.cacheRefreshStatus || null,
|
||||
} : null,
|
||||
worktreeStatus: payload.worktreeGuard?.status ?? null,
|
||||
nodeGuardStatus: payload.nodeGuard?.status ?? payload.audit?.nodeGuard?.status ?? null,
|
||||
lockState: payload.lockState ? {
|
||||
status: payload.lockState.status,
|
||||
phase: payload.lockState.phase,
|
||||
held: payload.lockState.held,
|
||||
holder: payload.lockState.ownerTaskId || payload.lockState.holderIdentity || null,
|
||||
stale: payload.lockState.stale,
|
||||
} : null,
|
||||
target: payload.target ? {
|
||||
ref: payload.target.ref,
|
||||
promotionCommit: payload.target.promotionCommit,
|
||||
shortCommitId: payload.target.shortCommitId,
|
||||
promotionSource: payload.target.promotionSource,
|
||||
} : payload.audit?.desiredState ? {
|
||||
targetCommit: payload.audit.desiredState.targetCommit,
|
||||
} : null,
|
||||
controlledDevCdStatus: payload.controlledDevCd?.status ?? payload.audit?.controlledDevCd?.status ?? null,
|
||||
auditStatus: payload.audit?.status ?? null,
|
||||
blockerTypes: payload.blockerTypes || payload.audit?.blockerTypes || [...new Set(compactBlockers(payload.blockers, 8).map((item) => item.scope).filter(Boolean))],
|
||||
blockers: compactBlockers(payload.blockers || payload.audit?.blockers, 6),
|
||||
nextSafeCommand: payload.nextSafeCommand,
|
||||
outputCompacted: "provider-host-ssh-stdout-limit",
|
||||
};
|
||||
text = JSON.stringify(compact);
|
||||
}
|
||||
if (Buffer.byteLength(text, "utf8") > 520) {
|
||||
const blockerTypes = payload.blockerTypes
|
||||
|| payload.audit?.blockerTypes
|
||||
|| [...new Set(compactBlockers(payload.blockers || payload.audit?.blockers, 8).map((item) => item.scope).filter(Boolean))];
|
||||
compact = {
|
||||
ok: payload.ok,
|
||||
env: payload.env,
|
||||
status: payload.status,
|
||||
error: payload.error ?? null,
|
||||
action: payload.action ?? null,
|
||||
remoteFullResultPath: fullResultPath,
|
||||
repoPath: payload.repo?.path ?? payload.workspace?.path ?? null,
|
||||
repoEphemeral: payload.repo?.ephemeral === true,
|
||||
cacheRefreshStatus: payload.repo?.cacheRefreshStatus ?? null,
|
||||
worktreeStatus: payload.worktreeGuard?.status ?? null,
|
||||
nodeGuardStatus: payload.nodeGuard?.status ?? payload.audit?.nodeGuard?.status ?? null,
|
||||
lockStatus: payload.lockState ? `${payload.lockState.status || "unknown"}/${payload.lockState.phase || "unknown"}/${payload.lockState.held === true ? "held" : "free"}` : null,
|
||||
targetCommit: payload.target?.promotionCommit ?? payload.audit?.desiredState?.targetCommit ?? null,
|
||||
blockerTypes: Array.isArray(blockerTypes) ? blockerTypes.slice(0, 4) : [],
|
||||
outputCompacted: true,
|
||||
};
|
||||
text = JSON.stringify(compact);
|
||||
}
|
||||
fs.writeFileSync(compactResultPath, `${text}\n`, { mode: 0o600 });
|
||||
console.log(text);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = decodeOptions();
|
||||
const timeoutMs = Math.min(Number(options.timeoutMs || 45000), 60000);
|
||||
@@ -1146,8 +1749,8 @@ async function main() {
|
||||
const dumpDir = path.join(os.homedir(), ".state", "unidesk-hwlab-cd", runId);
|
||||
await fsp.mkdir(dumpDir, { recursive: true, mode: 0o700 });
|
||||
|
||||
const repo = resolveRepo(options.repoPath || null);
|
||||
const action = options.action;
|
||||
const repo = await resolveRepo(options.repoPath || null, dumpDir, Math.min(timeoutMs, 55000));
|
||||
const base = {
|
||||
ok: false,
|
||||
env: "dev",
|
||||
@@ -1176,7 +1779,7 @@ async function main() {
|
||||
|
||||
if (repo.status !== "selected") {
|
||||
const blockers = collectBlockers({ repo, action });
|
||||
console.log(JSON.stringify({ ...base, status: "blocked", error: "hwlab-repo-not-found", repo, blockers, nextSafeCommand: nextSafeCommand(action, blockers) }, null, 2));
|
||||
emitResult({ ...base, status: "blocked", error: "hwlab-repo-not-found", repo, blockers, nextSafeCommand: nextSafeCommand(action, blockers) }, dumpDir, options.compactStdout === true);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -1218,7 +1821,7 @@ async function main() {
|
||||
]);
|
||||
const durability = runtimeDurabilityAudit(auditDesiredState, publicHealth, secretRefs, workload);
|
||||
const summary = auditSummary({ repo, git, guard, secretRefs, registry, lock, desired: auditDesiredState, workload, publicHealth, durability, controlled, dumpDir });
|
||||
console.log(JSON.stringify({
|
||||
emitResult({
|
||||
...base,
|
||||
ok: summary.ok,
|
||||
status: summary.status,
|
||||
@@ -1235,13 +1838,13 @@ async function main() {
|
||||
blockers: summary.blockers,
|
||||
blockerTypes: summary.blockerTypes,
|
||||
nextSafeCommand: nextSafeCommand(action, summary.blockers),
|
||||
}, null, 2));
|
||||
}, dumpDir, options.compactStdout === true);
|
||||
process.exitCode = summary.ok ? 0 : 1;
|
||||
return;
|
||||
}
|
||||
const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action });
|
||||
const ok = blockers.length === 0;
|
||||
console.log(JSON.stringify({
|
||||
emitResult({
|
||||
...base,
|
||||
ok,
|
||||
status: ok ? action === "status" ? "ready" : "prepared" : guard.refusal ? "refused" : "blocked",
|
||||
@@ -1263,7 +1866,7 @@ async function main() {
|
||||
reportDumpPath: controlled.command?.dump?.stdout || dumpDir,
|
||||
blockers,
|
||||
nextSafeCommand: nextSafeCommand(action, blockers),
|
||||
}, null, 2));
|
||||
}, dumpDir, options.compactStdout === true);
|
||||
process.exitCode = ok ? 0 : 1;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user