feat(hwlab): add dev cd audit
This commit is contained in:
Codex
+1
-1
@@ -49,7 +49,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
|
||||
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and merge blocked." },
|
||||
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
|
||||
{ command: "hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Bounded HWLAB DEV CD wrapper that calls HWLAB repo-owned scripts, forces D601 native k3s kubeconfig, refuses Docker Desktop control-plane signals, and writes full stdout/stderr dumps." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Bounded HWLAB DEV CD wrapper that calls HWLAB repo-owned scripts, forces D601 native k3s kubeconfig, refuses Docker Desktop control-plane signals, and exposes read-only post-recovery blocker classification." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
{ command: "schedule upsert-pgdata-backup [--time HH:MM] [--remote-base /SERVER_DATA/UNIDESK_PG_DATA]", description: "Create or update the daily PGDATA physical backup task that uploads monthly rotated archives to Baidu Netdisk." },
|
||||
|
||||
@@ -276,6 +276,168 @@ function readJsonSummary(repoPath, relativePath, fallback = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
function readJsonFile(repoPath, relativePath) {
|
||||
const absolutePath = path.join(repoPath, relativePath);
|
||||
try {
|
||||
const raw = fs.readFileSync(absolutePath, "utf8");
|
||||
return { ok: true, path: relativePath, exists: true, hash: sha256(raw), json: JSON.parse(raw), error: null };
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
path: relativePath,
|
||||
exists: fs.existsSync(absolutePath),
|
||||
hash: null,
|
||||
json: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function commitMatches(actual, expected) {
|
||||
if (typeof actual !== "string" || typeof expected !== "string" || actual.length === 0 || expected.length === 0) return false;
|
||||
return actual === expected || actual.startsWith(expected) || expected.startsWith(actual);
|
||||
}
|
||||
|
||||
function imageTag(image) {
|
||||
const value = String(image || "");
|
||||
const slash = value.lastIndexOf("/");
|
||||
const colon = value.lastIndexOf(":");
|
||||
return colon > slash ? value.slice(colon + 1) : null;
|
||||
}
|
||||
|
||||
function servicesFromManifest(manifest) {
|
||||
const services = Array.isArray(manifest?.services)
|
||||
? manifest.services
|
||||
: manifest?.services && typeof manifest.services === "object"
|
||||
? Object.values(manifest.services)
|
||||
: [];
|
||||
return services
|
||||
.filter((service) => service && typeof service === "object")
|
||||
.map((service) => ({
|
||||
serviceId: stringValue(service.serviceId) || stringValue(service.id) || stringValue(service.name),
|
||||
name: stringValue(service.name) || stringValue(service.serviceId) || stringValue(service.id),
|
||||
image: stringValue(service.image),
|
||||
imageTag: stringValue(service.imageTag) || imageTag(service.image),
|
||||
commitId: stringValue(service.commitId) || stringValue(service.imageTag) || imageTag(service.image),
|
||||
namespace: stringValue(service.namespace) || namespace,
|
||||
healthPath: stringValue(service.healthPath) || "/health/live",
|
||||
replicas: Number.isInteger(service.replicas) ? service.replicas : null,
|
||||
env: asRecord(service.env) || {},
|
||||
digest: stringValue(service.digest),
|
||||
publishState: stringValue(service.publishState) || stringValue(service.status),
|
||||
artifactRequired: service.artifactRequired !== false,
|
||||
}))
|
||||
.filter((service) => service.serviceId);
|
||||
}
|
||||
|
||||
function summarizeDeployFile(file) {
|
||||
const manifest = asRecord(file.json);
|
||||
const endpoints = asRecord(manifest?.publicEndpoints) || {};
|
||||
return {
|
||||
path: file.path,
|
||||
exists: file.exists,
|
||||
hash: file.hash,
|
||||
commitId: stringValue(manifest?.commitId),
|
||||
environment: stringValue(manifest?.environment),
|
||||
namespace: stringValue(manifest?.namespace) || namespace,
|
||||
endpoint: stringValue(manifest?.endpoint),
|
||||
serviceCount: servicesFromManifest(manifest).length,
|
||||
publicEndpoints: {
|
||||
frontend: stringValue(asRecord(endpoints.frontend)?.url),
|
||||
api: stringValue(asRecord(endpoints.api)?.url),
|
||||
},
|
||||
unavailableReason: file.ok ? null : file.error,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeArtifactFile(file) {
|
||||
const manifest = asRecord(file.json);
|
||||
const services = servicesFromManifest(manifest);
|
||||
const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 };
|
||||
for (const service of services) {
|
||||
if (/^sha256:[a-f0-9]{64}$/u.test(service.digest || "")) digestCounts.sha256 += 1;
|
||||
else if (service.digest === "not_published") digestCounts.notPublished += 1;
|
||||
else digestCounts.invalid += 1;
|
||||
}
|
||||
return {
|
||||
path: file.path,
|
||||
exists: file.exists,
|
||||
hash: file.hash,
|
||||
commitId: stringValue(manifest?.commitId),
|
||||
artifactState: stringValue(manifest?.artifactState),
|
||||
ciPublished: manifest?.publish?.ciPublished ?? null,
|
||||
registryVerified: manifest?.publish?.registryVerified ?? null,
|
||||
serviceCount: services.length,
|
||||
digestCounts,
|
||||
unavailableReason: file.ok ? null : file.error,
|
||||
};
|
||||
}
|
||||
|
||||
function buildDesiredStateAudit(repoPath, target) {
|
||||
const deployFile = readJsonFile(repoPath, "deploy/deploy.json");
|
||||
const catalogFile = readJsonFile(repoPath, "deploy/artifact-catalog.dev.json");
|
||||
const reportFile = readJsonFile(repoPath, "reports/dev-gate/dev-artifacts.json");
|
||||
const deployServices = servicesFromManifest(deployFile.json);
|
||||
const catalogServices = servicesFromManifest(catalogFile.json);
|
||||
const catalogById = new Map(catalogServices.map((service) => [service.serviceId, service]));
|
||||
const mismatches = [];
|
||||
const missing = [];
|
||||
for (const service of deployServices) {
|
||||
if (!service.image) continue;
|
||||
const catalog = catalogById.get(service.serviceId);
|
||||
if (!catalog && service.artifactRequired) {
|
||||
missing.push({ serviceId: service.serviceId, reason: "missing-from-artifact-catalog" });
|
||||
continue;
|
||||
}
|
||||
if (catalog?.image && catalog.image !== service.image) {
|
||||
mismatches.push({ serviceId: service.serviceId, deployImage: service.image, catalogImage: catalog.image });
|
||||
}
|
||||
if (catalog && catalog.artifactRequired && catalog.publishState && catalog.publishState !== "published") {
|
||||
missing.push({ serviceId: service.serviceId, reason: `artifact-${catalog.publishState}` });
|
||||
}
|
||||
}
|
||||
const deploySummary = summarizeDeployFile(deployFile);
|
||||
const catalogSummary = summarizeArtifactFile(catalogFile);
|
||||
const reportSummary = summarizeArtifactFile(reportFile);
|
||||
const targetCommit = stringValue(target?.shortCommitId) || stringValue(target?.promotionCommit) || deploySummary.commitId;
|
||||
const commitConverged = Boolean(
|
||||
targetCommit &&
|
||||
commitMatches(deploySummary.commitId, targetCommit) &&
|
||||
(!catalogSummary.commitId || commitMatches(catalogSummary.commitId, targetCommit))
|
||||
);
|
||||
const status = !deployFile.ok || !catalogFile.ok || missing.length > 0
|
||||
? "missing"
|
||||
: mismatches.length > 0 || !commitConverged
|
||||
? "mismatch"
|
||||
: "pass";
|
||||
return {
|
||||
status,
|
||||
targetCommit,
|
||||
commitConverged,
|
||||
deployJson: deploySummary,
|
||||
artifactCatalog: catalogSummary,
|
||||
artifactReport: reportSummary,
|
||||
imageConvergence: {
|
||||
status: missing.length === 0 && mismatches.length === 0 ? "pass" : missing.length > 0 ? "missing" : "mismatch",
|
||||
serviceCount: deployServices.length,
|
||||
catalogServiceCount: catalogServices.length,
|
||||
missing: missing.slice(0, 12),
|
||||
mismatches: mismatches.slice(0, 12),
|
||||
},
|
||||
services: deployServices.map((service) => ({
|
||||
serviceId: service.serviceId,
|
||||
image: service.image,
|
||||
imageTag: service.imageTag,
|
||||
catalogImage: catalogById.get(service.serviceId)?.image || null,
|
||||
catalogDigest: catalogById.get(service.serviceId)?.digest || null,
|
||||
replicas: service.replicas,
|
||||
})).slice(0, 30),
|
||||
serviceEnv: {
|
||||
cloudApi: asRecord(deployServices.find((service) => service.serviceId === "hwlab-cloud-api")?.env) || {},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function forbiddenKubeSignals(text) {
|
||||
const signals = [];
|
||||
if (/docker-desktop/iu.test(text)) signals.push("docker-desktop");
|
||||
@@ -490,6 +652,409 @@ async function secretPreflight(kubeconfig, guard, dumpDir, timeoutMs) {
|
||||
};
|
||||
}
|
||||
|
||||
async function registryAudit(kubeconfig, guard, dumpDir, timeoutMs) {
|
||||
const result = await runCaptured(["curl", "-fsS", "--max-time", "5", "http://127.0.0.1:5000/v2/"], process.cwd(), dumpDir, "registry-v2", { timeoutMs: Math.min(timeoutMs, 8000) });
|
||||
const reachable = result.ok || /401|unauthorized/iu.test(`${result.stdoutText}\n${result.stderrText}`);
|
||||
let k3sPullAccess = { status: "skipped", reason: "d601-native-k3s-guard-not-pass", pullFailureCount: null, pulledRegistryImageCount: null };
|
||||
if (guard.status === "pass") {
|
||||
const env = { ...process.env, KUBECONFIG: kubeconfig };
|
||||
const pods = await runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "registry-k3s-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) });
|
||||
const parsed = asRecord(parseJson(pods.stdoutText));
|
||||
const items = Array.isArray(parsed?.items) ? parsed.items : [];
|
||||
let pullFailureCount = 0;
|
||||
let pulledRegistryImageCount = 0;
|
||||
for (const item of items) {
|
||||
const statuses = Array.isArray(item?.status?.containerStatuses) ? item.status.containerStatuses : [];
|
||||
for (const status of statuses) {
|
||||
const image = String(status?.image || "");
|
||||
const imageID = String(status?.imageID || "");
|
||||
const waiting = String(status?.state?.waiting?.reason || "");
|
||||
if (/ErrImagePull|ImagePullBackOff|InvalidImageName/u.test(waiting)) pullFailureCount += 1;
|
||||
if (image.includes("127.0.0.1:5000/") && imageID.includes("sha256:")) pulledRegistryImageCount += 1;
|
||||
}
|
||||
}
|
||||
k3sPullAccess = {
|
||||
status: !pods.ok ? "unavailable" : pullFailureCount > 0 ? "blocked" : pulledRegistryImageCount > 0 ? "pass" : "degraded",
|
||||
pullFailureCount,
|
||||
pulledRegistryImageCount,
|
||||
command: commandView(pods),
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: reachable && !["blocked", "unavailable"].includes(k3sPullAccess.status) ? "pass" : reachable ? "degraded" : "blocked",
|
||||
endpoint: "http://127.0.0.1:5000/v2/",
|
||||
processHttpAccess: {
|
||||
status: reachable ? "pass" : "blocked",
|
||||
reachable,
|
||||
exitCode: result.exitCode,
|
||||
command: commandView(result),
|
||||
},
|
||||
k3sPullAccess,
|
||||
};
|
||||
}
|
||||
|
||||
function deploymentCondition(conditions, type) {
|
||||
return (Array.isArray(conditions) ? conditions : []).find((condition) => condition?.type === type) || null;
|
||||
}
|
||||
|
||||
function envValue(container, name) {
|
||||
const item = (Array.isArray(container?.env) ? container.env : []).find((entry) => entry?.name === name);
|
||||
return item?.value ?? null;
|
||||
}
|
||||
|
||||
async function workloadAudit(kubeconfig, guard, desired, dumpDir, timeoutMs) {
|
||||
if (guard.status !== "pass") {
|
||||
return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, deployments: [], currentImageConvergence: { status: "unavailable", unavailableReason: "d601-native-k3s-guard-not-pass" } };
|
||||
}
|
||||
const env = { ...process.env, KUBECONFIG: kubeconfig };
|
||||
const [deployments, pods, jobs] = await Promise.all([
|
||||
runCaptured(["kubectl", "-n", namespace, "get", "deployments", "-o", "json"], process.cwd(), dumpDir, "workload-deployments", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
|
||||
runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "workload-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
|
||||
runCaptured(["kubectl", "-n", namespace, "get", "jobs", "-o", "json"], process.cwd(), dumpDir, "runtime-jobs", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
|
||||
]);
|
||||
const desiredById = new Map((desired.services || []).map((service) => [service.serviceId, service]));
|
||||
const deploymentItems = Array.isArray(parseJson(deployments.stdoutText)?.items) ? parseJson(deployments.stdoutText).items : [];
|
||||
const summaries = deploymentItems.map((deployment) => {
|
||||
const metadata = asRecord(deployment.metadata) || {};
|
||||
const spec = asRecord(deployment.spec) || {};
|
||||
const status = asRecord(deployment.status) || {};
|
||||
const template = asRecord(spec.template) || {};
|
||||
const podSpec = asRecord(template.spec) || {};
|
||||
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
|
||||
const container = containers[0] || {};
|
||||
const labels = asRecord(metadata.labels) || {};
|
||||
const serviceId = stringValue(labels["hwlab.pikastech.local/service-id"]) || stringValue(labels["app.kubernetes.io/name"]) || stringValue(metadata.name);
|
||||
const desiredService = desiredById.get(serviceId);
|
||||
const image = stringValue(container.image);
|
||||
const desiredImage = desiredService?.image || null;
|
||||
const available = Number(status.availableReplicas || 0);
|
||||
const desiredReplicas = Number(spec.replicas || 0);
|
||||
const updated = Number(status.updatedReplicas || 0);
|
||||
const unavailable = Number(status.unavailableReplicas || 0);
|
||||
const progressing = deploymentCondition(status.conditions, "Progressing");
|
||||
const availableCondition = deploymentCondition(status.conditions, "Available");
|
||||
return {
|
||||
name: stringValue(metadata.name),
|
||||
serviceId,
|
||||
image,
|
||||
imageTag: imageTag(image),
|
||||
desiredImage,
|
||||
imageMatchesDesired: desiredImage ? image === desiredImage : null,
|
||||
revision: stringValue(asRecord(metadata.annotations)?.["deployment.kubernetes.io/revision"]),
|
||||
replicas: desiredReplicas,
|
||||
availableReplicas: available,
|
||||
updatedReplicas: updated,
|
||||
unavailableReplicas: unavailable,
|
||||
rolloutStatus: desiredReplicas === 0 || (available >= desiredReplicas && updated >= desiredReplicas && unavailable === 0) ? "healthy" : "unhealthy",
|
||||
conditions: {
|
||||
available: availableCondition ? { status: availableCondition.status ?? null, reason: availableCondition.reason ?? null } : null,
|
||||
progressing: progressing ? { status: progressing.status ?? null, reason: progressing.reason ?? null } : null,
|
||||
},
|
||||
commitEnv: envValue(container, "HWLAB_COMMIT_ID"),
|
||||
};
|
||||
});
|
||||
const podItems = Array.isArray(parseJson(pods.stdoutText)?.items) ? parseJson(pods.stdoutText).items : [];
|
||||
const waiting = [];
|
||||
for (const pod of podItems) {
|
||||
const podName = stringValue(pod?.metadata?.name);
|
||||
for (const status of Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []) {
|
||||
const reason = stringValue(status?.state?.waiting?.reason);
|
||||
if (reason) waiting.push({ pod: podName, container: status.name ?? null, reason, image: status.image ?? null });
|
||||
}
|
||||
}
|
||||
const jobItems = Array.isArray(parseJson(jobs.stdoutText)?.items) ? parseJson(jobs.stdoutText).items : [];
|
||||
const runtimeJobs = jobItems
|
||||
.filter((job) => /hwlab-runtime|runtime-db|runtime/i.test(String(job?.metadata?.name || "")))
|
||||
.map((job) => ({
|
||||
name: stringValue(job?.metadata?.name),
|
||||
succeeded: Number(job?.status?.succeeded || 0),
|
||||
failed: Number(job?.status?.failed || 0),
|
||||
active: Number(job?.status?.active || 0),
|
||||
completionTime: stringValue(job?.status?.completionTime),
|
||||
status: Number(job?.status?.failed || 0) > 0 ? "blocked" : Number(job?.status?.active || 0) > 0 ? "running" : Number(job?.status?.succeeded || 0) > 0 ? "pass" : "unknown",
|
||||
}))
|
||||
.slice(0, 12);
|
||||
const imageMismatches = summaries.filter((deployment) => deployment.imageMatchesDesired === false);
|
||||
const unhealthy = summaries.filter((deployment) => deployment.rolloutStatus === "unhealthy");
|
||||
return {
|
||||
status: !deployments.ok ? "unavailable" : unhealthy.length > 0 ? "unhealthy" : imageMismatches.length > 0 ? "mismatch" : "healthy",
|
||||
namespace,
|
||||
deploymentCount: summaries.length,
|
||||
deployments: summaries.slice(0, 30),
|
||||
currentImageConvergence: {
|
||||
status: imageMismatches.length === 0 ? "pass" : "mismatch",
|
||||
mismatches: imageMismatches.map((deployment) => ({ serviceId: deployment.serviceId, image: deployment.image, desiredImage: deployment.desiredImage })).slice(0, 12),
|
||||
},
|
||||
podWaiting: waiting.slice(0, 12),
|
||||
runtimeJobs,
|
||||
commands: [deployments, pods, jobs].map(commandView),
|
||||
};
|
||||
}
|
||||
|
||||
function compactHealth(body, expectedServiceId, expectedCommit) {
|
||||
const json = asRecord(body);
|
||||
const commit = stringValue(asRecord(json?.commit)?.id) || stringValue(json?.commitId) || stringValue(json?.revision) || stringValue(asRecord(json?.image)?.tag);
|
||||
const runtime = asRecord(json?.runtime);
|
||||
const db = asRecord(json?.db);
|
||||
const blockerCodes = Array.isArray(json?.blockerCodes) ? json.blockerCodes.map(String).slice(0, 12) : [];
|
||||
return {
|
||||
serviceId: stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id),
|
||||
environment: stringValue(json?.environment),
|
||||
status: stringValue(json?.status),
|
||||
ready: typeof json?.ready === "boolean" ? json.ready : null,
|
||||
observedCommit: commit,
|
||||
commitMatches: expectedCommit ? commitMatches(commit, expectedCommit) : null,
|
||||
serviceMatches: expectedServiceId ? (stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id)) === expectedServiceId : null,
|
||||
image: typeof json?.image === "string" ? json.image : stringValue(asRecord(json?.image)?.reference),
|
||||
blockerCodes,
|
||||
db: db === null ? null : {
|
||||
ready: db.ready ?? null,
|
||||
connected: db.connected ?? null,
|
||||
liveDbEvidence: db.liveDbEvidence ?? null,
|
||||
runtimeReadiness: asRecord(db.runtimeReadiness) ? {
|
||||
status: stringValue(db.runtimeReadiness.status),
|
||||
ready: db.runtimeReadiness.ready ?? null,
|
||||
blocker: stringValue(db.runtimeReadiness.blocker),
|
||||
} : null,
|
||||
},
|
||||
runtime: runtime === null ? null : {
|
||||
status: stringValue(runtime.status),
|
||||
ready: runtime.ready ?? null,
|
||||
durable: runtime.durable ?? null,
|
||||
durableRequested: runtime.durableRequested ?? null,
|
||||
liveRuntimeEvidence: runtime.liveRuntimeEvidence ?? null,
|
||||
blocker: stringValue(runtime.blocker),
|
||||
adapter: stringValue(runtime.adapter),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function httpHealthAudit(desired, dumpDir, timeoutMs) {
|
||||
const endpoints = [
|
||||
{ id: "cloud-web-16666", url: desired.deployJson.publicEndpoints.frontend || "http://74.48.78.17:16666", expectedServiceId: "hwlab-cloud-web" },
|
||||
{ id: "cloud-api-16667", url: desired.deployJson.publicEndpoints.api || desired.deployJson.endpoint || "http://74.48.78.17:16667", expectedServiceId: "hwlab-cloud-api" },
|
||||
];
|
||||
const expectedCommit = desired.targetCommit || desired.deployJson.commitId || null;
|
||||
const observations = [];
|
||||
for (const endpoint of endpoints) {
|
||||
const result = await runCaptured(["curl", "-fsS", "--max-time", "8", `${endpoint.url.replace(/\/+$/u, "")}/health/live`], process.cwd(), dumpDir, `public-health-${endpoint.id}`, { timeoutMs: Math.min(timeoutMs, 10000) });
|
||||
const parsed = parseJson(result.stdoutText);
|
||||
const health = compactHealth(parsed, endpoint.expectedServiceId, expectedCommit);
|
||||
const reachable = result.ok && parsed !== null;
|
||||
observations.push({
|
||||
id: endpoint.id,
|
||||
url: endpoint.url,
|
||||
status: reachable && health.serviceMatches !== false && health.commitMatches !== false ? "pass" : "blocked",
|
||||
reachable,
|
||||
expectedServiceId: endpoint.expectedServiceId,
|
||||
expectedCommit,
|
||||
health,
|
||||
command: commandView(result),
|
||||
});
|
||||
}
|
||||
return {
|
||||
status: observations.every((item) => item.status === "pass") ? "pass" : "blocked",
|
||||
expectedCommit,
|
||||
summary: {
|
||||
checked: observations.length,
|
||||
reachable: observations.filter((item) => item.reachable).length,
|
||||
commitMatches: observations.filter((item) => item.health.commitMatches === true).length,
|
||||
readinessPass: observations.filter((item) => item.health.ready !== false && !["blocked", "failed"].includes(String(item.health.status || ""))).length,
|
||||
ports: [16666, 16667],
|
||||
},
|
||||
endpoints: observations,
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeDurabilityAudit(desired, publicHealth, secretRefs, workload) {
|
||||
const api = publicHealth?.endpoints?.find((endpoint) => endpoint.id === "cloud-api-16667")?.health || null;
|
||||
const cloudApiEnv = asRecord(desired.serviceEnv?.cloudApi) || {};
|
||||
const configuredDurable = cloudApiEnv.HWLAB_CLOUD_RUNTIME_DURABLE === "true";
|
||||
const configuredAdapter = stringValue(cloudApiEnv.HWLAB_CLOUD_RUNTIME_ADAPTER);
|
||||
const dbSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db" && ref.secretKey === "database-url");
|
||||
const adminSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db-admin" && ref.secretKey === "admin-url");
|
||||
const runtime = asRecord(api?.runtime);
|
||||
const dbRuntimeReadiness = asRecord(api?.db?.runtimeReadiness);
|
||||
const liveReady = runtime?.durable === true && runtime?.ready !== false && runtime?.liveRuntimeEvidence === true && !["blocked", "degraded", "failed"].includes(String(runtime?.status || ""));
|
||||
const status = liveReady
|
||||
? "pass"
|
||||
: api
|
||||
? "risk"
|
||||
: configuredDurable && configuredAdapter === "postgres" && dbSecret?.keyPresent === true && adminSecret?.keyPresent === true
|
||||
? "unavailable"
|
||||
: "risk";
|
||||
return {
|
||||
status,
|
||||
configured: {
|
||||
adapter: configuredAdapter,
|
||||
durable: configuredDurable,
|
||||
dbSecretPresent: dbSecret?.keyPresent === true,
|
||||
adminSecretPresent: adminSecret?.keyPresent === true,
|
||||
},
|
||||
live: api ? {
|
||||
db: api.db,
|
||||
runtime: api.runtime,
|
||||
dbRuntimeReadiness,
|
||||
} : null,
|
||||
workloadEvidence: {
|
||||
cloudApiDeployment: (workload?.deployments || []).find((deployment) => deployment.serviceId === "hwlab-cloud-api") || null,
|
||||
},
|
||||
unavailableReason: api ? null : "public-health-cloud-api-unavailable",
|
||||
risk: status === "pass" ? null : "db-runtime-durability-risk",
|
||||
};
|
||||
}
|
||||
|
||||
function blocker(scope, summary, details = {}) {
|
||||
return { scope, summary, ...details };
|
||||
}
|
||||
|
||||
function classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }) {
|
||||
const blockers = [];
|
||||
if (guard?.status !== "pass") blockers.push(blocker("control-plane-unavailable", guard?.summary || "D601 native k3s guard did not pass."));
|
||||
if (guard?.refusalSignals?.length > 0) blockers.push(blocker("docker-desktop-context-risk", "Explicit D601 kubeconfig resolved to forbidden Docker Desktop control-plane signal.", { refusalSignals: guard.refusalSignals }));
|
||||
if (guard?.secondControlPlaneRisk) blockers.push(blocker("second-control-plane-risk", "Bare kubectl can observe hwlab-dev through a different control plane; audit is read-only and will not release or mutate locks."));
|
||||
if (repo.status !== "selected") blockers.push(blocker("workspace-unavailable", "No eligible HWLAB CD repo was selected."));
|
||||
for (const item of git?.blockers || []) {
|
||||
const scope = item.scope === "hwlab-git-clean" ? "dirty-worktree" : item.scope?.startsWith("hwlab-git") || item.scope?.includes("permission") ? "workspace-unavailable" : item.scope;
|
||||
blockers.push(blocker(scope, item.summary));
|
||||
}
|
||||
for (const item of secretRefs?.blockers || []) blockers.push(blocker("secret-missing", item.summary, { secretRef: item.scope }));
|
||||
if (registry?.status === "blocked" || registry?.status === "degraded") blockers.push(blocker("registry-unavailable", "Registry reachability or k3s pull evidence is not healthy.", { status: registry.status }));
|
||||
if (lock?.held === true) blockers.push(blocker("lease-held", `HWLAB DEV CD Lease is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.`, { phase: lock.phase, retryAfterSeconds: lock.retryAfterSeconds }));
|
||||
if (lock?.stale === true) blockers.push(blocker("lease-stale-candidate", "HWLAB DEV CD Lease appears stale; audit is read-only and did not release or break it.", { phase: lock.phase }));
|
||||
if (desired?.status === "missing") blockers.push(blocker("artifact-missing", "deploy/deploy.json or artifact catalog/report is missing or incomplete.", { imageConvergence: desired.imageConvergence }));
|
||||
if (desired?.status === "mismatch" || workload?.currentImageConvergence?.status === "mismatch") blockers.push(blocker("artifact-mismatch", "deploy/deploy.json, artifact catalog, or current workload images are not converged.", { desired: desired?.status, currentImageConvergence: workload?.currentImageConvergence }));
|
||||
if ((workload?.runtimeJobs || []).some((job) => job.status === "blocked" || job.status === "running")) blockers.push(blocker("runtime-job-blocked", "Runtime DB maintenance Job is blocked or still running.", { runtimeJobs: workload.runtimeJobs }));
|
||||
if (workload?.status === "unhealthy") blockers.push(blocker("rollout-unhealthy", "One or more hwlab-dev Deployments are not fully available.", { deployments: (workload.deployments || []).filter((item) => item.rolloutStatus === "unhealthy").slice(0, 8) }));
|
||||
if (publicHealth?.status === "blocked") blockers.push(blocker("public-tunnel-unhealthy", "16666/16667 public health did not pass read-only commit/readiness checks.", { summary: publicHealth.summary }));
|
||||
if (durability?.status === "risk") blockers.push(blocker("db-runtime-durability-risk", "DB/runtime durability is not proven by live cloud-api health.", { live: durability.live, configured: durability.configured }));
|
||||
if (controlled?.commandOk === false || controlled?.status === "blocked") blockers.push(blocker("runtime-job-blocked", "HWLAB repo-owned dev-cd-apply status reported blocked or failed.", { status: controlled?.status }));
|
||||
return blockers;
|
||||
}
|
||||
|
||||
function auditSummary({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled, dumpDir }) {
|
||||
const blockers = classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled });
|
||||
const blockerTypes = [...new Set(blockers.map((item) => item.scope))];
|
||||
return {
|
||||
ok: blockers.length === 0,
|
||||
status: blockers.length === 0 ? "pass" : guard?.refusal ? "refused" : "blocked",
|
||||
env: "dev",
|
||||
namespace,
|
||||
nodeGuard: {
|
||||
status: guard?.status ?? "unavailable",
|
||||
nodeName: guard?.requiredNodeName ?? requiredNodeName,
|
||||
nodeNames: guard?.nodeNames ?? [],
|
||||
requiredNodePresent: guard?.requiredNodePresent ?? false,
|
||||
currentContext: guard?.currentContext ?? null,
|
||||
apiServer: guard?.apiServer ?? null,
|
||||
kubeconfig: guard?.kubeconfig ?? nativeKubeconfig,
|
||||
refusalSignals: guard?.refusalSignals ?? [],
|
||||
secondControlPlaneRisk: guard?.secondControlPlaneRisk ?? false,
|
||||
defaultKubectlDiagnostic: guard?.defaultKubectlDiagnostic ?? null,
|
||||
},
|
||||
secrets: {
|
||||
status: secretRefs?.status ?? "unavailable",
|
||||
valuesRead: false,
|
||||
valuesPrinted: false,
|
||||
secretRefs: (secretRefs?.secretRefs || []).map((ref) => ({
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
exists: ref.exists === true,
|
||||
keyPresent: ref.keyPresent === true,
|
||||
status: ref.status,
|
||||
})),
|
||||
unavailableReason: secretRefs?.reason ?? null,
|
||||
},
|
||||
registry: {
|
||||
status: registry?.status ?? "unavailable",
|
||||
endpoint: registry?.endpoint ?? "http://127.0.0.1:5000/v2/",
|
||||
processHttpAccess: registry?.processHttpAccess ? { status: registry.processHttpAccess.status, reachable: registry.processHttpAccess.reachable } : null,
|
||||
k3sPullAccess: registry?.k3sPullAccess ? {
|
||||
status: registry.k3sPullAccess.status,
|
||||
pullFailureCount: registry.k3sPullAccess.pullFailureCount,
|
||||
pulledRegistryImageCount: registry.k3sPullAccess.pulledRegistryImageCount,
|
||||
} : null,
|
||||
},
|
||||
lease: {
|
||||
status: lock?.status ?? "unavailable",
|
||||
phase: lock?.phase ?? null,
|
||||
holder: lock?.ownerTaskId || lock?.holderIdentity || null,
|
||||
ageSeconds: lock?.updatedAt && Number.isFinite(Date.parse(lock.updatedAt)) ? Math.max(0, Math.floor((Date.now() - Date.parse(lock.updatedAt)) / 1000)) : null,
|
||||
retryAfterSeconds: lock?.retryAfterSeconds ?? null,
|
||||
staleClassification: lock?.stale === true ? "lease-stale-candidate" : lock?.held === true ? "lease-held" : "not-held",
|
||||
lockName: lock?.lockName ?? lockName,
|
||||
},
|
||||
desiredState: {
|
||||
status: desired?.status ?? "unavailable",
|
||||
targetCommit: desired?.targetCommit ?? null,
|
||||
deployJson: desired?.deployJson ?? null,
|
||||
artifactCatalog: desired?.artifactCatalog ?? null,
|
||||
artifactReport: desired?.artifactReport ?? null,
|
||||
imageConvergence: desired?.imageConvergence ?? null,
|
||||
},
|
||||
workload: {
|
||||
status: workload?.status ?? "unavailable",
|
||||
currentImageConvergence: workload?.currentImageConvergence ?? null,
|
||||
deploymentCount: workload?.deploymentCount ?? 0,
|
||||
deployments: (workload?.deployments || []).map((deployment) => ({
|
||||
name: deployment.name,
|
||||
serviceId: deployment.serviceId,
|
||||
image: deployment.image,
|
||||
imageTag: deployment.imageTag,
|
||||
desiredImage: deployment.desiredImage,
|
||||
revision: deployment.revision,
|
||||
replicas: deployment.replicas,
|
||||
availableReplicas: deployment.availableReplicas,
|
||||
rolloutStatus: deployment.rolloutStatus,
|
||||
commitEnv: deployment.commitEnv,
|
||||
})),
|
||||
podWaiting: workload?.podWaiting ?? [],
|
||||
runtimeJobs: workload?.runtimeJobs ?? [],
|
||||
},
|
||||
publicHealth: {
|
||||
status: publicHealth?.status ?? "unavailable",
|
||||
expectedCommit: publicHealth?.expectedCommit ?? null,
|
||||
summary: publicHealth?.summary ?? { checked: 0 },
|
||||
endpoints: (publicHealth?.endpoints || []).map((endpoint) => ({
|
||||
id: endpoint.id,
|
||||
url: endpoint.url,
|
||||
status: endpoint.status,
|
||||
reachable: endpoint.reachable,
|
||||
expectedServiceId: endpoint.expectedServiceId,
|
||||
expectedCommit: endpoint.expectedCommit,
|
||||
health: endpoint.health,
|
||||
})),
|
||||
},
|
||||
durability: {
|
||||
status: durability?.status ?? "unavailable",
|
||||
configured: durability?.configured ?? null,
|
||||
live: durability?.live ?? null,
|
||||
unavailableReason: durability?.unavailableReason ?? null,
|
||||
},
|
||||
controlledDevCd: {
|
||||
status: controlled?.status ?? "unavailable",
|
||||
commandOk: controlled?.commandOk ?? null,
|
||||
controlledEntrypoint: "scripts/dev-cd-apply.mjs",
|
||||
target: controlled?.target ?? null,
|
||||
blockers: controlled?.blockers ?? [],
|
||||
},
|
||||
blockers,
|
||||
blockerTypes,
|
||||
dumpPath: dumpDir,
|
||||
safety: {
|
||||
mutation: false,
|
||||
kubectlApplyExecuted: false,
|
||||
rolloutExecuted: false,
|
||||
liveVerifyExecuted: false,
|
||||
cdLockMutated: false,
|
||||
secretValuesRead: false,
|
||||
secretValuesPrinted: false,
|
||||
reportsWritten: false,
|
||||
repoReportsWritten: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeControlled(parsed, command) {
|
||||
const record = asRecord(parsed);
|
||||
const target = asRecord(record?.target);
|
||||
@@ -552,7 +1117,7 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio
|
||||
if (guard) {
|
||||
if (guard.refusal) blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary, refusal: true });
|
||||
else if (guard.status !== "pass") blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary });
|
||||
if ((action === "preflight" || action === "apply") && guard.secondControlPlaneRisk) {
|
||||
if ((action === "preflight" || action === "audit" || action === "apply") && guard.secondControlPlaneRisk) {
|
||||
blockers.push({ scope: "second-hwlab-dev-control-plane", summary: "Bare kubectl can observe hwlab-dev through a different control plane; refusing write-path planning." });
|
||||
}
|
||||
}
|
||||
@@ -568,7 +1133,8 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio
|
||||
}
|
||||
|
||||
function nextSafeCommand(action, blockers) {
|
||||
if (blockers.length > 0) return "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev";
|
||||
if (blockers.length > 0) return action === "audit" ? "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd audit --env dev" : "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev";
|
||||
if (action === "audit") return "audit is read-only; host commander may compare blockerTypes before deciding whether the unique CD runner should continue";
|
||||
if (action === "status") return "bun scripts/cli.ts hwlab cd apply --env dev --dry-run";
|
||||
return "host commander or the unique CD runner may decide whether to run node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report on D601; this wrapper did not apply or rollout";
|
||||
}
|
||||
@@ -603,6 +1169,8 @@ async function main() {
|
||||
secretValuesPrinted: false,
|
||||
cdLockMutated: false,
|
||||
prodTouched: false,
|
||||
reportsWritten: false,
|
||||
repoReportsWritten: false,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -625,17 +1193,15 @@ async function main() {
|
||||
artifactReport: readJsonSummary(repoPath, "reports/dev-gate/dev-artifacts.json"),
|
||||
};
|
||||
const lock = await lockStatus(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000));
|
||||
const needsSecretPreflight = action === "preflight" || action === "apply";
|
||||
const needsSecretPreflight = action === "preflight" || action === "audit" || action === "apply";
|
||||
const secretRefs = needsSecretPreflight
|
||||
? await secretPreflight(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000))
|
||||
: { status: "skipped", reason: "status-does-not-run-secret-preflight", namespace, mutation: false, blockers: [] };
|
||||
const preCommandBlockers = collectBlockers({ repo, git, guard, lock, secretRefs, action });
|
||||
const canRunControlled = preCommandBlockers.length === 0 || action === "status";
|
||||
const canRunControlled = preCommandBlockers.length === 0 || action === "status" || action === "audit";
|
||||
const controlled = canRunControlled && git.status === "pass" && guard.status === "pass"
|
||||
? await runDevCdApply(repoPath, options.kubeconfig || nativeKubeconfig, action === "status" ? "status" : "apply", dumpDir, Math.max(1000, timeoutMs - 5000))
|
||||
: { status: "skipped", commandOk: true, reason: "preflight-blockers-before-controlled-dev-cd-apply" };
|
||||
const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action });
|
||||
const ok = blockers.length === 0;
|
||||
const target = controlled.target || {
|
||||
ref: "deploy/deploy.json",
|
||||
promotionCommit: desiredState.deployJson.commitId || git.headCommit,
|
||||
@@ -643,6 +1209,38 @@ async function main() {
|
||||
promotionSource: "deploy/deploy.json",
|
||||
namespace,
|
||||
};
|
||||
if (action === "audit") {
|
||||
const auditDesiredState = buildDesiredStateAudit(repoPath, target);
|
||||
const [registry, workload, publicHealth] = await Promise.all([
|
||||
registryAudit(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)),
|
||||
workloadAudit(options.kubeconfig || nativeKubeconfig, guard, auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)),
|
||||
httpHealthAudit(auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)),
|
||||
]);
|
||||
const durability = runtimeDurabilityAudit(auditDesiredState, publicHealth, secretRefs, workload);
|
||||
const summary = auditSummary({ repo, git, guard, secretRefs, registry, lock, desired: auditDesiredState, workload, publicHealth, durability, controlled, dumpDir });
|
||||
console.log(JSON.stringify({
|
||||
...base,
|
||||
ok: summary.ok,
|
||||
status: summary.status,
|
||||
audit: summary,
|
||||
repo: {
|
||||
status: repo.status,
|
||||
path: repo.path,
|
||||
source: repo.source,
|
||||
rejected: repo.rejected,
|
||||
},
|
||||
workspace: { path: repoPath, source: repo.source },
|
||||
remoteDumpPath: dumpDir,
|
||||
reportDumpPath: controlled.command?.dump?.stdout || dumpDir,
|
||||
blockers: summary.blockers,
|
||||
blockerTypes: summary.blockerTypes,
|
||||
nextSafeCommand: nextSafeCommand(action, summary.blockers),
|
||||
}, null, 2));
|
||||
process.exitCode = summary.ok ? 0 : 1;
|
||||
return;
|
||||
}
|
||||
const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action });
|
||||
const ok = blockers.length === 0;
|
||||
console.log(JSON.stringify({
|
||||
...base,
|
||||
ok,
|
||||
|
||||
@@ -6,7 +6,7 @@ import { join } from "node:path";
|
||||
import { readConfig, repoRoot, rootPath, type UniDeskConfig } from "./config";
|
||||
import { d601NativeKubeconfig } from "./d601-k3s-guard";
|
||||
|
||||
type HwlabCdAction = "status" | "preflight" | "apply";
|
||||
type HwlabCdAction = "status" | "preflight" | "audit" | "apply";
|
||||
type HwlabCdEnvironment = "dev";
|
||||
type HwlabCdTransport = "frontend" | "local";
|
||||
|
||||
@@ -89,8 +89,8 @@ function envTransport(): HwlabCdTransport {
|
||||
|
||||
function parseOptions(args: string[]): HwlabCdOptions {
|
||||
const [scope, actionArg] = args;
|
||||
if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|apply --env dev");
|
||||
if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|apply");
|
||||
if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|audit|apply --env dev");
|
||||
if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "audit" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|audit|apply");
|
||||
|
||||
const options: HwlabCdOptions = {
|
||||
action: actionArg,
|
||||
@@ -498,6 +498,7 @@ export function hwlabHelp(): Record<string, unknown> {
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts hwlab cd status --env dev",
|
||||
"bun scripts/cli.ts hwlab cd audit --env dev",
|
||||
"bun scripts/cli.ts hwlab cd preflight --env dev",
|
||||
"bun scripts/cli.ts hwlab cd apply --env dev --dry-run",
|
||||
],
|
||||
@@ -508,7 +509,8 @@ export function hwlabHelp(): Record<string, unknown> {
|
||||
`default HWLAB CD repo is ${defaultHwlabCdRepoPath}; ${rejectedRunnerHistoryRepoPath} is rejected as runner history`,
|
||||
"deploy/deploy.json remains the authoritative desired-state source",
|
||||
"preflight/apply --dry-run check required SecretRef object/key metadata without reading or printing Secret values",
|
||||
"status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs with --skip-live-verify; no apply, rollout, lock mutation, live verification, DB write, or secret read is executed",
|
||||
"audit/status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs read-only/status paths with --skip-live-verify; no apply, rollout, lock mutation, DEV acceptance live verification, DB write, or secret read is executed",
|
||||
"audit adds bounded read-only kubectl get/curl health probes for blocker classification; full stdout/stderr stays in temp dump paths, not reports/",
|
||||
"real apply is structured refused and must remain with the host commander or unique CD runner",
|
||||
],
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user