Lyon
e2646763c0
Adds read-only D601 recovery diagnostics, fixture coverage, CLI wiring, and recovery hotfix runbook updates. Validated with recovery contract, check --files, scripts tsc, artifact matrix direct contract, and read-only live diagnostic.
1180 lines
44 KiB
TypeScript
1180 lines
44 KiB
TypeScript
import { lstatSync, readFileSync, readlinkSync, statSync } from "node:fs";
|
|
import { dirname, isAbsolute, resolve } from "node:path";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import { repoRoot, rootPath, type UniDeskConfig } from "./config";
|
|
|
|
type Severity = "info" | "yellow" | "red";
|
|
type Disposition = "safe-read-only" | "requires-manual-host-hotfix" | "forbidden-automatic-action";
|
|
type PathKind = "missing" | "directory" | "file" | "socket" | "symlink" | "other";
|
|
|
|
export interface RecoveryGuardrailRedline {
|
|
id: string;
|
|
severity: Severity;
|
|
disposition: Disposition;
|
|
summary: string;
|
|
evidence: Record<string, unknown>;
|
|
safeReadOnly: boolean;
|
|
requiresManualHostHotfix: boolean;
|
|
forbiddenAutomaticActions: string[];
|
|
recommendedNext: string[];
|
|
}
|
|
|
|
export interface RecoveryGuardrailsResult {
|
|
ok: boolean;
|
|
surface: "d601-recovery-guardrails";
|
|
mutation: false;
|
|
observedAt: string;
|
|
scope: {
|
|
nodeId: "D601";
|
|
environment: "host-read-only";
|
|
liveMutationAllowed: false;
|
|
};
|
|
safeReadOnly: {
|
|
filesRead: string[];
|
|
commandsAttempted: string[][];
|
|
note: string;
|
|
};
|
|
redlineSummary: {
|
|
red: number;
|
|
yellow: number;
|
|
safeReadOnly: boolean;
|
|
requiresManualHostHotfix: string[];
|
|
forbiddenAutomaticActions: string[];
|
|
};
|
|
redlines: RecoveryGuardrailRedline[];
|
|
checks: {
|
|
procMounts: ProcMountsReport;
|
|
kubeletValidationRisk: KubeletValidationRisk;
|
|
criSandboxes: CriSandboxReport;
|
|
codeQueueDeployWorktree: WorktreeReadinessReport;
|
|
hostPaths: HostPathReadinessReport;
|
|
mdtodoAdjacentHostPaths: MdtodoAdjacentReadiness;
|
|
containerCreating: ContainerCreatingReport;
|
|
};
|
|
}
|
|
|
|
export interface RecoveryGuardrailsFixture {
|
|
observedAt?: string;
|
|
procMountsText?: string;
|
|
criPodsJsonText?: string;
|
|
kubectlPodsJsonText?: string;
|
|
pathStates?: Record<string, PathStateFixture>;
|
|
manifestTexts?: Record<string, string>;
|
|
deployWorktreePath?: string;
|
|
compatibilitySymlinkPath?: string;
|
|
manifestPaths?: string[];
|
|
}
|
|
|
|
export interface PathStateFixture {
|
|
kind: PathKind;
|
|
target?: string;
|
|
targetExists?: boolean;
|
|
}
|
|
|
|
export interface CompactRecoveryGuardrailsResult {
|
|
ok: boolean;
|
|
surface: RecoveryGuardrailsResult["surface"];
|
|
mutation: false;
|
|
observedAt: string;
|
|
scope: RecoveryGuardrailsResult["scope"];
|
|
safeReadOnly: RecoveryGuardrailsResult["safeReadOnly"];
|
|
redlineSummary: RecoveryGuardrailsResult["redlineSummary"];
|
|
redlines: Array<Pick<RecoveryGuardrailRedline, "id" | "severity" | "disposition" | "summary" | "recommendedNext">>;
|
|
checks: {
|
|
procMounts: Pick<ProcMountsReport, "ok" | "source" | "totalLines" | "malformedLines" | "dockerDesktopHost9pLines" | "readError">;
|
|
kubeletValidationRisk: KubeletValidationRisk;
|
|
criSandboxes: Pick<CriSandboxReport, "ok" | "source" | "command" | "total" | "notReadyCount" | "staleNotReadyCount" | "unknownAgeCount" | "staleAfterMinutes" | "staleSandboxes" | "parseError">;
|
|
codeQueueDeployWorktree: WorktreeReadinessReport;
|
|
hostPaths: {
|
|
ok: boolean;
|
|
manifests: CompactHostPathManifest[];
|
|
totalEntries: number;
|
|
checkedCount: number;
|
|
missing: CompactHostPathProblem[];
|
|
typeMismatches: CompactHostPathProblem[];
|
|
directoryOrCreateMissing: CompactHostPathProblem[];
|
|
sensitiveUserDataBoundaries: string[];
|
|
};
|
|
mdtodoAdjacentHostPaths: MdtodoAdjacentReadiness;
|
|
containerCreating: Pick<ContainerCreatingReport, "ok" | "source" | "command" | "totalContainerCreating" | "opaqueContainerCreating" | "hostPathContainerCreating" | "parseError">;
|
|
};
|
|
}
|
|
|
|
interface CompactHostPathProblem {
|
|
hostPath: string;
|
|
hostPathType: string;
|
|
manifest: string;
|
|
workload: string;
|
|
volumeName: string | null;
|
|
problem: string | null;
|
|
severity: Severity;
|
|
disposition: Disposition;
|
|
reason: string;
|
|
}
|
|
|
|
interface CompactHostPathManifest {
|
|
path: string;
|
|
readError: string | null;
|
|
textBytes: number;
|
|
}
|
|
|
|
interface ProcMountsReport {
|
|
ok: boolean;
|
|
source: string;
|
|
totalLines: number;
|
|
malformedLines: MountLineFinding[];
|
|
dockerDesktopHost9pLines: MountLineFinding[];
|
|
readError: string | null;
|
|
}
|
|
|
|
interface MountLineFinding {
|
|
lineNumber: number;
|
|
fieldCount: number;
|
|
dockerDesktopHost9p: boolean;
|
|
textPreview: string;
|
|
reason: string;
|
|
}
|
|
|
|
interface KubeletValidationRisk {
|
|
ok: boolean;
|
|
risk: "none" | "kubelet-mount-table-validation-risk";
|
|
severity: Severity;
|
|
reason: string;
|
|
evidenceLineNumbers: number[];
|
|
}
|
|
|
|
interface CriSandboxReport {
|
|
ok: boolean;
|
|
source: "crictl" | "fixture" | "unavailable";
|
|
command: string[] | null;
|
|
commandResult: CommandResultSummary | null;
|
|
total: number;
|
|
notReadyCount: number;
|
|
staleNotReadyCount: number;
|
|
unknownAgeCount: number;
|
|
staleAfterMinutes: number;
|
|
staleSandboxes: CriSandboxSummary[];
|
|
parseError: string | null;
|
|
}
|
|
|
|
interface CommandResultSummary {
|
|
exitCode: number | null;
|
|
signal: NodeJS.Signals | null;
|
|
timedOut: boolean;
|
|
stdoutTail: string;
|
|
stderrTail: string;
|
|
}
|
|
|
|
interface CriSandboxSummary {
|
|
id: string;
|
|
name: string | null;
|
|
namespace: string | null;
|
|
state: string;
|
|
ageMinutes: number | null;
|
|
}
|
|
|
|
interface WorktreeReadinessReport {
|
|
ok: boolean;
|
|
canonicalPath: PathCheck;
|
|
compatibilitySymlink: SymlinkCheck;
|
|
requiredFor: string[];
|
|
}
|
|
|
|
interface SymlinkCheck extends PathCheck {
|
|
expectedTarget: string;
|
|
targetExists: boolean;
|
|
targetKind: PathKind;
|
|
}
|
|
|
|
interface PathCheck {
|
|
path: string;
|
|
exists: boolean;
|
|
kind: PathKind;
|
|
isSymlink: boolean;
|
|
symlinkTarget: string | null;
|
|
resolvedPath: string | null;
|
|
matchesExpectedType: boolean;
|
|
expectedType: string;
|
|
problem: string | null;
|
|
}
|
|
|
|
interface HostPathManifest {
|
|
path: string;
|
|
text: string;
|
|
readError: string | null;
|
|
}
|
|
|
|
interface HostPathEntry {
|
|
manifest: string;
|
|
documentIndex: number;
|
|
kind: string | null;
|
|
name: string | null;
|
|
namespace: string | null;
|
|
volumeName: string | null;
|
|
hostPath: string;
|
|
hostPathType: string;
|
|
}
|
|
|
|
interface HostPathProblem {
|
|
entry: HostPathEntry;
|
|
check: PathCheck;
|
|
severity: Severity;
|
|
disposition: Disposition;
|
|
reason: string;
|
|
}
|
|
|
|
interface HostPathReadinessReport {
|
|
ok: boolean;
|
|
manifests: HostPathManifest[];
|
|
totalEntries: number;
|
|
checkedEntries: HostPathEntry[];
|
|
missing: HostPathProblem[];
|
|
typeMismatches: HostPathProblem[];
|
|
directoryOrCreateMissing: HostPathProblem[];
|
|
sensitiveUserDataBoundaries: string[];
|
|
}
|
|
|
|
interface MdtodoAdjacentReadiness {
|
|
ok: boolean;
|
|
workspacePaths: string[];
|
|
logPaths: string[];
|
|
missingPaths: string[];
|
|
opaqueRisk: boolean;
|
|
userDataBoundary: string;
|
|
}
|
|
|
|
interface ContainerCreatingReport {
|
|
ok: boolean;
|
|
source: "kubectl" | "fixture" | "unavailable";
|
|
command: string[] | null;
|
|
commandResult: CommandResultSummary | null;
|
|
totalContainerCreating: number;
|
|
opaqueContainerCreating: ContainerCreatingFinding[];
|
|
hostPathContainerCreating: ContainerCreatingFinding[];
|
|
parseError: string | null;
|
|
}
|
|
|
|
interface ContainerCreatingFinding {
|
|
namespace: string;
|
|
pod: string;
|
|
container: string;
|
|
reason: string;
|
|
messagePreview: string;
|
|
classification: "opaque-containercreating-hostpath-risk" | "hostpath-mount-failure" | "containercreating-observed";
|
|
}
|
|
|
|
const staleSandboxAfterMinutes = 30;
|
|
const defaultCompatibilitySymlinkPath = "/home/ubuntu/cq-deploy";
|
|
const defaultManifestPaths = [
|
|
"src/components/microservices/k3sctl-adapter/k3s/code-queue.k8s.yaml",
|
|
"src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-code-queue.k8s.yaml",
|
|
"src/components/microservices/k3sctl-adapter/k3s/mdtodo.k8s.yaml",
|
|
"src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-mdtodo.k8s.yaml",
|
|
];
|
|
const forbiddenAutomaticActions = [
|
|
"systemctl restart k3s",
|
|
"service k3s restart",
|
|
"kubectl delete pod",
|
|
"crictl rmp",
|
|
"crictl rm",
|
|
"docker system prune",
|
|
"docker volume prune",
|
|
"rm -rf /home/ubuntu",
|
|
"git reset --hard live worktree",
|
|
];
|
|
|
|
export function runD601RecoveryGuardrails(config: UniDeskConfig, fixture: RecoveryGuardrailsFixture = {}): RecoveryGuardrailsResult {
|
|
const observedAt = fixture.observedAt ?? new Date().toISOString();
|
|
const filesRead: string[] = [];
|
|
const commandsAttempted: string[][] = [];
|
|
const codeQueueService = config.microservices.find((service) => service.id === "code-queue");
|
|
const deployWorktreePath = fixture.deployWorktreePath ?? codeQueueService?.development.worktreePath ?? "/home/ubuntu/unidesk-code-queue-deploy";
|
|
const compatibilitySymlinkPath = fixture.compatibilitySymlinkPath ?? defaultCompatibilitySymlinkPath;
|
|
const manifestPaths = fixture.manifestPaths ?? defaultManifestPaths;
|
|
|
|
const procMounts = fixture.procMountsText === undefined
|
|
? collectProcMounts("/proc/mounts", filesRead)
|
|
: parseProcMounts(fixture.procMountsText, "fixture:/proc/mounts");
|
|
const kubeletValidationRisk = classifyKubeletValidationRisk(procMounts);
|
|
const manifests = manifestPaths.map((manifestPath) => collectManifest(manifestPath, fixture, filesRead));
|
|
const hostPathEntries = manifests.flatMap((manifest) => manifest.readError === null ? extractHostPathEntries(manifest.path, manifest.text) : []);
|
|
const hostPaths = evaluateHostPaths(hostPathEntries, manifests, fixture.pathStates);
|
|
const worktree = evaluateWorktreeReadiness(deployWorktreePath, compatibilitySymlinkPath, fixture.pathStates);
|
|
const mdtodo = evaluateMdtodoAdjacentHostPaths(hostPaths);
|
|
const criSandboxes = fixture.criPodsJsonText === undefined
|
|
? collectCriSandboxes(observedAt, commandsAttempted)
|
|
: parseCriSandboxReport(fixture.criPodsJsonText, observedAt, "fixture", null);
|
|
const containerCreating = fixture.kubectlPodsJsonText === undefined
|
|
? collectContainerCreating(hostPaths, commandsAttempted)
|
|
: parseContainerCreatingReport(fixture.kubectlPodsJsonText, hostPaths, "fixture", null);
|
|
const redlines = buildRedlines({
|
|
procMounts,
|
|
kubeletValidationRisk,
|
|
criSandboxes,
|
|
worktree,
|
|
hostPaths,
|
|
mdtodo,
|
|
containerCreating,
|
|
});
|
|
const redlineSummary = {
|
|
red: redlines.filter((item) => item.severity === "red").length,
|
|
yellow: redlines.filter((item) => item.severity === "yellow").length,
|
|
safeReadOnly: true,
|
|
requiresManualHostHotfix: redlines.filter((item) => item.requiresManualHostHotfix).map((item) => item.id),
|
|
forbiddenAutomaticActions: Array.from(new Set(redlines.flatMap((item) => item.forbiddenAutomaticActions))),
|
|
};
|
|
return {
|
|
ok: redlineSummary.red === 0,
|
|
surface: "d601-recovery-guardrails",
|
|
mutation: false,
|
|
observedAt,
|
|
scope: {
|
|
nodeId: "D601",
|
|
environment: "host-read-only",
|
|
liveMutationAllowed: false,
|
|
},
|
|
safeReadOnly: {
|
|
filesRead,
|
|
commandsAttempted,
|
|
note: "Diagnostics only read files, inspect path metadata, and run optional bounded read-only kubectl/crictl probes. They never delete CRI sandboxes, prune Docker state, edit hostPath directories, restart k3s, apply manifests, or rollout workloads.",
|
|
},
|
|
redlineSummary,
|
|
redlines,
|
|
checks: {
|
|
procMounts,
|
|
kubeletValidationRisk,
|
|
criSandboxes,
|
|
codeQueueDeployWorktree: worktree,
|
|
hostPaths,
|
|
mdtodoAdjacentHostPaths: mdtodo,
|
|
containerCreating,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function compactD601RecoveryGuardrails(result: RecoveryGuardrailsResult): CompactRecoveryGuardrailsResult {
|
|
return {
|
|
ok: result.ok,
|
|
surface: result.surface,
|
|
mutation: false,
|
|
observedAt: result.observedAt,
|
|
scope: result.scope,
|
|
safeReadOnly: result.safeReadOnly,
|
|
redlineSummary: result.redlineSummary,
|
|
redlines: result.redlines.map((item) => ({
|
|
id: item.id,
|
|
severity: item.severity,
|
|
disposition: item.disposition,
|
|
summary: item.summary,
|
|
recommendedNext: item.recommendedNext,
|
|
})),
|
|
checks: {
|
|
procMounts: result.checks.procMounts,
|
|
kubeletValidationRisk: result.checks.kubeletValidationRisk,
|
|
criSandboxes: {
|
|
ok: result.checks.criSandboxes.ok,
|
|
source: result.checks.criSandboxes.source,
|
|
command: result.checks.criSandboxes.command,
|
|
total: result.checks.criSandboxes.total,
|
|
notReadyCount: result.checks.criSandboxes.notReadyCount,
|
|
staleNotReadyCount: result.checks.criSandboxes.staleNotReadyCount,
|
|
unknownAgeCount: result.checks.criSandboxes.unknownAgeCount,
|
|
staleAfterMinutes: result.checks.criSandboxes.staleAfterMinutes,
|
|
staleSandboxes: result.checks.criSandboxes.staleSandboxes,
|
|
parseError: result.checks.criSandboxes.parseError,
|
|
},
|
|
codeQueueDeployWorktree: result.checks.codeQueueDeployWorktree,
|
|
hostPaths: {
|
|
ok: result.checks.hostPaths.ok,
|
|
manifests: result.checks.hostPaths.manifests.map((manifest) => ({
|
|
path: manifest.path,
|
|
readError: manifest.readError,
|
|
textBytes: Buffer.byteLength(manifest.text, "utf8"),
|
|
})),
|
|
totalEntries: result.checks.hostPaths.totalEntries,
|
|
checkedCount: result.checks.hostPaths.checkedEntries.length,
|
|
missing: result.checks.hostPaths.missing.map(compactHostPathProblem),
|
|
typeMismatches: result.checks.hostPaths.typeMismatches.map(compactHostPathProblem),
|
|
directoryOrCreateMissing: result.checks.hostPaths.directoryOrCreateMissing.map(compactHostPathProblem),
|
|
sensitiveUserDataBoundaries: result.checks.hostPaths.sensitiveUserDataBoundaries,
|
|
},
|
|
mdtodoAdjacentHostPaths: result.checks.mdtodoAdjacentHostPaths,
|
|
containerCreating: {
|
|
ok: result.checks.containerCreating.ok,
|
|
source: result.checks.containerCreating.source,
|
|
command: result.checks.containerCreating.command,
|
|
totalContainerCreating: result.checks.containerCreating.totalContainerCreating,
|
|
opaqueContainerCreating: result.checks.containerCreating.opaqueContainerCreating,
|
|
hostPathContainerCreating: result.checks.containerCreating.hostPathContainerCreating,
|
|
parseError: result.checks.containerCreating.parseError,
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
export function parseProcMounts(text: string, source: string): ProcMountsReport {
|
|
const lines = text.split(/\r?\n/u).filter((line) => line.trim().length > 0);
|
|
const malformedLines: MountLineFinding[] = [];
|
|
const dockerDesktopHost9pLines: MountLineFinding[] = [];
|
|
lines.forEach((line, index) => {
|
|
const tokens = line.trim().split(/\s+/u);
|
|
const dockerDesktopHost9p = line.includes("/Docker/host") && /(?:^|\s)9p(?:\s|$)/u.test(line);
|
|
const finding: MountLineFinding = {
|
|
lineNumber: index + 1,
|
|
fieldCount: tokens.length,
|
|
dockerDesktopHost9p,
|
|
textPreview: safePreview(line, 260),
|
|
reason: tokens.length === 6 ? "docker-desktop-host-9p-line-observed" : "proc-mounts-line-field-count-not-six",
|
|
};
|
|
if (dockerDesktopHost9p) dockerDesktopHost9pLines.push(finding);
|
|
if (tokens.length !== 6) malformedLines.push(finding);
|
|
});
|
|
return {
|
|
ok: malformedLines.length === 0,
|
|
source,
|
|
totalLines: lines.length,
|
|
malformedLines,
|
|
dockerDesktopHost9pLines,
|
|
readError: null,
|
|
};
|
|
}
|
|
|
|
export function classifyKubeletValidationRisk(report: ProcMountsReport): KubeletValidationRisk {
|
|
const malformedDockerHost = report.malformedLines.filter((line) => line.dockerDesktopHost9p);
|
|
if (report.malformedLines.length === 0) {
|
|
return {
|
|
ok: true,
|
|
risk: "none",
|
|
severity: "info",
|
|
reason: report.dockerDesktopHost9pLines.length === 0
|
|
? "No malformed /proc/mounts lines were detected."
|
|
: "Docker Desktop /Docker/host 9p lines were observed but field counts are valid.",
|
|
evidenceLineNumbers: [],
|
|
};
|
|
}
|
|
return {
|
|
ok: false,
|
|
risk: "kubelet-mount-table-validation-risk",
|
|
severity: "red",
|
|
reason: malformedDockerHost.length > 0
|
|
? "Malformed Docker Desktop /Docker/host 9p mount rows can make kubelet mount-table validation reject hostPath setup after reboot."
|
|
: "Malformed /proc/mounts rows can make kubelet mount-table validation unreliable.",
|
|
evidenceLineNumbers: report.malformedLines.map((line) => line.lineNumber),
|
|
};
|
|
}
|
|
|
|
export function parseCriSandboxReport(
|
|
jsonText: string,
|
|
observedAt: string,
|
|
source: "crictl" | "fixture",
|
|
commandResult: CommandResultSummary | null,
|
|
): CriSandboxReport {
|
|
try {
|
|
const parsed = JSON.parse(jsonText) as unknown;
|
|
const items = sandboxItems(parsed);
|
|
const observedMs = Date.parse(observedAt);
|
|
const notReady = items.filter((item) => {
|
|
const state = String(recordField(item, "state") ?? "").toUpperCase();
|
|
return state.length === 0 || (!state.includes("READY") && state !== "SANDBOX_READY" && state !== "READY") || state.includes("NOTREADY");
|
|
});
|
|
const summaries = notReady.map((item) => sandboxSummary(item, observedMs));
|
|
const staleSandboxes = summaries.filter((item) => item.ageMinutes === null || item.ageMinutes >= staleSandboxAfterMinutes);
|
|
return {
|
|
ok: staleSandboxes.length === 0,
|
|
source,
|
|
command: source === "crictl" ? ["crictl", "pods", "-o", "json"] : null,
|
|
commandResult,
|
|
total: items.length,
|
|
notReadyCount: notReady.length,
|
|
staleNotReadyCount: staleSandboxes.length,
|
|
unknownAgeCount: summaries.filter((item) => item.ageMinutes === null).length,
|
|
staleAfterMinutes: staleSandboxAfterMinutes,
|
|
staleSandboxes,
|
|
parseError: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
source,
|
|
command: source === "crictl" ? ["crictl", "pods", "-o", "json"] : null,
|
|
commandResult,
|
|
total: 0,
|
|
notReadyCount: 0,
|
|
staleNotReadyCount: 0,
|
|
unknownAgeCount: 0,
|
|
staleAfterMinutes: staleSandboxAfterMinutes,
|
|
staleSandboxes: [],
|
|
parseError: errorMessage(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
export function extractHostPathEntries(manifestPath: string, text: string): HostPathEntry[] {
|
|
return splitManifestDocuments(text).flatMap((doc) => {
|
|
const lines = doc.raw.split(/\r?\n/u);
|
|
const entries: HostPathEntry[] = [];
|
|
for (let index = 0; index < lines.length; index += 1) {
|
|
if (!/^\s*hostPath:\s*$/u.test(lines[index] ?? "")) continue;
|
|
const volumeName = findVolumeName(lines, index);
|
|
const path = findScalarAfter(lines, index, "path");
|
|
if (path === null) continue;
|
|
entries.push({
|
|
manifest: manifestPath,
|
|
documentIndex: doc.index,
|
|
kind: doc.kind,
|
|
name: doc.name,
|
|
namespace: doc.namespace,
|
|
volumeName,
|
|
hostPath: path,
|
|
hostPathType: findScalarAfter(lines, index, "type") ?? "Unset",
|
|
});
|
|
}
|
|
return entries;
|
|
});
|
|
}
|
|
|
|
export function evaluateHostPaths(
|
|
entries: HostPathEntry[],
|
|
manifests: HostPathManifest[],
|
|
fixtures: Record<string, PathStateFixture> | undefined = undefined,
|
|
): HostPathReadinessReport {
|
|
const checkedEntries = dedupeHostPathEntries(entries);
|
|
const problems = checkedEntries.map((entry) => hostPathProblem(entry, checkPath(entry.hostPath, entry.hostPathType, fixtures))).filter((item): item is HostPathProblem => item !== null);
|
|
return {
|
|
ok: problems.filter((problem) => problem.severity === "red").length === 0,
|
|
manifests,
|
|
totalEntries: entries.length,
|
|
checkedEntries,
|
|
missing: problems.filter((problem) => problem.check.problem === "missing"),
|
|
typeMismatches: problems.filter((problem) => problem.check.problem === "type-mismatch"),
|
|
directoryOrCreateMissing: problems.filter((problem) => problem.check.problem === "directory-or-create-missing"),
|
|
sensitiveUserDataBoundaries: [
|
|
"/home/ubuntu/.codex/auth.json",
|
|
"/home/ubuntu/.ssh",
|
|
"/home/ubuntu/.agents/skills",
|
|
"/home/ubuntu/cq-deploy/.state/mdtodo-workspace",
|
|
"/home/ubuntu/unidesk-dev-mdtodo-workspace",
|
|
],
|
|
};
|
|
}
|
|
|
|
export function evaluateWorktreeReadiness(
|
|
canonicalPath: string,
|
|
compatibilitySymlinkPath = defaultCompatibilitySymlinkPath,
|
|
fixtures: Record<string, PathStateFixture> | undefined = undefined,
|
|
): WorktreeReadinessReport {
|
|
const canonicalPathCheck = checkPath(canonicalPath, "Directory", fixtures);
|
|
const symlink = checkPath(compatibilitySymlinkPath, "Directory", fixtures);
|
|
const expectedTarget = canonicalPath;
|
|
const targetPath = symlink.resolvedPath ?? resolveSymlinkTarget(compatibilitySymlinkPath, symlink.symlinkTarget);
|
|
const targetCheck = targetPath === null ? null : checkPath(targetPath, "Directory", fixtures);
|
|
return {
|
|
ok: canonicalPathCheck.matchesExpectedType && symlink.matchesExpectedType && (symlink.isSymlink ? targetCheck?.matchesExpectedType === true : true),
|
|
canonicalPath: canonicalPathCheck,
|
|
compatibilitySymlink: {
|
|
...symlink,
|
|
expectedTarget,
|
|
targetExists: targetCheck?.exists ?? false,
|
|
targetKind: targetCheck?.kind ?? "missing",
|
|
},
|
|
requiredFor: [
|
|
"config.json microservices.code-queue.development.worktreePath",
|
|
"production code-queue /root/unidesk and /app hostPath mapping through /home/ubuntu/cq-deploy",
|
|
"MDTODO production workspace/log hostPath adjacency under /home/ubuntu/cq-deploy/.state",
|
|
],
|
|
};
|
|
}
|
|
|
|
export function parseContainerCreatingReport(
|
|
jsonText: string,
|
|
hostPaths: HostPathReadinessReport,
|
|
source: "kubectl" | "fixture",
|
|
commandResult: CommandResultSummary | null,
|
|
): ContainerCreatingReport {
|
|
try {
|
|
const parsed = JSON.parse(jsonText) as unknown;
|
|
const pods = podItems(parsed);
|
|
const hostPathNeedles = missingHostPathNeedles(hostPaths);
|
|
const findings = pods.flatMap((pod) => containerCreatingFindings(pod, hostPathNeedles));
|
|
const opaque = findings.filter((item) => item.classification === "opaque-containercreating-hostpath-risk");
|
|
const hostPathFailures = findings.filter((item) => item.classification === "hostpath-mount-failure");
|
|
return {
|
|
ok: opaque.length === 0 && hostPathFailures.length === 0,
|
|
source,
|
|
command: source === "kubectl" ? ["kubectl", "get", "pods", "-A", "-o", "json"] : null,
|
|
commandResult,
|
|
totalContainerCreating: findings.length,
|
|
opaqueContainerCreating: opaque,
|
|
hostPathContainerCreating: hostPathFailures,
|
|
parseError: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
source,
|
|
command: source === "kubectl" ? ["kubectl", "get", "pods", "-A", "-o", "json"] : null,
|
|
commandResult,
|
|
totalContainerCreating: 0,
|
|
opaqueContainerCreating: [],
|
|
hostPathContainerCreating: [],
|
|
parseError: errorMessage(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
function collectProcMounts(path: string, filesRead: string[]): ProcMountsReport {
|
|
filesRead.push(path);
|
|
try {
|
|
return parseProcMounts(readFileSync(path, "utf8"), path);
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
source: path,
|
|
totalLines: 0,
|
|
malformedLines: [],
|
|
dockerDesktopHost9pLines: [],
|
|
readError: errorMessage(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
function collectManifest(manifestPath: string, fixture: RecoveryGuardrailsFixture, filesRead: string[]): HostPathManifest {
|
|
if (fixture.manifestTexts?.[manifestPath] !== undefined) {
|
|
return { path: manifestPath, text: fixture.manifestTexts[manifestPath] ?? "", readError: null };
|
|
}
|
|
const absolute = rootPath(manifestPath);
|
|
filesRead.push(absolute);
|
|
try {
|
|
return { path: manifestPath, text: readFileSync(absolute, "utf8"), readError: null };
|
|
} catch (error) {
|
|
return { path: manifestPath, text: "", readError: errorMessage(error) };
|
|
}
|
|
}
|
|
|
|
function collectCriSandboxes(observedAt: string, commandsAttempted: string[][]): CriSandboxReport {
|
|
const probe = runCommand(["sh", "-lc", "command -v crictl >/dev/null 2>&1"], repoRoot, { timeoutMs: 5_000 });
|
|
if (probe.exitCode !== 0) {
|
|
return unavailableCriSandboxReport("crictl binary is not available in this environment.");
|
|
}
|
|
const command = ["crictl", "pods", "-o", "json"];
|
|
commandsAttempted.push(command);
|
|
const result = runCommand(command, repoRoot, { timeoutMs: 10_000 });
|
|
const summary = summarizeCommandResult(result);
|
|
if (result.exitCode !== 0) {
|
|
return {
|
|
...unavailableCriSandboxReport("crictl read-only pod sandbox probe failed."),
|
|
command,
|
|
commandResult: summary,
|
|
};
|
|
}
|
|
return parseCriSandboxReport(result.stdout, observedAt, "crictl", summary);
|
|
}
|
|
|
|
function collectContainerCreating(hostPaths: HostPathReadinessReport, commandsAttempted: string[][]): ContainerCreatingReport {
|
|
const probe = runCommand(["sh", "-lc", "command -v kubectl >/dev/null 2>&1"], repoRoot, { timeoutMs: 5_000 });
|
|
if (probe.exitCode !== 0) {
|
|
return unavailableContainerCreatingReport("kubectl binary is not available in this environment.");
|
|
}
|
|
const command = ["kubectl", "get", "pods", "-A", "-o", "json"];
|
|
commandsAttempted.push(command);
|
|
const result = runCommand(command, repoRoot, { timeoutMs: 10_000 });
|
|
const summary = summarizeCommandResult(result);
|
|
if (result.exitCode !== 0) {
|
|
return {
|
|
...unavailableContainerCreatingReport("kubectl read-only pod status probe failed."),
|
|
command,
|
|
commandResult: summary,
|
|
};
|
|
}
|
|
return parseContainerCreatingReport(result.stdout, hostPaths, "kubectl", summary);
|
|
}
|
|
|
|
function unavailableCriSandboxReport(reason: string): CriSandboxReport {
|
|
return {
|
|
ok: true,
|
|
source: "unavailable",
|
|
command: null,
|
|
commandResult: null,
|
|
total: 0,
|
|
notReadyCount: 0,
|
|
staleNotReadyCount: 0,
|
|
unknownAgeCount: 0,
|
|
staleAfterMinutes: staleSandboxAfterMinutes,
|
|
staleSandboxes: [],
|
|
parseError: reason,
|
|
};
|
|
}
|
|
|
|
function unavailableContainerCreatingReport(reason: string): ContainerCreatingReport {
|
|
return {
|
|
ok: true,
|
|
source: "unavailable",
|
|
command: null,
|
|
commandResult: null,
|
|
totalContainerCreating: 0,
|
|
opaqueContainerCreating: [],
|
|
hostPathContainerCreating: [],
|
|
parseError: reason,
|
|
};
|
|
}
|
|
|
|
function evaluateMdtodoAdjacentHostPaths(hostPaths: HostPathReadinessReport): MdtodoAdjacentReadiness {
|
|
const mdtodoEntries = hostPaths.checkedEntries.filter((entry) => entry.hostPath.includes("mdtodo"));
|
|
const workspacePaths = mdtodoEntries.filter((entry) => entry.volumeName === "workspace" || entry.hostPath.includes("workspace")).map((entry) => entry.hostPath);
|
|
const logPaths = mdtodoEntries.filter((entry) => entry.volumeName === "logs" || entry.hostPath.includes("/logs")).map((entry) => entry.hostPath);
|
|
const missingPaths = Array.from(new Set([
|
|
...hostPaths.missing.filter((problem) => problem.entry.hostPath.includes("mdtodo")).map((problem) => problem.entry.hostPath),
|
|
...hostPaths.directoryOrCreateMissing.filter((problem) => problem.entry.hostPath.includes("mdtodo")).map((problem) => problem.entry.hostPath),
|
|
]));
|
|
return {
|
|
ok: missingPaths.length === 0 && workspacePaths.length > 0,
|
|
workspacePaths: Array.from(new Set(workspacePaths)),
|
|
logPaths: Array.from(new Set(logPaths)),
|
|
missingPaths,
|
|
opaqueRisk: missingPaths.length > 0,
|
|
userDataBoundary: "MDTODO workspace hostPaths are user data. Recovery diagnostics may report readiness, but must not create, delete, prune, reset, or replace the workspace automatically.",
|
|
};
|
|
}
|
|
|
|
function buildRedlines(reports: {
|
|
procMounts: ProcMountsReport;
|
|
kubeletValidationRisk: KubeletValidationRisk;
|
|
criSandboxes: CriSandboxReport;
|
|
worktree: WorktreeReadinessReport;
|
|
hostPaths: HostPathReadinessReport;
|
|
mdtodo: MdtodoAdjacentReadiness;
|
|
containerCreating: ContainerCreatingReport;
|
|
}): RecoveryGuardrailRedline[] {
|
|
const redlines: RecoveryGuardrailRedline[] = [];
|
|
if (!reports.procMounts.ok || !reports.kubeletValidationRisk.ok) {
|
|
redlines.push(redline(
|
|
"proc-mounts-malformed-kubelet-risk",
|
|
"red",
|
|
"requires-manual-host-hotfix",
|
|
"Malformed /proc/mounts lines create kubelet hostPath validation risk.",
|
|
{
|
|
source: reports.procMounts.source,
|
|
malformedLines: reports.procMounts.malformedLines,
|
|
kubeletValidationRisk: reports.kubeletValidationRisk,
|
|
},
|
|
[
|
|
"Host commander or user should repair the Docker Desktop/WSL mount-table condition and then run this read-only check again.",
|
|
"Do not restart k3s from Code Queue or this CLI without explicit host authorization.",
|
|
],
|
|
));
|
|
}
|
|
if (reports.criSandboxes.staleNotReadyCount > 0 || reports.criSandboxes.parseError !== null && reports.criSandboxes.source !== "unavailable") {
|
|
redlines.push(redline(
|
|
"stale-cri-sandboxes-observed",
|
|
reports.criSandboxes.staleNotReadyCount > 0 ? "red" : "yellow",
|
|
"requires-manual-host-hotfix",
|
|
"Stale or unreadable CRI pod sandbox state needs host-side review; automatic deletion is forbidden.",
|
|
{
|
|
source: reports.criSandboxes.source,
|
|
staleNotReadyCount: reports.criSandboxes.staleNotReadyCount,
|
|
unknownAgeCount: reports.criSandboxes.unknownAgeCount,
|
|
staleSandboxes: reports.criSandboxes.staleSandboxes,
|
|
parseError: reports.criSandboxes.parseError,
|
|
},
|
|
[
|
|
"Review CRI sandbox state on D601 host with bounded read-only crictl/kubectl commands.",
|
|
"Escalate to host commander/user before any sandbox cleanup.",
|
|
],
|
|
));
|
|
}
|
|
if (!reports.worktree.ok) {
|
|
redlines.push(redline(
|
|
"code-queue-deploy-worktree-not-ready",
|
|
"red",
|
|
"requires-manual-host-hotfix",
|
|
"Code Queue deploy worktree or compatibility symlink is missing or unresolved.",
|
|
reports.worktree as unknown as Record<string, unknown>,
|
|
[
|
|
"Restore the Git worktree/symlink on D601 host from pushed Git state.",
|
|
"Do not git reset --hard or replace a dirty live worktree without explicit user/host approval.",
|
|
],
|
|
));
|
|
}
|
|
if (!reports.hostPaths.ok || reports.hostPaths.directoryOrCreateMissing.length > 0) {
|
|
redlines.push(redline(
|
|
"required-hostpaths-not-ready",
|
|
reports.hostPaths.ok ? "yellow" : "red",
|
|
"requires-manual-host-hotfix",
|
|
"One or more k3s hostPath mounts required by Code Queue or MDTODO are missing or have the wrong type.",
|
|
{
|
|
missing: reports.hostPaths.missing,
|
|
typeMismatches: reports.hostPaths.typeMismatches,
|
|
directoryOrCreateMissing: reports.hostPaths.directoryOrCreateMissing,
|
|
sensitiveUserDataBoundaries: reports.hostPaths.sensitiveUserDataBoundaries,
|
|
},
|
|
[
|
|
"Repair only the specific missing hostPath on D601 host after reviewing whether it is user data, credential material, or cache/log state.",
|
|
"Do not mass-create, delete, prune, chmod recursively, or replace hostPath directories from Code Queue.",
|
|
],
|
|
));
|
|
}
|
|
if (!reports.mdtodo.ok) {
|
|
redlines.push(redline(
|
|
"mdtodo-adjacent-hostpaths-opaque",
|
|
"red",
|
|
"requires-manual-host-hotfix",
|
|
"MDTODO workspace/log hostPaths are not transparently ready, making ContainerCreating recovery ambiguous.",
|
|
reports.mdtodo as unknown as Record<string, unknown>,
|
|
[
|
|
"Verify the MDTODO workspace source and logs path on D601 host before restarting or rolling pods.",
|
|
"Ask the user before touching the MDTODO workspace because it contains user-authored Markdown data.",
|
|
],
|
|
));
|
|
}
|
|
if (reports.containerCreating.opaqueContainerCreating.length > 0 || reports.containerCreating.hostPathContainerCreating.length > 0) {
|
|
redlines.push(redline(
|
|
"opaque-containercreating-hostpath-risk",
|
|
"red",
|
|
"requires-manual-host-hotfix",
|
|
"Pods stuck in ContainerCreating have opaque or hostPath-looking evidence.",
|
|
{
|
|
opaqueContainerCreating: reports.containerCreating.opaqueContainerCreating,
|
|
hostPathContainerCreating: reports.containerCreating.hostPathContainerCreating,
|
|
},
|
|
[
|
|
"Use kubectl describe/get events on D601 host to confirm the exact mount failure.",
|
|
"Do not delete pods or sandboxes automatically; first fix the hostPath or kubelet mount-table condition.",
|
|
],
|
|
));
|
|
}
|
|
redlines.push(redline(
|
|
"destructive-recovery-actions-forbidden",
|
|
"yellow",
|
|
"forbidden-automatic-action",
|
|
"Recovery diagnostics must never perform destructive cleanup or rollout as an automatic follow-up.",
|
|
{ forbiddenAutomaticActions },
|
|
[
|
|
"Keep this CLI read-only.",
|
|
"Use ClaudeQQ/user approval for high-risk host actions such as k3s restart, CRI cleanup, user-data directory repair, or Code Queue runtime intervention.",
|
|
],
|
|
));
|
|
return redlines;
|
|
}
|
|
|
|
function redline(
|
|
id: string,
|
|
severity: Severity,
|
|
disposition: Disposition,
|
|
summary: string,
|
|
evidence: Record<string, unknown>,
|
|
recommendedNext: string[],
|
|
): RecoveryGuardrailRedline {
|
|
return {
|
|
id,
|
|
severity,
|
|
disposition,
|
|
summary,
|
|
evidence,
|
|
safeReadOnly: true,
|
|
requiresManualHostHotfix: disposition === "requires-manual-host-hotfix",
|
|
forbiddenAutomaticActions,
|
|
recommendedNext,
|
|
};
|
|
}
|
|
|
|
function hostPathProblem(entry: HostPathEntry, check: PathCheck): HostPathProblem | null {
|
|
if (check.problem === null) return null;
|
|
if (check.problem === "directory-or-create-missing") {
|
|
return {
|
|
entry,
|
|
check,
|
|
severity: "yellow",
|
|
disposition: "requires-manual-host-hotfix",
|
|
reason: "DirectoryOrCreate hostPath is missing; kubelet may create it only if mount-table validation and parent permissions are healthy.",
|
|
};
|
|
}
|
|
return {
|
|
entry,
|
|
check,
|
|
severity: "red",
|
|
disposition: "requires-manual-host-hotfix",
|
|
reason: check.problem,
|
|
};
|
|
}
|
|
|
|
function checkPath(path: string, expectedType: string, fixtures: Record<string, PathStateFixture> | undefined): PathCheck {
|
|
const fixture = fixtures?.[path];
|
|
if (fixture !== undefined) return checkFixturePath(path, expectedType, fixture);
|
|
try {
|
|
const lstat = lstatSync(path);
|
|
const isSymlink = lstat.isSymbolicLink();
|
|
const symlinkTarget = isSymlink ? readlinkSync(path) : null;
|
|
const resolvedPath = isSymlink ? resolveSymlinkTarget(path, symlinkTarget) : path;
|
|
const stat = statSync(resolvedPath ?? path);
|
|
const kind = stat.isDirectory() ? "directory" : stat.isFile() ? "file" : stat.isSocket() ? "socket" : isSymlink ? "symlink" : "other";
|
|
const matchesExpectedType = matchesType(kind, expectedType);
|
|
return {
|
|
path,
|
|
exists: true,
|
|
kind,
|
|
isSymlink,
|
|
symlinkTarget,
|
|
resolvedPath,
|
|
matchesExpectedType,
|
|
expectedType,
|
|
problem: matchesExpectedType ? null : "type-mismatch",
|
|
};
|
|
} catch (error) {
|
|
try {
|
|
const lstat = lstatSync(path);
|
|
if (lstat.isSymbolicLink()) {
|
|
const symlinkTarget = readlinkSync(path);
|
|
return {
|
|
path,
|
|
exists: true,
|
|
kind: "symlink",
|
|
isSymlink: true,
|
|
symlinkTarget,
|
|
resolvedPath: resolveSymlinkTarget(path, symlinkTarget),
|
|
matchesExpectedType: false,
|
|
expectedType,
|
|
problem: "symlink-target-missing",
|
|
};
|
|
}
|
|
} catch {
|
|
// Fall through to the generic missing/error report.
|
|
}
|
|
if (isMissingPathError(error)) return missingPathCheck(path, expectedType);
|
|
return {
|
|
path,
|
|
exists: false,
|
|
kind: "missing",
|
|
isSymlink: false,
|
|
symlinkTarget: null,
|
|
resolvedPath: null,
|
|
matchesExpectedType: false,
|
|
expectedType,
|
|
problem: errorMessage(error),
|
|
};
|
|
}
|
|
}
|
|
|
|
function checkFixturePath(path: string, expectedType: string, fixture: PathStateFixture): PathCheck {
|
|
const exists = fixture.kind !== "missing";
|
|
const resolvedPath = fixture.kind === "symlink" && fixture.target !== undefined ? resolveSymlinkTarget(path, fixture.target) : exists ? path : null;
|
|
const targetExists = fixture.kind !== "symlink" ? exists : fixture.targetExists === true;
|
|
const effectiveKind = fixture.kind === "symlink" && targetExists ? "directory" : fixture.kind;
|
|
const matchesExpectedType = exists && matchesType(effectiveKind, expectedType) && (fixture.kind !== "symlink" || targetExists);
|
|
return {
|
|
path,
|
|
exists,
|
|
kind: fixture.kind,
|
|
isSymlink: fixture.kind === "symlink",
|
|
symlinkTarget: fixture.target ?? null,
|
|
resolvedPath,
|
|
matchesExpectedType,
|
|
expectedType,
|
|
problem: matchesExpectedType ? null : expectedType === "DirectoryOrCreate" && !exists ? "directory-or-create-missing" : exists ? "type-mismatch" : "missing",
|
|
};
|
|
}
|
|
|
|
function missingPathCheck(path: string, expectedType: string): PathCheck {
|
|
return {
|
|
path,
|
|
exists: false,
|
|
kind: "missing",
|
|
isSymlink: false,
|
|
symlinkTarget: null,
|
|
resolvedPath: null,
|
|
matchesExpectedType: expectedType === "DirectoryOrCreate",
|
|
expectedType,
|
|
problem: expectedType === "DirectoryOrCreate" ? "directory-or-create-missing" : "missing",
|
|
};
|
|
}
|
|
|
|
function matchesType(kind: PathKind, expectedType: string): boolean {
|
|
if (expectedType === "Unset" || expectedType.length === 0) return kind !== "missing";
|
|
if (expectedType === "Directory" || expectedType === "DirectoryOrCreate") return kind === "directory";
|
|
if (expectedType === "File" || expectedType === "FileOrCreate") return kind === "file";
|
|
if (expectedType === "Socket") return kind === "socket";
|
|
return kind !== "missing";
|
|
}
|
|
|
|
function resolveSymlinkTarget(path: string, target: string | null): string | null {
|
|
if (target === null) return null;
|
|
return isAbsolute(target) ? target : resolve(dirname(path), target);
|
|
}
|
|
|
|
function dedupeHostPathEntries(entries: HostPathEntry[]): HostPathEntry[] {
|
|
const seen = new Set<string>();
|
|
const result: HostPathEntry[] = [];
|
|
for (const entry of entries) {
|
|
const key = `${entry.hostPath}\0${entry.hostPathType}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
result.push(entry);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function splitManifestDocuments(text: string): Array<{ index: number; raw: string; kind: string | null; name: string | null; namespace: string | null }> {
|
|
return text.split(/^---\s*$/mu)
|
|
.map((raw, index) => ({ index, raw: raw.trim() }))
|
|
.filter((doc) => doc.raw.length > 0)
|
|
.map((doc) => ({
|
|
...doc,
|
|
kind: scalarAfter(doc.raw, "kind"),
|
|
name: metadataScalarAfter(doc.raw, "name"),
|
|
namespace: metadataScalarAfter(doc.raw, "namespace"),
|
|
}));
|
|
}
|
|
|
|
function scalarAfter(text: string, key: string): string | null {
|
|
const match = text.match(new RegExp(`^\\s*${key}:\\s*"?([^"\\n#]+)"?\\s*(?:#.*)?$`, "mu"));
|
|
return match?.[1]?.trim() ?? null;
|
|
}
|
|
|
|
function metadataScalarAfter(text: string, key: string): string | null {
|
|
const metadataIndex = text.search(/^metadata:\s*$/mu);
|
|
if (metadataIndex < 0) return null;
|
|
return scalarAfter(text.slice(metadataIndex), key);
|
|
}
|
|
|
|
function findVolumeName(lines: string[], hostPathIndex: number): string | null {
|
|
for (let index = hostPathIndex - 1; index >= Math.max(0, hostPathIndex - 8); index -= 1) {
|
|
const match = (lines[index] ?? "").match(/^\s*-\s+name:\s*"?([^"\n#]+)"?/u);
|
|
if (match?.[1] !== undefined) return match[1].trim();
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function findScalarAfter(lines: string[], startIndex: number, key: string): string | null {
|
|
for (let index = startIndex + 1; index < Math.min(lines.length, startIndex + 8); index += 1) {
|
|
const match = (lines[index] ?? "").match(new RegExp(`^\\s*${key}:\\s*"?([^"\\n#]+)"?\\s*(?:#.*)?$`, "u"));
|
|
if (match?.[1] !== undefined) return match[1].trim();
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function sandboxItems(parsed: unknown): Record<string, unknown>[] {
|
|
const record = asRecord(parsed);
|
|
const items = record.items ?? record.sandboxes ?? record.pods ?? [];
|
|
return Array.isArray(items) ? items.flatMap((item) => typeof item === "object" && item !== null ? [item as Record<string, unknown>] : []) : [];
|
|
}
|
|
|
|
function sandboxSummary(item: Record<string, unknown>, observedMs: number): CriSandboxSummary {
|
|
const metadata = asRecord(recordField(item, "metadata"));
|
|
const createdAt = recordField(item, "createdAt") ?? recordField(item, "created_at") ?? recordField(item, "created");
|
|
const createdMs = typeof createdAt === "string" || typeof createdAt === "number" ? Number(createdAt) > 10_000_000_000_000 ? Math.floor(Number(createdAt) / 1_000_000) : Date.parse(String(createdAt)) : NaN;
|
|
const ageMinutes = Number.isFinite(createdMs) && Number.isFinite(observedMs) ? Math.max(0, Math.floor((observedMs - createdMs) / 60_000)) : null;
|
|
return {
|
|
id: String(recordField(item, "id") ?? recordField(item, "podSandboxId") ?? "unknown"),
|
|
name: stringOrNull(recordField(metadata, "name")),
|
|
namespace: stringOrNull(recordField(metadata, "namespace")),
|
|
state: String(recordField(item, "state") ?? "unknown"),
|
|
ageMinutes,
|
|
};
|
|
}
|
|
|
|
function podItems(parsed: unknown): Record<string, unknown>[] {
|
|
const record = asRecord(parsed);
|
|
const items = record.items ?? [];
|
|
return Array.isArray(items) ? items.flatMap((item) => typeof item === "object" && item !== null ? [item as Record<string, unknown>] : []) : [];
|
|
}
|
|
|
|
function containerCreatingFindings(pod: Record<string, unknown>, hostPathNeedles: string[]): ContainerCreatingFinding[] {
|
|
const metadata = asRecord(recordField(pod, "metadata"));
|
|
const status = asRecord(recordField(pod, "status"));
|
|
const namespace = stringOrNull(recordField(metadata, "namespace")) ?? "default";
|
|
const podName = stringOrNull(recordField(metadata, "name")) ?? "unknown";
|
|
const statuses = [
|
|
...statusArray(status, "initContainerStatuses"),
|
|
...statusArray(status, "containerStatuses"),
|
|
];
|
|
return statuses.flatMap((container) => {
|
|
const state = asRecord(recordField(container, "state"));
|
|
const waiting = asRecord(recordField(state, "waiting"));
|
|
const reason = stringOrNull(recordField(waiting, "reason")) ?? "";
|
|
if (reason !== "ContainerCreating") return [];
|
|
const message = stringOrNull(recordField(waiting, "message")) ?? "";
|
|
const classification = classifyContainerCreating(message, hostPathNeedles);
|
|
return [{
|
|
namespace,
|
|
pod: podName,
|
|
container: stringOrNull(recordField(container, "name")) ?? "unknown",
|
|
reason,
|
|
messagePreview: safePreview(message, 260),
|
|
classification,
|
|
}];
|
|
});
|
|
}
|
|
|
|
function classifyContainerCreating(message: string, hostPathNeedles: string[]): ContainerCreatingFinding["classification"] {
|
|
const lower = message.toLowerCase();
|
|
if (hostPathNeedles.some((needle) => needle.length > 0 && message.includes(needle)) || lower.includes("hostpath") || lower.includes("mountvolume") || lower.includes("not a directory") || lower.includes("no such file or directory")) {
|
|
return "hostpath-mount-failure";
|
|
}
|
|
if (message.trim().length === 0 && hostPathNeedles.length > 0) return "opaque-containercreating-hostpath-risk";
|
|
return "containercreating-observed";
|
|
}
|
|
|
|
function missingHostPathNeedles(hostPaths: HostPathReadinessReport): string[] {
|
|
return Array.from(new Set([
|
|
...hostPaths.missing.map((problem) => problem.entry.hostPath),
|
|
...hostPaths.typeMismatches.map((problem) => problem.entry.hostPath),
|
|
...hostPaths.directoryOrCreateMissing.map((problem) => problem.entry.hostPath),
|
|
]));
|
|
}
|
|
|
|
function statusArray(status: Record<string, unknown>, key: string): Record<string, unknown>[] {
|
|
const value = recordField(status, key);
|
|
return Array.isArray(value) ? value.flatMap((item) => typeof item === "object" && item !== null ? [item as Record<string, unknown>] : []) : [];
|
|
}
|
|
|
|
function summarizeCommandResult(result: CommandResult): CommandResultSummary {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
signal: result.signal,
|
|
timedOut: result.timedOut,
|
|
stdoutTail: result.stdout.length === 0 ? "" : `<omitted ${Buffer.byteLength(result.stdout, "utf8")} stdout bytes parsed as JSON>`,
|
|
stderrTail: result.stderr.slice(-2000),
|
|
};
|
|
}
|
|
|
|
function compactHostPathProblem(problem: HostPathProblem): CompactHostPathProblem {
|
|
return {
|
|
hostPath: problem.entry.hostPath,
|
|
hostPathType: problem.entry.hostPathType,
|
|
manifest: problem.entry.manifest,
|
|
workload: `${problem.entry.namespace ?? "default"}/${problem.entry.kind ?? "Unknown"}/${problem.entry.name ?? "unknown"}`,
|
|
volumeName: problem.entry.volumeName,
|
|
problem: problem.check.problem,
|
|
severity: problem.severity,
|
|
disposition: problem.disposition,
|
|
reason: problem.reason,
|
|
};
|
|
}
|
|
|
|
function recordField(record: Record<string, unknown>, key: string): unknown;
|
|
function recordField(record: unknown, key: string): unknown;
|
|
function recordField(record: unknown, key: string): unknown {
|
|
return typeof record === "object" && record !== null && key in record ? (record as Record<string, unknown>)[key] : undefined;
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function errorMessage(error: unknown): string {
|
|
return error instanceof Error ? error.message : String(error);
|
|
}
|
|
|
|
function isMissingPathError(error: unknown): boolean {
|
|
if (typeof error !== "object" || error === null || !("code" in error)) return false;
|
|
const code = (error as { code?: unknown }).code;
|
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
}
|
|
|
|
function safePreview(text: string, maxChars: number): string {
|
|
return text.length <= maxChars ? text : `${text.slice(0, maxChars)}...`;
|
|
}
|