import { lstatSync, readFileSync, readlinkSync, statSync } from "node:fs"; import { dirname, isAbsolute, resolve } from "node:path"; import { runCommand, type CommandResult } from "./command"; import { repoRoot, rootPath, type UniDeskConfig } from "./config"; type Severity = "info" | "yellow" | "red"; type Disposition = "safe-read-only" | "requires-manual-host-hotfix" | "forbidden-automatic-action"; type PathKind = "missing" | "directory" | "file" | "socket" | "symlink" | "other"; export interface RecoveryGuardrailRedline { id: string; severity: Severity; disposition: Disposition; summary: string; evidence: Record; safeReadOnly: boolean; requiresManualHostHotfix: boolean; forbiddenAutomaticActions: string[]; recommendedNext: string[]; } export interface RecoveryGuardrailsResult { ok: boolean; surface: "d601-recovery-guardrails"; mutation: false; observedAt: string; scope: { nodeId: "D601"; environment: "host-read-only"; liveMutationAllowed: false; }; safeReadOnly: { filesRead: string[]; commandsAttempted: string[][]; note: string; }; redlineSummary: { red: number; yellow: number; safeReadOnly: boolean; requiresManualHostHotfix: string[]; forbiddenAutomaticActions: string[]; }; redlines: RecoveryGuardrailRedline[]; checks: { procMounts: ProcMountsReport; kubeletValidationRisk: KubeletValidationRisk; criSandboxes: CriSandboxReport; codeQueueDeployWorktree: WorktreeReadinessReport; hostPaths: HostPathReadinessReport; mdtodoAdjacentHostPaths: MdtodoAdjacentReadiness; containerCreating: ContainerCreatingReport; }; } export interface RecoveryGuardrailsFixture { observedAt?: string; procMountsText?: string; criPodsJsonText?: string; kubectlPodsJsonText?: string; pathStates?: Record; manifestTexts?: Record; deployWorktreePath?: string; compatibilitySymlinkPath?: string; manifestPaths?: string[]; } export interface PathStateFixture { kind: PathKind; target?: string; targetExists?: boolean; } export interface CompactRecoveryGuardrailsResult { ok: boolean; surface: RecoveryGuardrailsResult["surface"]; mutation: false; observedAt: string; scope: RecoveryGuardrailsResult["scope"]; safeReadOnly: RecoveryGuardrailsResult["safeReadOnly"]; redlineSummary: RecoveryGuardrailsResult["redlineSummary"]; redlines: Array>; checks: { procMounts: Pick; kubeletValidationRisk: KubeletValidationRisk; criSandboxes: Pick; codeQueueDeployWorktree: WorktreeReadinessReport; hostPaths: { ok: boolean; manifests: CompactHostPathManifest[]; totalEntries: number; checkedCount: number; missing: CompactHostPathProblem[]; typeMismatches: CompactHostPathProblem[]; directoryOrCreateMissing: CompactHostPathProblem[]; sensitiveUserDataBoundaries: string[]; }; mdtodoAdjacentHostPaths: MdtodoAdjacentReadiness; containerCreating: Pick; }; } interface CompactHostPathProblem { hostPath: string; hostPathType: string; manifest: string; workload: string; volumeName: string | null; problem: string | null; severity: Severity; disposition: Disposition; reason: string; } interface CompactHostPathManifest { path: string; readError: string | null; textBytes: number; } interface ProcMountsReport { ok: boolean; source: string; totalLines: number; malformedLines: MountLineFinding[]; dockerDesktopHost9pLines: MountLineFinding[]; readError: string | null; } interface MountLineFinding { lineNumber: number; fieldCount: number; dockerDesktopHost9p: boolean; textPreview: string; reason: string; } interface KubeletValidationRisk { ok: boolean; risk: "none" | "kubelet-mount-table-validation-risk"; severity: Severity; reason: string; evidenceLineNumbers: number[]; } interface CriSandboxReport { ok: boolean; source: "crictl" | "fixture" | "unavailable"; command: string[] | null; commandResult: CommandResultSummary | null; total: number; notReadyCount: number; staleNotReadyCount: number; unknownAgeCount: number; staleAfterMinutes: number; staleSandboxes: CriSandboxSummary[]; parseError: string | null; } interface CommandResultSummary { exitCode: number | null; signal: NodeJS.Signals | null; timedOut: boolean; stdoutTail: string; stderrTail: string; } interface CriSandboxSummary { id: string; name: string | null; namespace: string | null; state: string; ageMinutes: number | null; } interface WorktreeReadinessReport { ok: boolean; canonicalPath: PathCheck; compatibilitySymlink: SymlinkCheck; requiredFor: string[]; } interface SymlinkCheck extends PathCheck { expectedTarget: string; targetExists: boolean; targetKind: PathKind; } interface PathCheck { path: string; exists: boolean; kind: PathKind; isSymlink: boolean; symlinkTarget: string | null; resolvedPath: string | null; matchesExpectedType: boolean; expectedType: string; problem: string | null; } interface HostPathManifest { path: string; text: string; readError: string | null; } interface HostPathEntry { manifest: string; documentIndex: number; kind: string | null; name: string | null; namespace: string | null; volumeName: string | null; hostPath: string; hostPathType: string; } interface HostPathProblem { entry: HostPathEntry; check: PathCheck; severity: Severity; disposition: Disposition; reason: string; } interface HostPathReadinessReport { ok: boolean; manifests: HostPathManifest[]; totalEntries: number; checkedEntries: HostPathEntry[]; missing: HostPathProblem[]; typeMismatches: HostPathProblem[]; directoryOrCreateMissing: HostPathProblem[]; sensitiveUserDataBoundaries: string[]; } interface MdtodoAdjacentReadiness { ok: boolean; workspacePaths: string[]; logPaths: string[]; missingPaths: string[]; opaqueRisk: boolean; userDataBoundary: string; } interface ContainerCreatingReport { ok: boolean; source: "kubectl" | "fixture" | "unavailable"; command: string[] | null; commandResult: CommandResultSummary | null; totalContainerCreating: number; opaqueContainerCreating: ContainerCreatingFinding[]; hostPathContainerCreating: ContainerCreatingFinding[]; parseError: string | null; } interface ContainerCreatingFinding { namespace: string; pod: string; container: string; reason: string; messagePreview: string; classification: "opaque-containercreating-hostpath-risk" | "hostpath-mount-failure" | "containercreating-observed"; } const staleSandboxAfterMinutes = 30; const defaultCompatibilitySymlinkPath = "/home/ubuntu/cq-deploy"; const defaultManifestPaths = [ "src/components/microservices/k3sctl-adapter/k3s/code-queue.k8s.yaml", "src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-code-queue.k8s.yaml", "src/components/microservices/k3sctl-adapter/k3s/mdtodo.k8s.yaml", "src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-mdtodo.k8s.yaml", ]; const forbiddenAutomaticActions = [ "systemctl restart k3s", "service k3s restart", "kubectl delete pod", "crictl rmp", "crictl rm", "docker system prune", "docker volume prune", "rm -rf /home/ubuntu", "git reset --hard live worktree", ]; export function runD601RecoveryGuardrails(config: UniDeskConfig, fixture: RecoveryGuardrailsFixture = {}): RecoveryGuardrailsResult { const observedAt = fixture.observedAt ?? new Date().toISOString(); const filesRead: string[] = []; const commandsAttempted: string[][] = []; const codeQueueService = config.microservices.find((service) => service.id === "code-queue"); const deployWorktreePath = fixture.deployWorktreePath ?? codeQueueService?.development.worktreePath ?? "/home/ubuntu/unidesk-code-queue-deploy"; const compatibilitySymlinkPath = fixture.compatibilitySymlinkPath ?? defaultCompatibilitySymlinkPath; const manifestPaths = fixture.manifestPaths ?? defaultManifestPaths; const procMounts = fixture.procMountsText === undefined ? collectProcMounts("/proc/mounts", filesRead) : parseProcMounts(fixture.procMountsText, "fixture:/proc/mounts"); const kubeletValidationRisk = classifyKubeletValidationRisk(procMounts); const manifests = manifestPaths.map((manifestPath) => collectManifest(manifestPath, fixture, filesRead)); const hostPathEntries = manifests.flatMap((manifest) => manifest.readError === null ? extractHostPathEntries(manifest.path, manifest.text) : []); const hostPaths = evaluateHostPaths(hostPathEntries, manifests, fixture.pathStates); const worktree = evaluateWorktreeReadiness(deployWorktreePath, compatibilitySymlinkPath, fixture.pathStates); const mdtodo = evaluateMdtodoAdjacentHostPaths(hostPaths); const criSandboxes = fixture.criPodsJsonText === undefined ? collectCriSandboxes(observedAt, commandsAttempted) : parseCriSandboxReport(fixture.criPodsJsonText, observedAt, "fixture", null); const containerCreating = fixture.kubectlPodsJsonText === undefined ? collectContainerCreating(hostPaths, commandsAttempted) : parseContainerCreatingReport(fixture.kubectlPodsJsonText, hostPaths, "fixture", null); const redlines = buildRedlines({ procMounts, kubeletValidationRisk, criSandboxes, worktree, hostPaths, mdtodo, containerCreating, }); const redlineSummary = { red: redlines.filter((item) => item.severity === "red").length, yellow: redlines.filter((item) => item.severity === "yellow").length, safeReadOnly: true, requiresManualHostHotfix: redlines.filter((item) => item.requiresManualHostHotfix).map((item) => item.id), forbiddenAutomaticActions: Array.from(new Set(redlines.flatMap((item) => item.forbiddenAutomaticActions))), }; return { ok: redlineSummary.red === 0, surface: "d601-recovery-guardrails", mutation: false, observedAt, scope: { nodeId: "D601", environment: "host-read-only", liveMutationAllowed: false, }, safeReadOnly: { filesRead, commandsAttempted, note: "Diagnostics only read files, inspect path metadata, and run optional bounded read-only kubectl/crictl probes. They never delete CRI sandboxes, prune Docker state, edit hostPath directories, restart k3s, apply manifests, or rollout workloads.", }, redlineSummary, redlines, checks: { procMounts, kubeletValidationRisk, criSandboxes, codeQueueDeployWorktree: worktree, hostPaths, mdtodoAdjacentHostPaths: mdtodo, containerCreating, }, }; } export function compactD601RecoveryGuardrails(result: RecoveryGuardrailsResult): CompactRecoveryGuardrailsResult { return { ok: result.ok, surface: result.surface, mutation: false, observedAt: result.observedAt, scope: result.scope, safeReadOnly: result.safeReadOnly, redlineSummary: result.redlineSummary, redlines: result.redlines.map((item) => ({ id: item.id, severity: item.severity, disposition: item.disposition, summary: item.summary, recommendedNext: item.recommendedNext, })), checks: { procMounts: result.checks.procMounts, kubeletValidationRisk: result.checks.kubeletValidationRisk, criSandboxes: { ok: result.checks.criSandboxes.ok, source: result.checks.criSandboxes.source, command: result.checks.criSandboxes.command, total: result.checks.criSandboxes.total, notReadyCount: result.checks.criSandboxes.notReadyCount, staleNotReadyCount: result.checks.criSandboxes.staleNotReadyCount, unknownAgeCount: result.checks.criSandboxes.unknownAgeCount, staleAfterMinutes: result.checks.criSandboxes.staleAfterMinutes, staleSandboxes: result.checks.criSandboxes.staleSandboxes, parseError: result.checks.criSandboxes.parseError, }, codeQueueDeployWorktree: result.checks.codeQueueDeployWorktree, hostPaths: { ok: result.checks.hostPaths.ok, manifests: result.checks.hostPaths.manifests.map((manifest) => ({ path: manifest.path, readError: manifest.readError, textBytes: Buffer.byteLength(manifest.text, "utf8"), })), totalEntries: result.checks.hostPaths.totalEntries, checkedCount: result.checks.hostPaths.checkedEntries.length, missing: result.checks.hostPaths.missing.map(compactHostPathProblem), typeMismatches: result.checks.hostPaths.typeMismatches.map(compactHostPathProblem), directoryOrCreateMissing: result.checks.hostPaths.directoryOrCreateMissing.map(compactHostPathProblem), sensitiveUserDataBoundaries: result.checks.hostPaths.sensitiveUserDataBoundaries, }, mdtodoAdjacentHostPaths: result.checks.mdtodoAdjacentHostPaths, containerCreating: { ok: result.checks.containerCreating.ok, source: result.checks.containerCreating.source, command: result.checks.containerCreating.command, totalContainerCreating: result.checks.containerCreating.totalContainerCreating, opaqueContainerCreating: result.checks.containerCreating.opaqueContainerCreating, hostPathContainerCreating: result.checks.containerCreating.hostPathContainerCreating, parseError: result.checks.containerCreating.parseError, }, }, }; } export function parseProcMounts(text: string, source: string): ProcMountsReport { const lines = text.split(/\r?\n/u).filter((line) => line.trim().length > 0); const malformedLines: MountLineFinding[] = []; const dockerDesktopHost9pLines: MountLineFinding[] = []; lines.forEach((line, index) => { const tokens = line.trim().split(/\s+/u); const dockerDesktopHost9p = line.includes("/Docker/host") && /(?:^|\s)9p(?:\s|$)/u.test(line); const finding: MountLineFinding = { lineNumber: index + 1, fieldCount: tokens.length, dockerDesktopHost9p, textPreview: safePreview(line, 260), reason: tokens.length === 6 ? "docker-desktop-host-9p-line-observed" : "proc-mounts-line-field-count-not-six", }; if (dockerDesktopHost9p) dockerDesktopHost9pLines.push(finding); if (tokens.length !== 6) malformedLines.push(finding); }); return { ok: malformedLines.length === 0, source, totalLines: lines.length, malformedLines, dockerDesktopHost9pLines, readError: null, }; } export function classifyKubeletValidationRisk(report: ProcMountsReport): KubeletValidationRisk { const malformedDockerHost = report.malformedLines.filter((line) => line.dockerDesktopHost9p); if (report.malformedLines.length === 0) { return { ok: true, risk: "none", severity: "info", reason: report.dockerDesktopHost9pLines.length === 0 ? "No malformed /proc/mounts lines were detected." : "Docker Desktop /Docker/host 9p lines were observed but field counts are valid.", evidenceLineNumbers: [], }; } return { ok: false, risk: "kubelet-mount-table-validation-risk", severity: "red", reason: malformedDockerHost.length > 0 ? "Malformed Docker Desktop /Docker/host 9p mount rows can make kubelet mount-table validation reject hostPath setup after reboot." : "Malformed /proc/mounts rows can make kubelet mount-table validation unreliable.", evidenceLineNumbers: report.malformedLines.map((line) => line.lineNumber), }; } export function parseCriSandboxReport( jsonText: string, observedAt: string, source: "crictl" | "fixture", commandResult: CommandResultSummary | null, ): CriSandboxReport { try { const parsed = JSON.parse(jsonText) as unknown; const items = sandboxItems(parsed); const observedMs = Date.parse(observedAt); const notReady = items.filter((item) => { const state = String(recordField(item, "state") ?? "").toUpperCase(); return state.length === 0 || (!state.includes("READY") && state !== "SANDBOX_READY" && state !== "READY") || state.includes("NOTREADY"); }); const summaries = notReady.map((item) => sandboxSummary(item, observedMs)); const staleSandboxes = summaries.filter((item) => item.ageMinutes === null || item.ageMinutes >= staleSandboxAfterMinutes); return { ok: staleSandboxes.length === 0, source, command: source === "crictl" ? ["crictl", "pods", "-o", "json"] : null, commandResult, total: items.length, notReadyCount: notReady.length, staleNotReadyCount: staleSandboxes.length, unknownAgeCount: summaries.filter((item) => item.ageMinutes === null).length, staleAfterMinutes: staleSandboxAfterMinutes, staleSandboxes, parseError: null, }; } catch (error) { return { ok: false, source, command: source === "crictl" ? ["crictl", "pods", "-o", "json"] : null, commandResult, total: 0, notReadyCount: 0, staleNotReadyCount: 0, unknownAgeCount: 0, staleAfterMinutes: staleSandboxAfterMinutes, staleSandboxes: [], parseError: errorMessage(error), }; } } export function extractHostPathEntries(manifestPath: string, text: string): HostPathEntry[] { return splitManifestDocuments(text).flatMap((doc) => { const lines = doc.raw.split(/\r?\n/u); const entries: HostPathEntry[] = []; for (let index = 0; index < lines.length; index += 1) { if (!/^\s*hostPath:\s*$/u.test(lines[index] ?? "")) continue; const volumeName = findVolumeName(lines, index); const path = findScalarAfter(lines, index, "path"); if (path === null) continue; entries.push({ manifest: manifestPath, documentIndex: doc.index, kind: doc.kind, name: doc.name, namespace: doc.namespace, volumeName, hostPath: path, hostPathType: findScalarAfter(lines, index, "type") ?? "Unset", }); } return entries; }); } export function evaluateHostPaths( entries: HostPathEntry[], manifests: HostPathManifest[], fixtures: Record | undefined = undefined, ): HostPathReadinessReport { const checkedEntries = dedupeHostPathEntries(entries); const problems = checkedEntries.map((entry) => hostPathProblem(entry, checkPath(entry.hostPath, entry.hostPathType, fixtures))).filter((item): item is HostPathProblem => item !== null); return { ok: problems.filter((problem) => problem.severity === "red").length === 0, manifests, totalEntries: entries.length, checkedEntries, missing: problems.filter((problem) => problem.check.problem === "missing"), typeMismatches: problems.filter((problem) => problem.check.problem === "type-mismatch"), directoryOrCreateMissing: problems.filter((problem) => problem.check.problem === "directory-or-create-missing"), sensitiveUserDataBoundaries: [ "/home/ubuntu/.codex/auth.json", "/home/ubuntu/.ssh", "/home/ubuntu/.agents/skills", "/home/ubuntu/cq-deploy/.state/mdtodo-workspace", "/home/ubuntu/unidesk-dev-mdtodo-workspace", ], }; } export function evaluateWorktreeReadiness( canonicalPath: string, compatibilitySymlinkPath = defaultCompatibilitySymlinkPath, fixtures: Record | undefined = undefined, ): WorktreeReadinessReport { const canonicalPathCheck = checkPath(canonicalPath, "Directory", fixtures); const symlink = checkPath(compatibilitySymlinkPath, "Directory", fixtures); const expectedTarget = canonicalPath; const targetPath = symlink.resolvedPath ?? resolveSymlinkTarget(compatibilitySymlinkPath, symlink.symlinkTarget); const targetCheck = targetPath === null ? null : checkPath(targetPath, "Directory", fixtures); return { ok: canonicalPathCheck.matchesExpectedType && symlink.matchesExpectedType && (symlink.isSymlink ? targetCheck?.matchesExpectedType === true : true), canonicalPath: canonicalPathCheck, compatibilitySymlink: { ...symlink, expectedTarget, targetExists: targetCheck?.exists ?? false, targetKind: targetCheck?.kind ?? "missing", }, requiredFor: [ "config.json microservices.code-queue.development.worktreePath", "production code-queue /root/unidesk and /app hostPath mapping through /home/ubuntu/cq-deploy", "MDTODO production workspace/log hostPath adjacency under /home/ubuntu/cq-deploy/.state", ], }; } export function parseContainerCreatingReport( jsonText: string, hostPaths: HostPathReadinessReport, source: "kubectl" | "fixture", commandResult: CommandResultSummary | null, ): ContainerCreatingReport { try { const parsed = JSON.parse(jsonText) as unknown; const pods = podItems(parsed); const hostPathNeedles = missingHostPathNeedles(hostPaths); const findings = pods.flatMap((pod) => containerCreatingFindings(pod, hostPathNeedles)); const opaque = findings.filter((item) => item.classification === "opaque-containercreating-hostpath-risk"); const hostPathFailures = findings.filter((item) => item.classification === "hostpath-mount-failure"); return { ok: opaque.length === 0 && hostPathFailures.length === 0, source, command: source === "kubectl" ? ["kubectl", "get", "pods", "-A", "-o", "json"] : null, commandResult, totalContainerCreating: findings.length, opaqueContainerCreating: opaque, hostPathContainerCreating: hostPathFailures, parseError: null, }; } catch (error) { return { ok: false, source, command: source === "kubectl" ? ["kubectl", "get", "pods", "-A", "-o", "json"] : null, commandResult, totalContainerCreating: 0, opaqueContainerCreating: [], hostPathContainerCreating: [], parseError: errorMessage(error), }; } } function collectProcMounts(path: string, filesRead: string[]): ProcMountsReport { filesRead.push(path); try { return parseProcMounts(readFileSync(path, "utf8"), path); } catch (error) { return { ok: false, source: path, totalLines: 0, malformedLines: [], dockerDesktopHost9pLines: [], readError: errorMessage(error), }; } } function collectManifest(manifestPath: string, fixture: RecoveryGuardrailsFixture, filesRead: string[]): HostPathManifest { if (fixture.manifestTexts?.[manifestPath] !== undefined) { return { path: manifestPath, text: fixture.manifestTexts[manifestPath] ?? "", readError: null }; } const absolute = rootPath(manifestPath); filesRead.push(absolute); try { return { path: manifestPath, text: readFileSync(absolute, "utf8"), readError: null }; } catch (error) { return { path: manifestPath, text: "", readError: errorMessage(error) }; } } function collectCriSandboxes(observedAt: string, commandsAttempted: string[][]): CriSandboxReport { const probe = runCommand(["sh", "-lc", "command -v crictl >/dev/null 2>&1"], repoRoot, { timeoutMs: 5_000 }); if (probe.exitCode !== 0) { return unavailableCriSandboxReport("crictl binary is not available in this environment."); } const command = ["crictl", "pods", "-o", "json"]; commandsAttempted.push(command); const result = runCommand(command, repoRoot, { timeoutMs: 10_000 }); const summary = summarizeCommandResult(result); if (result.exitCode !== 0) { return { ...unavailableCriSandboxReport("crictl read-only pod sandbox probe failed."), command, commandResult: summary, }; } return parseCriSandboxReport(result.stdout, observedAt, "crictl", summary); } function collectContainerCreating(hostPaths: HostPathReadinessReport, commandsAttempted: string[][]): ContainerCreatingReport { const probe = runCommand(["sh", "-lc", "command -v kubectl >/dev/null 2>&1"], repoRoot, { timeoutMs: 5_000 }); if (probe.exitCode !== 0) { return unavailableContainerCreatingReport("kubectl binary is not available in this environment."); } const command = ["kubectl", "get", "pods", "-A", "-o", "json"]; commandsAttempted.push(command); const result = runCommand(command, repoRoot, { timeoutMs: 10_000 }); const summary = summarizeCommandResult(result); if (result.exitCode !== 0) { return { ...unavailableContainerCreatingReport("kubectl read-only pod status probe failed."), command, commandResult: summary, }; } return parseContainerCreatingReport(result.stdout, hostPaths, "kubectl", summary); } function unavailableCriSandboxReport(reason: string): CriSandboxReport { return { ok: true, source: "unavailable", command: null, commandResult: null, total: 0, notReadyCount: 0, staleNotReadyCount: 0, unknownAgeCount: 0, staleAfterMinutes: staleSandboxAfterMinutes, staleSandboxes: [], parseError: reason, }; } function unavailableContainerCreatingReport(reason: string): ContainerCreatingReport { return { ok: true, source: "unavailable", command: null, commandResult: null, totalContainerCreating: 0, opaqueContainerCreating: [], hostPathContainerCreating: [], parseError: reason, }; } function evaluateMdtodoAdjacentHostPaths(hostPaths: HostPathReadinessReport): MdtodoAdjacentReadiness { const mdtodoEntries = hostPaths.checkedEntries.filter((entry) => entry.hostPath.includes("mdtodo")); const workspacePaths = mdtodoEntries.filter((entry) => entry.volumeName === "workspace" || entry.hostPath.includes("workspace")).map((entry) => entry.hostPath); const logPaths = mdtodoEntries.filter((entry) => entry.volumeName === "logs" || entry.hostPath.includes("/logs")).map((entry) => entry.hostPath); const missingPaths = Array.from(new Set([ ...hostPaths.missing.filter((problem) => problem.entry.hostPath.includes("mdtodo")).map((problem) => problem.entry.hostPath), ...hostPaths.directoryOrCreateMissing.filter((problem) => problem.entry.hostPath.includes("mdtodo")).map((problem) => problem.entry.hostPath), ])); return { ok: missingPaths.length === 0 && workspacePaths.length > 0, workspacePaths: Array.from(new Set(workspacePaths)), logPaths: Array.from(new Set(logPaths)), missingPaths, opaqueRisk: missingPaths.length > 0, userDataBoundary: "MDTODO workspace hostPaths are user data. Recovery diagnostics may report readiness, but must not create, delete, prune, reset, or replace the workspace automatically.", }; } function buildRedlines(reports: { procMounts: ProcMountsReport; kubeletValidationRisk: KubeletValidationRisk; criSandboxes: CriSandboxReport; worktree: WorktreeReadinessReport; hostPaths: HostPathReadinessReport; mdtodo: MdtodoAdjacentReadiness; containerCreating: ContainerCreatingReport; }): RecoveryGuardrailRedline[] { const redlines: RecoveryGuardrailRedline[] = []; if (!reports.procMounts.ok || !reports.kubeletValidationRisk.ok) { redlines.push(redline( "proc-mounts-malformed-kubelet-risk", "red", "requires-manual-host-hotfix", "Malformed /proc/mounts lines create kubelet hostPath validation risk.", { source: reports.procMounts.source, malformedLines: reports.procMounts.malformedLines, kubeletValidationRisk: reports.kubeletValidationRisk, }, [ "Host commander or user should repair the Docker Desktop/WSL mount-table condition and then run this read-only check again.", "Do not restart k3s from Code Queue or this CLI without explicit host authorization.", ], )); } if (reports.criSandboxes.staleNotReadyCount > 0 || reports.criSandboxes.parseError !== null && reports.criSandboxes.source !== "unavailable") { redlines.push(redline( "stale-cri-sandboxes-observed", reports.criSandboxes.staleNotReadyCount > 0 ? "red" : "yellow", "requires-manual-host-hotfix", "Stale or unreadable CRI pod sandbox state needs host-side review; automatic deletion is forbidden.", { source: reports.criSandboxes.source, staleNotReadyCount: reports.criSandboxes.staleNotReadyCount, unknownAgeCount: reports.criSandboxes.unknownAgeCount, staleSandboxes: reports.criSandboxes.staleSandboxes, parseError: reports.criSandboxes.parseError, }, [ "Review CRI sandbox state on D601 host with bounded read-only crictl/kubectl commands.", "Escalate to host commander/user before any sandbox cleanup.", ], )); } if (!reports.worktree.ok) { redlines.push(redline( "code-queue-deploy-worktree-not-ready", "red", "requires-manual-host-hotfix", "Code Queue deploy worktree or compatibility symlink is missing or unresolved.", reports.worktree as unknown as Record, [ "Restore the Git worktree/symlink on D601 host from pushed Git state.", "Do not git reset --hard or replace a dirty live worktree without explicit user/host approval.", ], )); } if (!reports.hostPaths.ok || reports.hostPaths.directoryOrCreateMissing.length > 0) { redlines.push(redline( "required-hostpaths-not-ready", reports.hostPaths.ok ? "yellow" : "red", "requires-manual-host-hotfix", "One or more k3s hostPath mounts required by Code Queue or MDTODO are missing or have the wrong type.", { missing: reports.hostPaths.missing, typeMismatches: reports.hostPaths.typeMismatches, directoryOrCreateMissing: reports.hostPaths.directoryOrCreateMissing, sensitiveUserDataBoundaries: reports.hostPaths.sensitiveUserDataBoundaries, }, [ "Repair only the specific missing hostPath on D601 host after reviewing whether it is user data, credential material, or cache/log state.", "Do not mass-create, delete, prune, chmod recursively, or replace hostPath directories from Code Queue.", ], )); } if (!reports.mdtodo.ok) { redlines.push(redline( "mdtodo-adjacent-hostpaths-opaque", "red", "requires-manual-host-hotfix", "MDTODO workspace/log hostPaths are not transparently ready, making ContainerCreating recovery ambiguous.", reports.mdtodo as unknown as Record, [ "Verify the MDTODO workspace source and logs path on D601 host before restarting or rolling pods.", "Ask the user before touching the MDTODO workspace because it contains user-authored Markdown data.", ], )); } if (reports.containerCreating.opaqueContainerCreating.length > 0 || reports.containerCreating.hostPathContainerCreating.length > 0) { redlines.push(redline( "opaque-containercreating-hostpath-risk", "red", "requires-manual-host-hotfix", "Pods stuck in ContainerCreating have opaque or hostPath-looking evidence.", { opaqueContainerCreating: reports.containerCreating.opaqueContainerCreating, hostPathContainerCreating: reports.containerCreating.hostPathContainerCreating, }, [ "Use kubectl describe/get events on D601 host to confirm the exact mount failure.", "Do not delete pods or sandboxes automatically; first fix the hostPath or kubelet mount-table condition.", ], )); } redlines.push(redline( "destructive-recovery-actions-forbidden", "yellow", "forbidden-automatic-action", "Recovery diagnostics must never perform destructive cleanup or rollout as an automatic follow-up.", { forbiddenAutomaticActions }, [ "Keep this CLI read-only.", "Use ClaudeQQ/user approval for high-risk host actions such as k3s restart, CRI cleanup, user-data directory repair, or Code Queue runtime intervention.", ], )); return redlines; } function redline( id: string, severity: Severity, disposition: Disposition, summary: string, evidence: Record, recommendedNext: string[], ): RecoveryGuardrailRedline { return { id, severity, disposition, summary, evidence, safeReadOnly: true, requiresManualHostHotfix: disposition === "requires-manual-host-hotfix", forbiddenAutomaticActions, recommendedNext, }; } function hostPathProblem(entry: HostPathEntry, check: PathCheck): HostPathProblem | null { if (check.problem === null) return null; if (check.problem === "directory-or-create-missing") { return { entry, check, severity: "yellow", disposition: "requires-manual-host-hotfix", reason: "DirectoryOrCreate hostPath is missing; kubelet may create it only if mount-table validation and parent permissions are healthy.", }; } return { entry, check, severity: "red", disposition: "requires-manual-host-hotfix", reason: check.problem, }; } function checkPath(path: string, expectedType: string, fixtures: Record | undefined): PathCheck { const fixture = fixtures?.[path]; if (fixture !== undefined) return checkFixturePath(path, expectedType, fixture); try { const lstat = lstatSync(path); const isSymlink = lstat.isSymbolicLink(); const symlinkTarget = isSymlink ? readlinkSync(path) : null; const resolvedPath = isSymlink ? resolveSymlinkTarget(path, symlinkTarget) : path; const stat = statSync(resolvedPath ?? path); const kind = stat.isDirectory() ? "directory" : stat.isFile() ? "file" : stat.isSocket() ? "socket" : isSymlink ? "symlink" : "other"; const matchesExpectedType = matchesType(kind, expectedType); return { path, exists: true, kind, isSymlink, symlinkTarget, resolvedPath, matchesExpectedType, expectedType, problem: matchesExpectedType ? null : "type-mismatch", }; } catch (error) { try { const lstat = lstatSync(path); if (lstat.isSymbolicLink()) { const symlinkTarget = readlinkSync(path); return { path, exists: true, kind: "symlink", isSymlink: true, symlinkTarget, resolvedPath: resolveSymlinkTarget(path, symlinkTarget), matchesExpectedType: false, expectedType, problem: "symlink-target-missing", }; } } catch { // Fall through to the generic missing/error report. } if (isMissingPathError(error)) return missingPathCheck(path, expectedType); return { path, exists: false, kind: "missing", isSymlink: false, symlinkTarget: null, resolvedPath: null, matchesExpectedType: false, expectedType, problem: errorMessage(error), }; } } function checkFixturePath(path: string, expectedType: string, fixture: PathStateFixture): PathCheck { const exists = fixture.kind !== "missing"; const resolvedPath = fixture.kind === "symlink" && fixture.target !== undefined ? resolveSymlinkTarget(path, fixture.target) : exists ? path : null; const targetExists = fixture.kind !== "symlink" ? exists : fixture.targetExists === true; const effectiveKind = fixture.kind === "symlink" && targetExists ? "directory" : fixture.kind; const matchesExpectedType = exists && matchesType(effectiveKind, expectedType) && (fixture.kind !== "symlink" || targetExists); return { path, exists, kind: fixture.kind, isSymlink: fixture.kind === "symlink", symlinkTarget: fixture.target ?? null, resolvedPath, matchesExpectedType, expectedType, problem: matchesExpectedType ? null : expectedType === "DirectoryOrCreate" && !exists ? "directory-or-create-missing" : exists ? "type-mismatch" : "missing", }; } function missingPathCheck(path: string, expectedType: string): PathCheck { return { path, exists: false, kind: "missing", isSymlink: false, symlinkTarget: null, resolvedPath: null, matchesExpectedType: expectedType === "DirectoryOrCreate", expectedType, problem: expectedType === "DirectoryOrCreate" ? "directory-or-create-missing" : "missing", }; } function matchesType(kind: PathKind, expectedType: string): boolean { if (expectedType === "Unset" || expectedType.length === 0) return kind !== "missing"; if (expectedType === "Directory" || expectedType === "DirectoryOrCreate") return kind === "directory"; if (expectedType === "File" || expectedType === "FileOrCreate") return kind === "file"; if (expectedType === "Socket") return kind === "socket"; return kind !== "missing"; } function resolveSymlinkTarget(path: string, target: string | null): string | null { if (target === null) return null; return isAbsolute(target) ? target : resolve(dirname(path), target); } function dedupeHostPathEntries(entries: HostPathEntry[]): HostPathEntry[] { const seen = new Set(); const result: HostPathEntry[] = []; for (const entry of entries) { const key = `${entry.hostPath}\0${entry.hostPathType}`; if (seen.has(key)) continue; seen.add(key); result.push(entry); } return result; } function splitManifestDocuments(text: string): Array<{ index: number; raw: string; kind: string | null; name: string | null; namespace: string | null }> { return text.split(/^---\s*$/mu) .map((raw, index) => ({ index, raw: raw.trim() })) .filter((doc) => doc.raw.length > 0) .map((doc) => ({ ...doc, kind: scalarAfter(doc.raw, "kind"), name: metadataScalarAfter(doc.raw, "name"), namespace: metadataScalarAfter(doc.raw, "namespace"), })); } function scalarAfter(text: string, key: string): string | null { const match = text.match(new RegExp(`^\\s*${key}:\\s*"?([^"\\n#]+)"?\\s*(?:#.*)?$`, "mu")); return match?.[1]?.trim() ?? null; } function metadataScalarAfter(text: string, key: string): string | null { const metadataIndex = text.search(/^metadata:\s*$/mu); if (metadataIndex < 0) return null; return scalarAfter(text.slice(metadataIndex), key); } function findVolumeName(lines: string[], hostPathIndex: number): string | null { for (let index = hostPathIndex - 1; index >= Math.max(0, hostPathIndex - 8); index -= 1) { const match = (lines[index] ?? "").match(/^\s*-\s+name:\s*"?([^"\n#]+)"?/u); if (match?.[1] !== undefined) return match[1].trim(); } return null; } function findScalarAfter(lines: string[], startIndex: number, key: string): string | null { for (let index = startIndex + 1; index < Math.min(lines.length, startIndex + 8); index += 1) { const match = (lines[index] ?? "").match(new RegExp(`^\\s*${key}:\\s*"?([^"\\n#]+)"?\\s*(?:#.*)?$`, "u")); if (match?.[1] !== undefined) return match[1].trim(); } return null; } function sandboxItems(parsed: unknown): Record[] { const record = asRecord(parsed); const items = record.items ?? record.sandboxes ?? record.pods ?? []; return Array.isArray(items) ? items.flatMap((item) => typeof item === "object" && item !== null ? [item as Record] : []) : []; } function sandboxSummary(item: Record, observedMs: number): CriSandboxSummary { const metadata = asRecord(recordField(item, "metadata")); const createdAt = recordField(item, "createdAt") ?? recordField(item, "created_at") ?? recordField(item, "created"); const createdMs = typeof createdAt === "string" || typeof createdAt === "number" ? Number(createdAt) > 10_000_000_000_000 ? Math.floor(Number(createdAt) / 1_000_000) : Date.parse(String(createdAt)) : NaN; const ageMinutes = Number.isFinite(createdMs) && Number.isFinite(observedMs) ? Math.max(0, Math.floor((observedMs - createdMs) / 60_000)) : null; return { id: String(recordField(item, "id") ?? recordField(item, "podSandboxId") ?? "unknown"), name: stringOrNull(recordField(metadata, "name")), namespace: stringOrNull(recordField(metadata, "namespace")), state: String(recordField(item, "state") ?? "unknown"), ageMinutes, }; } function podItems(parsed: unknown): Record[] { const record = asRecord(parsed); const items = record.items ?? []; return Array.isArray(items) ? items.flatMap((item) => typeof item === "object" && item !== null ? [item as Record] : []) : []; } function containerCreatingFindings(pod: Record, hostPathNeedles: string[]): ContainerCreatingFinding[] { const metadata = asRecord(recordField(pod, "metadata")); const status = asRecord(recordField(pod, "status")); const namespace = stringOrNull(recordField(metadata, "namespace")) ?? "default"; const podName = stringOrNull(recordField(metadata, "name")) ?? "unknown"; const statuses = [ ...statusArray(status, "initContainerStatuses"), ...statusArray(status, "containerStatuses"), ]; return statuses.flatMap((container) => { const state = asRecord(recordField(container, "state")); const waiting = asRecord(recordField(state, "waiting")); const reason = stringOrNull(recordField(waiting, "reason")) ?? ""; if (reason !== "ContainerCreating") return []; const message = stringOrNull(recordField(waiting, "message")) ?? ""; const classification = classifyContainerCreating(message, hostPathNeedles); return [{ namespace, pod: podName, container: stringOrNull(recordField(container, "name")) ?? "unknown", reason, messagePreview: safePreview(message, 260), classification, }]; }); } function classifyContainerCreating(message: string, hostPathNeedles: string[]): ContainerCreatingFinding["classification"] { const lower = message.toLowerCase(); if (hostPathNeedles.some((needle) => needle.length > 0 && message.includes(needle)) || lower.includes("hostpath") || lower.includes("mountvolume") || lower.includes("not a directory") || lower.includes("no such file or directory")) { return "hostpath-mount-failure"; } if (message.trim().length === 0 && hostPathNeedles.length > 0) return "opaque-containercreating-hostpath-risk"; return "containercreating-observed"; } function missingHostPathNeedles(hostPaths: HostPathReadinessReport): string[] { return Array.from(new Set([ ...hostPaths.missing.map((problem) => problem.entry.hostPath), ...hostPaths.typeMismatches.map((problem) => problem.entry.hostPath), ...hostPaths.directoryOrCreateMissing.map((problem) => problem.entry.hostPath), ])); } function statusArray(status: Record, key: string): Record[] { const value = recordField(status, key); return Array.isArray(value) ? value.flatMap((item) => typeof item === "object" && item !== null ? [item as Record] : []) : []; } function summarizeCommandResult(result: CommandResult): CommandResultSummary { return { exitCode: result.exitCode, signal: result.signal, timedOut: result.timedOut, stdoutTail: result.stdout.length === 0 ? "" : ``, stderrTail: result.stderr.slice(-2000), }; } function compactHostPathProblem(problem: HostPathProblem): CompactHostPathProblem { return { hostPath: problem.entry.hostPath, hostPathType: problem.entry.hostPathType, manifest: problem.entry.manifest, workload: `${problem.entry.namespace ?? "default"}/${problem.entry.kind ?? "Unknown"}/${problem.entry.name ?? "unknown"}`, volumeName: problem.entry.volumeName, problem: problem.check.problem, severity: problem.severity, disposition: problem.disposition, reason: problem.reason, }; } function recordField(record: Record, key: string): unknown; function recordField(record: unknown, key: string): unknown; function recordField(record: unknown, key: string): unknown { return typeof record === "object" && record !== null && key in record ? (record as Record)[key] : undefined; } function asRecord(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } function isMissingPathError(error: unknown): boolean { if (typeof error !== "object" || error === null || !("code" in error)) return false; const code = (error as { code?: unknown }).code; return code === "ENOENT" || code === "ENOTDIR"; } function safePreview(text: string, maxChars: number): string { return text.length <= maxChars ? text : `${text.slice(0, maxChars)}...`; }