feat: add d601 recovery guardrails

Adds read-only D601 recovery diagnostics, fixture coverage, CLI wiring, and recovery hotfix runbook updates. Validated with recovery contract, check --files, scripts tsc, artifact matrix direct contract, and read-only live diagnostic.
2026-05-23 21:18:44 +08:00
parent 6c44f66289
commit e2646763c0
9 changed files with 1406 additions and 5 deletions
@@ -4,7 +4,7 @@ import { isRebuildableService, rebuildService, stackLogs, stackStatus, startStac
 import { parseE2ERunOptions, runE2E } from "./src/e2e";
 import { emitError, emitJson } from "./src/output";
 import { jobWithTail, listJobs, listJobsSummary, readJob, runJob } from "./src/jobs";
-import { checkHelp, parseCheckOptions, runChecks } from "./src/check";
+import { checkHelp, parseCheckOptions, runChecks, runRecoveryGuardrailsCheck } from "./src/check";
 import { runSsh } from "./src/ssh";
 import { autoRemoteCiPublishUserServiceDryRunPlan, extractRemoteCliOptions, runRemoteCli } from "./src/remote";
 import { runMicroserviceCommand } from "./src/microservices";
@@ -291,6 +291,12 @@ async function main(): Promise<void> {
      emitJson(commandName, checkHelp());
      return;
    }
+    if (sub === "recovery-guardrails") {
+      const result = runRecoveryGuardrailsCheck(config);
+      emitJson(commandName, result, result.ok);
+      if (!result.ok) process.exitCode = 1;
+      return;
+    }
    const result = runChecks(config, parseCheckOptions(args.slice(1)));
    emitJson(commandName, result, result.ok);
    if (!result.ok) process.exitCode = 1;
@@ -0,0 +1,157 @@
+import { readConfig } from "./src/config";
+import { extractHostPathEntries, parseContainerCreatingReport, parseProcMounts, runD601RecoveryGuardrails, type RecoveryGuardrailsFixture } from "./src/recovery-guardrails";
+
+type JsonRecord = Record<string, unknown>;
+
+function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
+  if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
+}
+
+const malformedProcMounts = [
+  "proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0",
+  "drvfs /Docker/host 9p rw,dirsync,aname=drvfs;path=C:\\Program Files\\Docker\\Docker Desktop\\host 0 0",
+].join("\n");
+
+const manifest = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: code-queue-scheduler-dev
+  namespace: unidesk-dev
+spec:
+  template:
+    spec:
+      volumes:
+        - name: repo
+          hostPath:
+            path: /home/ubuntu/unidesk-dev-code-queue-deploy/code-queue
+            type: Directory
+        - name: state
+          hostPath:
+            path: /home/ubuntu/unidesk-dev-code-queue-deploy/state/code-queue
+            type: DirectoryOrCreate
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mdtodo-dev
+  namespace: unidesk-dev
+spec:
+  template:
+    spec:
+      volumes:
+        - name: workspace
+          hostPath:
+            path: /home/ubuntu/unidesk-dev-mdtodo-workspace
+            type: Directory
+        - name: logs
+          hostPath:
+            path: /home/ubuntu/cq-deploy/.state/mdtodo-dev/logs
+            type: DirectoryOrCreate
+`;
+
+const kubectlPods = {
+  items: [
+    {
+      metadata: { namespace: "unidesk-dev", name: "code-queue-scheduler-dev-abc" },
+      status: {
+        containerStatuses: [{
+          name: "code-queue",
+          state: {
+            waiting: {
+              reason: "ContainerCreating",
+              message: "MountVolume.SetUp failed for volume \"repo\": hostPath type check failed: /home/ubuntu/unidesk-dev-code-queue-deploy/code-queue is not a directory",
+            },
+          },
+        }],
+      },
+    },
+    {
+      metadata: { namespace: "unidesk-dev", name: "mdtodo-dev-def" },
+      status: {
+        containerStatuses: [{
+          name: "mdtodo",
+          state: { waiting: { reason: "ContainerCreating", message: "" } },
+        }],
+      },
+    },
+  ],
+};
+
+const staleCriPods = {
+  items: [
+    {
+      id: "sandbox-stale",
+      metadata: { name: "code-queue-scheduler-dev-abc", namespace: "unidesk-dev" },
+      state: "SANDBOX_NOTREADY",
+      createdAt: "2026-05-23T09:00:00.000Z",
+    },
+  ],
+};
+
+export function runD601RecoveryGuardrailsContract(): JsonRecord {
+  const procMounts = parseProcMounts(malformedProcMounts, "fixture:/proc/mounts");
+  assertCondition(procMounts.ok === false, "malformed /proc/mounts fixture should fail", procMounts);
+  assertCondition(procMounts.malformedLines.length === 1, "one malformed mount line expected", procMounts);
+  assertCondition(procMounts.malformedLines[0]?.dockerDesktopHost9p === true, "malformed line should be classified as Docker Desktop /Docker/host 9p", procMounts);
+
+  const entries = extractHostPathEntries("fixture.k8s.yaml", manifest);
+  assertCondition(entries.some((entry) => entry.hostPath === "/home/ubuntu/unidesk-dev-code-queue-deploy/code-queue"), "hostPath parser should extract repo path", entries);
+  assertCondition(entries.some((entry) => entry.hostPath === "/home/ubuntu/unidesk-dev-mdtodo-workspace"), "hostPath parser should extract MDTODO workspace path", entries);
+
+  const fixture: RecoveryGuardrailsFixture = {
+    observedAt: "2026-05-23T10:00:00.000Z",
+    procMountsText: malformedProcMounts,
+    criPodsJsonText: JSON.stringify(staleCriPods),
+    kubectlPodsJsonText: JSON.stringify(kubectlPods),
+    deployWorktreePath: "/home/ubuntu/unidesk-code-queue-deploy",
+    compatibilitySymlinkPath: "/home/ubuntu/cq-deploy",
+    manifestPaths: ["fixture.k8s.yaml"],
+    manifestTexts: { "fixture.k8s.yaml": manifest },
+    pathStates: {
+      "/home/ubuntu/unidesk-code-queue-deploy": { kind: "missing" },
+      "/home/ubuntu/cq-deploy": { kind: "symlink", target: "/home/ubuntu/unidesk-code-queue-deploy", targetExists: false },
+      "/home/ubuntu/unidesk-dev-code-queue-deploy/code-queue": { kind: "missing" },
+      "/home/ubuntu/unidesk-dev-code-queue-deploy/state/code-queue": { kind: "missing" },
+      "/home/ubuntu/unidesk-dev-mdtodo-workspace": { kind: "missing" },
+      "/home/ubuntu/cq-deploy/.state/mdtodo-dev/logs": { kind: "missing" },
+    },
+  };
+  const result = runD601RecoveryGuardrails(readConfig(), fixture);
+  assertCondition(result.scope.liveMutationAllowed === false && result.mutation === false, "guardrail result must be read-only", result.scope);
+  assertCondition(result.ok === false, "fixture should produce red guardrails", result.redlineSummary);
+  assertCondition(result.redlineSummary.requiresManualHostHotfix.includes("proc-mounts-malformed-kubelet-risk"), "malformed mount risk should require manual host hotfix", result.redlineSummary);
+  assertCondition(result.redlineSummary.requiresManualHostHotfix.includes("code-queue-deploy-worktree-not-ready"), "missing worktree/symlink target should require manual host hotfix", result.redlineSummary);
+  assertCondition(result.redlineSummary.requiresManualHostHotfix.includes("required-hostpaths-not-ready"), "missing hostPath should require manual host hotfix", result.redlineSummary);
+  assertCondition(result.redlineSummary.requiresManualHostHotfix.includes("opaque-containercreating-hostpath-risk"), "ContainerCreating hostPath risk should require manual host hotfix", result.redlineSummary);
+  assertCondition(result.checks.codeQueueDeployWorktree.canonicalPath.exists === false, "canonical deploy worktree should report missing", result.checks.codeQueueDeployWorktree);
+  assertCondition(result.checks.codeQueueDeployWorktree.compatibilitySymlink.isSymlink === true, "compat symlink should be visible", result.checks.codeQueueDeployWorktree);
+  assertCondition(result.checks.codeQueueDeployWorktree.compatibilitySymlink.targetExists === false, "compat symlink target should report missing", result.checks.codeQueueDeployWorktree);
+  assertCondition(result.checks.hostPaths.missing.some((problem) => problem.entry.hostPath === "/home/ubuntu/unidesk-dev-code-queue-deploy/code-queue"), "missing Directory hostPath should be red", result.checks.hostPaths.missing);
+  assertCondition(result.checks.hostPaths.directoryOrCreateMissing.some((problem) => problem.entry.hostPath === "/home/ubuntu/unidesk-dev-code-queue-deploy/state/code-queue"), "DirectoryOrCreate missing should be surfaced separately", result.checks.hostPaths.directoryOrCreateMissing);
+  assertCondition(result.checks.mdtodoAdjacentHostPaths.opaqueRisk === true, "MDTODO adjacent hostPath readiness should be opaque when workspace/logs missing", result.checks.mdtodoAdjacentHostPaths);
+  assertCondition(result.checks.criSandboxes.staleNotReadyCount === 1, "stale CRI sandbox should be counted", result.checks.criSandboxes);
+  assertCondition(result.checks.containerCreating.hostPathContainerCreating.length === 1, "explicit hostPath ContainerCreating should be classified", result.checks.containerCreating);
+  assertCondition(result.checks.containerCreating.opaqueContainerCreating.length === 1, "empty-message ContainerCreating should be classified as opaque when hostPaths are missing", result.checks.containerCreating);
+  assertCondition(result.redlineSummary.forbiddenAutomaticActions.includes("crictl rmp"), "CRI deletion should be explicitly forbidden", result.redlineSummary);
+  assertCondition(result.redlineSummary.forbiddenAutomaticActions.includes("systemctl restart k3s"), "k3s restart should be explicitly forbidden", result.redlineSummary);
+
+  const containerCreatingOnly = parseContainerCreatingReport(JSON.stringify(kubectlPods), result.checks.hostPaths, "fixture", null);
+  assertCondition(containerCreatingOnly.ok === false, "ContainerCreating parser should fail when opaque/hostPath findings exist", containerCreatingOnly);
+
+  return {
+    ok: true,
+    checks: [
+      "malformed /proc/mounts Docker Desktop /Docker/host 9p line detected",
+      "missing Code Queue worktree symlink and target reported",
+      "missing hostPath and DirectoryOrCreate readiness reported",
+      "opaque and explicit hostPath ContainerCreating classified",
+      "stale CRI sandbox count reported without cleanup",
+      "destructive prune/reset/restart/delete actions forbidden",
+    ],
+  };
+}
+
+if (import.meta.main) {
+  process.stdout.write(`${JSON.stringify(runD601RecoveryGuardrailsContract(), null, 2)}\n`);
+}
@@ -3,6 +3,7 @@ import { extname } from "node:path";
 import { runCommand } from "./command";
 import { type UniDeskConfig, repoRoot, rootPath } from "./config";
 import { composeConfig } from "./docker";
+import { compactD601RecoveryGuardrails, runD601RecoveryGuardrails } from "./recovery-guardrails";

 interface CheckItem {
  name: string;
@@ -27,6 +28,7 @@ const syntaxFiles = [
  "scripts/src/e2e.ts",
  "scripts/src/help.ts",
  "scripts/src/commander.ts",
+  "scripts/src/recovery-guardrails.ts",
  "scripts/src/server-cleanup.ts",
  "scripts/src/remote.ts",
  "scripts/host-codex-commander-contract-test.ts",
@@ -42,6 +44,7 @@ const syntaxFiles = [
  "scripts/code-queue-submit-summary-contract-test.ts",
  "scripts/code-queue-cli-read-terminal-contract-test.ts",
  "scripts/code-queue-gh-auth-redaction-contract-test.ts",
+  "scripts/d601-recovery-guardrails-contract-test.ts",
  "scripts/microservice-health-output-contract-test.ts",
  "scripts/code-queue-supervisor-disclosure-contract-test.ts",
  "scripts/code-queue-commander-view-contract-test.ts",
@@ -70,6 +73,7 @@ export interface CheckOptions {
  components: boolean;
  compose: boolean;
  logs: boolean;
+  recoveryGuardrails: boolean;
  rust: boolean;
 }

@@ -80,6 +84,7 @@ const defaultCheckOptions: CheckOptions = {
  components: false,
  compose: false,
  logs: false,
+  recoveryGuardrails: false,
  rust: false,
 };

@@ -87,7 +92,10 @@ export function checkHelp(): Record<string, unknown> {
  return {
    ok: true,
    command: "check",
-    usage: "bun scripts/cli.ts check [--syntax-only|--full|--files|--scripts-typecheck|--components|--compose|--logs|--rust]",
+    usage: [
+      "bun scripts/cli.ts check [--syntax-only|--full|--files|--scripts-typecheck|--components|--compose|--logs|--recovery-guardrails|--rust]",
+      "bun scripts/cli.ts check recovery-guardrails",
+    ],
    defaultMode: "syntax/config only; Rust is never compiled on the master server by default",
    options: [
      { name: "--syntax-only|--basic", description: "Run only config validation, Bun version and TypeScript syntax transpile." },
@@ -97,12 +105,18 @@ export function checkHelp(): Record<string, unknown> {
      { name: "--components", description: "Run component TypeScript typecheck." },
      { name: "--compose", description: "Render Docker Compose config." },
      { name: "--logs", description: "Check unified log rotation policy." },
+      { name: "--recovery-guardrails", description: "Run D601 k3s/Code Queue reboot recovery diagnostics in read-only mode." },
      { name: "--rust", description: "Run cargo check only when UNIDESK_D601_RUST_CHECK=1 is set inside D601 CI/dev execution." },
    ],
    rustBoundary: {
      masterServer: "do not run cargo check/build here",
      d601: "use deploy apply --env dev --service backend-core and CI with UNIDESK_D601_RUST_CHECK=1",
    },
+    recoveryGuardrailsBoundary: {
+      command: "bun scripts/cli.ts check recovery-guardrails",
+      mutation: false,
+      forbidden: ["restart k3s", "delete CRI sandboxes or pods", "modify hostPath directories", "deploy/rollout", "destructive prune/reset"],
+    },
  };
 }

@@ -116,6 +130,7 @@ export function parseCheckOptions(args: string[]): CheckOptions {
      options.components = true;
      options.compose = true;
      options.logs = true;
+      options.recoveryGuardrails = true;
    } else if (arg === "--files") {
      options.files = true;
    } else if (arg === "--scripts-typecheck") {
@@ -126,6 +141,8 @@ export function parseCheckOptions(args: string[]): CheckOptions {
      options.compose = true;
    } else if (arg === "--logs") {
      options.logs = true;
+    } else if (arg === "--recovery-guardrails") {
+      options.recoveryGuardrails = true;
    } else if (arg === "--rust") {
      options.rust = true;
    } else if (arg === "--basic" || arg === "--syntax-only") {
@@ -277,6 +294,10 @@ function skippedItem(name: string, reason: string, enableWith: string): CheckIte
  return { name, ok: true, detail: { skipped: true, reason, enableWith } };
 }

+export function runRecoveryGuardrailsCheck(config: UniDeskConfig): ReturnType<typeof compactD601RecoveryGuardrails> {
+  return compactD601RecoveryGuardrails(runD601RecoveryGuardrails(config));
+}
+
 export function runChecks(config: UniDeskConfig, options: CheckOptions = defaultCheckOptions): { ok: boolean; mode: string; options: CheckOptions; items: CheckItem[] } {
  const items: CheckItem[] = [
    { name: "config:validated", ok: true, detail: { project: config.project.name, runtime: config.runtime } },
@@ -352,8 +373,10 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
      fileItem("scripts/server-cleanup-plan-contract-test.ts"),
      fileItem("scripts/src/artifact-registry.ts"),
      fileItem("scripts/src/server-cleanup.ts"),
+      fileItem("scripts/src/recovery-guardrails.ts"),
      fileItem("scripts/src/auth-broker.ts"),
      fileItem("scripts/auth-broker-contract-test.ts"),
+      fileItem("scripts/d601-recovery-guardrails-contract-test.ts"),
      fileItem("src/components/microservices/auth-broker/Cargo.toml"),
      fileItem("src/components/microservices/auth-broker/Dockerfile"),
      fileItem("src/components/microservices/auth-broker/src/main.rs"),
@@ -403,6 +426,7 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
    items.push(commandItem("gh:pr-contract", ["bun", "scripts/gh-cli-pr-contract-test.ts"], 30_000));
    items.push(commandItem("playwright:cli-wrapper-contract", ["bun", "scripts/playwright-cli-contract-test.ts"], 30_000));
    items.push(commandItem("auth-broker:p0-contract", ["bun", "scripts/auth-broker-contract-test.ts"], 30_000));
+    items.push(commandItem("d601:recovery-guardrails-contract", ["bun", "scripts/d601-recovery-guardrails-contract-test.ts"], 30_000));
  } else {
    items.push(skippedItem("typescript:scripts", "scripts TypeScript typecheck is opt-in", "--scripts-typecheck or --full"));
    items.push(skippedItem("code-queue:prompt-observation-contract", "prompt observation contract is opt-in with script checks", "--scripts-typecheck or --full"));
@@ -439,12 +463,23 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
    items.push(skippedItem("gh:pr-contract", "GitHub PR CLI contract is opt-in with script checks", "--scripts-typecheck or --full"));
    items.push(skippedItem("playwright:cli-wrapper-contract", "Playwright wrapper/headless/session contract is opt-in with script checks", "--scripts-typecheck or --full"));
    items.push(skippedItem("auth-broker:p0-contract", "Auth Broker P0 skeleton and CLI adapter contract is opt-in with script checks", "--scripts-typecheck or --full"));
+    items.push(skippedItem("d601:recovery-guardrails-contract", "D601 recovery guardrails fixture contract is opt-in with script checks", "--scripts-typecheck or --full"));
  }
  if (options.logs) {
    items.push(unifiedLogRotationItem());
  } else {
    items.push(skippedItem("logs:unified-hourly-rotation", "policy scan is opt-in", "--logs or --full"));
  }
+  if (options.recoveryGuardrails) {
+    const recovery = runRecoveryGuardrailsCheck(config);
+    items.push({
+      name: "d601:recovery-guardrails",
+      ok: recovery.ok,
+      detail: recovery,
+    });
+  } else {
+    items.push(skippedItem("d601:recovery-guardrails", "D601 reboot recovery diagnostics are opt-in and read-only", "--recovery-guardrails or --full"));
+  }
  if (options.components) {
    items.push(commandItem("typescript:components", ["bunx", "tsc", "-p", "src/tsconfig.check.json", "--pretty", "false"], 180_000));
  } else {
@@ -9,7 +9,7 @@ export function rootHelp(): unknown {
      { command: "help", description: "List supported commands." },
      { command: "--main-server-ip <ip> <command>", description: "Run selected commands through the public frontend API; use --main-server-key only for legacy SSH transport." },
      { command: "config show", description: "Validate and print config.json as the single source of truth." },
-      { command: "check [--full|--files|--scripts-typecheck|--components|--compose|--logs|--rust]", description: "Run the lightweight default syntax/config gate; Rust is opt-in and only allowed from D601 CI/dev execution." },
+      { command: "check [--full|--files|--scripts-typecheck|--components|--compose|--logs|--recovery-guardrails|--rust] | check recovery-guardrails", description: "Run the lightweight default syntax/config gate or the low-noise read-only D601 recovery guardrails; Rust is opt-in and only allowed from D601 CI/dev execution." },
      { command: "server start", description: "Fire-and-forget build/start for database, backend-core, frontend, provider gateway, and managed main-server user services." },
      { command: "server stop", description: "Fire-and-forget docker-compose down for the fixed UniDesk stack." },
      { command: "server status", description: "Show fixed ports, containers, service health, and public URLs." },