fix: schedule web sentinel cadence in k3s

config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml

@@ -17,6 +17,7 @@ sentinel:
       checkoutPaths:
         - scripts
         - config
+        - config.json
         - package.json
         - bun.lock
         - bun.lockb

config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml

@@ -17,6 +17,7 @@ sentinel:
       checkoutPaths:
         - scripts
         - config
+        - config.json
         - package.json
         - bun.lock
         - bun.lockb

config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml

@@ -17,6 +17,7 @@ sentinel:
       checkoutPaths:
         - scripts
         - config
+        - config.json
         - package.json
         - bun.lock
         - bun.lockb

config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml

@@ -17,6 +17,7 @@ sentinel:
       checkoutPaths:
         - scripts
         - config
+        - config.json
         - package.json
         - bun.lock
         - bun.lockb

scripts/src/hwlab-node-web-sentinel-cicd.ts

@@ -119,6 +119,7 @@ interface SentinelCicdState {
   readonly configReady: boolean;
   readonly runtime: Record<string, unknown>;
   readonly cicd: Record<string, unknown>;
+  readonly scenarios: unknown;
   readonly publicExposure: Record<string, unknown>;
   readonly secrets: Record<string, unknown>;
   readonly controlPlaneTarget: Record<string, unknown>;
@@ -308,6 +309,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
   const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
   const runtime = recordTarget(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime);
   const cicd = recordTarget(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd);
+  const scenarios = readConfigRefTarget(sentinel.configRefs.scenarios);
   const publicExposure = recordTarget(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
   const secrets = recordTarget(readConfigRefTarget(sentinel.configRefs.secrets), sentinel.configRefs.secrets);
   const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef");
@@ -317,7 +319,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
   const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
   const sourceHead = resolveSourceHead(cicd, timeoutSeconds);
   const image = sentinelImagePlan(cicd, sourceHead);
-  const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, publicExposure, secrets, image);
+  const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image);
   const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
   return {
     spec,
@@ -326,6 +328,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
     configReady: configPlan.ok,
     runtime,
     cicd,
+    scenarios,
     publicExposure,
     secrets,
     controlPlaneTarget,
@@ -382,6 +385,7 @@ function sentinelDockerfile(baseImage: string, entrypoint: string): string {
     "WORKDIR /app",
     "COPY . /app",
     "RUN if [ -d /opt/hwlab-ci-node-deps/node_modules ]; then mkdir -p /app/node_modules; for dep in /opt/hwlab-ci-node-deps/node_modules/*; do ln -sf \"$dep\" \"/app/node_modules/$(basename \"$dep\")\"; done; fi",
+    "RUN printf '%s\\n' '#!/bin/sh' 'exec bun /app/scripts/ssh-cli.ts \"$@\"' > /usr/local/bin/trans && chmod 0755 /usr/local/bin/trans",
     "RUN bun scripts/verify-web-probe-sentinel-monitor-web.ts",
     "ENV NODE_ENV=production",
     `ENTRYPOINT ["bun", "${entrypoint}"]`,
@@ -409,6 +413,7 @@ function renderSentinelManifests(
   sentinelId: string,
   runtime: Record<string, unknown>,
   cicd: Record<string, unknown>,
+  scenarios: unknown,
   publicExposure: Record<string, unknown>,
   secrets: Record<string, unknown>,
   image: SentinelImagePlan,
@@ -429,6 +434,7 @@ function renderSentinelManifests(
   const pvcStorage = stringAt(runtime, "pvcStorage");
   const stateRoot = stringAt(runtime, "stateRoot");
   const sentinelEnv = sentinelContainerEnv(sentinelId, secrets);
+  const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv);
   return [
     {
       apiVersion: "v1",
@@ -504,6 +510,7 @@ function renderSentinelManifests(
       metadata: { name: serviceName, namespace, labels },
       spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
     },
+    ...(cadenceJob === null ? [] : [cadenceJob]),
     {
       apiVersion: "apps/v1",
       kind: "Deployment",
@@ -564,20 +571,166 @@ function renderSentinelManifests(
 
 function sentinelContainerEnv(sentinelId: string, secrets: Record<string, unknown>): readonly Record<string, unknown>[] {
   const env: Record<string, unknown>[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }];
+  const sourcesByPurpose = new Map<string, Record<string, unknown>>();
+  for (const source of arrayAt(secrets, "sources").map(record)) {
+    const purpose = stringAtNullable(source, "purpose");
+    if (purpose !== null) sourcesByPurpose.set(purpose, source);
+  }
+  const used = new Set(env.map((item) => String(item.name ?? "")));
+  const pushEnv = (item: Record<string, unknown>): void => {
+    const name = String(item.name ?? "");
+    if (name.length === 0 || used.has(name)) return;
+    used.add(name);
+    env.push(item);
+  };
   for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) {
     const secretName = stringAtNullable(runtimeSecret, "name");
     if (secretName === null) continue;
     for (const item of arrayAt(runtimeSecret, "data").map(record)) {
       const targetKey = stringAtNullable(item, "targetKey");
       const sourcePurpose = stringAtNullable(item, "sourcePurpose");
+      const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey");
+      if (targetKey !== null && sourceKey !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKey)) {
+        pushEnv({ name: sourceKey, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
+      }
       const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
       if (envName === null) continue;
-      env.push({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
+      pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
     }
   }
   return env;
 }
 
+function sentinelCadenceCronJobPlan(
+  spec: HwlabRuntimeLaneSpec,
+  sentinelId: string,
+  runtime: Record<string, unknown>,
+  cicd: Record<string, unknown>,
+  scenarios: unknown,
+  imageRef: string,
+  sentinelEnv: readonly Record<string, unknown>[],
+): Record<string, unknown> | null {
+  const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId");
+  if (scenarioId === null) return null;
+  const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null;
+  if (scenario === null) return null;
+  const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null;
+  const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds);
+  if (schedule === null) return null;
+  const namespace = stringAt(runtime, "namespace");
+  const deploymentName = stringAt(runtime, "deploymentName");
+  const serviceAccountName = stringAt(runtime, "serviceAccountName");
+  const timeoutSeconds = numberAtNullable(cicd, "targetValidation.maxSeconds") ?? numberAtNullable(scenario, "maxRunSeconds") ?? 300;
+  const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost");
+  const name = safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
+  const labels = {
+    "app.kubernetes.io/name": name,
+    "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
+    "app.kubernetes.io/component": "cadence-scheduler",
+    "app.kubernetes.io/managed-by": "unidesk",
+    "unidesk.ai/spec-ref": "PJ2026-01060508",
+    "unidesk.ai/node": spec.nodeId,
+    "unidesk.ai/lane": spec.lane,
+    "unidesk.ai/web-probe-sentinel-id": sentinelId,
+  };
+  return {
+    apiVersion: "batch/v1",
+    kind: "CronJob",
+    metadata: {
+      name,
+      namespace,
+      labels,
+      annotations: {
+        "unidesk.ai/cadence": String(scenario.cadence),
+        "unidesk.ai/target-validation-max-seconds": String(timeoutSeconds),
+      },
+    },
+    spec: {
+      schedule,
+      concurrencyPolicy: "Forbid",
+      successfulJobsHistoryLimit: 3,
+      failedJobsHistoryLimit: 5,
+      startingDeadlineSeconds: Math.max(60, cadenceSeconds),
+      jobTemplate: {
+        spec: {
+          activeDeadlineSeconds: timeoutSeconds + 60,
+          ttlSecondsAfterFinished: 86400,
+          backoffLimit: 0,
+          template: {
+            metadata: { labels },
+            spec: {
+              restartPolicy: "Never",
+              serviceAccountName,
+              containers: [{
+                name: "quick-verify",
+                image: imageRef,
+                imagePullPolicy: "IfNotPresent",
+                command: ["bun", "scripts/cli.ts"],
+                args: [
+                  "web-probe",
+                  "sentinel",
+                  "validate",
+                  "--node",
+                  spec.nodeId,
+                  "--lane",
+                  spec.lane,
+                  "--sentinel",
+                  sentinelId,
+                  "--quick-verify",
+                  "--confirm",
+                  "--wait",
+                  "--timeout-seconds",
+                  String(timeoutSeconds),
+                ],
+                env: [
+                  ...sentinelEnv,
+                  { name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" },
+                  ...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]),
+                ],
+              }],
+            },
+          },
+        },
+      },
+    },
+  };
+}
+
+function scenarioRows(value: unknown): Record<string, unknown>[] {
+  if (Array.isArray(value)) return value.map(record);
+  if (!isRecord(value)) return [];
+  if (Array.isArray(value.scenarios)) return value.scenarios.map(record);
+  if (isRecord(value.workflow)) return [value.workflow];
+  return [value];
+}
+
+function parseDurationSeconds(value: string): number | null {
+  const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
+  if (match === null) return null;
+  const amount = Number(match[1]);
+  const unit = match[2];
+  if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000));
+  if (unit === "s") return Math.max(60, amount);
+  if (unit === "m") return amount * 60;
+  if (unit === "h") return amount * 3600;
+  return null;
+}
+
+function cronScheduleForCadenceSeconds(seconds: number): string | null {
+  if (!Number.isFinite(seconds) || seconds <= 0) return null;
+  if (seconds <= 60) return "* * * * *";
+  if (seconds % 60 === 0) {
+    const minutes = Math.trunc(seconds / 60);
+    if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`;
+    if (minutes % 60 === 0) {
+      const hours = Math.trunc(minutes / 60);
+      if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`;
+      if (hours === 24) return "0 0 * * *";
+    }
+  }
+  return null;
+}
+
 function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null {
   if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null;
   const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, "");
@@ -601,7 +754,21 @@ function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: No
       const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
       if (envName === null || sourcePurpose === null || targetKey === null) continue;
       const source = sourcesByPurpose.get(sourcePurpose);
+      const runtimeValue = process.env[envName];
       if (source === undefined) {
+        if (runtimeValue !== undefined && runtimeValue.length > 0) {
+          env[envName] = runtimeValue;
+          items.push({
+            envName,
+            secretName,
+            targetKey,
+            sourcePurpose,
+            sourceMode: "runtime-env",
+            fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
+            valuesRedacted: true,
+          });
+          continue;
+        }
         missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
         continue;
       }
@@ -609,6 +776,21 @@ function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: No
       const sourceKey = stringAt(source, "sourceKey");
       const material = readSentinelSecretSourceValue(source);
       if (!material.ok) {
+        if (runtimeValue !== undefined && runtimeValue.length > 0) {
+          env[envName] = runtimeValue;
+          items.push({
+            envName,
+            secretName,
+            targetKey,
+            sourcePurpose,
+            sourceRef,
+            sourceKey,
+            sourceMode: "runtime-env",
+            fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
+            valuesRedacted: true,
+          });
+          continue;
+        }
         missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
         continue;
       }
@@ -2921,6 +3103,9 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p
   const servicePort = numberAt(state.runtime, "servicePort");
   const deploymentName = stringAt(state.runtime, "deploymentName");
   const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`;
+  if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") {
+    return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url);
+  }
   const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
   const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
   const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
@@ -2972,6 +3157,52 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p
   };
 }
 
+function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number, url: string): Record<string, unknown> {
+  const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
+  const fetchScript = [
+    "const method = process.env.SENTINEL_METHOD || 'GET';",
+    "const url = process.env.SENTINEL_URL || '';",
+    "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
+    "const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;",
+    "fetch(url, { method, headers, body: method === 'POST' ? body : undefined }).then(async (response) => {",
+    "  const text = await response.text();",
+    "  process.stdout.write(text);",
+    "  if (!response.ok) process.exit(22);",
+    "}).catch((error) => {",
+    "  console.error(error && error.stack ? error.stack : String(error));",
+    "  process.exit(23);",
+    "});",
+  ].join(" ");
+  const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
+  const result = runCommand(["node", "-e", fetchScript], repoRoot, {
+    timeoutMs: attemptTimeoutSeconds * 1000,
+    env: {
+      ...process.env,
+      SENTINEL_METHOD: method,
+      SENTINEL_URL: url,
+      SENTINEL_BODY_B64: bodyB64,
+    },
+  });
+  const parsed = parseJsonObject(result.stdout);
+  const compactBodyJson = compactSentinelServiceBodyJson(parsed);
+  return {
+    ok: result.exitCode === 0,
+    method,
+    path: pathWithQuery,
+    internalUrl: url,
+    httpStatus: result.exitCode === 0 ? 200 : null,
+    bodyJson: record(compactBodyJson),
+    bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "",
+    bodyBytes: Buffer.byteLength(result.stdout),
+    error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000),
+    proxyPath: null,
+    result: compactCommand(result),
+    attempts: [{ attempt: 1, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }],
+    transport: "direct-service",
+    valuesRedacted: true,
+  };
+}
+
 function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
   if (value === null || typeof value.renderedText !== "string") return value;
   return {
@@ -3890,9 +4121,11 @@ function readPromptSetForScenario(scenario: Record<string, unknown>): { ok: true
   const paths = secretSourcePaths(sourceRef);
   const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
   const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true };
-  if (!existsSync(sourcePath)) return { ok: false, error: "prompt-source-missing", summary };
-  const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
-  const raw = values[key];
+  const runtimeRaw = process.env[key];
+  const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
+  const raw = values[key] ?? runtimeRaw;
+  const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null;
+  if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary };
   if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary };
   const parsed = parsePromptJson(raw);
   if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary };
@@ -3901,6 +4134,7 @@ function readPromptSetForScenario(scenario: Record<string, unknown>): { ok: true
     prompts: parsed,
     summary: {
       ...summary,
+      sourceMode,
       promptCount: parsed.length,
       promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))),
       promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`),

scripts/src/hwlab-node/web-probe.ts

@@ -1256,11 +1256,12 @@ export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): Boo
   }
   const paths = secretSourcePaths(sourceRef);
   const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
-  if (!existsSync(sourcePath)) {
+  const runtimePassword = process.env[sourceKey];
+  if (!existsSync(sourcePath) && (runtimePassword === undefined || runtimePassword.length === 0)) {
     return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" };
   }
-  const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
-  const password = values[sourceKey];
+  const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
+  const password = values[sourceKey] ?? runtimePassword;
   if (password === undefined || password.length === 0) {
     return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" };
   }