From 6ebdec151b11c4e5d2c61e0a009ff7f049467a1e Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 28 Jun 2026 01:18:15 +0000 Subject: [PATCH] fix: schedule web sentinel cadence in k3s --- .../cicd.auth-session-switch.d601-v03.yaml | 1 + .../cicd.d518-v03.yaml | 1 + .../cicd.d601-v03.yaml | 1 + .../cicd.mdtodo.d601-v03.yaml | 1 + scripts/src/hwlab-node-web-sentinel-cicd.ts | 244 +++++++++++++++++- scripts/src/hwlab-node/web-probe.ts | 7 +- 6 files changed, 247 insertions(+), 8 deletions(-) diff --git a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml index 815b962e..a35e216c 100644 --- a/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.auth-session-switch.d601-v03.yaml @@ -17,6 +17,7 @@ sentinel: checkoutPaths: - scripts - config + - config.json - package.json - bun.lock - bun.lockb diff --git a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml index 05f65424..832e9153 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d518-v03.yaml @@ -17,6 +17,7 @@ sentinel: checkoutPaths: - scripts - config + - config.json - package.json - bun.lock - bun.lockb diff --git a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml index dc7b701b..ddb4d074 100644 --- a/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml @@ -17,6 +17,7 @@ sentinel: checkoutPaths: - scripts - config + - config.json - package.json - bun.lock - bun.lockb diff --git a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml index 7e8ab881..a340b359 100644 --- a/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml +++ b/config/hwlab-web-probe-sentinel/cicd.mdtodo.d601-v03.yaml @@ -17,6 +17,7 @@ sentinel: checkoutPaths: - scripts - config + - config.json - package.json - bun.lock - bun.lockb diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 2c54d788..515a234c 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -119,6 +119,7 @@ interface SentinelCicdState { readonly configReady: boolean; readonly runtime: Record; readonly cicd: Record; + readonly scenarios: unknown; readonly publicExposure: Record; readonly secrets: Record; readonly controlPlaneTarget: Record; @@ -308,6 +309,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); const runtime = recordTarget(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime); const cicd = recordTarget(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd); + const scenarios = readConfigRefTarget(sentinel.configRefs.scenarios); const publicExposure = recordTarget(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure); const secrets = recordTarget(readConfigRefTarget(sentinel.configRefs.secrets), sentinel.configRefs.secrets); const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef"); @@ -317,7 +319,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`); const sourceHead = resolveSourceHead(cicd, timeoutSeconds); const image = sentinelImagePlan(cicd, sourceHead); - const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, publicExposure, secrets, image); + const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image); const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; return { spec, @@ -326,6 +328,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string | configReady: configPlan.ok, runtime, cicd, + scenarios, publicExposure, secrets, controlPlaneTarget, @@ -382,6 +385,7 @@ function sentinelDockerfile(baseImage: string, entrypoint: string): string { "WORKDIR /app", "COPY . /app", "RUN if [ -d /opt/hwlab-ci-node-deps/node_modules ]; then mkdir -p /app/node_modules; for dep in /opt/hwlab-ci-node-deps/node_modules/*; do ln -sf \"$dep\" \"/app/node_modules/$(basename \"$dep\")\"; done; fi", + "RUN printf '%s\\n' '#!/bin/sh' 'exec bun /app/scripts/ssh-cli.ts \"$@\"' > /usr/local/bin/trans && chmod 0755 /usr/local/bin/trans", "RUN bun scripts/verify-web-probe-sentinel-monitor-web.ts", "ENV NODE_ENV=production", `ENTRYPOINT ["bun", "${entrypoint}"]`, @@ -409,6 +413,7 @@ function renderSentinelManifests( sentinelId: string, runtime: Record, cicd: Record, + scenarios: unknown, publicExposure: Record, secrets: Record, image: SentinelImagePlan, @@ -429,6 +434,7 @@ function renderSentinelManifests( const pvcStorage = stringAt(runtime, "pvcStorage"); const stateRoot = stringAt(runtime, "stateRoot"); const sentinelEnv = sentinelContainerEnv(sentinelId, secrets); + const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv); return [ { apiVersion: "v1", @@ -504,6 +510,7 @@ function renderSentinelManifests( metadata: { name: serviceName, namespace, labels }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] }, }, + ...(cadenceJob === null ? [] : [cadenceJob]), { apiVersion: "apps/v1", kind: "Deployment", @@ -564,20 +571,166 @@ function renderSentinelManifests( function sentinelContainerEnv(sentinelId: string, secrets: Record): readonly Record[] { const env: Record[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }]; + const sourcesByPurpose = new Map>(); + for (const source of arrayAt(secrets, "sources").map(record)) { + const purpose = stringAtNullable(source, "purpose"); + if (purpose !== null) sourcesByPurpose.set(purpose, source); + } + const used = new Set(env.map((item) => String(item.name ?? ""))); + const pushEnv = (item: Record): void => { + const name = String(item.name ?? ""); + if (name.length === 0 || used.has(name)) return; + used.add(name); + env.push(item); + }; for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) { const secretName = stringAtNullable(runtimeSecret, "name"); if (secretName === null) continue; for (const item of arrayAt(runtimeSecret, "data").map(record)) { const targetKey = stringAtNullable(item, "targetKey"); const sourcePurpose = stringAtNullable(item, "sourcePurpose"); + const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey"); + if (targetKey !== null && sourceKey !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKey)) { + pushEnv({ name: sourceKey, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); + } const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); if (envName === null) continue; - env.push({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); + pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } }); } } return env; } +function sentinelCadenceCronJobPlan( + spec: HwlabRuntimeLaneSpec, + sentinelId: string, + runtime: Record, + cicd: Record, + scenarios: unknown, + imageRef: string, + sentinelEnv: readonly Record[], +): Record | null { + const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId"); + if (scenarioId === null) return null; + const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null; + if (scenario === null) return null; + const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null; + const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds); + if (schedule === null) return null; + const namespace = stringAt(runtime, "namespace"); + const deploymentName = stringAt(runtime, "deploymentName"); + const serviceAccountName = stringAt(runtime, "serviceAccountName"); + const timeoutSeconds = numberAtNullable(cicd, "targetValidation.maxSeconds") ?? numberAtNullable(scenario, "maxRunSeconds") ?? 300; + const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost"); + const name = safeKubernetesSegment(`${deploymentName}-quick-verify`, 52); + const labels = { + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "hwlab-web-probe-sentinel", + "app.kubernetes.io/component": "cadence-scheduler", + "app.kubernetes.io/managed-by": "unidesk", + "unidesk.ai/spec-ref": "PJ2026-01060508", + "unidesk.ai/node": spec.nodeId, + "unidesk.ai/lane": spec.lane, + "unidesk.ai/web-probe-sentinel-id": sentinelId, + }; + return { + apiVersion: "batch/v1", + kind: "CronJob", + metadata: { + name, + namespace, + labels, + annotations: { + "unidesk.ai/cadence": String(scenario.cadence), + "unidesk.ai/target-validation-max-seconds": String(timeoutSeconds), + }, + }, + spec: { + schedule, + concurrencyPolicy: "Forbid", + successfulJobsHistoryLimit: 3, + failedJobsHistoryLimit: 5, + startingDeadlineSeconds: Math.max(60, cadenceSeconds), + jobTemplate: { + spec: { + activeDeadlineSeconds: timeoutSeconds + 60, + ttlSecondsAfterFinished: 86400, + backoffLimit: 0, + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + serviceAccountName, + containers: [{ + name: "quick-verify", + image: imageRef, + imagePullPolicy: "IfNotPresent", + command: ["bun", "scripts/cli.ts"], + args: [ + "web-probe", + "sentinel", + "validate", + "--node", + spec.nodeId, + "--lane", + spec.lane, + "--sentinel", + sentinelId, + "--quick-verify", + "--confirm", + "--wait", + "--timeout-seconds", + String(timeoutSeconds), + ], + env: [ + ...sentinelEnv, + { name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" }, + ...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]), + ], + }], + }, + }, + }, + }, + }, + }; +} + +function scenarioRows(value: unknown): Record[] { + if (Array.isArray(value)) return value.map(record); + if (!isRecord(value)) return []; + if (Array.isArray(value.scenarios)) return value.scenarios.map(record); + if (isRecord(value.workflow)) return [value.workflow]; + return [value]; +} + +function parseDurationSeconds(value: string): number | null { + const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim()); + if (match === null) return null; + const amount = Number(match[1]); + const unit = match[2]; + if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000)); + if (unit === "s") return Math.max(60, amount); + if (unit === "m") return amount * 60; + if (unit === "h") return amount * 3600; + return null; +} + +function cronScheduleForCadenceSeconds(seconds: number): string | null { + if (!Number.isFinite(seconds) || seconds <= 0) return null; + if (seconds <= 60) return "* * * * *"; + if (seconds % 60 === 0) { + const minutes = Math.trunc(seconds / 60); + if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`; + if (minutes % 60 === 0) { + const hours = Math.trunc(minutes / 60); + if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`; + if (hours === 24) return "0 0 * * *"; + } + } + return null; +} + function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null { if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null; const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, ""); @@ -601,7 +754,21 @@ function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: No const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey); if (envName === null || sourcePurpose === null || targetKey === null) continue; const source = sourcesByPurpose.get(sourcePurpose); + const runtimeValue = process.env[envName]; if (source === undefined) { + if (runtimeValue !== undefined && runtimeValue.length > 0) { + env[envName] = runtimeValue; + items.push({ + envName, + secretName, + targetKey, + sourcePurpose, + sourceMode: "runtime-env", + fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`, + valuesRedacted: true, + }); + continue; + } missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true }); continue; } @@ -609,6 +776,21 @@ function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: No const sourceKey = stringAt(source, "sourceKey"); const material = readSentinelSecretSourceValue(source); if (!material.ok) { + if (runtimeValue !== undefined && runtimeValue.length > 0) { + env[envName] = runtimeValue; + items.push({ + envName, + secretName, + targetKey, + sourcePurpose, + sourceRef, + sourceKey, + sourceMode: "runtime-env", + fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`, + valuesRedacted: true, + }); + continue; + } missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true }); continue; } @@ -2921,6 +3103,9 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p const servicePort = numberAt(state.runtime, "servicePort"); const deploymentName = stringAt(state.runtime, "deploymentName"); const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`; + if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") { + return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url); + } const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`; const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64"); @@ -2972,6 +3157,52 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p }; } +function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record | null, timeoutSeconds: number, url: string): Record { + const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64"); + const fetchScript = [ + "const method = process.env.SENTINEL_METHOD || 'GET';", + "const url = process.env.SENTINEL_URL || '';", + "const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');", + "const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;", + "fetch(url, { method, headers, body: method === 'POST' ? body : undefined }).then(async (response) => {", + " const text = await response.text();", + " process.stdout.write(text);", + " if (!response.ok) process.exit(22);", + "}).catch((error) => {", + " console.error(error && error.stack ? error.stack : String(error));", + " process.exit(23);", + "});", + ].join(" "); + const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60)); + const result = runCommand(["node", "-e", fetchScript], repoRoot, { + timeoutMs: attemptTimeoutSeconds * 1000, + env: { + ...process.env, + SENTINEL_METHOD: method, + SENTINEL_URL: url, + SENTINEL_BODY_B64: bodyB64, + }, + }); + const parsed = parseJsonObject(result.stdout); + const compactBodyJson = compactSentinelServiceBodyJson(parsed); + return { + ok: result.exitCode === 0, + method, + path: pathWithQuery, + internalUrl: url, + httpStatus: result.exitCode === 0 ? 200 : null, + bodyJson: record(compactBodyJson), + bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "", + bodyBytes: Buffer.byteLength(result.stdout), + error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000), + proxyPath: null, + result: compactCommand(result), + attempts: [{ attempt: 1, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }], + transport: "direct-service", + valuesRedacted: true, + }; +} + function compactSentinelServiceBodyJson(value: Record | null): unknown { if (value === null || typeof value.renderedText !== "string") return value; return { @@ -3890,9 +4121,11 @@ function readPromptSetForScenario(scenario: Record): { ok: true const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true }; - if (!existsSync(sourcePath)) return { ok: false, error: "prompt-source-missing", summary }; - const values = parseEnvFile(readFileSync(sourcePath, "utf8")); - const raw = values[key]; + const runtimeRaw = process.env[key]; + const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; + const raw = values[key] ?? runtimeRaw; + const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null; + if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary }; if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary }; const parsed = parsePromptJson(raw); if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary }; @@ -3901,6 +4134,7 @@ function readPromptSetForScenario(scenario: Record): { ok: true prompts: parsed, summary: { ...summary, + sourceMode, promptCount: parsed.length, promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))), promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`), diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index e18632f2..9f29397d 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -1256,11 +1256,12 @@ export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): Boo } const paths = secretSourcePaths(sourceRef); const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef); - if (!existsSync(sourcePath)) { + const runtimePassword = process.env[sourceKey]; + if (!existsSync(sourcePath) && (runtimePassword === undefined || runtimePassword.length === 0)) { return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" }; } - const values = parseEnvFile(readFileSync(sourcePath, "utf8")); - const password = values[sourceKey]; + const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; + const password = values[sourceKey] ?? runtimePassword; if (password === undefined || password.length === 0) { return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" }; }