mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: schedule web sentinel cadence in k3s

Browse Source
This commit is contained in:
Codex
2026-06-28 01:18:15 +00:00
parent 0b926a3332
commit 6ebdec151b
6 changed files with 247 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-web-sentinel-cicd.ts
+239 -5
View File
@@ -119,6 +119,7 @@ interface SentinelCicdState {
readonly configReady: boolean;
readonly runtime: Record<string, unknown>;
readonly cicd: Record<string, unknown>;
readonly scenarios: unknown;
readonly publicExposure: Record<string, unknown>;
readonly secrets: Record<string, unknown>;
readonly controlPlaneTarget: Record<string, unknown>;
@@ -308,6 +309,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
const configPlan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
const runtime = recordTarget(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime);
const cicd = recordTarget(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd);
const scenarios = readConfigRefTarget(sentinel.configRefs.scenarios);
const publicExposure = recordTarget(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
const secrets = recordTarget(readConfigRefTarget(sentinel.configRefs.secrets), sentinel.configRefs.secrets);
const controlPlaneRef = stringField(cicd, "controlPlaneConfigRef");
@@ -317,7 +319,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
const controlPlaneNode = recordTarget(valueAtPath(controlPlaneConfig, `nodes.${nodeId}`), `${configRefFile(controlPlaneRef)}#nodes.${nodeId}`);
const sourceHead = resolveSourceHead(cicd, timeoutSeconds);
const image = sentinelImagePlan(cicd, sourceHead);
const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, publicExposure, secrets, image);
const manifests = renderSentinelManifests(spec, sentinel.id, runtime, cicd, scenarios, publicExposure, secrets, image);
const manifestYaml = `${manifests.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`;
return {
spec,
@@ -326,6 +328,7 @@ function loadSentinelCicdState(spec: HwlabRuntimeLaneSpec, sentinelId: string |
configReady: configPlan.ok,
runtime,
cicd,
scenarios,
publicExposure,
secrets,
controlPlaneTarget,
@@ -382,6 +385,7 @@ function sentinelDockerfile(baseImage: string, entrypoint: string): string {
"WORKDIR /app",
"COPY . /app",
"RUN if [ -d /opt/hwlab-ci-node-deps/node_modules ]; then mkdir -p /app/node_modules; for dep in /opt/hwlab-ci-node-deps/node_modules/*; do ln -sf \"$dep\" \"/app/node_modules/$(basename \"$dep\")\"; done; fi",
"RUN printf '%s\\n' '#!/bin/sh' 'exec bun /app/scripts/ssh-cli.ts \"$@\"' > /usr/local/bin/trans && chmod 0755 /usr/local/bin/trans",
"RUN bun scripts/verify-web-probe-sentinel-monitor-web.ts",
"ENV NODE_ENV=production",
`ENTRYPOINT ["bun", "${entrypoint}"]`,
@@ -409,6 +413,7 @@ function renderSentinelManifests(
sentinelId: string,
runtime: Record<string, unknown>,
cicd: Record<string, unknown>,
scenarios: unknown,
publicExposure: Record<string, unknown>,
secrets: Record<string, unknown>,
image: SentinelImagePlan,
@@ -429,6 +434,7 @@ function renderSentinelManifests(
const pvcStorage = stringAt(runtime, "pvcStorage");
const stateRoot = stringAt(runtime, "stateRoot");
const sentinelEnv = sentinelContainerEnv(sentinelId, secrets);
const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv);
return [
{
apiVersion: "v1",
@@ -504,6 +510,7 @@ function renderSentinelManifests(
metadata: { name: serviceName, namespace, labels },
spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": deploymentName }, ports: [{ name: "http", port: servicePort, targetPort: "http" }] },
},
...(cadenceJob === null ? [] : [cadenceJob]),
{
apiVersion: "apps/v1",
kind: "Deployment",
@@ -564,20 +571,166 @@ function renderSentinelManifests(
function sentinelContainerEnv(sentinelId: string, secrets: Record<string, unknown>): readonly Record<string, unknown>[] {
const env: Record<string, unknown>[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }];
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
for (const source of arrayAt(secrets, "sources").map(record)) {
const purpose = stringAtNullable(source, "purpose");
if (purpose !== null) sourcesByPurpose.set(purpose, source);
}
const used = new Set(env.map((item) => String(item.name ?? "")));
const pushEnv = (item: Record<string, unknown>): void => {
const name = String(item.name ?? "");
if (name.length === 0 || used.has(name)) return;
used.add(name);
env.push(item);
};
for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) {
const secretName = stringAtNullable(runtimeSecret, "name");
if (secretName === null) continue;
for (const item of arrayAt(runtimeSecret, "data").map(record)) {
const targetKey = stringAtNullable(item, "targetKey");
const sourcePurpose = stringAtNullable(item, "sourcePurpose");
const sourceKey = sourcePurpose === null ? null : stringAtNullable(sourcesByPurpose.get(sourcePurpose), "sourceKey");
if (targetKey !== null && sourceKey !== null && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(sourceKey)) {
pushEnv({ name: sourceKey, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
}
const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
if (envName === null) continue;
env.push({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
pushEnv({ name: envName, valueFrom: { secretKeyRef: { name: secretName, key: targetKey } } });
}
}
return env;
}
function sentinelCadenceCronJobPlan(
spec: HwlabRuntimeLaneSpec,
sentinelId: string,
runtime: Record<string, unknown>,
cicd: Record<string, unknown>,
scenarios: unknown,
imageRef: string,
sentinelEnv: readonly Record<string, unknown>[],
): Record<string, unknown> | null {
const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId");
if (scenarioId === null) return null;
const scenario = scenarioRows(scenarios).find((item) => item.id === scenarioId && item.enabled !== false) ?? null;
if (scenario === null) return null;
const cadenceSeconds = typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null;
const schedule = cadenceSeconds === null ? null : cronScheduleForCadenceSeconds(cadenceSeconds);
if (schedule === null) return null;
const namespace = stringAt(runtime, "namespace");
const deploymentName = stringAt(runtime, "deploymentName");
const serviceAccountName = stringAt(runtime, "serviceAccountName");
const timeoutSeconds = numberAtNullable(cicd, "targetValidation.maxSeconds") ?? numberAtNullable(scenario, "maxRunSeconds") ?? 300;
const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost");
const name = safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
const labels = {
"app.kubernetes.io/name": name,
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
"app.kubernetes.io/component": "cadence-scheduler",
"app.kubernetes.io/managed-by": "unidesk",
"unidesk.ai/spec-ref": "PJ2026-01060508",
"unidesk.ai/node": spec.nodeId,
"unidesk.ai/lane": spec.lane,
"unidesk.ai/web-probe-sentinel-id": sentinelId,
};
return {
apiVersion: "batch/v1",
kind: "CronJob",
metadata: {
name,
namespace,
labels,
annotations: {
"unidesk.ai/cadence": String(scenario.cadence),
"unidesk.ai/target-validation-max-seconds": String(timeoutSeconds),
},
},
spec: {
schedule,
concurrencyPolicy: "Forbid",
successfulJobsHistoryLimit: 3,
failedJobsHistoryLimit: 5,
startingDeadlineSeconds: Math.max(60, cadenceSeconds),
jobTemplate: {
spec: {
activeDeadlineSeconds: timeoutSeconds + 60,
ttlSecondsAfterFinished: 86400,
backoffLimit: 0,
template: {
metadata: { labels },
spec: {
restartPolicy: "Never",
serviceAccountName,
containers: [{
name: "quick-verify",
image: imageRef,
imagePullPolicy: "IfNotPresent",
command: ["bun", "scripts/cli.ts"],
args: [
"web-probe",
"sentinel",
"validate",
"--node",
spec.nodeId,
"--lane",
spec.lane,
"--sentinel",
sentinelId,
"--quick-verify",
"--confirm",
"--wait",
"--timeout-seconds",
String(timeoutSeconds),
],
env: [
...sentinelEnv,
{ name: "UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE", value: "1" },
...(mainServerHost === null ? [] : [{ name: "UNIDESK_MAIN_SERVER_HOST", value: mainServerHost }]),
],
}],
},
},
},
},
},
};
}
function scenarioRows(value: unknown): Record<string, unknown>[] {
if (Array.isArray(value)) return value.map(record);
if (!isRecord(value)) return [];
if (Array.isArray(value.scenarios)) return value.scenarios.map(record);
if (isRecord(value.workflow)) return [value.workflow];
return [value];
}
function parseDurationSeconds(value: string): number | null {
const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
if (match === null) return null;
const amount = Number(match[1]);
const unit = match[2];
if (unit === "ms") return Math.max(60, Math.ceil(amount / 1000));
if (unit === "s") return Math.max(60, amount);
if (unit === "m") return amount * 60;
if (unit === "h") return amount * 3600;
return null;
}
function cronScheduleForCadenceSeconds(seconds: number): string | null {
if (!Number.isFinite(seconds) || seconds <= 0) return null;
if (seconds <= 60) return "* * * * *";
if (seconds % 60 === 0) {
const minutes = Math.trunc(seconds / 60);
if (minutes >= 1 && minutes <= 59) return `*/${minutes} * * * *`;
if (minutes % 60 === 0) {
const hours = Math.trunc(minutes / 60);
if (hours >= 1 && hours <= 23) return `0 */${hours} * * *`;
if (hours === 24) return "0 0 * * *";
}
}
return null;
}
function accountSecretEnvName(sourcePurpose: string, targetKey: string): string | null {
if (!/^account-[a-z0-9-]+$/u.test(sourcePurpose) || !targetKey.endsWith(".json")) return null;
const segment = sourcePurpose.toUpperCase().replace(/[^A-Z0-9]+/gu, "_").replace(/^_+|_+$/gu, "");
@@ -601,7 +754,21 @@ function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: No
const envName = sourcePurpose === null || targetKey === null ? null : accountSecretEnvName(sourcePurpose, targetKey);
if (envName === null || sourcePurpose === null || targetKey === null) continue;
const source = sourcesByPurpose.get(sourcePurpose);
const runtimeValue = process.env[envName];
if (source === undefined) {
if (runtimeValue !== undefined && runtimeValue.length > 0) {
env[envName] = runtimeValue;
items.push({
envName,
secretName,
targetKey,
sourcePurpose,
sourceMode: "runtime-env",
fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
valuesRedacted: true,
});
continue;
}
missing.push({ envName, secretName, targetKey, sourcePurpose, reason: "source-purpose-missing", valuesRedacted: true });
continue;
}
@@ -609,6 +776,21 @@ function quickVerifyAccountEnv(state: SentinelCicdState): { ok: boolean; env: No
const sourceKey = stringAt(source, "sourceKey");
const material = readSentinelSecretSourceValue(source);
if (!material.ok) {
if (runtimeValue !== undefined && runtimeValue.length > 0) {
env[envName] = runtimeValue;
items.push({
envName,
secretName,
targetKey,
sourcePurpose,
sourceRef,
sourceKey,
sourceMode: "runtime-env",
fingerprint: `sha256:${createHash("sha256").update(runtimeValue).digest("hex").slice(0, 16)}`,
valuesRedacted: true,
});
continue;
}
missing.push({ envName, secretName, targetKey, sourcePurpose, sourceRef, sourceKey, reason: material.error, sourcePath: material.sourcePath, valuesRedacted: true });
continue;
}
@@ -2921,6 +3103,9 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p
const servicePort = numberAt(state.runtime, "servicePort");
const deploymentName = stringAt(state.runtime, "deploymentName");
const url = `http://${serviceName}.${namespace}.svc.cluster.local:${servicePort}${pathWithQuery}`;
if (process.env.UNIDESK_WEB_PROBE_SENTINEL_DIRECT_SERVICE === "1") {
return callSentinelServiceDirect(method, pathWithQuery, body, timeoutSeconds, url);
}
const proxyPath = `/api/v1/namespaces/${namespace}/services/${serviceName}:${servicePort}/proxy${pathWithQuery}`;
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const pathB64 = Buffer.from(pathWithQuery, "utf8").toString("base64");
@@ -2972,6 +3157,52 @@ function callSentinelService(state: SentinelCicdState, method: "GET" | "POST", p
};
}
function callSentinelServiceDirect(method: "GET" | "POST", pathWithQuery: string, body: Record<string, unknown> | null, timeoutSeconds: number, url: string): Record<string, unknown> {
const bodyB64 = Buffer.from(body === null ? "" : JSON.stringify(body), "utf8").toString("base64");
const fetchScript = [
"const method = process.env.SENTINEL_METHOD || 'GET';",
"const url = process.env.SENTINEL_URL || '';",
"const body = Buffer.from(process.env.SENTINEL_BODY_B64 || '', 'base64').toString('utf8');",
"const headers = method === 'POST' ? { 'content-type': 'application/json' } : undefined;",
"fetch(url, { method, headers, body: method === 'POST' ? body : undefined }).then(async (response) => {",
" const text = await response.text();",
" process.stdout.write(text);",
" if (!response.ok) process.exit(22);",
"}).catch((error) => {",
" console.error(error && error.stack ? error.stack : String(error));",
" process.exit(23);",
"});",
].join(" ");
const attemptTimeoutSeconds = Math.max(5, Math.min(timeoutSeconds, method === "GET" ? 15 : 60));
const result = runCommand(["node", "-e", fetchScript], repoRoot, {
timeoutMs: attemptTimeoutSeconds * 1000,
env: {
...process.env,
SENTINEL_METHOD: method,
SENTINEL_URL: url,
SENTINEL_BODY_B64: bodyB64,
},
});
const parsed = parseJsonObject(result.stdout);
const compactBodyJson = compactSentinelServiceBodyJson(parsed);
return {
ok: result.exitCode === 0,
method,
path: pathWithQuery,
internalUrl: url,
httpStatus: result.exitCode === 0 ? 200 : null,
bodyJson: record(compactBodyJson),
bodyTextPreview: parsed === null ? clipTail(result.stdout, 4000) : "",
bodyBytes: Buffer.byteLength(result.stdout),
error: result.exitCode === 0 ? null : clipTail(`${result.stderr}${result.stdout}`, 1000),
proxyPath: null,
result: compactCommand(result),
attempts: [{ attempt: 1, ...compactCommand(result), parsedOk: parsed !== null, transport: "direct-service", valuesRedacted: true }],
transport: "direct-service",
valuesRedacted: true,
};
}
function compactSentinelServiceBodyJson(value: Record<string, unknown> | null): unknown {
if (value === null || typeof value.renderedText !== "string") return value;
return {
@@ -3890,9 +4121,11 @@ function readPromptSetForScenario(scenario: Record<string, unknown>): { ok: true
const paths = secretSourcePaths(sourceRef);
const sourcePath = paths.find((item) => existsSync(item)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
const summary = { sourceRef, sourceKey: key, sourcePath: displayPath(sourcePath), valuesRedacted: true };
if (!existsSync(sourcePath)) return { ok: false, error: "prompt-source-missing", summary };
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
const raw = values[key];
const runtimeRaw = process.env[key];
const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
const raw = values[key] ?? runtimeRaw;
const sourceMode = values[key] !== undefined ? "secret-source-file" : runtimeRaw !== undefined && runtimeRaw.length > 0 ? "runtime-env" : null;
if (!existsSync(sourcePath) && sourceMode === null) return { ok: false, error: "prompt-source-missing", summary };
if (raw === undefined || raw.length === 0) return { ok: false, error: "prompt-key-missing", summary };
const parsed = parsePromptJson(raw);
if (parsed.length === 0) return { ok: false, error: "prompt-json-empty", summary };
@@ -3901,6 +4134,7 @@ function readPromptSetForScenario(scenario: Record<string, unknown>): { ok: true
prompts: parsed,
summary: {
...summary,
sourceMode,
promptCount: parsed.length,
promptMarkers: parsed.map((item) => Array.from(new Set(Array.from(item.matchAll(/\bsentinel-(?:0[1-9]|10)\b/giu)).map((match) => match[0].toLowerCase())))),
promptTextHashes: parsed.map((item) => `sha256:${createHash("sha256").update(item).digest("hex").slice(0, 16)}`),
scripts/src/hwlab-node/web-probe.ts
+4 -3
View File
@@ -1256,11 +1256,12 @@ export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): Boo
}
const paths = secretSourcePaths(sourceRef);
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
if (!existsSync(sourcePath)) {
const runtimePassword = process.env[sourceKey];
if (!existsSync(sourcePath) && (runtimePassword === undefined || runtimePassword.length === 0)) {
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" };
}
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
const password = values[sourceKey];
const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
const password = values[sourceKey] ?? runtimePassword;
if (password === undefined || password.length === 0) {
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" };
}