feat(sentinel): restore JD01 cadence cronjob visibility
This commit is contained in:
Codex
@@ -6,6 +6,7 @@
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility.
|
||||
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
|
||||
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
@@ -20,6 +21,7 @@ import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
||||
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
|
||||
|
||||
export type WebProbeSentinelConfigAction = "plan" | "status";
|
||||
export type WebProbeSentinelImageAction = "status" | "build";
|
||||
@@ -178,6 +180,7 @@ interface SentinelObservedStatus {
|
||||
readonly gitops: Record<string, unknown>;
|
||||
readonly argo: Record<string, unknown>;
|
||||
readonly runtime: Record<string, unknown>;
|
||||
readonly cadence: Record<string, unknown>;
|
||||
readonly wait?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
@@ -215,7 +218,7 @@ export interface ChildCliResult {
|
||||
readonly result: CompactCommandResult & { stdoutTail: string; stderrTail: string };
|
||||
}
|
||||
|
||||
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-06-30-p14-sentinel-cicd-visibility";
|
||||
const SPEC_REF = "PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel";
|
||||
|
||||
export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options: WebProbeSentinelOptions): RenderedCliResult {
|
||||
if (options.kind === "config") return withWebProbeSentinelConfigRendered(webProbeSentinelConfigPlan(spec, options.action, options.sentinelId));
|
||||
@@ -317,6 +320,14 @@ function runSentinelControlPlane(state: SentinelCicdState, options: Extract<WebP
|
||||
objects: manifestObjectSummary(state.manifests),
|
||||
sha256: state.manifestSha256,
|
||||
},
|
||||
observability: webProbeSentinelOtelSummary({
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
namespace: stringAt(state.runtime, "namespace"),
|
||||
runtime: state.runtime,
|
||||
cicd: state.cicd,
|
||||
}),
|
||||
observed,
|
||||
warnings: observedWarnings,
|
||||
blocker: null,
|
||||
@@ -924,8 +935,24 @@ function renderSentinelManifests(
|
||||
const servicePort = numberAt(runtime, "servicePort");
|
||||
const pvcStorage = stringAt(runtime, "pvcStorage");
|
||||
const stateRoot = stringAt(runtime, "stateRoot");
|
||||
const sentinelEnv = sentinelContainerEnv(sentinelId, secrets);
|
||||
const sentinelEnv = sentinelContainerEnv(sentinelId, runtime, cicd, secrets);
|
||||
const cadenceJob = sentinelCadenceCronJobPlan(spec, sentinelId, runtime, cicd, scenarios, image.ref, sentinelEnv);
|
||||
if (cadenceJob !== null) {
|
||||
emitWebProbeSentinelSpan({
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
sentinelId,
|
||||
namespace,
|
||||
runtime,
|
||||
cicd,
|
||||
}, "web_probe_sentinel.cadence.cronjob_rendered", {
|
||||
cronJobName: record(cadenceJob.metadata).name ?? null,
|
||||
namespace,
|
||||
cadence: record(cadenceJob.metadata).annotations === undefined ? null : record(record(cadenceJob.metadata).annotations)["unidesk.ai/cadence"],
|
||||
schedule: record(cadenceJob.spec).schedule ?? null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
return [
|
||||
{
|
||||
apiVersion: "v1",
|
||||
@@ -1060,8 +1087,16 @@ function renderSentinelManifests(
|
||||
];
|
||||
}
|
||||
|
||||
function sentinelContainerEnv(sentinelId: string, secrets: Record<string, unknown>): readonly Record<string, unknown>[] {
|
||||
function sentinelContainerEnv(sentinelId: string, runtime: Record<string, unknown>, cicd: Record<string, unknown>, secrets: Record<string, unknown>): readonly Record<string, unknown>[] {
|
||||
const env: Record<string, unknown>[] = [{ name: "UNIDESK_WEB_PROBE_SENTINEL_ID", value: sentinelId }];
|
||||
const otelEnabled = booleanAtNullable(runtime, "observability.otel.enabled") ?? booleanAtNullable(cicd, "observability.otel.enabled") ?? false;
|
||||
const otelEndpoint = stringAtNullable(runtime, "observability.otel.tracesEndpoint")
|
||||
?? stringAtNullable(runtime, "observability.otel.endpoint")
|
||||
?? stringAtNullable(cicd, "observability.otel.tracesEndpoint")
|
||||
?? stringAtNullable(cicd, "observability.otel.endpoint");
|
||||
const otelServiceName = stringAtNullable(runtime, "observability.otel.serviceName") ?? stringAtNullable(cicd, "observability.otel.serviceName");
|
||||
const otelSampler = stringAtNullable(runtime, "observability.otel.sampler") ?? stringAtNullable(cicd, "observability.otel.sampler");
|
||||
const otelSamplerArg = stringAtNullable(runtime, "observability.otel.samplerArg") ?? stringAtNullable(cicd, "observability.otel.samplerArg");
|
||||
const sourcesByPurpose = new Map<string, Record<string, unknown>>();
|
||||
for (const source of arrayAt(secrets, "sources").map(record)) {
|
||||
const purpose = stringAtNullable(source, "purpose");
|
||||
@@ -1074,6 +1109,12 @@ function sentinelContainerEnv(sentinelId: string, secrets: Record<string, unknow
|
||||
used.add(name);
|
||||
env.push(item);
|
||||
};
|
||||
if (otelEnabled) {
|
||||
if (otelEndpoint !== null) pushEnv({ name: "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", value: otelEndpoint });
|
||||
if (otelServiceName !== null) pushEnv({ name: "OTEL_SERVICE_NAME", value: otelServiceName });
|
||||
if (otelSampler !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER", value: otelSampler });
|
||||
if (otelSamplerArg !== null) pushEnv({ name: "OTEL_TRACES_SAMPLER_ARG", value: otelSamplerArg });
|
||||
}
|
||||
for (const runtimeSecret of arrayAt(secrets, "runtimeSecrets").map(record)) {
|
||||
const secretName = stringAtNullable(runtimeSecret, "name");
|
||||
if (secretName === null) continue;
|
||||
@@ -1102,6 +1143,7 @@ function sentinelCadenceCronJobPlan(
|
||||
imageRef: string,
|
||||
sentinelEnv: readonly Record<string, unknown>[],
|
||||
): Record<string, unknown> | null {
|
||||
const scheduler = record(valueAtPath(cicd, "targetValidation.cadenceScheduler"));
|
||||
const cadenceSchedulerEnabled = booleanAtNullable(cicd, "targetValidation.cadenceScheduler.enabled") === true;
|
||||
if (!cadenceSchedulerEnabled) return null;
|
||||
const scenarioId = stringAtNullable(cicd, "targetValidation.scenarioId");
|
||||
@@ -1114,9 +1156,12 @@ function sentinelCadenceCronJobPlan(
|
||||
const namespace = stringAt(runtime, "namespace");
|
||||
const deploymentName = stringAt(runtime, "deploymentName");
|
||||
const serviceAccountName = stringAt(runtime, "serviceAccountName");
|
||||
const timeoutSeconds = numberAtNullable(cicd, "targetValidation.maxSeconds") ?? numberAtNullable(scenario, "maxRunSeconds") ?? 300;
|
||||
const timeoutSeconds = numberAt(cicd, "targetValidation.maxSeconds");
|
||||
const activeDeadlineSlackSeconds = numberAt(scheduler, "activeDeadlineSlackSeconds");
|
||||
const mainServerHost = stringAtNullable(cicd, "scheduler.mainServerHost");
|
||||
const name = safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
|
||||
const name = sentinelCadenceCronJobName(deploymentName);
|
||||
const concurrencyPolicy = stringAt(scheduler, "concurrencyPolicy");
|
||||
if (!["Allow", "Forbid", "Replace"].includes(concurrencyPolicy)) throw new Error("targetValidation.cadenceScheduler.concurrencyPolicy must be Allow, Forbid or Replace");
|
||||
const labels = {
|
||||
"app.kubernetes.io/name": name,
|
||||
"app.kubernetes.io/part-of": "hwlab-web-probe-sentinel",
|
||||
@@ -1137,19 +1182,20 @@ function sentinelCadenceCronJobPlan(
|
||||
annotations: {
|
||||
"unidesk.ai/cadence": String(scenario.cadence),
|
||||
"unidesk.ai/target-validation-max-seconds": String(timeoutSeconds),
|
||||
"unidesk.ai/source": "targetValidation.cadenceScheduler",
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
schedule,
|
||||
concurrencyPolicy: "Forbid",
|
||||
successfulJobsHistoryLimit: 3,
|
||||
failedJobsHistoryLimit: 5,
|
||||
startingDeadlineSeconds: Math.max(60, cadenceSeconds),
|
||||
concurrencyPolicy,
|
||||
successfulJobsHistoryLimit: numberAt(scheduler, "successfulJobsHistoryLimit"),
|
||||
failedJobsHistoryLimit: numberAt(scheduler, "failedJobsHistoryLimit"),
|
||||
startingDeadlineSeconds: numberAt(scheduler, "startingDeadlineSeconds"),
|
||||
jobTemplate: {
|
||||
spec: {
|
||||
activeDeadlineSeconds: timeoutSeconds + 60,
|
||||
ttlSecondsAfterFinished: 86400,
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: timeoutSeconds + activeDeadlineSlackSeconds,
|
||||
ttlSecondsAfterFinished: numberAt(scheduler, "ttlSecondsAfterFinished"),
|
||||
backoffLimit: numberAt(scheduler, "backoffLimit"),
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
@@ -1190,6 +1236,10 @@ function sentinelCadenceCronJobPlan(
|
||||
};
|
||||
}
|
||||
|
||||
function sentinelCadenceCronJobName(deploymentName: string): string {
|
||||
return safeKubernetesSegment(`${deploymentName}-quick-verify`, 52);
|
||||
}
|
||||
|
||||
function scenarioRows(value: unknown): Record<string, unknown>[] {
|
||||
if (Array.isArray(value)) return value.map(record);
|
||||
if (!isRecord(value)) return [];
|
||||
@@ -1605,6 +1655,7 @@ function sentinelSkippedObservedStatus(reason: string): SentinelObservedStatus {
|
||||
gitops: skipped,
|
||||
argo: skipped,
|
||||
runtime: skipped,
|
||||
cadence: skipped,
|
||||
wait: {
|
||||
polls: 0,
|
||||
elapsedMs: 0,
|
||||
@@ -1633,6 +1684,7 @@ function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
|
||||
gitops,
|
||||
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
|
||||
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
|
||||
cadence: probeCadenceCronJob(state, timeoutSeconds),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1668,13 +1720,15 @@ function sentinelObservedReady(value: Record<string, unknown> | SentinelObserved
|
||||
&& gitMirrorReady
|
||||
&& record(observed.gitops).ok === true
|
||||
&& record(observed.argo).ok === true
|
||||
&& record(observed.runtime).ok === true;
|
||||
&& record(observed.runtime).ok === true
|
||||
&& record(observed.cadence).ok === true;
|
||||
}
|
||||
|
||||
function sentinelObservedWarnings(value: Record<string, unknown> | SentinelObservedStatus | null): string[] {
|
||||
const observed = record(value);
|
||||
const argo = record(observed.argo);
|
||||
return mergeWarnings(argo.warning);
|
||||
const cadence = record(observed.cadence);
|
||||
return mergeWarnings(argo.warning, cadence.warning);
|
||||
}
|
||||
|
||||
function probeSourceMirror(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
@@ -1900,6 +1954,74 @@ function probeRuntimeObjects(state: SentinelCicdState, timeoutSeconds: number, e
|
||||
return { ok: result.exitCode === 0 && probe?.ok === true, probe, result: compactCommand(result) };
|
||||
}
|
||||
|
||||
function probeCadenceCronJob(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||
const expected = state.manifests.find((item) => item.kind === "CronJob") ?? null;
|
||||
if (expected === null) {
|
||||
return { ok: true, skipped: true, reason: "targetValidation.cadenceScheduler.disabled", valuesRedacted: true };
|
||||
}
|
||||
const metadata = record(expected.metadata);
|
||||
const spec = record(expected.spec);
|
||||
const namespace = stringAt(metadata, "namespace");
|
||||
const name = stringAt(metadata, "name");
|
||||
const expectedSchedule = stringAt(spec, "schedule");
|
||||
const script = [
|
||||
"set +e",
|
||||
`namespace=${shellQuote(namespace)}`,
|
||||
`cronjob=${shellQuote(name)}`,
|
||||
`sentinel=${shellQuote(state.sentinelId)}`,
|
||||
`expected_schedule=${shellQuote(expectedSchedule)}`,
|
||||
"tmp=$(mktemp -d)",
|
||||
"kubectl -n \"$namespace\" get cronjob \"$cronjob\" -o json >\"$tmp/cronjob.json\" 2>/dev/null; echo $? >\"$tmp/cronjob.rc\"",
|
||||
"kubectl -n \"$namespace\" get jobs -l \"unidesk.ai/web-probe-sentinel-id=$sentinel,app.kubernetes.io/component=cadence-scheduler\" -o json >\"$tmp/jobs.json\" 2>/dev/null; echo $? >\"$tmp/jobs.rc\"",
|
||||
"node - \"$tmp\" \"$namespace\" \"$cronjob\" \"$expected_schedule\" <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const [dir, namespace, cronJobName, expectedSchedule] = process.argv.slice(2);",
|
||||
"function rc(name){ try { return Number(fs.readFileSync(`${dir}/${name}.rc`, 'utf8').trim()); } catch { return 1; } }",
|
||||
"function json(name){ try { return JSON.parse(fs.readFileSync(`${dir}/${name}.json`, 'utf8')); } catch { return null; } }",
|
||||
"const cron = json('cronjob');",
|
||||
"const jobs = Array.isArray(json('jobs')?.items) ? json('jobs').items : [];",
|
||||
"const present = rc('cronjob') === 0 && !!cron;",
|
||||
"const schedule = cron?.spec?.schedule || null;",
|
||||
"const scheduleMatches = present && schedule === expectedSchedule;",
|
||||
"const suspended = cron?.spec?.suspend === true;",
|
||||
"const active = Array.isArray(cron?.status?.active) ? cron.status.active.length : 0;",
|
||||
"const sortedJobs = jobs.slice().sort((a,b)=>String(b?.metadata?.creationTimestamp||'').localeCompare(String(a?.metadata?.creationTimestamp||''))).slice(0,8);",
|
||||
"let code = null;",
|
||||
"if (!present) code = 'sentinel-cadence-cronjob-missing';",
|
||||
"else if (!scheduleMatches) code = 'sentinel-cadence-cronjob-schedule-mismatch';",
|
||||
"else if (suspended) code = 'sentinel-cadence-cronjob-suspended';",
|
||||
"const latestJob = sortedJobs[0] || null;",
|
||||
"console.log(JSON.stringify({ ok: code === null, code, present, namespace, name: cronJobName, schedule, expectedSchedule, scheduleMatches, suspended, lastScheduleTime: cron?.status?.lastScheduleTime || null, lastSuccessfulTime: cron?.status?.lastSuccessfulTime || null, active, jobCount: jobs.length, latestJobs: sortedJobs.map((job)=>({ name: job?.metadata?.name || null, createdAt: job?.metadata?.creationTimestamp || null, active: Number(job?.status?.active || 0), succeeded: Number(job?.status?.succeeded || 0), failed: Number(job?.status?.failed || 0), completionTime: job?.status?.completionTime || null, valuesRedacted:true })), latestJobName: latestJob?.metadata?.name || null, valuesRedacted: true }));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = runCommand(["trans", stringAt(state.controlPlaneNode, "kubeRoute"), "sh", "--", script], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||
const probe = parseJsonObject(result.stdout);
|
||||
const ok = result.exitCode === 0 && probe?.ok === true;
|
||||
emitWebProbeSentinelSpan({
|
||||
node: state.spec.nodeId,
|
||||
lane: state.spec.lane,
|
||||
sentinelId: state.sentinelId,
|
||||
namespace,
|
||||
runtime: state.runtime,
|
||||
cicd: state.cicd,
|
||||
}, "web_probe_sentinel.cadence.cronjob_observed", {
|
||||
cronJobName: name,
|
||||
namespace,
|
||||
schedule: expectedSchedule,
|
||||
status: ok ? "ok" : text(probe?.code ?? "unknown"),
|
||||
jobName: probe?.latestJobName ?? null,
|
||||
failureKind: probe?.code ?? null,
|
||||
valuesRedacted: true,
|
||||
}, ok);
|
||||
return {
|
||||
ok,
|
||||
probe,
|
||||
result: compactCommand(result),
|
||||
warning: ok ? null : `cadence CronJob is not ready: ${text(probe?.code ?? "probe-failed")}`,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function expectedRuntimeImageFromRegistry(state: SentinelCicdState, registry: Record<string, unknown>): string | null {
|
||||
const digest = nonEmptyString(record(record(registry).probe).digest);
|
||||
if (digest === null) return null;
|
||||
@@ -3816,6 +3938,7 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
const gitops = record(result.gitops);
|
||||
const argo = record(result.argo);
|
||||
const validation = record(result.validation);
|
||||
const observability = record(result.observability);
|
||||
const observed = record(result.observed);
|
||||
const sourceMirrorSync = record(result.sourceMirrorSync);
|
||||
const publish = record(result.publish);
|
||||
@@ -3841,6 +3964,8 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
|
||||
"",
|
||||
table(["SCENARIO", "MAX_SECONDS", "CI_WAIT", "QVERIFY", "SECOND_PATH"], [[validation.scenarioId, validation.maxSeconds, validation.controlPlaneWaitMaxSeconds ?? "-", validation.quickVerifyMode ?? "-", validation.automaticSecondPath]]),
|
||||
"",
|
||||
Object.keys(observability).length === 0 ? "OTEL\n-" : table(["ENABLED", "ENDPOINT", "SERVICE", "COVERAGE"], [[observability.enabled, observability.endpointConfigured, observability.serviceName, observability.coverage]]),
|
||||
"",
|
||||
renderObservedStatus(observed),
|
||||
"",
|
||||
Object.keys(sourceMirrorSync).length === 0 ? "SOURCE_MIRROR_SYNC\n-" : table(["OK", "PHASE", "JOB", "COMMIT", "ELAPSED"], [[sourceMirrorSync.ok, sourceMirrorSync.phase, sourceMirrorSync.jobName, short(record(sourceMirrorSync.payload).mirrorCommit), sourceMirrorSync.elapsedMs ?? "-"]]),
|
||||
@@ -3913,6 +4038,7 @@ function renderObservedStatus(observed: Record<string, unknown>): string {
|
||||
observedStatusRow("gitops", observed.gitops),
|
||||
observedStatusRow("argo", observed.argo),
|
||||
observedStatusRow("runtime", observed.runtime),
|
||||
observedStatusRow("cadence", observed.cadence),
|
||||
].filter((row) => row !== null);
|
||||
if (rows.length === 0) return "OBSERVED\n-";
|
||||
return table(["CHECK", "OK", "DETAIL", "EXIT", "TIMED_OUT", "PREVIEW"], rows);
|
||||
@@ -3944,6 +4070,11 @@ function observedDetail(name: string, item: Record<string, unknown>): string {
|
||||
const deployment = record(probe.deployment);
|
||||
return `ready=${deployment.readyReplicas ?? "-"} image=${short(deployment.image)}/${short(deployment.expectedImage)}`;
|
||||
}
|
||||
if (name === "cadence") {
|
||||
if (item.skipped === true) return `${item.reason ?? "skipped"}`;
|
||||
const probe = record(item.probe);
|
||||
return `${probe.code ?? "ok"} schedule=${probe.schedule ?? "-"}/${probe.expectedSchedule ?? "-"} last=${probe.lastScheduleTime ?? "-"} jobs=${probe.jobCount ?? "-"}`;
|
||||
}
|
||||
return "-";
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user