feat(gc): add JD01 remote pressure policy
This commit is contained in:
@@ -106,6 +106,44 @@ gc:
|
|||||||
limit: 50
|
limit: 50
|
||||||
resultLimit: 50
|
resultLimit: 50
|
||||||
full: false
|
full: false
|
||||||
|
remote:
|
||||||
|
targets:
|
||||||
|
JD01:
|
||||||
|
memoryPressure:
|
||||||
|
processPatterns:
|
||||||
|
- chrome
|
||||||
|
- chromium
|
||||||
|
- web-probe observe
|
||||||
|
- playwright
|
||||||
|
observeStateRoots:
|
||||||
|
- /root/workspace/hwlab-v03/.state/web-observe
|
||||||
|
- /root/workspace/hwlab-v03/.state/playwright-cli
|
||||||
|
staleRunMaxAgeHours: 6
|
||||||
|
pvcAttribution:
|
||||||
|
namespaces:
|
||||||
|
- agentrun-ci
|
||||||
|
- agentrun-v02
|
||||||
|
- hwlab-ci
|
||||||
|
- hwlab-dev
|
||||||
|
- hwlab-prod
|
||||||
|
- hwlab-v03
|
||||||
|
candidateNamespaces:
|
||||||
|
- agentrun-ci
|
||||||
|
- hwlab-ci
|
||||||
|
hwlabNode: JD01
|
||||||
|
hwlabLane: v03
|
||||||
|
agentrunNode: JD01
|
||||||
|
agentrunLane: v02
|
||||||
|
limit: 80
|
||||||
|
policyTimer:
|
||||||
|
enabled: true
|
||||||
|
name: unidesk-jd01-low-risk-gc
|
||||||
|
onCalendar: daily
|
||||||
|
randomizedDelaySec: 20min
|
||||||
|
journalTargetBytes: 256MiB
|
||||||
|
tmpMinAgeHours: 24
|
||||||
|
includeAptCache: true
|
||||||
|
includeToolCaches: false
|
||||||
policyTimer:
|
policyTimer:
|
||||||
journald:
|
journald:
|
||||||
systemMaxUse: 512MiB
|
systemMaxUse: 512MiB
|
||||||
|
|||||||
+957
-56
File diff suppressed because it is too large
Load Diff
@@ -1729,6 +1729,9 @@ function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number,
|
|||||||
const revision = nonEmptyString(revisionRaw);
|
const revision = nonEmptyString(revisionRaw);
|
||||||
const revisionMatches = expectedRevision === null || revision === expectedRevision;
|
const revisionMatches = expectedRevision === null || revision === expectedRevision;
|
||||||
const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy";
|
const healthy = result.exitCode === 0 && syncStatus === "Synced" && healthStatus === "Healthy";
|
||||||
|
const diagnostics = result.exitCode === 0 && !healthy
|
||||||
|
? probeArgoApplicationDiagnostics(state, timeoutSeconds, namespace, applicationName)
|
||||||
|
: null;
|
||||||
return {
|
return {
|
||||||
ok: healthy,
|
ok: healthy,
|
||||||
present: result.exitCode === 0,
|
present: result.exitCode === 0,
|
||||||
@@ -1741,10 +1744,80 @@ function probeArgoApplication(state: SentinelCicdState, timeoutSeconds: number,
|
|||||||
warning: healthy && !revisionMatches
|
warning: healthy && !revisionMatches
|
||||||
? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness."
|
? "Argo app is Synced/Healthy but status.sync.revision differs from current GitOps branch HEAD; in multi-sentinel GitOps this can happen when another sentinel path advances the branch. Runtime image/manifest checks remain authoritative for rollout readiness."
|
||||||
: null,
|
: null,
|
||||||
|
diagnostics,
|
||||||
result: compactCommand(result),
|
result: compactCommand(result),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function probeArgoApplicationDiagnostics(state: SentinelCicdState, timeoutSeconds: number, namespace: string, applicationName: string): Record<string, unknown> {
|
||||||
|
const route = stringAt(state.controlPlaneNode, "kubeRoute");
|
||||||
|
const appResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "application", applicationName, "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||||
|
const parsed = parseJsonObject(appResult.stdout);
|
||||||
|
const status = record(parsed?.status);
|
||||||
|
const resourcesRaw = Array.isArray(status.resources) ? status.resources : [];
|
||||||
|
const resources = resourcesRaw
|
||||||
|
.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
|
||||||
|
.map((item) => {
|
||||||
|
const health = record(item.health);
|
||||||
|
return {
|
||||||
|
kind: item.kind ?? null,
|
||||||
|
namespace: item.namespace ?? null,
|
||||||
|
name: item.name ?? null,
|
||||||
|
status: item.status ?? null,
|
||||||
|
healthStatus: health.status ?? null,
|
||||||
|
healthMessage: short(health.message),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const problemResources = resources
|
||||||
|
.filter((item) => {
|
||||||
|
const sync = nonEmptyString(item.status);
|
||||||
|
const health = nonEmptyString(item.healthStatus);
|
||||||
|
return (sync !== null && sync !== "Synced") || (health !== null && health !== "Healthy");
|
||||||
|
})
|
||||||
|
.slice(0, 12);
|
||||||
|
const conditions = (Array.isArray(status.conditions) ? status.conditions : [])
|
||||||
|
.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
|
||||||
|
.slice(-8)
|
||||||
|
.map((item) => ({
|
||||||
|
type: item.type ?? null,
|
||||||
|
message: short(item.message),
|
||||||
|
lastTransitionTime: item.lastTransitionTime ?? null,
|
||||||
|
}));
|
||||||
|
const operationState = record(status.operationState);
|
||||||
|
const eventResult = runCommand(["trans", route, "kubectl", "-n", namespace, "get", "events", "--field-selector", `involvedObject.name=${applicationName}`, "--sort-by=.lastTimestamp", "-o", "json"], repoRoot, { timeoutMs: Math.min(timeoutSeconds, 60) * 1000 });
|
||||||
|
const eventsJson = parseJsonObject(eventResult.stdout);
|
||||||
|
const events = (Array.isArray(eventsJson?.items) ? eventsJson.items : [])
|
||||||
|
.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item))
|
||||||
|
.slice(-8)
|
||||||
|
.map((item) => ({
|
||||||
|
type: item.type ?? null,
|
||||||
|
reason: item.reason ?? null,
|
||||||
|
message: short(item.message),
|
||||||
|
count: item.count ?? null,
|
||||||
|
lastTimestamp: item.lastTimestamp ?? item.eventTime ?? null,
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
ok: appResult.exitCode === 0,
|
||||||
|
resourceCount: resources.length,
|
||||||
|
problemResourceCount: problemResources.length,
|
||||||
|
problemResources,
|
||||||
|
conditions,
|
||||||
|
operationState: {
|
||||||
|
phase: operationState.phase ?? null,
|
||||||
|
message: short(operationState.message),
|
||||||
|
startedAt: operationState.startedAt ?? null,
|
||||||
|
finishedAt: operationState.finishedAt ?? null,
|
||||||
|
},
|
||||||
|
events,
|
||||||
|
result: compactCommand(appResult),
|
||||||
|
eventsResult: compactCommand(eventResult),
|
||||||
|
drillDown: {
|
||||||
|
application: `trans ${route} kubectl -n ${namespace} get application ${applicationName} -o json`,
|
||||||
|
events: `trans ${route} kubectl -n ${namespace} get events --field-selector involvedObject.name=${applicationName} --sort-by=.lastTimestamp`,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
function probeGitopsRuntimeManifest(state: SentinelCicdState, timeoutSeconds: number): Record<string, unknown> {
|
||||||
const namespace = stringAt(state.cicd, "builder.namespace");
|
const namespace = stringAt(state.cicd, "builder.namespace");
|
||||||
const repository = stringAt(state.controlPlaneTarget, "source.repository");
|
const repository = stringAt(state.controlPlaneTarget, "source.repository");
|
||||||
@@ -3857,7 +3930,15 @@ function observedDetail(name: string, item: Record<string, unknown>): string {
|
|||||||
if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`;
|
if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`;
|
||||||
if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`;
|
if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`;
|
||||||
if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`;
|
if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`;
|
||||||
if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}`;
|
if (name === "argo") {
|
||||||
|
const diagnostics = record(item.diagnostics);
|
||||||
|
const problems = Array.isArray(diagnostics.problemResources) ? diagnostics.problemResources : [];
|
||||||
|
const first = problems.find((entry) => typeof entry === "object" && entry !== null && !Array.isArray(entry)) as Record<string, unknown> | undefined;
|
||||||
|
const problemText = Number(diagnostics.problemResourceCount ?? 0) > 0
|
||||||
|
? ` degraded=${diagnostics.problemResourceCount}:${first?.kind ?? "-"} ${first?.namespace ?? "-"}/${first?.name ?? "-"} ${first?.healthStatus ?? first?.status ?? "-"}`
|
||||||
|
: "";
|
||||||
|
return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}${problemText}`;
|
||||||
|
}
|
||||||
if (name === "runtime") {
|
if (name === "runtime") {
|
||||||
const probe = record(item.probe);
|
const probe = record(item.probe);
|
||||||
const deployment = record(probe.deployment);
|
const deployment = record(probe.deployment);
|
||||||
|
|||||||
Reference in New Issue
Block a user