fix: warn on slow HWLAB v02 build taskruns
This commit is contained in:
@@ -58,6 +58,8 @@ const DEFAULT_MAX_CYCLES = 0;
|
||||
const DEFAULT_TIMEOUT_SECONDS = 1800;
|
||||
const G14_BRIEF_INDEX_ISSUE = 7;
|
||||
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||||
const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
|
||||
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
|
||||
|
||||
interface G14MonitorOptions {
|
||||
intervalSeconds: number;
|
||||
@@ -840,6 +842,7 @@ function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun:
|
||||
stderr: stderr.trim().slice(0, 2000),
|
||||
counts: { succeeded: 0, failed: 0, running: 0, unknown: 0 },
|
||||
items: [],
|
||||
performance: v02TaskRunPerformanceSummary([]),
|
||||
};
|
||||
}
|
||||
const items = text
|
||||
@@ -863,16 +866,71 @@ function taskRunsCompactFromText(text: string, commandOk: boolean, pipelineRun:
|
||||
running: items.filter((item) => item.status === "Unknown").length,
|
||||
unknown: items.filter((item) => item.status !== "True" && item.status !== "False" && item.status !== "Unknown").length,
|
||||
};
|
||||
const performance = v02TaskRunPerformanceSummary(items);
|
||||
const performanceWarning = performance.ok === false ? `; ${String(performance.summary ?? "")}` : "";
|
||||
return {
|
||||
ok: true,
|
||||
pipelineRun,
|
||||
counts,
|
||||
items,
|
||||
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}`,
|
||||
performance,
|
||||
summary: `taskruns succeeded=${counts.succeeded} failed=${counts.failed} running=${counts.running} unknown=${counts.unknown}${performanceWarning}`,
|
||||
disclosure: items.length > 0 ? "complete taskrun condition summary" : "no taskruns observed yet",
|
||||
};
|
||||
}
|
||||
|
||||
function v02TaskRunPipelineTaskName(name: string): string | null {
|
||||
const buildIndex = name.lastIndexOf("-build-");
|
||||
if (buildIndex >= 0) return name.slice(buildIndex + 1);
|
||||
const knownSuffixes = [
|
||||
"prepare-source",
|
||||
"plan-artifacts",
|
||||
"publish-artifact-catalog",
|
||||
"gitops-render",
|
||||
"gitops-promote",
|
||||
"runtime-ready",
|
||||
"collect-artifacts",
|
||||
];
|
||||
return knownSuffixes.find((suffix) => name.endsWith(`-${suffix}`)) ?? null;
|
||||
}
|
||||
|
||||
export function v02TaskRunPerformanceSummary(taskRuns: unknown[]): Record<string, unknown> {
|
||||
const slowTaskRuns: Record<string, unknown>[] = [];
|
||||
for (const itemRaw of taskRuns) {
|
||||
const item = record(itemRaw);
|
||||
const name = stringOrNull(item.name) ?? "";
|
||||
const pipelineTask = stringOrNull(item.pipelineTask) ?? v02TaskRunPipelineTaskName(name);
|
||||
const durationSeconds = typeof item.durationSeconds === "number" ? item.durationSeconds : null;
|
||||
if (!pipelineTask?.startsWith("build-") || durationSeconds === null || durationSeconds <= V02_BUILD_TASKRUN_WARNING_SECONDS) continue;
|
||||
const serviceId = pipelineTask.slice("build-".length) || null;
|
||||
slowTaskRuns.push({
|
||||
name,
|
||||
pipelineTask,
|
||||
serviceId,
|
||||
status: stringOrNull(item.status),
|
||||
reason: stringOrNull(item.reason),
|
||||
durationSeconds,
|
||||
budgetSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
||||
severity: durationSeconds >= V02_BUILD_TASKRUN_CRITICAL_SECONDS ? "critical" : "warning",
|
||||
message: `${pipelineTask} took ${durationSeconds}s, above v0.2 build TaskRun warning budget ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
||||
});
|
||||
}
|
||||
slowTaskRuns.sort((left, right) => Number(right.durationSeconds ?? 0) - Number(left.durationSeconds ?? 0));
|
||||
const worst = slowTaskRuns[0];
|
||||
return {
|
||||
ok: slowTaskRuns.length === 0,
|
||||
warningCount: slowTaskRuns.length,
|
||||
thresholds: {
|
||||
buildTaskRunWarningSeconds: V02_BUILD_TASKRUN_WARNING_SECONDS,
|
||||
buildTaskRunCriticalSeconds: V02_BUILD_TASKRUN_CRITICAL_SECONDS,
|
||||
},
|
||||
slowTaskRuns,
|
||||
summary: worst
|
||||
? `slow build taskruns=${slowTaskRuns.length}; worst=${String(worst.pipelineTask)} ${String(worst.durationSeconds)}s budget=${V02_BUILD_TASKRUN_WARNING_SECONDS}s`
|
||||
: `no build taskrun over ${V02_BUILD_TASKRUN_WARNING_SECONDS}s`,
|
||||
};
|
||||
}
|
||||
|
||||
function v02WebAssetsFromText(
|
||||
text: string,
|
||||
commandOk: boolean,
|
||||
|
||||
Reference in New Issue
Block a user