mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: surface HWLAB control-plane TaskRun failures

Browse Source
This commit is contained in:
Codex
2026-06-19 15:42:13 +00:00
parent 20c39269ce
commit 6ca6695c87
2 changed files with 325 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+225 -6
View File
@@ -2586,6 +2586,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds)
: { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 };
const waitedPipelineRun = record(pipelineWait.pipelineRun);
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
const postFlush = waitedPipelineRun.status === "True"
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
: null;
@@ -2600,6 +2601,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
pipelineRun,
before,
pipelineWait,
pipelineFailureSummary,
postFlush,
skipped: true,
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
@@ -2665,6 +2667,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
const pipelineWait = createOk ? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, scoped.timeoutSeconds) : null;
const waitedPipelineRun = record(pipelineWait?.pipelineRun);
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
const postFlush = waitedPipelineRun.status === "True"
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
: null;
@@ -2687,6 +2690,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
create: compactRuntimeCommand(create),
after,
pipelineWait,
pipelineFailureSummary,
postFlush,
createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
degradedReason: ok
@@ -3211,7 +3215,7 @@ function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScoped
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.status === "Unknown"
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
: null;
const workloads = namespaceExists
@@ -3401,9 +3405,13 @@ function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>,
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
podCount: pipelineRunDiagnostics.podCount ?? null,
failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null,
failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [],
failureSummary: pipelineRunDiagnostics.failureSummary ?? null,
pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
next: pipelineRunDiagnostics.next ?? null,
},
},
argo: {
@@ -3463,6 +3471,14 @@ function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: Re
if (reason === "pipelinerun-not-succeeded") {
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
}
if (reason === "node-runtime-ci-step-publish-failed") {
return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`;
}
if (reason === "node-runtime-ci-taskrun-failed") {
const next = record(record(status.pipelineRunDiagnostics).next);
const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null;
return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`;
}
if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
}
@@ -3491,11 +3507,28 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : [];
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : [];
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish");
const unscheduledPods = pods.filter((item) => item.scheduled === false);
const schedulingMessages = unscheduledPods
.map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
.filter((message) => message.length > 0);
const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
const failureSummary = failedTaskRunSummaries.length > 0
? {
failedTaskRunCount: failedTaskRuns.length,
failedStepCount: failedTaskRunSummaries.length,
stepPublishFailureCount: stepPublishFailures.length,
firstFailure: failedTaskRunSummaries[0] ?? null,
stepFailures: failedTaskRunSummaries.slice(0, 8),
nextAction: stepPublishFailures.length > 0
? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix."
: failedTaskRunSummaries.length > 0
? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger."
: null,
}
: null;
return {
ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
pipelineRun,
@@ -3503,6 +3536,10 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
pods,
taskRunCount: taskRuns.length,
podCount: pods.length,
failedTaskRunCount: failedTaskRuns.length,
failedTaskRuns: failedTaskRunSummaries,
stepPublishFailures,
failureSummary,
pendingTaskRuns,
unscheduledPods,
schedulingMessages,
@@ -3510,15 +3547,32 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
? "node-runtime-ci-pod-capacity-exhausted"
: unscheduledPods.length > 0
? "node-runtime-ci-pod-unschedulable"
: pendingTaskRuns.length > 0
? "node-runtime-ci-taskrun-pending"
: undefined,
: stepPublishFailures.length > 0
? "node-runtime-ci-step-publish-failed"
: failedTaskRunSummaries.length > 0
? "node-runtime-ci-taskrun-failed"
: pendingTaskRuns.length > 0
? "node-runtime-ci-taskrun-pending"
: undefined,
query: {
taskRuns: compactRuntimeCommand(taskRunsResult),
pods: compactRuntimeCommand(podsResult),
},
next: tooManyPods || unscheduledPods.length > 0
? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
: stepPublishFailures.length > 0
? {
sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`,
sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`,
failedStepLogs: stepPublishFailures[0]?.logCommand ?? null,
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`,
}
: failedTaskRunSummaries.length > 0
? {
failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null,
failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
}
: undefined,
};
}
@@ -3527,15 +3581,24 @@ function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): A
const items = Array.isArray(json.items) ? json.items.map(record) : [];
return items.map((item) => {
const metadata = record(item.metadata);
const labels = record(metadata.labels);
const spec = record(item.spec);
const taskRef = record(spec.taskRef);
const status = record(item.status);
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
const condition = conditions[0] ?? {};
const steps = nodeRuntimePipelineDiagnosticSteps(status.steps);
const failedSteps = steps.filter((step) => step.failed === true);
return {
name: metadata.name ?? null,
pipelineTask: labels["tekton.dev/pipelineTask"] ?? null,
taskRef: taskRef.name ?? null,
status: condition.status ?? null,
reason: condition.reason ?? null,
message: condition.message ?? null,
message: diagnosticText(condition.message),
podName: status.podName ?? null,
steps,
failedSteps,
};
});
}
@@ -3544,21 +3607,169 @@ function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array
const items = Array.isArray(json.items) ? json.items.map(record) : [];
return items.map((item) => {
const metadata = record(item.metadata);
const labels = record(metadata.labels);
const spec = record(item.spec);
const status = record(item.status);
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses);
const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses);
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
return {
name: metadata.name ?? null,
taskRun: labels["tekton.dev/taskRun"] ?? null,
phase: status.phase ?? null,
nodeName: spec.nodeName ?? null,
scheduled: scheduled === undefined ? null : scheduled.status === "True",
scheduledReason: scheduled?.reason ?? null,
scheduledMessage: scheduled?.message ?? null,
scheduledMessage: diagnosticText(scheduled?.message),
initContainers,
containers,
failedContainers,
};
});
}
function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array<Record<string, unknown>> {
const steps = Array.isArray(value) ? value.map(record) : [];
return steps.map((step) => {
const terminated = record(step.terminated);
const running = record(step.running);
const waiting = record(step.waiting);
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
return {
name: step.name ?? null,
container: step.container ?? null,
state,
failed: exitCode !== null && exitCode !== 0,
exitCode,
reason: terminatedReason ?? waitingReason,
message: diagnosticText(terminated.message ?? waiting.message),
startedAt: terminated.startedAt ?? running.startedAt ?? null,
finishedAt: terminated.finishedAt ?? null,
};
});
}
function nodeRuntimeContainerStatusSummaries(value: unknown): Array<Record<string, unknown>> {
const containers = Array.isArray(value) ? value.map(record) : [];
return containers.map((container) => {
const state = record(container.state);
const terminated = record(state.terminated);
const waiting = record(state.waiting);
const running = record(state.running);
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
return {
name: container.name ?? null,
ready: container.ready === true,
restartCount: typeof container.restartCount === "number" ? container.restartCount : null,
state: stateName,
failed: exitCode !== null && exitCode !== 0,
exitCode,
reason: terminatedReason ?? waitingReason,
message: diagnosticText(terminated.message ?? waiting.message),
startedAt: terminated.startedAt ?? running.startedAt ?? null,
finishedAt: terminated.finishedAt ?? null,
};
});
}
function nodeRuntimePipelineFailedTaskRunSummaries(
spec: HwlabRuntimeLaneSpec,
failedTaskRuns: Array<Record<string, unknown>>,
pods: Array<Record<string, unknown>>,
): Array<Record<string, unknown>> {
const summaries: Array<Record<string, unknown>> = [];
for (const taskRun of failedTaskRuns) {
const taskRunName = stringOrNull(taskRun.name);
const podName = stringOrNull(taskRun.podName);
const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {};
const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : [];
const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : [];
const failures = failedSteps.length > 0
? failedSteps
: failedContainers.map((container) => ({
name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null,
container: container.name ?? null,
state: container.state ?? null,
exitCode: container.exitCode ?? null,
reason: container.reason ?? null,
message: container.message ?? null,
}));
const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }];
for (const failure of effectiveFailures) {
const stepName = stringOrNull(failure.name);
const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`);
summaries.push({
taskRun: taskRunName,
pipelineTask: taskRun.pipelineTask ?? null,
taskRef: taskRun.taskRef ?? null,
taskRunStatus: taskRun.status ?? null,
taskRunReason: taskRun.reason ?? null,
taskRunMessage: diagnosticText(taskRun.message),
pod: podName,
podPhase: pod.phase ?? null,
nodeName: pod.nodeName ?? null,
step: stepName,
container: containerName,
containerState: failure.state ?? null,
terminationReason: failure.reason ?? null,
exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null,
message: diagnosticText(failure.message),
logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName),
taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]),
taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]),
podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]),
});
}
}
return summaries.slice(0, 16);
}
function nodeRuntimePipelineFailureSummary(value: unknown): Record<string, unknown> | null {
const recordValue = record(value);
const direct = record(recordValue.failureSummary);
if (Object.keys(direct).length > 0) return direct;
const diagnostics = record(recordValue.diagnostics);
const fromDiagnostics = record(diagnostics.failureSummary);
return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null;
}
function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string {
return nodeRuntimeK3sCommand(spec, [
"logs",
"--namespace", HWLAB_CI_NAMESPACE,
"--pod", podName,
...(containerName === null ? ["--all-containers"] : ["--container", containerName]),
"--tail", "200",
]);
}
function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string {
return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" ");
}
function stringOrNull(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
function diagnosticText(value: unknown): string | null {
if (typeof value !== "string") return null;
const trimmed = value.trim();
if (trimmed.length === 0) return null;
return trimmed
.replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://<redacted>")
.replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=<redacted>")
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer <redacted>")
.slice(0, 600);
}
function nodeRuntimeRenderToken(): string {
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
}
@@ -4970,11 +5181,15 @@ function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipel
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
if (status === "True" || status === "False") {
const ok = status === "True";
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: ok ? "succeeded" : "failed", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
return {
ok,
status: ok ? "succeeded" : "failed",
pipelineRun: last,
diagnostics,
failureSummary,
polls,
elapsedMs: Date.now() - startedAt,
degradedReason: ok ? undefined : "node-runtime-pipelinerun-failed",
@@ -4983,10 +5198,14 @@ function waitForNodeRuntimePipelineRunTerminal(spec: HwlabRuntimeLaneSpec, pipel
sleepSync(Math.min(10_000, Math.max(1000, deadline - Date.now())));
}
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "timeout", pipelineRun, polls, elapsedMs: Date.now() - startedAt });
const diagnostics = nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);
const failureSummary = nodeRuntimePipelineFailureSummary(diagnostics);
return {
ok: false,
status: "timeout",
pipelineRun: last,
diagnostics,
failureSummary,
polls,
elapsedMs: Date.now() - startedAt,
degradedReason: "node-runtime-pipelinerun-wait-timeout",