mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a93236010ca93d21f1d5104aeb77ed3e9c626feb
pikasTech-unidesk/scripts/src/hwlab-node/control-actions.ts
T
Codex e0af0f85ac refactor: split control-plane cli modules
2026-06-25 16:16:25 +00:00

624 lines
32 KiB
TypeScript
Raw Blame History

// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. control-actions module for scripts/src/hwlab-node-impl.ts.
// Moved mechanically from scripts/src/hwlab-node-impl.ts:2202-2799 for #903.
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
import { createHash, randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";
import { isCommandSuccess, resolveNodeRuntimeLaneHead, runNodeK3sArgs, runNodeK3sScript, shortSha } from "./cleanup";
import { HWLAB_CI_NAMESPACE } from "./entry";
import { nodeRuntimeExpected, parseNodeScopedDelegatedOptions } from "./plan";
import { renderNodeRuntimeControlPlane } from "./render";
import { compactRuntimeCommand, sshTcpPoolDiagnosticsFromCommand } from "./runtime-common";
import { commaListField, keyValueLinesFromText, numericField, shellQuote, statusText } from "./utils";
import { applyLocalNodeRuntimeControlPlaneFiles, applyNodeRuntimeControlPlaneFiles, cleanupLocalNodeRuntimeRenderDir, cleanupNodeRuntimeRenderDir, ensureNodeBaseImage, nodeRuntimeBaseImageStatus, syncNodeExternalPostgresSecrets, syncNodeLocalPostgresBootstrapSecret } from "./web-probe";
export function nodeRuntimeCleanupOwnedTaskRunsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
return text.split(/\r?\n/u).map((line) => {
const [name = "", pipelineRun = "", status = "", reason = ""] = line.trim().split("\t");
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
return {
name,
pipelineRun,
status: status || null,
reason: reason || null,
};
}).filter((item): item is Record<string, unknown> => item !== null);
}
export function nodeRuntimeCleanupOwnedPodsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
return text.split(/\r?\n/u).map((line) => {
const [name = "", pipelineRun = "", phase = "", nodeName = ""] = line.trim().split("\t");
if (name.length === 0 || !wanted.has(pipelineRun)) return null;
return {
name,
pipelineRun,
phase: phase || null,
nodeName: nodeName || null,
};
}).filter((item): item is Record<string, unknown> => item !== null);
}
export function nodeRuntimeCleanupOwnedPvcsFromText(text: string, wanted: Set<string>): Record<string, unknown>[] {
return text.split(/\r?\n/u).map((line) => {
const [name = "", volumeName = "", phase = "", ownerKind = "", ownerName = "", storage = ""] = line.trim().split("\t");
if (name.length === 0 || ownerKind !== "PipelineRun" || !wanted.has(ownerName)) return null;
return {
name,
pipelineRun: ownerName,
phase: phase || null,
volumeName: volumeName || null,
storage: storage || null,
};
}).filter((item): item is Record<string, unknown> => item !== null);
}
export function deleteNodeRuntimeCleanupRuns(spec: HwlabRuntimeLaneSpec, pipelineRunNames: string[], timeoutSeconds: number): CommandResult {
if (pipelineRunNames.length === 0) {
return {
command: [],
cwd: repoRoot,
exitCode: 0,
stdout: "no candidates",
stderr: "",
signal: null,
timedOut: false,
};
}
const script = [
"set -eu",
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
"names_file=$(mktemp)",
"cat > \"$names_file\"",
"deleted_pipeline_runs=$(grep -c . \"$names_file\" | tr -d ' ')",
"xargs -r -n 50 kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=false < \"$names_file\"",
"deleted_pod_groups=0",
"deleted_taskrun_groups=0",
"explicit_owned_cleanup=skipped-large-batch",
"if [ \"$deleted_pipeline_runs\" -le 40 ]; then",
" explicit_owned_cleanup=executed",
" while IFS= read -r pipeline_run; do",
" [ -n \"$pipeline_run\" ] || continue",
" kubectl -n \"$namespace\" delete pod -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" deleted_pod_groups=$((deleted_pod_groups + 1))",
" kubectl -n \"$namespace\" delete taskrun -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true",
" deleted_taskrun_groups=$((deleted_taskrun_groups + 1))",
" done < \"$names_file\"",
"fi",
"printf 'deletedPipelineRunCount\\t%s\\n' \"$deleted_pipeline_runs\"",
"printf 'deletedTaskRunLabelGroups\\t%s\\n' \"$deleted_taskrun_groups\"",
"printf 'deletedPodLabelGroups\\t%s\\n' \"$deleted_pod_groups\"",
"printf 'explicitOwnedCleanup\\t%s\\n' \"$explicit_owned_cleanup\"",
"rm -f \"$names_file\"",
].join("\n");
return runNodeK3sScript(spec, script, timeoutSeconds, pipelineRunNames.join("\n") + "\n");
}
export function nodeRuntimeCiObjectCounts(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const script = [
"set +e",
`namespace=${shellQuote(HWLAB_CI_NAMESPACE)}`,
"count_kind() { kubectl -n \"$namespace\" get \"$1\" -o name 2>/dev/null | wc -l | tr -d ' '; }",
"printf 'pipelineRuns\\t%s\\n' \"$(count_kind pipelinerun)\"",
"printf 'taskRuns\\t%s\\n' \"$(count_kind taskrun)\"",
"printf 'pods\\t%s\\n' \"$(count_kind pod)\"",
"printf 'pvcs\\t%s\\n' \"$(count_kind pvc)\"",
].join("\n");
const result = runNodeK3sScript(spec, script, 30);
const fields = keyValueLinesFromText(statusText(result));
return {
pipelineRuns: numericField(fields.pipelineRuns),
taskRuns: numericField(fields.taskRuns),
pods: numericField(fields.pods),
pvcs: numericField(fields.pvcs),
result: compactRuntimeCommand(result),
};
}
export function nodeRuntimeBaseImageCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const action = scoped.runtimeImageAction;
if (action === null) {
return {
ok: false,
command: `hwlab nodes control-plane runtime-image --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
degradedReason: "node-runtime-image-action-missing",
message: "runtime-image requires one of: status, preload, build",
expected: nodeRuntimeExpected(scoped.spec),
};
}
if (action !== "status" && action !== "preload" && action !== "build") {
return {
ok: false,
command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mutation: false,
degradedReason: "unsupported-node-runtime-image-action",
message: "runtime-image currently supports status/preload/build",
expected: nodeRuntimeExpected(scoped.spec),
};
}
const statusBefore = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds);
if (action === "status") {
return {
ok: statusBefore.ok,
command: `hwlab nodes control-plane runtime-image status --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: "status",
mutation: false,
status: statusBefore,
degradedReason: statusBefore.ok ? undefined : "node-runtime-base-image-not-ready",
next: statusBefore.ok ? undefined : {
preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm`,
},
};
}
if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-image preload/build requires --dry-run or --confirm");
const preload = ensureNodeBaseImage(scoped.spec, scoped.dryRun, scoped.timeoutSeconds);
const statusAfter = nodeRuntimeBaseImageStatus(scoped.spec, scoped.timeoutSeconds);
return {
ok: preload !== null && preload.ok === true && (scoped.dryRun || statusAfter.ok === true),
command: `hwlab nodes control-plane runtime-image ${action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-preload",
requestedAction: action,
effectiveAction: "preload",
mutation: !scoped.dryRun && preload !== null && preload.ok === true,
statusBefore,
preload,
statusAfter,
degradedReason: preload === null
? "node-runtime-base-image-source-missing"
: preload.ok === true && (scoped.dryRun || statusAfter.ok === true)
? undefined
: "node-runtime-base-image-seed-failed",
next: scoped.dryRun
? { preload: `bun scripts/cli.ts hwlab nodes control-plane runtime-image preload --node ${scoped.node} --lane ${scoped.lane} --confirm` }
: { triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun` },
};
}
export function nodeRuntimeMigration(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
if (scoped.allowLiveDbRead && scoped.confirm) throw new Error("control-plane runtime-migration accepts --allow-live-db-read only with dry-run/source-check mode, not --confirm");
if (!scoped.confirm && !scoped.dryRun) throw new Error("control-plane runtime-migration requires --dry-run or --confirm");
const head = resolveNodeRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
if (sourceCommit === null) {
return {
ok: false,
command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "source-head",
mutation: false,
degradedReason: "node-runtime-source-head-unresolved",
headProbe: compactRuntimeCommand(head.result),
};
}
const reportPath = `/tmp/hwlab-${scoped.node.toLowerCase()}-${scoped.lane}-runtime-migration-${shortSha(sourceCommit)}.json`;
const migrationArgs = scoped.dryRun
? [
...(scoped.allowLiveDbRead ? ["--dry-run", "--allow-live-db-read", "--confirm-dev"] : ["--check"]),
"--report",
reportPath,
]
: [
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--report",
reportPath,
];
const result = runNodeK3sArgs(spec, [
"kubectl",
"exec",
"-n",
spec.runtimeNamespace,
"deployment/hwlab-cloud-api",
"-c",
"hwlab-cloud-api",
"--",
"sh",
"-lc",
`cd /workspace/hwlab-boot/repo && exec bun cmd/hwlab-cloud-api/migrate.ts ${migrationArgs.map(shellQuote).join(" ")}`,
], scoped.timeoutSeconds);
const ok = isCommandSuccess(result);
return {
ok,
command: `hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? scoped.allowLiveDbRead ? "live-read-dry-run" : "source-check" : "confirmed-apply",
sourceCommit,
runtimeNamespace: spec.runtimeNamespace,
target: "deployment/hwlab-cloud-api -c hwlab-cloud-api",
workingDirectory: "/workspace/hwlab-boot/repo",
migrationCommand: ["bun", "cmd/hwlab-cloud-api/migrate.ts", ...migrationArgs],
reportPath,
mutation: !scoped.dryRun && ok,
result: compactRuntimeCommand(result),
degradedReason: ok ? undefined : "node-runtime-migration-failed",
next: scoped.dryRun
? {
liveReadDryRun: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --allow-live-db-read --dry-run`,
apply: `bun scripts/cli.ts hwlab nodes control-plane runtime-migration --node ${scoped.node} --lane ${scoped.lane} --confirm`,
}
: {
health: `curl -fsS --max-time 20 ${spec.publicApiUrl}/health/live`,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
},
};
}
export function nodeRuntimeApply(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const head = resolveNodeRuntimeLaneHead(spec);
const sourceCommit = head.sourceCommit;
if (sourceCommit === null) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase: "source-head",
degradedReason: "node-runtime-source-head-unresolved",
headProbe: compactRuntimeCommand(head.result),
};
}
const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
if (localPostgres !== null && localPostgres.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "local-postgres-secret-sync",
sourceCommit,
localPostgres,
degradedReason: "local-postgres-secret-sync-failed",
};
}
const secrets = syncNodeExternalPostgresSecrets(spec, scoped.dryRun, scoped.timeoutSeconds);
if (secrets !== null && secrets.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "external-postgres-secret-sync",
sourceCommit,
localPostgres,
secrets,
degradedReason: "external-postgres-secret-sync-failed",
};
}
const baseImage = ensureNodeBaseImage(spec, scoped.dryRun, scoped.timeoutSeconds);
if (baseImage !== null && baseImage.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "base-image-seed",
sourceCommit,
localPostgres,
secrets,
baseImage,
degradedReason: "node-runtime-base-image-seed-failed",
};
}
const render = renderNodeRuntimeControlPlane(spec, sourceCommit, scoped.timeoutSeconds);
if (!isCommandSuccess(render.result)) {
return {
ok: false,
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
phase: "source-render",
sourceCommit,
localPostgres,
secrets,
baseImage,
renderDir: render.renderDir,
renderLocation: render.location,
render: compactRuntimeCommand(render.result),
degradedReason: "node-runtime-control-plane-render-failed",
};
}
const apply = render.location === "local"
? applyLocalNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds)
: applyNodeRuntimeControlPlaneFiles(spec, render.renderDir, scoped.dryRun, scoped.timeoutSeconds);
const cleanup = render.location === "local"
? cleanupLocalNodeRuntimeRenderDir(spec, render)
: cleanupNodeRuntimeRenderDir(spec, render.renderDir);
const sshTcpPoolDiagnostics = sshTcpPoolDiagnosticsFromCommand(spec, apply);
return {
ok: isCommandSuccess(apply),
command: `hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-apply",
mutation: !scoped.dryRun && isCommandSuccess(apply),
sourceCommit,
expected: nodeRuntimeExpected(spec),
localPostgres,
secrets,
baseImage,
renderDir: render.renderDir,
renderLocation: render.location,
render: compactRuntimeCommand(render.result),
apply: compactRuntimeCommand(apply),
cleanupRenderDir: compactRuntimeCommand(cleanup),
diagnostics: sshTcpPoolDiagnostics,
degradedReason: isCommandSuccess(apply) ? undefined : "node-runtime-control-plane-apply-failed",
next: scoped.dryRun
? { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` }
: {
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
...(sshTcpPoolDiagnostics === null ? {} : { sshPoolStatus: `bun scripts/cli.ts debug ssh-pool ${scoped.node}` }),
},
};
}
export function nodeRuntimeRefresh(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const script = [
"set +e",
`app=${shellQuote(spec.app)}`,
"argo_namespace=argocd",
`dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
"before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
"if [ \"$dry_run\" = true ]; then",
" output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
"else",
" output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
"fi",
"code=$?",
"after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}' 2>/dev/null || true)",
"printf 'app\\t%s\\n' \"$app\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'before\\t%s\\n' \"$before\"",
"printf 'after\\t%s\\n' \"$after\"",
"printf 'annotateExitCode\\t%s\\n' \"$code\"",
"printf 'annotateOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"exit \"$code\"",
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
return {
ok: isCommandSuccess(result),
command: `hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-refresh",
mutation: !scoped.dryRun && isCommandSuccess(result),
argoApplication: fields.app || spec.app,
before: fields.before || null,
after: fields.after || null,
annotateExitCode: numericField(fields.annotateExitCode),
result: compactRuntimeCommand(result),
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
export function nodeRuntimeSync(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const localPostgres = syncNodeLocalPostgresBootstrapSecret(spec, scoped.dryRun, scoped.timeoutSeconds);
if (localPostgres !== null && localPostgres.ok !== true) {
return {
ok: false,
command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
phase: "local-postgres-secret-sync",
localPostgres,
degradedReason: "local-postgres-secret-sync-failed",
next: { apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm` },
};
}
const operation = {
operation: {
initiatedBy: { username: "unidesk-hwlab-node-cli" },
retry: { limit: 1 },
sync: {
prune: true,
revision: spec.gitopsBranch,
source: {
repoURL: spec.argoRepoUrl,
targetRevision: spec.gitopsBranch,
path: spec.runtimePath,
},
syncOptions: ["CreateNamespace=true"],
syncStrategy: { hook: {} },
},
},
};
const patchB64 = Buffer.from(JSON.stringify(operation), "utf8").toString("base64");
const script = [
"set +e",
`app=${shellQuote(spec.app)}`,
"argo_namespace=argocd",
`runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
`dry_run=${shellQuote(scoped.dryRun ? "true" : "false")}`,
`patch_b64=${shellQuote(patchB64)}`,
"terminate_patch_file=$(mktemp /tmp/hwlab-node-argocd-terminate.XXXXXX.json)",
"patch_file=$(mktemp /tmp/hwlab-node-argocd-sync.XXXXXX.json)",
"printf '{\"operation\":null}' > \"$terminate_patch_file\"",
"printf '%s' \"$patch_b64\" | base64 -d > \"$patch_file\"",
"before=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
"operation_phase=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.operationState.phase}' 2>/dev/null || true)",
"terminate_code=0",
"terminate_output=",
"terminated_operation=false",
"if [ \"$operation_phase\" = Running ]; then",
" if [ \"$dry_run\" = true ]; then",
" terminated_operation=would-terminate",
" else",
" terminate_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$terminate_patch_file\" -o name 2>&1)",
" terminate_code=$?",
" if [ \"$terminate_code\" -eq 0 ]; then terminated_operation=true; else terminated_operation=failed; fi",
" sleep 2",
" fi",
"fi",
"failed_hook_count=0",
"deleted_hook_count=0",
"failed_hooks=",
"deleted_hooks=",
"hook_delete_errors=",
"for job in $(kubectl -n \"$runtime_namespace\" get job -o name 2>/dev/null || true); do",
" tracking=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ index .metadata.annotations \"argocd.argoproj.io/tracking-id\" }}' 2>/dev/null || true)",
" failed=$(kubectl -n \"$runtime_namespace\" get \"$job\" -o go-template='{{ range .status.conditions }}{{ if or (eq .type \"Failed\") (eq .type \"FailureTarget\") }}{{ .status }} {{ end }}{{ end }}' 2>/dev/null || true)",
" case \"$tracking\" in \"$app:\"*) ;; *) continue ;; esac",
" case \"$failed\" in *True*) ;; *) continue ;; esac",
" hook_name=${job#job.batch/}",
" failed_hook_count=$((failed_hook_count + 1))",
" if [ -z \"$failed_hooks\" ]; then failed_hooks=$hook_name; else failed_hooks=$failed_hooks,$hook_name; fi",
" if [ \"$dry_run\" = false ]; then",
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$job\" --wait=false 2>&1)",
" delete_code=$?",
" if [ \"$delete_code\" -eq 0 ]; then",
" deleted_hook_count=$((deleted_hook_count + 1))",
" if [ -z \"$deleted_hooks\" ]; then deleted_hooks=$hook_name; else deleted_hooks=$deleted_hooks,$hook_name; fi",
" else",
" if [ -z \"$hook_delete_errors\" ]; then hook_delete_errors=\"$hook_name:$delete_code\"; else hook_delete_errors=\"$hook_delete_errors,$hook_name:$delete_code\"; fi",
" fi",
" fi",
"done",
"stale_statefulset_pod_count=0",
"deleted_stale_statefulset_pod_count=0",
"stale_statefulset_pods=",
"deleted_stale_statefulset_pods=",
"stale_statefulset_pod_delete_errors=",
"for sts in $(kubectl -n \"$runtime_namespace\" get statefulset -o name 2>/dev/null || true); do",
" sts_name=${sts#statefulset.apps/}",
" current_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.currentRevision}' 2>/dev/null || true)",
" update_revision=$(kubectl -n \"$runtime_namespace\" get \"$sts\" -o 'jsonpath={.status.updateRevision}' 2>/dev/null || true)",
" if [ -z \"$current_revision\" ] || [ -z \"$update_revision\" ] || [ \"$current_revision\" = \"$update_revision\" ]; then continue; fi",
" for pod in $(kubectl -n \"$runtime_namespace\" get pod -l \"controller-revision-hash=$current_revision\" -o name 2>/dev/null || true); do",
" pod_name=${pod#pod/}",
" owner=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.metadata.ownerReferences[0].kind}/{.metadata.ownerReferences[0].name}' 2>/dev/null || true)",
" if [ \"$owner\" != \"StatefulSet/$sts_name\" ]; then continue; fi",
" ready=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={.status.conditions[?(@.type==\"Ready\")].status}' 2>/dev/null || true)",
" waiting=$(kubectl -n \"$runtime_namespace\" get \"$pod\" -o 'jsonpath={range .status.containerStatuses[*]}{.state.waiting.reason}{\" \"}{end}' 2>/dev/null || true)",
" case \"$waiting\" in *ImagePullBackOff*|*ErrImagePull*|*CrashLoopBackOff*) ;; *) continue ;; esac",
" if [ \"$ready\" = True ]; then continue; fi",
" stale_statefulset_pod_count=$((stale_statefulset_pod_count + 1))",
" stale_item=\"$sts_name/$pod_name:$waiting\"",
" if [ -z \"$stale_statefulset_pods\" ]; then stale_statefulset_pods=$stale_item; else stale_statefulset_pods=$stale_statefulset_pods,$stale_item; fi",
" if [ \"$dry_run\" = false ]; then",
" delete_output=$(kubectl -n \"$runtime_namespace\" delete \"$pod\" --wait=false 2>&1)",
" delete_code=$?",
" if [ \"$delete_code\" -eq 0 ]; then",
" deleted_stale_statefulset_pod_count=$((deleted_stale_statefulset_pod_count + 1))",
" if [ -z \"$deleted_stale_statefulset_pods\" ]; then deleted_stale_statefulset_pods=$pod_name; else deleted_stale_statefulset_pods=$deleted_stale_statefulset_pods,$pod_name; fi",
" else",
" if [ -z \"$stale_statefulset_pod_delete_errors\" ]; then stale_statefulset_pod_delete_errors=\"$pod_name:$delete_code\"; else stale_statefulset_pod_delete_errors=\"$stale_statefulset_pod_delete_errors,$pod_name:$delete_code\"; fi",
" fi",
" fi",
" done",
"done",
"if [ \"$dry_run\" = true ]; then",
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" --dry-run=server -o name 2>&1)",
"else",
" output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type merge --patch-file \"$patch_file\" -o name 2>&1)",
"fi",
"code=$?",
"after=$(kubectl -n \"$argo_namespace\" get application \"$app\" -o 'jsonpath={.status.sync.status}{\"\\t\"}{.status.health.status}{\"\\t\"}{.status.sync.revision}{\"\\t\"}{.operation.sync.revision}{\"\\t\"}{.status.operationState.phase}{\"\\t\"}{.status.operationState.message}' 2>/dev/null || true)",
"printf 'app\\t%s\\n' \"$app\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'before\\t%s\\n' \"$before\"",
"printf 'after\\t%s\\n' \"$after\"",
"printf 'operationPhase\\t%s\\n' \"$operation_phase\"",
"printf 'terminatedOperation\\t%s\\n' \"$terminated_operation\"",
"printf 'terminateExitCode\\t%s\\n' \"$terminate_code\"",
"printf 'terminateOutput\\t%s\\n' \"$(printf '%s' \"$terminate_output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"printf 'failedHookCount\\t%s\\n' \"$failed_hook_count\"",
"printf 'failedHooks\\t%s\\n' \"$failed_hooks\"",
"printf 'deletedHookCount\\t%s\\n' \"$deleted_hook_count\"",
"printf 'deletedHooks\\t%s\\n' \"$deleted_hooks\"",
"printf 'hookDeleteErrors\\t%s\\n' \"$hook_delete_errors\"",
"printf 'staleStatefulSetPodCount\\t%s\\n' \"$stale_statefulset_pod_count\"",
"printf 'staleStatefulSetPods\\t%s\\n' \"$stale_statefulset_pods\"",
"printf 'deletedStaleStatefulSetPodCount\\t%s\\n' \"$deleted_stale_statefulset_pod_count\"",
"printf 'deletedStaleStatefulSetPods\\t%s\\n' \"$deleted_stale_statefulset_pods\"",
"printf 'staleStatefulSetPodDeleteErrors\\t%s\\n' \"$stale_statefulset_pod_delete_errors\"",
"printf 'patchExitCode\\t%s\\n' \"$code\"",
"printf 'patchOutput\\t%s\\n' \"$(printf '%s' \"$output\" | tr '\\n\\t' ' ' | cut -c1-500)\"",
"rm -f \"$patch_file\" \"$terminate_patch_file\"",
"exit \"$code\"",
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const fields = keyValueLinesFromText(statusText(result));
return {
ok: isCommandSuccess(result),
command: `hwlab nodes control-plane sync --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: scoped.dryRun ? "dry-run" : "confirmed-sync",
mutation: !scoped.dryRun && (isCommandSuccess(result) || localPostgres?.mutation === true),
localPostgres,
argoApplication: fields.app || spec.app,
syncSource: {
repoURL: spec.argoRepoUrl,
targetRevision: spec.gitopsBranch,
path: spec.runtimePath,
},
before: fields.before || null,
after: fields.after || null,
operationRecovery: {
operationPhase: fields.operationPhase || null,
terminatedOperation: fields.terminatedOperation || "false",
terminateExitCode: numericField(fields.terminateExitCode),
terminateOutput: fields.terminateOutput || null,
failedHookCount: numericField(fields.failedHookCount),
failedHooks: commaListField(fields.failedHooks),
deletedHookCount: numericField(fields.deletedHookCount),
deletedHooks: commaListField(fields.deletedHooks),
hookDeleteErrors: fields.hookDeleteErrors || null,
staleStatefulSetPodCount: numericField(fields.staleStatefulSetPodCount),
staleStatefulSetPods: commaListField(fields.staleStatefulSetPods),
deletedStaleStatefulSetPodCount: numericField(fields.deletedStaleStatefulSetPodCount),
deletedStaleStatefulSetPods: commaListField(fields.deletedStaleStatefulSetPods),
staleStatefulSetPodDeleteErrors: fields.staleStatefulSetPodDeleteErrors || null,
},
patchExitCode: numericField(fields.patchExitCode),
patchOutput: fields.patchOutput || null,
result: compactRuntimeCommand(result),
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
Reference in New Issue View Git Blame Copy Permalink