Codex
713 lines
38 KiB
TypeScript
713 lines
38 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. status module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:3281-3999 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { NodeRuntimeGitMirrorTargetSpec } from "./entry";
|
|
import { isCommandSuccess, runNodeK3sArgs, runNodeK3sScript, shortValue } from "./cleanup";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { nodeRuntimeGitMirrorGithubTransportSummary, nodeRuntimeGitMirrorJobManifest, nodeRuntimeGitMirrorJobName } from "./render";
|
|
import { compactRuntimeCommand, nodeRuntimeGitMirrorFlushPartialSuccess, nodeRuntimeGitMirrorRefSources, nodeRuntimeUnsupportedAction } from "./runtime-common";
|
|
import { parseJsonObject, record, shellQuote, statusText } from "./utils";
|
|
import { webObserveTable } from "./web-observe-render";
|
|
import { nodeRuntimeGitMirrorTarget, printNodeRuntimeTriggerProgress, sleepSync } from "./web-probe";
|
|
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
|
|
|
export function nodeRuntimeExternalPostgresSecretRows(secrets: Record<string, unknown>): unknown[][] {
|
|
if (secrets.required === false) return [["not-required", "-", "-", "-", "false"]];
|
|
if (secrets.required === true && secrets.ready !== true && secrets.degradedReason !== undefined) {
|
|
return [[webObserveText(secrets.degradedReason), "-", "-", "false", "false"]];
|
|
}
|
|
const cloudApi = record(secrets.cloudApi);
|
|
const openfga = record(secrets.openfga);
|
|
const datastoreUri = record(openfga.datastoreUri);
|
|
const authnPresharedKey = record(openfga.authnPresharedKey);
|
|
return [
|
|
["cloudApi", webObserveText(cloudApi.secret), webObserveText(cloudApi.key), webObserveText(cloudApi.present), webObserveText(cloudApi.valuesPrinted ?? false)],
|
|
["openfga.datastoreUri", webObserveText(datastoreUri.secret), webObserveText(datastoreUri.key), webObserveText(datastoreUri.present), webObserveText(datastoreUri.valuesPrinted ?? false)],
|
|
["openfga.authnPresharedKey", webObserveText(authnPresharedKey.secret), webObserveText(authnPresharedKey.key), webObserveText(authnPresharedKey.present), webObserveText(authnPresharedKey.valuesPrinted ?? false)],
|
|
];
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(mirror.namespace)}`,
|
|
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
|
|
`write_deploy=${shellQuote(mirror.serviceWriteName)}`,
|
|
`read_svc=${shellQuote(mirror.serviceReadName)}`,
|
|
`write_svc=${shellQuote(mirror.serviceWriteName)}`,
|
|
`cache_pvc=${shellQuote(mirror.cachePvcName)}`,
|
|
`cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`,
|
|
`github_transport_mode=${shellQuote(mirror.githubTransport.mode)}`,
|
|
`github_token_secret=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretName : "")}`,
|
|
`github_token_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretKey : "")}`,
|
|
`github_token_source_ref=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceRef : "")}`,
|
|
`github_token_source_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceKey : "")}`,
|
|
"deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }",
|
|
"exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }",
|
|
"endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }",
|
|
"github_transport_json=$(python3 - \"$github_transport_mode\" \"$namespace\" \"$github_token_secret\" \"$github_token_key\" \"$github_token_source_ref\" \"$github_token_source_key\" <<'PY'",
|
|
"import hashlib, json, subprocess, sys",
|
|
"mode, namespace, secret_name, secret_key, source_ref, source_key = sys.argv[1:7]",
|
|
"if mode != 'https':",
|
|
" print(json.dumps({'mode': mode, 'required': False, 'ready': True, 'valuesPrinted': False}))",
|
|
" raise SystemExit(0)",
|
|
"proc = subprocess.run(['kubectl', '-n', namespace, 'get', 'secret', secret_name, '-o', 'json'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)",
|
|
"exists = proc.returncode == 0",
|
|
"data = {}",
|
|
"if exists:",
|
|
" try:",
|
|
" data = json.loads(proc.stdout).get('data') or {}",
|
|
" except Exception:",
|
|
" data = {}",
|
|
"encoded = data.get(secret_key) if isinstance(data, dict) else None",
|
|
"present = isinstance(encoded, str) and len(encoded) > 0",
|
|
"fingerprint = 'sha256:' + hashlib.sha256(encoded.encode()).hexdigest()[:16] if present else None",
|
|
"print(json.dumps({'mode': mode, 'required': True, 'ready': exists and present, 'tokenSecretName': secret_name, 'tokenSecretKey': secret_key, 'tokenSourceRef': source_ref, 'tokenSourceKey': source_key, 'tokenSecretExists': exists, 'tokenKeyPresent': present, 'tokenKeyBytes': len(encoded) if present else 0, 'tokenFingerprint': fingerprint, 'valuesPrinted': False}))",
|
|
"PY",
|
|
")",
|
|
"summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)",
|
|
"if [ -z \"$summary_json\" ]; then summary_json='{}'; fi",
|
|
"read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")",
|
|
"write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")",
|
|
"read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")",
|
|
"write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")",
|
|
"read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")",
|
|
"write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")",
|
|
"cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")",
|
|
"cache_host_path_exists=false",
|
|
"if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi",
|
|
"SUMMARY_JSON=\"$summary_json\" GITHUB_TRANSPORT_JSON=\"$github_transport_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" node <<'NODE'",
|
|
"const summary = (() => { try { return JSON.parse(process.env.SUMMARY_JSON || '{}'); } catch { return {}; } })();",
|
|
"const githubTransport = (() => { try { return JSON.parse(process.env.GITHUB_TRANSPORT_JSON || '{}'); } catch { return {}; } })();",
|
|
"const env = process.env;",
|
|
"const ok = env.read_deployment_ready === 'true' && env.write_deployment_ready === 'true' && env.read_service_exists === 'true' && env.write_service_exists === 'true' && env.read_endpoints_ready === 'true' && env.write_endpoints_ready === 'true' && (env.cache_pvc_exists === 'true' || env.cache_host_path_exists === 'true') && githubTransport.ready !== false && summary.localSource;",
|
|
"console.log(JSON.stringify({",
|
|
" ok: Boolean(ok),",
|
|
" resources: {",
|
|
" readDeploymentReady: env.read_deployment_ready === 'true',",
|
|
" writeDeploymentReady: env.write_deployment_ready === 'true',",
|
|
" readServiceExists: env.read_service_exists === 'true',",
|
|
" writeServiceExists: env.write_service_exists === 'true',",
|
|
" readEndpointsReady: env.read_endpoints_ready === 'true',",
|
|
" writeEndpointsReady: env.write_endpoints_ready === 'true',",
|
|
" cachePvcExists: env.cache_pvc_exists === 'true',",
|
|
" cacheHostPathConfigured: Boolean(env.cache_host_path),",
|
|
" cacheHostPathExists: env.cache_host_path_exists === 'true'",
|
|
" },",
|
|
" githubTransport,",
|
|
" summary,",
|
|
" valuesPrinted: false",
|
|
"}));",
|
|
"NODE",
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const parsed = record(parseJsonObject(statusText(result)));
|
|
const summary = record(parsed.summary);
|
|
const resources = record(parsed.resources);
|
|
const githubTransport = record(parsed.githubTransport);
|
|
const ok = result.exitCode === 0 && parsed.ok === true;
|
|
const remoteText = `${result.stderr ?? ""}\n${result.stdout ?? ""}`;
|
|
const remoteUnavailable = /provider is not online|UNIDESK_SSH_PROVIDER_OFFLINE|ssh.*unreachable/iu.test(remoteText);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "status",
|
|
mutation: false,
|
|
namespace: mirror.namespace,
|
|
readUrl: spec.gitReadUrl,
|
|
writeUrl: spec.gitWriteUrl,
|
|
sourceBranch: mirror.sourceBranch,
|
|
gitopsBranch: mirror.gitopsBranch,
|
|
resources,
|
|
githubTransport,
|
|
summary,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
result: compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : remoteUnavailable ? "node-provider-unavailable" : "node-runtime-git-mirror-not-ready",
|
|
valuesPrinted: false,
|
|
next: {
|
|
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped);
|
|
if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
|
|
const spec = scoped.spec;
|
|
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const retryMaxAttempts = !scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1;
|
|
const attempts: Record<string, unknown>[] = [];
|
|
let finalAttempt: {
|
|
attempt: number;
|
|
retryLabel: string;
|
|
jobName: string;
|
|
manifest: Record<string, unknown>;
|
|
result: CommandResult;
|
|
partialSuccess: Record<string, unknown> | null;
|
|
retryableFailure: Record<string, unknown> | null;
|
|
actionSucceeded: boolean;
|
|
} | null = null;
|
|
|
|
for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) {
|
|
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
|
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
|
|
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName);
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const waitTimeoutSeconds = Math.max(5, Math.min(45, Math.max(5, scoped.timeoutSeconds - 10)));
|
|
const script = [
|
|
"set -eu",
|
|
`namespace=${shellQuote(mirror.namespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
"manifest_path=\"/tmp/$job.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
scoped.dryRun
|
|
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
|
: [
|
|
"kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null",
|
|
"kubectl create -f \"$manifest_path\"",
|
|
`deadline=$(( $(date +%s) + ${waitTimeoutSeconds} ))`,
|
|
"while :; do",
|
|
" status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)",
|
|
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
|
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
|
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
|
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi",
|
|
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi",
|
|
" sleep 2",
|
|
"done",
|
|
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
|
|
].join("\n"),
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
|
const actionSucceeded = isCommandSuccess(result);
|
|
const retryableFailure = !actionSucceeded
|
|
? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess, attempt, retryMaxAttempts)
|
|
: null;
|
|
const retryable = retryableFailure?.retryable === true;
|
|
const exhausted = retryable && attempt >= retryMaxAttempts;
|
|
attempts.push({
|
|
attempt,
|
|
retryLabel,
|
|
jobName,
|
|
ok: actionSucceeded,
|
|
exitCode: result.exitCode,
|
|
retryable,
|
|
exhausted,
|
|
partialSuccess: partialSuccess?.partialSuccess ?? null,
|
|
degradedReason: partialSuccess !== null
|
|
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
|
: actionSucceeded ? null : `node-runtime-git-mirror-${scoped.action}-failed`,
|
|
result: compactRuntimeCommand(result),
|
|
valuesPrinted: false,
|
|
});
|
|
finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded };
|
|
if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break;
|
|
const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt);
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: `git-mirror-${scoped.action}-retry`,
|
|
status: "waiting",
|
|
retryLabel,
|
|
nextRetryLabel: `${attempt + 1}/${retryMaxAttempts}`,
|
|
backoffMs,
|
|
jobName,
|
|
});
|
|
sleepSync(backoffMs);
|
|
}
|
|
|
|
if (finalAttempt === null) throw new Error("git-mirror run produced no attempts");
|
|
const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt;
|
|
const status = scoped.dryRun || !actionSucceeded
|
|
? undefined
|
|
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
|
|
const stopped = !actionSucceeded && (retryExhausted || retryableFailure?.stopped === true || retryableFailure === null);
|
|
return {
|
|
ok: actionSucceeded && (status === undefined || status.ok === true),
|
|
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
|
|
mutation: !scoped.dryRun && actionSucceeded,
|
|
namespace: mirror.namespace,
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
jobName,
|
|
manifest: scoped.dryRun ? manifest : undefined,
|
|
result: compactRuntimeCommand(result),
|
|
status,
|
|
retry: retryMaxAttempts > 1 ? {
|
|
policy: "exponential-backoff",
|
|
maxAttempts: retryMaxAttempts,
|
|
attempts,
|
|
exhausted: retryExhausted,
|
|
stopped,
|
|
valuesPrinted: false,
|
|
} : undefined,
|
|
retryableFailure: retryableFailure ?? undefined,
|
|
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
|
|
recovery: partialSuccess ?? undefined,
|
|
degradedReason: partialSuccess !== null
|
|
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
|
: actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
|
|
next: partialSuccess !== null
|
|
? partialSuccess.next
|
|
: scoped.dryRun
|
|
? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
|
|
: retryableFailure?.next ?? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorRetryableFailure(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
mirror: NodeRuntimeGitMirrorTargetSpec,
|
|
result: CommandResult,
|
|
partialSuccess: Record<string, unknown> | null,
|
|
attempt: number,
|
|
retryMaxAttempts: number,
|
|
): Record<string, unknown> | null {
|
|
const text = `${result.stdout}\n${result.stderr}`;
|
|
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
|
|
const sshProxyBannerFailure = mirror.egressProxy.mode !== "direct" && /Connection timed out during banner exchange|Connection to UNKNOWN port 65535 timed out|kex_exchange_identification|Connection closed by remote host/iu.test(text);
|
|
const waitTimeoutFailure = result.exitCode === 45
|
|
|| result.exitCode === 124
|
|
|| /UNIDESK_SSH_RUNTIME_TIMEOUT|ssh-runtime-timeout|ssh\/tran operation exceeded|job,pod/iu.test(text);
|
|
const scriptTransportMismatch =
|
|
(mirror.githubTransport.mode === "ssh" && /transport=https[\s\S]*https auth: missing GITHUB_TOKEN/iu.test(text))
|
|
|| (mirror.githubTransport.mode === "https" && /transport=ssh\b/iu.test(text));
|
|
if (scriptTransportMismatch) {
|
|
return {
|
|
retryable: false,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
|
retryExhausted: false,
|
|
stopped: true,
|
|
reason: `Git mirror script transport does not match YAML githubTransport.mode=${mirror.githubTransport.mode}; apply the node control-plane so the git-mirror ConfigMap is updated before retrying.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: {
|
|
applyControlPlane: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text);
|
|
if (authFailure) {
|
|
const authFix = mirror.githubTransport.mode === "ssh"
|
|
? {
|
|
fixSecret: `apply the YAML-declared SSH private key Secret ${mirror.secretName} through the node control-plane infra apply path`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
}
|
|
: {
|
|
fixSecret: "apply the YAML-declared githubTransport token source through the node control-plane infra apply path",
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
};
|
|
return {
|
|
retryable: false,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
|
retryExhausted: false,
|
|
stopped: true,
|
|
reason: `Git mirror GitHub ${mirror.githubTransport.mode} authentication failed. This is not retryable; fix the YAML-declared ${mirror.githubTransport.mode === "ssh" ? "SSH key Secret" : "token source/Secret"} instead of consuming retry budget.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: authFix,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const retryable = partialSuccess !== null
|
|
|| proxyConnectFailure
|
|
|| sshProxyBannerFailure
|
|
|| waitTimeoutFailure
|
|
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
|
if (!retryable) return null;
|
|
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
|
const exhausted = attempt >= retryMaxAttempts;
|
|
return {
|
|
retryable: true,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel,
|
|
retryExhausted: exhausted,
|
|
stopped: exhausted,
|
|
backoffPolicy: "exponential",
|
|
nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt),
|
|
reason: partialSuccess !== null
|
|
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
|
: waitTimeoutFailure
|
|
? "Git mirror job wait exceeded the controlled short-connection budget. Standard git-mirror retries with exponential backoff and stops when retry budget is exhausted."
|
|
: proxyConnectFailure || sshProxyBannerFailure
|
|
? "Git mirror job hit a retryable YAML-first SSH-over-proxy failure. Standard git-mirror keeps using the configured node-global proxy and stops when retry budget is exhausted."
|
|
: `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: exhausted
|
|
? {
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
stopped: "retry budget exhausted; standard git-mirror stopped instead of silently continuing",
|
|
}
|
|
: {
|
|
retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorRetryDelayMs(attempt: number): number {
|
|
return Math.min(15_000, 1000 * (2 ** Math.max(0, attempt - 1)));
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60);
|
|
const value = result.stdout.trim();
|
|
if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null;
|
|
return `http://${value}/${mirror.sourceRepository}.git`;
|
|
}
|
|
|
|
export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
|
|
const full = nodeScopedFullOutput(scoped);
|
|
const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
|
|
const beforeGithubTransport = record(before.githubTransport);
|
|
if (beforeGithubTransport.required === true && beforeGithubTransport.ready === false) {
|
|
return {
|
|
ok: false,
|
|
mode: "github-transport-auth-secret-not-ready",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
githubTransport: beforeGithubTransport,
|
|
degradedReason: "node-runtime-git-mirror-auth-secret-not-ready",
|
|
next: {
|
|
applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) {
|
|
const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before);
|
|
return {
|
|
ok: flush.ok === true,
|
|
mode: "already-current",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
flush,
|
|
degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed",
|
|
};
|
|
}
|
|
const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
|
|
const after = record(sync.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit;
|
|
const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after) : null;
|
|
const ok = sourceOk && (flush === null || flush.ok === true);
|
|
return {
|
|
ok,
|
|
mode: "synced-before-trigger",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync),
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
after: full ? sync.status ?? null : undefined,
|
|
flush,
|
|
degradedReason: ok
|
|
? undefined
|
|
: sourceOk
|
|
? "node-runtime-git-mirror-pre-flush-failed"
|
|
: "node-runtime-git-mirror-local-source-not-current-after-sync",
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeEnsureGitMirrorFlushed(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
phase: "pre" | "post" | "parallel",
|
|
sourceCommit: string,
|
|
pipelineRun: string | null,
|
|
statusInput: Record<string, unknown> | null = null,
|
|
): Record<string, unknown> {
|
|
const stage = `git-mirror-${phase}-flush`;
|
|
const full = nodeScopedFullOutput(scoped);
|
|
const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
|
|
if (before.ok !== true) {
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary });
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
mode: "status-failed",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
degradedReason: `node-runtime-git-mirror-${phase}-status-failed`,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const flushNeeded = nodeRuntimeGitMirrorNeedsFlush(before);
|
|
if (!flushNeeded) {
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "skipped", sourceCommit, pipelineRun, flushNeeded: false, ...beforeSummary });
|
|
return {
|
|
ok: true,
|
|
phase,
|
|
mode: "already-flushed",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
after: full ? before : undefined,
|
|
afterSummary: beforeSummary,
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary });
|
|
const flush = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true });
|
|
const after = record(flush.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after);
|
|
printNodeRuntimeTriggerProgress(scoped.spec, {
|
|
stage,
|
|
status: ok ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
flushNeeded: true,
|
|
jobName: flush.jobName ?? null,
|
|
...afterSummary,
|
|
});
|
|
return {
|
|
ok,
|
|
phase,
|
|
mode: "flushed",
|
|
executed: true,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
flush: full ? flush : compactNodeRuntimeGitMirrorRun(flush),
|
|
jobName: flush.jobName ?? null,
|
|
after: full ? (Object.keys(after).length > 0 ? after : null) : undefined,
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
degradedReason: ok ? undefined : `node-runtime-git-mirror-${phase}-flush-failed`,
|
|
next: ok ? undefined : flush.next ?? { flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeOpportunisticGitMirrorSync(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
sourceCommit: string,
|
|
pipelineRun: string,
|
|
): Record<string, unknown> | null {
|
|
const stage = "git-mirror-parallel-sync";
|
|
const full = nodeScopedFullOutput(scoped);
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun });
|
|
const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
|
|
const after = record(sync.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const ok = sync.ok === true;
|
|
printNodeRuntimeTriggerProgress(scoped.spec, {
|
|
stage,
|
|
status: ok ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
jobName: sync.jobName ?? null,
|
|
...afterSummary,
|
|
});
|
|
return {
|
|
ok,
|
|
phase: "parallel",
|
|
mode: "synced-during-pipelinerun-wait",
|
|
executed: true,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync),
|
|
jobName: sync.jobName ?? null,
|
|
after: full ? (Object.keys(after).length > 0 ? after : null) : undefined,
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
degradedReason: ok ? undefined : "node-runtime-git-mirror-parallel-sync-failed",
|
|
next: ok ? undefined : sync.next ?? { sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeOpportunisticGitMirrorFlush(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
sourceCommit: string,
|
|
pipelineRun: string,
|
|
): Record<string, unknown> | null {
|
|
const status = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
|
|
if (status.ok !== true || !nodeRuntimeGitMirrorNeedsFlush(status)) return null;
|
|
return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status);
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorNeedsFlush(status: Record<string, unknown>): boolean {
|
|
const summary = record(status.summary);
|
|
const localGitops = typeof summary.localGitops === "string" ? summary.localGitops : null;
|
|
const githubGitops = typeof summary.githubGitops === "string" ? summary.githubGitops : null;
|
|
return summary.pendingFlush === true
|
|
|| summary.flushNeeded === true
|
|
|| summary.githubInSync === false
|
|
|| (localGitops !== null && githubGitops !== null && localGitops !== githubGitops);
|
|
}
|
|
|
|
export function nodeScopedFullOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): boolean {
|
|
return scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw");
|
|
}
|
|
|
|
export function compactNodeRuntimeGitMirrorObservation(status: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
ok: status.ok === true,
|
|
mode: status.mode ?? null,
|
|
mutation: status.mutation === true,
|
|
summary: compactNodeRuntimeGitMirrorStatus(status),
|
|
degradedReason: status.degradedReason ?? null,
|
|
next: status.next ?? null,
|
|
};
|
|
}
|
|
|
|
export function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record<string, unknown> {
|
|
const status = record(result.status);
|
|
return {
|
|
ok: result.ok === true,
|
|
action: result.action ?? null,
|
|
mode: result.mode ?? null,
|
|
mutation: result.mutation === true,
|
|
jobName: result.jobName ?? null,
|
|
partialSuccess: result.partialSuccess ?? null,
|
|
fallback: result.fallback ?? null,
|
|
retry: result.retry ?? null,
|
|
githubTransport: result.githubTransport ?? null,
|
|
retryableFailure: result.retryableFailure ?? null,
|
|
degradedReason: result.degradedReason ?? null,
|
|
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
|
next: result.next ?? null,
|
|
};
|
|
}
|
|
|
|
export function withNodeRuntimeGitMirrorRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const status = record(result.status);
|
|
const summary = scoped.action === "status" ? record(result.summary) : record(status.summary);
|
|
const retry = record(result.retry);
|
|
const retryAttempts = Array.isArray(retry.attempts) ? retry.attempts.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
|
const retryText = typeof retry.maxAttempts === "number"
|
|
? `${retryAttempts.length}/${retry.maxAttempts}${retry.exhausted === true ? " exhausted" : ""}`
|
|
: "-";
|
|
const transport = record(result.githubTransport);
|
|
const commandResult = record(result.result);
|
|
const retryableFailure = record(result.retryableFailure);
|
|
const next = record(result.next);
|
|
const resultStatus = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
|
const lines = [
|
|
`hwlab nodes git-mirror ${scoped.action}`,
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "ACTION", "STATUS", "MODE", "JOB", "RETRY"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
scoped.action,
|
|
resultStatus,
|
|
webObserveText(result.mode),
|
|
webObserveShort(webObserveText(result.jobName), 36),
|
|
retryText,
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(summary).length === 0
|
|
? "REFS\n-"
|
|
: webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
|
[[
|
|
shortValue(summary.localSource),
|
|
shortValue(summary.githubSource),
|
|
shortValue(summary.localGitops),
|
|
shortValue(summary.githubGitops),
|
|
webObserveText(summary.pendingFlush),
|
|
webObserveText(summary.githubInSync),
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(transport).length === 0
|
|
? "TRANSPORT\n-"
|
|
: webObserveTable(
|
|
["MODE", "READY", "REQUIRED"],
|
|
[[webObserveText(transport.mode), webObserveText(transport.ready), webObserveText(transport.required)]],
|
|
),
|
|
"",
|
|
Object.keys(commandResult).length === 0
|
|
? "RESULT\n-"
|
|
: webObserveTable(
|
|
["EXIT", "TIMED_OUT", "STDOUT_BYTES", "STDERR_BYTES"],
|
|
[[
|
|
webObserveText(commandResult.exitCode),
|
|
webObserveText(commandResult.timedOut),
|
|
webObserveText(commandResult.stdoutBytes),
|
|
webObserveText(commandResult.stderrBytes),
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(retryableFailure).length === 0
|
|
? "RETRYABLE\n-"
|
|
: webObserveTable(
|
|
["RETRYABLE", "EXHAUSTED", "REASON"],
|
|
[[
|
|
webObserveText(retryableFailure.retryable),
|
|
webObserveText(retryableFailure.exhausted),
|
|
webObserveShort(webObserveText(retryableFailure.reason ?? retryableFailure.degradedReason), 100),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? next.sync ?? next.flush ?? next.retry ?? next.run ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded git-mirror summary; use --full or --raw for the complete JSON payload.",
|
|
];
|
|
return { ...result, renderedText: lines.join("\n"), contentType: "text/plain" };
|
|
}
|
|
|
|
export function compactNodeRuntimeGitMirrorStatus(status: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(status.summary);
|
|
return {
|
|
ok: status.ok === true,
|
|
localSource: summary.localSource ?? null,
|
|
githubSource: summary.githubSource ?? null,
|
|
localGitops: summary.localGitops ?? null,
|
|
githubGitops: summary.githubGitops ?? null,
|
|
pendingFlush: summary.pendingFlush === true,
|
|
flushNeeded: summary.flushNeeded === true,
|
|
githubInSync: summary.githubInSync === true,
|
|
};
|
|
}
|