Codex
895 lines
49 KiB
TypeScript
895 lines
49 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. status module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:3281-3999 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, hwlabRuntimeSourceSnapshotStageRefPrefix, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { NodeRuntimeGitMirrorTargetSpec } from "./entry";
|
|
import { isCommandSuccess, runNodeK3sArgs, runNodeK3sScript, shortValue } from "./cleanup";
|
|
import { parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { nodeRuntimeGitMirrorGithubTransportSummary, nodeRuntimeGitMirrorJobManifest, nodeRuntimeGitMirrorJobName } from "./render";
|
|
import { compactRuntimeCommand, nodeRuntimeGitMirrorFlushPartialSuccess, nodeRuntimeGitMirrorRefSources, nodeRuntimeUnsupportedAction } from "./runtime-common";
|
|
import { parseJsonObject, record, shellQuote, statusText } from "./utils";
|
|
import { webObserveTable } from "./web-observe-render";
|
|
import { nodeRuntimeGitMirrorTarget, printNodeRuntimeTriggerProgress, sleepSync } from "./web-probe";
|
|
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
|
|
|
type NodeScopedDelegatedOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
|
|
export type NodeRuntimeGitMirrorRunOptions = {
|
|
maxAttempts?: number;
|
|
deadlineMs?: number;
|
|
};
|
|
|
|
export function nodeRuntimeExternalPostgresSecretRows(secrets: Record<string, unknown>): unknown[][] {
|
|
if (secrets.required === false) return [["not-required", "-", "-", "-", "false"]];
|
|
if (secrets.required === true && secrets.ready !== true && secrets.degradedReason !== undefined) {
|
|
return [[webObserveText(secrets.degradedReason), "-", "-", "false", "false"]];
|
|
}
|
|
const cloudApi = record(secrets.cloudApi);
|
|
const openfga = record(secrets.openfga);
|
|
const datastoreUri = record(openfga.datastoreUri);
|
|
const authnPresharedKey = record(openfga.authnPresharedKey);
|
|
return [
|
|
["cloudApi", webObserveText(cloudApi.secret), webObserveText(cloudApi.key), webObserveText(cloudApi.present), webObserveText(cloudApi.valuesPrinted ?? false)],
|
|
["openfga.datastoreUri", webObserveText(datastoreUri.secret), webObserveText(datastoreUri.key), webObserveText(datastoreUri.present), webObserveText(datastoreUri.valuesPrinted ?? false)],
|
|
["openfga.authnPresharedKey", webObserveText(authnPresharedKey.secret), webObserveText(authnPresharedKey.key), webObserveText(authnPresharedKey.present), webObserveText(authnPresharedKey.valuesPrinted ?? false)],
|
|
];
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const script = [
|
|
"set +e",
|
|
`namespace=${shellQuote(mirror.namespace)}`,
|
|
`read_deploy=${shellQuote(mirror.serviceReadName)}`,
|
|
`write_deploy=${shellQuote(mirror.serviceWriteName)}`,
|
|
`read_svc=${shellQuote(mirror.serviceReadName)}`,
|
|
`write_svc=${shellQuote(mirror.serviceWriteName)}`,
|
|
`cache_pvc=${shellQuote(mirror.cachePvcName)}`,
|
|
`cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`,
|
|
`source_repository=${shellQuote(mirror.sourceRepository)}`,
|
|
`source_branch=${shellQuote(mirror.sourceBranch)}`,
|
|
`source_stage_ref_prefix=${shellQuote(hwlabRuntimeSourceSnapshotStageRefPrefix(spec))}`,
|
|
`gitops_branch=${shellQuote(mirror.gitopsBranch)}`,
|
|
`github_transport_mode=${shellQuote(mirror.githubTransport.mode)}`,
|
|
`github_ssh_secret=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.secretName : "")}`,
|
|
`github_ssh_private_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.privateKeySecretKey : "")}`,
|
|
`github_ssh_private_source_ref=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.privateKeySourceRef : "")}`,
|
|
`github_ssh_private_source_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.privateKeySourceKey : "")}`,
|
|
`github_ssh_known_hosts_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.knownHostsSecretKey ?? "" : "")}`,
|
|
`github_ssh_known_hosts_source_ref=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.knownHostsSourceRef ?? "" : "")}`,
|
|
`github_ssh_known_hosts_source_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.knownHostsSourceKey ?? "" : "")}`,
|
|
`github_token_secret=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretName : "")}`,
|
|
`github_token_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretKey : "")}`,
|
|
`github_token_source_ref=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceRef : "")}`,
|
|
`github_token_source_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceKey : "")}`,
|
|
"deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }",
|
|
"exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }",
|
|
"endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }",
|
|
"github_transport_json=$(python3 - \"$github_transport_mode\" \"$namespace\" \"$github_ssh_secret\" \"$github_ssh_private_key\" \"$github_ssh_private_source_ref\" \"$github_ssh_private_source_key\" \"$github_ssh_known_hosts_key\" \"$github_ssh_known_hosts_source_ref\" \"$github_ssh_known_hosts_source_key\" \"$github_token_secret\" \"$github_token_key\" \"$github_token_source_ref\" \"$github_token_source_key\" <<'PY'",
|
|
"import hashlib, json, subprocess, sys",
|
|
"mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14]",
|
|
"def read_secret(name):",
|
|
" proc = subprocess.run(['kubectl', '-n', namespace, 'get', 'secret', name, '-o', 'json'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)",
|
|
" if proc.returncode != 0:",
|
|
" return False, {}, {}",
|
|
" try:",
|
|
" obj = json.loads(proc.stdout)",
|
|
" except Exception:",
|
|
" obj = {}",
|
|
" return True, obj.get('data') or {}, obj.get('metadata', {}).get('annotations') or {}",
|
|
"def fingerprint(value):",
|
|
" return 'sha256:' + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None",
|
|
"if mode == 'ssh':",
|
|
" exists, data, annotations = read_secret(ssh_secret)",
|
|
" private_encoded = data.get(ssh_private_key) if isinstance(data, dict) else None",
|
|
" known_hosts_encoded = data.get(ssh_known_hosts_key) if ssh_known_hosts_key and isinstance(data, dict) else None",
|
|
" private_present = isinstance(private_encoded, str) and len(private_encoded) > 0",
|
|
" known_hosts_expected = bool(ssh_known_hosts_key)",
|
|
" known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0",
|
|
" print(json.dumps({'mode': mode, 'required': True, 'ready': exists and private_present and (not known_hosts_expected or known_hosts_present), 'secretName': ssh_secret, 'privateKeySecretKey': ssh_private_key, 'privateKeySourceRef': ssh_private_source_ref, 'privateKeySourceKey': ssh_private_source_key, 'privateKeySecretExists': exists, 'privateKeyPresent': private_present, 'privateKeyBytes': len(private_encoded) if private_present else 0, 'privateKeyFingerprint': annotations.get('unidesk.ai/private-key-fingerprint') or fingerprint(private_encoded), 'knownHostsSecretKey': ssh_known_hosts_key or None, 'knownHostsSourceRef': ssh_known_hosts_source_ref or None, 'knownHostsSourceKey': ssh_known_hosts_source_key or None, 'knownHostsPresent': (known_hosts_present if known_hosts_expected else None), 'knownHostsBytes': (len(known_hosts_encoded) if known_hosts_present else 0) if known_hosts_expected else None, 'knownHostsFingerprint': annotations.get('unidesk.ai/known-hosts-fingerprint') or fingerprint(known_hosts_encoded), 'valuesPrinted': False}))",
|
|
" raise SystemExit(0)",
|
|
"if mode != 'https':",
|
|
" print(json.dumps({'mode': mode, 'required': True, 'ready': False, 'valuesPrinted': False}))",
|
|
" raise SystemExit(0)",
|
|
"exists, data, _ = read_secret(token_secret)",
|
|
"encoded = data.get(token_key) if isinstance(data, dict) else None",
|
|
"present = isinstance(encoded, str) and len(encoded) > 0",
|
|
"print(json.dumps({'mode': mode, 'required': True, 'ready': exists and present, 'tokenSecretName': token_secret, 'tokenSecretKey': token_key, 'tokenSourceRef': token_source_ref, 'tokenSourceKey': token_source_key, 'tokenSecretExists': exists, 'tokenKeyPresent': present, 'tokenKeyBytes': len(encoded) if present else 0, 'tokenFingerprint': fingerprint(encoded), 'valuesPrinted': False}))",
|
|
"PY",
|
|
")",
|
|
"summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)",
|
|
"if ! SUMMARY_JSON=\"$summary_json\" python3 - <<'PY'; then",
|
|
"import json, os, sys",
|
|
"try:",
|
|
" value = json.loads(os.environ.get('SUMMARY_JSON') or '{}')",
|
|
"except Exception:",
|
|
" value = {}",
|
|
"sys.exit(0 if isinstance(value, dict) and value.get('localSource') else 1)",
|
|
"PY",
|
|
" summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 source_stage_ref_prefix=\\$3 gitops_branch=\\$4 node <<'NODE'",
|
|
"const { execFileSync } = require('node:child_process');",
|
|
"const { readFileSync, existsSync } = require('node:fs');",
|
|
"const repository = process.env.source_repository;",
|
|
"const sourceBranch = process.env.source_branch;",
|
|
"const sourceStageRefPrefix = process.env.source_stage_ref_prefix;",
|
|
"const gitopsBranch = process.env.gitops_branch;",
|
|
"const repoPath = '/cache/' + repository + '.git';",
|
|
"function readJson(path) { try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; } catch { return null; } }",
|
|
"function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }",
|
|
"const localSource = rev('refs/heads/' + sourceBranch);",
|
|
"const githubSource = rev('refs/mirror-stage/heads/' + sourceBranch);",
|
|
"const sourceStageRef = githubSource ? sourceStageRefPrefix.replace(/\\/+$/, '') + '/' + githubSource : null;",
|
|
"const sourceSnapshot = sourceStageRef ? rev(sourceStageRef) : null;",
|
|
"const localGitops = rev('refs/heads/' + gitopsBranch);",
|
|
"const githubGitops = rev('refs/mirror-stage/heads/' + gitopsBranch);",
|
|
"const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));",
|
|
"console.log(JSON.stringify({",
|
|
" localSource, githubSource, sourceAuthority: 'git-mirror-snapshot', sourceStageRef, sourceSnapshot, localGitops, githubGitops,",
|
|
" refSources: {",
|
|
" localSource: 'refs/heads/' + sourceBranch,",
|
|
" githubSource: 'refs/mirror-stage/heads/' + sourceBranch,",
|
|
" sourceSnapshot: sourceStageRef,",
|
|
" localGitops: 'refs/heads/' + gitopsBranch,",
|
|
" githubGitops: 'refs/mirror-stage/heads/' + gitopsBranch,",
|
|
" githubFieldsAreMirrorStageCache: true",
|
|
" },",
|
|
" pendingFlush,",
|
|
" flushNeeded: pendingFlush,",
|
|
" githubInSync: Boolean(localSource && githubSource && localSource === githubSource && sourceSnapshot === githubSource && localGitops && githubGitops && localGitops === githubGitops),",
|
|
" statusSource: 'cache-ref-fallback',",
|
|
" lastSync: readJson('/cache/HWLAB.last-sync.json'),",
|
|
" lastFlush: readJson('/cache/HWLAB.last-flush.json')",
|
|
"}));",
|
|
"NODE",
|
|
"\" sh \"$source_repository\" \"$source_branch\" \"$source_stage_ref_prefix\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)",
|
|
"fi",
|
|
"if [ -z \"$summary_json\" ]; then summary_json='{}'; fi",
|
|
"read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")",
|
|
"write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")",
|
|
"read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")",
|
|
"write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")",
|
|
"read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")",
|
|
"write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")",
|
|
"cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")",
|
|
"cache_host_path_exists=false",
|
|
"if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi",
|
|
"SUMMARY_JSON=\"$summary_json\" GITHUB_TRANSPORT_JSON=\"$github_transport_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" python3 - <<'PY'",
|
|
"import json, os",
|
|
"def load_env_json(name):",
|
|
" try:",
|
|
" value = json.loads(os.environ.get(name) or '{}')",
|
|
" return value if isinstance(value, dict) else {}",
|
|
" except Exception:",
|
|
" return {}",
|
|
"def truth(name):",
|
|
" return os.environ.get(name) == 'true'",
|
|
"summary = load_env_json('SUMMARY_JSON')",
|
|
"github_transport = load_env_json('GITHUB_TRANSPORT_JSON')",
|
|
"source_snapshot_ready = bool(summary.get('githubSource')) and summary.get('sourceSnapshot') == summary.get('githubSource')",
|
|
"ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource')) and source_snapshot_ready",
|
|
"print(json.dumps({",
|
|
" 'ok': bool(ok),",
|
|
" 'resources': {",
|
|
" 'readDeploymentReady': truth('read_deployment_ready'),",
|
|
" 'writeDeploymentReady': truth('write_deployment_ready'),",
|
|
" 'readServiceExists': truth('read_service_exists'),",
|
|
" 'writeServiceExists': truth('write_service_exists'),",
|
|
" 'readEndpointsReady': truth('read_endpoints_ready'),",
|
|
" 'writeEndpointsReady': truth('write_endpoints_ready'),",
|
|
" 'cachePvcExists': truth('cache_pvc_exists'),",
|
|
" 'cacheHostPathConfigured': bool(os.environ.get('cache_host_path')),",
|
|
" 'cacheHostPathExists': truth('cache_host_path_exists'),",
|
|
" },",
|
|
" 'githubTransport': github_transport,",
|
|
" 'summary': summary,",
|
|
" 'valuesPrinted': False,",
|
|
"}))",
|
|
"PY",
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
|
|
const parsed = record(parseJsonObject(statusText(result)));
|
|
const summary = record(parsed.summary);
|
|
const resources = record(parsed.resources);
|
|
const githubTransport = record(parsed.githubTransport);
|
|
const ok = result.exitCode === 0 && parsed.ok === true;
|
|
const remoteText = `${result.stderr ?? ""}\n${result.stdout ?? ""}`;
|
|
const remoteUnavailable = /provider is not online|UNIDESK_SSH_PROVIDER_OFFLINE|ssh.*unreachable/iu.test(remoteText);
|
|
return {
|
|
ok,
|
|
command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: "status",
|
|
mutation: false,
|
|
namespace: mirror.namespace,
|
|
readUrl: spec.gitReadUrl,
|
|
writeUrl: spec.gitWriteUrl,
|
|
sourceBranch: mirror.sourceBranch,
|
|
gitopsBranch: mirror.gitopsBranch,
|
|
resources,
|
|
githubTransport,
|
|
summary,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
result: compactRuntimeCommand(result),
|
|
degradedReason: ok ? undefined : remoteUnavailable ? "node-provider-unavailable" : "node-runtime-git-mirror-not-ready",
|
|
valuesPrinted: false,
|
|
next: {
|
|
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorRun(scoped: NodeScopedDelegatedOptions, options: NodeRuntimeGitMirrorRunOptions = {}): Record<string, unknown> {
|
|
if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped);
|
|
if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
|
|
const spec = scoped.spec;
|
|
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
|
const retryMaxAttempts = options.maxAttempts ?? (!scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1);
|
|
const attempts: Record<string, unknown>[] = [];
|
|
let finalAttempt: {
|
|
attempt: number;
|
|
retryLabel: string;
|
|
jobName: string;
|
|
manifest: Record<string, unknown>;
|
|
result: CommandResult;
|
|
partialSuccess: Record<string, unknown> | null;
|
|
retryableFailure: Record<string, unknown> | null;
|
|
actionSucceeded: boolean;
|
|
} | null = null;
|
|
|
|
for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) {
|
|
const attemptScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs, 5);
|
|
if (attemptScoped === null) break;
|
|
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
|
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
|
|
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName, {
|
|
discardStaleGitops: scoped.discardStaleGitops === true,
|
|
});
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
const waitTimeoutSeconds = Math.max(1, Math.min(45, Math.max(1, attemptScoped.timeoutSeconds - 10)));
|
|
const script = [
|
|
"set -eu",
|
|
`namespace=${shellQuote(mirror.namespace)}`,
|
|
`job=${shellQuote(jobName)}`,
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
"manifest_path=\"/tmp/$job.json\"",
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
scoped.dryRun
|
|
? "kubectl create --dry-run=server -f \"$manifest_path\" -o name"
|
|
: [
|
|
"kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null",
|
|
"kubectl create -f \"$manifest_path\"",
|
|
`deadline=$(( $(date +%s) + ${waitTimeoutSeconds} ))`,
|
|
"while :; do",
|
|
" status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)",
|
|
" succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')",
|
|
" failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')",
|
|
" if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi",
|
|
" if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi",
|
|
" if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi",
|
|
" sleep 2",
|
|
"done",
|
|
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
|
|
].join("\n"),
|
|
].join("\n");
|
|
const result = runNodeK3sScript(spec, script, attemptScoped.timeoutSeconds);
|
|
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
|
|
const actionSucceeded = isCommandSuccess(result);
|
|
const retryableFailure = !actionSucceeded
|
|
? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess, attempt, retryMaxAttempts)
|
|
: null;
|
|
const retryable = retryableFailure?.retryable === true;
|
|
const exhausted = retryable && attempt >= retryMaxAttempts;
|
|
attempts.push({
|
|
attempt,
|
|
retryLabel,
|
|
jobName,
|
|
ok: actionSucceeded,
|
|
exitCode: result.exitCode,
|
|
retryable,
|
|
exhausted,
|
|
partialSuccess: partialSuccess?.partialSuccess ?? null,
|
|
degradedReason: partialSuccess !== null
|
|
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
|
: actionSucceeded ? null : `node-runtime-git-mirror-${scoped.action}-failed`,
|
|
result: compactRuntimeCommand(result),
|
|
valuesPrinted: false,
|
|
});
|
|
finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded };
|
|
if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break;
|
|
const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt);
|
|
if (options.deadlineMs !== undefined && Date.now() + backoffMs >= options.deadlineMs) break;
|
|
printNodeRuntimeTriggerProgress(spec, {
|
|
stage: `git-mirror-${scoped.action}-retry`,
|
|
status: "waiting",
|
|
retryLabel,
|
|
nextRetryLabel: `${attempt + 1}/${retryMaxAttempts}`,
|
|
backoffMs,
|
|
jobName,
|
|
});
|
|
sleepSync(backoffMs);
|
|
}
|
|
|
|
if (finalAttempt === null) {
|
|
return nodeRuntimeGitMirrorBudgetExhausted(scoped);
|
|
}
|
|
const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt;
|
|
const statusScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs, 5);
|
|
const status = scoped.dryRun || !actionSucceeded || statusScoped === null
|
|
? undefined
|
|
: nodeRuntimeGitMirrorStatus({ ...statusScoped, action: "status", dryRun: true, confirm: false });
|
|
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
|
|
const stopped = !actionSucceeded && (retryExhausted || retryableFailure?.stopped === true || retryableFailure === null);
|
|
return {
|
|
ok: actionSucceeded && (status === undefined || status.ok === true),
|
|
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`,
|
|
mutation: !scoped.dryRun && actionSucceeded,
|
|
namespace: mirror.namespace,
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
jobName,
|
|
manifest: scoped.dryRun ? manifest : undefined,
|
|
result: compactRuntimeCommand(result),
|
|
status,
|
|
statusSkippedReason: statusScoped === null && !scoped.dryRun && actionSucceeded ? "node-runtime-cicd-budget-exhausted" : undefined,
|
|
retry: retryMaxAttempts > 1 ? {
|
|
policy: "exponential-backoff",
|
|
maxAttempts: retryMaxAttempts,
|
|
attempts,
|
|
exhausted: retryExhausted,
|
|
stopped,
|
|
valuesPrinted: false,
|
|
} : undefined,
|
|
retryableFailure: retryableFailure ?? undefined,
|
|
partialSuccess: partialSuccess?.partialSuccess ?? undefined,
|
|
recovery: partialSuccess ?? undefined,
|
|
degradedReason: partialSuccess !== null
|
|
? "node-runtime-git-mirror-flush-post-push-fetch-failed"
|
|
: actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`,
|
|
next: partialSuccess !== null
|
|
? partialSuccess.next
|
|
: scoped.dryRun
|
|
? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` }
|
|
: retryableFailure?.next ?? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
function nodeRuntimeScopedWithinDeadline(scoped: NodeScopedDelegatedOptions, deadlineMs?: number, minRemainingSeconds = 5): NodeScopedDelegatedOptions | null {
|
|
if (deadlineMs === undefined) return scoped;
|
|
const remainingSeconds = Math.floor((deadlineMs - Date.now()) / 1000);
|
|
if (remainingSeconds < minRemainingSeconds) return null;
|
|
return { ...scoped, timeoutSeconds: Math.max(1, Math.min(scoped.timeoutSeconds, remainingSeconds)) };
|
|
}
|
|
|
|
function nodeRuntimeGitMirrorBudgetExhausted(scoped: NodeScopedDelegatedOptions): Record<string, unknown> {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
mode: `confirmed-${scoped.action}`,
|
|
mutation: false,
|
|
status: "pending",
|
|
degradedReason: "node-runtime-cicd-budget-exhausted",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorRetryableFailure(
|
|
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
|
|
mirror: NodeRuntimeGitMirrorTargetSpec,
|
|
result: CommandResult,
|
|
partialSuccess: Record<string, unknown> | null,
|
|
attempt: number,
|
|
retryMaxAttempts: number,
|
|
): Record<string, unknown> | null {
|
|
const text = `${result.stdout}\n${result.stderr}`;
|
|
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
|
|
const sshProxyBannerFailure = mirror.egressProxy.mode !== "direct" && /Connection timed out during banner exchange|Connection to UNKNOWN port 65535 timed out|kex_exchange_identification|Connection closed by remote host/iu.test(text);
|
|
const waitTimeoutFailure = result.exitCode === 45
|
|
|| result.exitCode === 124
|
|
|| /UNIDESK_SSH_RUNTIME_TIMEOUT|ssh-runtime-timeout|ssh\/tran operation exceeded|job,pod/iu.test(text);
|
|
const scriptTransportMismatch =
|
|
(mirror.githubTransport.mode === "ssh" && /transport=https[\s\S]*https auth: missing GITHUB_TOKEN/iu.test(text))
|
|
|| (mirror.githubTransport.mode === "https" && /transport=ssh\b/iu.test(text));
|
|
if (scriptTransportMismatch) {
|
|
return {
|
|
retryable: false,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
|
retryExhausted: false,
|
|
stopped: true,
|
|
reason: `Git mirror script transport does not match YAML githubTransport.mode=${mirror.githubTransport.mode}; apply the node control-plane so the git-mirror ConfigMap is updated before retrying.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: {
|
|
applyControlPlane: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text);
|
|
if (authFailure) {
|
|
const authFix = mirror.githubTransport.mode === "ssh"
|
|
? {
|
|
fixSecret: `apply the YAML-declared SSH private key Secret ${mirror.secretName} through the node control-plane infra apply path`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
}
|
|
: {
|
|
fixSecret: "apply the YAML-declared githubTransport token source through the node control-plane infra apply path",
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
};
|
|
return {
|
|
retryable: false,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel: `${attempt}/${retryMaxAttempts}`,
|
|
retryExhausted: false,
|
|
stopped: true,
|
|
reason: `Git mirror GitHub ${mirror.githubTransport.mode} authentication failed. This is not retryable; fix the YAML-declared ${mirror.githubTransport.mode === "ssh" ? "SSH key Secret" : "token source/Secret"} instead of consuming retry budget.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: authFix,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const retryable = partialSuccess !== null
|
|
|| proxyConnectFailure
|
|
|| sshProxyBannerFailure
|
|
|| waitTimeoutFailure
|
|
|| /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text);
|
|
if (!retryable) return null;
|
|
const retryLabel = `${attempt}/${retryMaxAttempts}`;
|
|
const exhausted = attempt >= retryMaxAttempts;
|
|
return {
|
|
retryable: true,
|
|
retryAttempt: attempt,
|
|
retryMaxAttempts,
|
|
retryLabel,
|
|
retryExhausted: exhausted,
|
|
stopped: exhausted,
|
|
backoffPolicy: "exponential",
|
|
nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt),
|
|
reason: partialSuccess !== null
|
|
? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback."
|
|
: waitTimeoutFailure
|
|
? "Git mirror job wait exceeded the controlled short-connection budget. Standard git-mirror retries with exponential backoff and stops when retry budget is exhausted."
|
|
: proxyConnectFailure || sshProxyBannerFailure
|
|
? "Git mirror job hit a retryable YAML-first SSH-over-proxy failure. Standard git-mirror keeps using the configured proxy and stops when retry budget is exhausted."
|
|
: `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`,
|
|
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
|
|
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
|
|
next: exhausted
|
|
? {
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
stopped: "retry budget exhausted; standard git-mirror stopped instead of silently continuing",
|
|
}
|
|
: {
|
|
retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorRetryDelayMs(attempt: number): number {
|
|
return Math.min(15_000, 1000 * (2 ** Math.max(0, attempt - 1)));
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null {
|
|
const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60);
|
|
const value = result.stdout.trim();
|
|
if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null;
|
|
return `http://${value}/${mirror.sourceRepository}.git`;
|
|
}
|
|
|
|
export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: NodeScopedDelegatedOptions, sourceCommit: string, pipelineRun: string, options: NodeRuntimeGitMirrorRunOptions = {}): Record<string, unknown> {
|
|
const full = nodeScopedFullOutput(scoped);
|
|
const beforeScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
|
|
if (beforeScoped === null) {
|
|
return {
|
|
ok: false,
|
|
mode: "budget-exhausted-before-status",
|
|
sourceCommit,
|
|
status: "pending",
|
|
degradedReason: "node-runtime-cicd-budget-exhausted",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const before = nodeRuntimeGitMirrorStatus({ ...beforeScoped, action: "status", dryRun: true, confirm: false });
|
|
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
|
|
const beforeGithubTransport = record(before.githubTransport);
|
|
if (beforeGithubTransport.required === true && beforeGithubTransport.ready === false) {
|
|
return {
|
|
ok: false,
|
|
mode: "github-transport-auth-secret-not-ready",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
githubTransport: beforeGithubTransport,
|
|
degradedReason: "node-runtime-git-mirror-auth-secret-not-ready",
|
|
next: {
|
|
applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) {
|
|
const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before, options);
|
|
return {
|
|
ok: flush.ok === true,
|
|
mode: "already-current",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
flush,
|
|
degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed",
|
|
};
|
|
}
|
|
const syncScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
|
|
if (syncScoped === null) {
|
|
return {
|
|
ok: false,
|
|
mode: "budget-exhausted-before-sync",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
status: "pending",
|
|
degradedReason: "node-runtime-cicd-budget-exhausted",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const sync = nodeRuntimeGitMirrorRun({
|
|
...syncScoped,
|
|
domain: "git-mirror",
|
|
action: "sync",
|
|
confirm: true,
|
|
dryRun: false,
|
|
wait: true,
|
|
discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
|
|
}, options);
|
|
const after = record(sync.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit;
|
|
const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after, options) : null;
|
|
const ok = sourceOk && (flush === null || flush.ok === true);
|
|
return {
|
|
ok,
|
|
mode: "synced-before-trigger",
|
|
sourceCommit,
|
|
beforeSummary,
|
|
before: full ? before : undefined,
|
|
sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync),
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
after: full ? sync.status ?? null : undefined,
|
|
flush,
|
|
degradedReason: ok
|
|
? undefined
|
|
: sourceOk
|
|
? "node-runtime-git-mirror-pre-flush-failed"
|
|
: "node-runtime-git-mirror-local-source-not-current-after-sync",
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeEnsureGitMirrorFlushed(
|
|
scoped: NodeScopedDelegatedOptions,
|
|
phase: "pre" | "post" | "parallel",
|
|
sourceCommit: string,
|
|
pipelineRun: string | null,
|
|
statusInput: Record<string, unknown> | null = null,
|
|
options: NodeRuntimeGitMirrorRunOptions = {},
|
|
): Record<string, unknown> {
|
|
const stage = `git-mirror-${phase}-flush`;
|
|
const full = nodeScopedFullOutput(scoped);
|
|
const beforeScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
|
|
if (beforeScoped === null) {
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
mode: "budget-exhausted-before-status",
|
|
executed: false,
|
|
status: "pending",
|
|
degradedReason: "node-runtime-cicd-budget-exhausted",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...beforeScoped, action: "status", dryRun: true, confirm: false });
|
|
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
|
|
if (before.ok !== true) {
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary });
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
mode: "status-failed",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
degradedReason: `node-runtime-git-mirror-${phase}-status-failed`,
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const flushNeeded = nodeRuntimeGitMirrorNeedsFlush(before);
|
|
if (!flushNeeded) {
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "skipped", sourceCommit, pipelineRun, flushNeeded: false, ...beforeSummary });
|
|
return {
|
|
ok: true,
|
|
phase,
|
|
mode: "already-flushed",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
after: full ? before : undefined,
|
|
afterSummary: beforeSummary,
|
|
};
|
|
}
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary });
|
|
const flushScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
|
|
if (flushScoped === null) {
|
|
return {
|
|
ok: false,
|
|
phase,
|
|
mode: "budget-exhausted-before-flush",
|
|
executed: false,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
status: "pending",
|
|
degradedReason: "node-runtime-cicd-budget-exhausted",
|
|
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
|
|
};
|
|
}
|
|
const flush = nodeRuntimeGitMirrorRun({ ...flushScoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true }, options);
|
|
const after = record(flush.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after);
|
|
printNodeRuntimeTriggerProgress(scoped.spec, {
|
|
stage,
|
|
status: ok ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
flushNeeded: true,
|
|
jobName: flush.jobName ?? null,
|
|
...afterSummary,
|
|
});
|
|
return {
|
|
ok,
|
|
phase,
|
|
mode: "flushed",
|
|
executed: true,
|
|
before: full ? before : undefined,
|
|
beforeSummary,
|
|
flush: full ? flush : compactNodeRuntimeGitMirrorRun(flush),
|
|
jobName: flush.jobName ?? null,
|
|
after: full ? (Object.keys(after).length > 0 ? after : null) : undefined,
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
degradedReason: ok ? undefined : `node-runtime-git-mirror-${phase}-flush-failed`,
|
|
next: ok ? undefined : flush.next ?? { flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeOpportunisticGitMirrorSync(
|
|
scoped: NodeScopedDelegatedOptions,
|
|
sourceCommit: string,
|
|
pipelineRun: string,
|
|
options: NodeRuntimeGitMirrorRunOptions = {},
|
|
): Record<string, unknown> | null {
|
|
const stage = "git-mirror-parallel-sync";
|
|
const syncScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
|
|
if (syncScoped === null) return null;
|
|
const full = nodeScopedFullOutput(scoped);
|
|
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun });
|
|
const sync = nodeRuntimeGitMirrorRun({ ...syncScoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }, options);
|
|
const after = record(sync.status);
|
|
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
|
|
const ok = sync.ok === true;
|
|
printNodeRuntimeTriggerProgress(scoped.spec, {
|
|
stage,
|
|
status: ok ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
jobName: sync.jobName ?? null,
|
|
...afterSummary,
|
|
});
|
|
return {
|
|
ok,
|
|
phase: "parallel",
|
|
mode: "synced-during-pipelinerun-wait",
|
|
executed: true,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync),
|
|
jobName: sync.jobName ?? null,
|
|
after: full ? (Object.keys(after).length > 0 ? after : null) : undefined,
|
|
afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null,
|
|
degradedReason: ok ? undefined : "node-runtime-git-mirror-parallel-sync-failed",
|
|
next: ok ? undefined : sync.next ?? { sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` },
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeOpportunisticGitMirrorFlush(
|
|
scoped: NodeScopedDelegatedOptions,
|
|
sourceCommit: string,
|
|
pipelineRun: string,
|
|
options: NodeRuntimeGitMirrorRunOptions = {},
|
|
): Record<string, unknown> | null {
|
|
const statusScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
|
|
if (statusScoped === null) return null;
|
|
const status = nodeRuntimeGitMirrorStatus({ ...statusScoped, action: "status", dryRun: true, confirm: false });
|
|
if (status.ok !== true || !nodeRuntimeGitMirrorNeedsFlush(status)) return null;
|
|
return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status, options);
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorNeedsFlush(status: Record<string, unknown>): boolean {
|
|
const summary = record(status.summary);
|
|
const localGitops = typeof summary.localGitops === "string" ? summary.localGitops : null;
|
|
const githubGitops = typeof summary.githubGitops === "string" ? summary.githubGitops : null;
|
|
return summary.pendingFlush === true
|
|
|| summary.flushNeeded === true
|
|
|| (localGitops !== null && githubGitops !== null && localGitops !== githubGitops);
|
|
}
|
|
|
|
export function nodeScopedFullOutput(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): boolean {
|
|
return scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw");
|
|
}
|
|
|
|
export function compactNodeRuntimeGitMirrorObservation(status: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
ok: status.ok === true,
|
|
mode: status.mode ?? null,
|
|
mutation: status.mutation === true,
|
|
summary: compactNodeRuntimeGitMirrorStatus(status),
|
|
degradedReason: status.degradedReason ?? null,
|
|
next: status.next ?? null,
|
|
};
|
|
}
|
|
|
|
export function compactNodeRuntimeGitMirrorRun(result: Record<string, unknown>): Record<string, unknown> {
|
|
const status = record(result.status);
|
|
return {
|
|
ok: result.ok === true,
|
|
action: result.action ?? null,
|
|
mode: result.mode ?? null,
|
|
mutation: result.mutation === true,
|
|
jobName: result.jobName ?? null,
|
|
partialSuccess: result.partialSuccess ?? null,
|
|
fallback: result.fallback ?? null,
|
|
retry: result.retry ?? null,
|
|
githubTransport: result.githubTransport ?? null,
|
|
retryableFailure: result.retryableFailure ?? null,
|
|
degradedReason: result.degradedReason ?? null,
|
|
statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null,
|
|
next: result.next ?? null,
|
|
};
|
|
}
|
|
|
|
export function withNodeRuntimeGitMirrorRendered(result: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): RenderedCliResult {
|
|
const status = record(result.status);
|
|
const summary = scoped.action === "status" ? record(result.summary) : record(status.summary);
|
|
const retry = record(result.retry);
|
|
const retryAttempts = Array.isArray(retry.attempts) ? retry.attempts.map(record).filter((item): item is Record<string, unknown> => item !== null) : [];
|
|
const retryText = typeof retry.maxAttempts === "number"
|
|
? `${retryAttempts.length}/${retry.maxAttempts}${retry.exhausted === true ? " exhausted" : ""}`
|
|
: "-";
|
|
const transport = record(result.githubTransport);
|
|
const commandResult = record(result.result);
|
|
const retryableFailure = record(result.retryableFailure);
|
|
const next = record(result.next);
|
|
const resultStatus = result.ok === true ? "ok" : result.ok === false ? "failed" : "-";
|
|
const lines = [
|
|
`hwlab nodes git-mirror ${scoped.action}`,
|
|
"",
|
|
webObserveTable(
|
|
["NODE", "LANE", "ACTION", "STATUS", "MODE", "JOB", "RETRY"],
|
|
[[
|
|
scoped.node,
|
|
scoped.lane,
|
|
scoped.action,
|
|
resultStatus,
|
|
webObserveText(result.mode),
|
|
webObserveShort(webObserveText(result.jobName), 36),
|
|
retryText,
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(summary).length === 0
|
|
? "REFS\n-"
|
|
: webObserveTable(
|
|
["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"],
|
|
[[
|
|
shortValue(summary.localSource),
|
|
shortValue(summary.githubSource),
|
|
shortValue(summary.sourceSnapshot),
|
|
shortValue(summary.localGitops),
|
|
shortValue(summary.githubGitops),
|
|
webObserveText(summary.pendingFlush),
|
|
webObserveText(summary.githubInSync),
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(transport).length === 0
|
|
? "TRANSPORT\n-"
|
|
: webObserveTable(
|
|
["MODE", "READY", "REQUIRED"],
|
|
[[webObserveText(transport.mode), webObserveText(transport.ready), webObserveText(transport.required)]],
|
|
),
|
|
"",
|
|
Object.keys(commandResult).length === 0
|
|
? "RESULT\n-"
|
|
: webObserveTable(
|
|
["EXIT", "TIMED_OUT", "STDOUT_BYTES", "STDERR_BYTES"],
|
|
[[
|
|
webObserveText(commandResult.exitCode),
|
|
webObserveText(commandResult.timedOut),
|
|
webObserveText(commandResult.stdoutBytes),
|
|
webObserveText(commandResult.stderrBytes),
|
|
]],
|
|
),
|
|
"",
|
|
Object.keys(retryableFailure).length === 0
|
|
? "RETRYABLE\n-"
|
|
: webObserveTable(
|
|
["RETRYABLE", "EXHAUSTED", "REASON"],
|
|
[[
|
|
webObserveText(retryableFailure.retryable),
|
|
webObserveText(retryableFailure.exhausted),
|
|
webObserveShort(webObserveText(retryableFailure.reason ?? retryableFailure.degradedReason), 100),
|
|
]],
|
|
),
|
|
"",
|
|
"NEXT",
|
|
` ${next.status ?? next.sync ?? next.flush ?? next.retry ?? next.run ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`}`,
|
|
"",
|
|
"Disclosure:",
|
|
" default view is a bounded git-mirror summary; use --full or --raw for the complete JSON payload.",
|
|
];
|
|
return { ...result, renderedText: lines.join("\n"), contentType: "text/plain" };
|
|
}
|
|
|
|
export function compactNodeRuntimeGitMirrorStatus(status: Record<string, unknown>): Record<string, unknown> {
|
|
const summary = record(status.summary);
|
|
return {
|
|
ok: status.ok === true,
|
|
localSource: summary.localSource ?? null,
|
|
githubSource: summary.githubSource ?? null,
|
|
sourceAuthority: summary.sourceAuthority ?? null,
|
|
sourceStageRef: summary.sourceStageRef ?? null,
|
|
sourceSnapshot: summary.sourceSnapshot ?? null,
|
|
sourceSnapshotReady: Boolean(summary.githubSource && summary.sourceSnapshot === summary.githubSource),
|
|
localGitops: summary.localGitops ?? null,
|
|
githubGitops: summary.githubGitops ?? null,
|
|
pendingFlush: summary.pendingFlush === true,
|
|
flushNeeded: summary.flushNeeded === true,
|
|
githubInSync: summary.githubInSync === true,
|
|
};
|
|
}
|