// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. status module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:3281-3999 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { NodeRuntimeGitMirrorTargetSpec } from "./entry"; import { isCommandSuccess, runNodeK3sArgs, runNodeK3sScript, shortValue } from "./cleanup"; import { parseNodeScopedDelegatedOptions } from "./plan"; import { nodeRuntimeGitMirrorGithubTransportSummary, nodeRuntimeGitMirrorJobManifest, nodeRuntimeGitMirrorJobName } from "./render"; import { compactRuntimeCommand, nodeRuntimeGitMirrorFlushPartialSuccess, nodeRuntimeGitMirrorRefSources, nodeRuntimeUnsupportedAction } from "./runtime-common"; import { parseJsonObject, record, shellQuote, statusText } from "./utils"; import { webObserveTable } from "./web-observe-render"; import { nodeRuntimeGitMirrorTarget, printNodeRuntimeTriggerProgress, sleepSync } from "./web-probe"; import { webObserveShort, webObserveText } from "./web-probe-observe"; export function nodeRuntimeExternalPostgresSecretRows(secrets: Record): unknown[][] { if (secrets.required === false) return [["not-required", "-", "-", "-", "false"]]; if (secrets.required === true && secrets.ready !== true && secrets.degradedReason !== undefined) { return [[webObserveText(secrets.degradedReason), "-", "-", "false", "false"]]; } const cloudApi = record(secrets.cloudApi); const openfga = record(secrets.openfga); const datastoreUri = record(openfga.datastoreUri); const authnPresharedKey = record(openfga.authnPresharedKey); return [ ["cloudApi", webObserveText(cloudApi.secret), webObserveText(cloudApi.key), webObserveText(cloudApi.present), webObserveText(cloudApi.valuesPrinted ?? false)], ["openfga.datastoreUri", webObserveText(datastoreUri.secret), webObserveText(datastoreUri.key), webObserveText(datastoreUri.present), webObserveText(datastoreUri.valuesPrinted ?? false)], ["openfga.authnPresharedKey", webObserveText(authnPresharedKey.secret), webObserveText(authnPresharedKey.key), webObserveText(authnPresharedKey.present), webObserveText(authnPresharedKey.valuesPrinted ?? false)], ]; } export function nodeRuntimeGitMirrorStatus(scoped: ReturnType): Record { const spec = scoped.spec; const mirror = nodeRuntimeGitMirrorTarget(spec); const script = [ "set +e", `namespace=${shellQuote(mirror.namespace)}`, `read_deploy=${shellQuote(mirror.serviceReadName)}`, `write_deploy=${shellQuote(mirror.serviceWriteName)}`, `read_svc=${shellQuote(mirror.serviceReadName)}`, `write_svc=${shellQuote(mirror.serviceWriteName)}`, `cache_pvc=${shellQuote(mirror.cachePvcName)}`, `cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`, `source_repository=${shellQuote(mirror.sourceRepository)}`, `source_branch=${shellQuote(mirror.sourceBranch)}`, `gitops_branch=${shellQuote(mirror.gitopsBranch)}`, `github_transport_mode=${shellQuote(mirror.githubTransport.mode)}`, `github_ssh_secret=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.secretName : "")}`, `github_ssh_private_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.privateKeySecretKey : "")}`, `github_ssh_private_source_ref=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.privateKeySourceRef : "")}`, `github_ssh_private_source_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.privateKeySourceKey : "")}`, `github_ssh_known_hosts_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.knownHostsSecretKey ?? "" : "")}`, `github_ssh_known_hosts_source_ref=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.knownHostsSourceRef ?? "" : "")}`, `github_ssh_known_hosts_source_key=${shellQuote(mirror.githubTransport.mode === "ssh" ? mirror.githubTransport.knownHostsSourceKey ?? "" : "")}`, `github_token_secret=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretName : "")}`, `github_token_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSecretKey : "")}`, `github_token_source_ref=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceRef : "")}`, `github_token_source_key=${shellQuote(mirror.githubTransport.mode === "https" ? mirror.githubTransport.tokenSourceKey : "")}`, "deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }", "exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }", "endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }", "github_transport_json=$(python3 - \"$github_transport_mode\" \"$namespace\" \"$github_ssh_secret\" \"$github_ssh_private_key\" \"$github_ssh_private_source_ref\" \"$github_ssh_private_source_key\" \"$github_ssh_known_hosts_key\" \"$github_ssh_known_hosts_source_ref\" \"$github_ssh_known_hosts_source_key\" \"$github_token_secret\" \"$github_token_key\" \"$github_token_source_ref\" \"$github_token_source_key\" <<'PY'", "import hashlib, json, subprocess, sys", "mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14]", "def read_secret(name):", " proc = subprocess.run(['kubectl', '-n', namespace, 'get', 'secret', name, '-o', 'json'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)", " if proc.returncode != 0:", " return False, {}, {}", " try:", " obj = json.loads(proc.stdout)", " except Exception:", " obj = {}", " return True, obj.get('data') or {}, obj.get('metadata', {}).get('annotations') or {}", "def fingerprint(value):", " return 'sha256:' + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None", "if mode == 'ssh':", " exists, data, annotations = read_secret(ssh_secret)", " private_encoded = data.get(ssh_private_key) if isinstance(data, dict) else None", " known_hosts_encoded = data.get(ssh_known_hosts_key) if ssh_known_hosts_key and isinstance(data, dict) else None", " private_present = isinstance(private_encoded, str) and len(private_encoded) > 0", " known_hosts_expected = bool(ssh_known_hosts_key)", " known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0", " print(json.dumps({'mode': mode, 'required': True, 'ready': exists and private_present and (not known_hosts_expected or known_hosts_present), 'secretName': ssh_secret, 'privateKeySecretKey': ssh_private_key, 'privateKeySourceRef': ssh_private_source_ref, 'privateKeySourceKey': ssh_private_source_key, 'privateKeySecretExists': exists, 'privateKeyPresent': private_present, 'privateKeyBytes': len(private_encoded) if private_present else 0, 'privateKeyFingerprint': annotations.get('unidesk.ai/private-key-fingerprint') or fingerprint(private_encoded), 'knownHostsSecretKey': ssh_known_hosts_key or None, 'knownHostsSourceRef': ssh_known_hosts_source_ref or None, 'knownHostsSourceKey': ssh_known_hosts_source_key or None, 'knownHostsPresent': (known_hosts_present if known_hosts_expected else None), 'knownHostsBytes': (len(known_hosts_encoded) if known_hosts_present else 0) if known_hosts_expected else None, 'knownHostsFingerprint': annotations.get('unidesk.ai/known-hosts-fingerprint') or fingerprint(known_hosts_encoded), 'valuesPrinted': False}))", " raise SystemExit(0)", "if mode != 'https':", " print(json.dumps({'mode': mode, 'required': True, 'ready': False, 'valuesPrinted': False}))", " raise SystemExit(0)", "exists, data, _ = read_secret(token_secret)", "encoded = data.get(token_key) if isinstance(data, dict) else None", "present = isinstance(encoded, str) and len(encoded) > 0", "print(json.dumps({'mode': mode, 'required': True, 'ready': exists and present, 'tokenSecretName': token_secret, 'tokenSecretKey': token_key, 'tokenSourceRef': token_source_ref, 'tokenSourceKey': token_source_key, 'tokenSecretExists': exists, 'tokenKeyPresent': present, 'tokenKeyBytes': len(encoded) if present else 0, 'tokenFingerprint': fingerprint(encoded), 'valuesPrinted': False}))", "PY", ")", "summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)", "if ! SUMMARY_JSON=\"$summary_json\" python3 - <<'PY'; then", "import json, os, sys", "try:", " value = json.loads(os.environ.get('SUMMARY_JSON') or '{}')", "except Exception:", " value = {}", "sys.exit(0 if isinstance(value, dict) and value.get('localSource') else 1)", "PY", " summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc \"source_repository=\\$1 source_branch=\\$2 gitops_branch=\\$3 node <<'NODE'", "const { execFileSync } = require('node:child_process');", "const { readFileSync, existsSync } = require('node:fs');", "const repository = process.env.source_repository;", "const sourceBranch = process.env.source_branch;", "const gitopsBranch = process.env.gitops_branch;", "const repoPath = '/cache/' + repository + '.git';", "function readJson(path) { try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; } catch { return null; } }", "function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }", "const localSource = rev('refs/heads/' + sourceBranch);", "const githubSource = rev('refs/mirror-stage/heads/' + sourceBranch);", "const localGitops = rev('refs/heads/' + gitopsBranch);", "const githubGitops = rev('refs/mirror-stage/heads/' + gitopsBranch);", "const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));", "console.log(JSON.stringify({", " localSource, githubSource, localGitops, githubGitops,", " refSources: {", " localSource: 'refs/heads/' + sourceBranch,", " githubSource: 'refs/mirror-stage/heads/' + sourceBranch,", " localGitops: 'refs/heads/' + gitopsBranch,", " githubGitops: 'refs/mirror-stage/heads/' + gitopsBranch,", " githubFieldsAreMirrorStageCache: true", " },", " pendingFlush,", " flushNeeded: pendingFlush,", " githubInSync: Boolean(localSource && githubSource && localSource === githubSource && localGitops && githubGitops && localGitops === githubGitops),", " statusSource: 'cache-ref-fallback',", " lastSync: readJson('/cache/HWLAB.last-sync.json'),", " lastFlush: readJson('/cache/HWLAB.last-flush.json')", "}));", "NODE", "\" sh \"$source_repository\" \"$source_branch\" \"$gitops_branch\" 2>>/tmp/hwlab-node-gitmirror-status.err || true)", "fi", "if [ -z \"$summary_json\" ]; then summary_json='{}'; fi", "read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")", "write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")", "read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")", "write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")", "read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")", "write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")", "cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")", "cache_host_path_exists=false", "if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi", "SUMMARY_JSON=\"$summary_json\" GITHUB_TRANSPORT_JSON=\"$github_transport_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" python3 - <<'PY'", "import json, os", "def load_env_json(name):", " try:", " value = json.loads(os.environ.get(name) or '{}')", " return value if isinstance(value, dict) else {}", " except Exception:", " return {}", "def truth(name):", " return os.environ.get(name) == 'true'", "summary = load_env_json('SUMMARY_JSON')", "github_transport = load_env_json('GITHUB_TRANSPORT_JSON')", "ok = truth('read_deployment_ready') and truth('write_deployment_ready') and truth('read_service_exists') and truth('write_service_exists') and truth('read_endpoints_ready') and truth('write_endpoints_ready') and (truth('cache_pvc_exists') or truth('cache_host_path_exists')) and github_transport.get('ready') is not False and bool(summary.get('localSource'))", "print(json.dumps({", " 'ok': bool(ok),", " 'resources': {", " 'readDeploymentReady': truth('read_deployment_ready'),", " 'writeDeploymentReady': truth('write_deployment_ready'),", " 'readServiceExists': truth('read_service_exists'),", " 'writeServiceExists': truth('write_service_exists'),", " 'readEndpointsReady': truth('read_endpoints_ready'),", " 'writeEndpointsReady': truth('write_endpoints_ready'),", " 'cachePvcExists': truth('cache_pvc_exists'),", " 'cacheHostPathConfigured': bool(os.environ.get('cache_host_path')),", " 'cacheHostPathExists': truth('cache_host_path_exists'),", " },", " 'githubTransport': github_transport,", " 'summary': summary,", " 'valuesPrinted': False,", "}))", "PY", ].join("\n"); const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); const parsed = record(parseJsonObject(statusText(result))); const summary = record(parsed.summary); const resources = record(parsed.resources); const githubTransport = record(parsed.githubTransport); const ok = result.exitCode === 0 && parsed.ok === true; const remoteText = `${result.stderr ?? ""}\n${result.stdout ?? ""}`; const remoteUnavailable = /provider is not online|UNIDESK_SSH_PROVIDER_OFFLINE|ssh.*unreachable/iu.test(remoteText); return { ok, command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: "status", mutation: false, namespace: mirror.namespace, readUrl: spec.gitReadUrl, writeUrl: spec.gitWriteUrl, sourceBranch: mirror.sourceBranch, gitopsBranch: mirror.gitopsBranch, resources, githubTransport, summary, refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), result: compactRuntimeCommand(result), degradedReason: ok ? undefined : remoteUnavailable ? "node-provider-unavailable" : "node-runtime-git-mirror-not-ready", valuesPrinted: false, next: { sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`, flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`, }, }; } export function nodeRuntimeGitMirrorRun(scoped: ReturnType): Record { if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped); if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`); const spec = scoped.spec; const mirror = nodeRuntimeGitMirrorTarget(spec); const retryMaxAttempts = !scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1; const attempts: Record[] = []; let finalAttempt: { attempt: number; retryLabel: string; jobName: string; manifest: Record; result: CommandResult; partialSuccess: Record | null; retryableFailure: Record | null; actionSucceeded: boolean; } | null = null; for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) { const retryLabel = `${attempt}/${retryMaxAttempts}`; const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action); const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName, { discardStaleGitops: scoped.discardStaleGitops === true, }); const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); const waitTimeoutSeconds = Math.max(5, Math.min(45, Math.max(5, scoped.timeoutSeconds - 10))); const script = [ "set -eu", `namespace=${shellQuote(mirror.namespace)}`, `job=${shellQuote(jobName)}`, `manifest_b64=${shellQuote(manifestB64)}`, "manifest_path=\"/tmp/$job.json\"", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", scoped.dryRun ? "kubectl create --dry-run=server -f \"$manifest_path\" -o name" : [ "kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null", "kubectl create -f \"$manifest_path\"", `deadline=$(( $(date +%s) + ${waitTimeoutSeconds} ))`, "while :; do", " status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)", " succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')", " failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')", " if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi", " if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi", " if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi", " sleep 2", "done", "kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true", ].join("\n"), ].join("\n"); const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result); const actionSucceeded = isCommandSuccess(result); const retryableFailure = !actionSucceeded ? nodeRuntimeGitMirrorRetryableFailure(scoped, mirror, result, partialSuccess, attempt, retryMaxAttempts) : null; const retryable = retryableFailure?.retryable === true; const exhausted = retryable && attempt >= retryMaxAttempts; attempts.push({ attempt, retryLabel, jobName, ok: actionSucceeded, exitCode: result.exitCode, retryable, exhausted, partialSuccess: partialSuccess?.partialSuccess ?? null, degradedReason: partialSuccess !== null ? "node-runtime-git-mirror-flush-post-push-fetch-failed" : actionSucceeded ? null : `node-runtime-git-mirror-${scoped.action}-failed`, result: compactRuntimeCommand(result), valuesPrinted: false, }); finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded }; if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break; const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt); printNodeRuntimeTriggerProgress(spec, { stage: `git-mirror-${scoped.action}-retry`, status: "waiting", retryLabel, nextRetryLabel: `${attempt + 1}/${retryMaxAttempts}`, backoffMs, jobName, }); sleepSync(backoffMs); } if (finalAttempt === null) throw new Error("git-mirror run produced no attempts"); const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt; const status = scoped.dryRun || !actionSucceeded ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts; const stopped = !actionSucceeded && (retryExhausted || retryableFailure?.stopped === true || retryableFailure === null); return { ok: actionSucceeded && (status === undefined || status.ok === true), command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, node: scoped.node, lane: scoped.lane, mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`, mutation: !scoped.dryRun && actionSucceeded, namespace: mirror.namespace, githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror), jobName, manifest: scoped.dryRun ? manifest : undefined, result: compactRuntimeCommand(result), status, retry: retryMaxAttempts > 1 ? { policy: "exponential-backoff", maxAttempts: retryMaxAttempts, attempts, exhausted: retryExhausted, stopped, valuesPrinted: false, } : undefined, retryableFailure: retryableFailure ?? undefined, partialSuccess: partialSuccess?.partialSuccess ?? undefined, recovery: partialSuccess ?? undefined, degradedReason: partialSuccess !== null ? "node-runtime-git-mirror-flush-post-push-fetch-failed" : actionSucceeded ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`, next: partialSuccess !== null ? partialSuccess.next : scoped.dryRun ? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` } : retryableFailure?.next ?? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` }, }; } export function nodeRuntimeGitMirrorRetryableFailure( scoped: ReturnType, mirror: NodeRuntimeGitMirrorTargetSpec, result: CommandResult, partialSuccess: Record | null, attempt: number, retryMaxAttempts: number, ): Record | null { const text = `${result.stdout}\n${result.stderr}`; const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text); const sshProxyBannerFailure = mirror.egressProxy.mode !== "direct" && /Connection timed out during banner exchange|Connection to UNKNOWN port 65535 timed out|kex_exchange_identification|Connection closed by remote host/iu.test(text); const waitTimeoutFailure = result.exitCode === 45 || result.exitCode === 124 || /UNIDESK_SSH_RUNTIME_TIMEOUT|ssh-runtime-timeout|ssh\/tran operation exceeded|job,pod/iu.test(text); const scriptTransportMismatch = (mirror.githubTransport.mode === "ssh" && /transport=https[\s\S]*https auth: missing GITHUB_TOKEN/iu.test(text)) || (mirror.githubTransport.mode === "https" && /transport=ssh\b/iu.test(text)); if (scriptTransportMismatch) { return { retryable: false, retryAttempt: attempt, retryMaxAttempts, retryLabel: `${attempt}/${retryMaxAttempts}`, retryExhausted: false, stopped: true, reason: `Git mirror script transport does not match YAML githubTransport.mode=${mirror.githubTransport.mode}; apply the node control-plane so the git-mirror ConfigMap is updated before retrying.`, refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror), next: { applyControlPlane: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, }, valuesPrinted: false, }; } const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text); if (authFailure) { const authFix = mirror.githubTransport.mode === "ssh" ? { fixSecret: `apply the YAML-declared SSH private key Secret ${mirror.secretName} through the node control-plane infra apply path`, status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, } : { fixSecret: "apply the YAML-declared githubTransport token source through the node control-plane infra apply path", status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, }; return { retryable: false, retryAttempt: attempt, retryMaxAttempts, retryLabel: `${attempt}/${retryMaxAttempts}`, retryExhausted: false, stopped: true, reason: `Git mirror GitHub ${mirror.githubTransport.mode} authentication failed. This is not retryable; fix the YAML-declared ${mirror.githubTransport.mode === "ssh" ? "SSH key Secret" : "token source/Secret"} instead of consuming retry budget.`, refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror), next: authFix, valuesPrinted: false, }; } const retryable = partialSuccess !== null || proxyConnectFailure || sshProxyBannerFailure || waitTimeoutFailure || /kex_exchange_identification|Connection closed by remote host|Could not read from remote repository|ssh.github.com|github.com|fetch-pack|early EOF/iu.test(text); if (!retryable) return null; const retryLabel = `${attempt}/${retryMaxAttempts}`; const exhausted = attempt >= retryMaxAttempts; return { retryable: true, retryAttempt: attempt, retryMaxAttempts, retryLabel, retryExhausted: exhausted, stopped: exhausted, backoffPolicy: "exponential", nextRetryDelayMs: exhausted ? null : nodeRuntimeGitMirrorRetryDelayMs(attempt), reason: partialSuccess !== null ? "GitOps push appears to have succeeded, but the post-push fetch/recheck failed. Standard git-mirror stops without host workspace fallback." : waitTimeoutFailure ? "Git mirror job wait exceeded the controlled short-connection budget. Standard git-mirror retries with exponential backoff and stops when retry budget is exhausted." : proxyConnectFailure || sshProxyBannerFailure ? "Git mirror job hit a retryable YAML-first SSH-over-proxy failure. Standard git-mirror keeps using the configured proxy and stops when retry budget is exhausted." : `Git mirror job hit a retryable upstream GitHub ${mirror.githubTransport.mode} transport/fetch failure. Standard git-mirror stops without host workspace fallback.`, refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror), next: exhausted ? { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, stopped: "retry budget exhausted; standard git-mirror stopped instead of silently continuing", } : { retry: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`, status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, }, valuesPrinted: false, }; } export function nodeRuntimeGitMirrorRetryDelayMs(attempt: number): number { return Math.min(15_000, 1000 * (2 ** Math.max(0, attempt - 1))); } export function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mirror: NodeRuntimeGitMirrorTargetSpec): string | null { const result = runNodeK3sArgs(spec, ["kubectl", "-n", mirror.namespace, "get", "service", mirror.serviceWriteName, "-o", "jsonpath={.spec.clusterIP}{\":\"}{.spec.ports[0].port}"], 60); const value = result.stdout.trim(); if (result.exitCode !== 0 || value.length === 0 || value.startsWith(":")) return null; return `http://${value}/${mirror.sourceRepository}.git`; } export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType, sourceCommit: string, pipelineRun: string): Record { const full = nodeScopedFullOutput(scoped); const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); const beforeSummary = compactNodeRuntimeGitMirrorStatus(before); const beforeGithubTransport = record(before.githubTransport); if (beforeGithubTransport.required === true && beforeGithubTransport.ready === false) { return { ok: false, mode: "github-transport-auth-secret-not-ready", sourceCommit, beforeSummary, before: full ? before : undefined, githubTransport: beforeGithubTransport, degradedReason: "node-runtime-git-mirror-auth-secret-not-ready", next: { applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, }, }; } if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) { const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before); return { ok: flush.ok === true, mode: "already-current", sourceCommit, beforeSummary, before: full ? before : undefined, flush, degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed", }; } const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true, discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true, }); const after = record(sync.status); const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {}; const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit; const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after) : null; const ok = sourceOk && (flush === null || flush.ok === true); return { ok, mode: "synced-before-trigger", sourceCommit, beforeSummary, before: full ? before : undefined, sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync), afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null, after: full ? sync.status ?? null : undefined, flush, degradedReason: ok ? undefined : sourceOk ? "node-runtime-git-mirror-pre-flush-failed" : "node-runtime-git-mirror-local-source-not-current-after-sync", }; } export function nodeRuntimeEnsureGitMirrorFlushed( scoped: ReturnType, phase: "pre" | "post" | "parallel", sourceCommit: string, pipelineRun: string | null, statusInput: Record | null = null, ): Record { const stage = `git-mirror-${phase}-flush`; const full = nodeScopedFullOutput(scoped); const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); const beforeSummary = compactNodeRuntimeGitMirrorStatus(before); if (before.ok !== true) { printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary }); return { ok: false, phase, mode: "status-failed", executed: false, before: full ? before : undefined, beforeSummary, degradedReason: `node-runtime-git-mirror-${phase}-status-failed`, next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` }, }; } const flushNeeded = nodeRuntimeGitMirrorNeedsFlush(before); if (!flushNeeded) { printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "skipped", sourceCommit, pipelineRun, flushNeeded: false, ...beforeSummary }); return { ok: true, phase, mode: "already-flushed", executed: false, before: full ? before : undefined, beforeSummary, after: full ? before : undefined, afterSummary: beforeSummary, }; } printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary }); const flush = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true }); const after = record(flush.status); const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {}; const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after); printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: ok ? "succeeded" : "failed", sourceCommit, pipelineRun, flushNeeded: true, jobName: flush.jobName ?? null, ...afterSummary, }); return { ok, phase, mode: "flushed", executed: true, before: full ? before : undefined, beforeSummary, flush: full ? flush : compactNodeRuntimeGitMirrorRun(flush), jobName: flush.jobName ?? null, after: full ? (Object.keys(after).length > 0 ? after : null) : undefined, afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null, degradedReason: ok ? undefined : `node-runtime-git-mirror-${phase}-flush-failed`, next: ok ? undefined : flush.next ?? { flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` }, }; } export function nodeRuntimeOpportunisticGitMirrorSync( scoped: ReturnType, sourceCommit: string, pipelineRun: string, ): Record | null { const stage = "git-mirror-parallel-sync"; const full = nodeScopedFullOutput(scoped); printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun }); const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }); const after = record(sync.status); const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {}; const ok = sync.ok === true; printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: ok ? "succeeded" : "failed", sourceCommit, pipelineRun, jobName: sync.jobName ?? null, ...afterSummary, }); return { ok, phase: "parallel", mode: "synced-during-pipelinerun-wait", executed: true, sourceCommit, pipelineRun, sync: full ? sync : compactNodeRuntimeGitMirrorRun(sync), jobName: sync.jobName ?? null, after: full ? (Object.keys(after).length > 0 ? after : null) : undefined, afterSummary: Object.keys(afterSummary).length > 0 ? afterSummary : null, degradedReason: ok ? undefined : "node-runtime-git-mirror-parallel-sync-failed", next: ok ? undefined : sync.next ?? { sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait` }, }; } export function nodeRuntimeOpportunisticGitMirrorFlush( scoped: ReturnType, sourceCommit: string, pipelineRun: string, ): Record | null { const status = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); if (status.ok !== true || !nodeRuntimeGitMirrorNeedsFlush(status)) return null; return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status); } export function nodeRuntimeGitMirrorNeedsFlush(status: Record): boolean { const summary = record(status.summary); const localGitops = typeof summary.localGitops === "string" ? summary.localGitops : null; const githubGitops = typeof summary.githubGitops === "string" ? summary.githubGitops : null; return summary.pendingFlush === true || summary.flushNeeded === true || (localGitops !== null && githubGitops !== null && localGitops !== githubGitops); } export function nodeScopedFullOutput(scoped: ReturnType): boolean { return scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw"); } export function compactNodeRuntimeGitMirrorObservation(status: Record): Record { return { ok: status.ok === true, mode: status.mode ?? null, mutation: status.mutation === true, summary: compactNodeRuntimeGitMirrorStatus(status), degradedReason: status.degradedReason ?? null, next: status.next ?? null, }; } export function compactNodeRuntimeGitMirrorRun(result: Record): Record { const status = record(result.status); return { ok: result.ok === true, action: result.action ?? null, mode: result.mode ?? null, mutation: result.mutation === true, jobName: result.jobName ?? null, partialSuccess: result.partialSuccess ?? null, fallback: result.fallback ?? null, retry: result.retry ?? null, githubTransport: result.githubTransport ?? null, retryableFailure: result.retryableFailure ?? null, degradedReason: result.degradedReason ?? null, statusSummary: Object.keys(status).length > 0 ? compactNodeRuntimeGitMirrorStatus(status) : null, next: result.next ?? null, }; } export function withNodeRuntimeGitMirrorRendered(result: Record, scoped: ReturnType): RenderedCliResult { const status = record(result.status); const summary = scoped.action === "status" ? record(result.summary) : record(status.summary); const retry = record(result.retry); const retryAttempts = Array.isArray(retry.attempts) ? retry.attempts.map(record).filter((item): item is Record => item !== null) : []; const retryText = typeof retry.maxAttempts === "number" ? `${retryAttempts.length}/${retry.maxAttempts}${retry.exhausted === true ? " exhausted" : ""}` : "-"; const transport = record(result.githubTransport); const commandResult = record(result.result); const retryableFailure = record(result.retryableFailure); const next = record(result.next); const resultStatus = result.ok === true ? "ok" : result.ok === false ? "failed" : "-"; const lines = [ `hwlab nodes git-mirror ${scoped.action}`, "", webObserveTable( ["NODE", "LANE", "ACTION", "STATUS", "MODE", "JOB", "RETRY"], [[ scoped.node, scoped.lane, scoped.action, resultStatus, webObserveText(result.mode), webObserveShort(webObserveText(result.jobName), 36), retryText, ]], ), "", Object.keys(summary).length === 0 ? "REFS\n-" : webObserveTable( ["LOCAL_SOURCE", "GITHUB_SOURCE", "LOCAL_GITOPS", "GITHUB_GITOPS", "PENDING", "IN_SYNC"], [[ shortValue(summary.localSource), shortValue(summary.githubSource), shortValue(summary.localGitops), shortValue(summary.githubGitops), webObserveText(summary.pendingFlush), webObserveText(summary.githubInSync), ]], ), "", Object.keys(transport).length === 0 ? "TRANSPORT\n-" : webObserveTable( ["MODE", "READY", "REQUIRED"], [[webObserveText(transport.mode), webObserveText(transport.ready), webObserveText(transport.required)]], ), "", Object.keys(commandResult).length === 0 ? "RESULT\n-" : webObserveTable( ["EXIT", "TIMED_OUT", "STDOUT_BYTES", "STDERR_BYTES"], [[ webObserveText(commandResult.exitCode), webObserveText(commandResult.timedOut), webObserveText(commandResult.stdoutBytes), webObserveText(commandResult.stderrBytes), ]], ), "", Object.keys(retryableFailure).length === 0 ? "RETRYABLE\n-" : webObserveTable( ["RETRYABLE", "EXHAUSTED", "REASON"], [[ webObserveText(retryableFailure.retryable), webObserveText(retryableFailure.exhausted), webObserveShort(webObserveText(retryableFailure.reason ?? retryableFailure.degradedReason), 100), ]], ), "", "NEXT", ` ${next.status ?? next.sync ?? next.flush ?? next.retry ?? next.run ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`}`, "", "Disclosure:", " default view is a bounded git-mirror summary; use --full or --raw for the complete JSON payload.", ]; return { ...result, renderedText: lines.join("\n"), contentType: "text/plain" }; } export function compactNodeRuntimeGitMirrorStatus(status: Record): Record { const summary = record(status.summary); return { ok: status.ok === true, localSource: summary.localSource ?? null, githubSource: summary.githubSource ?? null, localGitops: summary.localGitops ?? null, githubGitops: summary.githubGitops ?? null, pendingFlush: summary.pendingFlush === true, flushNeeded: summary.flushNeeded === true, githubInSync: summary.githubInSync === true, }; }