diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 4c359290..7ebd70fd 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -56,7 +56,7 @@ targets: repository: pikasTech/HWLAB branch: v0.3 gitops: - branch: v0.3-d601-gitops + branch: v0.3-gitops path: deploy/gitops/node/d601/runtime-v03 gitMirror: namespace: devops-infra @@ -64,8 +64,9 @@ targets: serviceWriteName: git-mirror-write cachePvcName: hwlab-git-mirror-cache cachePvcStorage: 20Gi + cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d601-v03-git-mirror-cache servicePort: 8080 - deploymentReplicas: 0 + deploymentReplicas: 1 secretName: git-mirror-github-ssh syncConfigMapName: git-mirror-sync-script syncJobPrefix: git-mirror-hwlab-d601-v03-sync-manual diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 44849484..9f22df9d 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -122,6 +122,13 @@ lanes: registryPrefix: 127.0.0.1:5000/hwlab baseImage: 127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim baseImageSource: node:20-bookworm-slim + serviceIds: + - hwlab-cloud-api + - hwlab-cloud-web + - hwlab-gateway + - hwlab-edge-proxy + - hwlab-agent-skills + - hwlab-user-billing buildkit: sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless stepEnv: diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 72b8571d..a4c873b7 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -80,6 +80,7 @@ interface ControlPlaneTargetSpec { serviceWriteName: string; cachePvcName: string; cachePvcStorage: string; + cacheHostPath: string | null; servicePort: number; deploymentReplicas: number; secretName: string; @@ -245,7 +246,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") - && boolField(gitMirror, "cachePvcExists") + && (boolField(gitMirror, "cachePvcExists") || boolField(gitMirror, "cacheHostPathReady")) && boolField(registry, "ready") && boolField(registry, "toolsImageReady") && boolField(argo, "installed") @@ -272,6 +273,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta gitMirrorReadServiceExists: boolField(gitMirror, "readServiceExists"), gitMirrorWriteServiceExists: boolField(gitMirror, "writeServiceExists"), gitMirrorCachePvcExists: boolField(gitMirror, "cachePvcExists"), + gitMirrorCacheHostPathReady: boolField(gitMirror, "cacheHostPathReady"), gitMirrorReadReady: boolField(gitMirror, "readDeploymentReady"), gitMirrorWriteReady: boolField(gitMirror, "writeDeploymentReady"), argoInstalled: boolField(argo, "installed"), @@ -781,6 +783,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa serviceWriteName, cachePvcName: stringField(gitMirror, "cachePvcName", `${path}.gitMirror`), cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), + cacheHostPath: optionalStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`) ?? null, servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), deploymentReplicas: nonNegativeIntegerField(gitMirror, "deploymentReplicas", `${path}.gitMirror`), secretName: stringField(gitMirror, "secretName", `${path}.gitMirror`), @@ -820,22 +823,28 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa } manifests.push( { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: target.tekton.serviceAccountName, namespace: target.ciNamespace, labels } }, - { - apiVersion: "v1", - kind: "PersistentVolumeClaim", - metadata: { name: target.gitMirror.cachePvcName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, - spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: target.gitMirror.cachePvcStorage } } }, - }, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, data: { - "repositories.json": JSON.stringify([{ repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), - "sync.sh": "#!/bin/sh\nset -eu\necho d601-hwlab-git-mirror-sync-placeholder\ncat /etc/git-mirror/repositories.json\n", - "flush.sh": "#!/bin/sh\nset -eu\necho d601-hwlab-git-mirror-flush-placeholder\ncat /etc/git-mirror/repositories.json\n", + "repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), + "server.js": gitMirrorServerJs(), + "status.sh": gitMirrorStatusShell(), + "sync.sh": gitMirrorSyncShell(_node, target), + "flush.sh": gitMirrorFlushShell(_node, target), }, }, + ); + if (target.gitMirror.cacheHostPath === null) { + manifests.push({ + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { name: target.gitMirror.cachePvcName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, + spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: target.gitMirror.cachePvcStorage } } }, + }); + } + manifests.push( service(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), service(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), gitMirrorDeployment(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target, "read"), @@ -888,14 +897,301 @@ function gitMirrorDeployment(name: string, namespace: string, labels: Record= 0 ? crlf + 4 : (lf >= 0 ? lf + 2 : -1); + if (headerEnd < 0) return null; + const headerText = buffer.slice(0, headerEnd).toString('latin1').trim(); + const rest = buffer.slice(headerEnd); + const headers = {}; + let status = 200; + for (const line of headerText.split(/\r?\n/u)) { + const index = line.indexOf(':'); + if (index < 0) continue; + const key = line.slice(0, index).trim(); + const value = line.slice(index + 1).trim(); + if (key.toLowerCase() === 'status') { + const parsed = Number.parseInt(value.split(' ')[0] || '', 10); + if (Number.isInteger(parsed)) status = parsed; + } else { + headers[key] = value; + } + } + return { status, headers, rest }; +} + +function handleGit(req, res) { + const url = new URL(req.url || '/', 'http://git-mirror.local'); + const env = { + ...process.env, + GIT_PROJECT_ROOT: projectRoot, + GIT_HTTP_EXPORT_ALL: '1', + PATH_INFO: decodeURIComponent(url.pathname), + REQUEST_METHOD: req.method || 'GET', + QUERY_STRING: url.search.slice(1), + CONTENT_TYPE: req.headers['content-type'] || '', + CONTENT_LENGTH: req.headers['content-length'] || '', + REMOTE_USER: 'git', + }; + const child = spawn('git', ['http-backend'], { env }); + let pending = Buffer.alloc(0); + let headersSent = false; + child.stderr.on('data', (chunk) => process.stderr.write(chunk)); + child.on('error', (error) => { + if (!headersSent) { + res.writeHead(500, { 'content-type': 'text/plain' }); + headersSent = true; + } + res.end(String(error && error.message || error)); + }); + child.stdout.on('data', (chunk) => { + if (headersSent) { + res.write(chunk); + return; + } + pending = Buffer.concat([pending, chunk]); + const parsed = parseHeaders(pending); + if (!parsed) return; + headersSent = true; + res.writeHead(parsed.status, parsed.headers); + if (parsed.rest.length) res.write(parsed.rest); + }); + child.on('close', (code) => { + if (!headersSent) { + res.writeHead(code === 0 ? 200 : 500, { 'content-type': 'text/plain' }); + headersSent = true; + if (pending.length) res.write(pending); + } + res.end(); + }); + req.pipe(child.stdin); +} + +http.createServer((req, res) => { + if ((req.url || '').startsWith('/healthz')) return sendHealth(res); + return handleGit(req, res); +}).listen(port, '0.0.0.0', () => { + console.log(JSON.stringify({ event: 'git-mirror-http-started', port, projectRoot, mode: process.env.GIT_MIRROR_MODE || null })); +}); +`; +} + +function gitMirrorStatusShell(): string { + return String.raw`#!/bin/sh +set -eu +node <<'NODE' +const { execFileSync } = require('node:child_process'); +const { readFileSync, existsSync } = require('node:fs'); +const repositories = JSON.parse(readFileSync('/etc/git-mirror/repositories.json', 'utf8')); +function readJson(path) { + try { + return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; + } catch { + return null; + } +} +function rev(repo, ref) { + try { + return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); + } catch { + return null; + } +} +const items = {}; +for (const spec of repositories) { + const repoPath = '/cache/' + spec.repository + '.git'; + const localSource = rev(repoPath, 'refs/heads/' + spec.sourceBranch); + const githubSource = rev(repoPath, 'refs/mirror-stage/heads/' + spec.sourceBranch); + const localGitops = rev(repoPath, 'refs/heads/' + spec.gitopsBranch); + const githubGitops = rev(repoPath, 'refs/mirror-stage/heads/' + spec.gitopsBranch); + items[spec.key] = { + repository: spec.repository, + sourceBranch: spec.sourceBranch, + localSource, + githubSource, + gitopsBranch: spec.gitopsBranch, + localGitops, + githubGitops, + sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), + gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), + pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)), + }; +} +const first = items[repositories[0]?.key] || {}; +const pendingFlush = Object.values(items).some((item) => Boolean(item.pendingFlush)); +console.log(JSON.stringify({ + localSource: first.localSource || null, + githubSource: first.githubSource || null, + localGitops: first.localGitops || null, + githubGitops: first.githubGitops || null, + pendingFlush, + flushNeeded: pendingFlush, + githubInSync: Object.values(items).every((item) => item.sourceInSync === true && item.gitopsInSync === true), + repositories: items, + lastSync: readJson('/cache/HWLAB.last-sync.json'), + lastFlush: readJson('/cache/HWLAB.last-flush.json'), +})); +NODE +`; +} + +function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { + const proxyHost = node.egressProxy?.serviceName === undefined ? "127.0.0.1" : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; + const proxyPort = node.egressProxy?.port ?? 10808; + return [ + "mkdir -p /root/.ssh", + "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", + "chmod 0400 /root/.ssh/id_rsa", + "cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'", + "#!/usr/bin/env node", + "const net = require('node:net');", + "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", + "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", + "const targetPort = Number.parseInt(targetPortRaw || '', 10);", + "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);", + "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", + "let buffer = Buffer.alloc(0);", + "socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });", + "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", + "socket.on('error', () => process.exit(66));", + "function onData(chunk) {", + " buffer = Buffer.concat([buffer, chunk]);", + " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", + " if (headerEnd === -1 && buffer.length < 8192) return;", + " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", + " const statusLine = head.split('\\r\\n', 1)[0] || '';", + " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", + " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }", + " socket.off('data', onData);", + " socket.setTimeout(0);", + " const rest = buffer.slice(headerEnd + 4);", + " if (rest.length) process.stdout.write(rest);", + " process.stdin.pipe(socket);", + " socket.pipe(process.stdout);", + "}", + "socket.on('data', onData);", + "socket.on('close', () => process.exit(0));", + "NODE_PROXY", + "chmod 0700 /tmp/hwlab-github-proxy-connect.cjs", + `export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p'`)}`, + `repository=${shQuote(target.source.repository)}`, + `source_branch=${shQuote(target.source.branch)}`, + `gitops_branch=${shQuote(target.gitops.branch)}`, + "repo=\"/cache/${repository}.git\"", + "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", + ].join("\n"); +} + +function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { + return [ + "#!/bin/sh", + "set -eu", + "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", + gitMirrorProxyPrelude(node, target), + "mkdir -p \"$(dirname \"$repo\")\"", + "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", + " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", + "else", + " rm -rf \"$repo\"", + " git init --bare \"$repo\"", + " git --git-dir=\"$repo\" remote add origin \"$remote\"", + "fi", + "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", + "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", + "git --git-dir=\"$repo\" config http.uploadpack true", + "git --git-dir=\"$repo\" config http.receivepack true", + "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", + "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", + "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", + "if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", + " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + " if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then", + " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", + " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", + " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", + " fi", + "fi", + "git --git-dir=\"$repo\" update-server-info", + "export repository source_branch gitops_branch started_at", + "node <<'NODE' | tee /cache/HWLAB.last-sync.json", + "const { execFileSync } = require('node:child_process');", + "const repository = process.env.repository;", + "const sourceBranch = process.env.source_branch;", + "const gitopsBranch = process.env.gitops_branch;", + "const repoPath = `/cache/${repository}.git`;", + "function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }", + "const localSource = rev(`refs/heads/${sourceBranch}`);", + "const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);", + "const localGitops = rev(`refs/heads/${gitopsBranch}`);", + "const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);", + "const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));", + "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush }));", + "NODE", + "cat /cache/HWLAB.last-sync.json", + "", + ].join("\n"); +} + +function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { + return [ + "#!/bin/sh", + "set -eu", + "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", + gitMirrorProxyPrelude(node, target), + "test -d \"$repo/objects\"", + "git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", + "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "if [ -n \"$local_gitops\" ]; then", + " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", + " git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", + "fi", + "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", + "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", + "export repository gitops_branch started_at local_gitops github_gitops pending", + "node <<'NODE' | tee /cache/HWLAB.last-flush.json", + "const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: 'succeeded', startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true' };", + "console.log(JSON.stringify(payload));", + "NODE", + "cat /cache/HWLAB.last-flush.json", + "", + ].join("\n"); +} + function argoDesiredManifest(target: ControlPlaneTargetSpec): Record[] { return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; } @@ -973,6 +1269,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS writeUrl: target.gitMirror.writeUrl, cachePvc: target.gitMirror.cachePvcName, cachePvcStorage: target.gitMirror.cachePvcStorage, + cacheHostPath: target.gitMirror.cacheHostPath, servicePort: target.gitMirror.servicePort, deploymentReplicas: target.gitMirror.deploymentReplicas, syncConfigMap: target.gitMirror.syncConfigMapName, @@ -1020,6 +1317,7 @@ write_deploy=${shQuote(target.gitMirror.serviceWriteName)} read_svc=${shQuote(target.gitMirror.serviceReadName)} write_svc=${shQuote(target.gitMirror.serviceWriteName)} cache_pvc=${shQuote(target.gitMirror.cachePvcName)} +cache_host_path=${shQuote(target.gitMirror.cacheHostPath ?? "")} pipeline=${shQuote(target.tekton.pipelineName)} service_account=${shQuote(target.tekton.serviceAccountName)} argo_ns=${shQuote(target.argo.namespace)} @@ -1042,6 +1340,8 @@ tools_repo=\${tools_repo_tag%:*} tools_tag=\${tools_repo_tag##*:} tools_image_ready=false if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi +cache_host_path_ready=false +if [ -n "$cache_host_path" ] && kubectl -n "$gitmirror_ns" exec deploy/"$read_deploy" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_ready=true; fi python3 - "$required_crds_json" "$argo_deployments_json" "$argo_statefulsets_json" <<'PY' >/tmp/hwlab-node-status-fragments.json import json, subprocess, sys required_crds=json.loads(sys.argv[1]) @@ -1067,7 +1367,7 @@ print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crd PY argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"tekton":{"installed":$(kubectl get crd pipelines.tekton.dev pipelineruns.tekton.dev >/dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} JSON `; } @@ -1139,7 +1439,7 @@ function statusNext( || !boolField(gitMirror, "namespaceExists") || !boolField(gitMirror, "readServiceExists") || !boolField(gitMirror, "writeServiceExists") - || !boolField(gitMirror, "cachePvcExists"); + || (!boolField(gitMirror, "cachePvcExists") && !boolField(gitMirror, "cacheHostPathReady")); const blockers: string[] = []; if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 2e6bf429..8a41f166 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -5,7 +5,7 @@ import { repoRoot, rootPath, type Config } from "./config"; import { runCommand, type CommandResult } from "./command"; import { startJob } from "./jobs"; import { runHwlabG14Command } from "./hwlab-g14"; -import { hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; +import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimePublicExposureSpec } from "./hwlab-node-lanes"; type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; @@ -79,6 +79,24 @@ interface RuntimeSecretSpec { fieldManager: string; } +interface NodeRuntimeGitMirrorTargetSpec { + id: string; + node: string; + lane: string; + namespace: string; + serviceReadName: string; + serviceWriteName: string; + cachePvcName: string; + cacheHostPath: string | null; + secretName: string; + syncJobPrefix: string; + flushJobPrefix: string; + toolsImage: string; + sourceRepository: string; + sourceBranch: string; + gitopsBranch: string; +} + const MASTER_ADMIN_API_KEY_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env"; const MASTER_ADMIN_API_KEY_KEY = "api-key"; const BOOTSTRAP_ADMIN_PASSWORD_HASH_KEY = "password-hash"; @@ -179,16 +197,12 @@ async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomai return nodeRuntimeUnsupportedAction(scoped); } if (domain === "git-mirror" && scoped.node !== defaultSpec.nodeId) { - return { - ok: false, - command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, - node: scoped.node, - lane: scoped.lane, - mutation: false, - degradedReason: "node-scoped-git-mirror-action-not-enabled", - message: "D601 runtime GitOps is declared in UniDesk YAML and triggered by node-scoped control-plane commands; this command intentionally does not delegate D601 git-mirror actions to G14.", - expected: nodeRuntimeExpected(scoped.spec), - }; + if (scoped.action === "status") return nodeRuntimeGitMirrorStatus(scoped); + if (scoped.action === "sync" || scoped.action === "flush") { + if (scoped.confirm && !scoped.dryRun && !scoped.wait) return startNodeDelegatedJob(scoped); + return nodeRuntimeGitMirrorRun(scoped); + } + return nodeRuntimeUnsupportedAction(scoped); } if (domain === "control-plane" && scoped.action === "allow-endpoint-bridge") { return runNodeEndpointBridge(scoped); @@ -955,6 +969,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType): Record { + const spec = scoped.spec; + const mirror = nodeRuntimeGitMirrorTarget(spec); + const script = [ + "set +e", + `namespace=${shellQuote(mirror.namespace)}`, + `read_deploy=${shellQuote(mirror.serviceReadName)}`, + `write_deploy=${shellQuote(mirror.serviceWriteName)}`, + `read_svc=${shellQuote(mirror.serviceReadName)}`, + `write_svc=${shellQuote(mirror.serviceWriteName)}`, + `cache_pvc=${shellQuote(mirror.cachePvcName)}`, + `cache_host_path=${shellQuote(mirror.cacheHostPath ?? "")}`, + "deploy_ready() { desired=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n \"$1\" get deploy \"$2\" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n \"$desired\" ] && [ \"$desired\" -gt 0 ] 2>/dev/null && [ \"${ready:-0}\" = \"$desired\" ] && printf true || printf false; }", + "exists_res() { kubectl -n \"$1\" get \"$2\" \"$3\" >/dev/null 2>&1 && printf true || printf false; }", + "endpoint_ready() { endpoints=$(kubectl -n \"$1\" get endpoints \"$2\" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n \"$endpoints\" ] && printf true || printf false; }", + "summary_json=$(kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc '/etc/git-mirror/status.sh' 2>/tmp/hwlab-node-gitmirror-status.err || true)", + "if [ -z \"$summary_json\" ]; then summary_json='{}'; fi", + "read_deployment_ready=$(deploy_ready \"$namespace\" \"$read_deploy\")", + "write_deployment_ready=$(deploy_ready \"$namespace\" \"$write_deploy\")", + "read_service_exists=$(exists_res \"$namespace\" service \"$read_svc\")", + "write_service_exists=$(exists_res \"$namespace\" service \"$write_svc\")", + "read_endpoints_ready=$(endpoint_ready \"$namespace\" \"$read_svc\")", + "write_endpoints_ready=$(endpoint_ready \"$namespace\" \"$write_svc\")", + "cache_pvc_exists=$(exists_res \"$namespace\" pvc \"$cache_pvc\")", + "cache_host_path_exists=false", + "if [ -n \"$cache_host_path\" ] && kubectl -n \"$namespace\" exec deploy/\"$read_deploy\" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_exists=true; fi", + "SUMMARY_JSON=\"$summary_json\" read_deployment_ready=\"$read_deployment_ready\" write_deployment_ready=\"$write_deployment_ready\" read_service_exists=\"$read_service_exists\" write_service_exists=\"$write_service_exists\" read_endpoints_ready=\"$read_endpoints_ready\" write_endpoints_ready=\"$write_endpoints_ready\" cache_pvc_exists=\"$cache_pvc_exists\" cache_host_path=\"$cache_host_path\" cache_host_path_exists=\"$cache_host_path_exists\" node <<'NODE'", + "const summary = (() => { try { return JSON.parse(process.env.SUMMARY_JSON || '{}'); } catch { return {}; } })();", + "const env = process.env;", + "const ok = env.read_deployment_ready === 'true' && env.write_deployment_ready === 'true' && env.read_service_exists === 'true' && env.write_service_exists === 'true' && env.read_endpoints_ready === 'true' && env.write_endpoints_ready === 'true' && (env.cache_pvc_exists === 'true' || env.cache_host_path_exists === 'true') && summary.localSource;", + "console.log(JSON.stringify({", + " ok: Boolean(ok),", + " resources: {", + " readDeploymentReady: env.read_deployment_ready === 'true',", + " writeDeploymentReady: env.write_deployment_ready === 'true',", + " readServiceExists: env.read_service_exists === 'true',", + " writeServiceExists: env.write_service_exists === 'true',", + " readEndpointsReady: env.read_endpoints_ready === 'true',", + " writeEndpointsReady: env.write_endpoints_ready === 'true',", + " cachePvcExists: env.cache_pvc_exists === 'true',", + " cacheHostPathConfigured: Boolean(env.cache_host_path),", + " cacheHostPathExists: env.cache_host_path_exists === 'true'", + " },", + " summary,", + " valuesPrinted: false", + "}));", + "NODE", + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const parsed = parseJsonObject(statusText(result)); + const summary = record(parsed.summary); + const resources = record(parsed.resources); + const ok = result.exitCode === 0 && parsed.ok === true; + return { + ok, + command: `hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: "status", + mutation: false, + namespace: mirror.namespace, + readUrl: spec.gitReadUrl, + writeUrl: spec.gitWriteUrl, + sourceBranch: mirror.sourceBranch, + gitopsBranch: mirror.gitopsBranch, + resources, + summary, + result: compactRuntimeCommand(result), + degradedReason: ok ? undefined : "node-runtime-git-mirror-not-ready", + valuesPrinted: false, + next: { + sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm`, + flush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm`, + }, + }; +} + +function nodeRuntimeGitMirrorRun(scoped: ReturnType): Record { + if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped); + if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`); + const spec = scoped.spec; + const mirror = nodeRuntimeGitMirrorTarget(spec); + const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action); + const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName); + const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + const script = [ + "set -eu", + `namespace=${shellQuote(mirror.namespace)}`, + `job=${shellQuote(jobName)}`, + `manifest_b64=${shellQuote(manifestB64)}`, + "manifest_path=\"/tmp/$job.json\"", + "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", + scoped.dryRun + ? "kubectl create --dry-run=server -f \"$manifest_path\" -o name" + : [ + "kubectl delete job -n \"$namespace\" \"$job\" --ignore-not-found=true >/dev/null", + "kubectl create -f \"$manifest_path\"", + `deadline=$(( $(date +%s) + ${scoped.timeoutSeconds} ))`, + "while :; do", + " status=$(kubectl get job -n \"$namespace\" \"$job\" -o jsonpath='succeeded={.status.succeeded} failed={.status.failed}' 2>/dev/null || true)", + " succeeded=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"succeeded\") print a[2]; }}')", + " failed=$(printf '%s\\n' \"$status\" | awk '{for (i = 1; i <= NF; i++) { split($i, a, \"=\"); if (a[1] == \"failed\") print a[2]; }}')", + " if [ \"${succeeded:-0}\" = \"1\" ]; then break; fi", + " if [ \"${failed:-0}\" != \"\" ] && [ \"${failed:-0}\" != \"0\" ]; then kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true; exit 44; fi", + " if [ \"$(date +%s)\" -ge \"$deadline\" ]; then kubectl get job,pod -n \"$namespace\" -l job-name=\"$job\" -o wide || true; exit 45; fi", + " sleep 2", + "done", + "kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true", + ].join("\n"), + ].join("\n"); + const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds); + const status = scoped.dryRun || !isCommandSuccess(result) ? undefined : nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); + return { + ok: isCommandSuccess(result) && (status === undefined || status.ok === true), + command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`, + node: scoped.node, + lane: scoped.lane, + mode: scoped.dryRun ? "dry-run" : `confirmed-${scoped.action}`, + mutation: !scoped.dryRun && isCommandSuccess(result), + namespace: mirror.namespace, + jobName, + manifest: scoped.dryRun ? manifest : undefined, + result: compactRuntimeCommand(result), + status, + degradedReason: isCommandSuccess(result) ? status?.ok === false ? "node-runtime-git-mirror-status-failed-after-action" : undefined : `node-runtime-git-mirror-${scoped.action}-failed`, + next: scoped.dryRun + ? { run: `bun scripts/cli.ts hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane} --confirm` } + : { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` }, + }; +} + +function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType, sourceCommit: string): Record { + const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false }); + const beforeSummary = record(before.summary); + if (before.ok === true && beforeSummary.localSource === sourceCommit) { + return { ok: true, mode: "already-current", sourceCommit, before }; + } + const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }); + const after = record(sync.status); + const afterSummary = record(after.summary); + const ok = sync.ok === true && afterSummary.localSource === sourceCommit; + return { + ok, + mode: "synced-before-trigger", + sourceCommit, + before, + sync, + after: sync.status ?? null, + degradedReason: ok ? undefined : "node-runtime-git-mirror-local-source-not-current-after-sync", + }; +} + +function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string { + const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix; + return `${prefix}-${Date.now().toString(36)}`.slice(0, 63); +} + +function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record { + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace: mirror.namespace, + labels: { + "app.kubernetes.io/name": "git-mirror", + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "app.kubernetes.io/component": `${action}-controller`, + "hwlab.pikastech.local/node": mirror.node, + "hwlab.pikastech.local/lane": mirror.lane, + "hwlab.pikastech.local/trigger": "manual-cli", + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 600, + ttlSecondsAfterFinished: 3600, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "git-mirror", + "app.kubernetes.io/part-of": "hwlab-node-control-plane", + "app.kubernetes.io/component": `${action}-controller`, + "hwlab.pikastech.local/node": mirror.node, + "hwlab.pikastech.local/lane": mirror.lane, + }, + }, + spec: { + restartPolicy: "Never", + volumes: [ + { name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, + { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } }, + { name: "script", configMap: { name: "git-mirror-sync-script", defaultMode: 0o755 } }, + ], + containers: [{ + name: action, + image: mirror.toolsImage, + imagePullPolicy: "IfNotPresent", + command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"], + volumeMounts: [ + { name: "cache", mountPath: "/cache" }, + { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }, + { name: "script", mountPath: "/script", readOnly: true }, + ], + }], + }, + }, + }, + }; +} + function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { const spec = scoped.spec; const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); @@ -2798,6 +3044,36 @@ function externalPostgresSecretStatus(spec: HwlabRuntimeLaneSpec, namespaceExist }; } +function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRuntimeGitMirrorTargetSpec { + const configPath = rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); + const parsed = record(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown); + const targets = Array.isArray(parsed.targets) ? parsed.targets : []; + const target = targets.map((item) => record(item)).find((item) => item.node === spec.nodeId && item.lane === spec.lane); + if (target === undefined) throw new Error(`no gitMirror target for node=${spec.nodeId} lane=${spec.lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); + const gitMirror = record(target.gitMirror); + const source = record(target.source); + const gitops = record(target.gitops); + const tekton = record(target.tekton); + const toolsImage = record(tekton.toolsImage); + return { + id: stringValue(target.id, "target.id"), + node: stringValue(target.node, "target.node"), + lane: stringValue(target.lane, "target.lane"), + namespace: stringValue(gitMirror.namespace, "gitMirror.namespace"), + serviceReadName: stringValue(gitMirror.serviceReadName, "gitMirror.serviceReadName"), + serviceWriteName: stringValue(gitMirror.serviceWriteName, "gitMirror.serviceWriteName"), + cachePvcName: stringValue(gitMirror.cachePvcName, "gitMirror.cachePvcName"), + cacheHostPath: optionalStringValue(gitMirror.cacheHostPath, "gitMirror.cacheHostPath"), + secretName: stringValue(gitMirror.secretName, "gitMirror.secretName"), + syncJobPrefix: stringValue(gitMirror.syncJobPrefix, "gitMirror.syncJobPrefix"), + flushJobPrefix: stringValue(gitMirror.flushJobPrefix, "gitMirror.flushJobPrefix"), + toolsImage: stringValue(toolsImage.output, "tekton.toolsImage.output"), + sourceRepository: stringValue(source.repository, "source.repository"), + sourceBranch: stringValue(source.branch, "source.branch"), + gitopsBranch: stringValue(gitops.branch, "gitops.branch"), + }; +} + function secretKeyStatus(spec: HwlabRuntimeLaneSpec, secret: string, key: string): Record { const result = runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "secret", secret, "-o", `go-template={{ index .data ${JSON.stringify(key)} }}`], 60); return { @@ -5385,6 +5661,38 @@ function assertNodeId(value: string): void { if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`--node must be a simple node id, got ${value}`); } +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function stringValue(value: unknown, path: string): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`); + return value; +} + +function optionalStringValue(value: unknown, path: string): string | null { + if (value === undefined || value === null) return null; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string when set`); + return value; +} + +function parseJsonObject(text: string): Record { + const trimmed = text.trim(); + if (trimmed.length === 0) return {}; + try { + return record(JSON.parse(trimmed) as unknown); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { + return record(JSON.parse(trimmed.slice(start, end + 1)) as unknown); + } catch {} + } + } + return {}; +} + function shellQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; }