mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add JD01 YAML-first deployment support

Browse Source
This commit is contained in:
Codex
2026-06-29 08:13:34 +00:00
parent fe917bec4a
commit 076c4b643d
49 changed files with 10909 additions and 5223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/render.ts
+129 -5
View File
@@ -45,7 +45,12 @@ export function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSp
return `${prefix}-${Date.now().toString(36)}`.slice(0, 63);
}
export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
export function nodeRuntimeGitMirrorJobManifest(
mirror: NodeRuntimeGitMirrorTargetSpec,
action: "sync" | "flush",
jobName: string,
options: { discardStaleGitops?: boolean } = {},
): Record<string, unknown> {
const volumes: Record<string, unknown>[] = [
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
@@ -88,6 +93,7 @@ export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTarg
},
},
spec: {
...(mirror.egressProxy.mode === "host-route" ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}),
restartPolicy: "Never",
volumes,
containers: [{
@@ -95,7 +101,11 @@ export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTarg
image: mirror.toolsImage,
imagePullPolicy: mirror.toolsImagePullPolicy,
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
env: [...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror)],
env: [
...nodeRuntimeGitMirrorProxyEnv(mirror),
...nodeRuntimeGitMirrorGithubTransportEnv(mirror),
...(options.discardStaleGitops === true ? [{ name: "UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS", value: "true" }] : []),
],
volumeMounts,
}],
},
@@ -107,7 +117,7 @@ export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTarg
export function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
const proxy = mirror.egressProxy;
if (proxy.mode === "direct") return [];
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
const proxyUrl = proxy.mode === "host-route" ? proxy.proxyUrl : `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
const noProxy = proxy.noProxy.join(",");
return [
{ name: "HTTP_PROXY", value: proxyUrl },
@@ -1101,7 +1111,7 @@ export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec,
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
@@ -1186,7 +1196,7 @@ export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, s
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
@@ -1283,6 +1293,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
" runtimeStore: overlay.runtimeStore,",
" codeAgentRuntime: overlay.codeAgentRuntime,",
" observability: overlay.observability,",
" deployYamlGitMirror: overlay.deployYamlGitMirror,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" dockerProxyHttp: overlay.dockerProxyHttp,",
" dockerProxyHttps: overlay.dockerProxyHttps,",
@@ -1332,6 +1343,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
"fs.writeFileSync(file, YAML.stringify(doc));",
"console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));",
"NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;",
@@ -2200,6 +2212,117 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
" if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
"}",
"function patchGitMirrorHostRouteYaml() {",
" const mirror = overlay.gitMirror || {};",
" const proxy = mirror.egressProxy || {};",
" if (proxy.mode !== 'host-route') return;",
" if (!YAML) throw new Error('yaml module is required to patch git-mirror host-route proxy');",
" const requireString = (pathName, value) => {",
" if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror host-route proxy');",
" return value;",
" };",
" const proxyUrl = requireString('gitMirror.egressProxy.proxyUrl', proxy.proxyUrl);",
" const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);",
" let proxyEndpoint;",
" try { proxyEndpoint = new URL(proxyUrl); } catch (error) { throw new Error(`overlay.gitMirror.egressProxy.proxyUrl is invalid: ${error.message}`); }",
" if (proxyEndpoint.protocol !== 'http:') throw new Error('overlay.gitMirror.egressProxy.proxyUrl must use http:// for host-route');",
" const proxyHost = proxyEndpoint.hostname;",
" const proxyPort = Number(proxyEndpoint.port || '80');",
" if (!proxyHost || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65535) throw new Error('overlay.gitMirror.egressProxy.proxyUrl must include a valid host and port');",
" const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.join(',') : '';",
" const envEntries = [",
" ['HTTP_PROXY', proxyUrl],",
" ['HTTPS_PROXY', proxyUrl],",
" ['ALL_PROXY', proxyUrl],",
" ['http_proxy', proxyUrl],",
" ['https_proxy', proxyUrl],",
" ['all_proxy', proxyUrl],",
" ['NO_PROXY', noProxy],",
" ['no_proxy', noProxy],",
" ];",
" function readDocs(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
" function flattenDocs(docs) {",
" const manifests = [];",
" for (const doc of docs) {",
" if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);",
" else manifests.push(doc);",
" }",
" return manifests;",
" }",
" function setContainerEnv(container, name, value) {",
" if (!container || typeof container !== 'object') return false;",
" container.env = Array.isArray(container.env) ? container.env : [];",
" let item = container.env.find((env) => env && env.name === name);",
" if (!item) { item = { name }; container.env.push(item); }",
" const changed = item.value !== value || item.valueFrom !== undefined;",
" item.value = value;",
" delete item.valueFrom;",
" return changed;",
" }",
" function patchProxyScript(script, key) {",
" let next = String(script || '');",
" if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);",
" next = next.replace(/node \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) \\S+ \\d+/g, `node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort}`);",
" const missing = [];",
" for (const [name, value] of envEntries) {",
" let replaced = false;",
" const exportPattern = new RegExp(`(^|\\\\n)([ \\\\t]*export ${escapeRegExp(name)}=)[^\\\\n]*`, 'g');",
" next = next.replace(exportPattern, (_match, prefix, left) => { replaced = true; return `${prefix}${left}${shellSingle(value)}`; });",
" if (!replaced) missing.push([name, value]);",
" }",
" if (missing.length > 0) {",
" const block = missing.map(([name, value]) => `export ${name}=${shellSingle(value)}`).join('\\\\n');",
" if (next.includes('\\\\nset -eu\\\\n')) next = next.replace('\\\\nset -eu\\\\n', `\\\\nset -eu\\\\n${block}\\\\n`);",
" else next = `${block}\\\\n${next}`;",
" }",
" next = next.replace(/mode=node-global/g, 'mode=host-route');",
" return next;",
" }",
" const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');",
" if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);",
" const docs = readDocs(gitMirrorFile);",
" const manifests = flattenDocs(docs);",
" let configMapChanged = false;",
" let workloadChanged = false;",
" const workloadNames = new Set([mirror.serviceReadName, mirror.serviceWriteName].filter((value) => typeof value === 'string' && value.length > 0));",
" for (const doc of manifests) {",
" if (!doc || typeof doc !== 'object') continue;",
" if (doc.kind === 'ConfigMap' && doc.metadata && doc.metadata.name === configMapName) {",
" doc.data = doc.data || {};",
" doc.data['sync.sh'] = patchProxyScript(doc.data['sync.sh'], 'sync.sh');",
" doc.data['flush.sh'] = patchProxyScript(doc.data['flush.sh'], 'flush.sh');",
" configMapChanged = true;",
" continue;",
" }",
" if (!['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(doc.kind)) continue;",
" const name = doc.metadata && doc.metadata.name ? String(doc.metadata.name) : '';",
" const labels = doc.metadata && doc.metadata.labels && typeof doc.metadata.labels === 'object' ? doc.metadata.labels : {};",
" if (!workloadNames.has(name) && labels['app.kubernetes.io/name'] !== 'git-mirror' && !name.includes('git-mirror')) continue;",
" doc.spec = doc.spec || {};",
" doc.spec.template = doc.spec.template || {};",
" doc.spec.template.metadata = doc.spec.template.metadata || {};",
" doc.spec.template.metadata.annotations = doc.spec.template.metadata.annotations || {};",
" doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-egress-proxy'] = 'host-route';",
" doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-host-proxy-config-ref'] = String(proxy.hostProxyConfigRef || '');",
" doc.spec.template.spec = doc.spec.template.spec || {};",
" if (proxy.podHostNetwork) {",
" doc.spec.template.spec.hostNetwork = true;",
" doc.spec.template.spec.dnsPolicy = 'ClusterFirstWithHostNet';",
" } else {",
" delete doc.spec.template.spec.hostNetwork;",
" delete doc.spec.template.spec.dnsPolicy;",
" }",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(doc.spec.template.spec[group]) ? doc.spec.template.spec[group] : []) {",
" for (const [envName, envValue] of envEntries) setContainerEnv(container, envName, envValue);",
" }",
" }",
" workloadChanged = true;",
" }",
" if (!configMapChanged) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
" if (!workloadChanged) throw new Error(`generated git-mirror workload for host-route proxy was not found in ${gitMirrorFile}`);",
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
"}",
"const structured = patchStructuredPipeline();",
"function replaceParamDefault(name, value) {",
" const namePattern = escapeRegExp(name);",
@@ -2253,6 +2376,7 @@ export function nodeRuntimePipelinePostprocessScript(): string[] {
"if (text.includes('/yaml/-/yaml-') || text.includes('bun add --no-save --ignore-scripts') || text.includes('npm install --package-lock=false --no-save')) { throw new Error(`generated pipeline still downloads yaml during prepare-source in ${pipelinePath}`); }",
"if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }",
"fs.writeFileSync(pipelinePath, text);",
"patchGitMirrorHostRouteYaml();",
"patchGitMirrorTransportYaml();",
"function patchArgoYaml(filePath) {",
" if (!YAML || !fs.existsSync(filePath)) return;",