fix: avoid full npm install in D601 prepare-source
This commit is contained in:
Codex
+122
-14
@@ -1023,6 +1023,62 @@ function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec):
|
||||
return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
|
||||
}
|
||||
|
||||
function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
|
||||
const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
|
||||
const retryCount = Math.max(0, Math.floor(retries));
|
||||
const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
||||
return [
|
||||
`yaml_registry=${shellQuote(registry)}`,
|
||||
`yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`,
|
||||
`yaml_fetch_retries=${shellQuote(String(retryCount))}`,
|
||||
`yaml_dependency_context=${shellQuote(safeContext)}`,
|
||||
"yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }",
|
||||
"yaml_npm_debug_log_tail() {",
|
||||
" yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"",
|
||||
" if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi",
|
||||
" yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"",
|
||||
" if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then",
|
||||
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2",
|
||||
" tail -n 80 \"$yaml_npm_log\" >&2 || true",
|
||||
" fi",
|
||||
"}",
|
||||
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
|
||||
" mkdir -p node_modules/yaml",
|
||||
" yaml_tarball=\"${yaml_registry%/}/yaml/-/yaml-2.8.3.tgz\"",
|
||||
" yaml_tgz=\"$(mktemp)\"",
|
||||
" if command -v curl >/dev/null 2>&1 && command -v tar >/dev/null 2>&1; then",
|
||||
" if timeout \"$yaml_fetch_timeout\" curl -fsSL --retry \"$yaml_fetch_retries\" --connect-timeout 10 -o \"$yaml_tgz\" \"$yaml_tarball\"; then",
|
||||
" tar -xzf \"$yaml_tgz\" -C node_modules/yaml --strip-components=1",
|
||||
" yaml_dependency_log installed tarball",
|
||||
" else",
|
||||
" yaml_dependency_log tarball-failed tarball",
|
||||
" fi",
|
||||
" fi",
|
||||
" rm -f \"$yaml_tgz\"",
|
||||
"fi",
|
||||
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then",
|
||||
" rm -rf node_modules/yaml",
|
||||
" if timeout \"$yaml_fetch_timeout\" bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then",
|
||||
" yaml_dependency_log installed bun",
|
||||
" else",
|
||||
" yaml_dependency_log bun-failed bun",
|
||||
" fi",
|
||||
"fi",
|
||||
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
|
||||
" rm -rf node_modules/yaml",
|
||||
" command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }",
|
||||
" if npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then",
|
||||
" yaml_dependency_log installed npm",
|
||||
" else",
|
||||
" yaml_dependency_log npm-failed npm",
|
||||
" yaml_npm_debug_log_tail",
|
||||
" exit 34",
|
||||
" fi",
|
||||
"fi",
|
||||
"node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }",
|
||||
];
|
||||
}
|
||||
|
||||
function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
||||
const token = nodeRuntimeRenderToken();
|
||||
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
|
||||
@@ -1043,16 +1099,7 @@ function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceC
|
||||
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
||||
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
||||
"cd \"$worktree_dir\"",
|
||||
"if [ ! -d node_modules/yaml ]; then",
|
||||
" if [ -f package-lock.json ]; then",
|
||||
" npm ci --ignore-scripts --no-audit --prefer-offline",
|
||||
" elif [ -f bun.lock ] || [ -f bun.lockb ]; then",
|
||||
" bun install --frozen-lockfile --ignore-scripts",
|
||||
" else",
|
||||
" echo \"control-plane render cannot install dependencies: no lockfile found\" >&2",
|
||||
" exit 42",
|
||||
" fi",
|
||||
"fi",
|
||||
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"),
|
||||
"node - \"$overlay_b64\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const YAML = require('yaml');",
|
||||
@@ -1133,10 +1180,8 @@ function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCo
|
||||
"run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
|
||||
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
||||
"cd \"$worktree_dir\"",
|
||||
"if [ ! -d node_modules/yaml ]; then",
|
||||
" echo \"phase=local-install-yaml\" >&2",
|
||||
" npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3",
|
||||
"fi",
|
||||
"echo \"phase=local-install-yaml\" >&2",
|
||||
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
|
||||
"node - \"$overlay_b64\" <<'NODE'",
|
||||
"const fs = require('fs');",
|
||||
"const YAML = require('yaml');",
|
||||
@@ -1226,6 +1271,61 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,",
|
||||
"};",
|
||||
"const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };",
|
||||
"function prepareSourceDependencyScript() {",
|
||||
" const registry = String(overlay.npmRegistry || 'https://registry.npmjs.org/');",
|
||||
" const timeoutSeconds = Math.max(15, Math.ceil(Number(overlay.npmFetchTimeoutMs || 120000) / 1000));",
|
||||
" const retryCount = Math.max(0, Math.floor(Number(overlay.npmRetries || 3)));",
|
||||
" return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"",
|
||||
"node <<'NODE_UNIDESK_YAML_DEPENDENCY'",
|
||||
"const { spawnSync } = require('node:child_process');",
|
||||
"const fs = require('node:fs');",
|
||||
"const os = require('node:os');",
|
||||
"const path = require('node:path');",
|
||||
"const registry = ${JSON.stringify(registry)};",
|
||||
"const timeoutMs = ${JSON.stringify(timeoutSeconds * 1000)};",
|
||||
"const timeoutSeconds = ${JSON.stringify(timeoutSeconds)};",
|
||||
"const retryCount = ${JSON.stringify(retryCount)};",
|
||||
"const dependency = 'yaml';",
|
||||
"const version = '2.8.3';",
|
||||
"function emit(status, extra = {}) { console.error(JSON.stringify({ event: 'prepare-source-dependencies', status, dependency, ...extra })); }",
|
||||
"function hasYaml() { try { require.resolve('yaml'); return true; } catch { return false; } }",
|
||||
"function run(command, args) {",
|
||||
" const result = spawnSync(command, args, { stdio: 'inherit', env: process.env, timeout: timeoutMs });",
|
||||
" if (result.error) console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'command-error', command, error: result.error.message }));",
|
||||
" return result.status === 0;",
|
||||
"}",
|
||||
"function tailNpmLog() {",
|
||||
" const dir = path.join(process.env.HOME || '/tmp', '.npm', '_logs');",
|
||||
" if (!fs.existsSync(dir)) return;",
|
||||
" const files = fs.readdirSync(dir).filter((name) => name.includes('debug') && name.endsWith('.log')).sort();",
|
||||
" const file = files[files.length - 1];",
|
||||
" if (!file) return;",
|
||||
" const full = path.join(dir, file);",
|
||||
" console.error(JSON.stringify({ event: 'prepare-source-dependencies', status: 'npm-debug-log', path: full }));",
|
||||
" console.error(fs.readFileSync(full, 'utf8').split(/\\r?\\n/u).slice(-80).join('\\n'));",
|
||||
"}",
|
||||
"if (hasYaml()) { emit('cached'); process.exit(0); }",
|
||||
"fs.mkdirSync('node_modules/yaml', { recursive: true });",
|
||||
"const tarball = registry.replace(/\\/+$/u, '') + '/yaml/-/yaml-' + version + '.tgz';",
|
||||
"const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hwlab-yaml-'));",
|
||||
"const tgz = path.join(tmpDir, 'yaml.tgz');",
|
||||
"if (run('curl', ['-fsSL', '--retry', String(retryCount), '--connect-timeout', '10', '--max-time', String(timeoutSeconds), '-o', tgz, tarball]) && run('tar', ['-xzf', tgz, '-C', 'node_modules/yaml', '--strip-components=1'])) emit('installed', { manager: 'tarball' });",
|
||||
"else emit('tarball-failed', { manager: 'tarball' });",
|
||||
"fs.rmSync(tmpDir, { recursive: true, force: true });",
|
||||
"if (!hasYaml()) {",
|
||||
" fs.rmSync('node_modules/yaml', { recursive: true, force: true });",
|
||||
" if (run('bun', ['add', '--no-save', '--ignore-scripts', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'bun' });",
|
||||
" else emit('bun-failed', { manager: 'bun' });",
|
||||
"}",
|
||||
"if (!hasYaml()) {",
|
||||
" fs.rmSync('node_modules/yaml', { recursive: true, force: true });",
|
||||
" if (run('npm', ['install', '--package-lock=false', '--no-save', '--ignore-scripts', '--no-audit', '--no-fund', '--omit=dev', '--registry', registry, 'yaml@' + version]) && hasYaml()) emit('installed', { manager: 'npm' });",
|
||||
" else { emit('npm-failed', { manager: 'npm' }); tailNpmLog(); process.exit(34); }",
|
||||
"}",
|
||||
"if (!hasYaml()) { emit('failed', { reason: 'unresolved' }); process.exit(34); }",
|
||||
"NODE_UNIDESK_YAML_DEPENDENCY",
|
||||
"ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;",
|
||||
"}",
|
||||
"function deployYamlOverlayScript() {",
|
||||
" const runtimeOverlay = JSON.stringify({",
|
||||
" nodeId: overlay.nodeId,",
|
||||
@@ -1248,6 +1348,7 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" dockerNoProxyList: overlay.dockerNoProxyList,",
|
||||
" npmRegistry: overlay.npmRegistry,",
|
||||
" npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,",
|
||||
" npmRetries: overlay.npmRetries,",
|
||||
" });",
|
||||
" return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'",
|
||||
"const fs = require('fs');",
|
||||
@@ -1679,6 +1780,10 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
"}",
|
||||
"function patchScript(script) {",
|
||||
" let result = String(script || '');",
|
||||
" const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');",
|
||||
" if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {",
|
||||
" result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());",
|
||||
" }",
|
||||
" const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';",
|
||||
" if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
|
||||
" result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);",
|
||||
@@ -1828,6 +1933,8 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
|
||||
" changed = true;",
|
||||
"}",
|
||||
"if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }",
|
||||
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }",
|
||||
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && !text.includes('NODE_UNIDESK_YAML_DEPENDENCY')) { throw new Error(`generated pipeline missing UniDesk yaml dependency install in ${pipelinePath}`); }",
|
||||
"fs.writeFileSync(pipelinePath, text);",
|
||||
"function patchArgoYaml(filePath) {",
|
||||
" if (!YAML || !fs.existsSync(filePath)) return;",
|
||||
@@ -1887,6 +1994,7 @@ function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<string, un
|
||||
dockerNoProxyList: spec.networkProfile.dockerBuildProxy.noProxy,
|
||||
npmRegistry: spec.downloadProfile.npm.registry,
|
||||
npmFetchTimeoutMs: spec.downloadProfile.npm.fetchTimeoutSeconds * 1000,
|
||||
npmRetries: spec.downloadProfile.npm.retries,
|
||||
stepEnv: spec.stepEnv,
|
||||
observability: spec.observability,
|
||||
runtimeImageRewrites: spec.runtimeImageRewrites,
|
||||
|
||||
Reference in New Issue
Block a user