diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index fcbe217f..33287f75 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -97,6 +97,7 @@ lanes: baseImageSource: node:20-bookworm-slim serviceIds: - hwlab-cloud-api + - hwlab-workbench-runtime - hwlab-user-billing - hwlab-cloud-web - hwlab-gateway @@ -153,11 +154,12 @@ lanes: baseImageSource: node:20-bookworm-slim serviceIds: - hwlab-cloud-api + - hwlab-workbench-runtime + - hwlab-user-billing - hwlab-cloud-web - hwlab-gateway - hwlab-edge-proxy - hwlab-agent-skills - - hwlab-user-billing buildkit: sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless stepEnv: diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index a19d72a6..f5b4193e 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -153,7 +153,7 @@ targets: sourceRef: platform-infra/master-vpn-subscription.env sourceKey: MASTER_VPN_SUBSCRIPTION_URL sourceType: subscription-url - preferredOutbound: vless-reality + preferredOutbound: hysteria2 applyToSub2Api: true applyToSentinel: true healthProbeUrl: https://www.gstatic.com/generate_204 diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 24f2c916..161da79f 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -476,6 +476,7 @@ function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record | null { const text = `${result.stdout}\n${result.stderr}`; const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text); + const scriptTransportMismatch = + (mirror.githubTransport.mode === "ssh" && /transport=https[\s\S]*https auth: missing GITHUB_TOKEN/iu.test(text)) + || (mirror.githubTransport.mode === "https" && /transport=ssh\b/iu.test(text)); + if (scriptTransportMismatch) { + return { + retryable: false, + retryAttempt: attempt, + retryMaxAttempts, + retryLabel: `${attempt}/${retryMaxAttempts}`, + retryExhausted: false, + stopped: true, + reason: `Git mirror script transport does not match YAML githubTransport.mode=${mirror.githubTransport.mode}; apply the node control-plane so the git-mirror ConfigMap is updated before retrying.`, + refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror), + githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror), + next: { + applyControlPlane: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, + status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`, + }, + valuesPrinted: false, + }; + } const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text); if (authFailure) { return { @@ -3777,10 +3806,12 @@ function nodeRuntimeStatusCommand(scoped: ReturnType { - const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60); - const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60); - const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : []; - const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : []; + const taskRunTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineTask"}}{{"\\t"}}{{if .spec.taskRef}}{{.spec.taskRef.name}}{{end}}{{"\\t"}}{{with index .status.conditions 0}}{{.status}}{{"\\t"}}{{.reason}}{{else}}{{"\\t"}}{{end}}{{"\\t"}}{{.status.podName}}{{"\\t"}}{{with index .status.conditions 0}}{{printf "%.600s" .message}}{{end}}{{"\\n"}}{{end}}`; + const podTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/taskRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "PodScheduled"}}{{.status}}|{{.reason}}|{{printf "%.600s" .message}}{{end}}{{end}}{{"\\t"}}{{range .status.initContainerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\t"}}{{range .status.containerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\n"}}{{end}}`; + const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${taskRunTemplate}`], 60); + const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${podTemplate}`], 60); + const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRunsFromTsv(taskRunsResult.stdout) : []; + const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPodsFromTsv(podsResult.stdout) : []; const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False"); const failedTaskRuns = taskRuns.filter((item) => item.status === "False"); const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods); @@ -3852,6 +3883,67 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR }; } +function nodeRuntimePipelineDiagnosticTaskRunsFromTsv(text: string): Array> { + return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => { + const parts = line.split("\t"); + const [name = "", pipelineTask = "", taskRef = "", status = "", reason = "", podName = "", ...messageParts] = parts; + const message = messageParts.join("\t"); + return { + name: stringOrNull(name), + pipelineTask: stringOrNull(pipelineTask), + taskRef: stringOrNull(taskRef), + status: stringOrNull(status), + reason: stringOrNull(reason), + message: diagnosticText(message), + podName: stringOrNull(podName), + steps: [], + failedSteps: [], + }; + }); +} + +function nodeRuntimePipelineDiagnosticPodsFromTsv(text: string): Array> { + return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => { + const [name = "", taskRun = "", phase = "", nodeName = "", scheduledRaw = "", initRaw = "", containersRaw = ""] = line.split("\t"); + const [scheduledStatus = "", scheduledReason = "", scheduledMessage = ""] = scheduledRaw.split("|"); + const initContainers = nodeRuntimePipelineContainerStatusesFromCompact(initRaw); + const containers = nodeRuntimePipelineContainerStatusesFromCompact(containersRaw); + const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true); + return { + name: stringOrNull(name), + taskRun: stringOrNull(taskRun), + phase: stringOrNull(phase), + nodeName: stringOrNull(nodeName), + scheduled: scheduledStatus.length === 0 ? null : scheduledStatus === "True", + scheduledReason: stringOrNull(scheduledReason), + scheduledMessage: diagnosticText(scheduledMessage), + initContainers, + containers, + failedContainers, + }; + }); +} + +function nodeRuntimePipelineContainerStatusesFromCompact(text: string): Array> { + return text.split(",").map((item) => item.trim()).filter(Boolean).map((item) => { + const [name = "", stateOrExit = "", reason = ""] = item.split(":"); + const exitCode = /^[0-9]+$/u.test(stateOrExit) ? Number(stateOrExit) : null; + const state = exitCode !== null ? "terminated" : stateOrExit === "waiting" ? "waiting" : stateOrExit === "running" ? "running" : null; + return { + name: stringOrNull(name), + ready: null, + restartCount: null, + state, + failed: exitCode !== null && exitCode !== 0, + exitCode, + reason: stringOrNull(reason), + message: null, + startedAt: null, + finishedAt: null, + }; + }); +} + function nodeRuntimePipelineDiagnosticTaskRuns(json: Record): Array> { const items = Array.isArray(json.items) ? json.items.map(record) : []; return items.map((item) => { @@ -5135,6 +5227,87 @@ function nodeRuntimePipelinePostprocessScript(): string[] { " }", " return false;", "}", + "function patchGitMirrorTransportYaml() {", + " const mirror = overlay.gitMirror || {};", + " const transport = mirror.githubTransport || {};", + " if (transport.mode !== 'https') return;", + " if (!YAML) throw new Error('yaml module is required to patch git-mirror githubTransport=https');", + " const requireString = (pathName, value) => {", + " if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror githubTransport=https');", + " return value;", + " };", + " const repository = requireString('gitMirror.sourceRepository', mirror.sourceRepository);", + " const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);", + " const tokenSecretName = requireString('gitMirror.githubTransport.tokenSecretName', transport.tokenSecretName);", + " const tokenSecretKey = requireString('gitMirror.githubTransport.tokenSecretKey', transport.tokenSecretKey);", + " const tokenSourceRef = requireString('gitMirror.githubTransport.tokenSourceRef', transport.tokenSourceRef);", + " const username = typeof transport.username === 'string' && transport.username ? transport.username : 'x-access-token';", + " const remoteUrl = `https://github.com/${repository}.git`;", + " const proxySummary = `transport=https proxy=HTTP_PROXY authSecret=${tokenSecretName} authKey=${tokenSecretKey} authSourceRef=${tokenSourceRef} source=yaml`;", + " const askpassBlock = [", + " 'if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo \\'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env\\' >&2; exit 64; fi',", + " \"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'\",", + " '#!/bin/sh',", + " 'case \"$1\" in',", + " ' *Username*) printf \\'%s\\\\n\\' \"${GITHUB_USERNAME:-x-access-token}\" ;;',", + " ' *Password*) printf \\'%s\\\\n\\' \"$GITHUB_TOKEN\" ;;',", + " \" *) printf '\\\\n' ;;\",", + " 'esac',", + " 'SH_ASKPASS',", + " 'chmod 0700 /tmp/hwlab-git-askpass.sh',", + " `export GITHUB_USERNAME=${shellSingle(username)}`,", + " 'export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh',", + " 'export GIT_TERMINAL_PROMPT=0',", + " 'unset GIT_SSH',", + " 'unset GIT_SSH_COMMAND',", + " ].join('\\n');", + " function patchScript(script, key) {", + " let next = String(script || '');", + " if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);", + " let remoteReplaced = false;", + " next = next.replace(/repo_url=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `repo_url=${shellSingle(remoteUrl)}`; });", + " next = next.replace(/remote=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `remote=${shellSingle(remoteUrl)}`; });", + " if (!remoteReplaced) throw new Error(`generated git-mirror ${key} missing remote url assignment for githubTransport=https`);", + " next = next.replace(/transport=ssh ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);", + " next = next.replace(/ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);", + " next = next.replace(/mkdir -p ([^\\n]*?) \\/root\\/\\.ssh/g, 'mkdir -p $1');", + " next = next.replace(/\\n[ \\t]*mkdir -p \\/root\\/\\.ssh\\n/g, '\\n');", + " next = next.replace(/\\n[ \\t]*cp \\/git-ssh\\/ssh-privatekey[^\\n]*\\n[ \\t]*chmod 0?400[^\\n]*\\n/g, '\\n');", + " next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) <<'NODE_PROXY'[\\s\\S]*?\\nNODE_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs)\\n/g, '\\n');", + " next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-git-ssh-proxy\\.sh <<'SH_PROXY'[\\s\\S]*?\\nSH_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-git-ssh-proxy\\.sh\\n/g, '\\n');", + " next = next.replace(/\\n[ \\t]*export GIT_SSH=.*\\n/g, '\\n');", + " next = next.replace(/\\n[ \\t]*export GIT_SSH_COMMAND=.*\\n/g, '\\n');", + " next = next.replace(/\\n[ \\t]*unset GIT_SSH_COMMAND\\n/g, '\\n');", + " if (!next.includes('hwlab git-mirror https auth: missing GITHUB_TOKEN secret env')) {", + " const noProxyExport = /\\nexport no_proxy=[^\\n]*\\n/;", + " if (noProxyExport.test(next)) next = next.replace(noProxyExport, (match) => `${match}${askpassBlock}\\n`);", + " else if (next.includes('\\nset -eu\\n')) next = next.replace('\\nset -eu\\n', `\\nset -eu\\n${askpassBlock}\\n`);", + " else next = `${askpassBlock}\\n${next}`;", + " }", + " if (next.includes('/git-ssh') || next.includes('ssh://git@') || next.includes('GIT_SSH=')) throw new Error(`generated git-mirror ${key} still contains ssh transport after githubTransport=https patch`);", + " if (!next.includes('GIT_ASKPASS') || !next.includes('GITHUB_TOKEN')) throw new Error(`generated git-mirror ${key} missing https auth after githubTransport=https patch`);", + " return next;", + " }", + " const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');", + " if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);", + " const docs = YAML.parseAllDocuments(fs.readFileSync(gitMirrorFile, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", + " const manifests = [];", + " for (const doc of docs) {", + " if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);", + " else manifests.push(doc);", + " }", + " let changed = false;", + " for (const doc of manifests) {", + " if (!doc || typeof doc !== 'object' || doc.kind !== 'ConfigMap') continue;", + " if (!doc.metadata || doc.metadata.name !== configMapName) continue;", + " doc.data = doc.data || {};", + " doc.data['sync.sh'] = patchScript(doc.data['sync.sh'], 'sync.sh');", + " doc.data['flush.sh'] = patchScript(doc.data['flush.sh'], 'flush.sh');", + " changed = true;", + " }", + " if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);", + " fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", + "}", "const structured = patchStructuredPipeline();", "function replaceParamDefault(name, value) {", " const namePattern = escapeRegExp(name);", @@ -5187,6 +5360,7 @@ function nodeRuntimePipelinePostprocessScript(): string[] { "if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && !text.includes('NODE_UNIDESK_YAML_DEPENDENCY')) { throw new Error(`generated pipeline missing UniDesk yaml dependency install in ${pipelinePath}`); }", "if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }", "fs.writeFileSync(pipelinePath, text);", + "patchGitMirrorTransportYaml();", "function patchArgoYaml(filePath) {", " if (!YAML || !fs.existsSync(filePath)) return;", " const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", @@ -5304,6 +5478,7 @@ function httpProxyEndpoint(value: string): { host: string; port: number } | null function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] { return [ + `${renderDir}/devops-infra/git-mirror.yaml`, `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, `${renderDir}/${spec.tektonDir}/rbac.yaml`, `${renderDir}/${spec.tektonDir}/pipeline.yaml`,