fix d601 workbench runtime deployment and proxy flush

This commit is contained in:
Codex
2026-06-21 13:02:36 +00:00
parent 543605f3fb
commit 2513222fb5
3 changed files with 184 additions and 7 deletions
+3 -1
View File
@@ -97,6 +97,7 @@ lanes:
baseImageSource: node:20-bookworm-slim
serviceIds:
- hwlab-cloud-api
- hwlab-workbench-runtime
- hwlab-user-billing
- hwlab-cloud-web
- hwlab-gateway
@@ -153,11 +154,12 @@ lanes:
baseImageSource: node:20-bookworm-slim
serviceIds:
- hwlab-cloud-api
- hwlab-workbench-runtime
- hwlab-user-billing
- hwlab-cloud-web
- hwlab-gateway
- hwlab-edge-proxy
- hwlab-agent-skills
- hwlab-user-billing
buildkit:
sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless
stepEnv:
+1 -1
View File
@@ -153,7 +153,7 @@ targets:
sourceRef: platform-infra/master-vpn-subscription.env
sourceKey: MASTER_VPN_SUBSCRIPTION_URL
sourceType: subscription-url
preferredOutbound: vless-reality
preferredOutbound: hysteria2
applyToSub2Api: true
applyToSentinel: true
healthProbeUrl: https://www.gstatic.com/generate_204
+180 -5
View File
@@ -476,6 +476,7 @@ function nodeRuntimeExpected(spec: HwlabRuntimeLaneSpec): Record<string, unknown
image: spec.baseImage,
sourceImage: spec.baseImageSource ?? null,
},
serviceIds: spec.serviceIds,
buildkit: spec.buildkit === undefined ? null : {
sidecarImage: spec.buildkit.sidecarImage,
},
@@ -1827,6 +1828,10 @@ function nodeRuntimePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: st
return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`;
}
function nodeRuntimeRerunPipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string {
return `${nodeRuntimePipelineRunName(spec, sourceCommit)}-r${Date.now().toString(36)}`.slice(0, 63);
}
function nodeRuntimeGitopsRoot(spec: HwlabRuntimeLaneSpec): string {
const suffix = `/${spec.runtimeRenderDir}`;
if (spec.runtimePath.endsWith(suffix)) return spec.runtimePath.slice(0, -suffix.length);
@@ -2646,7 +2651,8 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
headProbe: compactRuntimeCommand(head.result),
};
}
const pipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
const basePipelineRun = nodeRuntimePipelineRunName(spec, sourceCommit);
const pipelineRun = scoped.rerun ? nodeRuntimeRerunPipelineRunName(spec, sourceCommit) : basePipelineRun;
printNodeRuntimeTriggerProgress(spec, { stage: "source-head", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "probe-existing-pipelinerun", status: "started", sourceCommit, pipelineRun });
const before = getNodeRuntimePipelineRun(spec, pipelineRun);
@@ -2661,6 +2667,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
mode: "dry-run",
sourceCommit,
pipelineRun,
rerunOf: scoped.rerun ? basePipelineRun : null,
rerun: scoped.rerun,
before,
gitMirror: nodeScopedFullOutput(scoped) ? gitMirror : compactNodeRuntimeGitMirrorObservation(gitMirror),
@@ -2776,6 +2783,7 @@ function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDele
mutation: createOk,
sourceCommit,
pipelineRun,
rerunOf: scoped.rerun ? basePipelineRun : null,
rerun: scoped.rerun,
before,
gitMirror,
@@ -3012,6 +3020,27 @@ function nodeRuntimeGitMirrorRetryableFailure(
): Record<string, unknown> | null {
const text = `${result.stdout}\n${result.stderr}`;
const proxyConnectFailure = /hwlab git-mirror proxy-connect:/iu.test(text);
const scriptTransportMismatch =
(mirror.githubTransport.mode === "ssh" && /transport=https[\s\S]*https auth: missing GITHUB_TOKEN/iu.test(text))
|| (mirror.githubTransport.mode === "https" && /transport=ssh\b/iu.test(text));
if (scriptTransportMismatch) {
return {
retryable: false,
retryAttempt: attempt,
retryMaxAttempts,
retryLabel: `${attempt}/${retryMaxAttempts}`,
retryExhausted: false,
stopped: true,
reason: `Git mirror script transport does not match YAML githubTransport.mode=${mirror.githubTransport.mode}; apply the node control-plane so the git-mirror ConfigMap is updated before retrying.`,
refSources: nodeRuntimeGitMirrorRefSources(scoped, mirror),
githubTransport: nodeRuntimeGitMirrorGithubTransportSummary(mirror),
next: {
applyControlPlane: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`,
},
valuesPrinted: false,
};
}
const authFailure = /Authentication failed|Invalid username or password|Repository not found|could not read Username|could not read Password|terminal prompts disabled|https auth: missing GITHUB_TOKEN/iu.test(text);
if (authFailure) {
return {
@@ -3777,10 +3806,12 @@ function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDeleg
}
function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", "json"], 60);
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRuns(parseJsonRecordFromText(taskRunsResult.stdout)) : [];
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPods(parseJsonRecordFromText(podsResult.stdout)) : [];
const taskRunTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineTask"}}{{"\\t"}}{{if .spec.taskRef}}{{.spec.taskRef.name}}{{end}}{{"\\t"}}{{with index .status.conditions 0}}{{.status}}{{"\\t"}}{{.reason}}{{else}}{{"\\t"}}{{end}}{{"\\t"}}{{.status.podName}}{{"\\t"}}{{with index .status.conditions 0}}{{printf "%.600s" .message}}{{end}}{{"\\n"}}{{end}}`;
const podTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/taskRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "PodScheduled"}}{{.status}}|{{.reason}}|{{printf "%.600s" .message}}{{end}}{{end}}{{"\\t"}}{{range .status.initContainerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\t"}}{{range .status.containerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\n"}}{{end}}`;
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${taskRunTemplate}`], 60);
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${podTemplate}`], 60);
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRunsFromTsv(taskRunsResult.stdout) : [];
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPodsFromTsv(podsResult.stdout) : [];
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
@@ -3852,6 +3883,67 @@ function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineR
};
}
function nodeRuntimePipelineDiagnosticTaskRunsFromTsv(text: string): Array<Record<string, unknown>> {
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
const parts = line.split("\t");
const [name = "", pipelineTask = "", taskRef = "", status = "", reason = "", podName = "", ...messageParts] = parts;
const message = messageParts.join("\t");
return {
name: stringOrNull(name),
pipelineTask: stringOrNull(pipelineTask),
taskRef: stringOrNull(taskRef),
status: stringOrNull(status),
reason: stringOrNull(reason),
message: diagnosticText(message),
podName: stringOrNull(podName),
steps: [],
failedSteps: [],
};
});
}
function nodeRuntimePipelineDiagnosticPodsFromTsv(text: string): Array<Record<string, unknown>> {
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
const [name = "", taskRun = "", phase = "", nodeName = "", scheduledRaw = "", initRaw = "", containersRaw = ""] = line.split("\t");
const [scheduledStatus = "", scheduledReason = "", scheduledMessage = ""] = scheduledRaw.split("|");
const initContainers = nodeRuntimePipelineContainerStatusesFromCompact(initRaw);
const containers = nodeRuntimePipelineContainerStatusesFromCompact(containersRaw);
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
return {
name: stringOrNull(name),
taskRun: stringOrNull(taskRun),
phase: stringOrNull(phase),
nodeName: stringOrNull(nodeName),
scheduled: scheduledStatus.length === 0 ? null : scheduledStatus === "True",
scheduledReason: stringOrNull(scheduledReason),
scheduledMessage: diagnosticText(scheduledMessage),
initContainers,
containers,
failedContainers,
};
});
}
function nodeRuntimePipelineContainerStatusesFromCompact(text: string): Array<Record<string, unknown>> {
return text.split(",").map((item) => item.trim()).filter(Boolean).map((item) => {
const [name = "", stateOrExit = "", reason = ""] = item.split(":");
const exitCode = /^[0-9]+$/u.test(stateOrExit) ? Number(stateOrExit) : null;
const state = exitCode !== null ? "terminated" : stateOrExit === "waiting" ? "waiting" : stateOrExit === "running" ? "running" : null;
return {
name: stringOrNull(name),
ready: null,
restartCount: null,
state,
failed: exitCode !== null && exitCode !== 0,
exitCode,
reason: stringOrNull(reason),
message: null,
startedAt: null,
finishedAt: null,
};
});
}
function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): Array<Record<string, unknown>> {
const items = Array.isArray(json.items) ? json.items.map(record) : [];
return items.map((item) => {
@@ -5135,6 +5227,87 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
" }",
" return false;",
"}",
"function patchGitMirrorTransportYaml() {",
" const mirror = overlay.gitMirror || {};",
" const transport = mirror.githubTransport || {};",
" if (transport.mode !== 'https') return;",
" if (!YAML) throw new Error('yaml module is required to patch git-mirror githubTransport=https');",
" const requireString = (pathName, value) => {",
" if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror githubTransport=https');",
" return value;",
" };",
" const repository = requireString('gitMirror.sourceRepository', mirror.sourceRepository);",
" const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);",
" const tokenSecretName = requireString('gitMirror.githubTransport.tokenSecretName', transport.tokenSecretName);",
" const tokenSecretKey = requireString('gitMirror.githubTransport.tokenSecretKey', transport.tokenSecretKey);",
" const tokenSourceRef = requireString('gitMirror.githubTransport.tokenSourceRef', transport.tokenSourceRef);",
" const username = typeof transport.username === 'string' && transport.username ? transport.username : 'x-access-token';",
" const remoteUrl = `https://github.com/${repository}.git`;",
" const proxySummary = `transport=https proxy=HTTP_PROXY authSecret=${tokenSecretName} authKey=${tokenSecretKey} authSourceRef=${tokenSourceRef} source=yaml`;",
" const askpassBlock = [",
" 'if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo \\'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env\\' >&2; exit 64; fi',",
" \"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'\",",
" '#!/bin/sh',",
" 'case \"$1\" in',",
" ' *Username*) printf \\'%s\\\\n\\' \"${GITHUB_USERNAME:-x-access-token}\" ;;',",
" ' *Password*) printf \\'%s\\\\n\\' \"$GITHUB_TOKEN\" ;;',",
" \" *) printf '\\\\n' ;;\",",
" 'esac',",
" 'SH_ASKPASS',",
" 'chmod 0700 /tmp/hwlab-git-askpass.sh',",
" `export GITHUB_USERNAME=${shellSingle(username)}`,",
" 'export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh',",
" 'export GIT_TERMINAL_PROMPT=0',",
" 'unset GIT_SSH',",
" 'unset GIT_SSH_COMMAND',",
" ].join('\\n');",
" function patchScript(script, key) {",
" let next = String(script || '');",
" if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);",
" let remoteReplaced = false;",
" next = next.replace(/repo_url=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `repo_url=${shellSingle(remoteUrl)}`; });",
" next = next.replace(/remote=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `remote=${shellSingle(remoteUrl)}`; });",
" if (!remoteReplaced) throw new Error(`generated git-mirror ${key} missing remote url assignment for githubTransport=https`);",
" next = next.replace(/transport=ssh ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
" next = next.replace(/ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
" next = next.replace(/mkdir -p ([^\\n]*?) \\/root\\/\\.ssh/g, 'mkdir -p $1');",
" next = next.replace(/\\n[ \\t]*mkdir -p \\/root\\/\\.ssh\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*cp \\/git-ssh\\/ssh-privatekey[^\\n]*\\n[ \\t]*chmod 0?400[^\\n]*\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) <<'NODE_PROXY'[\\s\\S]*?\\nNODE_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs)\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-git-ssh-proxy\\.sh <<'SH_PROXY'[\\s\\S]*?\\nSH_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-git-ssh-proxy\\.sh\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*export GIT_SSH=.*\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*export GIT_SSH_COMMAND=.*\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*unset GIT_SSH_COMMAND\\n/g, '\\n');",
" if (!next.includes('hwlab git-mirror https auth: missing GITHUB_TOKEN secret env')) {",
" const noProxyExport = /\\nexport no_proxy=[^\\n]*\\n/;",
" if (noProxyExport.test(next)) next = next.replace(noProxyExport, (match) => `${match}${askpassBlock}\\n`);",
" else if (next.includes('\\nset -eu\\n')) next = next.replace('\\nset -eu\\n', `\\nset -eu\\n${askpassBlock}\\n`);",
" else next = `${askpassBlock}\\n${next}`;",
" }",
" if (next.includes('/git-ssh') || next.includes('ssh://git@') || next.includes('GIT_SSH=')) throw new Error(`generated git-mirror ${key} still contains ssh transport after githubTransport=https patch`);",
" if (!next.includes('GIT_ASKPASS') || !next.includes('GITHUB_TOKEN')) throw new Error(`generated git-mirror ${key} missing https auth after githubTransport=https patch`);",
" return next;",
" }",
" const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');",
" if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);",
" const docs = YAML.parseAllDocuments(fs.readFileSync(gitMirrorFile, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
" const manifests = [];",
" for (const doc of docs) {",
" if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);",
" else manifests.push(doc);",
" }",
" let changed = false;",
" for (const doc of manifests) {",
" if (!doc || typeof doc !== 'object' || doc.kind !== 'ConfigMap') continue;",
" if (!doc.metadata || doc.metadata.name !== configMapName) continue;",
" doc.data = doc.data || {};",
" doc.data['sync.sh'] = patchScript(doc.data['sync.sh'], 'sync.sh');",
" doc.data['flush.sh'] = patchScript(doc.data['flush.sh'], 'flush.sh');",
" changed = true;",
" }",
" if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
"}",
"const structured = patchStructuredPipeline();",
"function replaceParamDefault(name, value) {",
" const namePattern = escapeRegExp(name);",
@@ -5187,6 +5360,7 @@ function nodeRuntimePipelinePostprocessScript(): string[] {
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && !text.includes('NODE_UNIDESK_YAML_DEPENDENCY')) { throw new Error(`generated pipeline missing UniDesk yaml dependency install in ${pipelinePath}`); }",
"if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }",
"fs.writeFileSync(pipelinePath, text);",
"patchGitMirrorTransportYaml();",
"function patchArgoYaml(filePath) {",
" if (!YAML || !fs.existsSync(filePath)) return;",
" const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
@@ -5304,6 +5478,7 @@ function httpProxyEndpoint(value: string): { host: string; port: number } | null
function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string): string[] {
return [
`${renderDir}/devops-infra/git-mirror.yaml`,
`${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`,
`${renderDir}/${spec.tektonDir}/rbac.yaml`,
`${renderDir}/${spec.tektonDir}/pipeline.yaml`,