diff --git a/config/deploy-ssh-identities.yaml b/config/deploy-ssh-identities.yaml index 7cb7667a..7457fbc6 100644 --- a/config/deploy-ssh-identities.yaml +++ b/config/deploy-ssh-identities.yaml @@ -54,7 +54,8 @@ targets: route: JD01 homeDir: /root egress: - mode: direct + mode: http-connect-proxy + proxyUrl: http://127.0.0.1:10808 identities: - github.com G14: diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 361d6d8d..c95f7e95 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -785,6 +785,14 @@ lanes: node: JD01 workspace: /root/workspace/hwlab-v03 sourceWorkspace: + git: + remoteName: origin + remoteUrl: git@github.com:pikasTech/HWLAB.git + identityTarget: JD01 + identityId: github.com + proxyEnvPath: /etc/unidesk/proxy.env + verifyRemote: true + requireUpToDate: true requiredCommands: - git - node diff --git a/config/platform-infra/host-proxy.yaml b/config/platform-infra/host-proxy.yaml index 82f697cb..c8bbfff4 100644 --- a/config/platform-infra/host-proxy.yaml +++ b/config/platform-infra/host-proxy.yaml @@ -106,6 +106,11 @@ targets: apt: /etc/apt/apt.conf.d/90unidesk-proxy dockerSystemdDropIn: /etc/systemd/system/docker.service.d/10-unidesk-proxy.conf k3sSystemdDropIn: /etc/systemd/system/k3s.service.d/10-unidesk-proxy.conf + trans: + hostProxyEnv: + enabled: true + envFileRef: files.envFile + applyTo: host-posix-commands apply: reloadSystemd: true restartDocker: true diff --git a/scripts/src/deploy-ssh-identity.ts b/scripts/src/deploy-ssh-identity.ts index fbde5d43..f9398b12 100644 --- a/scripts/src/deploy-ssh-identity.ts +++ b/scripts/src/deploy-ssh-identity.ts @@ -623,24 +623,55 @@ async function distributeGithubIdentity(config: UniDeskConfig, target: DeploySsh `host=${shellQuote(identitySpec.host)}`, `egress_mode=${shellQuote(target.egress.mode)}`, `proxy_url=${target.egress.mode === "http-connect-proxy" ? shellQuote(target.egress.proxyUrl) : "''"}`, - "git_ssh_proxy=/tmp/unidesk-git-ssh-http-connect.py", + "ssh_dir=\"$HOME/.ssh\"", + "git_ssh_proxy=\"$ssh_dir/unidesk-git-ssh-http-connect.py\"", + "ssh_config=\"$ssh_dir/config\"", "if [ \"$egress_mode\" = \"http-connect-proxy\" ]; then", " curl -fsSL --max-time 20 -x \"$proxy_url\" \"https://$host\" -o /dev/null", "else", " curl -fsSL --max-time 20 \"https://$host\" -o /dev/null", "fi", + "tmp_config=$(mktemp)", + "touch \"$ssh_config\"", + "chmod 600 \"$ssh_config\"", + "awk '/^# BEGIN unidesk managed github-ssh-identity$/{skip=1; next} /^# END unidesk managed github-ssh-identity$/{skip=0; next} !skip{print}' \"$ssh_config\" > \"$tmp_config\"", "if [ \"$egress_mode\" = \"http-connect-proxy\" ]; then", " cat > \"$git_ssh_proxy\" <<'UNIDESK_GIT_SSH_PROXY'", proxyPython, "UNIDESK_GIT_SSH_PROXY", " chmod 700 \"$git_ssh_proxy\"", - " export UNIDESK_GIT_SSH_HTTP_PROXY=\"$proxy_url\"", - " export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o 'ProxyCommand=$git_ssh_proxy %h %p'\"", - " auth_output=$(ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -o \"ProxyCommand=$git_ssh_proxy %h %p\" -T \"git@$host\" 2>&1 || true)", + " {", + " printf '%s\\n' '# BEGIN unidesk managed github-ssh-identity'", + " printf 'Host %s\\n' \"$host\"", + " printf ' HostName %s\\n' \"$host\"", + " printf '%s\\n' ' User git'", + " printf ' IdentityFile %s/.ssh/id_ed25519\\n' \"$HOME\"", + " printf '%s\\n' ' IdentitiesOnly yes'", + " printf '%s\\n' ' BatchMode yes'", + " printf '%s\\n' ' StrictHostKeyChecking yes'", + " printf ' UserKnownHostsFile %s/.ssh/known_hosts\\n' \"$HOME\"", + " printf ' ProxyCommand env UNIDESK_GIT_SSH_HTTP_PROXY=%s %s %%h %%p\\n' \"$proxy_url\" \"$git_ssh_proxy\"", + " printf '%s\\n' '# END unidesk managed github-ssh-identity'", + " cat \"$tmp_config\"", + " } > \"$ssh_config\"", "else", - " export GIT_SSH_COMMAND=\"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519\"", - " auth_output=$(ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$HOME/.ssh/known_hosts -i $HOME/.ssh/id_ed25519 -T \"git@$host\" 2>&1 || true)", + " {", + " printf '%s\\n' '# BEGIN unidesk managed github-ssh-identity'", + " printf 'Host %s\\n' \"$host\"", + " printf ' HostName %s\\n' \"$host\"", + " printf '%s\\n' ' User git'", + " printf ' IdentityFile %s/.ssh/id_ed25519\\n' \"$HOME\"", + " printf '%s\\n' ' IdentitiesOnly yes'", + " printf '%s\\n' ' BatchMode yes'", + " printf '%s\\n' ' StrictHostKeyChecking yes'", + " printf ' UserKnownHostsFile %s/.ssh/known_hosts\\n' \"$HOME\"", + " printf '%s\\n' '# END unidesk managed github-ssh-identity'", + " cat \"$tmp_config\"", + " } > \"$ssh_config\"", "fi", + "rm -f \"$tmp_config\"", + "chmod 600 \"$ssh_config\"", + "auth_output=$(ssh -F \"$ssh_config\" -T \"git@$host\" 2>&1 || true)", "printf '%s\\n' \"$auth_output\"", "printf '%s\\n' \"$auth_output\" | grep -q 'successfully authenticated'", ].join("\n"); diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index 67c4f7b2..edd78326 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -353,6 +353,7 @@ export interface HwlabRuntimeCodeAgentRuntimeSpec { } export interface HwlabRuntimeSourceWorkspaceSpec { + readonly git?: HwlabRuntimeSourceWorkspaceGitSpec; readonly requiredCommands: readonly string[]; readonly requiredFiles: readonly string[]; readonly hostDependencies?: HwlabRuntimeSourceWorkspaceHostDependenciesSpec; @@ -365,6 +366,16 @@ export interface HwlabRuntimeSourceWorkspaceSpec { }; } +export interface HwlabRuntimeSourceWorkspaceGitSpec { + readonly remoteName: string; + readonly remoteUrl: string; + readonly identityTarget?: string; + readonly identityId?: string; + readonly proxyEnvPath?: string; + readonly verifyRemote: boolean; + readonly requireUpToDate: boolean; +} + export interface HwlabRuntimeSourceWorkspaceHostDependenciesSpec { readonly checkCommands: readonly string[]; readonly stateDir: string; @@ -873,11 +884,31 @@ function absoluteHostPathField(value: string, path: string): string { return value; } +function gitRemoteNameField(value: string, path: string): string { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a git remote name`); + return value; +} + +function sourceWorkspaceGitConfig(value: unknown, path: string): HwlabRuntimeSourceWorkspaceGitSpec | undefined { + if (value === undefined) return undefined; + const raw = asRecord(value, path); + return { + remoteName: gitRemoteNameField(stringField(raw, "remoteName", path), `${path}.remoteName`), + remoteUrl: stringField(raw, "remoteUrl", path), + ...(raw.identityTarget === undefined ? {} : { identityTarget: stringField(raw, "identityTarget", path) }), + ...(raw.identityId === undefined ? {} : { identityId: stringField(raw, "identityId", path) }), + ...(raw.proxyEnvPath === undefined ? {} : { proxyEnvPath: absoluteHostPathField(stringField(raw, "proxyEnvPath", path), `${path}.proxyEnvPath`) }), + verifyRemote: booleanField(raw, "verifyRemote", path), + requireUpToDate: booleanField(raw, "requireUpToDate", path), + }; +} + function sourceWorkspaceConfig(value: unknown, path: string): HwlabRuntimeSourceWorkspaceSpec | undefined { if (value === undefined) return undefined; const raw = asRecord(value, path); const install = asRecord(raw.install, `${path}.install`); return { + git: sourceWorkspaceGitConfig(raw.git, `${path}.git`), requiredCommands: stringArrayField(raw, "requiredCommands", path) .map((item, index) => commandNameField(item, `${path}.requiredCommands[${index}]`)), requiredFiles: stringArrayField(raw, "requiredFiles", path) diff --git a/scripts/src/hwlab-node/source-workspace.ts b/scripts/src/hwlab-node/source-workspace.ts index 9a3f0502..d06c0dac 100644 --- a/scripts/src/hwlab-node/source-workspace.ts +++ b/scripts/src/hwlab-node/source-workspace.ts @@ -455,11 +455,16 @@ function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, so return [ "set +e", `workspace=${shellQuote(spec.workspace)}`, + `expected_branch=${shellQuote(spec.sourceBranch)}`, `state_dir=${shellQuote(stateDir)}`, `dependency_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.dependencyCommand, "utf8").toString("base64"))}`, `browser_command_b64=${shellQuote(Buffer.from(sourceWorkspace.install.browserCommand, "utf8").toString("base64"))}`, `required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`, `timeout_seconds=${String(sourceWorkspace.install.timeoutSeconds)}`, + `remote_name=${shellQuote(sourceWorkspace.git?.remoteName ?? "origin")}`, + `remote_url=${shellQuote(sourceWorkspace.git?.remoteUrl ?? "")}`, + `proxy_env_path=${shellQuote(sourceWorkspace.git?.proxyEnvPath ?? "")}`, + `git_timeout=${String(spec.downloadProfile.git.timeoutSeconds)}`, `http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`, `https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`, `all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`, @@ -489,11 +494,15 @@ function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, so "NODE", "(", " set +e", - " export HTTP_PROXY=\"$http_proxy_value\" HTTPS_PROXY=\"$https_proxy_value\" ALL_PROXY=\"$all_proxy_value\" NO_PROXY=\"$no_proxy_value\"", - " export http_proxy=\"$http_proxy_value\" https_proxy=\"$https_proxy_value\" all_proxy=\"$all_proxy_value\" no_proxy=\"$no_proxy_value\"", + " if [ -n \"$proxy_env_path\" ] && [ -f \"$proxy_env_path\" ]; then . \"$proxy_env_path\" 2>/dev/null || true; fi", + " export HTTP_PROXY=\"${HTTP_PROXY:-$http_proxy_value}\" HTTPS_PROXY=\"${HTTPS_PROXY:-$https_proxy_value}\" ALL_PROXY=\"${ALL_PROXY:-$all_proxy_value}\" NO_PROXY=\"${NO_PROXY:-$no_proxy_value}\"", + " export http_proxy=\"${http_proxy:-$HTTP_PROXY}\" https_proxy=\"${https_proxy:-$HTTPS_PROXY}\" all_proxy=\"${all_proxy:-$ALL_PROXY}\" no_proxy=\"${no_proxy:-$NO_PROXY}\"", " failure_kind=", " dependency_exit=1", " browser_exit=1", + " git_fetch_exit=0", + " git_update_exit=0", + " remote_url_after=", " missing_files=", " source_commit=", " status_short=", @@ -511,15 +520,36 @@ function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, so " failure_kind=workspace-dirty", " printf '%s\\n' \"source workspace is dirty; refusing to install dependencies\" >>\"$stderr_file\"", " else", - " timeout \"$timeout_seconds\" /bin/bash \"$dependency_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"", - " dependency_exit=$?", - " if [ \"$dependency_exit\" -eq 0 ]; then", - " timeout \"$timeout_seconds\" /bin/bash \"$browser_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"", - " browser_exit=$?", - " else", - " failure_kind=dependency-command-failed", + " if [ -n \"$remote_url\" ]; then", + " git remote set-url \"$remote_name\" \"$remote_url\" >>\"$stdout_file\" 2>>\"$stderr_file\" || git remote add \"$remote_name\" \"$remote_url\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " remote_url_after=$(git remote get-url \"$remote_name\" 2>/dev/null || true)", + " timeout \"$git_timeout\" git fetch \"$remote_name\" \"$expected_branch:refs/remotes/$remote_name/$expected_branch\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " git_fetch_exit=$?", + " if [ \"$git_fetch_exit\" -ne 0 ]; then", + " failure_kind=git-fetch-failed", + " else", + " current_branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)", + " if [ \"$current_branch\" = \"$expected_branch\" ]; then", + " git merge --ff-only \"refs/remotes/$remote_name/$expected_branch\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " git_update_exit=$?", + " else", + " git checkout -B \"$expected_branch\" \"refs/remotes/$remote_name/$expected_branch\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " git_update_exit=$?", + " fi", + " if [ \"$git_update_exit\" -ne 0 ]; then failure_kind=git-update-failed; fi", + " fi", + " fi", + " if [ -z \"$failure_kind\" ]; then", + " timeout \"$timeout_seconds\" /bin/bash \"$dependency_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " dependency_exit=$?", + " if [ \"$dependency_exit\" -eq 0 ]; then", + " timeout \"$timeout_seconds\" /bin/bash \"$browser_file\" >>\"$stdout_file\" 2>>\"$stderr_file\"", + " browser_exit=$?", + " else", + " failure_kind=dependency-command-failed", + " fi", + " if [ \"$dependency_exit\" -eq 0 ] && [ \"$browser_exit\" -ne 0 ]; then failure_kind=browser-command-failed; fi", " fi", - " if [ \"$dependency_exit\" -eq 0 ] && [ \"$browser_exit\" -ne 0 ]; then failure_kind=browser-command-failed; fi", " while IFS= read -r file_path; do", " [ -z \"$file_path\" ] && continue", " if [ ! -e \"$workspace/$file_path\" ]; then missing_files=\"$missing_files${missing_files:+,}$file_path\"; fi", @@ -532,15 +562,17 @@ function sourceWorkspaceHostBootstrapSubmitScript(spec: HwlabRuntimeLaneSpec, so " npx_version=$(npx --version 2>/dev/null || true)", " fi", " fi", - " JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" FAILURE_KIND=\"$failure_kind\" DEPENDENCY_EXIT=\"$dependency_exit\" BROWSER_EXIT=\"$browser_exit\" MISSING_FILES=\"$missing_files\" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" NPX_VERSION=\"$npx_version\" node <<'NODE' >\"$status_file\"", + " JOB_ID=\"$job_id\" WORKSPACE=\"$workspace\" STATE_DIR=\"$state_dir\" STATUS_FILE=\"$status_file\" STDOUT_FILE=\"$stdout_file\" STDERR_FILE=\"$stderr_file\" COMMAND_SHA=\"$command_sha\" FAILURE_KIND=\"$failure_kind\" DEPENDENCY_EXIT=\"$dependency_exit\" BROWSER_EXIT=\"$browser_exit\" GIT_FETCH_EXIT=\"$git_fetch_exit\" GIT_UPDATE_EXIT=\"$git_update_exit\" REMOTE_URL_AFTER=\"$remote_url_after\" MISSING_FILES=\"$missing_files\" SOURCE_COMMIT=\"$source_commit\" STATUS_SHORT=\"$status_short\" NODE_VERSION=\"$node_version\" NPM_VERSION=\"$npm_version\" NPX_VERSION=\"$npx_version\" node <<'NODE' >\"$status_file\"", "const fs = require('fs');", "const tail = (file) => { try { return fs.readFileSync(file, 'utf8').slice(-4000); } catch { return ''; } };", "const missingFiles = (process.env.MISSING_FILES || '').split(',').filter(Boolean);", "const dependencyExit = Number(process.env.DEPENDENCY_EXIT || '1');", "const browserExit = Number(process.env.BROWSER_EXIT || '1');", + "const gitFetchExit = Number(process.env.GIT_FETCH_EXIT || '0');", + "const gitUpdateExit = Number(process.env.GIT_UPDATE_EXIT || '0');", "const statusShort = process.env.STATUS_SHORT || '';", "const ok = dependencyExit === 0 && browserExit === 0 && missingFiles.length === 0 && !statusShort && !process.env.FAILURE_KIND;", - "console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, failureKind: process.env.FAILURE_KIND || null, dependencyExitCode: dependencyExit, browserExitCode: browserExit, missingFiles, sourceCommit: process.env.SOURCE_COMMIT || null, workspaceClean: !statusShort, statusShort: statusShort || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, npxVersion: process.env.NPX_VERSION || null, stdoutTail: tail(process.env.STDOUT_FILE), stderrTail: tail(process.env.STDERR_FILE), finishedAt: new Date().toISOString(), valuesPrinted: false }));", + "console.log(JSON.stringify({ ok, status: ok ? 'succeeded' : 'failed', jobId: process.env.JOB_ID, workspace: process.env.WORKSPACE, stateDir: process.env.STATE_DIR, statusFile: process.env.STATUS_FILE, stdoutFile: process.env.STDOUT_FILE, stderrFile: process.env.STDERR_FILE, installCommandSha256: process.env.COMMAND_SHA, failureKind: process.env.FAILURE_KIND || null, dependencyExitCode: dependencyExit, browserExitCode: browserExit, gitFetchExitCode: gitFetchExit, gitUpdateExitCode: gitUpdateExit, remoteUrlAfter: process.env.REMOTE_URL_AFTER || null, missingFiles, sourceCommit: process.env.SOURCE_COMMIT || null, workspaceClean: !statusShort, statusShort: statusShort || null, nodeVersion: process.env.NODE_VERSION || null, npmVersion: process.env.NPM_VERSION || null, npxVersion: process.env.NPX_VERSION || null, stdoutTail: tail(process.env.STDOUT_FILE), stderrTail: tail(process.env.STDERR_FILE), finishedAt: new Date().toISOString(), valuesPrinted: false }));", "NODE", ") >/dev/null 2>&1 &", "pid=$!", @@ -599,6 +631,15 @@ function sourceWorkspaceExpected(spec: HwlabRuntimeLaneSpec, sourceWorkspace: Hw return { workspace: spec.workspace, sourceBranch: spec.sourceBranch, + sourceWorkspaceGit: sourceWorkspace.git === undefined ? null : { + remoteName: sourceWorkspace.git.remoteName, + remoteUrl: sourceWorkspace.git.remoteUrl, + identityTarget: sourceWorkspace.git.identityTarget ?? null, + identityId: sourceWorkspace.git.identityId ?? null, + proxyEnvPath: sourceWorkspace.git.proxyEnvPath ?? null, + verifyRemote: sourceWorkspace.git.verifyRemote, + requireUpToDate: sourceWorkspace.git.requireUpToDate, + }, git: { url: spec.gitUrl, readUrl: spec.gitReadUrl, @@ -634,12 +675,18 @@ function sourceWorkspaceHostDependenciesExpected(hostDependencies: HwlabRuntimeS function sourceWorkspaceStatusFromFields(fields: Record, exitCode: number | null): Record { const missingCommands = commaListField(fields.missingCommands); const missingFiles = commaListField(fields.missingFiles); + const remoteReachabilityRequired = fields.remoteReachabilityRequired === "yes"; + const remoteUpToDateRequired = fields.remoteUpToDateRequired === "yes"; + const remoteReachable = fields.remoteReachable === "yes"; + const remoteHead = fields.remoteHead || ""; const ok = exitCode === 0 && fields.workspaceExists === "yes" && fields.gitDirExists === "yes" && fields.workspaceClean === "yes" && fields.branch === fields.expectedBranch && fields.remoteMatchesYaml === "yes" + && (!remoteReachabilityRequired || remoteReachable) + && (!remoteUpToDateRequired || remoteHead.length === 0 || fields.localHead === remoteHead) && missingCommands.length === 0 && missingFiles.length === 0 && fields.playwrightPackagePresent === "yes" @@ -656,8 +703,16 @@ function sourceWorkspaceStatusFromFields(fields: Record, exitCod branch: fields.branch || null, detached: fields.detached === "yes", localHead: fields.localHead || null, + expectedRemoteUrl: fields.expectedRemoteUrl || null, remoteUrl: fields.remoteUrl || null, remoteMatchesYaml: fields.remoteMatchesYaml === "yes", + remoteReachabilityRequired, + remoteUpToDateRequired, + remoteReachable, + remoteHead: remoteHead || null, + remoteProbeExitCode: numericField(fields.remoteProbeExitCode), + remoteProbeErrorTail: fields.remoteProbeErrorTail || null, + remoteUpToDate: remoteHead.length === 0 ? null : fields.localHead === remoteHead, requiredCommands: commaListField(fields.requiredCommands), missingCommands, requiredFiles: commaListField(fields.requiredFiles), @@ -795,11 +850,20 @@ function sourceWorkspaceHostDependenciesSubmitScript( function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace: HwlabRuntimeSourceWorkspaceSpec): string { const requiredCommands = shellLinesText(sourceWorkspace.requiredCommands); const requiredFiles = shellLinesText(sourceWorkspace.requiredFiles); - const remoteUrls = shellLinesText([spec.gitUrl, spec.gitReadUrl, spec.gitWriteUrl]); + const remoteUrls = shellLinesText(sourceWorkspace.git === undefined ? [spec.gitUrl, spec.gitReadUrl, spec.gitWriteUrl] : [sourceWorkspace.git.remoteUrl]); return [ "set +e", `workspace=${shellQuote(spec.workspace)}`, `expected_branch=${shellQuote(spec.sourceBranch)}`, + `remote_name=${shellQuote(sourceWorkspace.git?.remoteName ?? "origin")}`, + `verify_remote=${shellQuote(sourceWorkspace.git?.verifyRemote === true ? "yes" : "no")}`, + `require_up_to_date=${shellQuote(sourceWorkspace.git?.requireUpToDate === true ? "yes" : "no")}`, + `proxy_env_path=${shellQuote(sourceWorkspace.git?.proxyEnvPath ?? "")}`, + `git_timeout=${String(spec.downloadProfile.git.timeoutSeconds)}`, + `http_proxy_value=${shellQuote(spec.networkProfile.proxy.http)}`, + `https_proxy_value=${shellQuote(spec.networkProfile.proxy.https)}`, + `all_proxy_value=${shellQuote(spec.networkProfile.proxy.all)}`, + `no_proxy_value=${shellQuote(spec.networkProfile.proxy.noProxy.join(","))}`, `required_commands_b64=${shellQuote(Buffer.from(requiredCommands, "utf8").toString("base64"))}`, `required_files_b64=${shellQuote(Buffer.from(requiredFiles, "utf8").toString("base64"))}`, `remote_urls_b64=${shellQuote(Buffer.from(remoteUrls, "utf8").toString("base64"))}`, @@ -808,6 +872,7 @@ function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace "printf '%s' \"$required_commands_b64\" | base64 -d >\"$tmp_dir/commands.txt\"", "printf '%s' \"$required_files_b64\" | base64 -d >\"$tmp_dir/files.txt\"", "printf '%s' \"$remote_urls_b64\" | base64 -d >\"$tmp_dir/remotes.txt\"", + "expected_remote_url=$(head -n 1 \"$tmp_dir/remotes.txt\" 2>/dev/null || true)", "workspace_exists=no", "git_dir_exists=no", "workspace_clean=no", @@ -816,6 +881,10 @@ function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace "local_head=", "remote_url=", "remote_matches_yaml=no", + "remote_reachable=skipped", + "remote_head=", + "remote_probe_exit=", + "remote_probe_error_tail=", "status_short=", "if [ -d \"$workspace\" ]; then", " workspace_exists=yes", @@ -824,10 +893,20 @@ function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace " branch=$(git -C \"$workspace\" rev-parse --abbrev-ref HEAD 2>/dev/null || true)", " [ \"$branch\" = HEAD ] && detached=yes", " local_head=$(git -C \"$workspace\" rev-parse HEAD 2>/dev/null || true)", - " remote_url=$(git -C \"$workspace\" remote get-url origin 2>/dev/null || true)", + " remote_url=$(git -C \"$workspace\" remote get-url \"$remote_name\" 2>/dev/null || true)", " status_short=$(git -C \"$workspace\" status --short 2>/dev/null || true)", " [ -z \"$status_short\" ] && workspace_clean=yes", " while IFS= read -r expected_remote; do [ \"$remote_url\" = \"$expected_remote\" ] && remote_matches_yaml=yes; done <\"$tmp_dir/remotes.txt\"", + " if [ \"$verify_remote\" = yes ] && [ -n \"$expected_remote_url\" ]; then", + " if [ -n \"$proxy_env_path\" ] && [ -f \"$proxy_env_path\" ]; then . \"$proxy_env_path\" 2>/dev/null || true; fi", + " export HTTP_PROXY=\"${HTTP_PROXY:-$http_proxy_value}\" HTTPS_PROXY=\"${HTTPS_PROXY:-$https_proxy_value}\" ALL_PROXY=\"${ALL_PROXY:-$all_proxy_value}\" NO_PROXY=\"${NO_PROXY:-$no_proxy_value}\"", + " export http_proxy=\"${http_proxy:-$HTTP_PROXY}\" https_proxy=\"${https_proxy:-$HTTPS_PROXY}\" all_proxy=\"${all_proxy:-$ALL_PROXY}\" no_proxy=\"${no_proxy:-$NO_PROXY}\"", + " timeout \"$git_timeout\" git -C \"$workspace\" ls-remote \"$expected_remote_url\" \"refs/heads/$expected_branch\" >\"$tmp_dir/remote.out\" 2>\"$tmp_dir/remote.err\"", + " remote_probe_exit=$?", + " remote_head=$(awk '{print $1}' \"$tmp_dir/remote.out\" | head -n 1)", + " remote_probe_error_tail=$(tail -c 1000 \"$tmp_dir/remote.err\" 2>/dev/null | tr '\\n\\t' ' ' | cut -c1-1000)", + " if [ \"$remote_probe_exit\" = 0 ] && [ -n \"$remote_head\" ]; then remote_reachable=yes; else remote_reachable=no; fi", + " fi", " fi", "fi", "missing_commands=", @@ -907,8 +986,15 @@ function sourceWorkspaceStatusScript(spec: HwlabRuntimeLaneSpec, sourceWorkspace "printf 'branch\\t%s\\n' \"$branch\"", "printf 'detached\\t%s\\n' \"$detached\"", "printf 'localHead\\t%s\\n' \"$local_head\"", + "printf 'expectedRemoteUrl\\t%s\\n' \"$expected_remote_url\"", "printf 'remoteUrl\\t%s\\n' \"$remote_url\"", "printf 'remoteMatchesYaml\\t%s\\n' \"$remote_matches_yaml\"", + "printf 'remoteReachabilityRequired\\t%s\\n' \"$verify_remote\"", + "printf 'remoteUpToDateRequired\\t%s\\n' \"$require_up_to_date\"", + "printf 'remoteReachable\\t%s\\n' \"$remote_reachable\"", + "printf 'remoteHead\\t%s\\n' \"$remote_head\"", + "printf 'remoteProbeExitCode\\t%s\\n' \"$remote_probe_exit\"", + "printf 'remoteProbeErrorTail\\t%s\\n' \"$remote_probe_error_tail\"", "printf 'requiredCommands\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/commands.txt\" | sed 's/,$//')\"", "printf 'missingCommands\\t%s\\n' \"$missing_commands\"", "printf 'requiredFiles\\t%s\\n' \"$(tr '\\n' ',' <\"$tmp_dir/files.txt\" | sed 's/,$//')\"", diff --git a/scripts/src/platform-infra-host-proxy.ts b/scripts/src/platform-infra-host-proxy.ts index 800c1855..5d07ae53 100644 --- a/scripts/src/platform-infra-host-proxy.ts +++ b/scripts/src/platform-infra-host-proxy.ts @@ -8,6 +8,7 @@ import { resolveEgressProxySourceRef, type MasterShadowsocksSourceSpec } from ". import type { RenderedCliResult } from "./output"; import { shQuote } from "./platform-infra-public-service"; import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; +import { transHostProxyEnvSummary } from "./trans-host-proxy"; const configPath = "config/platform-infra/host-proxy.yaml"; @@ -853,6 +854,9 @@ function targetSummary(target: HostProxyTarget): Record { serviceName: client.serviceName, listen: `${client.listenHost}:${client.listenPort}`, }, + trans: { + hostProxyEnv: transHostProxyEnvSummary(target.id), + }, }; } @@ -924,6 +928,8 @@ function renderPlan(result: Record): RenderedCliResult { "", ...table(["PROXY", "NO_PROXY_COUNT", "HYUEAPI", "VALUES"], [[text(env.proxyUrl), text(env.noProxyCount), "preserved", "printed=false"]]), "", + ...transHostProxyEnvTable(target), + "", "NEXT", ` dry-run: ${text(next.dryRun, "")}`, ` apply: ${text(next.apply, "")}`, @@ -955,6 +961,8 @@ function renderApply(result: Record): RenderedCliResult { "", ...table(["SERVICE_ACTIVE", "BINARY_SHA", "HOST_PROBE", "POD_PROBE", "VALUES"], [[text(client.serviceActive), text(client.binarySha256Ok), text(probe.status), text(podProbe.status, "skipped"), "printed=false"]]), "", + ...transHostProxyEnvTable(target), + "", `NEXT\n ${text(record(result.next).status, "")}`, ]; return { ok: result.ok !== false, command: "platform-infra egress-proxy host apply", renderedText: lines.join("\n"), contentType: "text/plain" }; @@ -983,11 +991,22 @@ function renderStatus(result: Record): RenderedCliResult { "", ...table(["NO_PROXY_HYUEAPI", "NO_PROXY_WILDCARD", "VALUES"], [[text(noProxy.hyueapi), text(noProxy.wildcardHyueapi), "printed=false"]]), "", + ...transHostProxyEnvTable(target), + "", "Disclosure: file content, Secret values and generated proxy config are not printed.", ]; return { ok: result.ok !== false, command: "platform-infra egress-proxy host status", renderedText: lines.join("\n"), contentType: "text/plain" }; } +function transHostProxyEnvTable(target: Record): string[] { + const trans = record(target.trans); + const rule = record(trans.hostProxyEnv); + return table( + ["TRANS_PROXY_ENV", "ENV_REF", "APPLY_TO", "ENV_FILE"], + [[text(rule.enabled, "false"), text(rule.envFileRef, ""), text(rule.applyTo, "disabled"), text(rule.envFile, "")]], + ); +} + function option(args: string[], name: string): string | null { const index = args.indexOf(name); if (index === -1) return null; diff --git a/scripts/src/ssh.ts b/scripts/src/ssh.ts index 0a6dc793..58ce8d82 100644 --- a/scripts/src/ssh.ts +++ b/scripts/src/ssh.ts @@ -22,6 +22,7 @@ import { type SshRemoteCommandExecutor, type SshRemoteCommandStreamHandlers, } from "./ssh-file-transfer"; +import { readTransHostProxyEnvRule, type TransHostProxyEnvRule } from "./trans-host-proxy"; export interface ParsedSshArgs { remoteCommand: string | null; @@ -1064,7 +1065,8 @@ export function parseSshInvocation(target: string, args: string[]): ParsedSshInv if ((operationArgs[0] ?? "") === "k3s") { throw new Error(`ssh k3s shorthand is unsupported; use route syntax instead: trans ${route.providerId}:k3s ${operationArgs.slice(1).join(" ")}`.trim()); } - return { providerId: route.providerId, route, parsed: parseSshArgs(operationArgs, route.raw) }; + const parsed = parseSshArgs(operationArgs, route.raw); + return { providerId: route.providerId, route, parsed: withTransHostProxyEnv(route, parsed) }; } export function parseSshRoute(target: string): ParsedSshRoute { @@ -2428,6 +2430,22 @@ function shellScriptStdinPrefix(): string { return `${shellScriptPrelude()}\n`; } +function withTransHostProxyEnv(route: ParsedSshRoute, parsed: ParsedSshArgs): ParsedSshArgs { + if (route.plane !== "host" || parsed.remoteCommand === null) return parsed; + const rule = readTransHostProxyEnvRule(route.providerId); + if (rule === null || rule.applyTo !== "host-posix-commands") return parsed; + return { ...parsed, remoteCommand: `${transHostProxyEnvPrelude(rule)}; ${parsed.remoteCommand}` }; +} + +function transHostProxyEnvPrelude(rule: TransHostProxyEnvRule): string { + const envFile = shellQuote(rule.envFile); + return [ + `UNIDESK_TRANS_HOST_PROXY_ENV=${envFile}`, + 'if [ -f "$UNIDESK_TRANS_HOST_PROXY_ENV" ]; then set -a; . "$UNIDESK_TRANS_HOST_PROXY_ENV"; unidesk_trans_host_proxy_env_rc=$?; set +a; [ "$unidesk_trans_host_proxy_env_rc" -eq 0 ] || exit "$unidesk_trans_host_proxy_env_rc"; fi', + `export UNIDESK_TRANS_HOST_PROXY_ENV_SOURCE=${shellQuote(rule.configPathLabel)}`, + ].join("; "); +} + function buildShellCommand(args: string[], shell: "sh" | "bash", commandName: string): ParsedSshArgs { const scriptArgs: string[] = []; for (let index = 0; index < args.length; index += 1) { diff --git a/scripts/src/trans-host-proxy.ts b/scripts/src/trans-host-proxy.ts new file mode 100644 index 00000000..97ab97cf --- /dev/null +++ b/scripts/src/trans-host-proxy.ts @@ -0,0 +1,87 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { repoRoot } from "./config"; + +export const transHostProxyConfigPath = "config/platform-infra/host-proxy.yaml"; + +export type TransHostProxyEnvApplyTo = "host-posix-commands"; + +export interface TransHostProxyEnvRule { + targetId: string; + enabled: boolean; + envFile: string; + envFileRef: "files.envFile"; + applyTo: TransHostProxyEnvApplyTo; + configPath: string; + configPathLabel: string; +} + +export function readTransHostProxyEnvRule(targetId: string): TransHostProxyEnvRule | null { + const absolutePath = join(repoRoot, transHostProxyConfigPath); + if (!existsSync(absolutePath)) return null; + const root = record(Bun.YAML.parse(readFileSync(absolutePath, "utf8")), transHostProxyConfigPath); + if (root.kind !== "platform-infra-host-proxy") throw new Error(`${transHostProxyConfigPath}.kind must be platform-infra-host-proxy`); + const targets = record(root.targets, `${transHostProxyConfigPath}.targets`); + const targetRaw = targets[targetId]; + if (targetRaw === undefined) return null; + const target = record(targetRaw, `${transHostProxyConfigPath}.targets.${targetId}`); + if (target.trans === undefined) return null; + const targetEnabled = booleanField(target, "enabled", `${transHostProxyConfigPath}.targets.${targetId}`); + if (!targetEnabled) return null; + const trans = record(target.trans, `${transHostProxyConfigPath}#targets.${targetId}.trans`); + if (trans.hostProxyEnv === undefined) return null; + const raw = record(trans.hostProxyEnv, `${transHostProxyConfigPath}#targets.${targetId}.trans.hostProxyEnv`); + const enabled = booleanField(raw, "enabled", `${transHostProxyConfigPath}#targets.${targetId}.trans.hostProxyEnv`); + if (!enabled) return null; + const envFileRef = enumField(raw, "envFileRef", `${transHostProxyConfigPath}.targets.${targetId}.trans.hostProxyEnv`, ["files.envFile"] as const); + const files = record(target.files, `${transHostProxyConfigPath}.targets.${targetId}.files`); + return { + targetId, + enabled, + envFile: absolutePathField(files, "envFile", `${transHostProxyConfigPath}.targets.${targetId}.files`), + envFileRef, + applyTo: enumField(raw, "applyTo", `${transHostProxyConfigPath}#targets.${targetId}.trans.hostProxyEnv`, ["host-posix-commands"] as const), + configPath: transHostProxyConfigPath, + configPathLabel: `${transHostProxyConfigPath}#targets.${targetId}.trans.hostProxyEnv`, + }; +} + +export function transHostProxyEnvSummary(targetId: string): Record | null { + const rule = readTransHostProxyEnvRule(targetId); + if (rule === null) return null; + return { + enabled: rule.enabled, + envFileRef: rule.envFileRef, + envFile: rule.envFile, + applyTo: rule.applyTo, + source: rule.configPathLabel, + }; +} + +function record(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as Record; +} + +function booleanField(raw: Record, key: string, label: string): boolean { + const value = raw[key]; + if (typeof value !== "boolean") throw new Error(`${label}.${key} must be a boolean`); + return value; +} + +function enumField(raw: Record, key: string, label: string, allowed: T): T[number] { + const value = raw[key]; + if (typeof value !== "string" || !(allowed as readonly string[]).includes(value)) { + throw new Error(`${label}.${key} must be one of: ${allowed.join(", ")}`); + } + return value as T[number]; +} + +function absolutePathField(raw: Record, key: string, label: string): string { + const value = raw[key]; + if (typeof value !== "string" || !value.startsWith("/")) throw new Error(`${label}.${key} must be an absolute path`); + if (value.includes("\0") || value.includes("\n")) throw new Error(`${label}.${key} must not contain control characters`); + return value; +}