fix: preserve Sub2API host env keys
Pipelines as Code CI / hwlab-web-probe-sentinel-nc01- Success

This commit is contained in:
Codex
2026-07-09 14:45:37 +02:00
parent 121a28544c
commit 2bcbc634e0
4 changed files with 118 additions and 18 deletions
+2
View File
@@ -96,6 +96,8 @@ targets:
redisPort: 16379 redisPort: 16379
databaseHost: 127.0.0.1 databaseHost: 127.0.0.1
databaseSslMode: require databaseSslMode: require
preserveEnvKeys:
- API_KEY
noProxy: noProxy:
- localhost - localhost
- 127.0.0.1 - 127.0.0.1
+4
View File
@@ -224,6 +224,7 @@ export function parseHostDockerConfig(value: unknown, path: string, runtimeMode:
redisPort: integerField(record, "redisPort", `${path}.hostDocker`), redisPort: integerField(record, "redisPort", `${path}.hostDocker`),
databaseHost: stringField(record, "databaseHost", `${path}.hostDocker`), databaseHost: stringField(record, "databaseHost", `${path}.hostDocker`),
databaseSslMode: stringField(record, "databaseSslMode", `${path}.hostDocker`), databaseSslMode: stringField(record, "databaseSslMode", `${path}.hostDocker`),
preserveEnvKeys: record.preserveEnvKeys === undefined ? [] : stringArrayField(record, "preserveEnvKeys", `${path}.hostDocker`),
noProxy: stringArrayField(record, "noProxy", `${path}.hostDocker`), noProxy: stringArrayField(record, "noProxy", `${path}.hostDocker`),
}; };
validateProxyName(hostDocker.projectName, `${path}.hostDocker.projectName`); validateProxyName(hostDocker.projectName, `${path}.hostDocker.projectName`);
@@ -234,6 +235,9 @@ export function parseHostDockerConfig(value: unknown, path: string, runtimeMode:
validatePort(hostDocker.redisPort, `${path}.hostDocker.redisPort`); validatePort(hostDocker.redisPort, `${path}.hostDocker.redisPort`);
validateHostOrIp(hostDocker.databaseHost, `${path}.hostDocker.databaseHost`); validateHostOrIp(hostDocker.databaseHost, `${path}.hostDocker.databaseHost`);
if (!/^[A-Za-z0-9_-]+$/u.test(hostDocker.databaseSslMode)) throw new Error(`${configPath}.${path}.hostDocker.databaseSslMode has an unsupported format`); if (!/^[A-Za-z0-9_-]+$/u.test(hostDocker.databaseSslMode)) throw new Error(`${configPath}.${path}.hostDocker.databaseSslMode has an unsupported format`);
for (const key of hostDocker.preserveEnvKeys) {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`${configPath}.${path}.hostDocker.preserveEnvKeys contains an unsupported env key`);
}
return hostDocker; return hostDocker;
} }
+1
View File
@@ -140,6 +140,7 @@ export interface Sub2ApiHostDockerConfig {
redisPort: number; redisPort: number;
databaseHost: string; databaseHost: string;
databaseSslMode: string; databaseSslMode: string;
preserveEnvKeys: string[];
noProxy: string[]; noProxy: string[];
} }
+111 -18
View File
@@ -25,6 +25,7 @@ export function hostDockerPlanSummary(sub2api: Sub2ApiConfig, target: Sub2ApiTar
databaseHost: database.host, databaseHost: database.host,
databasePort: database.port, databasePort: database.port,
databaseSslMode: database.sslMode, databaseSslMode: database.sslMode,
preserveEnvKeys: host.preserveEnvKeys,
noProxy: host.noProxy, noProxy: host.noProxy,
valuesPrinted: false, valuesPrinted: false,
}; };
@@ -226,6 +227,7 @@ export function hostDockerApplyScript(sub2api: Sub2ApiConfig, target: Sub2ApiTar
const env = hostDockerEnvFile(sub2api, target, secretMaterial); const env = hostDockerEnvFile(sub2api, target, secretMaterial);
const composeB64 = Buffer.from(compose, "utf8").toString("base64"); const composeB64 = Buffer.from(compose, "utf8").toString("base64");
const envB64 = Buffer.from(env, "utf8").toString("base64"); const envB64 = Buffer.from(env, "utf8").toString("base64");
const preserveEnvKeysB64 = Buffer.from(host.preserveEnvKeys.join("\n"), "utf8").toString("base64");
const image = targetImage(sub2api, target); const image = targetImage(sub2api, target);
const dependencyImages = targetDependencyImages(sub2api, target); const dependencyImages = targetDependencyImages(sub2api, target);
const pullCommand = image.pullPolicy === "Always" const pullCommand = image.pullPolicy === "Always"
@@ -246,10 +248,15 @@ tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT trap 'rm -rf "$tmp"' EXIT
compose_tmp="$tmp/docker-compose.yml" compose_tmp="$tmp/docker-compose.yml"
env_tmp="$tmp/sub2api.env" env_tmp="$tmp/sub2api.env"
preserve_keys="$tmp/preserve-env-keys.txt"
existing_env="$tmp/existing.env"
printf '%s' '${composeB64}' | base64 -d >"$compose_tmp" printf '%s' '${composeB64}' | base64 -d >"$compose_tmp"
printf '%s' '${envB64}' | base64 -d >"$env_tmp" printf '%s' '${envB64}' | base64 -d >"$env_tmp"
printf '%s' '${preserveEnvKeysB64}' | base64 -d >"$preserve_keys"
docker_out="$tmp/docker.out" docker_out="$tmp/docker.out"
docker_err="$tmp/docker.err" docker_err="$tmp/docker.err"
preserve_out="$tmp/preserve-env.out"
preserve_err="$tmp/preserve-env.err"
install_out="$tmp/install.out" install_out="$tmp/install.out"
install_err="$tmp/install.err" install_err="$tmp/install.err"
pull_out="$tmp/pull.out" pull_out="$tmp/pull.out"
@@ -286,21 +293,100 @@ else
: >"$docker_out" : >"$docker_out"
printf '%s\\n' 'docker daemon or docker compose is not accessible' >"$docker_err" printf '%s\\n' 'docker daemon or docker compose is not accessible' >"$docker_err"
fi fi
preserve_rc=1
install_rc=1 install_rc=1
if [ "$docker_rc" -eq 0 ]; then if [ "$docker_rc" -eq 0 ]; then
sudo mkdir -p ${shQuote(host.workDir)} ${shQuote(host.appDataDir)} "$(dirname ${shQuote(host.composePath)})" "$(dirname ${shQuote(host.envPath)})" >"$install_out" 2>"$install_err" preserve_rc=0
install_rc=$? : >"$existing_env"
if [ "$install_rc" -eq 0 ]; then if [ -s "$preserve_keys" ] && [ -f ${shQuote(host.envPath)} ]; then
sudo install -m 0644 "$compose_tmp" ${shQuote(host.composePath)} >>"$install_out" 2>>"$install_err" if ! cat ${shQuote(host.envPath)} >"$existing_env" 2>"$preserve_err"; then
install_rc=$? if ! sudo cat ${shQuote(host.envPath)} >"$existing_env" 2>>"$preserve_err"; then
preserve_rc=1
fi
fi
else
: >"$preserve_err"
fi fi
if [ "$install_rc" -eq 0 ]; then if [ "$preserve_rc" -eq 0 ]; then
current_uid="$(id -u)" python3 - "$env_tmp" "$existing_env" "$preserve_keys" >"$preserve_out" 2>"$preserve_err" <<'PY'
current_gid="$(id -g)" import json
sudo install -m 0600 -o "$current_uid" -g "$current_gid" "$env_tmp" ${shQuote(host.envPath)} >>"$install_out" 2>>"$install_err" import re
import sys
env_path, existing_path, keys_path = sys.argv[1:]
keys = []
with open(keys_path, encoding="utf-8", errors="replace") as handle:
for line in handle:
key = line.strip()
if key:
keys.append(key)
existing = {}
try:
lines = open(existing_path, encoding="utf-8", errors="replace").read().splitlines()
except FileNotFoundError:
lines = []
for line in lines:
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key = stripped.split("=", 1)[0].strip()
if re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key):
existing[key] = line
current = open(env_path, encoding="utf-8", errors="replace").read()
current_keys = set()
for line in current.splitlines():
stripped = line.strip()
if stripped and not stripped.startswith("#") and "=" in stripped:
current_keys.add(stripped.split("=", 1)[0].strip())
append_lines = []
missing = []
skipped_present = []
for key in keys:
if key in current_keys:
skipped_present.append(key)
elif key in existing:
append_lines.append(existing[key])
else:
missing.append(key)
if append_lines:
with open(env_path, "a", encoding="utf-8") as handle:
if not current.endswith("\\n"):
handle.write("\\n")
handle.write("\\n".join(append_lines) + "\\n")
print(json.dumps({
"ok": True,
"keys": keys,
"preservedKeys": [line.split("=", 1)[0] for line in append_lines],
"missingKeys": missing,
"skippedAlreadyRenderedKeys": skipped_present,
"valuesPrinted": False,
}, ensure_ascii=False))
PY
preserve_rc=$?
else
printf '{"ok":false,"error":"read-existing-env-failed","valuesPrinted":false}\\n' >"$preserve_out"
fi
if [ "$preserve_rc" -eq 0 ]; then
sudo mkdir -p ${shQuote(host.workDir)} ${shQuote(host.appDataDir)} "$(dirname ${shQuote(host.composePath)})" "$(dirname ${shQuote(host.envPath)})" >"$install_out" 2>"$install_err"
install_rc=$? install_rc=$?
if [ "$install_rc" -eq 0 ]; then
sudo install -m 0644 "$compose_tmp" ${shQuote(host.composePath)} >>"$install_out" 2>>"$install_err"
install_rc=$?
fi
if [ "$install_rc" -eq 0 ]; then
current_uid="$(id -u)"
current_gid="$(id -g)"
sudo install -m 0600 -o "$current_uid" -g "$current_gid" "$env_tmp" ${shQuote(host.envPath)} >>"$install_out" 2>>"$install_err"
install_rc=$?
fi
else
: >"$install_out"
printf '%s\\n' 'skipped because preserve env failed' >"$install_err"
install_rc=1
fi fi
else else
: >"$preserve_out"
: >"$preserve_err"
: >"$install_out" : >"$install_out"
: >"$install_err" : >"$install_err"
fi fi
@@ -355,13 +441,13 @@ else
: >"$health_out" : >"$health_out"
printf '%s\\n' 'skipped because compose up failed' >"$health_err" printf '%s\\n' 'skipped because compose up failed' >"$health_err"
fi fi
python3 - "$docker_rc" "$install_rc" "$pull_rc" "$up_rc" "$health_rc" "$compose_mode" "$docker_out" "$docker_err" "$install_out" "$install_err" "$pull_out" "$pull_err" "$up_out" "$up_err" "$health_out" "$health_err" <<'PY' python3 - "$docker_rc" "$preserve_rc" "$install_rc" "$pull_rc" "$up_rc" "$health_rc" "$compose_mode" "$docker_out" "$docker_err" "$preserve_out" "$preserve_err" "$install_out" "$install_err" "$pull_out" "$pull_err" "$up_out" "$up_err" "$health_out" "$health_err" <<'PY'
import json import json
import sys import sys
docker_rc, install_rc, pull_rc, up_rc, health_rc = [int(value) for value in sys.argv[1:6]] docker_rc, preserve_rc, install_rc, pull_rc, up_rc, health_rc = [int(value) for value in sys.argv[1:7]]
docker_mode = sys.argv[6] docker_mode = sys.argv[7]
paths = sys.argv[7:] paths = sys.argv[8:]
def text(path, limit=4000): def text(path, limit=4000):
try: try:
@@ -369,8 +455,14 @@ def text(path, limit=4000):
except FileNotFoundError: except FileNotFoundError:
return "" return ""
def parsed(path):
try:
return json.load(open(path, encoding="utf-8", errors="replace"))
except Exception:
return None
payload = { payload = {
"ok": docker_rc == 0 and install_rc == 0 and pull_rc == 0 and up_rc == 0 and health_rc == 0, "ok": docker_rc == 0 and preserve_rc == 0 and install_rc == 0 and pull_rc == 0 and up_rc == 0 and health_rc == 0,
"target": "${target.id}", "target": "${target.id}",
"runtimeMode": "host-docker", "runtimeMode": "host-docker",
"route": "${target.route}", "route": "${target.route}",
@@ -387,10 +479,11 @@ payload = {
}, },
"steps": { "steps": {
"docker": {"exitCode": docker_rc, "stdout": text(paths[0]), "stderr": text(paths[1])}, "docker": {"exitCode": docker_rc, "stdout": text(paths[0]), "stderr": text(paths[1])},
"installFiles": {"exitCode": install_rc, "stdout": text(paths[2]), "stderr": text(paths[3])}, "preserveEnv": {"exitCode": preserve_rc, "summary": parsed(paths[2]), "stdout": text(paths[2]), "stderr": text(paths[3])},
"pull": {"exitCode": pull_rc, "stdout": text(paths[4]), "stderr": text(paths[5])}, "installFiles": {"exitCode": install_rc, "stdout": text(paths[4]), "stderr": text(paths[5])},
"composeUp": {"exitCode": up_rc, "stdout": text(paths[6]), "stderr": text(paths[7])}, "pull": {"exitCode": pull_rc, "stdout": text(paths[6]), "stderr": text(paths[7])},
"localHealth": {"exitCode": health_rc, "stdout": text(paths[8]), "stderr": text(paths[9])}, "composeUp": {"exitCode": up_rc, "stdout": text(paths[8]), "stderr": text(paths[9])},
"localHealth": {"exitCode": health_rc, "stdout": text(paths[10]), "stderr": text(paths[11])},
}, },
"valuesPrinted": False, "valuesPrinted": False,
} }