mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add JD01 YAML-first deployment support

Browse Source
This commit is contained in:
Codex
2026-06-29 08:13:34 +00:00
parent fe917bec4a
commit 076c4b643d
49 changed files with 10909 additions and 5223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/web-probe.ts
+153 -34
View File
@@ -49,12 +49,28 @@ export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<str
egressProxy: gitMirror.egressProxy.mode === "direct" ? {
mode: "direct",
required: false,
} : gitMirror.egressProxy.mode === "host-route" ? {
...gitMirror.egressProxy,
mode: "host-route",
required: true,
} : {
...gitMirror.egressProxy,
mode: "node-global",
required: true,
},
};
const deployYamlGitMirror = {
...renderGitMirror,
egressProxy: renderGitMirror.egressProxy.mode !== "host-route" ? renderGitMirror.egressProxy : {
mode: "node-global",
required: true,
clientName: renderGitMirror.egressProxy.clientName,
namespace: "platform-infra",
serviceName: renderGitMirror.egressProxy.clientName,
port: httpProxyEndpoint(renderGitMirror.egressProxy.proxyUrl)?.port ?? 10808,
noProxy: renderGitMirror.egressProxy.noProxy,
},
};
return {
nodeId: spec.nodeId,
lane: spec.lane,
@@ -75,6 +91,7 @@ export function nodeRuntimeRenderOverlay(spec: HwlabRuntimeLaneSpec): Record<str
toolsImage: gitMirror.toolsImage,
toolsImagePullPolicy: gitMirror.toolsImagePullPolicy,
gitMirror: renderGitMirror,
deployYamlGitMirror,
networkProfileId: spec.networkProfileId,
downloadProfileId: spec.downloadProfileId,
gitSshProxyHost: gitSshProxy?.host,
@@ -704,6 +721,7 @@ export function readLocalPostgresPasswordMaterial(input: { sourceRef: string; so
}
export function localSecretSourcePaths(sourceRef: string): string[] {
if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef);
const marker = "/.worktree/";
const index = repoRoot.indexOf(marker);
const paths = index >= 0
@@ -1034,6 +1052,16 @@ export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean,
" if [ \"$mode\" = build-moonbridge ]; then",
" action=build",
" source_present_before=build-source",
" effective_build_container_http_proxy=\"$build_container_http_proxy\"",
" effective_build_container_https_proxy=\"$build_container_https_proxy\"",
" effective_build_container_all_proxy=\"$build_container_all_proxy\"",
" effective_docker_build_add_host_args=\"$docker_build_add_host_args\"",
" if [ \"$docker_network_mode\" = host ]; then",
" effective_build_container_http_proxy=\"$build_http_proxy\"",
" effective_build_container_https_proxy=\"$build_https_proxy\"",
" effective_build_container_all_proxy=\"$build_all_proxy\"",
" effective_docker_build_add_host_args=\"\"",
" fi",
" tmpdir=$(mktemp -d /tmp/hwlab-node-runtime-image-$id.XXXXXX)",
" dockerfile=\"$tmpdir/Dockerfile\"",
" cat > \"$dockerfile\" <<'DOCKERFILE'",
@@ -1069,7 +1097,7 @@ export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean,
"USER 65532:65532",
"ENTRYPOINT [\"/app/moonbridge\"]",
"DOCKERFILE",
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$build_container_https_proxy\" --build-arg ALL_PROXY=\"$build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$build_container_http_proxy\" --build-arg https_proxy=\"$build_container_https_proxy\" --build-arg all_proxy=\"$build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
" if env HTTP_PROXY=\"$build_http_proxy\" HTTPS_PROXY=\"$build_https_proxy\" ALL_PROXY=\"$build_all_proxy\" NO_PROXY=\"$build_no_proxy\" http_proxy=\"$build_http_proxy\" https_proxy=\"$build_https_proxy\" all_proxy=\"$build_all_proxy\" no_proxy=\"$build_no_proxy\" docker build $effective_docker_build_add_host_args --network \"$docker_network_mode\" --build-arg BUILDER_IMAGE=\"$builder_image\" --build-arg MOONBRIDGE_REPO=\"$source_repo\" --build-arg MOONBRIDGE_REF=\"$source_ref\" --build-arg GOPROXY_VALUE=\"$go_proxy\" --build-arg HTTP_PROXY=\"$effective_build_container_http_proxy\" --build-arg HTTPS_PROXY=\"$effective_build_container_https_proxy\" --build-arg ALL_PROXY=\"$effective_build_container_all_proxy\" --build-arg NO_PROXY=\"$build_no_proxy\" --build-arg http_proxy=\"$effective_build_container_http_proxy\" --build-arg https_proxy=\"$effective_build_container_https_proxy\" --build-arg all_proxy=\"$effective_build_container_all_proxy\" --build-arg no_proxy=\"$build_no_proxy\" -t \"$target\" -f \"$dockerfile\" \"$tmpdir\" >/tmp/hwlab-node-runtime-image-$id-build.out 2>&1; then",
" docker push \"$target\" >/tmp/hwlab-node-runtime-image-$id-push.out 2>&1 || { cat /tmp/hwlab-node-runtime-image-$id-push.out >&2 2>/dev/null || true; failed=true; }",
" else",
" cat /tmp/hwlab-node-runtime-image-$id-build.out >&2 2>/dev/null || true",
@@ -1122,9 +1150,12 @@ export function ensureNodeBaseImage(spec: HwlabRuntimeLaneSpec, dryRun: boolean,
? runNodeHostScriptAsync(spec, script, Math.min(timeoutSeconds, 600), "runtime-image-build")
: runNodeHostScript(spec, script, Math.min(timeoutSeconds, 300));
const imageRows = parseNodeRuntimeImageRows(statusText(result));
const dependencyCount = nodeRuntimeImageDependencies(spec).length;
const imageRowsComplete = imageRows.length === dependencyCount
&& imageRows.every((image) => image.presentAfter === true || image.registryTagPresent === true);
const base = imageRows.find((image) => image.id === "base") ?? {};
return {
ok: isCommandSuccess(result),
ok: isCommandSuccess(result) || imageRowsComplete,
dryRun,
target: base.target ?? spec.baseImage,
source: base.source ?? spec.baseImageSource,
@@ -1196,6 +1227,7 @@ export function readSecretSourceValue(secretRoot: string, sourceRef: string, key
}
export function secretSourcePaths(sourceRef: string): string[] {
if (sourceRef.startsWith(".env/")) return ownerFileSourcePaths(sourceRef);
const paths = [join(repoRoot, ".state", "secrets", sourceRef)];
const marker = "/.worktree/";
const index = repoRoot.indexOf(marker);
@@ -1203,6 +1235,14 @@ export function secretSourcePaths(sourceRef: string): string[] {
return [...new Set(paths)];
}
function ownerFileSourcePaths(sourceRef: string): string[] {
if (sourceRef.includes("..") || sourceRef.includes("\0")) return [];
const marker = "/.worktree/";
const index = repoRoot.indexOf(marker);
const roots = index >= 0 ? [repoRoot.slice(0, index), repoRoot] : [repoRoot];
return [...new Set(roots.map((root) => join(root, sourceRef)))];
}
export function displayRepoPath(path: string): string {
const normalizedRoot = repoRoot.replace(/\/+$/u, "");
if (path === normalizedRoot) return ".";
@@ -1225,27 +1265,28 @@ export function hwlabPasswordHash(password: string): string {
export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): BootstrapAdminSecretMaterial {
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null;
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "bootstrap-admin-yaml-source-missing" };
}
const paths = secretSourcePaths(sourceRef);
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
if (!existsSync(sourcePath)) {
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, passwordHash: null, error: "secret-source-missing" };
}
const values = parseEnvFile(readFileSync(sourcePath, "utf8"));
const password = values[sourceKey];
if (password === undefined || password.length === 0) {
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, passwordHash: null, error: "secret-key-missing" };
return bootstrapAdminSecretMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing");
}
const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine);
if (!password.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error);
const username = readBootstrapAdminUsername(spec);
if (!username.ok) return bootstrapAdminSecretMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error);
return {
ok: true,
username: username.value,
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
usernameFingerprint: shortSecretFingerprint(username.value),
sourceRef,
sourceKey,
sourcePath,
sourceLine,
sourcePath: password.sourcePath,
sourcePresent: true,
sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
passwordHash: hwlabPasswordHash(password),
sourceFingerprint: shortSecretFingerprint(password.value),
passwordHash: hwlabPasswordHash(password.value),
error: null,
};
}
@@ -1253,32 +1294,96 @@ export function readBootstrapAdminSecretMaterial(spec: RuntimeSecretSpec): Boots
export function readBootstrapAdminPasswordMaterial(spec: RuntimeSecretSpec): BootstrapAdminPasswordMaterial {
const sourceRef = spec.bootstrapAdminPasswordSourceRef;
const sourceKey = spec.bootstrapAdminPasswordSourceKey;
const sourceLine = spec.bootstrapAdminPasswordSourceLine ?? null;
if (sourceRef === undefined || sourceKey === undefined || spec.bootstrapAdminPasswordHashTransform === undefined) {
return { ok: false, sourceRef: sourceRef ?? null, sourceKey: sourceKey ?? null, sourcePath: null, sourcePresent: false, sourceFingerprint: null, password: null, error: "bootstrap-admin-yaml-source-missing" };
}
const paths = secretSourcePaths(sourceRef);
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
const runtimePassword = process.env[sourceKey];
if (!existsSync(sourcePath) && (runtimePassword === undefined || runtimePassword.length === 0)) {
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: false, sourceFingerprint: null, password: null, error: "secret-source-missing" };
}
const values = existsSync(sourcePath) ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {};
const password = values[sourceKey] ?? runtimePassword;
if (password === undefined || password.length === 0) {
return { ok: false, sourceRef, sourceKey, sourcePath, sourcePresent: true, sourceFingerprint: null, password: null, error: "secret-key-missing" };
return bootstrapAdminPasswordMaterialError(spec, sourceRef ?? null, sourceKey ?? null, sourceLine, null, null, "bootstrap-admin-yaml-source-missing");
}
const password = readSecretSourceScalar(sourceRef, sourceKey, sourceLine);
if (!password.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, password.error);
const username = readBootstrapAdminUsername(spec);
if (!username.ok) return bootstrapAdminPasswordMaterialError(spec, sourceRef, sourceKey, sourceLine, password.sourcePath, password.sourcePresent, username.error);
return {
ok: true,
username: username.value,
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
usernameFingerprint: shortSecretFingerprint(username.value),
sourceRef,
sourceKey,
sourcePath,
sourceLine,
sourcePath: password.sourcePath,
sourcePresent: true,
sourceFingerprint: `sha256:${createHash("sha256").update(password).digest("hex").slice(0, 16)}`,
password,
sourceFingerprint: shortSecretFingerprint(password.value),
password: password.value,
error: null,
};
}
function readBootstrapAdminUsername(spec: RuntimeSecretSpec): { ok: true; value: string } | { ok: false; error: string } {
const ref = spec.bootstrapAdminUsernameSourceRef;
if (ref === undefined) return { ok: true, value: spec.bootstrapAdminUsername };
const key = spec.bootstrapAdminUsernameSourceKey ?? "username";
const material = readSecretSourceScalar(ref, key, spec.bootstrapAdminUsernameSourceLine ?? null);
if (!material.ok) return { ok: false, error: `username-${material.error}` };
return { ok: true, value: material.value };
}
function readSecretSourceScalar(sourceRef: string, sourceKey: string, sourceLine: number | null): { ok: true; value: string; sourcePath: string; sourcePresent: true } | { ok: false; sourcePath: string; sourcePresent: boolean; error: string } {
const paths = secretSourcePaths(sourceRef);
const sourcePath = paths.find((candidate) => existsSync(candidate)) ?? paths[0] ?? join(repoRoot, ".state", "secrets", sourceRef);
if (!existsSync(sourcePath)) return { ok: false, sourcePath, sourcePresent: false, error: "secret-source-missing" };
const text = readFileSync(sourcePath, "utf8");
if (sourceLine !== null) {
const line = text.split(/\r?\n/u)[sourceLine - 1]?.replace(/\r$/u, "") ?? "";
if (line.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-line-missing" };
return { ok: true, value: line, sourcePath, sourcePresent: true };
}
const values = parseEnvFile(text);
const runtimeValue = process.env[sourceKey];
const value = values[sourceKey] ?? runtimeValue;
if (value === undefined || value.length === 0) return { ok: false, sourcePath, sourcePresent: true, error: "secret-key-missing" };
return { ok: true, value, sourcePath, sourcePresent: true };
}
function bootstrapAdminSecretMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminSecretMaterial {
return {
ok: false,
username: null,
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
usernameFingerprint: null,
sourceRef,
sourceKey,
sourceLine,
sourcePath,
sourcePresent: sourcePresent === true,
sourceFingerprint: null,
passwordHash: null,
error,
};
}
function bootstrapAdminPasswordMaterialError(spec: RuntimeSecretSpec, sourceRef: string | null, sourceKey: string | null, sourceLine: number | null, sourcePath: string | null, sourcePresent: boolean | null, error: string): BootstrapAdminPasswordMaterial {
return {
ok: false,
username: null,
usernameSourceRef: spec.bootstrapAdminUsernameSourceRef ?? null,
usernameSourceKey: spec.bootstrapAdminUsernameSourceKey ?? null,
usernameSourceLine: spec.bootstrapAdminUsernameSourceLine ?? null,
usernameFingerprint: null,
sourceRef,
sourceKey,
sourceLine,
sourcePath,
sourcePresent: sourcePresent === true,
sourceFingerprint: null,
password: null,
error,
};
}
export function parseEnvFile(text: string): Record<string, string> {
const values: Record<string, string> = {};
for (const rawLine of text.split(/\r?\n/u)) {
@@ -1547,11 +1652,13 @@ export function nodeRuntimeGitMirrorTarget(spec: HwlabRuntimeLaneSpec): NodeRunt
const gitMirror = record(target.gitMirror);
const gitMirrorEgressProxy = record(gitMirror.egressProxy);
const gitMirrorEgressMode = stringValue(gitMirrorEgressProxy.mode, "gitMirror.egressProxy.mode");
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global or direct for node=${spec.nodeId} lane=${spec.lane}`);
if (gitMirrorEgressMode !== "node-global" && gitMirrorEgressMode !== "host-route" && gitMirrorEgressMode !== "direct") throw new Error(`gitMirror.egressProxy.mode must be node-global, host-route, or direct for node=${spec.nodeId} lane=${spec.lane}`);
const nodeEgressProxy = gitMirrorEgressMode === "direct"
? { mode: "direct" as const, required: false as const }
: nodeRuntimeGitMirrorEgressProxySpec(record(node.egressProxy), `nodes.${spec.nodeId}.egressProxy`);
if (gitMirrorEgressMode === "node-global" && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
if ((gitMirrorEgressMode === "node-global" || gitMirrorEgressMode === "host-route") && gitMirrorEgressProxy.required !== true) throw new Error(`gitMirror.egressProxy.required must be true for node=${spec.nodeId} lane=${spec.lane}`);
if (gitMirrorEgressMode === "node-global" && nodeEgressProxy.mode !== "k8s-service-cluster-ip") throw new Error(`gitMirror.egressProxy.mode=node-global requires nodes.${spec.nodeId}.egressProxy.mode=k8s-service-cluster-ip`);
if (gitMirrorEgressMode === "host-route" && nodeEgressProxy.mode !== "host-route") throw new Error(`gitMirror.egressProxy.mode=host-route requires nodes.${spec.nodeId}.egressProxy.mode=host-route`);
const githubTransport = nodeRuntimeGitMirrorGithubTransportSpec(record(gitMirror.githubTransport), "gitMirror.githubTransport");
const source = record(target.source);
const gitops = record(target.gitops);
@@ -1626,7 +1733,19 @@ function gitMirrorSecretSourceEncoding(raw: unknown, path: string): "plain" | "b
export function nodeRuntimeGitMirrorEgressProxySpec(raw: Record<string, unknown>, path: string): NodeRuntimeGitMirrorEgressProxySpec {
const mode = stringValue(raw.mode, `${path}.mode`);
if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`);
if (mode === "host-route") {
const noProxyRaw = raw.noProxy;
if (!Array.isArray(noProxyRaw)) throw new Error(`${path}.noProxy must be an array`);
return {
mode,
clientName: stringValue(raw.clientName, `${path}.clientName`),
hostProxyConfigRef: stringValue(raw.hostProxyConfigRef, `${path}.hostProxyConfigRef`),
proxyEnvPath: stringValue(raw.proxyEnvPath, `${path}.proxyEnvPath`),
proxyUrl: stringValue(raw.proxyUrl, `${path}.proxyUrl`),
noProxy: noProxyRaw.map((item, index) => stringValue(item, `${path}.noProxy[${index}]`)),
};
}
if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`);
const port = Number(raw.port);
if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`${path}.port must be a TCP port`);
const sourceConfigRef = optionalStringValue(raw.sourceConfigRef, `${path}.sourceConfigRef`) ?? null;