Codex
447 lines
18 KiB
TypeScript
447 lines
18 KiB
TypeScript
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { resolve } from "node:path";
|
|
import { spawnSync } from "node:child_process";
|
|
|
|
import {
|
|
resolveSelfMediaDeliveryTarget,
|
|
resolveSelfMediaDeliveryTargetByConfigRef,
|
|
selfMediaDeliveryConfigRef,
|
|
selfMediaEffectiveDeployment,
|
|
type SelfMediaDeliveryTarget,
|
|
} from "./selfmedia-config";
|
|
import { stableJsonSha256 } from "./stable-json";
|
|
|
|
export interface SelfMediaRendererBinding {
|
|
readonly consumer: {
|
|
readonly id: string;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly namespace: string;
|
|
readonly pipeline: string;
|
|
readonly pipelineRunPrefix: string;
|
|
readonly sourceArtifact: {
|
|
readonly configRef: string;
|
|
readonly mode: string;
|
|
readonly renderer: string;
|
|
};
|
|
};
|
|
readonly repository: {
|
|
readonly id: string;
|
|
readonly params: Readonly<Record<string, string>>;
|
|
};
|
|
}
|
|
|
|
export interface SelfMediaRenderedPipeline {
|
|
readonly pipeline: Record<string, unknown>;
|
|
readonly configRef: string;
|
|
readonly effectiveConfigSha256: string;
|
|
readonly sourceWorktreeRemote: string;
|
|
}
|
|
|
|
export function renderSelfMediaDesiredPipeline(binding: SelfMediaRendererBinding, sourceWorktree: string): SelfMediaRenderedPipeline {
|
|
const target = resolveSelfMediaDeliveryTargetByConfigRef(binding.consumer.sourceArtifact.configRef);
|
|
const configRef = selfMediaDeliveryConfigRef(target.id);
|
|
if (binding.consumer.sourceArtifact.renderer !== "selfmedia-runtime" || binding.consumer.sourceArtifact.mode !== "embedded-pipeline-spec") {
|
|
throw new Error("selfmedia-runtime requires embedded-pipeline-spec");
|
|
}
|
|
if (binding.consumer.sourceArtifact.configRef !== configRef) {
|
|
throw new Error(`sourceArtifact.configRef must equal resolved owning selector ${configRef}; observed ${binding.consumer.sourceArtifact.configRef}`);
|
|
}
|
|
assertBinding(target, binding);
|
|
assertServiceRepositoryContract(sourceWorktree);
|
|
const effectiveDeployment = selfMediaEffectiveDeployment(target);
|
|
validateRuntimeRender(sourceWorktree, effectiveDeployment);
|
|
const effectiveDeploymentB64 = Buffer.from(`${Bun.YAML.stringify(effectiveDeployment).trim()}\n`, "utf8").toString("base64");
|
|
return {
|
|
pipeline: pipeline(target, effectiveDeploymentB64),
|
|
configRef,
|
|
effectiveConfigSha256: stableJsonSha256(target),
|
|
sourceWorktreeRemote: target.source.worktreeRemote,
|
|
};
|
|
}
|
|
|
|
export function selfMediaSourceWorktreeRemote(targetId: string): string {
|
|
return resolveSelfMediaDeliveryTarget(targetId).source.worktreeRemote;
|
|
}
|
|
|
|
function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: string): Record<string, unknown> {
|
|
const params = [
|
|
"revision",
|
|
"source-snapshot-prefix",
|
|
"git-read-url",
|
|
"gitops-read-url",
|
|
"gitops-write-url",
|
|
"gitops-username",
|
|
"gitops-secret-name",
|
|
];
|
|
const literalDefaults: Readonly<Record<string, string>> = {
|
|
"source-snapshot-prefix": target.source.snapshotPrefix,
|
|
"git-read-url": target.source.readUrl,
|
|
"gitops-read-url": target.gitops.readUrl,
|
|
"gitops-write-url": target.gitops.writeUrl,
|
|
"gitops-username": target.gitops.credentialUsername,
|
|
"gitops-secret-name": target.gitops.credentialSecretName,
|
|
};
|
|
const defaults = target.ci.parameterBinding === "literal" ? literalDefaults : {};
|
|
const taskParams = params.map((name) => ({ name, type: "string", ...(defaults[name] === undefined ? {} : { default: defaults[name] }) }));
|
|
const taskParamBindings = params.map((name) => ({ name, value: `$(params.${name})` }));
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "Pipeline",
|
|
metadata: {
|
|
name: target.ci.pipeline,
|
|
namespace: target.ci.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": target.ci.pipeline,
|
|
"app.kubernetes.io/part-of": "selfmedia-factory",
|
|
"app.kubernetes.io/managed-by": "unidesk",
|
|
},
|
|
},
|
|
spec: {
|
|
params: taskParams,
|
|
tasks: [{
|
|
name: "build-and-publish",
|
|
params: taskParamBindings,
|
|
taskSpec: {
|
|
params: taskParams,
|
|
volumes: [
|
|
{ name: "workspace", emptyDir: { sizeLimit: target.ci.workspaceSize } },
|
|
{
|
|
name: "buildkit-state",
|
|
hostPath: { path: target.build.envReuse.statePath, type: "DirectoryOrCreate" },
|
|
},
|
|
{ name: "tmp", emptyDir: { sizeLimit: "4Gi" } },
|
|
{
|
|
name: "gitops-token",
|
|
secret: {
|
|
secretName: "$(params.gitops-secret-name)",
|
|
defaultMode: 288,
|
|
items: [{ key: target.gitops.credentialTokenKey, path: "token" }],
|
|
},
|
|
},
|
|
],
|
|
steps: [
|
|
sourceStep(target, effectiveDeploymentB64),
|
|
prepareBuildkitStep(target),
|
|
envReuseStep(target),
|
|
imageBuildStep(target),
|
|
gitOpsPublishStep(target),
|
|
],
|
|
},
|
|
}],
|
|
},
|
|
};
|
|
}
|
|
|
|
function sourceStep(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: string): Record<string, unknown> {
|
|
const proxy = target.build.proxy;
|
|
return {
|
|
name: "immutable-source",
|
|
image: target.ci.toolImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
workingDir: "/workspace",
|
|
env: [
|
|
{ name: "HTTP_PROXY", value: requiredString(proxy.http, "build.proxy.http") },
|
|
{ name: "HTTPS_PROXY", value: requiredString(proxy.https, "build.proxy.https") },
|
|
{ name: "ALL_PROXY", value: requiredString(proxy.all, "build.proxy.all") },
|
|
{ name: "NO_PROXY", value: requiredStringArray(proxy.noProxy, "build.proxy.noProxy").join(",") },
|
|
],
|
|
script: `#!/bin/sh
|
|
set -eu
|
|
stage_started_seconds=$(date +%s)
|
|
source_commit="$(params.revision)"
|
|
snapshot_prefix="$(params.source-snapshot-prefix)"
|
|
rm -rf /workspace/source /workspace/release
|
|
askpass=/tmp/selfmedia-source-git-askpass
|
|
cat >"$askpass" <<'ASKPASS'
|
|
#!/bin/sh
|
|
case "$1" in
|
|
*Username*) printf '%s' "$SELFMEDIA_GIT_USERNAME" ;;
|
|
*Password*) cat /var/run/selfmedia-gitops/token ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
ASKPASS
|
|
chmod 700 "$askpass"
|
|
export SELFMEDIA_GIT_USERNAME="$(params.gitops-username)"
|
|
export GIT_ASKPASS="$askpass"
|
|
export GIT_TERMINAL_PROMPT=0
|
|
git clone --filter=blob:none --no-checkout "$(params.git-read-url)" /workspace/source
|
|
cd /workspace/source
|
|
git fetch --depth=1 --filter=blob:none origin "+$snapshot_prefix/$source_commit:refs/remotes/origin/selfmedia-source-snapshot"
|
|
git checkout --detach "$source_commit"
|
|
test "$(git rev-parse HEAD)" = "$source_commit"
|
|
rm -f "$askpass"
|
|
printf '%s' '${effectiveDeploymentB64}' | base64 -d > /workspace/effective-deployment.yaml
|
|
bun install --frozen-lockfile
|
|
bun deploy/scripts/prepare-build.ts \\
|
|
--config /workspace/effective-deployment.yaml \\
|
|
--source-root /workspace/source \\
|
|
--source-commit "$source_commit" \\
|
|
--source-snapshot-prefix "$snapshot_prefix" \\
|
|
--output-dir /workspace/release
|
|
stage_finished_seconds=$(date +%s)
|
|
printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareSeconds":%s},"valuesPrinted":false}\n' "$((stage_finished_seconds-stage_started_seconds))"
|
|
`,
|
|
securityContext: nonRootSecurity(),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "tmp", mountPath: "/tmp" },
|
|
{ name: "gitops-token", mountPath: "/var/run/selfmedia-gitops", readOnly: true },
|
|
],
|
|
};
|
|
}
|
|
|
|
function prepareBuildkitStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
|
|
return {
|
|
name: "prepare-buildkit-state",
|
|
image: target.ci.toolImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
script: "#!/bin/sh\nset -eu\nmkdir -p /home/user/.local/share/buildkit/.unidesk\nchown 1000:1000 /home/user/.local/share/buildkit /home/user/.local/share/buildkit/.unidesk\n",
|
|
securityContext: { runAsUser: 0, runAsGroup: 0 },
|
|
volumeMounts: [{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }],
|
|
};
|
|
}
|
|
|
|
function envReuseStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
|
|
const semanticPackageManifests = target.build.envReuse.semanticPackageManifests;
|
|
const semanticPolicyFile = `/workspace/source/${semanticPackageManifests.policyFile}`;
|
|
const semanticRenderer = `/workspace/source/${semanticPackageManifests.renderer}`;
|
|
const identityFiles = [
|
|
...target.build.envReuse.identityFiles.map((path) => `/workspace/source/${path}`),
|
|
semanticPolicyFile,
|
|
semanticRenderer,
|
|
];
|
|
const identityInputs = [...identityFiles, "/workspace/release/codex-version"]
|
|
.map(shellSingleQuote)
|
|
.join(" ");
|
|
return {
|
|
name: "env-reuse",
|
|
image: target.ci.toolImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
script: `#!/bin/sh
|
|
set -eu
|
|
stage_started_seconds=$(date +%s)
|
|
identity_file=/home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity
|
|
for input in ${identityInputs}; do test -f "$input"; done
|
|
rm -rf /workspace/semantic-package-manifests
|
|
bun ${shellSingleQuote(semanticRenderer)} \
|
|
--config ${shellSingleQuote(semanticPolicyFile)} \
|
|
--source-root /workspace/source \
|
|
--output-root /workspace/semantic-package-manifests
|
|
test -f /workspace/semantic-package-manifests/identity.json
|
|
env_identity="sha256:$(sha256sum ${identityInputs} /workspace/semantic-package-manifests/identity.json | sha256sum | awk '{print $1}')"
|
|
previous_identity=""
|
|
if [ -f "$identity_file" ]; then previous_identity=$(cat "$identity_file"); fi
|
|
if [ "$previous_identity" = "$env_identity" ]; then
|
|
reuse_status=hit
|
|
previous_present=true
|
|
else
|
|
reuse_status=miss
|
|
if [ -n "$previous_identity" ]; then previous_present=true; else previous_present=false; fi
|
|
fi
|
|
printf '%s\n' "$env_identity" > /workspace/env-identity
|
|
stage_finished_seconds=$(date +%s)
|
|
printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-semantic-package-manifest-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentitySeconds":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_seconds-stage_started_seconds))"
|
|
`,
|
|
securityContext: nonRootSecurity(),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
],
|
|
};
|
|
}
|
|
|
|
function imageBuildStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
|
|
return {
|
|
name: "image-build",
|
|
image: target.ci.buildkitImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
workingDir: "/workspace",
|
|
env: [
|
|
{ name: "SOURCE_COMMIT", value: "$(params.revision)" },
|
|
{ name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
|
|
],
|
|
script: `#!/bin/sh
|
|
set -eu
|
|
stage_started_seconds=$(date +%s)
|
|
/bin/sh /workspace/source/deploy/scripts/build-image.sh
|
|
cp /workspace/env-identity /home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity
|
|
stage_finished_seconds=$(date +%s)
|
|
printf '{"ok":true,"phase":"image-build-complete","envIdentity":"%s","stageTimings":{"imageBuildSeconds":%s},"valuesPrinted":false}\n' "$(cat /workspace/env-identity)" "$((stage_finished_seconds-stage_started_seconds))"
|
|
`,
|
|
securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
|
|
{ name: "tmp", mountPath: "/tmp" },
|
|
],
|
|
};
|
|
}
|
|
|
|
function gitOpsPublishStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
|
|
return {
|
|
name: "gitops-publish",
|
|
image: target.ci.toolImage,
|
|
imagePullPolicy: "IfNotPresent",
|
|
workingDir: "/workspace",
|
|
env: [{ name: "SOURCE_COMMIT", value: "$(params.revision)" }],
|
|
script: `#!/bin/sh
|
|
set -eu
|
|
stage_started_seconds=$(date +%s)
|
|
askpass=/tmp/selfmedia-git-askpass
|
|
cat >"$askpass" <<'ASKPASS'
|
|
#!/bin/sh
|
|
case "$1" in
|
|
*Username*) printf '%s' "$SELFMEDIA_GIT_USERNAME" ;;
|
|
*Password*) cat /var/run/selfmedia-gitops/token ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
ASKPASS
|
|
chmod 700 "$askpass"
|
|
export SELFMEDIA_GIT_USERNAME="$(params.gitops-username)"
|
|
export GIT_ASKPASS="$askpass"
|
|
export GIT_TERMINAL_PROMPT=0
|
|
bun /workspace/source/deploy/scripts/publish-gitops.ts \\
|
|
--config /workspace/effective-deployment.yaml \\
|
|
--source-root /workspace/source \\
|
|
--metadata /workspace/build-metadata.json \\
|
|
--source-commit "$SOURCE_COMMIT" \\
|
|
--worktree /workspace/selfmedia-gitops
|
|
rm -f "$askpass"
|
|
stage_finished_seconds=$(date +%s)
|
|
printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishSeconds":%s},"valuesPrinted":false}\n' "$((stage_finished_seconds-stage_started_seconds))"
|
|
`,
|
|
securityContext: nonRootSecurity(),
|
|
volumeMounts: [
|
|
{ name: "workspace", mountPath: "/workspace" },
|
|
{ name: "tmp", mountPath: "/tmp" },
|
|
{ name: "gitops-token", mountPath: "/var/run/selfmedia-gitops", readOnly: true },
|
|
],
|
|
};
|
|
}
|
|
|
|
function assertBinding(target: SelfMediaDeliveryTarget, binding: SelfMediaRendererBinding): void {
|
|
const expected: Record<string, string> = {
|
|
node: target.node,
|
|
lane: target.lane,
|
|
namespace: target.ci.namespace,
|
|
pipeline: target.ci.pipeline,
|
|
pipelineRunPrefix: target.ci.pipelineRunPrefix,
|
|
};
|
|
const observed: Record<string, string> = {
|
|
node: binding.consumer.node,
|
|
lane: binding.consumer.lane,
|
|
namespace: binding.consumer.namespace,
|
|
pipeline: binding.consumer.pipeline,
|
|
pipelineRunPrefix: binding.consumer.pipelineRunPrefix,
|
|
};
|
|
for (const [key, value] of Object.entries(expected)) {
|
|
if (observed[key] !== value) throw new Error(`PaC ${binding.consumer.id} ${key}=${observed[key]} does not match selfmedia owner ${value}`);
|
|
}
|
|
const requiredParams: Readonly<Record<string, string>> = {
|
|
node: target.node,
|
|
source_branch: target.source.branch,
|
|
source_snapshot_prefix: target.source.snapshotPrefix,
|
|
git_read_url: target.source.readUrl,
|
|
gitops_read_url: target.gitops.readUrl,
|
|
gitops_write_url: target.gitops.writeUrl,
|
|
gitops_branch: target.gitops.branch,
|
|
gitops_username: target.gitops.credentialUsername,
|
|
gitops_secret_name: target.gitops.credentialSecretName,
|
|
pipeline_name: target.ci.pipeline,
|
|
pipeline_run_prefix: target.ci.pipelineRunPrefix,
|
|
service_account: target.ci.serviceAccountName,
|
|
image_repository: target.build.imageRepository,
|
|
pipeline_timeout: target.ci.pipelineTimeout,
|
|
};
|
|
for (const [key, value] of Object.entries(requiredParams)) {
|
|
if (binding.repository.params[key] !== value) throw new Error(`PaC ${binding.consumer.id} repository params.${key} must match selfmedia owner`);
|
|
}
|
|
}
|
|
|
|
function assertServiceRepositoryContract(sourceWorktree: string): void {
|
|
const required = [
|
|
"deploy/Dockerfile",
|
|
"deploy/scripts/prepare-build.ts",
|
|
"deploy/scripts/build-image.sh",
|
|
"deploy/scripts/publish-gitops.ts",
|
|
"deploy/scripts/render-runtime.ts",
|
|
];
|
|
for (const file of required) {
|
|
if (!existsSync(resolve(sourceWorktree, file))) throw new Error(`selfmedia source repository is missing renderer contract file ${file}`);
|
|
}
|
|
}
|
|
|
|
function validateRuntimeRender(sourceWorktree: string, effectiveDeployment: Record<string, unknown>): void {
|
|
const temporary = mkdtempSync(resolve(tmpdir(), "unidesk-selfmedia-render-"));
|
|
const configPath = resolve(temporary, "effective-deployment.yaml");
|
|
try {
|
|
writeFileSync(configPath, `${Bun.YAML.stringify(effectiveDeployment).trim()}\n`, "utf8");
|
|
const result = spawnSync("bun", [
|
|
"deploy/scripts/render-runtime.ts",
|
|
"--config", configPath,
|
|
"--image", `127.0.0.1:5000/selfmedia/newsroom@sha256:${"a".repeat(64)}`,
|
|
"--source-commit", "b".repeat(40),
|
|
], {
|
|
cwd: sourceWorktree,
|
|
encoding: "utf8",
|
|
maxBuffer: 16 * 1024 * 1024,
|
|
timeout: 60_000,
|
|
});
|
|
if (result.error !== undefined) throw result.error;
|
|
if (result.status !== 0) {
|
|
const evidence = `${result.stderr || result.stdout || "render-runtime failed"}`.trim().slice(-2000);
|
|
throw new Error(`selfmedia effective deployment failed service renderer validation: ${evidence}`);
|
|
}
|
|
const output = result.stdout;
|
|
const publicExposure = requiredRecord(effectiveDeployment.publicExposure, "deployment.publicExposure");
|
|
const required = ["kind: Deployment", "kind: Service", "kind: PersistentVolumeClaim", "kind: NetworkPolicy", `hostPort: ${requiredInteger(publicExposure.hostPort, "deployment.publicExposure.hostPort")}`, "automountServiceAccountToken: false"];
|
|
for (const marker of required) {
|
|
if (!output.includes(marker)) throw new Error(`selfmedia service renderer output is missing ${marker}`);
|
|
}
|
|
const forbidden = ["kind: Secret", "kind: Role\n", "kind: RoleBinding", "kind: ClusterRole", "kind: ClusterRoleBinding"];
|
|
for (const marker of forbidden) {
|
|
if (output.includes(marker)) throw new Error(`selfmedia service renderer output contains forbidden ${marker.trim()}`);
|
|
}
|
|
if ((output.match(/kind: PersistentVolumeClaim/gu) ?? []).length !== 2) throw new Error("selfmedia service renderer must produce exactly two PVCs");
|
|
if ((output.match(/kind: NetworkPolicy/gu) ?? []).length < 3) throw new Error("selfmedia service renderer must produce at least three NetworkPolicies");
|
|
} finally {
|
|
rmSync(temporary, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function nonRootSecurity(): Record<string, unknown> {
|
|
return {
|
|
runAsUser: 1000,
|
|
runAsGroup: 1000,
|
|
runAsNonRoot: true,
|
|
allowPrivilegeEscalation: false,
|
|
capabilities: { drop: ["ALL"] },
|
|
};
|
|
}
|
|
|
|
function requiredString(value: unknown, path: string): string {
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
function requiredStringArray(value: unknown, path: string): string[] {
|
|
if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path} must be an array of non-empty strings`);
|
|
return value as string[];
|
|
}
|
|
|
|
function requiredRecord(value: unknown, path: string): Record<string, unknown> {
|
|
if (value === null || typeof value !== "object" || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function requiredInteger(value: unknown, path: string): number {
|
|
if (!Number.isInteger(value)) throw new Error(`${path} must be an integer`);
|
|
return value as number;
|
|
}
|
|
|
|
function shellSingleQuote(value: string): string {
|
|
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
}
|