diff --git a/config/selfmedia.yaml b/config/selfmedia.yaml index ec4d9586..3529f125 100644 --- a/config/selfmedia.yaml +++ b/config/selfmedia.yaml @@ -34,6 +34,16 @@ delivery: dockerfile: deploy/Dockerfile imageRepository: 127.0.0.1:5000/selfmedia/newsroom networkMode: host + envReuse: + enabled: true + mode: buildkit-state-host-path + statePath: /var/lib/unidesk/selfmedia-ci/buildkit-state + identityFiles: + - package.json + - bun.lock + - web/package.json + - web/bun.lock + - deploy/Dockerfile proxy: http: http://127.0.0.1:10808 https: http://127.0.0.1:10808 diff --git a/scripts/native/cicd/pac-status-evaluator.cjs b/scripts/native/cicd/pac-status-evaluator.cjs index f15cf7a7..f1a0db17 100644 --- a/scripts/native/cicd/pac-status-evaluator.cjs +++ b/scripts/native/cicd/pac-status-evaluator.cjs @@ -505,6 +505,9 @@ function digestOf(item) { function extractPacArtifactEvidence(recordsInput, logText) { const records = Array.isArray(recordsInput) ? recordsInput.map(record) : []; + const envReuseRecord = [...records].reverse().find((item) => item.phase === "env-reuse") || null; + const imageBuildRecord = [...records].reverse().find((item) => item.phase === "image-build" || item.phase === "image-build-complete") || null; + const stageTimings = Object.assign({}, ...records.map((item) => record(item.stageTimings))); const sourceObservation = extractPacSourceObservation(records); const catalog = [...records].reverse().find((item) => item.status === "published" && Array.isArray(item.services) @@ -562,10 +565,13 @@ function extractPacArtifactEvidence(recordsInput, logText) { } if (image !== null) { const env = record(image.envReuse); + const reuseEnv = record(envReuseRecord?.envReuse); return { - imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status) || (stringOrNull(image.digestRef) !== null ? "built" : null), - envIdentity: stringOrNull(image.envIdentity), - envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.mode) || stringOrNull(image.envReuseStatus), + imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status) + || (imageBuildRecord !== null || stringOrNull(image.digestRef) !== null ? "built" : null), + envIdentity: stringOrNull(image.envIdentity) || stringOrNull(envReuseRecord?.envIdentity), + envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.status) || stringOrNull(image.envReuseStatus) + || stringOrNull(reuseEnv.dependencyReuse) || stringOrNull(reuseEnv.status) || stringOrNull(envReuseRecord?.envReuseStatus), nodeDepsReuse: null, buildCache: null, digest: digestOf(image) || (digests.length === 1 ? digests[0] : null), @@ -576,6 +582,7 @@ function extractPacArtifactEvidence(recordsInput, logText) { baselineSourceCommit: stringOrNull(image.baselineSourceCommit), action: stringOrNull(image.action), reason: stringOrNull(image.reason), + stageTimings, sourceObservation, valuesPrinted: false, }; diff --git a/scripts/src/platform-infra-pipelines-as-code-remote.sh b/scripts/src/platform-infra-pipelines-as-code-remote.sh index 4ac6f318..14287745 100644 --- a/scripts/src/platform-infra-pipelines-as-code-remote.sh +++ b/scripts/src/platform-infra-pipelines-as-code-remote.sh @@ -703,6 +703,12 @@ function recursiveEnvReuse(value, state) { if (typeof value.reusedServiceCount === 'number') state.serviceReusedCount = value.reusedServiceCount; if (typeof value.envReuse === 'string') state.status = value.envReuse; if (typeof value.envReuseStatus === 'string') state.status = value.envReuseStatus; + if (typeof value.envIdentity === 'string') state.identity = value.envIdentity; + if (value.stageTimings && typeof value.stageTimings === 'object' && !Array.isArray(value.stageTimings)) { + for (const [key, timing] of Object.entries(value.stageTimings)) { + if (typeof timing === 'number' && Number.isFinite(timing)) state.stageTimings[key] = timing; + } + } if (value.envReuse && typeof value.envReuse === 'object') { const env = value.envReuse; if (typeof env.mode === 'string') state.status = env.mode; @@ -742,7 +748,7 @@ function boundedFailureRecord(value) { } function envReuseForPipelineRun(namespace, name, taskRuns) { - const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null }; + const state = { status: null, identity: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, stageTimings: {}, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null }; const matching = (taskRuns.items || []).filter((item) => item.metadata?.labels?.['tekton.dev/pipelineRun'] === name); const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null); const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean); @@ -889,9 +895,11 @@ function rowFor(consumer, item, taskRuns) { classification, envReuse: { status: evidence.status, + identity: evidence.identity, buildSkippedCount: evidence.buildSkippedCount, serviceReusedCount: evidence.serviceReusedCount, buildCache: evidence.buildCache, + stageTimings: evidence.stageTimings, source: evidence.source, }, sourceObservation, diff --git a/scripts/src/platform-infra-pipelines-as-code.ts b/scripts/src/platform-infra-pipelines-as-code.ts index 97ce662f..f5edc327 100644 --- a/scripts/src/platform-infra-pipelines-as-code.ts +++ b/scripts/src/platform-infra-pipelines-as-code.ts @@ -2304,6 +2304,7 @@ export function renderStatus(result: Record): RenderedCliResult "", "IMAGE / GITOPS", ...table(["IMAGE_STATUS", "ENV_REUSE", "ENV_ID", "DIGEST", "GITOPS"], [[stringValue(artifact.imageStatus), stringValue(artifact.envReuse), stringValue(artifact.envIdentity), short(stringValue(artifact.digest), 18), short(stringValue(artifact.gitopsCommit))]]), + ` stage-timings: ${stageTimingsText(record(artifact.stageTimings))}`, "", "ARGO", ...table(["SYNC", "HEALTH", "REVISION", "RELATION", "TARGET_REVISION"], [[ @@ -2909,6 +2910,13 @@ function envReuseText(envReuse: Record): string { return parts.length === 0 ? "-" : parts.join(","); } +function stageTimingsText(timings: Record): string { + const parts = Object.entries(timings) + .filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1])) + .map(([name, milliseconds]) => `${name}=${Math.round(milliseconds)}ms`); + return parts.length === 0 ? "-" : parts.join(","); +} + function registryText(registry: Record): string { if (Object.keys(registry).length === 0) return "-"; if (registry.status === "not-configured" || registry.applicability === "not-configured") return "N/A"; @@ -2970,6 +2978,8 @@ function renderHistoryDetail(row: Record): string[] { ["errorExitCode", stringValue(typedError.exitCode)], ["errorSummary", compactLine(stringValue(typedError.stderrSummary ?? typedError.message))], ["envReuseSource", stringValue(envReuse.source)], + ["envIdentity", short(stringValue(envReuse.identity), 24)], + ["stageTimings", stageTimingsText(record(envReuse.stageTimings))], ["deliveryMode", stringValue(sourceObservation.mode)], ["deliveryValid", stringValue(sourceObservation.valid)], ["deliverySourceMatched", stringValue(sourceObservation.sourceMatched)], diff --git a/scripts/src/selfmedia-config.ts b/scripts/src/selfmedia-config.ts index 7cbb60e5..e1d6a412 100644 --- a/scripts/src/selfmedia-config.ts +++ b/scripts/src/selfmedia-config.ts @@ -33,6 +33,12 @@ export interface SelfMediaDeliveryTarget { readonly dockerfile: string; readonly imageRepository: string; readonly networkMode: string; + readonly envReuse: { + readonly enabled: true; + readonly mode: "buildkit-state-host-path"; + readonly statePath: string; + readonly identityFiles: readonly string[]; + }; readonly proxy: Record; }; readonly gitops: Record & { @@ -144,6 +150,7 @@ function parseDeliveryTarget(id: string, value: Record): SelfMe const ci = record(value.ci, `${path}.ci`); const source = record(value.source, `${path}.source`); const build = record(value.build, `${path}.build`); + const envReuse = record(build.envReuse, `${path}.build.envReuse`); const proxy = record(build.proxy, `${path}.build.proxy`); const gitops = record(value.gitops, `${path}.gitops`); const deployment = record(value.deployment, `${path}.deployment`); @@ -161,6 +168,12 @@ function parseDeliveryTarget(id: string, value: Record): SelfMe validateAccessSecret(deployment, `${path}.deployment`); const gitopsWriteUrl = text(gitops.writeUrl, `${path}.gitops.writeUrl`); if (!gitopsWriteUrl.startsWith("http://gitea-http.")) throw new Error(`${path}.gitops.writeUrl must use internal Gitea authority`); + if (boolean(envReuse.enabled, `${path}.build.envReuse.enabled`) !== true) throw new Error(`${path}.build.envReuse.enabled must be true`); + const envReuseMode = text(envReuse.mode, `${path}.build.envReuse.mode`); + if (envReuseMode !== "buildkit-state-host-path") throw new Error(`${path}.build.envReuse.mode must be buildkit-state-host-path`); + const identityFiles = stringArray(envReuse.identityFiles, `${path}.build.envReuse.identityFiles`) + .map((item, index) => relativePath(item, `${path}.build.envReuse.identityFiles[${index}]`)); + if (identityFiles.length === 0 || new Set(identityFiles).size !== identityFiles.length) throw new Error(`${path}.build.envReuse.identityFiles must be non-empty and unique`); return { id, node: id, @@ -190,6 +203,12 @@ function parseDeliveryTarget(id: string, value: Record): SelfMe dockerfile: relativePath(build.dockerfile, `${path}.build.dockerfile`), imageRepository: text(build.imageRepository, `${path}.build.imageRepository`), networkMode: text(build.networkMode, `${path}.build.networkMode`), + envReuse: { + enabled: true, + mode: "buildkit-state-host-path", + statePath: absolutePath(envReuse.statePath, `${path}.build.envReuse.statePath`), + identityFiles, + }, proxy: structuredClone(proxy), }, gitops: { diff --git a/scripts/src/selfmedia-delivery-renderer.ts b/scripts/src/selfmedia-delivery-renderer.ts index acffbfe9..bab3d09a 100644 --- a/scripts/src/selfmedia-delivery-renderer.ts +++ b/scripts/src/selfmedia-delivery-renderer.ts @@ -98,7 +98,10 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin params: taskParams, volumes: [ { name: "workspace", emptyDir: { sizeLimit: target.ci.workspaceSize } }, - { name: "buildkit-state", emptyDir: { sizeLimit: "12Gi" } }, + { + name: "buildkit-state", + hostPath: { path: target.build.envReuse.statePath, type: "DirectoryOrCreate" }, + }, { name: "tmp", emptyDir: { sizeLimit: "4Gi" } }, { name: "gitops-token", @@ -112,6 +115,7 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin steps: [ sourceStep(target, effectiveDeploymentB64), prepareBuildkitStep(target), + envReuseStep(target), imageBuildStep(target), gitOpsPublishStep(target), ], @@ -136,6 +140,7 @@ function sourceStep(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: str ], script: `#!/bin/sh set -eu +stage_started_ms=$(date +%s%3N) source_commit="$(params.revision)" snapshot_prefix="$(params.source-snapshot-prefix)" rm -rf /workspace/source /workspace/release @@ -166,6 +171,8 @@ bun deploy/scripts/prepare-build.ts \\ --source-commit "$source_commit" \\ --source-snapshot-prefix "$snapshot_prefix" \\ --output-dir /workspace/release +stage_finished_ms=$(date +%s%3N) +printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))" `, securityContext: nonRootSecurity(), volumeMounts: [ @@ -181,12 +188,48 @@ function prepareBuildkitStep(target: SelfMediaDeliveryTarget): Record { + const identityFiles = target.build.envReuse.identityFiles.map((path) => `/workspace/source/${path}`); + const identityInputs = [...identityFiles, "/workspace/release/codex-version"] + .map(shellSingleQuote) + .join(" "); + return { + name: "env-reuse", + image: target.ci.toolImage, + imagePullPolicy: "IfNotPresent", + script: `#!/bin/sh +set -eu +stage_started_ms=$(date +%s%3N) +identity_file=/home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity +for input in ${identityInputs}; do test -f "$input"; done +env_identity="sha256:$(sha256sum ${identityInputs} | sha256sum | awk '{print $1}')" +previous_identity="" +if [ -f "$identity_file" ]; then previous_identity=$(cat "$identity_file"); fi +if [ "$previous_identity" = "$env_identity" ]; then + reuse_status=hit + previous_present=true +else + reuse_status=miss + if [ -n "$previous_identity" ]; then previous_present=true; else previous_present=false; fi +fi +printf '%s\n' "$env_identity" > /workspace/env-identity +stage_finished_ms=$(date +%s%3N) +printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentityMs":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_ms-stage_started_ms))" +`, + securityContext: nonRootSecurity(), + volumeMounts: [ + { name: "workspace", mountPath: "/workspace" }, + { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }, + ], + }; +} + function imageBuildStep(target: SelfMediaDeliveryTarget): Record { return { name: "image-build", @@ -197,7 +240,14 @@ function imageBuildStep(target: SelfMediaDeliveryTarget): Record"$askpass" <<'ASKPASS' #!/bin/sh @@ -236,6 +287,8 @@ bun /workspace/source/deploy/scripts/publish-gitops.ts \\ --source-commit "$SOURCE_COMMIT" \\ --worktree /workspace/selfmedia-gitops rm -f "$askpass" +stage_finished_ms=$(date +%s%3N) +printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))" `, securityContext: nonRootSecurity(), volumeMounts: [ @@ -354,3 +407,7 @@ function requiredStringArray(value: unknown, path: string): string[] { if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path} must be an array of non-empty strings`); return value as string[]; } + +function shellSingleQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +}