Merge pull request #2028 from pikasTech/perf/selfmedia-env-reuse

perf: 复用 SELFMEDIA 构建环境

config/selfmedia.yaml

@@ -34,6 +34,16 @@ delivery:
         dockerfile: deploy/Dockerfile
         imageRepository: 127.0.0.1:5000/selfmedia/newsroom
         networkMode: host
+        envReuse:
+          enabled: true
+          mode: buildkit-state-host-path
+          statePath: /var/lib/unidesk/selfmedia-ci/buildkit-state
+          identityFiles:
+            - package.json
+            - bun.lock
+            - web/package.json
+            - web/bun.lock
+            - deploy/Dockerfile
         proxy:
           http: http://127.0.0.1:10808
           https: http://127.0.0.1:10808

scripts/native/cicd/pac-status-evaluator.cjs

@@ -505,6 +505,9 @@ function digestOf(item) {
 
 function extractPacArtifactEvidence(recordsInput, logText) {
   const records = Array.isArray(recordsInput) ? recordsInput.map(record) : [];
+  const envReuseRecord = [...records].reverse().find((item) => item.phase === "env-reuse") || null;
+  const imageBuildRecord = [...records].reverse().find((item) => item.phase === "image-build" || item.phase === "image-build-complete") || null;
+  const stageTimings = Object.assign({}, ...records.map((item) => record(item.stageTimings)));
   const sourceObservation = extractPacSourceObservation(records);
   const catalog = [...records].reverse().find((item) => item.status === "published"
     && Array.isArray(item.services)
@@ -562,10 +565,13 @@ function extractPacArtifactEvidence(recordsInput, logText) {
   }
   if (image !== null) {
     const env = record(image.envReuse);
+    const reuseEnv = record(envReuseRecord?.envReuse);
     return {
-      imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status) || (stringOrNull(image.digestRef) !== null ? "built" : null),
-      envIdentity: stringOrNull(image.envIdentity),
-      envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.mode) || stringOrNull(image.envReuseStatus),
+      imageStatus: stringOrNull(image.imageStatus) || stringOrNull(image.status)
+        || (imageBuildRecord !== null || stringOrNull(image.digestRef) !== null ? "built" : null),
+      envIdentity: stringOrNull(image.envIdentity) || stringOrNull(envReuseRecord?.envIdentity),
+      envReuse: stringOrNull(env.dependencyReuse) || stringOrNull(env.status) || stringOrNull(image.envReuseStatus)
+        || stringOrNull(reuseEnv.dependencyReuse) || stringOrNull(reuseEnv.status) || stringOrNull(envReuseRecord?.envReuseStatus),
       nodeDepsReuse: null,
       buildCache: null,
       digest: digestOf(image) || (digests.length === 1 ? digests[0] : null),
@@ -576,6 +582,7 @@ function extractPacArtifactEvidence(recordsInput, logText) {
       baselineSourceCommit: stringOrNull(image.baselineSourceCommit),
       action: stringOrNull(image.action),
       reason: stringOrNull(image.reason),
+      stageTimings,
       sourceObservation,
       valuesPrinted: false,
     };

scripts/src/platform-infra-pipelines-as-code-remote.sh

@@ -703,6 +703,12 @@ function recursiveEnvReuse(value, state) {
   if (typeof value.reusedServiceCount === 'number') state.serviceReusedCount = value.reusedServiceCount;
   if (typeof value.envReuse === 'string') state.status = value.envReuse;
   if (typeof value.envReuseStatus === 'string') state.status = value.envReuseStatus;
+  if (typeof value.envIdentity === 'string') state.identity = value.envIdentity;
+  if (value.stageTimings && typeof value.stageTimings === 'object' && !Array.isArray(value.stageTimings)) {
+    for (const [key, timing] of Object.entries(value.stageTimings)) {
+      if (typeof timing === 'number' && Number.isFinite(timing)) state.stageTimings[key] = timing;
+    }
+  }
   if (value.envReuse && typeof value.envReuse === 'object') {
     const env = value.envReuse;
     if (typeof env.mode === 'string') state.status = env.mode;
@@ -742,7 +748,7 @@ function boundedFailureRecord(value) {
 }
 
 function envReuseForPipelineRun(namespace, name, taskRuns) {
-  const state = { status: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null };
+  const state = { status: null, identity: null, buildSkippedCount: null, serviceReusedCount: null, buildCache: null, stageTimings: {}, source: 'pipeline-task-logs', sourceObservation: null, artifact: null, collector: null, traceId: null, firstBreak: null, error: null };
   const matching = (taskRuns.items || []).filter((item) => item.metadata?.labels?.['tekton.dev/pipelineRun'] === name);
   const contractTaskRuns = matching.filter((item) => taskTerminalRecord(item) !== null);
   const terminalRecords = contractTaskRuns.map(taskTerminalRecord).filter(Boolean);
@@ -889,9 +895,11 @@ function rowFor(consumer, item, taskRuns) {
     classification,
     envReuse: {
       status: evidence.status,
+      identity: evidence.identity,
       buildSkippedCount: evidence.buildSkippedCount,
       serviceReusedCount: evidence.serviceReusedCount,
       buildCache: evidence.buildCache,
+      stageTimings: evidence.stageTimings,
       source: evidence.source,
     },
     sourceObservation,

scripts/src/platform-infra-pipelines-as-code.ts

@@ -2304,6 +2304,7 @@ export function renderStatus(result: Record<string, unknown>): RenderedCliResult
     "",
     "IMAGE / GITOPS",
     ...table(["IMAGE_STATUS", "ENV_REUSE", "ENV_ID", "DIGEST", "GITOPS"], [[stringValue(artifact.imageStatus), stringValue(artifact.envReuse), stringValue(artifact.envIdentity), short(stringValue(artifact.digest), 18), short(stringValue(artifact.gitopsCommit))]]),
+    `  stage-timings: ${stageTimingsText(record(artifact.stageTimings))}`,
     "",
     "ARGO",
     ...table(["SYNC", "HEALTH", "REVISION", "RELATION", "TARGET_REVISION"], [[
@@ -2909,6 +2910,13 @@ function envReuseText(envReuse: Record<string, unknown>): string {
   return parts.length === 0 ? "-" : parts.join(",");
 }
 
+function stageTimingsText(timings: Record<string, unknown>): string {
+  const parts = Object.entries(timings)
+    .filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1]))
+    .map(([name, milliseconds]) => `${name}=${Math.round(milliseconds)}ms`);
+  return parts.length === 0 ? "-" : parts.join(",");
+}
+
 function registryText(registry: Record<string, unknown>): string {
   if (Object.keys(registry).length === 0) return "-";
   if (registry.status === "not-configured" || registry.applicability === "not-configured") return "N/A";
@@ -2970,6 +2978,8 @@ function renderHistoryDetail(row: Record<string, unknown>): string[] {
       ["errorExitCode", stringValue(typedError.exitCode)],
       ["errorSummary", compactLine(stringValue(typedError.stderrSummary ?? typedError.message))],
       ["envReuseSource", stringValue(envReuse.source)],
+      ["envIdentity", short(stringValue(envReuse.identity), 24)],
+      ["stageTimings", stageTimingsText(record(envReuse.stageTimings))],
       ["deliveryMode", stringValue(sourceObservation.mode)],
       ["deliveryValid", stringValue(sourceObservation.valid)],
       ["deliverySourceMatched", stringValue(sourceObservation.sourceMatched)],

scripts/src/selfmedia-config.ts

@@ -33,6 +33,12 @@ export interface SelfMediaDeliveryTarget {
     readonly dockerfile: string;
     readonly imageRepository: string;
     readonly networkMode: string;
+    readonly envReuse: {
+      readonly enabled: true;
+      readonly mode: "buildkit-state-host-path";
+      readonly statePath: string;
+      readonly identityFiles: readonly string[];
+    };
     readonly proxy: Record<string, unknown>;
   };
   readonly gitops: Record<string, unknown> & {
@@ -144,6 +150,7 @@ function parseDeliveryTarget(id: string, value: Record<string, unknown>): SelfMe
   const ci = record(value.ci, `${path}.ci`);
   const source = record(value.source, `${path}.source`);
   const build = record(value.build, `${path}.build`);
+  const envReuse = record(build.envReuse, `${path}.build.envReuse`);
   const proxy = record(build.proxy, `${path}.build.proxy`);
   const gitops = record(value.gitops, `${path}.gitops`);
   const deployment = record(value.deployment, `${path}.deployment`);
@@ -161,6 +168,12 @@ function parseDeliveryTarget(id: string, value: Record<string, unknown>): SelfMe
   validateAccessSecret(deployment, `${path}.deployment`);
   const gitopsWriteUrl = text(gitops.writeUrl, `${path}.gitops.writeUrl`);
   if (!gitopsWriteUrl.startsWith("http://gitea-http.")) throw new Error(`${path}.gitops.writeUrl must use internal Gitea authority`);
+  if (boolean(envReuse.enabled, `${path}.build.envReuse.enabled`) !== true) throw new Error(`${path}.build.envReuse.enabled must be true`);
+  const envReuseMode = text(envReuse.mode, `${path}.build.envReuse.mode`);
+  if (envReuseMode !== "buildkit-state-host-path") throw new Error(`${path}.build.envReuse.mode must be buildkit-state-host-path`);
+  const identityFiles = stringArray(envReuse.identityFiles, `${path}.build.envReuse.identityFiles`)
+    .map((item, index) => relativePath(item, `${path}.build.envReuse.identityFiles[${index}]`));
+  if (identityFiles.length === 0 || new Set(identityFiles).size !== identityFiles.length) throw new Error(`${path}.build.envReuse.identityFiles must be non-empty and unique`);
   return {
     id,
     node: id,
@@ -190,6 +203,12 @@ function parseDeliveryTarget(id: string, value: Record<string, unknown>): SelfMe
       dockerfile: relativePath(build.dockerfile, `${path}.build.dockerfile`),
       imageRepository: text(build.imageRepository, `${path}.build.imageRepository`),
       networkMode: text(build.networkMode, `${path}.build.networkMode`),
+      envReuse: {
+        enabled: true,
+        mode: "buildkit-state-host-path",
+        statePath: absolutePath(envReuse.statePath, `${path}.build.envReuse.statePath`),
+        identityFiles,
+      },
       proxy: structuredClone(proxy),
     },
     gitops: {

scripts/src/selfmedia-delivery-renderer.ts

@@ -98,7 +98,10 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin
           params: taskParams,
           volumes: [
             { name: "workspace", emptyDir: { sizeLimit: target.ci.workspaceSize } },
-            { name: "buildkit-state", emptyDir: { sizeLimit: "12Gi" } },
+            {
+              name: "buildkit-state",
+              hostPath: { path: target.build.envReuse.statePath, type: "DirectoryOrCreate" },
+            },
             { name: "tmp", emptyDir: { sizeLimit: "4Gi" } },
             {
               name: "gitops-token",
@@ -112,6 +115,7 @@ function pipeline(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: strin
           steps: [
             sourceStep(target, effectiveDeploymentB64),
             prepareBuildkitStep(target),
+            envReuseStep(target),
             imageBuildStep(target),
             gitOpsPublishStep(target),
           ],
@@ -136,6 +140,7 @@ function sourceStep(target: SelfMediaDeliveryTarget, effectiveDeploymentB64: str
     ],
     script: `#!/bin/sh
 set -eu
+stage_started_ms=$(date +%s%3N)
 source_commit="$(params.revision)"
 snapshot_prefix="$(params.source-snapshot-prefix)"
 rm -rf /workspace/source /workspace/release
@@ -166,6 +171,8 @@ bun deploy/scripts/prepare-build.ts \\
   --source-commit "$source_commit" \\
   --source-snapshot-prefix "$snapshot_prefix" \\
   --output-dir /workspace/release
+stage_finished_ms=$(date +%s%3N)
+printf '{"ok":true,"phase":"source-prepare","stageTimings":{"sourcePrepareMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))"
 `,
     securityContext: nonRootSecurity(),
     volumeMounts: [
@@ -181,12 +188,48 @@ function prepareBuildkitStep(target: SelfMediaDeliveryTarget): Record<string, un
     name: "prepare-buildkit-state",
     image: target.ci.toolImage,
     imagePullPolicy: "IfNotPresent",
-    script: "#!/bin/sh\nset -eu\nmkdir -p /home/user/.local/share/buildkit\nchown -R 1000:1000 /home/user/.local/share/buildkit\n",
+    script: "#!/bin/sh\nset -eu\nmkdir -p /home/user/.local/share/buildkit/.unidesk\nchown 1000:1000 /home/user/.local/share/buildkit /home/user/.local/share/buildkit/.unidesk\n",
     securityContext: { runAsUser: 0, runAsGroup: 0 },
     volumeMounts: [{ name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" }],
   };
 }
 
+function envReuseStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
+  const identityFiles = target.build.envReuse.identityFiles.map((path) => `/workspace/source/${path}`);
+  const identityInputs = [...identityFiles, "/workspace/release/codex-version"]
+    .map(shellSingleQuote)
+    .join(" ");
+  return {
+    name: "env-reuse",
+    image: target.ci.toolImage,
+    imagePullPolicy: "IfNotPresent",
+    script: `#!/bin/sh
+set -eu
+stage_started_ms=$(date +%s%3N)
+identity_file=/home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity
+for input in ${identityInputs}; do test -f "$input"; done
+env_identity="sha256:$(sha256sum ${identityInputs} | sha256sum | awk '{print $1}')"
+previous_identity=""
+if [ -f "$identity_file" ]; then previous_identity=$(cat "$identity_file"); fi
+if [ "$previous_identity" = "$env_identity" ]; then
+  reuse_status=hit
+  previous_present=true
+else
+  reuse_status=miss
+  if [ -n "$previous_identity" ]; then previous_present=true; else previous_present=false; fi
+fi
+printf '%s\n' "$env_identity" > /workspace/env-identity
+stage_finished_ms=$(date +%s%3N)
+printf '{"ok":true,"phase":"env-reuse","envIdentity":"%s","envReuseStatus":"%s","envReuse":{"status":"%s","mode":"${target.build.envReuse.mode}","identitySource":"owning-yaml-source-artifact","previousIdentityPresent":%s},"stageTimings":{"envIdentityMs":%s},"valuesPrinted":false}\n' "$env_identity" "$reuse_status" "$reuse_status" "$previous_present" "$((stage_finished_ms-stage_started_ms))"
+`,
+    securityContext: nonRootSecurity(),
+    volumeMounts: [
+      { name: "workspace", mountPath: "/workspace" },
+      { name: "buildkit-state", mountPath: "/home/user/.local/share/buildkit" },
+    ],
+  };
+}
+
 function imageBuildStep(target: SelfMediaDeliveryTarget): Record<string, unknown> {
   return {
     name: "image-build",
@@ -197,7 +240,14 @@ function imageBuildStep(target: SelfMediaDeliveryTarget): Record<string, unknown
       { name: "SOURCE_COMMIT", value: "$(params.revision)" },
       { name: "BUILDKITD_FLAGS", value: "--oci-worker-no-process-sandbox --oci-worker-net=host --allow-insecure-entitlement network.host" },
     ],
-    script: "#!/bin/sh\nset -eu\nexec /bin/sh /workspace/source/deploy/scripts/build-image.sh\n",
+    script: `#!/bin/sh
+set -eu
+stage_started_ms=$(date +%s%3N)
+/bin/sh /workspace/source/deploy/scripts/build-image.sh
+cp /workspace/env-identity /home/user/.local/share/buildkit/.unidesk/selfmedia-env-identity
+stage_finished_ms=$(date +%s%3N)
+printf '{"ok":true,"phase":"image-build-complete","envIdentity":"%s","stageTimings":{"imageBuildMs":%s},"valuesPrinted":false}\n' "$(cat /workspace/env-identity)" "$((stage_finished_ms-stage_started_ms))"
+`,
     securityContext: { privileged: true, runAsUser: 1000, runAsGroup: 1000 },
     volumeMounts: [
       { name: "workspace", mountPath: "/workspace" },
@@ -216,6 +266,7 @@ function gitOpsPublishStep(target: SelfMediaDeliveryTarget): Record<string, unkn
     env: [{ name: "SOURCE_COMMIT", value: "$(params.revision)" }],
     script: `#!/bin/sh
 set -eu
+stage_started_ms=$(date +%s%3N)
 askpass=/tmp/selfmedia-git-askpass
 cat >"$askpass" <<'ASKPASS'
 #!/bin/sh
@@ -236,6 +287,8 @@ bun /workspace/source/deploy/scripts/publish-gitops.ts \\
   --source-commit "$SOURCE_COMMIT" \\
   --worktree /workspace/selfmedia-gitops
 rm -f "$askpass"
+stage_finished_ms=$(date +%s%3N)
+printf '{"ok":true,"phase":"gitops-stage","stageTimings":{"gitopsPublishMs":%s},"valuesPrinted":false}\n' "$((stage_finished_ms-stage_started_ms))"
 `,
     securityContext: nonRootSecurity(),
     volumeMounts: [
@@ -354,3 +407,7 @@ function requiredStringArray(value: unknown, path: string): string[] {
   if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path} must be an array of non-empty strings`);
   return value as string[];
 }
+
+function shellSingleQuote(value: string): string {
+  return `'${value.replaceAll("'", `'"'"'`)}'`;
+}