fix: surface HWLAB v0.2 rollout evidence

2026-05-31 15:07:37 +00:00
parent 1761f319e4
commit 7a0ddaec90
2 changed files with 315 additions and 27 deletions
@@ -1,4 +1,4 @@
-import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02ControlPlaneRenderScript, v02PipelineServiceIds } from "./src/hwlab-g14";
+import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02PipelineServiceIds } from "./src/hwlab-g14";

 function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
  if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
@@ -85,6 +85,76 @@ assertCondition(
  v02PipelineServiceIds(),
 );

+const falseGreenPassed = v02FalseGreenGuard({
+  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  pipelineRun: { exists: true, status: "True" },
+  taskRuns: {
+    items: [
+      { name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-api", status: "True" },
+      { name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-web", status: "True" },
+    ],
+  },
+  planArtifacts: {
+    ok: true,
+    buildServices: ["hwlab-cloud-api"],
+    reusedServices: ["hwlab-cloud-web"],
+    artifactProvenanceAudit: {
+      ok: true,
+      unsafeReuseServices: [],
+    },
+  },
+  runtimeWorkloads: {
+    ok: true,
+    items: [
+      { serviceId: "hwlab-cloud-api", artifactSourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
+      { serviceId: "hwlab-cloud-web", artifactSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" },
+    ],
+  },
+});
+assertCondition(
+  falseGreenPassed.ok === true && falseGreenPassed.state === "passed",
+  "v0.2 false-green guard should pass when built runtime services and reuse provenance are proven",
+  falseGreenPassed,
+);
+
+const falseGreenReuseMissingAudit = v02FalseGreenGuard({
+  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  pipelineRun: { exists: true, status: "True" },
+  taskRuns: { items: [{ name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-api", status: "True" }] },
+  planArtifacts: {
+    ok: true,
+    buildServices: ["hwlab-cloud-api"],
+    reusedServices: ["hwlab-cloud-web"],
+    artifactProvenanceAudit: null,
+  },
+  runtimeWorkloads: { ok: true, items: [{ serviceId: "hwlab-cloud-api", artifactSourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }] },
+});
+assertCondition(
+  falseGreenReuseMissingAudit.ok === true
+    && JSON.stringify(falseGreenReuseMissingAudit.provenanceWarnings).includes("artifact-provenance-audit-missing-for-reuse"),
+  "v0.2 false-green guard should not fail a completed rollout only because optional reuse provenance is missing",
+  falseGreenReuseMissingAudit,
+);
+
+const falseGreenRuntimeMismatch = v02FalseGreenGuard({
+  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  pipelineRun: { exists: true, status: "True" },
+  taskRuns: { items: [{ name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-api", status: "True" }] },
+  planArtifacts: {
+    ok: true,
+    buildServices: ["hwlab-cloud-api"],
+    reusedServices: [],
+    artifactProvenanceAudit: null,
+  },
+  runtimeWorkloads: { ok: true, items: [{ serviceId: "hwlab-cloud-api", artifactSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }] },
+});
+assertCondition(
+  falseGreenRuntimeMismatch.ok === false
+    && JSON.stringify(falseGreenRuntimeMismatch.failures).includes("artifact-source-commit-mismatch"),
+  "v0.2 false-green guard must fail when a built service still runs an old artifact",
+  falseGreenRuntimeMismatch,
+);
+
 const prBody = [
  "## 背景",
  "",
@@ -194,6 +264,7 @@ console.log(JSON.stringify({
    "git mirror status exposes pending flush and controlled flush command",
    "v0.2 control-plane render uses a detached temp worktree",
    "v0.2 PipelineRun service matrix excludes hwlab-cli",
+    "v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance",
    "rollout brief includes natural-language changelog before automatic diff summary",
    "semantic changelog extracts Chinese summary sections",
    "rollout brief includes lazy-build reused/rebuild metrics and service durations",
@@ -715,12 +715,40 @@ function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): C
    `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
    `if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`,
    `if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`,
+    `if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`,
+    `section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`,
    `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`,
    `section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`,
  ].join("\n");
  return g14K3s(["script", "--", script], 60_000);
 }

+function v02PlanArtifactsLogScript(): string {
+  return [
+    "pipeline_run=\"$1\"",
+    `namespace=${shellQuote(CI_NAMESPACE)}`,
+    "[ -n \"$pipeline_run\" ] || exit 0",
+    "pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)",
+    "for pod in $pods; do",
+    "  printf '__POD__\\t%s\\n' \"$pod\"",
+    "  kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true",
+    "done",
+  ].join("\n");
+}
+
+function v02RuntimeWorkloadsColumns(): string {
+  return [
+    "custom-columns=KIND:.kind",
+    "NAME:.metadata.name",
+    "SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id",
+    "ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit",
+    "SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit",
+    "IMAGE:.spec.template.spec.containers[0].image",
+    "READY:.status.readyReplicas",
+    "REPLICAS:.status.replicas",
+  ].join(",");
+}
+
 function v02WebAssetsProbeScript(): string {
  return [
    "set +e",
@@ -871,6 +899,164 @@ function v02WebAssetsFromText(
  };
 }

+function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record<string, unknown> {
+  const events: Record<string, unknown>[] = [];
+  const pods: string[] = [];
+  for (const rawLine of text.split(/\r?\n/u)) {
+    const line = rawLine.trim();
+    if (line.startsWith("__POD__")) {
+      const [, pod = ""] = line.split("\t");
+      if (pod.length > 0) pods.push(pod);
+      continue;
+    }
+    if (!line.startsWith("{")) continue;
+    try {
+      const event = record(JSON.parse(line) as unknown);
+      if (event.event === "g14-ci-plan") events.push(event);
+    } catch {
+      // Ignore non-JSON log lines; the raw log remains visible through pod logs.
+    }
+  }
+  const latest = events.at(-1) ?? null;
+  if (!commandOk || latest === null) {
+    return {
+      ok: false,
+      pipelineRun,
+      pods,
+      eventFound: latest !== null,
+      degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed",
+      exitCode,
+      stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
+    };
+  }
+  const audit = record(latest.artifactProvenanceAudit);
+  const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
+  const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : [];
+  return {
+    ok: true,
+    pipelineRun,
+    pods,
+    sourceCommitId: stringOrNull(latest.sourceCommitId),
+    affectedServices: stringArray(latest.affectedServices),
+    rolloutServices: stringArray(latest.rolloutServices),
+    buildServices: stringArray(latest.buildServices),
+    reusedServices: stringArray(latest.reusedServices),
+    buildSkippedCount: numericValue(latest.buildSkippedCount),
+    artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null,
+    summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`,
+    disclosure: "parsed from plan-artifacts g14-ci-plan log event",
+  };
+}
+
+function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record<string, unknown> {
+  if (!commandOk) {
+    return {
+      ok: false,
+      items: [],
+      exitCode,
+      stderr: stderr.trim().slice(0, 2000),
+    };
+  }
+  const items = text
+    .split(/\r?\n/u)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u);
+      return {
+        kind,
+        name,
+        serviceId: serviceId === "<none>" ? null : serviceId,
+        artifactSourceCommit: artifactSourceCommit === "<none>" ? null : artifactSourceCommit,
+        sourceCommit: sourceCommit === "<none>" ? null : sourceCommit,
+        image: image === "<none>" ? null : image,
+        readyReplicas: numericField(ready),
+        replicas: numericField(replicas),
+      };
+    });
+  return {
+    ok: true,
+    items,
+    summary: `runtime deployments=${items.length}`,
+  };
+}
+
+export function v02FalseGreenGuard(input: {
+  sourceCommit: string | null;
+  pipelineRun: Record<string, unknown> | null;
+  taskRuns: Record<string, unknown>;
+  planArtifacts: Record<string, unknown>;
+  runtimeWorkloads: Record<string, unknown>;
+}): Record<string, unknown> {
+  const status = stringOrNull(input.pipelineRun?.status);
+  if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) {
+    return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" };
+  }
+  if (status !== "True") {
+    return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` };
+  }
+  if (input.planArtifacts.ok !== true) {
+    return {
+      ok: false,
+      state: "missing-plan-artifacts-evidence",
+      summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible",
+      planArtifacts: input.planArtifacts,
+    };
+  }
+  const sourceCommit = input.sourceCommit;
+  const buildServices = stringArray(input.planArtifacts.buildServices);
+  const reusedServices = stringArray(input.planArtifacts.reusedServices);
+  const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : [];
+  const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : [];
+  const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item]));
+  const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True"));
+  const runtimeMismatches = buildServices
+    .map((serviceId) => {
+      const workload = workloadByService.get(serviceId);
+      if (!workload) return { serviceId, reason: "runtime-workload-missing" };
+      if (workload.artifactSourceCommit !== sourceCommit) {
+        return {
+          serviceId,
+          reason: "artifact-source-commit-mismatch",
+          runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null,
+          expectedSourceCommit: sourceCommit,
+          image: workload.image ?? null,
+        };
+      }
+      return null;
+    })
+    .filter((item): item is Record<string, unknown> => item !== null);
+  const audit = record(input.planArtifacts.artifactProvenanceAudit);
+  const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : [];
+  const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null;
+  const provenanceWarning = !auditPresent && reusedServices.length > 0
+    ? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices }
+    : null;
+  const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true;
+  const failures = [
+    ...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]),
+    ...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]),
+    ...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })),
+    ...runtimeMismatches,
+  ];
+  return {
+    ok: failures.length === 0,
+    state: failures.length === 0 ? "passed" : "failed",
+    summary: failures.length === 0
+      ? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}`
+      : `false-green risk: ${failures.length} invariant violation(s)`,
+    sourceCommit,
+    buildServices,
+    reusedServices,
+    provenanceOk,
+    provenanceAuditPresent: auditPresent,
+    provenanceWarnings: provenanceWarning ? [provenanceWarning] : [],
+    missingBuildTaskRuns,
+    runtimeMismatches,
+    failures,
+  };
+}
+
 function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null {
  if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null;
  const activeItems = activePipelineRuns.map((item) => record(item));
@@ -886,10 +1072,20 @@ function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string

 function numericField(value: string | undefined): number | null {
  if (value === undefined || value.trim().length === 0) return null;
+  if (value === "<none>") return null;
  const parsed = Number(value);
  return Number.isFinite(parsed) ? parsed : null;
 }

+function numericValue(value: unknown): number | null {
+  const parsed = typeof value === "number" ? value : Number(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function stringArray(value: unknown): string[] {
+  return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
+}
+
 function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record<string, unknown> {
  if (!commandOk) {
    return {
@@ -1372,6 +1568,8 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
  const argo = sections.argo;
  const pipelineRunSection = sections.pipelineRun;
  const taskRunsSection = sections.taskRuns;
+  const planArtifactsSection = sections.planArtifacts;
+  const runtimeWorkloadsSection = sections.runtimeWorkloads;
  const webAssetsSection = sections.webAssets;
  const recentPipelineRuns = listV02PipelineRunsCompactFromText(
    sections.recentPipelineRuns?.stdout ?? "",
@@ -1384,6 +1582,45 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
  );
  const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
  const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
+  const pipelineRunInfo = pipelineRun === null
+    ? null
+    : pipelineRunCompactFromText(
+      pipelineRun,
+      pipelineRunSection?.stdout ?? "",
+      shellSectionOk(pipelineRunSection),
+      `kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
+      pipelineRunSection?.exitCode ?? null,
+      bundle.stderr,
+    );
+  const taskRuns = taskRunsCompactFromText(
+    taskRunsSection?.stdout ?? "",
+    shellSectionOk(taskRunsSection),
+    pipelineRun,
+    taskRunsSection?.exitCode ?? null,
+    bundle.stderr,
+  );
+  const planArtifacts = v02PlanArtifactsFromText(
+    planArtifactsSection?.stdout ?? "",
+    shellSectionOk(planArtifactsSection),
+    pipelineRun,
+    planArtifactsSection?.exitCode ?? null,
+    bundle.stderr,
+  );
+  const runtimeWorkloads = v02RuntimeWorkloadsFromText(
+    runtimeWorkloadsSection?.stdout ?? "",
+    shellSectionOk(runtimeWorkloadsSection),
+    runtimeWorkloadsSection?.exitCode ?? null,
+    bundle.stderr,
+  );
+  const webAssets = v02WebAssetsFromText(
+    webAssetsSection?.stdout ?? "",
+    shellSectionOk(webAssetsSection),
+    sourceCommit,
+    syncRevision,
+    webAssetsSection?.exitCode ?? null,
+    bundle.stderr,
+    activePipelineRuns,
+  );
  return {
    ok: sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo),
    command: "hwlab g14 control-plane status --lane v02",
@@ -1416,32 +1653,12 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record<string
      fields: { targetRevision, path, syncRevision, syncStatus, health },
      exitCode: argo?.exitCode ?? null,
    },
-    pipelineRun: pipelineRun === null
-      ? null
-      : pipelineRunCompactFromText(
-        pipelineRun,
-        pipelineRunSection?.stdout ?? "",
-        shellSectionOk(pipelineRunSection),
-        `kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
-        pipelineRunSection?.exitCode ?? null,
-        bundle.stderr,
-      ),
-    taskRuns: taskRunsCompactFromText(
-      taskRunsSection?.stdout ?? "",
-      shellSectionOk(taskRunsSection),
-      pipelineRun,
-      taskRunsSection?.exitCode ?? null,
-      bundle.stderr,
-    ),
-    webAssets: v02WebAssetsFromText(
-      webAssetsSection?.stdout ?? "",
-      shellSectionOk(webAssetsSection),
-      sourceCommit,
-      syncRevision,
-      webAssetsSection?.exitCode ?? null,
-      bundle.stderr,
-      activePipelineRuns,
-    ),
+    pipelineRun: pipelineRunInfo,
+    taskRuns,
+    planArtifacts,
+    runtimeWorkloads,
+    falseGreenGuard: v02FalseGreenGuard({ sourceCommit, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads }),
+    webAssets,
    activePipelineRuns,
    recentPipelineRuns,
    query: {