diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 606cb103..5353e50c 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -1,4 +1,4 @@ -import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02ControlPlaneRenderScript, v02PipelineServiceIds } from "./src/hwlab-g14"; +import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02PipelineServiceIds } from "./src/hwlab-g14"; function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); @@ -85,6 +85,76 @@ assertCondition( v02PipelineServiceIds(), ); +const falseGreenPassed = v02FalseGreenGuard({ + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + pipelineRun: { exists: true, status: "True" }, + taskRuns: { + items: [ + { name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-api", status: "True" }, + { name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-web", status: "True" }, + ], + }, + planArtifacts: { + ok: true, + buildServices: ["hwlab-cloud-api"], + reusedServices: ["hwlab-cloud-web"], + artifactProvenanceAudit: { + ok: true, + unsafeReuseServices: [], + }, + }, + runtimeWorkloads: { + ok: true, + items: [ + { serviceId: "hwlab-cloud-api", artifactSourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + { serviceId: "hwlab-cloud-web", artifactSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, + ], + }, +}); +assertCondition( + falseGreenPassed.ok === true && falseGreenPassed.state === "passed", + "v0.2 false-green guard should pass when built runtime services and reuse provenance are proven", + falseGreenPassed, +); + +const falseGreenReuseMissingAudit = v02FalseGreenGuard({ + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + pipelineRun: { exists: true, status: "True" }, + taskRuns: { items: [{ name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-api", status: "True" }] }, + planArtifacts: { + ok: true, + buildServices: ["hwlab-cloud-api"], + reusedServices: ["hwlab-cloud-web"], + artifactProvenanceAudit: null, + }, + runtimeWorkloads: { ok: true, items: [{ serviceId: "hwlab-cloud-api", artifactSourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }] }, +}); +assertCondition( + falseGreenReuseMissingAudit.ok === true + && JSON.stringify(falseGreenReuseMissingAudit.provenanceWarnings).includes("artifact-provenance-audit-missing-for-reuse"), + "v0.2 false-green guard should not fail a completed rollout only because optional reuse provenance is missing", + falseGreenReuseMissingAudit, +); + +const falseGreenRuntimeMismatch = v02FalseGreenGuard({ + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + pipelineRun: { exists: true, status: "True" }, + taskRuns: { items: [{ name: "hwlab-v02-ci-poll-aaaaaaaaaaaa-build-hwlab-cloud-api", status: "True" }] }, + planArtifacts: { + ok: true, + buildServices: ["hwlab-cloud-api"], + reusedServices: [], + artifactProvenanceAudit: null, + }, + runtimeWorkloads: { ok: true, items: [{ serviceId: "hwlab-cloud-api", artifactSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }] }, +}); +assertCondition( + falseGreenRuntimeMismatch.ok === false + && JSON.stringify(falseGreenRuntimeMismatch.failures).includes("artifact-source-commit-mismatch"), + "v0.2 false-green guard must fail when a built service still runs an old artifact", + falseGreenRuntimeMismatch, +); + const prBody = [ "## 背景", "", @@ -194,6 +264,7 @@ console.log(JSON.stringify({ "git mirror status exposes pending flush and controlled flush command", "v0.2 control-plane render uses a detached temp worktree", "v0.2 PipelineRun service matrix excludes hwlab-cli", + "v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance", "rollout brief includes natural-language changelog before automatic diff summary", "semantic changelog extracts Chinese summary sections", "rollout brief includes lazy-build reused/rebuild metrics and service durations", diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 21f10eec..18ec62bf 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -715,12 +715,40 @@ function v02ControlPlaneStatusBundle(sourceCommit: string | null | undefined): C `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(V02_APP)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`, `if [ -n "$pipeline_run" ]; then section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'; else section pipelineRun sh -c 'true'; fi`, `if [ -n "$pipeline_run" ]; then section taskRuns kubectl get taskrun -n ${shellQuote(CI_NAMESPACE)} -l "tekton.dev/pipelineRun=$pipeline_run" -o 'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\t"}{.status.startTime}{"\\t"}{.status.completionTime}{"\\n"}{end}'; else section taskRuns sh -c 'true'; fi`, + `if [ -n "$pipeline_run" ]; then section planArtifacts sh -c ${shellQuote(v02PlanArtifactsLogScript())} plan-artifacts "$pipeline_run"; else section planArtifacts sh -c 'true'; fi`, + `section runtimeWorkloads kubectl get deploy,statefulset,job -n hwlab-v02 -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(v02RuntimeWorkloadsColumns())} --no-headers`, `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l hwlab.pikastech.local/gitops-target=v02 -o ${shellQuote(pipelineRunRowsJsonPath())}`, `section webAssets sh -c ${shellQuote(v02WebAssetsProbeScript())}`, ].join("\n"); return g14K3s(["script", "--", script], 60_000); } +function v02PlanArtifactsLogScript(): string { + return [ + "pipeline_run=\"$1\"", + `namespace=${shellQuote(CI_NAMESPACE)}`, + "[ -n \"$pipeline_run\" ] || exit 0", + "pods=$(kubectl -n \"$namespace\" get pods -l \"tekton.dev/pipelineRun=$pipeline_run,tekton.dev/pipelineTask=plan-artifacts\" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)", + "for pod in $pods; do", + " printf '__POD__\\t%s\\n' \"$pod\"", + " kubectl -n \"$namespace\" logs \"$pod\" --all-containers --tail=200 2>/dev/null || true", + "done", + ].join("\n"); +} + +function v02RuntimeWorkloadsColumns(): string { + return [ + "custom-columns=KIND:.kind", + "NAME:.metadata.name", + "SERVICE:.metadata.labels.hwlab\\.pikastech\\.local/service-id", + "ARTIFACT:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/artifact-source-commit", + "SOURCE:.spec.template.metadata.annotations.hwlab\\.pikastech\\.local/source-commit", + "IMAGE:.spec.template.spec.containers[0].image", + "READY:.status.readyReplicas", + "REPLICAS:.status.replicas", + ].join(","); +} + function v02WebAssetsProbeScript(): string { return [ "set +e", @@ -871,6 +899,164 @@ function v02WebAssetsFromText( }; } +function v02PlanArtifactsFromText(text: string, commandOk: boolean, pipelineRun: string | null, exitCode: number | null, stderr: string): Record { + const events: Record[] = []; + const pods: string[] = []; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.startsWith("__POD__")) { + const [, pod = ""] = line.split("\t"); + if (pod.length > 0) pods.push(pod); + continue; + } + if (!line.startsWith("{")) continue; + try { + const event = record(JSON.parse(line) as unknown); + if (event.event === "g14-ci-plan") events.push(event); + } catch { + // Ignore non-JSON log lines; the raw log remains visible through pod logs. + } + } + const latest = events.at(-1) ?? null; + if (!commandOk || latest === null) { + return { + ok: false, + pipelineRun, + pods, + eventFound: latest !== null, + degradedReason: commandOk ? "g14-ci-plan-event-not-found" : "plan-artifacts-log-query-failed", + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + }; + } + const audit = record(latest.artifactProvenanceAudit); + const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : []; + const provenanceRebuildServices = Array.isArray(audit.provenanceRebuildServices) ? audit.provenanceRebuildServices : []; + return { + ok: true, + pipelineRun, + pods, + sourceCommitId: stringOrNull(latest.sourceCommitId), + affectedServices: stringArray(latest.affectedServices), + rolloutServices: stringArray(latest.rolloutServices), + buildServices: stringArray(latest.buildServices), + reusedServices: stringArray(latest.reusedServices), + buildSkippedCount: numericValue(latest.buildSkippedCount), + artifactProvenanceAudit: Object.keys(audit).length > 0 ? audit : null, + summary: `build=${stringArray(latest.buildServices).length} reuse=${stringArray(latest.reusedServices).length} unsafeReuse=${unsafeReuseServices.length} provenanceRebuild=${provenanceRebuildServices.length}`, + disclosure: "parsed from plan-artifacts g14-ci-plan log event", + }; +} + +function v02RuntimeWorkloadsFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string): Record { + if (!commandOk) { + return { + ok: false, + items: [], + exitCode, + stderr: stderr.trim().slice(0, 2000), + }; + } + const items = text + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [kind = "", name = "", serviceId = "", artifactSourceCommit = "", sourceCommit = "", image = "", ready = "", replicas = ""] = line.split(/\s+/u); + return { + kind, + name, + serviceId: serviceId === "" ? null : serviceId, + artifactSourceCommit: artifactSourceCommit === "" ? null : artifactSourceCommit, + sourceCommit: sourceCommit === "" ? null : sourceCommit, + image: image === "" ? null : image, + readyReplicas: numericField(ready), + replicas: numericField(replicas), + }; + }); + return { + ok: true, + items, + summary: `runtime deployments=${items.length}`, + }; +} + +export function v02FalseGreenGuard(input: { + sourceCommit: string | null; + pipelineRun: Record | null; + taskRuns: Record; + planArtifacts: Record; + runtimeWorkloads: Record; +}): Record { + const status = stringOrNull(input.pipelineRun?.status); + if (input.sourceCommit === null || input.pipelineRun === null || input.pipelineRun.exists === false) { + return { ok: null, state: "not-started", summary: "current source commit has no PipelineRun yet" }; + } + if (status !== "True") { + return { ok: null, state: "pipeline-not-final", summary: `PipelineRun status=${status ?? "unknown"}` }; + } + if (input.planArtifacts.ok !== true) { + return { + ok: false, + state: "missing-plan-artifacts-evidence", + summary: "PipelineRun succeeded but plan-artifacts g14-ci-plan evidence was not visible", + planArtifacts: input.planArtifacts, + }; + } + const sourceCommit = input.sourceCommit; + const buildServices = stringArray(input.planArtifacts.buildServices); + const reusedServices = stringArray(input.planArtifacts.reusedServices); + const taskItems = Array.isArray(input.taskRuns.items) ? input.taskRuns.items.map((item) => record(item)) : []; + const workloadItems = Array.isArray(input.runtimeWorkloads.items) ? input.runtimeWorkloads.items.map((item) => record(item)) : []; + const workloadByService = new Map(workloadItems.map((item) => [String(item.serviceId ?? ""), item])); + const missingBuildTaskRuns = buildServices.filter((serviceId) => !taskItems.some((item) => String(item.name ?? "").endsWith(`build-${serviceId}`) && item.status === "True")); + const runtimeMismatches = buildServices + .map((serviceId) => { + const workload = workloadByService.get(serviceId); + if (!workload) return { serviceId, reason: "runtime-workload-missing" }; + if (workload.artifactSourceCommit !== sourceCommit) { + return { + serviceId, + reason: "artifact-source-commit-mismatch", + runtimeArtifactSourceCommit: workload.artifactSourceCommit ?? null, + expectedSourceCommit: sourceCommit, + image: workload.image ?? null, + }; + } + return null; + }) + .filter((item): item is Record => item !== null); + const audit = record(input.planArtifacts.artifactProvenanceAudit); + const unsafeReuseServices = Array.isArray(audit.unsafeReuseServices) ? audit.unsafeReuseServices : []; + const auditPresent = input.planArtifacts.artifactProvenanceAudit !== null; + const provenanceWarning = !auditPresent && reusedServices.length > 0 + ? { reason: "artifact-provenance-audit-missing-for-reuse", reusedServices } + : null; + const provenanceOk = auditPresent ? audit.ok !== false && unsafeReuseServices.length === 0 : true; + const failures = [ + ...(input.runtimeWorkloads.ok === true ? [] : [{ reason: "runtime-workload-query-failed" }]), + ...(provenanceOk ? [] : [{ reason: "artifact-provenance-audit-failed", unsafeReuseServices }]), + ...missingBuildTaskRuns.map((serviceId) => ({ serviceId, reason: "expected-build-taskrun-not-succeeded" })), + ...runtimeMismatches, + ]; + return { + ok: failures.length === 0, + state: failures.length === 0 ? "passed" : "failed", + summary: failures.length === 0 + ? `artifact provenance and built runtime services align with ${shortSha(sourceCommit)}` + : `false-green risk: ${failures.length} invariant violation(s)`, + sourceCommit, + buildServices, + reusedServices, + provenanceOk, + provenanceAuditPresent: auditPresent, + provenanceWarnings: provenanceWarning ? [provenanceWarning] : [], + missingBuildTaskRuns, + runtimeMismatches, + failures, + }; +} + function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string | null, activePipelineRuns: unknown[]): string | null { if (!apiRevision || !sourceCommit || apiRevision === sourceCommit) return null; const activeItems = activePipelineRuns.map((item) => record(item)); @@ -886,10 +1072,20 @@ function webAssetsRevisionNote(apiRevision: string | null, sourceCommit: string function numericField(value: string | undefined): number | null { if (value === undefined || value.trim().length === 0) return null; + if (value === "") return null; const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } +function numericValue(value: unknown): number | null { + const parsed = typeof value === "number" ? value : Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : []; +} + function listV02PipelineRunsCompactFromText(text: string, commandOk: boolean, command: string[] | string, exitCode: number | null, stderr: string, limit = 8, nowMs = Date.now()): Record { if (!commandOk) { return { @@ -1372,6 +1568,8 @@ function v02ControlPlaneStatus(sourceCommitInput?: string | null): Record