From a9df554378ac2c2c512bb4326294cceaee3ca990 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 27 Jun 2026 13:11:29 +0000 Subject: [PATCH] fix: grant hwlab runtime observer rbac --- config/hwlab-node-control-plane.yaml | 8 ++ scripts/src/hwlab-node-control-plane.ts | 114 ++++++++++++++++++++++-- 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index c8750694..a25f359a 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -204,6 +204,10 @@ targets: sourceRefFrom: gitMirror.githubTransport privateKeySecretKey: ssh-privatekey knownHostsSecretKey: known_hosts + runtimeObserverRbac: + namespace: hwlab-v03 + roleName: hwlab-d601-v03-runtime-observer + roleBindingName: hwlab-d601-v03-runtime-observer toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: Always @@ -361,6 +365,10 @@ targets: sourceRefFrom: gitMirror.githubTransport privateKeySecretKey: ssh-privatekey knownHostsSecretKey: known_hosts + runtimeObserverRbac: + namespace: hwlab-v03 + roleName: hwlab-d518-v03-runtime-observer + roleBindingName: hwlab-d518-v03-runtime-observer toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: Always diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 36588a8f..366702f2 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -140,6 +140,12 @@ interface ControlPlaneTektonGitWorkspaceSecretSpec { knownHostsSecretKey: string; } +interface ControlPlaneTektonRuntimeObserverRbacSpec { + namespace: string; + roleName: string; + roleBindingName: string; +} + interface ControlPlaneNodeSpec { id: string; route: string; @@ -201,6 +207,7 @@ interface ControlPlaneTargetSpec { serviceAccountName: string; pipelineRunPrefix: string; gitWorkspaceSecret: ControlPlaneTektonGitWorkspaceSecretSpec; + runtimeObserverRbac: ControlPlaneTektonRuntimeObserverRbacSpec; toolsImage: { output: string; imagePullPolicy: "Always" | "IfNotPresent" | "Never"; @@ -374,6 +381,8 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta const tekton = record(components.tekton); const ciNamespace = record(components.ciNamespace); const ciGitWorkspaceSecret = record(ciNamespace.gitWorkspaceSecret); + const runtimeNamespace = record(components.runtimeNamespace); + const runtimeObserverRbac = record(runtimeNamespace.runtimeObserverRbac); const registry = record(components.registry); const k3sNodeConfig = record(components.k3sNodeConfig); const k3sNodeConfigReady = node.k3s === null @@ -385,6 +394,8 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && boolField(tekton, "installed") && boolField(ciNamespace, "exists") && boolField(ciGitWorkspaceSecret, "ready") + && boolField(runtimeNamespace, "exists") + && boolField(runtimeObserverRbac, "ready") && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") @@ -414,6 +425,8 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta tektonInstalled: boolField(tekton, "installed"), ciNamespaceExists: boolField(ciNamespace, "exists"), ciGitWorkspaceSecretReady: boolField(ciGitWorkspaceSecret, "ready"), + runtimeNamespaceExists: boolField(runtimeNamespace, "exists"), + runtimeObserverRbacReady: boolField(runtimeObserverRbac, "ready"), gitMirrorNamespaceExists: boolField(gitMirror, "namespaceExists"), gitMirrorReadServiceExists: boolField(gitMirror, "readServiceExists"), gitMirrorWriteServiceExists: boolField(gitMirror, "writeServiceExists"), @@ -432,7 +445,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta toolsImageReady: boolField(registry, "toolsImageReady"), }, result: compactCommandResult(result), - next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace, k3sNodeConfig), + next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace, runtimeNamespace, k3sNodeConfig), }; } @@ -1568,6 +1581,22 @@ function tektonGitWorkspaceSecretSpec( }; } +function tektonRuntimeObserverRbacSpec( + value: unknown, + path: string, + runtimeNamespace: string, +): ControlPlaneTektonRuntimeObserverRbacSpec { + const raw = asRecord(value, path); + const namespace = stringField(raw, "namespace", path); + const roleName = stringField(raw, "roleName", path); + const roleBindingName = stringField(raw, "roleBindingName", path); + validateKubernetesName(namespace, `${path}.namespace`); + validateKubernetesName(roleName, `${path}.roleName`); + validateKubernetesName(roleBindingName, `${path}.roleBindingName`); + if (namespace !== runtimeNamespace) throw new Error(`${path}.namespace must equal targets[].runtimeNamespace ${runtimeNamespace}`); + return { namespace, roleName, roleBindingName }; +} + function secretSourceEncodingField(raw: Record, key: string, path: string): "plain" | "base64" { const value = stringField(raw, key, path); if (value !== "plain" && value !== "base64") throw new Error(`${path}.${key} must be plain or base64`); @@ -1591,13 +1620,14 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const githubTransport = gitMirrorGithubTransportSpec(asRecord(gitMirror.githubTransport, `${path}.gitMirror.githubTransport`), `${path}.gitMirror.githubTransport`); const sourceRepository = stringField(source, "repository", `${path}.source`); const ciNamespace = stringField(raw, "ciNamespace", path); + const runtimeNamespace = stringField(raw, "runtimeNamespace", path); return { id: stringField(raw, "id", path), node, lane, enabled: booleanField(raw, "enabled", path), ciNamespace, - runtimeNamespace: stringField(raw, "runtimeNamespace", path), + runtimeNamespace, source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) }, gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) }, gitMirror: { @@ -1623,6 +1653,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), gitWorkspaceSecret: tektonGitWorkspaceSecretSpec(tekton.gitWorkspaceSecret, `${path}.tekton.gitWorkspaceSecret`, ciNamespace, githubTransport), + runtimeObserverRbac: tektonRuntimeObserverRbacSpec(tekton.runtimeObserverRbac, `${path}.tekton.runtimeObserverRbac`, runtimeNamespace), toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`), }, ciBuildBenchmarks: ciBuildBenchmarkProfileSpecs(raw.ciBuildBenchmarks, `${path}.ciBuildBenchmarks`), @@ -1642,14 +1673,20 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa "hwlab.pikastech.local/node": target.node, "hwlab.pikastech.local/lane": target.lane, }; - const manifests: Record[] = [ - { apiVersion: "v1", kind: "Namespace", metadata: { name: target.ciNamespace, labels } }, - ]; - if (target.gitMirror.namespace !== target.ciNamespace) { - manifests.push({ apiVersion: "v1", kind: "Namespace", metadata: { name: target.gitMirror.namespace, labels } }); - } + const manifests: Record[] = []; + const namespaces = new Set(); + const addNamespace = (name: string): void => { + if (namespaces.has(name)) return; + namespaces.add(name); + manifests.push({ apiVersion: "v1", kind: "Namespace", metadata: { name, labels } }); + }; + addNamespace(target.ciNamespace); + addNamespace(target.runtimeNamespace); + addNamespace(target.gitMirror.namespace); manifests.push( { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: target.tekton.serviceAccountName, namespace: target.ciNamespace, labels } }, + tektonRuntimeObserverRole(target, labels), + tektonRuntimeObserverRoleBinding(target, labels), { apiVersion: "v1", kind: "ConfigMap", @@ -1710,6 +1747,51 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa return manifests; } +function tektonRuntimeObserverRole(target: ControlPlaneTargetSpec, labels: Record): Record { + const rbac = target.tekton.runtimeObserverRbac; + return { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { + name: rbac.roleName, + namespace: rbac.namespace, + labels: { ...labels, "app.kubernetes.io/name": "hwlab-runtime-observer" }, + }, + rules: [ + { + apiGroups: ["apps"], + resources: ["deployments", "statefulsets"], + verbs: ["get", "list", "watch"], + }, + ], + }; +} + +function tektonRuntimeObserverRoleBinding(target: ControlPlaneTargetSpec, labels: Record): Record { + const rbac = target.tekton.runtimeObserverRbac; + return { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { + name: rbac.roleBindingName, + namespace: rbac.namespace, + labels: { ...labels, "app.kubernetes.io/name": "hwlab-runtime-observer" }, + }, + subjects: [ + { + kind: "ServiceAccount", + name: target.tekton.serviceAccountName, + namespace: target.ciNamespace, + }, + ], + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: rbac.roleName, + }, + }; +} + function gitMirrorGithubSshSecret(target: ControlPlaneTargetSpec, labels: Record): Record | null { const transport = target.gitMirror.githubTransport; if (transport.mode !== "ssh") return null; @@ -2392,6 +2474,7 @@ function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec) pipelineRunPrefix: target.tekton.pipelineRunPrefix, serviceAccount: target.tekton.serviceAccountName, gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), + runtimeObserverRbac: target.tekton.runtimeObserverRbac, toolsImage: target.tekton.toolsImage, argoApplication: target.argo.applicationName, argoInstall: { @@ -2431,6 +2514,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS pipelineRunPrefix: target.tekton.pipelineRunPrefix, serviceAccount: target.tekton.serviceAccountName, gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), + runtimeObserverRbac: target.tekton.runtimeObserverRbac, toolsImage: target.tekton.toolsImage, argoNamespace: target.argo.namespace, argoApplication: target.argo.applicationName, @@ -2602,6 +2686,8 @@ github_token_source_key=${shQuote(target.gitMirror.githubTransport.mode === "htt gitmirror_egress_proxy_json=${shQuote(gitMirrorEgressProxyJson)} pipeline=${shQuote(target.tekton.pipelineName)} service_account=${shQuote(target.tekton.serviceAccountName)} +runtime_observer_role=${shQuote(target.tekton.runtimeObserverRbac.roleName)} +runtime_observer_rolebinding=${shQuote(target.tekton.runtimeObserverRbac.roleBindingName)} ci_git_secret=${shQuote(target.tekton.gitWorkspaceSecret.name)} ci_git_private_key=${shQuote(target.tekton.gitWorkspaceSecret.privateKeySecretKey)} ci_git_known_hosts_key=${shQuote(target.tekton.gitWorkspaceSecret.knownHostsSecretKey)} @@ -2629,6 +2715,13 @@ exists_res() { kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 && printf true || p deploy_ready() { desired=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } sts_ready() { desired=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } endpoint_ready() { endpoints=$(kubectl -n "$1" get endpoints "$2" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n "$endpoints" ] && printf true || printf false; } +can_i_runtime() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$runtime_ns" >/dev/null 2>&1 && printf true || printf false; } +runtime_observer_role_exists=$(exists_res "$runtime_ns" role "$runtime_observer_role") +runtime_observer_rolebinding_exists=$(exists_res "$runtime_ns" rolebinding "$runtime_observer_rolebinding") +runtime_observer_can_list_deployments=$(can_i_runtime list deployments.apps) +runtime_observer_can_list_statefulsets=$(can_i_runtime list statefulsets.apps) +runtime_observer_ready=false +if [ "$runtime_observer_role_exists" = true ] && [ "$runtime_observer_rolebinding_exists" = true ] && [ "$runtime_observer_can_list_deployments" = true ] && [ "$runtime_observer_can_list_statefulsets" = true ]; then runtime_observer_ready=true; fi github_transport_json=$(python3 - "$github_transport_mode" "$gitmirror_ns" "$github_ssh_secret" "$github_ssh_private_key" "$github_ssh_private_source_ref" "$github_ssh_private_source_key" "$github_ssh_known_hosts_key" "$github_ssh_known_hosts_source_ref" "$github_ssh_known_hosts_source_key" "$github_token_secret" "$github_token_key" "$github_token_source_ref" "$github_token_source_key" <<'PY' import hashlib, json, subprocess, sys mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14] @@ -2814,7 +2907,7 @@ print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crd PY argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"k3sNodeConfig":$k3s_fragment,"tekton":{"installed":$(kubectl get crd pipelines.tekton.dev pipelineruns.tekton.dev >/dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} JSON `; } @@ -3133,6 +3226,7 @@ function statusNext( gitMirror: Record, argo: Record, ciNamespace: Record, + runtimeNamespace: Record, k3sNodeConfig: Record, ): Record { const bootstrapMissing = !boolField(ciNamespace, "exists") @@ -3146,6 +3240,8 @@ function statusNext( if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); if (!boolField(record(ciNamespace.gitWorkspaceSecret), "ready")) blockers.push("ci-git-workspace-secret-not-ready"); + if (!boolField(runtimeNamespace, "exists")) blockers.push("runtime-namespace-missing"); + if (!boolField(record(runtimeNamespace.runtimeObserverRbac), "ready")) blockers.push("runtime-observer-rbac-not-ready"); if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); const gitMirrorGithubTransport = record(gitMirror.githubTransport); if (gitMirrorGithubTransport.required === true && !boolField(gitMirrorGithubTransport, "ready")) blockers.push("git-mirror-github-token-secret-not-ready");