diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index a25f359a..2ecf9114 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -208,6 +208,10 @@ targets: namespace: hwlab-v03 roleName: hwlab-d601-v03-runtime-observer roleBindingName: hwlab-d601-v03-runtime-observer + argoObserverRbac: + namespace: argocd + roleName: hwlab-d601-v03-argo-observer + roleBindingName: hwlab-d601-v03-argo-observer toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: Always @@ -369,6 +373,10 @@ targets: namespace: hwlab-v03 roleName: hwlab-d518-v03-runtime-observer roleBindingName: hwlab-d518-v03-runtime-observer + argoObserverRbac: + namespace: argocd + roleName: hwlab-d518-v03-argo-observer + roleBindingName: hwlab-d518-v03-argo-observer toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: Always diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 366702f2..1af64cf9 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -146,6 +146,12 @@ interface ControlPlaneTektonRuntimeObserverRbacSpec { roleBindingName: string; } +interface ControlPlaneTektonArgoObserverRbacSpec { + namespace: string; + roleName: string; + roleBindingName: string; +} + interface ControlPlaneNodeSpec { id: string; route: string; @@ -208,6 +214,7 @@ interface ControlPlaneTargetSpec { pipelineRunPrefix: string; gitWorkspaceSecret: ControlPlaneTektonGitWorkspaceSecretSpec; runtimeObserverRbac: ControlPlaneTektonRuntimeObserverRbacSpec; + argoObserverRbac: ControlPlaneTektonArgoObserverRbacSpec; toolsImage: { output: string; imagePullPolicy: "Always" | "IfNotPresent" | "Never"; @@ -376,6 +383,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta const components = record(status.components); const argo = record(components.argo); const argoInstall = record(argo.install); + const argoObserverRbac = record(argo.argoObserverRbac); const gitMirror = record(components.gitMirror); const gitMirrorGithubTransport = record(gitMirror.githubTransport); const tekton = record(components.tekton); @@ -406,6 +414,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && boolField(argo, "installed") && boolField(argo, "projectExists") && boolField(argo, "applicationExists") + && boolField(argoObserverRbac, "ready") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") && boolField(argoInstall, "statefulSetsReady"); @@ -438,6 +447,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta argoInstalled: boolField(argo, "installed"), argoProjectExists: boolField(argo, "projectExists"), argoApplicationExists: boolField(argo, "applicationExists"), + argoObserverRbacReady: boolField(argoObserverRbac, "ready"), argoCrdsReady: boolField(argoInstall, "crdsReady"), argoDeploymentsReady: boolField(argoInstall, "deploymentsReady"), argoStatefulSetsReady: boolField(argoInstall, "statefulSetsReady"), @@ -1597,6 +1607,22 @@ function tektonRuntimeObserverRbacSpec( return { namespace, roleName, roleBindingName }; } +function tektonArgoObserverRbacSpec( + value: unknown, + path: string, + argoNamespace: string, +): ControlPlaneTektonArgoObserverRbacSpec { + const raw = asRecord(value, path); + const namespace = stringField(raw, "namespace", path); + const roleName = stringField(raw, "roleName", path); + const roleBindingName = stringField(raw, "roleBindingName", path); + validateKubernetesName(namespace, `${path}.namespace`); + validateKubernetesName(roleName, `${path}.roleName`); + validateKubernetesName(roleBindingName, `${path}.roleBindingName`); + if (namespace !== argoNamespace) throw new Error(`${path}.namespace must equal targets[].argo.namespace ${argoNamespace}`); + return { namespace, roleName, roleBindingName }; +} + function secretSourceEncodingField(raw: Record, key: string, path: string): "plain" | "base64" { const value = stringField(raw, key, path); if (value !== "plain" && value !== "base64") throw new Error(`${path}.${key} must be plain or base64`); @@ -1621,6 +1647,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const sourceRepository = stringField(source, "repository", `${path}.source`); const ciNamespace = stringField(raw, "ciNamespace", path); const runtimeNamespace = stringField(raw, "runtimeNamespace", path); + const argoNamespace = stringField(argo, "namespace", `${path}.argo`); return { id: stringField(raw, "id", path), node, @@ -1654,11 +1681,12 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), gitWorkspaceSecret: tektonGitWorkspaceSecretSpec(tekton.gitWorkspaceSecret, `${path}.tekton.gitWorkspaceSecret`, ciNamespace, githubTransport), runtimeObserverRbac: tektonRuntimeObserverRbacSpec(tekton.runtimeObserverRbac, `${path}.tekton.runtimeObserverRbac`, runtimeNamespace), + argoObserverRbac: tektonArgoObserverRbacSpec(tekton.argoObserverRbac, `${path}.tekton.argoObserverRbac`, argoNamespace), toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`), }, ciBuildBenchmarks: ciBuildBenchmarkProfileSpecs(raw.ciBuildBenchmarks, `${path}.ciBuildBenchmarks`), argo: { - namespace: stringField(argo, "namespace", `${path}.argo`), + namespace: argoNamespace, projectName: stringField(argo, "projectName", `${path}.argo`), applicationName: stringField(argo, "applicationName", `${path}.argo`), applicationFile: stringField(argo, "applicationFile", `${path}.argo`), @@ -1687,6 +1715,8 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: target.tekton.serviceAccountName, namespace: target.ciNamespace, labels } }, tektonRuntimeObserverRole(target, labels), tektonRuntimeObserverRoleBinding(target, labels), + tektonArgoObserverRole(target, labels), + tektonArgoObserverRoleBinding(target, labels), { apiVersion: "v1", kind: "ConfigMap", @@ -1792,6 +1822,51 @@ function tektonRuntimeObserverRoleBinding(target: ControlPlaneTargetSpec, labels }; } +function tektonArgoObserverRole(target: ControlPlaneTargetSpec, labels: Record): Record { + const rbac = target.tekton.argoObserverRbac; + return { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { + name: rbac.roleName, + namespace: rbac.namespace, + labels: { ...labels, "app.kubernetes.io/name": "hwlab-argo-observer" }, + }, + rules: [ + { + apiGroups: ["argoproj.io"], + resources: ["applications"], + verbs: ["get", "list", "watch"], + }, + ], + }; +} + +function tektonArgoObserverRoleBinding(target: ControlPlaneTargetSpec, labels: Record): Record { + const rbac = target.tekton.argoObserverRbac; + return { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { + name: rbac.roleBindingName, + namespace: rbac.namespace, + labels: { ...labels, "app.kubernetes.io/name": "hwlab-argo-observer" }, + }, + subjects: [ + { + kind: "ServiceAccount", + name: target.tekton.serviceAccountName, + namespace: target.ciNamespace, + }, + ], + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: rbac.roleName, + }, + }; +} + function gitMirrorGithubSshSecret(target: ControlPlaneTargetSpec, labels: Record): Record | null { const transport = target.gitMirror.githubTransport; if (transport.mode !== "ssh") return null; @@ -2475,6 +2550,7 @@ function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec) serviceAccount: target.tekton.serviceAccountName, gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), runtimeObserverRbac: target.tekton.runtimeObserverRbac, + argoObserverRbac: target.tekton.argoObserverRbac, toolsImage: target.tekton.toolsImage, argoApplication: target.argo.applicationName, argoInstall: { @@ -2515,6 +2591,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS serviceAccount: target.tekton.serviceAccountName, gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), runtimeObserverRbac: target.tekton.runtimeObserverRbac, + argoObserverRbac: target.tekton.argoObserverRbac, toolsImage: target.tekton.toolsImage, argoNamespace: target.argo.namespace, argoApplication: target.argo.applicationName, @@ -2699,6 +2776,8 @@ ci_git_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode == argo_ns=${shQuote(target.argo.namespace)} argo_project=${shQuote(target.argo.projectName)} argo_app=${shQuote(target.argo.applicationName)} +argo_observer_role=${shQuote(target.tekton.argoObserverRbac.roleName)} +argo_observer_rolebinding=${shQuote(target.tekton.argoObserverRbac.roleBindingName)} registry=${shQuote(nodeSpec.registry.endpoint)} tools_image=${shQuote(target.tekton.toolsImage.output)} required_crds_json=${shQuote(requiredCrds)} @@ -2716,12 +2795,18 @@ deploy_ready() { desired=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.spec.r sts_ready() { desired=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } endpoint_ready() { endpoints=$(kubectl -n "$1" get endpoints "$2" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n "$endpoints" ] && printf true || printf false; } can_i_runtime() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$runtime_ns" >/dev/null 2>&1 && printf true || printf false; } +can_i_argo() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$argo_ns" >/dev/null 2>&1 && printf true || printf false; } runtime_observer_role_exists=$(exists_res "$runtime_ns" role "$runtime_observer_role") runtime_observer_rolebinding_exists=$(exists_res "$runtime_ns" rolebinding "$runtime_observer_rolebinding") runtime_observer_can_list_deployments=$(can_i_runtime list deployments.apps) runtime_observer_can_list_statefulsets=$(can_i_runtime list statefulsets.apps) runtime_observer_ready=false if [ "$runtime_observer_role_exists" = true ] && [ "$runtime_observer_rolebinding_exists" = true ] && [ "$runtime_observer_can_list_deployments" = true ] && [ "$runtime_observer_can_list_statefulsets" = true ]; then runtime_observer_ready=true; fi +argo_observer_role_exists=$(exists_res "$argo_ns" role "$argo_observer_role") +argo_observer_rolebinding_exists=$(exists_res "$argo_ns" rolebinding "$argo_observer_rolebinding") +argo_observer_can_get_application=$(can_i_argo get applications.argoproj.io) +argo_observer_ready=false +if [ "$argo_observer_role_exists" = true ] && [ "$argo_observer_rolebinding_exists" = true ] && [ "$argo_observer_can_get_application" = true ]; then argo_observer_ready=true; fi github_transport_json=$(python3 - "$github_transport_mode" "$gitmirror_ns" "$github_ssh_secret" "$github_ssh_private_key" "$github_ssh_private_source_ref" "$github_ssh_private_source_key" "$github_ssh_known_hosts_key" "$github_ssh_known_hosts_source_ref" "$github_ssh_known_hosts_source_key" "$github_token_secret" "$github_token_key" "$github_token_source_ref" "$github_token_source_key" <<'PY' import hashlib, json, subprocess, sys mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14] @@ -2907,7 +2992,7 @@ print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crd PY argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"k3sNodeConfig":$k3s_fragment,"tekton":{"installed":$(kubectl get crd pipelines.tekton.dev pipelineruns.tekton.dev >/dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"argoObserverRbac":{"roleName":"$argo_observer_role","roleExists":$argo_observer_role_exists,"roleBindingName":"$argo_observer_rolebinding","roleBindingExists":$argo_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canGetApplication":$argo_observer_can_get_application,"ready":$argo_observer_ready},"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} JSON `; } @@ -3252,6 +3337,7 @@ function statusNext( else if (!boolField(argoInstall, "statefulSetsReady")) blockers.push("argocd-statefulsets-not-ready"); else if (!boolField(argo, "projectExists")) blockers.push("argocd-project-missing"); else if (!boolField(argo, "applicationExists")) blockers.push("argocd-application-missing"); + if (!boolField(record(argo.argoObserverRbac), "ready")) blockers.push("argocd-observer-rbac-not-ready"); const next: Record = { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, dryRun: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --dry-run`,