diff --git a/config/agentrun.yaml b/config/agentrun.yaml index d1a6d90c..f96eabf5 100644 --- a/config/agentrun.yaml +++ b/config/agentrun.yaml @@ -746,7 +746,7 @@ controlPlane: readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/agentrun.git cachePvc: hwlab-git-mirror-cache - cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-jd01-v03-git-mirror-cache + cacheHostPath: null sshSecretName: git-mirror-github-ssh githubProxy: host: 127.0.0.1 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index e9a5e822..bf41b3a2 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -56,6 +56,7 @@ nodes: kubelet: maxPods: 500 registry: + mode: host-docker endpoint: 127.0.0.1:5000 egressProxy: mode: k8s-service-cluster-ip @@ -124,6 +125,7 @@ nodes: kubelet: maxPods: 500 registry: + mode: host-docker endpoint: 127.0.0.1:5000 egressProxy: mode: k8s-service-cluster-ip @@ -215,7 +217,19 @@ nodes: kubelet: maxPods: 500 registry: + mode: k8s-workload endpoint: 127.0.0.1:5000 + namespace: devops-infra + deploymentName: node-local-registry + serviceName: node-local-registry + pvcName: node-local-registry-storage + storage: 20Gi + image: docker.m.daocloud.io/library/registry:2 + imagePullPolicy: IfNotPresent + containerPort: 5000 + listenHost: 127.0.0.1 + listenPort: 5000 + hostNetwork: true egressProxy: mode: host-route clientName: jd01-host-proxy @@ -465,7 +479,7 @@ targets: serviceWriteName: git-mirror-write cachePvcName: hwlab-git-mirror-cache cachePvcStorage: 20Gi - cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-jd01-v03-git-mirror-cache + cacheHostPath: null servicePort: 8080 readContainerPort: 8080 writeContainerPort: 8081 diff --git a/scripts/src/hwlab-node-control-plane-model.ts b/scripts/src/hwlab-node-control-plane-model.ts index 3a2412ce..99b52f9a 100644 --- a/scripts/src/hwlab-node-control-plane-model.ts +++ b/scripts/src/hwlab-node-control-plane-model.ts @@ -209,10 +209,33 @@ export interface ControlPlaneNodeSpec { route: string; kubeRoute: string; k3s: ControlPlaneK3sNodeSpec | null; - registry: { endpoint: string }; + registry: ControlPlaneRegistrySpec; egressProxy: ControlPlaneEgressProxySpec | null; } +export type ControlPlaneRegistrySpec = ControlPlaneHostDockerRegistrySpec | ControlPlaneK8sWorkloadRegistrySpec; + +export interface ControlPlaneHostDockerRegistrySpec { + mode: "host-docker"; + endpoint: string; +} + +export interface ControlPlaneK8sWorkloadRegistrySpec { + mode: "k8s-workload"; + endpoint: string; + namespace: string; + deploymentName: string; + serviceName: string; + pvcName: string; + storage: string; + image: string; + imagePullPolicy: "Always" | "IfNotPresent" | "Never"; + containerPort: number; + listenHost: string; + listenPort: number; + hostNetwork: boolean; +} + export interface ControlPlaneK3sNodeSpec { serviceName: string; dropInPath: string; diff --git a/scripts/src/hwlab-node-control-plane-registry.ts b/scripts/src/hwlab-node-control-plane-registry.ts new file mode 100644 index 00000000..c87da41d --- /dev/null +++ b/scripts/src/hwlab-node-control-plane-registry.ts @@ -0,0 +1,75 @@ +import type { ControlPlaneRegistrySpec } from "./hwlab-node-control-plane-model"; + +export function controlPlaneRegistrySummary(registry: ControlPlaneRegistrySpec): Record { + if (registry.mode === "host-docker") { + return { + mode: registry.mode, + endpoint: registry.endpoint, + }; + } + return { + mode: registry.mode, + endpoint: registry.endpoint, + namespace: registry.namespace, + deploymentName: registry.deploymentName, + serviceName: registry.serviceName, + pvcName: registry.pvcName, + storage: registry.storage, + image: registry.image, + imagePullPolicy: registry.imagePullPolicy, + containerPort: registry.containerPort, + listenHost: registry.listenHost, + listenPort: registry.listenPort, + hostNetwork: registry.hostNetwork, + }; +} + +export function registryInfraManifest(registry: ControlPlaneRegistrySpec, labels: Record): Record[] { + if (registry.mode !== "k8s-workload") return []; + const workloadLabels = { ...labels, "app.kubernetes.io/name": registry.deploymentName }; + const podSpec: Record = { + containers: [{ + name: "registry", + image: registry.image, + imagePullPolicy: registry.imagePullPolicy, + env: [{ name: "REGISTRY_HTTP_ADDR", value: `${registry.listenHost}:${registry.listenPort}` }], + ports: [{ name: "http", containerPort: registry.containerPort }], + volumeMounts: [{ name: "storage", mountPath: "/var/lib/registry" }], + readinessProbe: { httpGet: { path: "/v2/", port: "http", host: registry.listenHost }, initialDelaySeconds: 3, periodSeconds: 10 }, + livenessProbe: { httpGet: { path: "/v2/", port: "http", host: registry.listenHost }, initialDelaySeconds: 10, periodSeconds: 30 }, + }], + volumes: [{ name: "storage", persistentVolumeClaim: { claimName: registry.pvcName } }], + }; + if (registry.hostNetwork) { + podSpec.hostNetwork = true; + podSpec.dnsPolicy = "ClusterFirstWithHostNet"; + } + return [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { name: registry.pvcName, namespace: registry.namespace, labels: workloadLabels }, + spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: registry.storage } } }, + }, + { + apiVersion: "v1", + kind: "Service", + metadata: { name: registry.serviceName, namespace: registry.namespace, labels: workloadLabels }, + spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": registry.deploymentName }, ports: [{ name: "http", port: registry.containerPort, targetPort: "http" }] }, + }, + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: registry.deploymentName, namespace: registry.namespace, labels: workloadLabels }, + spec: { + replicas: 1, + selector: { matchLabels: { "app.kubernetes.io/name": registry.deploymentName } }, + strategy: { type: "Recreate" }, + template: { + metadata: { labels: workloadLabels }, + spec: podSpec, + }, + }, + }, + ]; +} diff --git a/scripts/src/hwlab-node-control-plane-runtime.ts b/scripts/src/hwlab-node-control-plane-runtime.ts index 0e3413f5..bc4ad11a 100644 --- a/scripts/src/hwlab-node-control-plane-runtime.ts +++ b/scripts/src/hwlab-node-control-plane-runtime.ts @@ -12,11 +12,13 @@ import { type ControlPlaneK3sInstallSpec, type ControlPlaneK3sNodeSpec, type ControlPlaneNodeSpec, + type ControlPlaneRegistrySpec, type ControlPlaneRuntimeProxySpec, type ControlPlaneTargetSpec, type TektonInstallOptions, type ToolsImageOptions, } from "./hwlab-node-control-plane-model"; +import { controlPlaneRegistrySummary } from "./hwlab-node-control-plane-registry"; export function argoDesiredManifest(target: ControlPlaneTargetSpec): Record[] { return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; @@ -60,7 +62,7 @@ export function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTarg ciNamespace: target.ciNamespace, runtimeNamespace: target.runtimeNamespace, k3sNodeConfig: k3sNodeConfigPlan(node), - registry: node.registry.endpoint, + registry: controlPlaneRegistrySummary(node.registry), egressProxy: controlPlaneEgressProxySummary(node.egressProxy), sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch, @@ -147,7 +149,7 @@ export function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlane expectedDeployments: target.argo.install.expectedDeployments, expectedStatefulSets: target.argo.install.expectedStatefulSets, }, - registry: node.registry.endpoint, + registry: controlPlaneRegistrySummary(node.registry), imagePolicy: { noPrivateInputImages: true, buildInput: { sourceKind: target.tekton.toolsImage.sourceKind, context: target.tekton.toolsImage.context, dockerfile: target.tekton.toolsImage.dockerfile ?? null, dockerfileInline: target.tekton.toolsImage.dockerfileInline ?? null, composeFile: target.tekton.toolsImage.composeFile ?? null, publicBaseImages: target.tekton.toolsImage.publicBaseImages }, @@ -713,6 +715,7 @@ argo_app=${shQuote(target.argo.applicationName)} argo_observer_role=${shQuote(target.tekton.argoObserverRbac.roleName)} argo_observer_rolebinding=${shQuote(target.tekton.argoObserverRbac.roleBindingName)} registry=${shQuote(nodeSpec.registry.endpoint)} +registry_spec_json=${shQuote(JSON.stringify(controlPlaneRegistrySummary(nodeSpec.registry)))} tools_image=${shQuote(target.tekton.toolsImage.output)} tekton_required_crds_json=${shQuote(tektonRequiredCrds)} tekton_deployment_namespaces_json=${shQuote(tektonDeploymentNamespaces)} @@ -838,8 +841,67 @@ print(json.dumps({ })) PY ) -registry_ready=false -if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi +registry_endpoint_ready=false +if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_endpoint_ready=true; fi +registry_workload_json=$(python3 - "$registry_spec_json" "$registry_endpoint_ready" <<'PY' +import json, subprocess, sys +spec=json.loads(sys.argv[1]) +endpoint_ready=sys.argv[2] == "true" +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +def exists(kind, namespace, name): + return run(["kubectl", "-n", namespace, "get", kind, name]).returncode == 0 +def deploy_ready(namespace, name): + proc=run(["kubectl", "-n", namespace, "get", "deploy", name, "-o", "json"]) + if proc.returncode != 0: + return {"exists": False, "ready": False, "desired": None, "readyReplicas": None} + try: + obj=json.loads(proc.stdout or "{}") + except Exception: + obj={} + desired=int(obj.get("spec", {}).get("replicas") or 0) + ready=int(obj.get("status", {}).get("readyReplicas") or 0) + return {"exists": True, "ready": desired > 0 and ready == desired, "desired": desired, "readyReplicas": ready} +if spec.get("mode") != "k8s-workload": + print(json.dumps({"mode": spec.get("mode") or "host-docker", "managed": False, "ready": endpoint_ready, "endpointReady": endpoint_ready, "valuesPrinted": False})) + raise SystemExit(0) +ns=str(spec.get("namespace") or "") +deploy=str(spec.get("deploymentName") or "") +svc=str(spec.get("serviceName") or "") +pvc=str(spec.get("pvcName") or "") +deployment=deploy_ready(ns, deploy) +namespace_exists=run(["kubectl", "get", "ns", ns]).returncode == 0 +pvc_exists=exists("pvc", ns, pvc) +service_exists=exists("service", ns, svc) +ready=endpoint_ready and namespace_exists and pvc_exists and service_exists and deployment.get("ready") is True +print(json.dumps({ + "mode": "k8s-workload", + "managed": True, + "ready": ready, + "endpointReady": endpoint_ready, + "namespace": ns, + "namespaceExists": namespace_exists, + "deploymentName": deploy, + "deployment": deployment, + "serviceName": svc, + "serviceExists": service_exists, + "pvcName": pvc, + "pvcExists": pvc_exists, + "seededFromOldRegistry": False, + "zeroSeeded": True, + "valuesPrinted": False, +})) +PY +) +registry_ready=$(python3 - "$registry_workload_json" <<'PY' +import json, sys +try: + data=json.loads(sys.argv[1] or "{}") +except Exception: + data={} +print("true" if data.get("ready") is True else "false") +PY +) tools_repo_tag=\${tools_image#\${registry}/} tools_repo=\${tools_repo_tag%:*} tools_tag=\${tools_repo_tag##*:} @@ -1114,7 +1176,7 @@ print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crd PY argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') cat </dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"argoObserverRbac":{"roleName":"$argo_observer_role","roleExists":$argo_observer_role_exists,"roleBindingName":"$argo_observer_rolebinding","roleBindingExists":$argo_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canGetApplication":$argo_observer_can_get_application,"ready":$argo_observer_ready},"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"k3sNodeConfig":$k3s_fragment,"tekton":{"installed":$tekton_installed,"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook),"install":$tekton_fragment},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"argoObserverRbac":{"roleName":"$argo_observer_role","roleExists":$argo_observer_role_exists,"roleBindingName":"$argo_observer_rolebinding","roleBindingExists":$argo_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canGetApplication":$argo_observer_can_get_application,"ready":$argo_observer_ready},"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"endpointReady":$registry_endpoint_ready,"workload":$registry_workload_json,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}} JSON `; } @@ -1126,26 +1188,154 @@ set +e manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml) printf %s ${shQuote(encoded)} | base64 -d >"$manifest" field_manager=${shQuote(controlPlaneFieldManager(target))} +${registryPreApplyScript(node)} kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err kubectl_rc=$? ${k3sApplyScriptFragment(node.k3s, target)} -python3 - "$kubectl_rc" "$k3s_report_file" <<'PY' +${registryPostApplyScript(node)} +python3 - "$kubectl_rc" "$k3s_report_file" "$registry_pre_report_file" "$registry_report_file" <<'PY' import json, pathlib, sys k3s_report = {} try: k3s_report = json.loads(pathlib.Path(sys.argv[2]).read_text(errors='replace')) except Exception as exc: k3s_report = {"managed": None, "ok": False, "parseError": str(exc)} +registry_pre = {} +try: + registry_pre = json.loads(pathlib.Path(sys.argv[3]).read_text(errors='replace')) +except Exception as exc: + registry_pre = {"managed": None, "ok": False, "parseError": str(exc)} +registry_report = {} +try: + registry_report = json.loads(pathlib.Path(sys.argv[4]).read_text(errors='replace')) +except Exception as exc: + registry_report = {"managed": None, "ok": False, "parseError": str(exc)} out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else '' err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else '' -print(json.dumps({'k3sNodeConfig': k3s_report, 'kubernetesApply': {'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}}, ensure_ascii=False)) +print(json.dumps({'k3sNodeConfig': k3s_report, 'registryMigration': {'preApply': registry_pre, 'postApply': registry_report}, 'kubernetesApply': {'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}}, ensure_ascii=False)) PY rm -f "$manifest" +if [ "$registry_pre_rc" != 0 ]; then exit "$registry_pre_rc"; fi if [ "$kubectl_rc" != 0 ]; then exit "$kubectl_rc"; fi +if [ "$registry_rc" != 0 ]; then exit "$registry_rc"; fi exit "$k3s_rc" `; } +export function registryPreApplyScript(node: ControlPlaneNodeSpec): string { + if (node.registry.mode !== "k8s-workload") { + return ` +registry_pre_report_file=$(mktemp /tmp/hwlab-node-registry-pre.XXXXXX.json) +printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$registry_pre_report_file" +registry_pre_rc=0 +`; + } + const installRegistry = node.k3s?.install?.localRegistry ?? null; + return ` +registry_pre_report_file=$(mktemp /tmp/hwlab-node-registry-pre.XXXXXX.json) +host_registry_name=${shQuote(installRegistry?.containerName ?? "")} +registry_pre_rc=0 +host_registry_exists=false +host_registry_running=false +docker_stop_rc=0 +if [ -n "$host_registry_name" ] && command -v docker >/dev/null 2>&1; then + if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -Fx "$host_registry_name" >/dev/null 2>&1; then + host_registry_exists=true + if docker ps --format '{{.Names}}' 2>/dev/null | grep -Fx "$host_registry_name" >/dev/null 2>&1; then host_registry_running=true; fi + docker stop "$host_registry_name" >/tmp/hwlab-registry-docker-stop.out 2>/tmp/hwlab-registry-docker-stop.err || docker_stop_rc=$? + if [ "$docker_stop_rc" != 0 ]; then registry_pre_rc=$docker_stop_rc; fi + fi +fi +python3 - "$host_registry_exists" "$host_registry_running" "$docker_stop_rc" "$host_registry_name" <<'PY' >"$registry_pre_report_file" +import json, pathlib, sys +def tail(path): + p = pathlib.Path(path) + return p.read_text(errors='replace')[-1000:] if p.exists() else '' +print(json.dumps({ + "managed": True, + "ok": int(sys.argv[3] or "0") == 0, + "mutation": sys.argv[1] == "true", + "mode": "k8s-workload", + "hostDockerRegistryName": sys.argv[4] or None, + "hostDockerRegistryExists": sys.argv[1] == "true", + "hostDockerRegistryWasRunning": sys.argv[2] == "true", + "stoppedBeforeApply": sys.argv[1] == "true" and int(sys.argv[3] or "0") == 0, + "seededFromOldRegistry": False, + "zeroSeeded": True, + "dockerStopExitCode": int(sys.argv[3] or "0"), + "dockerStopStderrTail": tail('/tmp/hwlab-registry-docker-stop.err'), + "valuesPrinted": False, +}, ensure_ascii=False)) +PY +`; +} + +export function registryPostApplyScript(node: ControlPlaneNodeSpec): string { + if (node.registry.mode !== "k8s-workload") { + return ` +registry_report_file=$(mktemp /tmp/hwlab-node-registry.XXXXXX.json) +printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$registry_report_file" +registry_rc=0 +`; + } + return ` +registry_report_file=$(mktemp /tmp/hwlab-node-registry.XXXXXX.json) +registry_ns=${shQuote(node.registry.namespace)} +registry_deploy=${shQuote(node.registry.deploymentName)} +registry_pvc=${shQuote(node.registry.pvcName)} +registry_endpoint=${shQuote(node.registry.endpoint)} +registry_rc=0 +registry_pv= +registry_pv_path= +registry_pvc_bound=false +for _ in $(seq 1 60); do + registry_phase=$(kubectl -n "$registry_ns" get pvc "$registry_pvc" -o 'jsonpath={.status.phase}' 2>/dev/null || true) + if [ "$registry_phase" = Bound ]; then registry_pvc_bound=true; break; fi + sleep 2 +done +if [ "$registry_pvc_bound" = true ]; then + registry_pv=$(kubectl -n "$registry_ns" get pvc "$registry_pvc" -o 'jsonpath={.spec.volumeName}' 2>/dev/null || true) + if [ -n "$registry_pv" ]; then + registry_pv_path=$(kubectl get pv "$registry_pv" -o 'jsonpath={.spec.hostPath.path}' 2>/dev/null || true) + if [ -z "$registry_pv_path" ]; then registry_pv_path=$(kubectl get pv "$registry_pv" -o 'jsonpath={.spec.local.path}' 2>/dev/null || true); fi + fi +fi +kubectl -n "$registry_ns" delete pod -l "app.kubernetes.io/name=$registry_deploy" --ignore-not-found >/tmp/hwlab-registry-pod-delete.out 2>/tmp/hwlab-registry-pod-delete.err || true +kubectl -n "$registry_ns" rollout status deploy/"$registry_deploy" --timeout=180s >/tmp/hwlab-registry-rollout.out 2>/tmp/hwlab-registry-rollout.err +registry_rollout_rc=$? +registry_endpoint_ready=false +if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry_endpoint/v2/" >/tmp/hwlab-registry-endpoint.out 2>/tmp/hwlab-registry-endpoint.err && registry_endpoint_ready=true; fi +if [ "$registry_rollout_rc" != 0 ] || [ "$registry_endpoint_ready" != true ]; then registry_rc=1; fi +python3 - "$registry_rc" "$registry_pvc_bound" "$registry_pv" "$registry_pv_path" "$registry_rollout_rc" "$registry_endpoint_ready" "$registry_ns" "$registry_deploy" "$registry_pvc" "$registry_endpoint" <<'PY' >"$registry_report_file" +import json, pathlib, sys +def tail(path): + p = pathlib.Path(path) + return p.read_text(errors='replace')[-1000:] if p.exists() else '' +payload = { + "managed": True, + "ok": int(sys.argv[1]) == 0, + "mutation": True, + "mode": "k8s-workload", + "namespace": sys.argv[7], + "deploymentName": sys.argv[8], + "pvcName": sys.argv[9], + "endpoint": sys.argv[10], + "pvcBound": sys.argv[2] == "true", + "pvName": sys.argv[3] or None, + "pvPathPresent": bool(sys.argv[4]), + "seededFromOldRegistry": False, + "zeroSeeded": True, + "rolloutExitCode": int(sys.argv[5] or "1"), + "endpointReady": sys.argv[6] == "true", + "rolloutStdoutTail": tail('/tmp/hwlab-registry-rollout.out'), + "rolloutStderrTail": tail('/tmp/hwlab-registry-rollout.err'), + "valuesPrinted": False, +} +print(json.dumps(payload, ensure_ascii=False)) +PY +`; +} + export function controlPlaneFieldManager(target: ControlPlaneTargetSpec): string { return `unidesk-hwlab-${target.node.toLowerCase()}-${target.lane}-control-plane`; } @@ -1395,7 +1585,7 @@ export function toolsImageStatus(node: ControlPlaneNodeSpec, target: ControlPlan toolsImageReady: boolean; result: Record; } { - const result = runTransK3s(node.kubeRoute, registryStatusScript(node.registry.endpoint, target.tekton.toolsImage.output), timeoutSeconds); + const result = runTransK3s(node.kubeRoute, registryStatusScript(node.registry, target.tekton.toolsImage.output), timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; return { @@ -1492,13 +1682,49 @@ export function statusNext( return next; } -export function registryStatusScript(registryEndpoint: string, toolsImage: string): string { +export function registryStatusScript(registrySpec: ControlPlaneRegistrySpec, toolsImage: string): string { + const registryEndpoint = registrySpec.endpoint; + const registrySpecJson = JSON.stringify(controlPlaneRegistrySummary(registrySpec)); return ` set +e registry=${shQuote(registryEndpoint)} +registry_spec_json=${shQuote(registrySpecJson)} tools_image=${shQuote(toolsImage)} -registry_ready=false -if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi +registry_endpoint_ready=false +if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_endpoint_ready=true; fi +registry_workload_json=$(python3 - "$registry_spec_json" "$registry_endpoint_ready" <<'PY' +import json, subprocess, sys +spec=json.loads(sys.argv[1]) +endpoint_ready=sys.argv[2] == "true" +def run(args): + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +def exists(kind, namespace, name): + return run(["kubectl", "-n", namespace, "get", kind, name]).returncode == 0 +def deploy_ready(namespace, name): + proc=run(["kubectl", "-n", namespace, "get", "deploy", name, "-o", "json"]) + if proc.returncode != 0: + return False + data=json.loads(proc.stdout or "{}") + desired=int(data.get("spec", {}).get("replicas") or 0) + ready=int(data.get("status", {}).get("readyReplicas") or 0) + return desired > 0 and ready == desired +if spec.get("mode") != "k8s-workload": + print(json.dumps({"mode": spec.get("mode") or "host-docker", "managed": False, "ready": endpoint_ready, "endpointReady": endpoint_ready, "valuesPrinted": False})) + raise SystemExit(0) +ns=str(spec.get("namespace") or "") +ready=endpoint_ready and run(["kubectl", "get", "ns", ns]).returncode == 0 and exists("pvc", ns, str(spec.get("pvcName") or "")) and exists("service", ns, str(spec.get("serviceName") or "")) and deploy_ready(ns, str(spec.get("deploymentName") or "")) +print(json.dumps({"mode":"k8s-workload","managed":True,"ready":ready,"endpointReady":endpoint_ready,"namespace":ns,"deploymentName":spec.get("deploymentName"),"serviceName":spec.get("serviceName"),"pvcName":spec.get("pvcName"),"seededFromOldRegistry":False,"zeroSeeded":True,"valuesPrinted":False})) +PY +) +registry_ready=$(python3 - "$registry_workload_json" <<'PY' +import json, sys +try: + data=json.loads(sys.argv[1] or "{}") +except Exception: + data={} +print("true" if data.get("ready") is True else "false") +PY +) tools_repo_tag=\${tools_image#\${registry}/} tools_repo=\${tools_repo_tag%:*} tools_tag=\${tools_repo_tag##*:} @@ -1506,7 +1732,7 @@ tools_image_ready=false manifest_accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 -H "Accept: $manifest_accept" "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi cat <): ControlPlaneNodeSpe route: stringField(raw, "route", `nodes.${id}`), kubeRoute: stringField(raw, "kubeRoute", `nodes.${id}`), k3s, - registry: { endpoint: stringField(registry, "endpoint", `nodes.${id}.registry`) }, + registry: registrySpec(registry, `nodes.${id}.registry`), egressProxy, }; } +function registrySpec(raw: Record, path: string): ControlPlaneRegistrySpec { + const mode = stringField(raw, "mode", path); + const endpoint = stringField(raw, "endpoint", path); + if (mode === "host-docker") return { mode, endpoint }; + if (mode !== "k8s-workload") throw new Error(`${path}.mode must be host-docker or k8s-workload`); + const imagePullPolicy = optionalStringField(raw, "imagePullPolicy", path) ?? "IfNotPresent"; + if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") { + throw new Error(`${path}.imagePullPolicy must be Always, IfNotPresent, or Never`); + } + const namespace = stringField(raw, "namespace", path); + const deploymentName = stringField(raw, "deploymentName", path); + const serviceName = stringField(raw, "serviceName", path); + const pvcName = stringField(raw, "pvcName", path); + validateKubernetesName(namespace, `${path}.namespace`); + validateKubernetesName(deploymentName, `${path}.deploymentName`); + validateKubernetesName(serviceName, `${path}.serviceName`); + validateKubernetesName(pvcName, `${path}.pvcName`); + const image = stringField(raw, "image", path); + validatePublicBaseImage(image, `${path}.image`); + const hostNetwork = booleanField(raw, "hostNetwork", path); + const listenHost = stringField(raw, "listenHost", path); + if (listenHost !== "127.0.0.1" && listenHost !== "0.0.0.0") throw new Error(`${path}.listenHost must be 127.0.0.1 or 0.0.0.0`); + return { + mode, + endpoint, + namespace, + deploymentName, + serviceName, + pvcName, + storage: stringField(raw, "storage", path), + image, + imagePullPolicy, + containerPort: numberField(raw, "containerPort", path), + listenHost, + listenPort: numberField(raw, "listenPort", path), + hostNetwork, + }; +} + function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3sNodeSpec { const kubelet = asRecord(raw.kubelet, `${path}.kubelet`); const serviceName = stringField(raw, "serviceName", path); @@ -1872,7 +1913,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa serviceWriteName, cachePvcName: stringField(gitMirror, "cachePvcName", `${path}.gitMirror`), cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), - cacheHostPath: optionalStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`) ?? null, + cacheHostPath: nullableStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`), servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), readContainerPort: numberField(gitMirror, "readContainerPort", `${path}.gitMirror`), writeContainerPort: numberField(gitMirror, "writeContainerPort", `${path}.gitMirror`), @@ -1923,6 +1964,8 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa addNamespace(target.ciNamespace); addNamespace(target.runtimeNamespace); addNamespace(target.gitMirror.namespace); + if (_node.registry.mode === "k8s-workload") addNamespace(_node.registry.namespace); + manifests.push(...registryInfraManifest(_node.registry, labels)); manifests.push( { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: target.tekton.serviceAccountName, namespace: target.ciNamespace, labels } }, tektonRuntimeObserverRole(target, labels), @@ -2791,6 +2834,13 @@ function optionalStringField(obj: Record, key: string, path: st return value; } +function nullableStringField(obj: Record, key: string, path: string): string | null { + const value = obj[key]; + if (value === undefined || value === null) return null; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string when set`); + return value; +} + function absoluteConfigPathField(obj: Record, key: string, path: string): string { const value = stringField(obj, key, path); if (!value.startsWith("/") || value.includes("\0") || value.includes("..")) throw new Error(`${path}.${key} must be an absolute path without '..'`);