diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 38a114b2..d45c9875 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -7,6 +7,7 @@ metadata: - 491 - 1010 - 1119 + - 1148 imagePolicy: requireReproducibleBuildSource: true forbidPrivateOrNodeLocalImagesAsInputs: true @@ -83,6 +84,74 @@ nodes: - 74.48.78.17 - hyueapi.com - .hyueapi.com + D518: + route: D518 + kubeRoute: D518:k3s + k3s: + serviceName: k3s + dropInPath: /etc/systemd/system/k3s.service.d/20-unidesk-node-config.conf + nodeStatusName: d518 + execStartPre: + - - -/usr/bin/umount + - /Docker/host + serverArgs: + - server + - --disable + - traefik + - --disable + - servicelb + - --disable + - metrics-server + - --node-name + - D518 + - --node-label + - unidesk.ai/node-id=D518 + - --node-label + - unidesk.ai/provider-id=D518 + - --tls-san + - 127.0.0.1 + - --tls-san + - host.docker.internal + - --write-kubeconfig-mode + - "644" + - --kubelet-arg + - image-gc-high-threshold=95 + - --kubelet-arg + - image-gc-low-threshold=90 + - --kubelet-arg + - max-pods=500 + kubelet: + maxPods: 500 + registry: + endpoint: 127.0.0.1:5000 + egressProxy: + mode: k8s-service-cluster-ip + clientName: d518-global-proxy + namespace: platform-infra + serviceName: sub2api-egress-proxy + port: 10808 + sourceConfigRef: config/platform-infra/egress-proxy-sources.yaml#sources.master-shadowsocks + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 10.0.0.0/8 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + - hyueapi.com + - .hyueapi.com targets: - id: d601-v03 @@ -228,3 +297,146 @@ targets: expectedStatefulSets: - argocd-application-controller readinessTimeoutSeconds: 600 + - id: d518-v03 + node: D518 + lane: v03 + enabled: true + ciNamespace: hwlab-ci + runtimeNamespace: hwlab-v03 + source: + repository: pikasTech/HWLAB + branch: v0.3 + gitops: + branch: v0.3-gitops + path: deploy/gitops/node/d518/runtime-v03 + gitMirror: + namespace: devops-infra + serviceReadName: git-mirror-http + serviceWriteName: git-mirror-write + cachePvcName: hwlab-git-mirror-cache + cachePvcStorage: 20Gi + cacheHostPath: /var/lib/rancher/k3s/storage/hwlab-d518-v03-git-mirror-cache + servicePort: 8080 + deploymentReplicas: 1 + secretName: git-mirror-github-ssh + syncConfigMapName: git-mirror-sync-script + syncJobPrefix: git-mirror-hwlab-d518-v03-sync-manual + flushJobPrefix: git-mirror-hwlab-d518-v03-flush-manual + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git + egressProxy: + mode: node-global + required: true + githubTransport: + mode: ssh + tekton: + pipelineName: hwlab-d518-v03-ci-image-publish + serviceAccountName: hwlab-d518-v03-tekton-runner + pipelineRunPrefix: hwlab-d518-v03-ci-poll + toolsImage: + output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + imagePullPolicy: Always + sourceKind: dockerfile + context: . + dockerfileInline: + filename: hwlab-ci-node-tools.public.Dockerfile + lines: + - FROM docker.io/library/golang:1.24-bookworm AS golang-toolchain + - FROM 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + - ARG HTTP_PROXY + - ARG HTTPS_PROXY + - ARG ALL_PROXY + - ARG NO_PROXY + - ARG http_proxy + - ARG https_proxy + - ARG all_proxy + - ARG no_proxy + - COPY --from=golang-toolchain /usr/local/go /usr/local/go + - ENV PATH=/usr/local/go/bin:$PATH + - RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx + - ENV HWLAB_CI_NODE_DEPS=/opt/hwlab-ci-node-deps/node_modules + - RUN set -eu; export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}"; export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$HTTP_PROXY}}"; export ALL_PROXY="${ALL_PROXY:-${all_proxy:-}}"; export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"; export http_proxy="$HTTP_PROXY"; export https_proxy="$HTTPS_PROXY"; export all_proxy="$ALL_PROXY"; export no_proxy="$NO_PROXY"; export npm_config_registry="https://registry.npmmirror.com/"; export BUN_CONFIG_REGISTRY="https://registry.npmmirror.com/"; export npm_config_noproxy="$NO_PROXY"; if [ -n "$HTTP_PROXY" ]; then export npm_config_proxy="$HTTP_PROXY"; fi; if [ -n "$HTTPS_PROXY" ]; then export npm_config_https_proxy="$HTTPS_PROXY"; fi; export npm_config_fetch_retries=2; export npm_config_fetch_retry_mintimeout=2000; export npm_config_fetch_retry_maxtimeout=16000; export npm_config_fetch_timeout=120000; proxy_label="${HTTP_PROXY:+HTTP_PROXY}"; proxy_label="${proxy_label:-none}"; mkdir -p /opt/hwlab-ci-node-deps; cd /opt/hwlab-ci-node-deps; printf '{"private":true,"dependencies":{}}\n' > package.json; ok=0; delay=2; for attempt in 1 2 3 4 5; do echo "{\"event\":\"tools-yaml-node-npm-install\",\"attempt\":\"$attempt/5\",\"registry\":\"$npm_config_registry\",\"proxy\":\"$proxy_label\"}" >&2; if timeout 180s npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3; then ok=1; break; fi; if [ "$attempt" = 5 ]; then break; fi; echo "{\"event\":\"tools-yaml-node-npm-install\",\"status\":\"retrying\",\"attempt\":\"$attempt/5\",\"sleepSeconds\":$delay}" >&2; sleep "$delay"; delay=$((delay * 2)); done; test "$ok" = 1; node --input-type=module -e 'import("/opt/hwlab-ci-node-deps/node_modules/yaml/browser/dist/index.js").then((yaml)=>console.log("yaml-ok", typeof yaml.parse))' + - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V && go version + buildArgs: {} + buildNetwork: host + publicBaseImages: + - docker.io/library/node:22-bookworm-slim + - docker.io/library/golang:1.24-bookworm + - docker.io/oven/bun:1.3.13 + - docker.io/buildpack-deps:bookworm-scm + - docker.io/library/python:3.12-slim-bookworm + - docker.io/docker:29-cli + buildOwner: D518 + buildMode: node-local + ciBuildBenchmarks: + - profile: no-mirror-full + runtimeLaneConfigRef: config/hwlab-node-lanes.yaml#lanes.v03.targets.D518 + pipelineRunPrefix: hwlab-d518-v03-ci-bench + catalogPathTemplate: .unidesk/ci-build-benchmark/{profile}/{pipelineRun}/artifact-catalog.json + imageTagMode: full + pipelineTimeoutSeconds: 7200 + cachePolicy: + noPipelineRunReuse: true + forceFullBuild: true + forbidGitopsCatalogReuse: true + forbidDependencyCache: true + forbidBuildkitCache: true + forbidRegistryMirror: true + forbidLocalPreheatedImages: true + timings: + requiredStages: + - source-fetch + - dependency-install + - base-image-pull + - service-image-build + - registry-push + - pipeline-total + failureFamilies: + - dns + - proxy-connect + - tls-timeout + - rate-limit + - auth + - cache-hit-forbidden + - image-policy + - build-script + - registry-push + argo: + namespace: argocd + projectName: hwlab-d518 + applicationName: hwlab-node-v03 + applicationFile: application-v03.yaml + install: + enabled: true + sourceKind: url + version: v3.4.2 + manifestUrl: https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.2/manifests/install.yaml + fieldManager: unidesk-hwlab-node-argocd + imagePullPolicy: IfNotPresent + preloadImages: + - 127.0.0.1:5000/hwlab/argocd:v3.4.2 + - 127.0.0.1:5000/hwlab/dex:v2.45.0 + - 127.0.0.1:5000/hwlab/redis:8.2.3-alpine + imageRewrites: + - source: quay.io/argoproj/argocd:v3.4.2 + pullImage: quay.m.daocloud.io/argoproj/argocd:v3.4.2 + target: 127.0.0.1:5000/hwlab/argocd:v3.4.2 + - source: ghcr.io/dexidp/dex:v2.45.0 + pullImage: ghcr.m.daocloud.io/dexidp/dex:v2.45.0 + target: 127.0.0.1:5000/hwlab/dex:v2.45.0 + - source: public.ecr.aws/docker/library/redis:8.2.3-alpine + pullImage: docker.m.daocloud.io/library/redis:8.2.3-alpine + target: 127.0.0.1:5000/hwlab/redis:8.2.3-alpine + requiredCrds: + - applications.argoproj.io + - appprojects.argoproj.io + expectedDeployments: + - argocd-applicationset-controller + - argocd-dex-server + - argocd-notifications-controller + - argocd-redis + - argocd-repo-server + - argocd-server + expectedStatefulSets: + - argocd-application-controller + readinessTimeoutSeconds: 600 diff --git a/config/hwlab-node-lanes.yaml b/config/hwlab-node-lanes.yaml index 07a1f830..337df7aa 100644 --- a/config/hwlab-node-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -27,6 +27,13 @@ nodes: gitopsRoot: deploy/gitops/node networkProfile: d601-node-ci-egress downloadProfile: d601-node-default + D518: + route: D518 + kubeRoute: D518:k3s + sourceWorkspace: /home/ubuntu/workspace/hwlab-v03 + gitopsRoot: deploy/gitops/node + networkProfile: d518-node-ci-egress + downloadProfile: d518-node-default lanes: v02: @@ -69,6 +76,7 @@ lanes: apiUrl: http://74.48.78.17:19667 v03: node: G14 + activeTarget: D518 minor: 3 version: v0.3 sourceBranch: v0.3 @@ -486,6 +494,206 @@ lanes: envKey: DATASTORE_URI authnKey: authn-preshared-key role: hwlab_d601_v03_app + D518: + node: D518 + workspace: /home/ubuntu/workspace/hwlab-v03 + cicdRepo: /home/ubuntu/workspace/hwlab-v03-cicd.git + cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock + app: hwlab-node-v03 + pipeline: hwlab-d518-v03-ci-image-publish + pipelineRunPrefix: hwlab-d518-v03-ci-poll + serviceAccountName: hwlab-d518-v03-tekton-runner + controlPlaneFieldManager: unidesk-hwlab-d518-v03-control-plane + git: + url: git@github.com:pikasTech/HWLAB.git + readUrl: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + writeUrl: http://git-mirror-write.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + argo: + repoURL: http://git-mirror-http.devops-infra.svc.cluster.local:8080/pikasTech/HWLAB.git + gitopsBranch: v0.3-gitops + catalogPath: deploy/artifact-catalog.d518-v03.json + runtime: + path: deploy/gitops/node/d518/runtime-v03 + namespace: hwlab-v03 + renderDir: runtime-v03 + runtimeStore: + postgres: + mode: local-k3s + secretName: hwlab-v03-postgres + statefulSet: hwlab-v03-postgres + serviceName: hwlab-v03-postgres + adminUser: hwlab_v03 + adminPasswordSourceRef: hwlab/d518-v03-postgres.env + adminPasswordSourceKey: HWLAB_V03_POSTGRES_PASSWORD + cloudApi: + secretName: hwlab-cloud-api-v03-db + secretKey: database-url + database: hwlab_v03 + role: hwlab_v03 + openfga: + secretName: hwlab-v03-openfga + secretKey: datastore-uri + authnKey: authn-preshared-key + postgresPasswordKey: postgres-password + database: hwlab_openfga + role: hwlab_openfga + poolMax: 16 + connectionTimeoutMs: 5000 + queryRetryMaxAttempts: 5 + queryRetryInitialDelayMs: 250 + queryRetryMaxDelayMs: 5000 + webProbe: + browserProxyMode: direct + defaultOrigin: + mode: public + baseUrl: https://hwlab.pikapython.com + authLogin: + maxAttempts: 6 + requestTimeoutMs: 30000 + initialDelayMs: 500 + maxDelayMs: 10000 + alertThresholds: + sameOriginApiSlowMs: 10000 + partialApiSlowMs: 10000 + longLivedStreamOpenSlowMs: 10000 + visibleLoadingSlowMs: 10000 + turnTimingSampleSlackSeconds: 3 + turnElapsedSevereTimeoutSeconds: 120 + uncommandedStateChangeCommandWindowMs: 10000 + scrollJumpCommandWindowMs: 8000 + scrollJumpFromY: 250 + scrollJumpToY: 40 + sessionRailFallbackRatio: 0.5 + projectManagement: + enabled: true + targetPaths: + - /projects + - /projects/mdtodo + readinessSelectors: + - '[data-testid="project-management-root"]' + - '[data-testid="project-management-mdtodo"]' + naturalApiPathPrefixes: + - /v1/project-management/ + - /v1/workbench/launches + - /v1/agent/chat + commandAllowlist: + - gotoProjectMdtodo + - openMdtodoSourceConfig + - closeMdtodoSourceConfig + - configureMdtodoHwpodSource + - probeMdtodoSource + - reindexMdtodoSource + - selectProjectSource + - selectMdtodoSource + - selectMdtodoFile + - selectMdtodoTask + - expandMdtodoTask + - openMdtodoReportPreview + - toggleMdtodoReportFullscreen + - editMdtodoTaskInline + - editMdtodoTaskTitle + - editMdtodoTaskBody + - toggleMdtodoTaskStatus + - addMdtodoRootTask + - addMdtodoSubTask + - continueMdtodoTask + - deleteMdtodoTask + - launchWorkbenchFromTask + - launchWorkbenchFromMdtodo + launchRoute: /v1/workbench/launches + slowApiBudgetMs: 10000 + tektonDir: tekton-v03 + argoApplicationFile: application-v03.yaml + registryPrefix: 127.0.0.1:5000/hwlab + baseImage: 127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim + baseImageSource: node:20-bookworm-slim + serviceIds: + - hwlab-cloud-api + - hwlab-workbench-runtime + - hwlab-user-billing + - hwlab-project-management + - hwlab-cloud-web + - hwlab-gateway + - hwlab-edge-proxy + - hwlab-agent-skills + buildkit: + sidecarImage: 127.0.0.1:5000/hwlab/buildkit:rootless + stepEnv: + HOME: /tekton/home + XDG_CONFIG_HOME: /tekton/home/.config + observability: + prometheusOperator: false + runtimeImageRewrites: + - source: fatedier/frpc:v0.68.1 + target: 127.0.0.1:5000/hwlab/frpc:v0.68.1 + public: + webUrl: https://hwlab.pikapython.com + apiUrl: https://hwlab.pikapython.com + bootstrapAdmin: + username: admin + displayName: HWLAB v0.3 Admin + passwordSourceRef: hwlab/d518-v03-bootstrap-admin.env + passwordSourceKey: HWLAB_BOOTSTRAP_ADMIN_PASSWORD + passwordHashTransform: hwlab-sha256 + secretName: hwlab-v03-bootstrap-admin + secretKey: password-hash + rollout: + deployment: hwlab-cloud-api + publicExposure: + mode: pk01-caddy-frp + publicBaseUrl: https://hwlab.pikapython.com + hostname: hwlab.pikapython.com + expectedA: 82.156.23.220 + frpc: + serverAddr: 82.156.23.220 + serverPort: 22000 + tokenSourceRef: platform-infra/pk01-frp.env + tokenSourceKey: FRP_TOKEN + secretName: hwlab-v03-frpc-secrets + secretKey: frpc.toml + tokenKey: token + webProxy: + name: hwlab-d518-v03-cloud-web + remotePort: 22196 + localIP: hwlab-cloud-web.hwlab-v03.svc.cluster.local + localPort: 8080 + apiProxy: + name: hwlab-d518-v03-edge-proxy + remotePort: 22195 + localIP: hwlab-edge-proxy.hwlab-v03.svc.cluster.local + localPort: 6667 + caddy: + route: PK01 + configPath: /etc/caddy/Caddyfile + serviceName: caddy + email: ops@pikapython.com + tls: auto + responseHeaderTimeoutSeconds: 600 + externalPostgres: + provider: PK01 + configRef: config/platform-db/postgres-pk01.yaml + serviceName: d518-pk01-platform-postgres + endpointAddress: 82.156.23.220 + port: 5432 + runtimeAccess: + routeName: d518-pk01-postgres + endpointAddress: d518-tcp-egress-gateway.unidesk.svc.cluster.local + port: 25432 + sslmode: require + database: hwlab_d518_v03 + cloudApi: + secretName: hwlab-cloud-api-v03-db + secretKey: database-url + sourceRef: hwlab/d518-v03-cloud-api-db.env + envKey: DATABASE_URL + role: hwlab_d518_v03_app + openfga: + secretName: hwlab-v03-openfga + secretKey: datastore-uri + sourceRef: hwlab/d518-v03-openfga-db.env + envKey: DATASTORE_URI + authnKey: authn-preshared-key + role: hwlab_d518_v03_app networkProfiles: node-ci-egress: @@ -592,6 +800,43 @@ networkProfiles: - .svc - .svc.cluster.local - .cluster.local + d518-node-ci-egress: + proxy: + http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + all: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local + - kubernetes + - kubernetes.default + - kubernetes.default.svc + - 10.0.0.0/8 + - 10.42.0.0/16 + - 10.43.0.0/16 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + dockerBuildProxy: + http: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + https: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + all: http://sub2api-egress-proxy.platform-infra.svc.cluster.local:10808 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - 127.0.0.1:5000 + - localhost:5000 + - .svc + - .svc.cluster.local + - .cluster.local downloadProfiles: node-default: @@ -634,6 +879,26 @@ downloadProfiles: retries: 3 connectTimeoutSeconds: 10 maxTimeSeconds: 120 + d518-node-default: + git: + proxyMode: inherit + retries: 3 + timeoutSeconds: 60 + npm: + registry: https://registry.npmmirror.com/ + retries: 3 + fetchTimeoutSeconds: 120 + pip: + indexUrl: https://pypi.org/simple + retries: 3 + timeoutSeconds: 120 + docker: + registryMirrors: [] + pullRetries: 3 + curl: + retries: 3 + connectTimeoutSeconds: 10 + maxTimeSeconds: 120 d601-node-no-mirror-benchmark: git: proxyMode: inherit diff --git a/scripts/src/hwlab-node-lanes.ts b/scripts/src/hwlab-node-lanes.ts index fc47a735..ca6a2bd6 100644 --- a/scripts/src/hwlab-node-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -352,6 +352,7 @@ export const HWLAB_NODE_LANE_CONFIG_PATH = "config/hwlab-node-lanes.yaml"; interface HwlabLaneConfig { readonly id: HwlabRuntimeLane; readonly node: string; + readonly activeTarget?: string; readonly minor: number; readonly version: string; readonly sourceBranch: string; @@ -572,6 +573,7 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa return { id, node: stringField(raw, "node", `lanes.${id}`), + activeTarget: optionalStringField(raw, "activeTarget", `lanes.${id}`), minor, version, sourceBranch: stringField(raw, "sourceBranch", `lanes.${id}`), @@ -639,6 +641,7 @@ function laneTargetConfig(id: HwlabRuntimeLane, nodeId: string, baseRaw: Record< observability: mergeOptionalRecord(baseRaw.observability, targetRaw.observability), }; delete merged.targets; + delete merged.activeTarget; return laneConfig(id, merged); } @@ -1160,6 +1163,16 @@ function readHwlabNodeLaneConfig(): HwlabNodeLaneConfig { if (defaultLane === undefined || defaultLane.node !== defaultTarget.node) { throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH}.defaults must reference a declared lane target`); } + for (const lane of Object.values(lanes)) { + if (lane.activeTarget === undefined) continue; + if (nodes[lane.activeTarget] === undefined) { + throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH}.lanes.${lane.id}.activeTarget references missing node ${lane.activeTarget}`); + } + const target = laneTargets[lane.id]?.[lane.activeTarget]; + if (target === undefined && lane.node !== lane.activeTarget) { + throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH}.lanes.${lane.id}.activeTarget must reference lanes.${lane.id}.targets.${lane.activeTarget}`); + } + } return { defaultTarget, requiredNoProxy, nodes, lanes, laneTargets, networkProfiles, downloadProfiles }; } @@ -1270,6 +1283,19 @@ export function hwlabDefaultRuntimeTarget(): { readonly node: string; readonly l return { ...HWLAB_NODE_LANE_CONFIG.defaultTarget }; } +export function hwlabRuntimeActiveTarget(lane: HwlabRuntimeLane): { readonly node: string; readonly lane: HwlabRuntimeLane } { + const config = HWLAB_NODE_LANE_CONFIG.lanes[lane]; + return { + node: config.activeTarget ?? config.node, + lane, + }; +} + +export function hwlabRuntimeActiveLaneSpec(lane: HwlabRuntimeLane): HwlabRuntimeLaneSpec { + const target = hwlabRuntimeActiveTarget(lane); + return hwlabRuntimeLaneSpecForNode(target.lane, target.node); +} + export function hwlabRuntimeLaneConfigPath(): string { return HWLAB_NODE_LANE_CONFIG_PATH; }