Compare commits
10 Commits
f35fde8f65
...
345b1cb7bb
| Author | SHA1 | Date | |
|---|---|---|---|
| 345b1cb7bb | |||
| 496b5fc729 | |||
| 2e00e749a5 | |||
| 3039195c5b | |||
| ab3566435c | |||
| e956a0ec2a | |||
| 06ba3e579a | |||
| c9fbf48990 | |||
| ff40e1fdea | |||
| de984e06f7 |
@@ -9,6 +9,7 @@
|
||||
- 高耦合任务先串行定锚:公共类型/契约、source-of-truth、请求治理 authority、共享 runtime policy、同一大文件或同一状态机的修改,应先由主代理或一个子代理形成基线,其他子代理基于基线继续。
|
||||
- 任务要按成功率分层:调查、工具增强、业务修复、验证、PR review、上线 closeout 可以并行,但“修同一个根因的两套实现”通常不应并行。
|
||||
- 主代理需要持续调度,而不是把多个子任务排队串行执行。用户明确要求并行且任务在不同 worktree 时,应同时派发能并行的子任务,并通过 issue/PR 状态汇总。
|
||||
- 派发 CI/CD 或运行面调优子任务时,主代理不要在 prompt 或评论里反复解释历史;用 issue/comment 链接引用既有结论,把下一步限定为可由子代理自主真实触发的单步 gate,并要求子代理在该 gate 内小闭环调优到通过后再提交 PR。
|
||||
|
||||
## 模型与思考等级
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台。本文
|
||||
|
||||
## P0: 文件体积与脚本分流
|
||||
|
||||
- P0: 任何源码/CLI 文件超过 3000 行必须先按职责拆分再继续,禁止继续追加绕过。
|
||||
- P0: 任何源码/CLI 文件超过 3000 行必须先按职责差分拆到 2000 行以下再继续,禁止卡在 3000 行边界反复触发。
|
||||
- P0: 禁止把 shell/Node/Python 等脚本作为大段字符串内嵌;脚本必须放入原生后缀文件(如 `.sh`/`.mjs`/`.py`)并从文件加载。
|
||||
|
||||
## P0: 主 worktree 同步提交第一原则
|
||||
|
||||
@@ -719,7 +719,7 @@ controlPlane:
|
||||
- hyueapi.com
|
||||
- .hyueapi.com
|
||||
retention:
|
||||
maxRunners: 20
|
||||
maxRunners: 3
|
||||
cleanupOrder: oldest-inactive-last-active-first
|
||||
activeHeartbeatMaxAgeMs: 900000
|
||||
selectors:
|
||||
@@ -734,6 +734,13 @@ controlPlane:
|
||||
ageBasedCleanup:
|
||||
enabled: false
|
||||
maxAgeHours: 48
|
||||
sessionPvcRetention:
|
||||
enabled: true
|
||||
prefixes:
|
||||
- agentrun-v01-session-
|
||||
- agentrun-v02-session-
|
||||
- agentrun-jd01-v02-session-
|
||||
maxDeletePerRun: 1000
|
||||
cancelLifecycle:
|
||||
deliveryMode: manager-epoch
|
||||
gracefulAbortMs: 15000
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# SPEC: GH-1548 Gitea mirror/Actions visibility and controlled Docker builder POC
|
||||
apiVersion: unidesk.pikapython.com/v1alpha1
|
||||
kind: CicdGiteaActionsPoc
|
||||
metadata:
|
||||
id: gitea-actions-builder-poc
|
||||
owner: UniDesk
|
||||
issue: https://github.com/pikasTech/unidesk/issues/1548
|
||||
specRef: GH-1548
|
||||
version: draft-2026-07-05-p0-gitea-actions-builder-poc
|
||||
|
||||
spec:
|
||||
scope:
|
||||
phase: p1-spec-p2-poc
|
||||
productionFollowerReplacement: false
|
||||
rolloutEnabled: false
|
||||
rolloutMode: diagnostic-only
|
||||
preferredTarget: agentrun-jd01-v02
|
||||
decisionGate: env-reuse-must-pass-before-replacement
|
||||
|
||||
migration:
|
||||
issue: https://github.com/pikasTech/unidesk/issues/1549
|
||||
primaryEntrypoint: bun scripts/cli.ts cicd gitea-actions-poc plan
|
||||
replacementTarget: gitea-actions-driven-cicd
|
||||
branchFollower:
|
||||
status: deprecated
|
||||
mode: migration-only
|
||||
reason: self-maintained branch observation and controller state machine are being replaced by Gitea mirror and Gitea Actions.
|
||||
allowedDuringMigration:
|
||||
- read-only-status
|
||||
- debug-migration-evidence
|
||||
- existing-production-controller-until-cutover
|
||||
frozenCapabilities:
|
||||
- new-controller-loop-features
|
||||
- new-self-maintained-branch-observer
|
||||
- source-commit-driven-big-loop-debugging
|
||||
- new-fallbacks-around-gitea-actions
|
||||
finalDisposition: remove-apply-run-once-or-convert-to-readonly-archive-after-cutover
|
||||
|
||||
sourceAuthority:
|
||||
mode: immutable-snapshot-ref
|
||||
allowMutableBranchAsCiSource: false
|
||||
allowHostWorktree: false
|
||||
existingMirrorRef: config/cicd-branch-followers.yaml#followers.agentrun-jd01-v02.nativeStatus.source.gitMirrorReadUrl
|
||||
giteaMirror:
|
||||
enabledForPoc: true
|
||||
role: internal-github-upstream-mirror
|
||||
namespace: devops-infra
|
||||
serviceName: gitea-http
|
||||
internalBaseUrl: http://gitea-http.devops-infra.svc.cluster.local:3000
|
||||
snapshotRefPrefix: refs/unidesk/snapshots/gitea-actions/agentrun-v0.2
|
||||
mirrorLagStatus: required
|
||||
forceSyncEntry: gitea-ui-or-controlled-cli
|
||||
repositories:
|
||||
- key: agentrun
|
||||
repository: pikasTech/agentrun
|
||||
upstreamBranch: v0.2
|
||||
gitopsBranch: jd01-v0.2-gitops
|
||||
- key: unidesk
|
||||
repository: pikasTech/unidesk
|
||||
upstreamBranch: master
|
||||
|
||||
actions:
|
||||
enabledForPoc: true
|
||||
role: visibility-and-event-orchestration
|
||||
workflowSource:
|
||||
repository: pikasTech/agentrun
|
||||
path: .gitea/workflows/unidesk-agentrun-jd01-v02.yaml
|
||||
runner:
|
||||
mode: dedicated-act-runner
|
||||
trustBoundary: internal-only
|
||||
label: unidesk-ci-builder
|
||||
credentialsRef:
|
||||
sourceRef: cicd/gitea-actions-runner.env
|
||||
targetKey: token
|
||||
allowRuntimeNamespaceToken: false
|
||||
trigger:
|
||||
primary: actions-to-tekton-eventlistener
|
||||
fallback: actions-to-unidesk-controlled-api
|
||||
payloadMustInclude:
|
||||
- sourceCommit
|
||||
- snapshotRef
|
||||
- repository
|
||||
- branch
|
||||
- reusePlanArtifactRef
|
||||
logPolicy:
|
||||
defaultBoundedSummary: true
|
||||
fullLogRequiresDrillDown: true
|
||||
secretRedaction: sourceRef-presence-fingerprint-only
|
||||
|
||||
runtimePlane:
|
||||
dockerAllowed: false
|
||||
buildAllowed: false
|
||||
dockerSocketAllowed: false
|
||||
hostWorktreeAllowed: false
|
||||
sourceAuthority: immutable-snapshot-ref
|
||||
deployMode: gitops-argo-pull-built-image
|
||||
statusAuthority:
|
||||
- argo-application
|
||||
- kubernetes-workload-status
|
||||
- runtime-health
|
||||
- provenance-artifact
|
||||
|
||||
buildPlane:
|
||||
dockerAllowed: true
|
||||
buildAllowed: true
|
||||
dockerScope: ci-build-plane-only
|
||||
mode: controlled-docker-or-buildkit-builder
|
||||
engineCandidates:
|
||||
- native-docker-daemon
|
||||
- buildkit
|
||||
selectedEngineForPoc: buildkit
|
||||
forbidMasterServer: true
|
||||
forbidRuntimeNode: true
|
||||
endpoint:
|
||||
kind: buildkit
|
||||
ref: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.buildkitImage
|
||||
registry:
|
||||
ref: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.registryPrefix
|
||||
credentials:
|
||||
registry:
|
||||
sourceRef: cicd/registry-builder.env
|
||||
targetKey: REGISTRY_AUTH
|
||||
builder:
|
||||
sourceRef: cicd/buildkit-builder.env
|
||||
targetKey: BUILDKIT_AUTH
|
||||
cache:
|
||||
policyRef: config/cicd-gitea-actions-poc.yaml#spec.buildPlane.cache
|
||||
mode: yaml-first-buildkit-cache
|
||||
gcRequired: true
|
||||
provenance:
|
||||
required: true
|
||||
fields:
|
||||
- sourceCommit
|
||||
- snapshotRef
|
||||
- envIdentity
|
||||
- imageDigest
|
||||
- recipeHash
|
||||
- baseImageDigest
|
||||
- reuseDecision
|
||||
- builderId
|
||||
- actionsRunId
|
||||
- pipelineRunName
|
||||
|
||||
reuse:
|
||||
p0NoRegression: true
|
||||
sourceTruth: gitops/reuse.ymal
|
||||
sourceRead: immutable-snapshot
|
||||
existingParser: scripts/src/cicd-reuse-config.ts
|
||||
existingAgentRunPlanner: scripts/src/cicd-agentrun-reuse.ts
|
||||
ciConsumptionRequired: true
|
||||
requiredDecisions:
|
||||
- skipImageBuild
|
||||
- reuseEnvImage
|
||||
requiredArtifacts:
|
||||
- affectedServices
|
||||
- buildServices
|
||||
- reusedServices
|
||||
- skipImageBuild
|
||||
- reuseEnvImage
|
||||
- artifactProvenanceAudit
|
||||
noRegressionChecks:
|
||||
- runtime-reuse-hit-skips-rollout
|
||||
- env-unchanged-skips-env-image-build
|
||||
- ci-builder-consumes-reuse-plan
|
||||
- runtime-status-links-env-image-provenance
|
||||
|
||||
tekton:
|
||||
enabledForPoc: true
|
||||
triggerMode: eventlistener-or-controlled-pipelinerun
|
||||
eventListener:
|
||||
namespace: agentrun-ci
|
||||
name: gitea-actions-agentrun-jd01-v02
|
||||
serviceAccountRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.serviceAccountName
|
||||
pipeline:
|
||||
namespaceRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.namespace
|
||||
nameRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.pipeline
|
||||
runPrefixRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.pipelineRunPrefix
|
||||
sourceParameters:
|
||||
- sourceCommit
|
||||
- snapshotRef
|
||||
- reusePlanArtifactRef
|
||||
|
||||
argo:
|
||||
enabledForPoc: true
|
||||
role: runtime-closeout
|
||||
namespaceRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.gitops.argoNamespace
|
||||
applicationRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.gitops.argoApplication
|
||||
statusOnlyFromNativeObjects: true
|
||||
|
||||
budgets:
|
||||
endToEndSeconds: 120
|
||||
sourceSyncSeconds: 20
|
||||
actionsDispatchSeconds: 20
|
||||
reusePlanSeconds: 20
|
||||
buildOrReuseSeconds: 70
|
||||
gitopsArgoCloseoutSeconds: 50
|
||||
statusSeconds: 35
|
||||
|
||||
targets:
|
||||
- id: agentrun-jd01-v02
|
||||
enabled: true
|
||||
repository: pikasTech/agentrun
|
||||
branch: v0.2
|
||||
node: JD01
|
||||
lane: jd01-v02
|
||||
baseline:
|
||||
currentBranchFollowerSeconds: 105.7
|
||||
budgetRef: config/cicd-gitea-actions-poc.yaml#spec.budgets.endToEndSeconds
|
||||
source:
|
||||
branchRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.source.branch
|
||||
snapshotRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.source.sourceSnapshot.stageRefPrefix
|
||||
currentMirrorReadUrlRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.gitMirror.readUrl
|
||||
actions:
|
||||
workflowRef: config/cicd-gitea-actions-poc.yaml#spec.actions.workflowSource
|
||||
tekton:
|
||||
pipelineRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.pipeline
|
||||
pipelineRunPrefixRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.ci.pipelineRunPrefix
|
||||
argo:
|
||||
applicationRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.gitops.argoApplication
|
||||
runtime:
|
||||
namespaceRef: config/agentrun.yaml#controlPlane.lanes.jd01-v02.runtime.namespace
|
||||
workload: Deployment/agentrun-mgr
|
||||
closeout:
|
||||
healthPath: /health
|
||||
requiredEvidence:
|
||||
- sourceCommit
|
||||
- snapshotRef
|
||||
- actionsRunId
|
||||
- pipelineRunName
|
||||
- builderJobName
|
||||
- gitopsRevision
|
||||
- argoHealth
|
||||
- runtimeTargetSha
|
||||
- imageDigest
|
||||
- envIdentity
|
||||
- reuseDecision
|
||||
|
||||
stages:
|
||||
- id: mirror-sync
|
||||
owner: gitea-mirror
|
||||
statusAuthority: gitea-repository-and-snapshot-ref
|
||||
output: immutableSnapshotRef
|
||||
- id: actions-dispatch
|
||||
owner: gitea-actions
|
||||
statusAuthority: actions-run-api
|
||||
output: actionsRunId
|
||||
- id: reuse-plan
|
||||
owner: unidesk-cli
|
||||
statusAuthority: reuse-plan-artifact
|
||||
output: affectedServices-buildServices-reusedServices
|
||||
- id: build-or-reuse
|
||||
owner: controlled-builder-plane
|
||||
statusAuthority: provenance-artifact
|
||||
output: imageDigest-or-reuseDecision
|
||||
- id: tekton-pipelinerun
|
||||
owner: tekton
|
||||
statusAuthority: pipelineRun-taskRun-status
|
||||
output: pipelineRunName
|
||||
- id: gitops-publish
|
||||
owner: unidesk-cli-or-tekton
|
||||
statusAuthority: gitops-commit
|
||||
output: gitopsRevision
|
||||
- id: argo-closeout
|
||||
owner: argo-cd
|
||||
statusAuthority: argo-application
|
||||
output: sync-health
|
||||
- id: runtime-provenance
|
||||
owner: unidesk-status
|
||||
statusAuthority: k8s-native-status-and-health
|
||||
output: runtimeTargetSha-health-provenance
|
||||
|
||||
componentSurvey:
|
||||
- component: gitea-mirror
|
||||
role: internal-source-mirror
|
||||
maturity: candidate
|
||||
directReuse: yes-for-visible-pull-mirror
|
||||
docs: https://docs.gitea.com/usage/repo-mirror
|
||||
risk: must-map-existing-snapshot-and-flush-semantics
|
||||
- component: gitea-actions
|
||||
role: visibility-event-orchestration
|
||||
maturity: production-capable-candidate
|
||||
directReuse: yes-as-visibility-layer-not-final-truth
|
||||
docs: https://docs.gitea.com/usage/actions/overview
|
||||
risk: github-actions-compatibility-gaps-and-runner-trust-boundary
|
||||
- component: tekton-triggers
|
||||
role: actions-to-pipelinerun-bridge
|
||||
maturity: candidate
|
||||
directReuse: yes-for-eventlistener-to-pipelinerun
|
||||
docs: https://tekton.dev/docs/triggers/eventlisteners/
|
||||
risk: eventlistener-rbac-and-ingress-exposure-must-be-bounded
|
||||
- component: argo-cd
|
||||
role: gitops-runtime-closeout
|
||||
maturity: existing-production-component
|
||||
directReuse: keep-existing
|
||||
docs: https://argo-cd.readthedocs.io/en/stable/core_concepts/
|
||||
risk: actions-must-not-parse-human-argo-output
|
||||
|
||||
statusProjection:
|
||||
mode: bounded-summary
|
||||
defaultOutputMustNotDump: true
|
||||
requiredFields:
|
||||
- target
|
||||
- sourceCommit
|
||||
- snapshotRef
|
||||
- giteaMirrorLag
|
||||
- actionsRunId
|
||||
- pipelineRunName
|
||||
- builderJobName
|
||||
- buildServices
|
||||
- reusedServices
|
||||
- skipImageBuild
|
||||
- reuseEnvImage
|
||||
- imageDigest
|
||||
- gitopsRevision
|
||||
- argoSync
|
||||
- argoHealth
|
||||
- runtimeTargetSha
|
||||
- health
|
||||
- elapsedSeconds
|
||||
drillDown:
|
||||
actionsRun: gitea-actions-run
|
||||
tekton: cicd-pipelinerun
|
||||
builder: cicd-builder-job
|
||||
argo: argo-application
|
||||
runtime: cicd-runtime
|
||||
|
||||
next:
|
||||
plan: bun scripts/cli.ts cicd gitea-actions-poc plan
|
||||
status: bun scripts/cli.ts cicd gitea-actions-poc status
|
||||
existingFollowerStatus: bun scripts/cli.ts cicd branch-follower status --follower agentrun-jd01-v02
|
||||
pocIssue: https://github.com/pikasTech/unidesk/issues/1548
|
||||
@@ -0,0 +1,85 @@
|
||||
version: 1
|
||||
kind: platform-infra-gitea
|
||||
|
||||
metadata:
|
||||
id: gitea-internal-mirror
|
||||
owner: unidesk
|
||||
spec: GH-1548
|
||||
relatedIssues:
|
||||
- 1548
|
||||
- 1549
|
||||
|
||||
defaults:
|
||||
targetId: JD01
|
||||
|
||||
migration:
|
||||
role: gitea-actions-driven-cicd-source-authority
|
||||
replaces: branch-follower-self-maintained-branch-observer
|
||||
parentConfigRef: config/cicd-gitea-actions-poc.yaml#spec.sourceAuthority.giteaMirror
|
||||
envReusePolicy: preserve-existing-runtime-env-reuse
|
||||
buildPlane: controlled-docker-or-buildkit-outside-runtime
|
||||
runtimePlane: k3s-gitea-service-zero-docker
|
||||
|
||||
targets:
|
||||
- id: JD01
|
||||
route: JD01:k3s
|
||||
namespace: devops-infra
|
||||
role: active-poc
|
||||
enabled: true
|
||||
createNamespace: true
|
||||
storageClassName: local-path
|
||||
|
||||
app:
|
||||
name: gitea
|
||||
statefulSetName: gitea
|
||||
serviceName: gitea-http
|
||||
replicas: 1
|
||||
image:
|
||||
repository: docker.gitea.com/gitea
|
||||
tag: 1.26.4-rootless
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
type: ClusterIP
|
||||
httpPort: 3000
|
||||
sshPort: 2222
|
||||
server:
|
||||
domain: gitea-http.devops-infra.svc.cluster.local
|
||||
rootUrl: http://gitea-http.devops-infra.svc.cluster.local:3000/
|
||||
sshDomain: gitea-http.devops-infra.svc.cluster.local
|
||||
protocol: http
|
||||
startSshServer: true
|
||||
database:
|
||||
type: sqlite3
|
||||
path: /var/lib/gitea/gitea.db
|
||||
actions:
|
||||
enabled: true
|
||||
registration:
|
||||
disabled: true
|
||||
storage:
|
||||
data:
|
||||
size: 8Gi
|
||||
mountPath: /var/lib/gitea
|
||||
config:
|
||||
size: 1Gi
|
||||
mountPath: /etc/gitea
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
probes:
|
||||
healthPath: /api/healthz
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 12
|
||||
|
||||
validation:
|
||||
waitTimeoutSeconds: 55
|
||||
healthPath: /api/healthz
|
||||
+40
-1
@@ -133,17 +133,56 @@ gc:
|
||||
hwlabNode: JD01
|
||||
hwlabLane: v03
|
||||
agentrunNode: JD01
|
||||
agentrunLane: v02
|
||||
agentrunLane: jd01-v02
|
||||
limit: 80
|
||||
containerdImageCache:
|
||||
enabled: true
|
||||
runtimeEndpoint: unix:///run/k3s/containerd/containerd.sock
|
||||
namespace: k8s.io
|
||||
ciNamespaces:
|
||||
- hwlab-ci
|
||||
- agentrun-ci
|
||||
hostContainerdCache:
|
||||
enabled: true
|
||||
root: /var/lib/containerd
|
||||
address: /run/containerd/containerd.sock
|
||||
namespaces:
|
||||
- default
|
||||
orphanCleanup:
|
||||
enabled: true
|
||||
overlaySnapshotsRoot: /var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots
|
||||
contentBlobRoot: /var/lib/containerd/io.containerd.content.v1.content/blobs/sha256
|
||||
maxDeletePerRun: 100
|
||||
localPathStorage:
|
||||
enabled: true
|
||||
root: /var/lib/rancher/k3s/storage
|
||||
orphanDirPrefixes:
|
||||
- pvc-
|
||||
orphanMinAgeMinutes: 0
|
||||
maxDeletePerRun: 100
|
||||
policyTimer:
|
||||
enabled: true
|
||||
name: unidesk-jd01-low-risk-gc
|
||||
onCalendar: daily
|
||||
randomizedDelaySec: 20min
|
||||
stateDir: /var/lib/unidesk-gc
|
||||
configDir: /etc/unidesk-gc
|
||||
journalTargetBytes: 256MiB
|
||||
tmpMinAgeHours: 24
|
||||
includeAptCache: true
|
||||
includeToolCaches: false
|
||||
includeWebObserveArtifacts: true
|
||||
includeK3sImageCache: true
|
||||
includeHostContainerdCache: true
|
||||
includeLocalPathOrphans: true
|
||||
agentrunSessionPvcs:
|
||||
enabled: true
|
||||
namespace: agentrun-v02
|
||||
prefixes:
|
||||
- agentrun-v01-session-
|
||||
- agentrun-v02-session-
|
||||
- agentrun-jd01-v02-session-
|
||||
maxDeletePerRun: 100
|
||||
policyTimer:
|
||||
journald:
|
||||
systemMaxUse: 512MiB
|
||||
|
||||
@@ -14,6 +14,8 @@ Local worktrees, D601 runtime files, copied scripts, copied images, ad-hoc Kuber
|
||||
|
||||
When stable release lanes such as `release/v1` are enabled, the desired-state ref must be explicit in the command, job log and deploy output. Until that support exists, commands that are documented to read `origin/master:deploy.json` must keep doing so and must not silently switch to another branch or a dirty manifest.
|
||||
|
||||
Source and CLI files must not be kept near the 3000-line split boundary. Once a file exceeds 3000 lines, split it by responsibility until the original file is below 2000 lines before continuing feature or fix work. Do not make token-preserving micro-edits that leave the file just under or exactly at 3000 lines; that only guarantees the next small change will trigger the same split problem again.
|
||||
|
||||
## Prohibited Deployment Truth
|
||||
|
||||
The following practices are not acceptable as the long-term or hidden source of a working environment:
|
||||
|
||||
@@ -93,6 +93,16 @@ JD01 远端 plan 必须适配短连接:`snapshot` 和轻量 `plan` 返回有
|
||||
|
||||
JD01 PVC 归因必须按 YAML 配置的 namespace 集合读取 k8s API,不得复用 G14 专属 namespace 硬编码。报告至少包含 namespace、PVC、PV、host path、requested size、estimated actual bytes、active mount pods、owner/session/PipelineRun/runId、phase 和 reclaim policy。默认只做 plan 和归因;删除 PVC/PV、local-path host path、k3s storage、containerd snapshot/blob 或 workload 对象必须通过对应高层 retention 子命令和 GitOps/运行面 owner 判定,不能由 remote GC 扩大成 raw `kubectl delete` 或 host path 删除。
|
||||
|
||||
JD01/AgentRun 这类 PVC retention 确认入口必须适配短连接:确认步骤只提交经过 plan 选中的 Kubernetes 删除请求并快速返回,不能等待 local-path PV 后端同步回收完成;收敛状态通过下一次 dry-run、`gc remote JD01 status` 或专用 status 子命令查询。若一次提交在 transport 窗口内仍不稳定,应降低 YAML/CLI 批量,而不是改成手工 raw kubectl 或 host path 删除。
|
||||
|
||||
JD01 local-path storage 中没有 PV 引用的 orphan 目录只能通过 `gc remote JD01 plan|run --include-local-path-orphans` 进入候选。该入口必须从 YAML 读取 storage root、目录前缀 allowlist 和年龄策略,只允许删除 root 的直接子目录,且执行前重新确认无 PV 引用、无 symlink、无打开 fd/cwd;不得把它扩大成通用 `/var/lib/rancher/k3s/storage` 清空或 raw host path 删除。
|
||||
|
||||
JD01 host containerd 只能通过 `gc remote JD01 plan|run --include-host-containerd-cache` 进入候选。该入口必须从 YAML 读取 containerd root、socket address 和 namespace allowlist;只有 host containerd 目标 namespace 中没有 task/container 时才允许执行 `ctr images prune --all`,不得直接删除 `/var/lib/containerd` 下的 content、snapshot 或 metadata 路径。
|
||||
|
||||
当 host containerd 的 `ctr` 元数据中 images、containers、tasks、leases、snapshots 和 content 全为空,但 YAML allowlist 下仍残留 overlay snapshot 目录或 content blob 文件时,才能把它们分类为 orphan state。orphan state 清理仍必须通过 `--include-host-containerd-cache` 的 plan/run,执行前重新检查元数据为空、路径在 YAML root 下、名称匹配受控形态、无 symlink、无打开 fd/cwd;不得删除 metadata DB 或扩大到 containerd root。
|
||||
|
||||
JD01 增长降速 policy 由 `gc remote JD01 policy plan|install|status` 管理。policy install 会把原生 Python runner 和 JSON config 写入 YAML 指定的目标路径,再由 systemd timer 周期触发;runner 不读取 host worktree,也不依赖临时 CLI 输出解析。启用阶段必须逐项来自 `config/unidesk-cli.yaml#gc.remote.targets.JD01.policyTimer`:低风险 journal/apt/tmp、可选 tool cache、Web observe stale artifact、AgentRun session PVC、k3s CRI image prune、host containerd orphan state 和 local-path orphan。每个中风险阶段仍使用各自的 owner-aware 保护条件,失败时记录到 policy state,不得扩大成 raw `rm -rf`、raw kubectl 或 containerd metadata 删除。
|
||||
|
||||
JD01 Web observe artifact 是一等 GC 对象。state root 必须来自 YAML;候选按 run 聚合并读取 `manifest.json`、`heartbeat.json`、`pid`、report sha 和 top files。年龄判定以 manifest/heartbeat 的 started/completed/updated 字段、pid 存活和打开 fd 检查为准,不以目录 mtime 为唯一依据,因为手动 GC 或目录遍历可能刷新 mtime。active run、pid alive、open fd、未生成必要 report 的 run 均为 protected。safe 候选只覆盖超过 YAML retention 且可重建的 raw samples、browser-process、network/trace、screenshot 等大 artifact;长期保留 report summary、report json/md、最终截图或诊断摘要由 YAML cap/retention 策略控制。
|
||||
|
||||
JD01 Chrome 内存治理应优先管理 observer runner 生命周期,而不是孤立清理 Chrome 进程。Web probe sentinel 和 quick-verify 启动 observer 后,所有终态路径(成功、blocked、失败、timeout、异常)都必须执行 YAML 控制的 `web-probe observe stop`/force stop 流程,并验证对应 runner/Chrome process tree 退出;observe runner 自身也必须从 scenario/YAML 获得最大运行时长或 max samples 兜底,即使调用方退出也会停止采样并关闭 browser。browser freeze policy 只能作为异常保护,不替代正常任务生命周期结束后的 stop。
|
||||
|
||||
@@ -223,6 +223,11 @@ export interface AgentRunRunnerRetentionSpec {
|
||||
readonly enabled: boolean;
|
||||
readonly maxAgeHours: number | null;
|
||||
};
|
||||
readonly sessionPvcRetention: {
|
||||
readonly enabled: boolean;
|
||||
readonly prefixes: readonly string[];
|
||||
readonly maxDeletePerRun: number;
|
||||
};
|
||||
}
|
||||
|
||||
export type AgentRunCancelLifecycleStage = "accepted" | "persisted" | "delivered" | "aborting" | "terminalized" | "fenced" | "late-write-rejected";
|
||||
@@ -701,6 +706,14 @@ function parseCancelLifecycleStages(input: unknown, path: string): readonly Agen
|
||||
function parseRunnerRetention(input: Record<string, unknown>, path: string): AgentRunRunnerRetentionSpec {
|
||||
const selectors = recordField(input, "selectors", path);
|
||||
const ageBasedCleanup = recordField(input, "ageBasedCleanup", path);
|
||||
const sessionPvcRetentionRaw = input.sessionPvcRetention;
|
||||
const sessionPvcRetention = typeof sessionPvcRetentionRaw === "object" && sessionPvcRetentionRaw !== null && !Array.isArray(sessionPvcRetentionRaw)
|
||||
? sessionPvcRetentionRaw as Record<string, unknown>
|
||||
: {};
|
||||
const sessionPvcPrefixes = sessionPvcRetention.prefixes === undefined ? [] : stringArrayField(sessionPvcRetention, "prefixes", `${path}.sessionPvcRetention`);
|
||||
for (const [index, prefix] of sessionPvcPrefixes.entries()) {
|
||||
if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9-])?$/u.test(prefix)) throw new Error(`${path}.sessionPvcRetention.prefixes[${index}] must be a lowercase Kubernetes PVC name prefix`);
|
||||
}
|
||||
return {
|
||||
maxRunners: positiveIntegerField(input, "maxRunners", path),
|
||||
cleanupOrder: enumField(input, "cleanupOrder", path, ["oldest-inactive-last-active-first"]),
|
||||
@@ -716,6 +729,11 @@ function parseRunnerRetention(input: Record<string, unknown>, path: string): Age
|
||||
enabled: booleanField(ageBasedCleanup, "enabled", `${path}.ageBasedCleanup`),
|
||||
maxAgeHours: optionalPositiveIntegerField(ageBasedCleanup, "maxAgeHours", `${path}.ageBasedCleanup`) ?? null,
|
||||
},
|
||||
sessionPvcRetention: {
|
||||
enabled: sessionPvcRetention.enabled === undefined ? false : booleanField(sessionPvcRetention, "enabled", `${path}.sessionPvcRetention`),
|
||||
prefixes: sessionPvcPrefixes,
|
||||
maxDeletePerRun: optionalPositiveIntegerField(sessionPvcRetention, "maxDeletePerRun", `${path}.sessionPvcRetention`) ?? 100,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
|
||||
function runJson(args) {
|
||||
return JSON.parse(execFileSync("kubectl", args, { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }));
|
||||
}
|
||||
|
||||
function duBytes(path) {
|
||||
if (!path || !path.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
||||
const result = spawnSync("du", ["-sb", path], { encoding: "utf8", timeout: 8000 });
|
||||
if (result.status !== 0) return null;
|
||||
const value = Number(result.stdout.trim().split(/\s+/u)[0]);
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
const namespace = process.env.NAMESPACE;
|
||||
const confirm = process.env.CONFIRM === "true";
|
||||
const enabled = process.env.ENABLED === "true";
|
||||
const limit = Math.max(1, Math.min(Number(process.env.LIMIT || "100"), 1000));
|
||||
const prefixes = JSON.parse(Buffer.from(process.env.PREFIXES_JSON_B64 || "W10=", "base64").toString("utf8"));
|
||||
|
||||
if (!enabled) {
|
||||
console.log(JSON.stringify({ ok: false, error: "session-pvc-retention-disabled", selectedPvcCount: 0, mutation: false }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (!namespace || !Array.isArray(prefixes) || prefixes.length === 0) throw new Error("session PVC cleanup requires namespace and YAML prefixes");
|
||||
|
||||
const pvData = runJson(["get", "pv", "-o", "json"]);
|
||||
const pvcData = runJson(["-n", namespace, "get", "pvc", "-o", "json"]);
|
||||
const podData = runJson(["-n", namespace, "get", "pod", "-o", "json"]);
|
||||
const pvs = new Map((pvData.items || []).map((pv) => [pv.metadata?.name, pv]));
|
||||
const activeClaims = new Map();
|
||||
for (const pod of podData.items || []) {
|
||||
const phase = pod.status?.phase;
|
||||
if (phase === "Succeeded" || phase === "Failed") continue;
|
||||
for (const volume of pod.spec?.volumes || []) {
|
||||
const claim = volume.persistentVolumeClaim?.claimName;
|
||||
if (!claim) continue;
|
||||
const list = activeClaims.get(claim) || [];
|
||||
list.push(pod.metadata?.name);
|
||||
activeClaims.set(claim, list);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = [];
|
||||
const protectedRows = [];
|
||||
for (const pvc of pvcData.items || []) {
|
||||
const name = pvc.metadata?.name || "";
|
||||
const matchedPrefix = prefixes.find((prefix) => name.startsWith(prefix));
|
||||
if (!matchedPrefix) continue;
|
||||
const activeMountPods = activeClaims.get(name) || [];
|
||||
const pv = pvs.get(pvc.spec?.volumeName);
|
||||
const storageClass = pvc.spec?.storageClassName || pv?.spec?.storageClassName || null;
|
||||
const reclaimPolicy = pv?.spec?.persistentVolumeReclaimPolicy || null;
|
||||
const hostPath = pv?.spec?.hostPath?.path || pv?.spec?.local?.path || null;
|
||||
const row = {
|
||||
namespace,
|
||||
pvc: name,
|
||||
volume: pvc.spec?.volumeName || null,
|
||||
matchedPrefix,
|
||||
phase: pvc.status?.phase || null,
|
||||
pvPhase: pv?.status?.phase || null,
|
||||
storageClass,
|
||||
reclaimPolicy,
|
||||
activeMountCount: activeMountPods.length,
|
||||
activeMountPods: activeMountPods.slice(0, 5),
|
||||
estimatedBytes: duBytes(hostPath),
|
||||
};
|
||||
if (activeMountPods.length > 0 || storageClass !== "local-path" || reclaimPolicy !== "Delete") {
|
||||
protectedRows.push({ ...row, reason: activeMountPods.length > 0 ? "active-mount" : "not-local-path-delete" });
|
||||
} else {
|
||||
candidates.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
candidates.sort((a, b) => (b.estimatedBytes || 0) - (a.estimatedBytes || 0));
|
||||
const selected = candidates.slice(0, limit);
|
||||
const result = {
|
||||
ok: true,
|
||||
planKind: "agentrun-session-pvc-retention",
|
||||
namespace,
|
||||
dryRun: !confirm,
|
||||
mutation: confirm,
|
||||
criteria: { prefixes, storageClass: "local-path", reclaimPolicy: "Delete", requireNoActiveMount: true, limit },
|
||||
candidatePvcCount: candidates.length,
|
||||
selectedPvcCount: selected.length,
|
||||
protectedPvcCount: protectedRows.length,
|
||||
estimatedReclaimBytes: selected.reduce((sum, item) => sum + (item.estimatedBytes || 0), 0),
|
||||
selectedPreview: selected.slice(0, 12),
|
||||
protectedPreview: protectedRows.slice(0, 12),
|
||||
deletedPvcCount: 0,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
|
||||
if (confirm && selected.length > 0) {
|
||||
for (let index = 0; index < selected.length; index += 50) {
|
||||
execFileSync("kubectl", ["-n", namespace, "delete", "pvc", "--wait=false", ...selected.slice(index, index + 50).map((item) => item.pvc)], { encoding: "utf8", maxBuffer: 1024 * 1024 });
|
||||
}
|
||||
result.deletedPvcCount = selected.length;
|
||||
result.deleteMode = "submit-only-wait-false";
|
||||
}
|
||||
|
||||
console.log(JSON.stringify(result));
|
||||
@@ -33,7 +33,7 @@ import {
|
||||
} from "../agentrun-manifests";
|
||||
import { sha256Fingerprint } from "../platform-infra-ops-library";
|
||||
|
||||
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, ConfirmOptions, GitMirrorOptions, LaneConfirmOptions, RefreshOptions, SecretSyncOptions, StatusOptions } from "./options";
|
||||
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, ConfirmOptions, GitMirrorOptions, LaneConfirmOptions, RefreshOptions, SecretSyncOptions, StatusOptions } from "./options";
|
||||
import { agentRunControlPlaneStatusCommand } from "./public-exposure";
|
||||
import { applyYamlScript, manifestObjectRef, yamlLaneGitMirrorStatusScript } from "./secrets";
|
||||
import { compactAgentRunLaneStatusTarget, compactLaneSecretsStatus } from "./trigger";
|
||||
@@ -193,6 +193,18 @@ export function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPv
|
||||
};
|
||||
}
|
||||
|
||||
export function parseCleanupSessionPvcsOptions(args: string[]): CleanupSessionPvcsOptions {
|
||||
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds", "--node", "--lane"]));
|
||||
const base = parseConfirmOptions(args);
|
||||
return {
|
||||
...base,
|
||||
node: optionValue(args, "--node") ?? null,
|
||||
lane: optionValue(args, "--lane") ?? null,
|
||||
limit: positiveIntegerOption(args, "--limit", 100, 1000),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateOptions(args: string[], booleanOptions: Set<string>, valueOptions: Set<string>): void {
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
|
||||
@@ -34,7 +34,7 @@ import {
|
||||
import { sha256Fingerprint } from "../platform-infra-ops-library";
|
||||
|
||||
import type { AgentRunResourceVerb, AgentRunRestCompatGroup } from "./utils";
|
||||
import { controlPlaneApply, controlPlanePlan, parseCleanupReleasedPvOptions, parseCleanupRunnersOptions, parseCleanupRunsOptions, parseConfirmOptions, parseGitMirrorOptions, parseLaneConfirmOptions, parseRefreshOptions, parseSecretSyncOptions, status } from "./control-plane";
|
||||
import { controlPlaneApply, controlPlanePlan, parseCleanupReleasedPvOptions, parseCleanupRunnersOptions, parseCleanupRunsOptions, parseCleanupSessionPvcsOptions, parseConfirmOptions, parseGitMirrorOptions, parseLaneConfirmOptions, parseRefreshOptions, parseSecretSyncOptions, status } from "./control-plane";
|
||||
import { gitMirrorStatus } from "./git-mirror";
|
||||
import { agentRunExplain, isRecord, parseGitMirrorStatusOptions, parseStatusOptions, parseTriggerOptions } from "./options";
|
||||
import { renderAgentRunControlPlaneActionSummary, renderAgentRunControlPlanePlanSummary, renderAgentRunControlPlaneStatusSummary } from "./public-exposure";
|
||||
@@ -43,7 +43,7 @@ import { agentRunGetKindHelp, runAgentRunResourceCommand } from "./resource-acti
|
||||
import { runAgentRunRestCompatCommand, runGitMirrorJob, startAsyncAgentRunJob } from "./rest-bridge";
|
||||
import { exposeAgentRun, restartYamlLane, secretSync, triggerCurrent } from "./trigger";
|
||||
import { unsupported } from "./utils";
|
||||
import { cleanupReleasedPvs, cleanupRunners, cleanupRuns, refresh } from "./yaml-lane";
|
||||
import { cleanupReleasedPvs, cleanupRunners, cleanupRuns, cleanupSessionPvcs, refresh } from "./yaml-lane";
|
||||
|
||||
export function agentRunHelp(): unknown {
|
||||
return {
|
||||
@@ -143,6 +143,9 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str
|
||||
return options.full || options.raw ? result : renderAgentRunControlPlaneActionSummary(result, "AGENTRUN RUNNER CLEANUP");
|
||||
}
|
||||
if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(actionArgs));
|
||||
if (action === "cleanup-session-pvcs") {
|
||||
return await cleanupSessionPvcs(config, parseCleanupSessionPvcsOptions(actionArgs));
|
||||
}
|
||||
if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs));
|
||||
}
|
||||
if (group === "git-mirror") {
|
||||
@@ -271,7 +274,7 @@ export function agentRunHelpText(args: string[]): string {
|
||||
return [
|
||||
"Usage: bun scripts/cli.ts agentrun control-plane <action> [options]",
|
||||
"",
|
||||
"Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-released-pvs",
|
||||
"Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-session-pvcs, cleanup-released-pvs",
|
||||
"Examples:",
|
||||
" bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02",
|
||||
" bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run",
|
||||
@@ -283,6 +286,7 @@ export function agentRunHelpText(args: string[]): string {
|
||||
" bun scripts/cli.ts agentrun control-plane expose --dry-run",
|
||||
" bun scripts/cli.ts agentrun control-plane trigger-current --dry-run",
|
||||
" bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run",
|
||||
" bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node JD01 --lane jd01-v02 --dry-run",
|
||||
" bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -265,6 +265,13 @@ export interface CleanupReleasedPvOptions extends ConfirmOptions {
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface CleanupSessionPvcsOptions extends ConfirmOptions {
|
||||
node: string | null;
|
||||
lane: string | null;
|
||||
limit: number;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
export interface DisclosureOptions {
|
||||
full: boolean;
|
||||
raw: boolean;
|
||||
|
||||
@@ -35,7 +35,7 @@ import {
|
||||
} from "../agentrun-manifests";
|
||||
import { sha256Fingerprint } from "../platform-infra-ops-library";
|
||||
|
||||
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, RefreshOptions } from "./options";
|
||||
import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, RefreshOptions } from "./options";
|
||||
import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror";
|
||||
import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets";
|
||||
import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils";
|
||||
@@ -204,6 +204,55 @@ export async function cleanupReleasedPvs(config: UniDeskConfig, options: Cleanup
|
||||
};
|
||||
}
|
||||
|
||||
export async function cleanupSessionPvcs(config: UniDeskConfig, options: CleanupSessionPvcsOptions): Promise<Record<string, unknown>> {
|
||||
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
||||
const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupSessionPvcsScript(options, spec)]);
|
||||
const payload = captureJsonPayload(result);
|
||||
const ok = result.exitCode === 0 && payload.ok !== false;
|
||||
const base = {
|
||||
...payload,
|
||||
ok,
|
||||
command: "agentrun control-plane cleanup-session-pvcs",
|
||||
configPath,
|
||||
target: { node: spec.nodeId, lane: spec.lane, namespace: spec.runtime.namespace },
|
||||
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
||||
namespace: spec.runtime.namespace,
|
||||
retention: spec.deployment.runner.retention.sessionPvcRetention,
|
||||
probe: result.exitCode === 0 ? undefined : compactCapture(result, { full: true, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
||||
};
|
||||
if (options.dryRun || !options.confirm) {
|
||||
return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm` } };
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
dryRun: false,
|
||||
mutation: true,
|
||||
followUp: {
|
||||
dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`,
|
||||
diskPressure: `bun scripts/cli.ts gc remote ${spec.nodeId} status --limit 20`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function cleanupSessionPvcsScript(options: CleanupSessionPvcsOptions, spec: AgentRunLaneSpec): string {
|
||||
const retention = spec.deployment.runner.retention.sessionPvcRetention;
|
||||
const script = readFileSync(rootPath("scripts/src/agentrun/cleanup-session-pvcs.mjs"), "utf8");
|
||||
return [
|
||||
"set -eu",
|
||||
`namespace=${shQuote(spec.runtime.namespace)}`,
|
||||
`confirm=${options.confirm && !options.dryRun ? "true" : "false"}`,
|
||||
`limit=${String(Math.min(options.limit, retention.maxDeletePerRun))}`,
|
||||
`enabled=${retention.enabled ? "true" : "false"}`,
|
||||
`prefixes_json_b64=${shQuote(Buffer.from(JSON.stringify(retention.prefixes), "utf8").toString("base64"))}`,
|
||||
"tmp_dir=$(mktemp -d)",
|
||||
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
||||
"cat > \"$tmp_dir/cleanup-session-pvcs.mjs\" <<'NODE'",
|
||||
script,
|
||||
"NODE",
|
||||
"env NAMESPACE=\"$namespace\" CONFIRM=\"$confirm\" LIMIT=\"$limit\" ENABLED=\"$enabled\" PREFIXES_JSON_B64=\"$prefixes_json_b64\" node \"$tmp_dir/cleanup-session-pvcs.mjs\"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string {
|
||||
const retention = spec.deployment.runner.retention;
|
||||
const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64");
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
// SPEC: GH-1548 Gitea mirror/Actions visibility and controlled Docker builder POC.
|
||||
// Responsibility: read-only YAML-first plan/status for the proposed CI/CD governance split.
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { isAbsolute } from "node:path";
|
||||
import { rootPath, type UniDeskConfig } from "./config";
|
||||
import { renderMachine } from "./cicd-render";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
|
||||
const DEFAULT_CONFIG_PATH = "config/cicd-gitea-actions-poc.yaml";
|
||||
|
||||
type OutputMode = "text" | "json" | "yaml";
|
||||
type Action = "plan" | "status" | "help";
|
||||
|
||||
interface Options {
|
||||
action: Action;
|
||||
configPath: string;
|
||||
output: OutputMode;
|
||||
targetId: string | null;
|
||||
}
|
||||
|
||||
interface LoadedPoc {
|
||||
configPath: string;
|
||||
root: Record<string, unknown>;
|
||||
spec: Record<string, unknown>;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export function cicdGiteaActionsPocHelp(): unknown {
|
||||
return {
|
||||
command: "cicd gitea-actions-poc plan|status",
|
||||
output: "text by default; use --json, --raw, or -o json|yaml for machine output",
|
||||
usage: [
|
||||
"bun scripts/cli.ts cicd gitea-actions-poc plan",
|
||||
"bun scripts/cli.ts cicd gitea-actions-poc status",
|
||||
"bun scripts/cli.ts cicd gitea-actions-poc plan --target agentrun-jd01-v02",
|
||||
],
|
||||
config: DEFAULT_CONFIG_PATH,
|
||||
issue: "https://github.com/pikasTech/unidesk/issues/1548",
|
||||
description: "Read-only P1/P2 plan for replacing branch-follower responsibilities with Gitea mirror, Gitea Actions visibility, controlled Docker/BuildKit builder plane, existing Tekton, existing Argo CD, and bounded UniDesk status while preserving env reuse.",
|
||||
};
|
||||
}
|
||||
|
||||
export async function runGiteaActionsPocCommand(_config: UniDeskConfig | null, args: string[], alias = "gitea-actions-poc"): Promise<RenderedCliResult> {
|
||||
const options = parseOptions(args);
|
||||
const command = `cicd ${alias}${options.action === "help" ? "" : ` ${options.action}`}`;
|
||||
if (options.action === "help") return renderMachine(command, cicdGiteaActionsPocHelp(), options.output === "yaml" ? "yaml" : "json");
|
||||
const loaded = loadPoc(options.configPath);
|
||||
const payload = options.action === "plan" ? buildPlan(loaded, options) : buildStatus(loaded, options);
|
||||
if (options.output === "json") return renderMachine(command, payload, "json", payload.ok !== false);
|
||||
if (options.output === "yaml") return renderMachine(command, payload, "yaml", payload.ok !== false);
|
||||
return {
|
||||
ok: payload.ok !== false,
|
||||
command,
|
||||
renderedText: options.action === "plan" ? renderPlanHuman(payload) : renderStatusHuman(payload),
|
||||
contentType: "text/plain",
|
||||
};
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): Options {
|
||||
const actionToken = args[0];
|
||||
const action: Action = actionToken === undefined || isHelpToken(actionToken) ? "help" : parseAction(actionToken);
|
||||
const options: Options = {
|
||||
action,
|
||||
configPath: DEFAULT_CONFIG_PATH,
|
||||
output: "text",
|
||||
targetId: null,
|
||||
};
|
||||
for (let index = action === "help" ? 0 : 1; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === undefined || isHelpToken(arg)) {
|
||||
options.action = "help";
|
||||
continue;
|
||||
}
|
||||
if (arg === "--json" || arg === "--raw") {
|
||||
options.output = "json";
|
||||
continue;
|
||||
}
|
||||
if (arg === "-o" || arg === "--output") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined) throw new Error(`${arg} requires text, json, or yaml`);
|
||||
options.output = parseOutput(value, arg);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("-o=")) {
|
||||
options.output = parseOutput(arg.slice(3), "-o");
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--output=")) {
|
||||
options.output = parseOutput(arg.slice("--output=".length), "--output");
|
||||
continue;
|
||||
}
|
||||
if (arg === "--config") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.length === 0) throw new Error("--config requires a path");
|
||||
options.configPath = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--config=")) {
|
||||
options.configPath = arg.slice("--config=".length);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--target") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.length === 0) throw new Error("--target requires a target id");
|
||||
options.targetId = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--target=")) {
|
||||
options.targetId = arg.slice("--target=".length);
|
||||
continue;
|
||||
}
|
||||
if (arg === "--confirm") throw new Error("cicd gitea-actions-poc is read-only in GH-1548 P1/P2; --confirm is not accepted");
|
||||
if (arg === "--dry-run" || arg === "--all") continue;
|
||||
throw new Error(`unsupported cicd gitea-actions-poc option: ${arg}`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function parseAction(value: string): Action {
|
||||
if (value === "plan" || value === "status") return value;
|
||||
if (isHelpToken(value)) return "help";
|
||||
if (value === "apply" || value === "run-once" || value === "trigger-current") {
|
||||
throw new Error(`cicd gitea-actions-poc ${value} is intentionally unavailable; GH-1548 first stage is read-only plan/status`);
|
||||
}
|
||||
throw new Error("cicd gitea-actions-poc usage: cicd gitea-actions-poc plan|status [--target <id>] [--config <path>]");
|
||||
}
|
||||
|
||||
function parseOutput(value: string, flag: string): OutputMode {
|
||||
if (value === "text" || value === "json" || value === "yaml") return value;
|
||||
throw new Error(`${flag} must be text, json, or yaml`);
|
||||
}
|
||||
|
||||
function isHelpToken(value: string): boolean {
|
||||
return value === "help" || value === "--help" || value === "-h";
|
||||
}
|
||||
|
||||
function loadPoc(configPath: string): LoadedPoc {
|
||||
const absolutePath = isAbsolute(configPath) ? configPath : rootPath(configPath);
|
||||
if (!existsSync(absolutePath)) throw new Error(`${configPath} does not exist`);
|
||||
const root = record(Bun.YAML.parse(readFileSync(absolutePath, "utf8")), configPath);
|
||||
const spec = record(root.spec, `${configPath}.spec`);
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
validateRoot(root, spec, configPath, errors, warnings);
|
||||
return { configPath, root, spec, errors, warnings };
|
||||
}
|
||||
|
||||
function validateRoot(root: Record<string, unknown>, spec: Record<string, unknown>, configPath: string, errors: string[], warnings: string[]): void {
|
||||
if (stringOrNull(root.kind) !== "CicdGiteaActionsPoc") errors.push(`${configPath}.kind must be CicdGiteaActionsPoc`);
|
||||
const scope = optionalRecord(spec.scope);
|
||||
if (scope?.productionFollowerReplacement !== false) errors.push(`${configPath}.spec.scope.productionFollowerReplacement must be false for first-stage POC`);
|
||||
if (scope?.rolloutEnabled !== false) errors.push(`${configPath}.spec.scope.rolloutEnabled must be false for first-stage POC`);
|
||||
const runtimePlane = optionalRecord(spec.runtimePlane);
|
||||
if (runtimePlane?.dockerAllowed !== false) errors.push(`${configPath}.spec.runtimePlane.dockerAllowed must be false`);
|
||||
if (runtimePlane?.buildAllowed !== false) errors.push(`${configPath}.spec.runtimePlane.buildAllowed must be false`);
|
||||
if (runtimePlane?.dockerSocketAllowed !== false) errors.push(`${configPath}.spec.runtimePlane.dockerSocketAllowed must be false`);
|
||||
const buildPlane = optionalRecord(spec.buildPlane);
|
||||
if (buildPlane?.dockerAllowed !== true) errors.push(`${configPath}.spec.buildPlane.dockerAllowed must be true for the controlled builder POC`);
|
||||
if (buildPlane?.forbidMasterServer !== true) errors.push(`${configPath}.spec.buildPlane.forbidMasterServer must be true`);
|
||||
if (buildPlane?.forbidRuntimeNode !== true) errors.push(`${configPath}.spec.buildPlane.forbidRuntimeNode must be true`);
|
||||
const reuse = optionalRecord(spec.reuse);
|
||||
if (reuse?.p0NoRegression !== true) errors.push(`${configPath}.spec.reuse.p0NoRegression must be true`);
|
||||
if (reuse?.ciConsumptionRequired !== true) errors.push(`${configPath}.spec.reuse.ciConsumptionRequired must be true`);
|
||||
const decisions = stringArray(reuse?.requiredDecisions);
|
||||
if (!decisions.includes("skipImageBuild")) errors.push(`${configPath}.spec.reuse.requiredDecisions must include skipImageBuild`);
|
||||
if (!decisions.includes("reuseEnvImage")) errors.push(`${configPath}.spec.reuse.requiredDecisions must include reuseEnvImage`);
|
||||
const targets = arrayRecords(spec.targets);
|
||||
if (targets.length === 0) errors.push(`${configPath}.spec.targets must declare at least one POC target`);
|
||||
const enabledTargets = targets.filter((target) => target.enabled !== false);
|
||||
if (enabledTargets.length === 0) warnings.push(`${configPath}.spec.targets has no enabled target`);
|
||||
if (arrayRecords(spec.componentSurvey).length === 0) warnings.push(`${configPath}.spec.componentSurvey is empty; component maturity evidence will be invisible`);
|
||||
}
|
||||
|
||||
function buildPlan(loaded: LoadedPoc, options: Options): Record<string, unknown> {
|
||||
const targets = selectedTargets(loaded, options);
|
||||
const scope = optionalRecord(loaded.spec.scope);
|
||||
return {
|
||||
ok: loaded.errors.length === 0 && targets.errors.length === 0,
|
||||
action: "plan",
|
||||
configPath: loaded.configPath,
|
||||
issue: stringOrNull(optionalRecord(loaded.root.metadata)?.issue),
|
||||
specRef: stringOrNull(optionalRecord(loaded.root.metadata)?.specRef),
|
||||
phase: stringOrNull(scope?.phase),
|
||||
rolloutEnabled: scope?.rolloutEnabled === true,
|
||||
productionFollowerReplacement: scope?.productionFollowerReplacement === true,
|
||||
migration: compactMigration(optionalRecord(loaded.spec.migration)),
|
||||
sourceAuthority: compactSourceAuthority(optionalRecord(loaded.spec.sourceAuthority)),
|
||||
planes: [
|
||||
compactRuntimePlane(optionalRecord(loaded.spec.runtimePlane)),
|
||||
compactBuildPlane(optionalRecord(loaded.spec.buildPlane)),
|
||||
],
|
||||
reuse: compactReuse(optionalRecord(loaded.spec.reuse)),
|
||||
targets: targets.items.map(compactTarget),
|
||||
stages: arrayRecords(loaded.spec.stages).map(compactStage),
|
||||
componentSurvey: arrayRecords(loaded.spec.componentSurvey).map(compactComponent),
|
||||
statusProjection: compactStatusProjection(optionalRecord(loaded.spec.statusProjection)),
|
||||
budgets: loaded.spec.budgets ?? null,
|
||||
errors: [...loaded.errors, ...targets.errors],
|
||||
warnings: loaded.warnings,
|
||||
valuesRedacted: true,
|
||||
next: nextCommands(loaded, targets.items[0]),
|
||||
};
|
||||
}
|
||||
|
||||
function buildStatus(loaded: LoadedPoc, options: Options): Record<string, unknown> {
|
||||
const plan = buildPlan(loaded, options);
|
||||
const targets = arrayRecords(plan.targets);
|
||||
return {
|
||||
...plan,
|
||||
action: "status",
|
||||
statusSource: "config-only",
|
||||
statusMode: "declared-poc-not-applied",
|
||||
checks: targets.map((target) => ({
|
||||
target: target.id,
|
||||
source: "declared",
|
||||
giteaMirror: "not-applied",
|
||||
actionsRun: "not-applied",
|
||||
tekton: "existing-component",
|
||||
builderPlane: "declared-controlled-docker",
|
||||
argo: "existing-component",
|
||||
runtimePlane: "zero-docker-required",
|
||||
reuse: "p0-required-not-yet-proven-in-poc",
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function selectedTargets(loaded: LoadedPoc, options: Options): { items: Record<string, unknown>[]; errors: string[] } {
|
||||
const targets = arrayRecords(loaded.spec.targets);
|
||||
if (options.targetId === null) return { items: targets, errors: [] };
|
||||
const selected = targets.filter((target) => stringOrNull(target.id) === options.targetId);
|
||||
return selected.length > 0 ? { items: selected, errors: [] } : { items: [], errors: [`target ${options.targetId} not found in ${loaded.configPath}`] };
|
||||
}
|
||||
|
||||
function compactSourceAuthority(value: Record<string, unknown> | null): Record<string, unknown> {
|
||||
const mirror = optionalRecord(value?.giteaMirror);
|
||||
return {
|
||||
mode: stringOrNull(value?.mode),
|
||||
allowMutableBranchAsCiSource: value?.allowMutableBranchAsCiSource === true,
|
||||
allowHostWorktree: value?.allowHostWorktree === true,
|
||||
existingMirrorRef: stringOrNull(value?.existingMirrorRef),
|
||||
giteaMirrorEnabledForPoc: mirror?.enabledForPoc === true,
|
||||
giteaInternalBaseUrl: stringOrNull(mirror?.internalBaseUrl),
|
||||
snapshotRefPrefix: stringOrNull(mirror?.snapshotRefPrefix),
|
||||
repositories: arrayRecords(mirror?.repositories).map((repo) => `${repo.repository ?? "-"}@${repo.upstreamBranch ?? "-"}`),
|
||||
};
|
||||
}
|
||||
|
||||
function compactMigration(value: Record<string, unknown> | null): Record<string, unknown> {
|
||||
const branchFollower = optionalRecord(value?.branchFollower);
|
||||
return {
|
||||
issue: stringOrNull(value?.issue),
|
||||
primaryEntrypoint: stringOrNull(value?.primaryEntrypoint),
|
||||
replacementTarget: stringOrNull(value?.replacementTarget),
|
||||
branchFollower: {
|
||||
status: stringOrNull(branchFollower?.status),
|
||||
mode: stringOrNull(branchFollower?.mode),
|
||||
reason: stringOrNull(branchFollower?.reason),
|
||||
allowedDuringMigration: stringArray(branchFollower?.allowedDuringMigration),
|
||||
frozenCapabilities: stringArray(branchFollower?.frozenCapabilities),
|
||||
finalDisposition: stringOrNull(branchFollower?.finalDisposition),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function compactRuntimePlane(value: Record<string, unknown> | null): Record<string, unknown> {
|
||||
return {
|
||||
plane: "runtime",
|
||||
dockerAllowed: value?.dockerAllowed === true,
|
||||
buildAllowed: value?.buildAllowed === true,
|
||||
dockerSocketAllowed: value?.dockerSocketAllowed === true,
|
||||
hostWorktreeAllowed: value?.hostWorktreeAllowed === true,
|
||||
sourceAuthority: stringOrNull(value?.sourceAuthority),
|
||||
deployMode: stringOrNull(value?.deployMode),
|
||||
statusAuthority: stringArray(value?.statusAuthority),
|
||||
};
|
||||
}
|
||||
|
||||
function compactBuildPlane(value: Record<string, unknown> | null): Record<string, unknown> {
|
||||
return {
|
||||
plane: "ci-build",
|
||||
dockerAllowed: value?.dockerAllowed === true,
|
||||
buildAllowed: value?.buildAllowed === true,
|
||||
dockerScope: stringOrNull(value?.dockerScope),
|
||||
mode: stringOrNull(value?.mode),
|
||||
engineCandidates: stringArray(value?.engineCandidates),
|
||||
selectedEngineForPoc: stringOrNull(value?.selectedEngineForPoc),
|
||||
forbidMasterServer: value?.forbidMasterServer === true,
|
||||
forbidRuntimeNode: value?.forbidRuntimeNode === true,
|
||||
endpointRef: stringOrNull(optionalRecord(value?.endpoint)?.ref),
|
||||
registryRef: stringOrNull(optionalRecord(value?.registry)?.ref),
|
||||
provenanceRequired: optionalRecord(value?.provenance)?.required === true,
|
||||
provenanceFields: stringArray(optionalRecord(value?.provenance)?.fields),
|
||||
};
|
||||
}
|
||||
|
||||
function compactReuse(value: Record<string, unknown> | null): Record<string, unknown> {
|
||||
return {
|
||||
p0NoRegression: value?.p0NoRegression === true,
|
||||
sourceTruth: stringOrNull(value?.sourceTruth),
|
||||
sourceRead: stringOrNull(value?.sourceRead),
|
||||
existingParser: stringOrNull(value?.existingParser),
|
||||
existingAgentRunPlanner: stringOrNull(value?.existingAgentRunPlanner),
|
||||
ciConsumptionRequired: value?.ciConsumptionRequired === true,
|
||||
requiredDecisions: stringArray(value?.requiredDecisions),
|
||||
requiredArtifacts: stringArray(value?.requiredArtifacts),
|
||||
noRegressionChecks: stringArray(value?.noRegressionChecks),
|
||||
};
|
||||
}
|
||||
|
||||
function compactTarget(value: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
id: stringOrNull(value.id),
|
||||
enabled: value.enabled !== false,
|
||||
repository: stringOrNull(value.repository),
|
||||
branch: stringOrNull(value.branch),
|
||||
node: stringOrNull(value.node),
|
||||
lane: stringOrNull(value.lane),
|
||||
baselineSeconds: optionalRecord(value.baseline)?.currentBranchFollowerSeconds ?? null,
|
||||
budgetRef: stringOrNull(optionalRecord(value.baseline)?.budgetRef),
|
||||
snapshotRef: stringOrNull(optionalRecord(value.source)?.snapshotRef),
|
||||
mirrorReadUrlRef: stringOrNull(optionalRecord(value.source)?.currentMirrorReadUrlRef),
|
||||
workflowRef: stringOrNull(optionalRecord(value.actions)?.workflowRef),
|
||||
pipelineRef: stringOrNull(optionalRecord(value.tekton)?.pipelineRef),
|
||||
argoApplicationRef: stringOrNull(optionalRecord(value.argo)?.applicationRef),
|
||||
runtimeNamespaceRef: stringOrNull(optionalRecord(value.runtime)?.namespaceRef),
|
||||
workload: stringOrNull(optionalRecord(value.runtime)?.workload),
|
||||
healthPath: stringOrNull(optionalRecord(value.closeout)?.healthPath),
|
||||
requiredEvidence: stringArray(optionalRecord(value.closeout)?.requiredEvidence),
|
||||
};
|
||||
}
|
||||
|
||||
function compactStage(value: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
id: stringOrNull(value.id),
|
||||
owner: stringOrNull(value.owner),
|
||||
statusAuthority: stringOrNull(value.statusAuthority),
|
||||
output: stringOrNull(value.output),
|
||||
};
|
||||
}
|
||||
|
||||
function compactComponent(value: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
component: stringOrNull(value.component),
|
||||
role: stringOrNull(value.role),
|
||||
maturity: stringOrNull(value.maturity),
|
||||
directReuse: stringOrNull(value.directReuse),
|
||||
docs: stringOrNull(value.docs),
|
||||
risk: stringOrNull(value.risk),
|
||||
};
|
||||
}
|
||||
|
||||
function compactStatusProjection(value: Record<string, unknown> | null): Record<string, unknown> {
|
||||
return {
|
||||
mode: stringOrNull(value?.mode),
|
||||
defaultOutputMustNotDump: value?.defaultOutputMustNotDump === true,
|
||||
requiredFields: stringArray(value?.requiredFields),
|
||||
drillDown: optionalRecord(value?.drillDown) ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
function nextCommands(loaded: LoadedPoc, firstTarget: Record<string, unknown> | undefined): Record<string, unknown> {
|
||||
const declared = optionalRecord(loaded.spec.next);
|
||||
const targetId = stringOrNull(firstTarget?.id) ?? "agentrun-jd01-v02";
|
||||
return {
|
||||
plan: stringOrNull(declared?.plan) ?? "bun scripts/cli.ts cicd gitea-actions-poc plan",
|
||||
status: stringOrNull(declared?.status) ?? "bun scripts/cli.ts cicd gitea-actions-poc status",
|
||||
existingFollowerStatus: stringOrNull(declared?.existingFollowerStatus) ?? `bun scripts/cli.ts cicd branch-follower status --follower ${targetId}`,
|
||||
pocIssue: stringOrNull(declared?.pocIssue),
|
||||
};
|
||||
}
|
||||
|
||||
function renderPlanHuman(payload: Record<string, unknown>): string {
|
||||
const next = optionalRecord(payload.next);
|
||||
const planes = arrayRecords(payload.planes);
|
||||
const reuse = optionalRecord(payload.reuse);
|
||||
const migration = optionalRecord(payload.migration);
|
||||
const branchFollower = optionalRecord(migration?.branchFollower);
|
||||
const statusProjection = optionalRecord(payload.statusProjection);
|
||||
const errors = stringArray(payload.errors);
|
||||
const warnings = stringArray(payload.warnings);
|
||||
return [
|
||||
`CI/CD GITEA-ACTIONS POC PLAN (${payload.ok === false ? "blocked" : "ok"})`,
|
||||
"",
|
||||
table(["TARGET", "SOURCE", "NODE/LANE", "BASELINE", "SNAPSHOT", "TEKTON", "ARGO", "RUNTIME"], arrayRecords(payload.targets).map((target) => [
|
||||
target.id,
|
||||
`${target.repository ?? "-"}@${target.branch ?? "-"}`,
|
||||
`${target.node ?? "-"}/${target.lane ?? "-"}`,
|
||||
target.baselineSeconds === null ? "-" : `${target.baselineSeconds}s`,
|
||||
target.snapshotRef,
|
||||
target.pipelineRef,
|
||||
target.argoApplicationRef,
|
||||
target.workload,
|
||||
])),
|
||||
"",
|
||||
"SOURCE AUTHORITY",
|
||||
sourceAuthorityLine(optionalRecord(payload.sourceAuthority)),
|
||||
"",
|
||||
"MIGRATION POLICY",
|
||||
`replacement=${migration?.replacementTarget ?? "-"} primary=${migration?.primaryEntrypoint ?? "-"} branchFollower=${branchFollower?.status ?? "-"}:${branchFollower?.mode ?? "-"}`,
|
||||
`frozen=${stringArray(branchFollower?.frozenCapabilities).join(",") || "-"}`,
|
||||
"",
|
||||
table(["PLANE", "DOCKER", "BUILDS", "MODE", "ENGINE", "MASTER", "RUNTIME_NODE", "AUTHORITY"], planes.map((plane) => [
|
||||
plane.plane,
|
||||
boolText(plane.dockerAllowed),
|
||||
boolText(plane.buildAllowed),
|
||||
plane.mode ?? plane.deployMode ?? "-",
|
||||
plane.selectedEngineForPoc ?? "-",
|
||||
plane.forbidMasterServer === true ? "forbidden" : "-",
|
||||
plane.forbidRuntimeNode === true ? "forbidden" : "-",
|
||||
plane.sourceAuthority ?? plane.endpointRef ?? "-",
|
||||
])),
|
||||
"",
|
||||
"REUSE CONTRACT",
|
||||
`source=${reuse?.sourceTruth ?? "-"} p0=${boolText(reuse?.p0NoRegression)} ciConsumption=${boolText(reuse?.ciConsumptionRequired)} decisions=${stringArray(reuse?.requiredDecisions).join(",") || "-"}`,
|
||||
`artifacts=${stringArray(reuse?.requiredArtifacts).join(",") || "-"}`,
|
||||
"",
|
||||
table(["STAGE", "OWNER", "STATUS_AUTHORITY", "OUTPUT"], arrayRecords(payload.stages).map((stage) => [stage.id, stage.owner, stage.statusAuthority, stage.output])),
|
||||
"",
|
||||
table(["COMPONENT", "ROLE", "MATURITY", "DIRECT_REUSE", "RISK"], arrayRecords(payload.componentSurvey).map((component) => [component.component, component.role, component.maturity, component.directReuse, component.risk])),
|
||||
"",
|
||||
"STATUS FIELDS",
|
||||
stringArray(statusProjection?.requiredFields).join(", ") || "-",
|
||||
warnings.length === 0 ? "" : `\nWARNINGS\n${warnings.map((item) => `- ${item}`).join("\n")}`,
|
||||
errors.length === 0 ? "" : `\nERRORS\n${errors.map((item) => `- ${item}`).join("\n")}`,
|
||||
"",
|
||||
"NEXT",
|
||||
`status: ${next?.status ?? "-"}`,
|
||||
`existing-follower: ${next?.existingFollowerStatus ?? "-"}`,
|
||||
`issue: ${next?.pocIssue ?? payload.issue ?? "-"}`,
|
||||
"",
|
||||
].filter((line) => line !== "").join("\n");
|
||||
}
|
||||
|
||||
function renderStatusHuman(payload: Record<string, unknown>): string {
|
||||
const next = optionalRecord(payload.next);
|
||||
const migration = optionalRecord(payload.migration);
|
||||
const branchFollower = optionalRecord(migration?.branchFollower);
|
||||
const errors = stringArray(payload.errors);
|
||||
return [
|
||||
`CI/CD GITEA-ACTIONS POC STATUS (${payload.ok === false ? "blocked" : "declared-only"})`,
|
||||
"",
|
||||
`statusSource=${payload.statusSource ?? "-"} mode=${payload.statusMode ?? "-"}`,
|
||||
`replacement=${migration?.replacementTarget ?? "-"} branchFollower=${branchFollower?.status ?? "-"}:${branchFollower?.mode ?? "-"}`,
|
||||
"",
|
||||
table(["TARGET", "SOURCE", "GITEA", "ACTIONS", "TEKTON", "BUILDER", "ARGO", "RUNTIME", "REUSE"], arrayRecords(payload.checks).map((check) => [
|
||||
check.target,
|
||||
check.source,
|
||||
check.giteaMirror,
|
||||
check.actionsRun,
|
||||
check.tekton,
|
||||
check.builderPlane,
|
||||
check.argo,
|
||||
check.runtimePlane,
|
||||
check.reuse,
|
||||
])),
|
||||
errors.length === 0 ? "" : `\nERRORS\n${errors.map((item) => `- ${item}`).join("\n")}`,
|
||||
"",
|
||||
"NEXT",
|
||||
`plan: ${next?.plan ?? "-"}`,
|
||||
`existing-follower: ${next?.existingFollowerStatus ?? "-"}`,
|
||||
"",
|
||||
].filter((line) => line !== "").join("\n");
|
||||
}
|
||||
|
||||
function sourceAuthorityLine(value: Record<string, unknown> | null): string {
|
||||
return [
|
||||
`mode=${value?.mode ?? "-"}`,
|
||||
`mutableBranch=${boolText(value?.allowMutableBranchAsCiSource)}`,
|
||||
`hostWorktree=${boolText(value?.allowHostWorktree)}`,
|
||||
`giteaEnabled=${boolText(value?.giteaMirrorEnabledForPoc)}`,
|
||||
`snapshot=${value?.snapshotRefPrefix ?? "-"}`,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function record(value: unknown, path: string): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be a YAML object`);
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function optionalRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => typeof item === "object" && item !== null && !Array.isArray(item)) : [];
|
||||
}
|
||||
|
||||
function stringArray(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
|
||||
}
|
||||
|
||||
function stringOrNull(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function boolText(value: unknown): string {
|
||||
return value === true ? "true" : value === false ? "false" : "-";
|
||||
}
|
||||
|
||||
function table(headers: readonly string[], rows: readonly (readonly unknown[])[]): string {
|
||||
const normalized = rows.map((row) => headers.map((_, index) => cell(row[index])));
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...normalized.map((row) => row[index]?.length ?? 0)));
|
||||
const format = (row: readonly string[]) => row.map((value, index) => value.padEnd(widths[index] ?? 0)).join(" ").trimEnd();
|
||||
return [format(headers), format(headers.map((header) => "-".repeat(header.length))), ...normalized.map(format)].join("\n");
|
||||
}
|
||||
|
||||
function cell(value: unknown): string {
|
||||
if (value === null || value === undefined || value === "") return "-";
|
||||
const text = String(value).replace(/\s+/gu, " ");
|
||||
return text.length > 96 ? `${text.slice(0, 93)}...` : text;
|
||||
}
|
||||
@@ -4,6 +4,10 @@
|
||||
export function buildCicdHelp(configPath: string, spec: string): unknown {
|
||||
return {
|
||||
command: "cicd branch-follower plan|apply|status|run-once|debug-step|cleanup-state|events|logs|taskrun|job|runtime|gate",
|
||||
deprecated: true,
|
||||
mode: "migration-only",
|
||||
replacement: "bun scripts/cli.ts cicd gitea-actions-poc plan",
|
||||
issue: "https://github.com/pikasTech/unidesk/issues/1549",
|
||||
output: "text by default; use --json, --raw, or -o json|yaml for machine output",
|
||||
usage: [
|
||||
"bun scripts/cli.ts cicd branch-follower plan",
|
||||
@@ -27,6 +31,6 @@ export function buildCicdHelp(configPath: string, spec: string): unknown {
|
||||
],
|
||||
config: configPath,
|
||||
spec,
|
||||
description: "Deploy and inspect the YAML-first Kubernetes branch follower that follows HWLAB v0.3, AgentRun v0.2, and the selected web-probe sentinel master lane without using host worktrees as source authority.",
|
||||
description: "Deprecated migration-only branch follower controller. Existing production status/debug remains available during cutover, but new CI/CD replacement work should use Gitea mirror and Gitea Actions.",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -52,6 +52,10 @@ function renderPlanHuman(payload: Record<string, unknown>): string {
|
||||
return [
|
||||
`CI/CD BRANCH-FOLLOWER PLAN (${payload.ok === false ? "blocked" : "ok"})`,
|
||||
"",
|
||||
"DEPRECATION",
|
||||
"branch-follower=migration-only replacement=\"bun scripts/cli.ts cicd gitea-actions-poc plan\" issue=https://github.com/pikasTech/unidesk/issues/1549",
|
||||
"Do not add new branch-follower controller-loop features; use the Gitea Actions driven path from #1548/#1549.",
|
||||
"",
|
||||
table(["FOLLOWER", "ENABLED", "ADAPTER", "SOURCE", "TARGET", "BUDGET", "REFS", "CHECKS"], rows),
|
||||
"",
|
||||
"SOURCE AUTHORITY",
|
||||
|
||||
+23
-4
@@ -1,13 +1,32 @@
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower.
|
||||
// Responsibility: thin CI/CD top-level route entry; branch-follower logic lives in responsibility modules.
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower and GH-1548 Gitea Actions POC.
|
||||
// Responsibility: thin CI/CD top-level route entry; subcommand logic lives in responsibility modules.
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { renderMachine } from "./cicd-render";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { cicdHelp as branchFollowerHelp, runCicdCommand as runBranchFollowerCommand } from "./cicd-branch-follower";
|
||||
import { cicdGiteaActionsPocHelp, runGiteaActionsPocCommand } from "./cicd-gitea-actions-poc";
|
||||
|
||||
export function cicdHelp(): unknown {
|
||||
return branchFollowerHelp();
|
||||
return {
|
||||
command: "cicd gitea-actions-poc|branch-follower",
|
||||
output: "text by default for subcommands; top-level help is json",
|
||||
migration: {
|
||||
issue: "https://github.com/pikasTech/unidesk/issues/1549",
|
||||
primary: "cicd gitea-actions-poc",
|
||||
deprecated: "cicd branch-follower",
|
||||
note: "New CI/CD replacement work should use the Gitea Actions driven path; branch-follower is migration-only.",
|
||||
},
|
||||
subcommands: [
|
||||
cicdGiteaActionsPocHelp(),
|
||||
branchFollowerHelp(),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export async function runCicdCommand(config: UniDeskConfig | null, args: string[]): Promise<RenderedCliResult> {
|
||||
return await runBranchFollowerCommand(config, args);
|
||||
const top = args[0];
|
||||
if (top === undefined || top === "help" || top === "--help" || top === "-h") return renderMachine("cicd", cicdHelp(), "json");
|
||||
if (top === "branch-follower") return await runBranchFollowerCommand(config, args);
|
||||
if (top === "gitea-actions-poc" || top === "gitea-builder-poc") return await runGiteaActionsPocCommand(config, args.slice(1), top);
|
||||
throw new Error("cicd usage: cicd gitea-actions-poc|branch-follower");
|
||||
}
|
||||
|
||||
+94
-4
@@ -142,6 +142,85 @@ function stringValue(value: unknown): string | null {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown): number | null {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function rounded(value: number | null): number | null {
|
||||
return value === null ? null : Math.round(value * 100) / 100;
|
||||
}
|
||||
|
||||
function providerHostPressureSummary(providerId: string): unknown {
|
||||
const response = coreInternalFetch("/api/nodes/system-status?limit=24");
|
||||
const body = recordValue(recordValue(response).body);
|
||||
const systemStatuses = arrayValue(body.systemStatuses);
|
||||
const item = systemStatuses
|
||||
.map((entry) => recordValue(entry))
|
||||
.find((entry) => entry.providerId === providerId) ?? null;
|
||||
if (item === null) {
|
||||
return {
|
||||
ok: false,
|
||||
degradedReason: "system-status-not-found",
|
||||
systemStatusFetch: response,
|
||||
};
|
||||
}
|
||||
const current = recordValue(item.current);
|
||||
if (Object.keys(current).length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
providerId,
|
||||
degradedReason: "system-status-current-missing",
|
||||
updatedAt: item.updatedAt ?? null,
|
||||
nodeStatus: item.nodeStatus ?? null,
|
||||
classification: "host-pressure-snapshot-unavailable",
|
||||
};
|
||||
}
|
||||
const cpu = recordValue(current.cpu);
|
||||
const memory = recordValue(current.memory);
|
||||
const disk = recordValue(current.disk);
|
||||
const cores = numberValue(cpu.cores);
|
||||
const load1 = numberValue(cpu.load1);
|
||||
const load5 = numberValue(cpu.load5);
|
||||
const load15 = numberValue(cpu.load15);
|
||||
const load1PerCore = cores !== null && cores > 0 && load1 !== null ? load1 / cores : null;
|
||||
const memoryPercent = numberValue(memory.percent);
|
||||
const memoryAvailableBytes = numberValue(memory.availableBytes);
|
||||
const swapTotalBytes = numberValue(memory.swapTotalBytes);
|
||||
const diskPercent = numberValue(disk.percent);
|
||||
const signals: string[] = [];
|
||||
if (load1PerCore !== null && load1PerCore >= 4) signals.push("load1-per-core>=4");
|
||||
if (memoryPercent !== null && memoryPercent >= 90) signals.push("memory-percent>=90");
|
||||
if (memoryAvailableBytes !== null && memoryAvailableBytes < 512 * 1024 * 1024) signals.push("memory-available<512MiB");
|
||||
if (swapTotalBytes === 0) signals.push("swap-disabled");
|
||||
if (diskPercent !== null && diskPercent >= 90) signals.push("disk-percent>=90");
|
||||
return {
|
||||
ok: current.ok ?? null,
|
||||
providerId,
|
||||
updatedAt: item.updatedAt ?? null,
|
||||
collectedAt: current.collectedAt ?? null,
|
||||
cpu: {
|
||||
cores,
|
||||
load1,
|
||||
load5,
|
||||
load15,
|
||||
load1PerCore: rounded(load1PerCore),
|
||||
},
|
||||
memory: {
|
||||
percent: memoryPercent,
|
||||
availableBytes: memoryAvailableBytes,
|
||||
swapTotalBytes,
|
||||
},
|
||||
disk: {
|
||||
percent: diskPercent,
|
||||
availableBytes: numberValue(disk.availableBytes),
|
||||
mount: disk.mount ?? null,
|
||||
},
|
||||
signals,
|
||||
classification: signals.length > 0 ? "host-pressure-signals-present" : "host-pressure-signals-not-observed",
|
||||
};
|
||||
}
|
||||
|
||||
export async function debugSshPool(_config: UniDeskConfig, providerId: string): Promise<unknown> {
|
||||
const nodesResponse = await coreInternalFetch("/api/nodes");
|
||||
const body = recordValue(recordValue(nodesResponse).body);
|
||||
@@ -159,6 +238,7 @@ export async function debugSshPool(_config: UniDeskConfig, providerId: string):
|
||||
};
|
||||
}
|
||||
const labels = recordValue(node.labels);
|
||||
const nodeStatus = stringValue(node.status);
|
||||
const pool = {
|
||||
transport: stringValue(labels.providerGatewaySshDataTransport),
|
||||
host: stringValue(labels.providerGatewaySshDataHost),
|
||||
@@ -173,19 +253,27 @@ export async function debugSshPool(_config: UniDeskConfig, providerId: string):
|
||||
const ready = Number(pool.ready ?? 0);
|
||||
const claimed = Number(pool.claimed ?? 0);
|
||||
const desired = Number(pool.desired ?? 0);
|
||||
const ok = pool.transport === "tcp-pool" && Number.isFinite(ready) && ready > 0;
|
||||
const providerOnline = nodeStatus === "online";
|
||||
const poolLabelReady = pool.transport === "tcp-pool" && Number.isFinite(ready) && ready > 0;
|
||||
const executionPathReady = providerOnline && poolLabelReady;
|
||||
return {
|
||||
ok,
|
||||
ok: executionPathReady,
|
||||
providerId,
|
||||
providerOnline,
|
||||
poolLabelReady,
|
||||
executionPathReady,
|
||||
node: {
|
||||
providerId: node.providerId,
|
||||
name: node.name,
|
||||
status: node.status,
|
||||
status: nodeStatus,
|
||||
lastHeartbeat: node.lastHeartbeat ?? null,
|
||||
updatedAt: node.updatedAt ?? null,
|
||||
},
|
||||
pool,
|
||||
classification: ok
|
||||
hostPressure: providerHostPressureSummary(providerId),
|
||||
classification: !providerOnline
|
||||
? (nodeStatus === "offline" ? "provider-offline" : "provider-not-online")
|
||||
: poolLabelReady
|
||||
? "ssh-tcp-pool-ready"
|
||||
: pool.transport !== "tcp-pool"
|
||||
? "provider-gateway-upgrade-required"
|
||||
@@ -195,7 +283,9 @@ export async function debugSshPool(_config: UniDeskConfig, providerId: string):
|
||||
next: {
|
||||
smoke: `trans ${providerId} argv true`,
|
||||
fullHealth: "bun scripts/cli.ts debug health",
|
||||
gcDegradedSnapshot: `bun scripts/cli.ts gc remote ${providerId} snapshot --no-save`,
|
||||
},
|
||||
note: "Pool labels are provider-reported channel state; use the smoke command or gc degraded output to verify execution path.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
def k3s_crictl_base():
|
||||
endpoint = str(CONTAINERD_CONFIG.get("runtimeEndpoint") or "unix:///run/k3s/containerd/containerd.sock")
|
||||
return ["crictl", "--runtime-endpoint", endpoint]
|
||||
|
||||
def shell_single_quote(value):
|
||||
return "'" + str(value).replace("'", "'\"'\"'") + "'"
|
||||
|
||||
def k3s_crictl_json(args, timeout=30):
|
||||
result = command(k3s_crictl_base() + args + ["-o", "json"], timeout)
|
||||
if result["exitCode"] != 0:
|
||||
return None, result
|
||||
try:
|
||||
return json.loads(result["stdout"] or "{}"), result
|
||||
except Exception:
|
||||
return None, result
|
||||
|
||||
def ci_activity_snapshot_for_prune():
|
||||
namespaces = config_list(CONTAINERD_CONFIG, "ciNamespaces", ["hwlab-ci", "agentrun-ci"])
|
||||
active = []
|
||||
commands = []
|
||||
for namespace in namespaces:
|
||||
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun,job -n %s --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" && $2 != \"Complete\" && $2 != \"Failed\" {print}' | head -20" % shell_single_quote(namespace)], 15)
|
||||
commands.append({"namespace": namespace, "command": bounded(result)})
|
||||
for line in (result.get("stdout") or "").splitlines():
|
||||
if line.strip():
|
||||
active.append({"namespace": namespace, "line": line.strip()})
|
||||
return {"ok": True, "activeCount": len(active), "activePreview": active[:20], "commands": commands}
|
||||
|
||||
def compact_ci_activity(activity):
|
||||
return {
|
||||
"ok": activity.get("ok"),
|
||||
"activeCount": activity.get("activeCount"),
|
||||
"activePreview": activity.get("activePreview") or [],
|
||||
}
|
||||
|
||||
def compact_image_ref(ref):
|
||||
ref = str(ref or "")
|
||||
return ref if len(ref) <= 120 else ref[:117] + "..."
|
||||
|
||||
def k3s_cri_image_rows():
|
||||
images, image_cmd = k3s_crictl_json(["images"], 45)
|
||||
containers, container_cmd = k3s_crictl_json(["ps", "-a"], 30)
|
||||
if images is None:
|
||||
return None, {"ok": False, "reason": "crictl-images-failed", "command": bounded(image_cmd)}
|
||||
if containers is None:
|
||||
return None, {"ok": False, "reason": "crictl-ps-failed", "command": bounded(container_cmd)}
|
||||
used = set()
|
||||
for container in containers.get("containers") or []:
|
||||
for key in ["imageRef", "image", "imageId"]:
|
||||
value = container.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
used.add(value)
|
||||
image = container.get("image") or {}
|
||||
if isinstance(image, dict):
|
||||
for key in ["image", "annotations", "userSpecifiedImage"]:
|
||||
value = image.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
used.add(value)
|
||||
rows = []
|
||||
for image in images.get("images") or []:
|
||||
refs = []
|
||||
for key in ["repoTags", "repoDigests"]:
|
||||
value = image.get(key)
|
||||
if isinstance(value, list):
|
||||
refs.extend([str(item) for item in value if item])
|
||||
image_id = str(image.get("id") or "")
|
||||
pinned = bool(image.get("pinned"))
|
||||
size = safe_int(image.get("size_") or image.get("size") or 0)
|
||||
in_use = pinned or image_id in used or any(ref in used for ref in refs)
|
||||
rows.append({"id": image_id, "refs": refs, "sizeBytes": size, "inUse": in_use, "pinned": pinned})
|
||||
return rows, {"ok": True, "imageCommand": bounded(image_cmd), "containerCommand": bounded(container_cmd)}
|
||||
|
||||
def k3s_image_cache_candidate():
|
||||
if not config_bool(CONTAINERD_CONFIG, "enabled", False):
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:disabled",
|
||||
"kind": "k3s-cri-image-prune-disabled",
|
||||
"risk": "blocked",
|
||||
"description": "K3s CRI image prune is disabled in YAML",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.containerdImageCache.enabled" % PROVIDER_ID,
|
||||
}
|
||||
activity = ci_activity_snapshot_for_prune()
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:ci-active",
|
||||
"kind": "k3s-cri-image-prune-blocked",
|
||||
"risk": "blocked",
|
||||
"description": "K3s CRI image prune is blocked while CI workloads are active",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"ciActivity": compact_ci_activity(activity),
|
||||
}
|
||||
rows, meta = k3s_cri_image_rows()
|
||||
if rows is None:
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:unavailable",
|
||||
"kind": "k3s-cri-image-prune-unavailable",
|
||||
"risk": "blocked",
|
||||
"description": "K3s CRI image list is unavailable",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"diagnostic": meta,
|
||||
}
|
||||
unused = [row for row in rows if not row.get("inUse")]
|
||||
estimated = sum(safe_int(row.get("sizeBytes")) for row in unused)
|
||||
if estimated <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:unused",
|
||||
"kind": "k3s-cri-image-prune",
|
||||
"risk": "medium",
|
||||
"description": "Prune unused k3s CRI images through crictl rmi --prune; no containerd paths are deleted directly",
|
||||
"sizeBytes": estimated,
|
||||
"estimatedReclaimBytes": estimated,
|
||||
"imageCount": len(rows),
|
||||
"unusedImageCount": len(unused),
|
||||
"unusedPreview": [{"id": row.get("id"), "refs": [compact_image_ref(ref) for ref in (row.get("refs") or [])[:2]], "sizeBytes": row.get("sizeBytes")} for row in unused[:3]],
|
||||
"ciActivity": compact_ci_activity(activity),
|
||||
"action": {"command": k3s_crictl_base() + ["rmi", "--prune"], "mode": "cri-unused-images-only"},
|
||||
}
|
||||
|
||||
def execute_k3s_image_cache_prune():
|
||||
activity = ci_activity_snapshot_for_prune()
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
raise RuntimeError("refusing k3s image prune while CI workloads are active")
|
||||
before = du_size("/var/lib/rancher/k3s/agent/containerd", 45) or 0
|
||||
result = command(k3s_crictl_base() + ["rmi", "--prune"], 300)
|
||||
if result["exitCode"] != 0:
|
||||
raise RuntimeError((result["stderr"] or result["stdout"] or "crictl rmi --prune failed").strip())
|
||||
after = du_size("/var/lib/rancher/k3s/agent/containerd", 45) or 0
|
||||
return {"reclaimedBytes": max(0, before - after), "commandOutput": bounded(result), "ciActivity": compact_ci_activity(activity)}
|
||||
|
||||
def host_ctr_base(namespace=None):
|
||||
address = config_str(HOST_CONTAINERD_CONFIG, "address", "")
|
||||
args = ["ctr"]
|
||||
if address:
|
||||
args.extend(["--address", address])
|
||||
if namespace:
|
||||
args.extend(["-n", namespace])
|
||||
return args
|
||||
|
||||
def host_ctr(args, timeout=30, namespace=None):
|
||||
return command(host_ctr_base(namespace) + args, timeout)
|
||||
|
||||
def host_containerd_namespaces():
|
||||
configured = config_list(HOST_CONTAINERD_CONFIG, "namespaces", [])
|
||||
if configured:
|
||||
return configured, {"source": "yaml", "command": None}
|
||||
result = host_ctr(["namespaces", "list", "-q"], 20)
|
||||
if result["exitCode"] != 0:
|
||||
return [], {"source": "ctr", "command": bounded(result), "error": "ctr-namespaces-failed"}
|
||||
return [line.strip() for line in (result.get("stdout") or "").splitlines() if line.strip()], {"source": "ctr", "command": bounded(result)}
|
||||
|
||||
def host_containerd_activity():
|
||||
if not config_bool(HOST_CONTAINERD_CONFIG, "enabled", False):
|
||||
return {"ok": False, "reason": "host-containerd-cache-disabled", "activeCount": 0}
|
||||
root = config_str(HOST_CONTAINERD_CONFIG, "root", "")
|
||||
if not root or not os.path.isdir(root):
|
||||
return {"ok": False, "reason": "host-containerd-root-unavailable", "root": root, "activeCount": 0}
|
||||
namespaces, namespace_meta = host_containerd_namespaces()
|
||||
active = []
|
||||
commands = []
|
||||
for namespace in namespaces:
|
||||
task_result = host_ctr(["tasks", "list", "-q"], 20, namespace)
|
||||
container_result = host_ctr(["containers", "list", "-q"], 20, namespace)
|
||||
image_result = host_ctr(["images", "list", "-q"], 20, namespace)
|
||||
lease_result = host_ctr(["leases", "list", "-q"], 20, namespace)
|
||||
snapshot_result = host_ctr(["snapshots", "list"], 20, namespace)
|
||||
content_result = host_ctr(["content", "list"], 20, namespace)
|
||||
snapshot_lines = table_data_lines(snapshot_result.get("stdout") or "", "KEY")
|
||||
content_lines = table_data_lines(content_result.get("stdout") or "", "DIGEST")
|
||||
commands.append({
|
||||
"namespace": namespace,
|
||||
"tasks": bounded(task_result),
|
||||
"containers": bounded(container_result),
|
||||
"images": bounded(image_result),
|
||||
"leases": bounded(lease_result),
|
||||
"snapshots": bounded(snapshot_result),
|
||||
"content": bounded(content_result),
|
||||
})
|
||||
for kind, result in [("task", task_result), ("container", container_result), ("lease", lease_result)]:
|
||||
if result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": kind, "state": "unknown", "reason": "ctr-list-failed"})
|
||||
continue
|
||||
for line in (result.get("stdout") or "").splitlines():
|
||||
if line.strip():
|
||||
active.append({"namespace": namespace, "kind": kind, "name": line.strip()})
|
||||
if snapshot_result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": "snapshot", "state": "unknown", "reason": "ctr-list-failed"})
|
||||
for line in snapshot_lines:
|
||||
active.append({"namespace": namespace, "kind": "snapshot", "name": line.split()[0] if line.split() else line})
|
||||
if content_result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": "content", "state": "unknown", "reason": "ctr-list-failed"})
|
||||
for line in content_lines:
|
||||
active.append({"namespace": namespace, "kind": "content", "name": line.split()[0] if line.split() else line})
|
||||
return {
|
||||
"ok": True,
|
||||
"root": root,
|
||||
"namespaces": namespaces,
|
||||
"namespaceMeta": namespace_meta,
|
||||
"activeCount": len(active),
|
||||
"activePreview": active[:20],
|
||||
"commands": commands,
|
||||
}
|
||||
|
||||
def compact_host_containerd_activity(activity):
|
||||
return {
|
||||
"ok": activity.get("ok"),
|
||||
"reason": activity.get("reason"),
|
||||
"root": activity.get("root"),
|
||||
"namespaces": activity.get("namespaces"),
|
||||
"activeCount": activity.get("activeCount"),
|
||||
"activePreview": activity.get("activePreview") or [],
|
||||
}
|
||||
|
||||
def table_data_lines(stdout, header_prefix):
|
||||
lines = [line.strip() for line in str(stdout or "").splitlines() if line.strip()]
|
||||
return [line for line in lines if not line.startswith(header_prefix)]
|
||||
|
||||
def host_containerd_orphan_config():
|
||||
value = HOST_CONTAINERD_CONFIG.get("orphanCleanup") if isinstance(HOST_CONTAINERD_CONFIG, dict) else None
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
def direct_child_paths(root, predicate):
|
||||
if not root or not os.path.isdir(root) or os.path.islink(root):
|
||||
return []
|
||||
rows = []
|
||||
for name in sorted(os.listdir(root)):
|
||||
path = os.path.realpath(os.path.abspath(os.path.join(root, name)))
|
||||
if os.path.dirname(path) != os.path.realpath(os.path.abspath(root)):
|
||||
continue
|
||||
if not predicate(name, path):
|
||||
continue
|
||||
rows.append({"name": name, "path": path, "estimatedReclaimBytes": du_size(path, 10) or path_size(path)})
|
||||
return rows
|
||||
|
||||
def host_containerd_orphan_rows(activity):
|
||||
cfg = host_containerd_orphan_config()
|
||||
if not config_bool(cfg, "enabled", False):
|
||||
return [], {"ok": False, "reason": "host-containerd-orphan-cleanup-disabled"}
|
||||
if not activity.get("ok") or int(activity.get("activeCount") or 0) > 0:
|
||||
return [], {"ok": False, "reason": "host-containerd-metadata-not-empty", "activity": compact_host_containerd_activity(activity)}
|
||||
overlay_root = os.path.realpath(os.path.abspath(config_str(cfg, "overlaySnapshotsRoot", "")))
|
||||
content_root = os.path.realpath(os.path.abspath(config_str(cfg, "contentBlobRoot", "")))
|
||||
root = os.path.realpath(os.path.abspath(config_str(HOST_CONTAINERD_CONFIG, "root", "")))
|
||||
if not root or not overlay_root.startswith(root.rstrip("/") + "/") or not content_root.startswith(root.rstrip("/") + "/"):
|
||||
return [], {"ok": False, "reason": "host-containerd-orphan-root-outside-containerd-root", "root": root}
|
||||
open_roots = []
|
||||
for candidate_root in [overlay_root, content_root]:
|
||||
if os.path.exists(candidate_root) and path_has_open_fd(candidate_root):
|
||||
open_roots.append(candidate_root)
|
||||
if open_roots:
|
||||
return [], {"ok": False, "reason": "host-containerd-orphan-root-open-fd", "openRoots": open_roots}
|
||||
overlay_rows = direct_child_paths(overlay_root, lambda name, path: os.path.isdir(path) and not os.path.islink(path) and re.match(r"^[0-9]+$", name) is not None)
|
||||
content_rows = direct_child_paths(content_root, lambda name, path: os.path.isfile(path) and not os.path.islink(path) and re.match(r"^[0-9a-f]{64}$", name) is not None)
|
||||
safe_rows = []
|
||||
for kind, rows in [("overlay-snapshot-dir", overlay_rows), ("content-blob-file", content_rows)]:
|
||||
for row in rows:
|
||||
safe_rows.append({**row, "kind": kind})
|
||||
safe_rows.sort(key=lambda item: safe_int(item.get("estimatedReclaimBytes")), reverse=True)
|
||||
return safe_rows, {
|
||||
"ok": True,
|
||||
"root": root,
|
||||
"overlaySnapshotsRoot": overlay_root,
|
||||
"contentBlobRoot": content_root,
|
||||
"overlayCandidateCount": len(overlay_rows),
|
||||
"contentCandidateCount": len(content_rows),
|
||||
"protectedCount": 0,
|
||||
"protectedPreview": [],
|
||||
}
|
||||
|
||||
def host_containerd_orphan_candidate(activity):
|
||||
rows, meta = host_containerd_orphan_rows(activity)
|
||||
if not meta.get("ok"):
|
||||
return None
|
||||
limit = int(OPTIONS.get("limit") or 50)
|
||||
selected = rows[:limit]
|
||||
estimated = sum(safe_int(row.get("estimatedReclaimBytes")) for row in selected)
|
||||
if estimated <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "host-containerd-orphan-state:delete",
|
||||
"kind": "host-containerd-orphan-state-delete",
|
||||
"risk": "medium",
|
||||
"description": "Delete YAML-allowlisted host containerd orphan snapshot/content files only when ctr metadata has no tasks, containers, leases, images, snapshots or content",
|
||||
"path": meta.get("root"),
|
||||
"sizeBytes": estimated,
|
||||
"estimatedReclaimBytes": estimated,
|
||||
"orphanCount": len(rows),
|
||||
"selectedOrphanCount": len(selected),
|
||||
"overlayCandidateCount": meta.get("overlayCandidateCount"),
|
||||
"contentCandidateCount": meta.get("contentCandidateCount"),
|
||||
"protectedCount": meta.get("protectedCount"),
|
||||
"selectedPreview": [{"kind": row.get("kind"), "name": row.get("name"), "estimatedReclaimBytes": row.get("estimatedReclaimBytes")} for row in selected[:8]],
|
||||
"protectedPreview": meta.get("protectedPreview"),
|
||||
"action": {"op": "remove-yaml-allowlisted-host-containerd-orphans", "limit": limit},
|
||||
}
|
||||
|
||||
def host_containerd_cache_candidate():
|
||||
activity = host_containerd_activity()
|
||||
if not activity.get("ok"):
|
||||
return {
|
||||
"id": "host-containerd-cache:unavailable",
|
||||
"kind": "host-containerd-cache-unavailable",
|
||||
"risk": "blocked",
|
||||
"description": "Host containerd cache cleanup is disabled or unavailable by YAML",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"diagnostic": compact_host_containerd_activity(activity),
|
||||
}
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
return {
|
||||
"id": "host-containerd-cache:active",
|
||||
"kind": "host-containerd-cache-blocked",
|
||||
"risk": "blocked",
|
||||
"description": "Host containerd cache prune is blocked while host containerd tasks or containers exist",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"activity": compact_host_containerd_activity(activity),
|
||||
}
|
||||
orphan = host_containerd_orphan_candidate(activity)
|
||||
if orphan:
|
||||
return orphan
|
||||
root = activity.get("root") or ""
|
||||
size = du_size(root, 45) or 0
|
||||
if size <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "host-containerd-cache:prune-unused",
|
||||
"kind": "host-containerd-cache-prune",
|
||||
"risk": "medium",
|
||||
"description": "Prune host containerd images in YAML-selected namespaces only when no host containerd tasks or containers exist",
|
||||
"path": root,
|
||||
"sizeBytes": size,
|
||||
"estimatedReclaimBytes": size,
|
||||
"activity": compact_host_containerd_activity(activity),
|
||||
"action": {"command": "ctr images prune --all per namespace", "mode": "host-containerd-unused-images-only"},
|
||||
}
|
||||
|
||||
def execute_host_containerd_cache_prune():
|
||||
activity = host_containerd_activity()
|
||||
if not activity.get("ok"):
|
||||
raise RuntimeError("host containerd cache cleanup unavailable: %s" % activity.get("reason"))
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
raise RuntimeError("refusing host containerd prune while tasks or containers exist")
|
||||
root = activity.get("root") or ""
|
||||
before = du_size(root, 45) or 0
|
||||
results = []
|
||||
for namespace in activity.get("namespaces") or []:
|
||||
result = host_ctr(["images", "prune", "--all"], 300, namespace)
|
||||
results.append({"namespace": namespace, "imagesPrune": bounded(result)})
|
||||
if result["exitCode"] != 0:
|
||||
raise RuntimeError("host containerd image prune failed in namespace %s: %s" % (namespace, (result.get("stderr") or result.get("stdout") or "").strip()))
|
||||
after = du_size(root, 45) or 0
|
||||
return {
|
||||
"reclaimedBytes": max(0, before - after),
|
||||
"activity": compact_host_containerd_activity(activity),
|
||||
"commandResults": results[:8],
|
||||
}
|
||||
|
||||
def execute_host_containerd_orphan_cleanup():
|
||||
activity = host_containerd_activity()
|
||||
rows, meta = host_containerd_orphan_rows(activity)
|
||||
if not meta.get("ok"):
|
||||
raise RuntimeError("host containerd orphan cleanup unavailable: %s" % meta.get("reason"))
|
||||
for root_path in [meta.get("overlaySnapshotsRoot"), meta.get("contentBlobRoot")]:
|
||||
if root_path and os.path.exists(root_path) and path_has_open_fd(root_path):
|
||||
raise RuntimeError("refusing host containerd orphan cleanup with open fd/cwd under root: %s" % root_path)
|
||||
limit = int(OPTIONS.get("limit") or 50)
|
||||
selected = rows[:limit]
|
||||
reclaimed = 0
|
||||
deleted = []
|
||||
for row in selected:
|
||||
path = row.get("path")
|
||||
before = du_size(path, 10) or path_size(path)
|
||||
if row.get("kind") == "overlay-snapshot-dir":
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif row.get("kind") == "content-blob-file":
|
||||
os.unlink(path)
|
||||
else:
|
||||
raise RuntimeError("unsupported host containerd orphan kind: %s" % row.get("kind"))
|
||||
reclaimed += before
|
||||
deleted.append({"kind": row.get("kind"), "name": row.get("name"), "reclaimedBytes": before})
|
||||
return {
|
||||
"reclaimedBytes": reclaimed,
|
||||
"deletedOrphanCount": len(deleted),
|
||||
"deletedPreview": deleted[:12],
|
||||
"root": meta.get("root"),
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { classifySshTcpPoolFailure, type SshCaptureResult } from "./ssh";
|
||||
|
||||
export function remoteGcDegradedFailure(providerId: string, action: string, result: SshCaptureResult): Record<string, unknown> {
|
||||
const text = `${result.stderr}\n${result.stdout}`;
|
||||
const failureKind = classifySshTcpPoolFailure(text);
|
||||
const timeout = result.exitCode === 124 || text.includes("ssh-runtime-timeout");
|
||||
const providerOffline = text.includes(`provider is not online: ${providerId}`) || text.includes("provider is not online");
|
||||
const degradedReason = failureKind ?? (providerOffline ? "provider-offline" : timeout ? "ssh-runtime-timeout" : "remote-command-failed");
|
||||
return {
|
||||
ok: false,
|
||||
degraded: true,
|
||||
providerId,
|
||||
action: `gc remote ${action}`,
|
||||
degradedReason,
|
||||
transport: {
|
||||
sshTcpPoolFailureKind: failureKind,
|
||||
providerOffline,
|
||||
sshRuntimeTimeout: timeout,
|
||||
exitCode: result.exitCode,
|
||||
},
|
||||
safeCandidateCount: null,
|
||||
runAllowed: false,
|
||||
mutation: false,
|
||||
summary: failureKind !== null
|
||||
? `remote GC could not acquire a provider data channel: ${failureKind}`
|
||||
: providerOffline
|
||||
? `provider ${providerId} is offline from the controlled CLI transport view`
|
||||
: timeout
|
||||
? "remote GC did not complete before the SSH runtime timeout"
|
||||
: "remote GC command failed before producing a valid plan",
|
||||
next: {
|
||||
sshPool: `bun scripts/cli.ts debug ssh-pool ${providerId}`,
|
||||
fullHealth: "bun scripts/cli.ts debug health",
|
||||
smoke: `trans ${providerId} argv true`,
|
||||
retryPlan: `bun scripts/cli.ts gc remote ${providerId} plan --no-snapshot-save`,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
def registry_growth_snapshot():
|
||||
summary = {
|
||||
"path": REGISTRY_ROOT,
|
||||
"sizeBytes": du_size(REGISTRY_ROOT, 60) or 0,
|
||||
}
|
||||
summary["sizeHuman"] = fmt_bytes(summary["sizeBytes"])
|
||||
if OPTIONS.get("hwlabRegistry", False):
|
||||
plan = plan_registry_retention()
|
||||
retention = dict(plan.get("summary") or {})
|
||||
for key in ["registrySizeBytes", "estimatedReclaimBytes"]:
|
||||
if key in retention:
|
||||
retention[key.replace("Bytes", "Human")] = fmt_bytes(retention.get(key) or 0)
|
||||
summary["retentionPlan"] = retention
|
||||
else:
|
||||
summary["retentionPlan"] = {
|
||||
"skipped": True,
|
||||
"reason": "rerun snapshot with --include-hwlab-registry to compute tag/revision retention counters",
|
||||
}
|
||||
summary["cadence"] = {
|
||||
"dryRun": "daily or before/after every v0.2 CI/CD burst",
|
||||
"maintenanceRun": "weekly, or when root >=80%, or when registry growth exceeds the agreed daily threshold",
|
||||
"planCommand": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
|
||||
"snapshotCommand": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit 12" % PROVIDER_ID,
|
||||
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --include-hwlab-registry --target-use-percent 70 --limit 50" % PROVIDER_ID,
|
||||
"defaultRetention": {
|
||||
"keepPerRepo": int(OPTIONS.get("registryKeepPerRepo") or 20),
|
||||
"minAgeHours": float(OPTIONS.get("registryMinAgeHours") or 48),
|
||||
"protects": ["current workload refs", "digest closure", "protected tags", "recent tags", "newest N tags per repo"],
|
||||
},
|
||||
}
|
||||
return summary
|
||||
|
||||
def growth_watermark_policy(root_disk):
|
||||
use_percent = root_disk.get("usePercent") if isinstance(root_disk, dict) else None
|
||||
if use_percent is None:
|
||||
state = "unknown"
|
||||
action = "collect-snapshot"
|
||||
elif use_percent < 75:
|
||||
state = "healthy"
|
||||
action = "observe-trend"
|
||||
elif use_percent < 80:
|
||||
state = "watch"
|
||||
action = "run-dry-run-plan"
|
||||
elif use_percent < 85:
|
||||
state = "maintenance"
|
||||
action = "schedule-owner-aware-retention"
|
||||
else:
|
||||
state = "emergency"
|
||||
action = "restore-runtime-then-file-evidence"
|
||||
return {
|
||||
"state": state,
|
||||
"recommendedAction": action,
|
||||
"watermarks": [
|
||||
{"range": "<75%", "action": "trend only"},
|
||||
{"range": "75%-80%", "action": "run dry-run plan and identify source"},
|
||||
{"range": "80%-85%", "action": "small owner-aware retention run"},
|
||||
{"range": ">=85%", "action": "runtime recovery first, then root-cause growth source"},
|
||||
],
|
||||
"growthThresholdPolicy": "If bytes/day remains high for consecutive snapshots, act before 80%; exact threshold should be set from the first week of saved snapshots.",
|
||||
}
|
||||
|
||||
def snapshot_metric_map(snapshot):
|
||||
metrics = {}
|
||||
root = snapshot.get("rootDisk") or {}
|
||||
if isinstance(root, dict) and root.get("usedBytes") is not None:
|
||||
metrics["root.usedBytes"] = {"value": safe_int(root.get("usedBytes")), "unit": "bytes", "label": "root used bytes"}
|
||||
for item in snapshot.get("sources") or []:
|
||||
if not isinstance(item, dict) or item.get("sizeBytes") is None:
|
||||
continue
|
||||
key = "source.%s.sizeBytes" % item.get("id")
|
||||
metrics[key] = {"value": safe_int(item.get("sizeBytes")), "unit": "bytes", "label": item.get("label") or item.get("id")}
|
||||
storage = ((snapshot.get("ciStorage") or {}).get("byOwnerGroup") or {})
|
||||
if not storage:
|
||||
storage = ((snapshot.get("pvcAttribution") or {}).get("byOwnerGroup") or {})
|
||||
for owner, value in storage.items():
|
||||
metrics["ciStorage.%s.estimatedBytes" % owner] = {"value": safe_int((value or {}).get("estimatedBytes")), "unit": "bytes", "label": "CI storage %s" % owner}
|
||||
memory = snapshot.get("memoryPressure") or {}
|
||||
memory_summary = memory.get("summary") or {}
|
||||
if memory_summary.get("matchedRssBytes") is not None:
|
||||
metrics["memoryPressure.matchedRssBytes"] = {"value": safe_int(memory_summary.get("matchedRssBytes")), "unit": "bytes", "label": "matched observer/chrome RSS"}
|
||||
if memory_summary.get("observeStateBytes") is not None:
|
||||
metrics["memoryPressure.observeStateBytes"] = {"value": safe_int(memory_summary.get("observeStateBytes")), "unit": "bytes", "label": "web observe state bytes"}
|
||||
for key in ["matchedProcessCount", "activeObserverSignals", "staleObserverSignals"]:
|
||||
if memory_summary.get(key) is not None:
|
||||
metrics["memoryPressure.%s" % key] = {"value": safe_int(memory_summary.get(key)), "unit": "count", "label": "memory pressure %s" % key}
|
||||
registry = snapshot.get("registry") or {}
|
||||
retention = registry.get("retentionPlan") or {}
|
||||
for key in ["totalTags", "totalRevisions", "deleteTags", "deleteRevisions", "estimatedReclaimBytes"]:
|
||||
if key in retention and retention.get(key) is not None:
|
||||
unit = "bytes" if key.endswith("Bytes") else "count"
|
||||
metrics["registry.%s" % key] = {"value": safe_int(retention.get(key)), "unit": unit, "label": "registry %s" % key}
|
||||
return metrics
|
||||
|
||||
def delta_metric_rows(before, after):
|
||||
before_metrics = snapshot_metric_map(before)
|
||||
after_metrics = snapshot_metric_map(after)
|
||||
before_ts = iso_to_epoch(before.get("observedAt"))
|
||||
after_ts = iso_to_epoch(after.get("observedAt"))
|
||||
seconds = (after_ts - before_ts) if before_ts is not None and after_ts is not None else None
|
||||
rows = []
|
||||
for key in sorted(set(before_metrics.keys()) | set(after_metrics.keys())):
|
||||
old = before_metrics.get(key, {"value": 0, "unit": (after_metrics.get(key) or {}).get("unit"), "label": key})
|
||||
new = after_metrics.get(key, {"value": 0, "unit": old.get("unit"), "label": old.get("label")})
|
||||
delta = safe_int(new.get("value")) - safe_int(old.get("value"))
|
||||
row = {
|
||||
"key": key,
|
||||
"label": new.get("label") or old.get("label") or key,
|
||||
"unit": new.get("unit") or old.get("unit"),
|
||||
"before": old.get("value"),
|
||||
"after": new.get("value"),
|
||||
"delta": delta,
|
||||
}
|
||||
if row["unit"] == "bytes":
|
||||
row["beforeHuman"] = fmt_bytes(row["before"] or 0)
|
||||
row["afterHuman"] = fmt_bytes(row["after"] or 0)
|
||||
row["deltaHuman"] = ("-" if delta < 0 else "") + fmt_bytes(abs(delta))
|
||||
if seconds and seconds > 0:
|
||||
per_day = int(delta * 86400 / seconds)
|
||||
row["perDayBytes"] = per_day
|
||||
row["perDayHuman"] = ("-" if per_day < 0 else "") + fmt_bytes(abs(per_day)) + "/day"
|
||||
rows.append(row)
|
||||
rows.sort(key=lambda item: safe_int(item.get("delta")), reverse=True)
|
||||
return {"durationSeconds": seconds, "metrics": rows}
|
||||
|
||||
def growth_trend_payload(points):
|
||||
points = [point for point in points if isinstance(point, dict)]
|
||||
if len(points) < 2:
|
||||
return {
|
||||
"pointCount": len(points),
|
||||
"state": "insufficient-history",
|
||||
"message": "Run snapshot at least twice to compute deltas.",
|
||||
}
|
||||
latest_delta = delta_metric_rows(points[-2], points[-1])
|
||||
window_delta = delta_metric_rows(points[0], points[-1])
|
||||
def rate_warning(delta):
|
||||
seconds = delta.get("durationSeconds")
|
||||
if seconds is not None and seconds < 3600:
|
||||
return {
|
||||
"code": "short-window-rate-noisy",
|
||||
"message": "Per-day rates from windows shorter than 1 hour are directional only; use daily snapshots for governance decisions.",
|
||||
"durationSeconds": seconds,
|
||||
}
|
||||
return None
|
||||
return {
|
||||
"pointCount": len(points),
|
||||
"oldestAt": points[0].get("observedAt"),
|
||||
"latestAt": points[-1].get("observedAt"),
|
||||
"latestDelta": {
|
||||
"durationSeconds": latest_delta.get("durationSeconds"),
|
||||
"rateWarning": rate_warning(latest_delta),
|
||||
"topGrowingBytes": [row for row in latest_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
|
||||
"topShrinkingBytes": [row for row in reversed(latest_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
|
||||
"registryCounters": [row for row in latest_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
|
||||
},
|
||||
"windowDelta": {
|
||||
"durationSeconds": window_delta.get("durationSeconds"),
|
||||
"rateWarning": rate_warning(window_delta),
|
||||
"topGrowingBytes": [row for row in window_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
|
||||
"topShrinkingBytes": [row for row in reversed(window_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
|
||||
"registryCounters": [row for row in window_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
|
||||
},
|
||||
}
|
||||
|
||||
def compact_metric_rows(rows, limit=3):
|
||||
compact = []
|
||||
for row in (rows or [])[:limit]:
|
||||
compact.append({
|
||||
"key": row.get("key"),
|
||||
"label": row.get("label"),
|
||||
"unit": row.get("unit"),
|
||||
"delta": row.get("delta"),
|
||||
"deltaHuman": row.get("deltaHuman"),
|
||||
"perDayHuman": row.get("perDayHuman"),
|
||||
})
|
||||
return compact
|
||||
|
||||
def compact_trend_payload(payload):
|
||||
if payload.get("state") == "insufficient-history":
|
||||
return payload
|
||||
latest = payload.get("latestDelta") or {}
|
||||
window = payload.get("windowDelta") or {}
|
||||
return {
|
||||
"pointCount": payload.get("pointCount"),
|
||||
"oldestAt": payload.get("oldestAt"),
|
||||
"latestAt": payload.get("latestAt"),
|
||||
"latestDelta": {
|
||||
"durationSeconds": latest.get("durationSeconds"),
|
||||
"rateWarning": latest.get("rateWarning"),
|
||||
"topGrowingBytes": compact_metric_rows(latest.get("topGrowingBytes") or [], 1),
|
||||
"topShrinkingBytes": compact_metric_rows(latest.get("topShrinkingBytes") or [], 1),
|
||||
"registryCounters": compact_metric_rows(latest.get("registryCounters") or [], 1),
|
||||
},
|
||||
"windowDelta": {
|
||||
"durationSeconds": window.get("durationSeconds"),
|
||||
"rateWarning": window.get("rateWarning"),
|
||||
"topGrowingBytes": compact_metric_rows(window.get("topGrowingBytes") or [], 1),
|
||||
"topShrinkingBytes": compact_metric_rows(window.get("topShrinkingBytes") or [], 1),
|
||||
"registryCounters": compact_metric_rows(window.get("registryCounters") or [], 1),
|
||||
},
|
||||
"fullDisclosure": "rerun trend --full for all metric rows",
|
||||
}
|
||||
|
||||
def compact_growth_point(item):
|
||||
registry = item.get("registry") or {}
|
||||
retention = registry.get("retentionPlan") or {}
|
||||
ci_storage = item.get("ciStorage") or {}
|
||||
containerd = item.get("containerd") or {}
|
||||
memory = item.get("memoryPressure") or {}
|
||||
memory_summary = memory.get("summary") or {}
|
||||
observe = (memory.get("webObserve") or {})
|
||||
return {
|
||||
"observedAt": item.get("observedAt"),
|
||||
"rootDisk": item.get("rootDisk"),
|
||||
"sourceCount": len(item.get("sources") or []),
|
||||
"registry": {
|
||||
"sizeBytes": registry.get("sizeBytes"),
|
||||
"sizeHuman": registry.get("sizeHuman"),
|
||||
"totalTags": retention.get("totalTags"),
|
||||
"totalRevisions": retention.get("totalRevisions"),
|
||||
"deleteTags": retention.get("deleteTags"),
|
||||
"deleteRevisions": retention.get("deleteRevisions"),
|
||||
"estimatedReclaimBytes": retention.get("estimatedReclaimBytes"),
|
||||
"estimatedReclaimHuman": retention.get("estimatedReclaimHuman"),
|
||||
},
|
||||
"ciStorage": {
|
||||
"pvcCount": ci_storage.get("pvcCount"),
|
||||
"estimatedBytes": ci_storage.get("estimatedBytes"),
|
||||
"estimatedHuman": ci_storage.get("estimatedHuman"),
|
||||
"byOwnerGroup": ci_storage.get("byOwnerGroup"),
|
||||
},
|
||||
"containerd": {
|
||||
"state": containerd.get("state"),
|
||||
"cleanupSupported": containerd.get("cleanupSupported"),
|
||||
},
|
||||
"memoryPressure": {
|
||||
"matchedProcessCount": memory_summary.get("matchedProcessCount"),
|
||||
"matchedRssBytes": memory_summary.get("matchedRssBytes"),
|
||||
"matchedRssHuman": memory_summary.get("matchedRssHuman"),
|
||||
"activeObserverSignals": memory_summary.get("activeObserverSignals"),
|
||||
"staleObserverSignals": memory_summary.get("staleObserverSignals"),
|
||||
"observeStateBytes": memory_summary.get("observeStateBytes"),
|
||||
"observeStateHuman": memory_summary.get("observeStateHuman"),
|
||||
"webObserveRootCount": observe.get("rootCount"),
|
||||
},
|
||||
}
|
||||
|
||||
def collect_growth_snapshot(observed_at, preflight):
|
||||
root_disk = df_snapshot()
|
||||
sources = disk_source_snapshot()
|
||||
ci_storage = ci_storage_snapshot()
|
||||
memory_pressure = collect_memory_pressure()
|
||||
compact_pvc = compact_pvc_attribution(ci_storage)
|
||||
if bool(OPTIONS.get("full")):
|
||||
public_pvc = ci_storage
|
||||
public_memory = memory_pressure
|
||||
else:
|
||||
public_pvc = compact_ci_storage_summary(ci_storage)
|
||||
public_memory = compact_memory_summary(memory_pressure)
|
||||
registry = registry_growth_snapshot()
|
||||
containerd = containerd_breakdown_snapshot()
|
||||
commands = {
|
||||
"snapshot": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"registryPlan": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
|
||||
"hwlabCiRetention": ((ci_storage.get("handoff") or {}).get("hwlab") or {}).get("dryRun"),
|
||||
"agentrunRetention": ((ci_storage.get("handoff") or {}).get("agentrun") or {}).get("dryRun"),
|
||||
"remotePolicy": "bun scripts/cli.ts gc remote %s policy plan" % PROVIDER_ID,
|
||||
}
|
||||
if not bool(OPTIONS.get("full")):
|
||||
commands = {
|
||||
"trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"status": "bun scripts/cli.ts gc remote %s status --limit %s" % (PROVIDER_ID, int(OPTIONS.get("limit") or 50)),
|
||||
"full": "bun scripts/cli.ts gc remote %s snapshot --full --no-save" % PROVIDER_ID,
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "gc remote snapshot",
|
||||
"providerId": PROVIDER_ID,
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"diagnosticStateMutation": bool(OPTIONS.get("saveSnapshot", True)),
|
||||
"observedAt": observed_at,
|
||||
"rootDisk": root_disk,
|
||||
"clusterPreflight": preflight,
|
||||
"sources": sources,
|
||||
"registry": registry,
|
||||
"pvcAttribution": public_pvc,
|
||||
"memoryPressure": public_memory,
|
||||
"containerd": containerd,
|
||||
"policy": growth_watermark_policy(root_disk or {}),
|
||||
"commands": commands,
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def command(args, timeout=60):
|
||||
started = time.time()
|
||||
try:
|
||||
result = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
||||
return {
|
||||
"exitCode": result.returncode,
|
||||
"elapsedMs": int((time.time() - started) * 1000),
|
||||
"stdoutTail": result.stdout[-1200:],
|
||||
"stderrTail": result.stderr[-1200:],
|
||||
}
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return {
|
||||
"exitCode": 124,
|
||||
"elapsedMs": int((time.time() - started) * 1000),
|
||||
"stdoutTail": (exc.stdout or "")[-1200:] if isinstance(exc.stdout, str) else "",
|
||||
"stderrTail": (exc.stderr or "")[-1200:] if isinstance(exc.stderr, str) else "",
|
||||
"timedOut": True,
|
||||
}
|
||||
|
||||
|
||||
def load_config(path):
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def safe_int(value, default=0):
|
||||
try:
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def du_size(path):
|
||||
result = command(["du", "-sb", path], 15)
|
||||
if result["exitCode"] != 0:
|
||||
return 0
|
||||
try:
|
||||
return int((result["stdoutTail"] or "0").split()[0])
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def path_size(path):
|
||||
try:
|
||||
stat = os.lstat(path)
|
||||
except OSError:
|
||||
return 0
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
return du_size(path)
|
||||
return int(getattr(stat, "st_blocks", 0)) * 512
|
||||
|
||||
|
||||
def path_has_open_fd(path):
|
||||
resolved = os.path.realpath(path)
|
||||
prefix = resolved.rstrip("/") + "/"
|
||||
try:
|
||||
pids = [name for name in os.listdir("/proc") if name.isdigit()]
|
||||
except OSError:
|
||||
return True
|
||||
for pid in pids:
|
||||
base = os.path.join("/proc", pid)
|
||||
for name in ["cwd", "root"]:
|
||||
try:
|
||||
target = os.path.realpath(os.readlink(os.path.join(base, name)))
|
||||
except OSError:
|
||||
continue
|
||||
if target == resolved or target.startswith(prefix):
|
||||
return True
|
||||
fd_dir = os.path.join(base, "fd")
|
||||
try:
|
||||
fds = os.listdir(fd_dir)
|
||||
except OSError:
|
||||
continue
|
||||
for fd in fds:
|
||||
try:
|
||||
target = os.path.realpath(os.readlink(os.path.join(fd_dir, fd)))
|
||||
except OSError:
|
||||
continue
|
||||
if target == resolved or target.startswith(prefix):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def run_json(args, timeout=45):
|
||||
started = time.time()
|
||||
try:
|
||||
process = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return None, {
|
||||
"exitCode": 124,
|
||||
"elapsedMs": int((time.time() - started) * 1000),
|
||||
"stdoutTail": (exc.stdout or "")[-1200:] if isinstance(exc.stdout, str) else "",
|
||||
"stderrTail": (exc.stderr or "")[-1200:] if isinstance(exc.stderr, str) else "",
|
||||
"timedOut": True,
|
||||
}
|
||||
result = {
|
||||
"exitCode": process.returncode,
|
||||
"elapsedMs": int((time.time() - started) * 1000),
|
||||
"stdoutTail": process.stdout[-1200:],
|
||||
"stderrTail": process.stderr[-1200:],
|
||||
}
|
||||
if process.returncode != 0:
|
||||
return None, result
|
||||
try:
|
||||
return json.loads(process.stdout or "{}"), result
|
||||
except Exception:
|
||||
return None, result
|
||||
|
||||
|
||||
def write_state(config, payload):
|
||||
state_path = config.get("statePath")
|
||||
if not state_path:
|
||||
return
|
||||
os.makedirs(os.path.dirname(state_path), mode=0o755, exist_ok=True)
|
||||
tmp = "%s.tmp.%d" % (state_path, os.getpid())
|
||||
with open(tmp, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, sort_keys=True)
|
||||
handle.write("\n")
|
||||
os.replace(tmp, state_path)
|
||||
|
||||
|
||||
def run_journal(config):
|
||||
target = safe_int(config.get("journalTargetBytes"), 268435456)
|
||||
result = command(["journalctl", "--vacuum-size=%d" % target], 30)
|
||||
return {"id": "journal", "ok": result["exitCode"] == 0, "command": result}
|
||||
|
||||
|
||||
def run_apt(config):
|
||||
if not config.get("includeAptCache"):
|
||||
return {"id": "apt-cache", "ok": True, "skipped": True}
|
||||
before = du_size("/var/cache/apt/archives")
|
||||
result = command(["apt-get", "clean"], 30)
|
||||
after = du_size("/var/cache/apt/archives")
|
||||
return {"id": "apt-cache", "ok": result["exitCode"] == 0, "reclaimedBytes": max(0, before - after), "command": result}
|
||||
|
||||
|
||||
def run_tmp(config):
|
||||
prefixes = list(config.get("tmpPrefixAllowlist") or [])
|
||||
protected = set(config.get("tmpExactProtect") or [])
|
||||
cutoff = time.time() - float(config.get("tmpMinAgeHours") or 24) * 3600.0
|
||||
deleted = 0
|
||||
reclaimed = 0
|
||||
if not os.path.isdir("/tmp"):
|
||||
return {"id": "tmp-allowlist", "ok": True, "deletedCount": 0, "reclaimedBytes": 0}
|
||||
for name in os.listdir("/tmp"):
|
||||
path = os.path.join("/tmp", name)
|
||||
if path in protected or not any(name.startswith(prefix) for prefix in prefixes):
|
||||
continue
|
||||
try:
|
||||
stat = os.lstat(path)
|
||||
except OSError:
|
||||
continue
|
||||
if stat.st_mtime >= cutoff or path_has_open_fd(path):
|
||||
continue
|
||||
before = path_size(path)
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
else:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
deleted += 1
|
||||
reclaimed += before
|
||||
return {"id": "tmp-allowlist", "ok": True, "deletedCount": deleted, "reclaimedBytes": reclaimed}
|
||||
|
||||
|
||||
def run_tool_caches(config):
|
||||
paths = list(config.get("toolCachePaths") or [])
|
||||
reclaimed = 0
|
||||
deleted = 0
|
||||
for path in paths:
|
||||
resolved = os.path.abspath(path)
|
||||
if resolved != path or os.path.islink(resolved) or resolved in ["/", "/root", "/root/.npm", "/root/.bun"]:
|
||||
continue
|
||||
before = path_size(resolved)
|
||||
if os.path.isdir(resolved):
|
||||
shutil.rmtree(resolved, ignore_errors=True)
|
||||
elif os.path.exists(resolved):
|
||||
try:
|
||||
os.unlink(resolved)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
continue
|
||||
deleted += 1
|
||||
reclaimed += before
|
||||
return {"id": "tool-caches", "ok": True, "deletedCount": deleted, "reclaimedBytes": reclaimed}
|
||||
|
||||
|
||||
def pid_alive(pid):
|
||||
try:
|
||||
value = int(pid)
|
||||
except Exception:
|
||||
return False
|
||||
return value > 0 and os.path.exists("/proc/%d" % value)
|
||||
|
||||
|
||||
def read_json_file(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def run_web_observe(config):
|
||||
cfg = config.get("webObserve") or {}
|
||||
if not cfg.get("enabled"):
|
||||
return {"id": "web-observe-artifacts", "ok": True, "skipped": True}
|
||||
roots = [os.path.abspath(item) for item in (cfg.get("observeStateRoots") or cfg.get("webObserveRoots") or []) if isinstance(item, str) and item.startswith("/")]
|
||||
stale_hours = float(cfg.get("staleRunMaxAgeHours") or 6)
|
||||
cutoff = time.time() - stale_hours * 3600.0
|
||||
deleted = 0
|
||||
reclaimed = 0
|
||||
protected = 0
|
||||
for root in roots:
|
||||
if not os.path.isdir(root):
|
||||
continue
|
||||
for name in os.listdir(root):
|
||||
path = os.path.abspath(os.path.join(root, name))
|
||||
if os.path.dirname(path) != root or os.path.islink(path) or not os.path.isdir(path):
|
||||
continue
|
||||
manifest = read_json_file(os.path.join(path, "manifest.json"))
|
||||
heartbeat = read_json_file(os.path.join(path, "heartbeat.json"))
|
||||
pid = manifest.get("pid") or heartbeat.get("pid") or read_json_file(os.path.join(path, "pid")).get("pid")
|
||||
try:
|
||||
stat = os.lstat(path)
|
||||
except OSError:
|
||||
continue
|
||||
completed = bool(manifest.get("completedAt") or heartbeat.get("completedAt") or manifest.get("status") in ["completed", "failed", "blocked"])
|
||||
stale = completed or stat.st_mtime < cutoff
|
||||
if pid_alive(pid) or not stale or path_has_open_fd(path):
|
||||
protected += 1
|
||||
continue
|
||||
before = path_size(path)
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
deleted += 1
|
||||
reclaimed += before
|
||||
return {"id": "web-observe-artifacts", "ok": True, "deletedCount": deleted, "protectedCount": protected, "reclaimedBytes": reclaimed}
|
||||
|
||||
|
||||
def active_claims(namespace):
|
||||
pods, _ = run_json(["kubectl", "-n", namespace, "get", "pod", "-o", "json"], 30)
|
||||
claims = {}
|
||||
for pod in (pods or {}).get("items") or []:
|
||||
if ((pod.get("status") or {}).get("phase")) in ["Succeeded", "Failed"]:
|
||||
continue
|
||||
pod_name = ((pod.get("metadata") or {}).get("name")) or ""
|
||||
for volume in ((pod.get("spec") or {}).get("volumes") or []):
|
||||
claim = (volume.get("persistentVolumeClaim") or {}).get("claimName")
|
||||
if claim:
|
||||
claims.setdefault(claim, []).append(pod_name)
|
||||
return claims
|
||||
|
||||
|
||||
def run_session_pvcs(config):
|
||||
cfg = config.get("agentrunSessionPvcs") or {}
|
||||
if not cfg.get("enabled"):
|
||||
return {"id": "agentrun-session-pvcs", "ok": True, "skipped": True}
|
||||
namespace = cfg.get("namespace")
|
||||
prefixes = list(cfg.get("prefixes") or [])
|
||||
limit = max(1, min(safe_int(cfg.get("maxDeletePerRun"), 100), 1000))
|
||||
if not namespace or not prefixes:
|
||||
return {"id": "agentrun-session-pvcs", "ok": False, "error": "namespace-or-prefixes-missing"}
|
||||
pv_data, pv_cmd = run_json(["kubectl", "get", "pv", "-o", "json"], 30)
|
||||
pvc_data, pvc_cmd = run_json(["kubectl", "-n", namespace, "get", "pvc", "-o", "json"], 30)
|
||||
if pv_data is None or pvc_data is None:
|
||||
return {"id": "agentrun-session-pvcs", "ok": False, "pvCommand": pv_cmd, "pvcCommand": pvc_cmd}
|
||||
pvs = {((pv.get("metadata") or {}).get("name")): pv for pv in pv_data.get("items") or []}
|
||||
active = active_claims(namespace)
|
||||
selected = []
|
||||
protected = 0
|
||||
for pvc in pvc_data.get("items") or []:
|
||||
name = ((pvc.get("metadata") or {}).get("name")) or ""
|
||||
if not any(name.startswith(prefix) for prefix in prefixes):
|
||||
continue
|
||||
pv = pvs.get((pvc.get("spec") or {}).get("volumeName")) or {}
|
||||
storage_class = (pvc.get("spec") or {}).get("storageClassName") or (pv.get("spec") or {}).get("storageClassName")
|
||||
reclaim_policy = (pv.get("spec") or {}).get("persistentVolumeReclaimPolicy")
|
||||
if active.get(name) or storage_class != "local-path" or reclaim_policy != "Delete":
|
||||
protected += 1
|
||||
continue
|
||||
selected.append(name)
|
||||
selected = selected[:limit]
|
||||
if selected:
|
||||
result = command(["kubectl", "-n", namespace, "delete", "pvc", "--wait=false"] + selected, 45)
|
||||
ok = result["exitCode"] == 0
|
||||
else:
|
||||
result = None
|
||||
ok = True
|
||||
return {"id": "agentrun-session-pvcs", "ok": ok, "deletedPvcCount": len(selected), "protectedCount": protected, "command": result}
|
||||
|
||||
|
||||
def ci_active(config):
|
||||
namespaces = list((config.get("k3sImageCache") or {}).get("ciNamespaces") or [])
|
||||
active = []
|
||||
for namespace in namespaces:
|
||||
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun,job -n %s --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" && $2 != \"Complete\" && $2 != \"Failed\" {print}' | head -20" % namespace], 15)
|
||||
for line in (result.get("stdoutTail") or "").splitlines():
|
||||
if line.strip():
|
||||
active.append({"namespace": namespace, "line": line.strip()})
|
||||
return active
|
||||
|
||||
|
||||
def run_k3s_images(config):
|
||||
cfg = config.get("k3sImageCache") or {}
|
||||
if not cfg.get("enabled"):
|
||||
return {"id": "k3s-image-cache", "ok": True, "skipped": True}
|
||||
active = ci_active(config)
|
||||
if active:
|
||||
return {"id": "k3s-image-cache", "ok": True, "skipped": True, "reason": "ci-active", "activePreview": active[:5]}
|
||||
endpoint = cfg.get("runtimeEndpoint") or "unix:///run/k3s/containerd/containerd.sock"
|
||||
before = du_size("/var/lib/rancher/k3s/agent/containerd")
|
||||
result = command(["crictl", "--runtime-endpoint", endpoint, "rmi", "--prune"], 300)
|
||||
after = du_size("/var/lib/rancher/k3s/agent/containerd")
|
||||
return {"id": "k3s-image-cache", "ok": result["exitCode"] == 0, "reclaimedBytes": max(0, before - after), "command": result}
|
||||
|
||||
|
||||
def table_data_lines(stdout, header_prefix):
|
||||
return [line.strip() for line in str(stdout or "").splitlines() if line.strip() and not line.startswith(header_prefix)]
|
||||
|
||||
|
||||
def host_containerd_empty(config):
|
||||
cfg = config.get("hostContainerdCache") or {}
|
||||
address = cfg.get("address") or "/run/containerd/containerd.sock"
|
||||
namespaces = cfg.get("namespaces") or ["default"]
|
||||
active = []
|
||||
for namespace in namespaces:
|
||||
base = ["ctr", "--address", address, "-n", namespace]
|
||||
for kind, args, header in [
|
||||
("image", ["images", "list", "-q"], ""),
|
||||
("container", ["containers", "list", "-q"], ""),
|
||||
("task", ["tasks", "list", "-q"], ""),
|
||||
("lease", ["leases", "list", "-q"], ""),
|
||||
("snapshot", ["snapshots", "list"], "KEY"),
|
||||
("content", ["content", "list"], "DIGEST"),
|
||||
]:
|
||||
result = command(base + args, 20)
|
||||
if result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": kind, "state": "unknown"})
|
||||
continue
|
||||
lines = table_data_lines(result.get("stdoutTail") or "", header) if header else [line for line in (result.get("stdoutTail") or "").splitlines() if line.strip()]
|
||||
for line in lines:
|
||||
active.append({"namespace": namespace, "kind": kind, "name": line.split()[0] if line.split() else line})
|
||||
return active
|
||||
|
||||
|
||||
def direct_child_rows(root, predicate):
|
||||
rows = []
|
||||
if not root or not os.path.isdir(root) or os.path.islink(root):
|
||||
return rows
|
||||
real_root = os.path.realpath(os.path.abspath(root))
|
||||
for name in sorted(os.listdir(real_root)):
|
||||
path = os.path.realpath(os.path.abspath(os.path.join(real_root, name)))
|
||||
if os.path.dirname(path) != real_root or not predicate(name, path):
|
||||
continue
|
||||
rows.append({"name": name, "path": path, "sizeBytes": path_size(path)})
|
||||
rows.sort(key=lambda item: item["sizeBytes"], reverse=True)
|
||||
return rows
|
||||
|
||||
|
||||
def run_host_containerd(config):
|
||||
cfg = config.get("hostContainerdCache") or {}
|
||||
if not cfg.get("enabled"):
|
||||
return {"id": "host-containerd-orphans", "ok": True, "skipped": True}
|
||||
active = host_containerd_empty(config)
|
||||
if active:
|
||||
return {"id": "host-containerd-orphans", "ok": True, "skipped": True, "reason": "metadata-not-empty", "activePreview": active[:8]}
|
||||
orphan = cfg.get("orphanCleanup") or {}
|
||||
if not orphan.get("enabled"):
|
||||
return {"id": "host-containerd-orphans", "ok": True, "skipped": True, "reason": "orphan-cleanup-disabled"}
|
||||
overlay_root = os.path.realpath(os.path.abspath(orphan.get("overlaySnapshotsRoot") or ""))
|
||||
content_root = os.path.realpath(os.path.abspath(orphan.get("contentBlobRoot") or ""))
|
||||
root = os.path.realpath(os.path.abspath(cfg.get("root") or ""))
|
||||
if not root or not overlay_root.startswith(root + "/") or not content_root.startswith(root + "/"):
|
||||
return {"id": "host-containerd-orphans", "ok": False, "error": "orphan-root-outside-containerd-root"}
|
||||
for root_path in [overlay_root, content_root]:
|
||||
if os.path.exists(root_path) and path_has_open_fd(root_path):
|
||||
return {"id": "host-containerd-orphans", "ok": True, "skipped": True, "reason": "open-fd", "root": root_path}
|
||||
rows = direct_child_rows(overlay_root, lambda name, path: os.path.isdir(path) and not os.path.islink(path) and re.match(r"^[0-9]+$", name) is not None)
|
||||
rows += direct_child_rows(content_root, lambda name, path: os.path.isfile(path) and not os.path.islink(path) and re.match(r"^[0-9a-f]{64}$", name) is not None)
|
||||
rows.sort(key=lambda item: item["sizeBytes"], reverse=True)
|
||||
limit = max(1, min(safe_int(orphan.get("maxDeletePerRun"), 100), 1000))
|
||||
reclaimed = 0
|
||||
deleted = 0
|
||||
for row in rows[:limit]:
|
||||
before = path_size(row["path"])
|
||||
if os.path.isdir(row["path"]):
|
||||
shutil.rmtree(row["path"], ignore_errors=True)
|
||||
else:
|
||||
try:
|
||||
os.unlink(row["path"])
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
reclaimed += before
|
||||
deleted += 1
|
||||
return {"id": "host-containerd-orphans", "ok": True, "deletedCount": deleted, "candidateCount": len(rows), "reclaimedBytes": reclaimed}
|
||||
|
||||
|
||||
def run_local_path_orphans(config):
|
||||
cfg = config.get("localPathStorage") or {}
|
||||
if not cfg.get("enabled"):
|
||||
return {"id": "local-path-orphans", "ok": True, "skipped": True}
|
||||
root = os.path.realpath(os.path.abspath(cfg.get("root") or ""))
|
||||
prefixes = list(cfg.get("orphanDirPrefixes") or [])
|
||||
if not root or not prefixes or not os.path.isdir(root):
|
||||
return {"id": "local-path-orphans", "ok": False, "error": "root-or-prefix-missing"}
|
||||
pv_data, pv_cmd = run_json(["kubectl", "get", "pv", "-o", "json"], 30)
|
||||
if pv_data is None:
|
||||
return {"id": "local-path-orphans", "ok": False, "pvCommand": pv_cmd}
|
||||
referenced = set()
|
||||
for pv in pv_data.get("items") or []:
|
||||
spec = pv.get("spec") or {}
|
||||
path = ((spec.get("hostPath") or {}).get("path")) or ((spec.get("local") or {}).get("path"))
|
||||
if path:
|
||||
referenced.add(os.path.realpath(os.path.abspath(path)))
|
||||
rows = []
|
||||
for name in os.listdir(root):
|
||||
path = os.path.realpath(os.path.abspath(os.path.join(root, name)))
|
||||
if os.path.dirname(path) != root or not any(name.startswith(prefix) for prefix in prefixes) or path in referenced or path_has_open_fd(path):
|
||||
continue
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
rows.append({"name": name, "path": path, "sizeBytes": path_size(path)})
|
||||
rows.sort(key=lambda item: item["sizeBytes"], reverse=True)
|
||||
limit = max(1, min(safe_int(cfg.get("maxDeletePerRun"), 100), 1000))
|
||||
reclaimed = 0
|
||||
deleted = 0
|
||||
for row in rows[:limit]:
|
||||
before = path_size(row["path"])
|
||||
shutil.rmtree(row["path"], ignore_errors=True)
|
||||
reclaimed += before
|
||||
deleted += 1
|
||||
return {"id": "local-path-orphans", "ok": True, "deletedCount": deleted, "candidateCount": len(rows), "reclaimedBytes": reclaimed}
|
||||
|
||||
|
||||
def main():
|
||||
config_path = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
config = load_config(config_path)
|
||||
results = []
|
||||
for fn in [run_journal, run_apt, run_tmp, run_tool_caches, run_web_observe, run_session_pvcs, run_k3s_images, run_host_containerd, run_local_path_orphans]:
|
||||
try:
|
||||
results.append(fn(config))
|
||||
except Exception as exc:
|
||||
results.append({"id": getattr(fn, "__name__", "stage"), "ok": False, "error": str(exc)})
|
||||
payload = {
|
||||
"ok": all(item.get("ok") for item in results),
|
||||
"observedAt": now_iso(),
|
||||
"providerId": config.get("providerId"),
|
||||
"unitName": config.get("unitName"),
|
||||
"resultCount": len(results),
|
||||
"failedCount": len([item for item in results if not item.get("ok")]),
|
||||
"reclaimedBytes": sum(safe_int(item.get("reclaimedBytes")) for item in results),
|
||||
"results": results,
|
||||
}
|
||||
write_state(config, payload)
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
return 0 if payload["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,383 @@
|
||||
def pv_host_path(pv):
|
||||
spec = (pv or {}).get("spec") or {}
|
||||
host_path = (spec.get("hostPath") or {}).get("path")
|
||||
if isinstance(host_path, str) and host_path:
|
||||
return host_path
|
||||
local_path = (spec.get("local") or {}).get("path")
|
||||
if isinstance(local_path, str) and local_path:
|
||||
return local_path
|
||||
return None
|
||||
|
||||
def pvc_owner_group(namespace, owner):
|
||||
owner = str(owner or "")
|
||||
if namespace == "agentrun-ci":
|
||||
return "agentrun"
|
||||
if namespace == "hwlab-ci":
|
||||
if owner.startswith("agentrun-"):
|
||||
return "agentrun"
|
||||
return "hwlab"
|
||||
if namespace.startswith("hwlab-"):
|
||||
return "hwlab-runtime"
|
||||
return "other"
|
||||
|
||||
def parse_k8s_quantity(value):
|
||||
if value is None:
|
||||
return None
|
||||
raw = str(value).strip()
|
||||
match = re.match(r"^([0-9]+(?:\.[0-9]+)?)(Ki|Mi|Gi|Ti|K|M|G|T)?$", raw)
|
||||
if not match:
|
||||
return None
|
||||
multiplier = {
|
||||
None: 1,
|
||||
"K": 1000,
|
||||
"M": 1000**2,
|
||||
"G": 1000**3,
|
||||
"T": 1000**4,
|
||||
"Ki": 1024,
|
||||
"Mi": 1024**2,
|
||||
"Gi": 1024**3,
|
||||
"Ti": 1024**4,
|
||||
}.get(match.group(2), 1)
|
||||
return int(float(match.group(1)) * multiplier)
|
||||
|
||||
def metadata_owner(meta):
|
||||
refs = meta.get("ownerReferences") or []
|
||||
if refs:
|
||||
first = refs[0] or {}
|
||||
return first.get("kind"), first.get("name"), [{"kind": item.get("kind"), "name": item.get("name")} for item in refs[:5]]
|
||||
labels = meta.get("labels") or {}
|
||||
annotations = meta.get("annotations") or {}
|
||||
for key in [
|
||||
"tekton.dev/pipelineRun",
|
||||
"tekton.dev/taskRun",
|
||||
"agentrun.unidesk/run-id",
|
||||
"hwlab.unidesk/run-id",
|
||||
"app.kubernetes.io/instance",
|
||||
]:
|
||||
value = labels.get(key) or annotations.get(key)
|
||||
if value:
|
||||
return "Label", value, []
|
||||
return None, None, []
|
||||
|
||||
def ci_storage_snapshot():
|
||||
namespaces = set(config_list(PVC_CONFIG, "namespaces", ["hwlab-ci", "agentrun-ci"]))
|
||||
candidate_namespaces = set(config_list(PVC_CONFIG, "candidateNamespaces", []))
|
||||
hwlab_node = config_str(PVC_CONFIG, "hwlabNode", PROVIDER_ID)
|
||||
hwlab_lane = config_str(PVC_CONFIG, "hwlabLane", "v03")
|
||||
agentrun_node = config_str(PVC_CONFIG, "agentrunNode", PROVIDER_ID)
|
||||
agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "v02")
|
||||
limit = config_int(PVC_CONFIG, "limit", int(OPTIONS.get("limit") or 50), minimum=1, maximum=5000)
|
||||
pv_data = kubectl_json(["get", "pv"], 30) or {}
|
||||
pvc_data = kubectl_json(["get", "pvc", "-A"], 30) or {}
|
||||
pod_data = kubectl_json(["get", "pod", "-A"], 30) or {}
|
||||
pvs = {}
|
||||
for pv in pv_data.get("items") or []:
|
||||
meta = pv.get("metadata") or {}
|
||||
name = meta.get("name")
|
||||
if name:
|
||||
pvs[name] = pv
|
||||
mounts = {}
|
||||
for pod in pod_data.get("items") or []:
|
||||
meta = pod.get("metadata") or {}
|
||||
ns = str(meta.get("namespace") or "")
|
||||
pod_name = str(meta.get("name") or "")
|
||||
phase = str(((pod.get("status") or {}).get("phase")) or "")
|
||||
if phase in set(["Succeeded", "Failed"]):
|
||||
continue
|
||||
spec = pod.get("spec") or {}
|
||||
for vol in spec.get("volumes") or []:
|
||||
claim = (vol.get("persistentVolumeClaim") or {}).get("claimName")
|
||||
if claim:
|
||||
mounts.setdefault((ns, claim), []).append(pod_name)
|
||||
rows = []
|
||||
for pvc in pvc_data.get("items") or []:
|
||||
meta = pvc.get("metadata") or {}
|
||||
spec = pvc.get("spec") or {}
|
||||
status = pvc.get("status") or {}
|
||||
ns = str(meta.get("namespace") or "")
|
||||
name = str(meta.get("name") or "")
|
||||
if ns not in namespaces:
|
||||
continue
|
||||
volume = str(spec.get("volumeName") or "")
|
||||
pv = pvs.get(volume) or {}
|
||||
pv_spec = pv.get("spec") or {}
|
||||
pv_meta = pv.get("metadata") or {}
|
||||
owner_kind, owner_name, owner_refs = metadata_owner(meta)
|
||||
requested = parse_k8s_quantity((((spec.get("resources") or {}).get("requests") or {}).get("storage")))
|
||||
host_path = pv_host_path(pv)
|
||||
active = sorted(mounts.get((ns, name), []))
|
||||
estimated = du_size(host_path, 8) if host_path else None
|
||||
candidate_reasons = []
|
||||
if not active:
|
||||
candidate_reasons.append("no-active-mount-observed")
|
||||
if status.get("phase") != "Bound":
|
||||
candidate_reasons.append("pvc-not-bound")
|
||||
if (pv.get("status") or {}).get("phase") == "Released":
|
||||
candidate_reasons.append("pv-released")
|
||||
review_candidate = ns in candidate_namespaces and len(candidate_reasons) > 0
|
||||
rows.append({
|
||||
"namespace": ns,
|
||||
"pvc": name,
|
||||
"volume": volume or None,
|
||||
"phase": status.get("phase"),
|
||||
"pvPhase": (pv.get("status") or {}).get("phase"),
|
||||
"ownerKind": owner_kind,
|
||||
"owner": owner_name,
|
||||
"ownerRefs": owner_refs,
|
||||
"ownerGroup": pvc_owner_group(ns, owner_name),
|
||||
"storageClass": spec.get("storageClassName") or pv_spec.get("storageClassName"),
|
||||
"reclaimPolicy": pv_spec.get("persistentVolumeReclaimPolicy"),
|
||||
"requestedBytes": requested,
|
||||
"requestedHuman": fmt_bytes(requested or 0),
|
||||
"hostPath": host_path,
|
||||
"pvCreatedAt": (pv_meta.get("creationTimestamp") if isinstance(pv_meta, dict) else None),
|
||||
"pvcCreatedAt": meta.get("creationTimestamp"),
|
||||
"activeMountPods": active,
|
||||
"estimatedBytes": estimated,
|
||||
"estimatedHuman": fmt_bytes(estimated or 0),
|
||||
"reviewCandidate": review_candidate,
|
||||
"reviewReasons": candidate_reasons,
|
||||
"dryRunOnly": True,
|
||||
})
|
||||
rows.sort(key=lambda item: safe_int(item.get("estimatedBytes")), reverse=True)
|
||||
by_namespace = {}
|
||||
by_owner_group = {}
|
||||
for row in rows:
|
||||
for bucket, key in [(by_namespace, row.get("namespace") or "unknown"), (by_owner_group, row.get("ownerGroup") or "unknown")]:
|
||||
current = bucket.setdefault(key, {"count": 0, "estimatedBytes": 0, "activeMountCount": 0})
|
||||
current["count"] += 1
|
||||
current["estimatedBytes"] += safe_int(row.get("estimatedBytes"))
|
||||
current["activeMountCount"] += len(row.get("activeMountPods") or [])
|
||||
current["estimatedHuman"] = fmt_bytes(current["estimatedBytes"])
|
||||
review_candidates = [row for row in rows if row.get("reviewCandidate")]
|
||||
return {
|
||||
"scope": "YAML-configured PVC namespaces",
|
||||
"configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.pvcAttribution" % PROVIDER_ID,
|
||||
"namespaces": sorted(namespaces),
|
||||
"candidateNamespaces": sorted(candidate_namespaces),
|
||||
"pvcCount": len(rows),
|
||||
"reviewCandidateCount": len(review_candidates),
|
||||
"estimatedBytes": sum(safe_int(row.get("estimatedBytes")) for row in rows),
|
||||
"estimatedHuman": fmt_bytes(sum(safe_int(row.get("estimatedBytes")) for row in rows)),
|
||||
"requestedBytes": sum(safe_int(row.get("requestedBytes")) for row in rows),
|
||||
"requestedHuman": fmt_bytes(sum(safe_int(row.get("requestedBytes")) for row in rows)),
|
||||
"byNamespace": by_namespace,
|
||||
"byOwnerGroup": by_owner_group,
|
||||
"topPvcs": rows[:limit],
|
||||
"reviewCandidates": review_candidates[:limit],
|
||||
"handoff": {
|
||||
"hwlab": {
|
||||
"dryRun": "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node %s --lane %s --min-age-minutes 30 --limit 200 --dry-run" % (hwlab_node, hwlab_lane),
|
||||
"releasedPvs": "bun scripts/cli.ts hwlab nodes control-plane cleanup-released-pvs --node %s --lane %s --limit 200 --dry-run" % (hwlab_node, hwlab_lane),
|
||||
},
|
||||
"agentrun": {
|
||||
"dryRun": "bun scripts/cli.ts agentrun control-plane cleanup-runs --node %s --lane %s --min-age-minutes 30 --limit 200 --dry-run" % (agentrun_node, agentrun_lane),
|
||||
"releasedPvs": "bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node %s --lane %s --limit 200 --dry-run" % (agentrun_node, agentrun_lane),
|
||||
},
|
||||
},
|
||||
"policy": "analysis-only; remote GC never deletes PVC/PV/local-path data and only hands off to owner-aware retention commands",
|
||||
}
|
||||
|
||||
def compact_pvc_row(row):
|
||||
return {
|
||||
"namespace": row.get("namespace"),
|
||||
"pvc": row.get("pvc"),
|
||||
"phase": row.get("phase"),
|
||||
"pvPhase": row.get("pvPhase"),
|
||||
"ownerKind": row.get("ownerKind"),
|
||||
"owner": row.get("owner"),
|
||||
"ownerGroup": row.get("ownerGroup"),
|
||||
"estimatedBytes": row.get("estimatedBytes"),
|
||||
"estimatedHuman": row.get("estimatedHuman"),
|
||||
"activeMountCount": len(row.get("activeMountPods") or []),
|
||||
"reviewCandidate": row.get("reviewCandidate"),
|
||||
"reviewReasons": row.get("reviewReasons"),
|
||||
}
|
||||
|
||||
def compact_pvc_attribution(payload):
|
||||
if bool(OPTIONS.get("full")):
|
||||
return payload
|
||||
top = payload.get("topPvcs") or []
|
||||
review = payload.get("reviewCandidates") or []
|
||||
compact_top = [compact_pvc_row(row) for row in top[:8] if isinstance(row, dict)]
|
||||
return {
|
||||
"configSource": payload.get("configSource"),
|
||||
"candidateNamespaces": payload.get("candidateNamespaces"),
|
||||
"pvcCount": payload.get("pvcCount"),
|
||||
"reviewCandidateCount": payload.get("reviewCandidateCount"),
|
||||
"estimatedBytes": payload.get("estimatedBytes"),
|
||||
"estimatedHuman": payload.get("estimatedHuman"),
|
||||
"byNamespace": payload.get("byNamespace"),
|
||||
"byOwnerGroup": payload.get("byOwnerGroup"),
|
||||
"topPvcs": compact_top,
|
||||
"reviewCandidates": [compact_pvc_row(row) for row in review[:2] if isinstance(row, dict)],
|
||||
"handoff": payload.get("handoff"),
|
||||
"compacted": True,
|
||||
"fullDisclosure": "rerun with --full for hostPath, creation timestamps and complete row details",
|
||||
}
|
||||
|
||||
def compact_ci_storage_summary(payload):
|
||||
return {
|
||||
"scope": payload.get("scope"),
|
||||
"configSource": payload.get("configSource"),
|
||||
"pvcCount": payload.get("pvcCount"),
|
||||
"reviewCandidateCount": payload.get("reviewCandidateCount"),
|
||||
"estimatedBytes": payload.get("estimatedBytes"),
|
||||
"estimatedHuman": payload.get("estimatedHuman"),
|
||||
"requestedBytes": payload.get("requestedBytes"),
|
||||
"requestedHuman": payload.get("requestedHuman"),
|
||||
"compacted": True,
|
||||
"fullDisclosure": "use pvcAttribution or --full for row-level details",
|
||||
}
|
||||
|
||||
def local_path_storage_root():
|
||||
root = config_str(LOCAL_PATH_CONFIG, "root", "")
|
||||
if not root:
|
||||
return ""
|
||||
return os.path.realpath(os.path.abspath(root))
|
||||
|
||||
def local_path_orphan_prefixes():
|
||||
return config_list(LOCAL_PATH_CONFIG, "orphanDirPrefixes", [])
|
||||
|
||||
def is_direct_local_path_child(root, path):
|
||||
resolved = os.path.realpath(os.path.abspath(path))
|
||||
return os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/")
|
||||
|
||||
def local_path_referenced_paths(root):
|
||||
pv_data = kubectl_json(["get", "pv"], 30) or {}
|
||||
referenced = set()
|
||||
for pv in pv_data.get("items") or []:
|
||||
host_path = pv_host_path(pv)
|
||||
if not host_path:
|
||||
continue
|
||||
resolved = os.path.realpath(os.path.abspath(host_path))
|
||||
if resolved == root or resolved.startswith(root.rstrip("/") + "/"):
|
||||
referenced.add(resolved)
|
||||
return referenced
|
||||
|
||||
def assert_local_path_orphan(path, referenced=None):
|
||||
root = local_path_storage_root()
|
||||
if not root:
|
||||
raise RuntimeError("localPathStorage.root is not configured")
|
||||
prefixes = local_path_orphan_prefixes()
|
||||
resolved = os.path.realpath(os.path.abspath(path))
|
||||
name = os.path.basename(resolved)
|
||||
if not is_direct_local_path_child(root, resolved):
|
||||
raise RuntimeError("refusing to remove local-path orphan outside configured direct storage root: %s" % path)
|
||||
if os.path.islink(path) or not os.path.isdir(resolved):
|
||||
raise RuntimeError("refusing to remove non-directory or symlink local-path orphan: %s" % path)
|
||||
if not prefixes or not any(name.startswith(prefix) for prefix in prefixes):
|
||||
raise RuntimeError("refusing to remove local-path orphan outside YAML prefix allowlist: %s" % path)
|
||||
refs = referenced if referenced is not None else local_path_referenced_paths(root)
|
||||
for ref in refs:
|
||||
if resolved == ref or ref.startswith(resolved.rstrip("/") + "/") or resolved.startswith(ref.rstrip("/") + "/"):
|
||||
raise RuntimeError("refusing to remove local-path path still referenced by PV: %s" % path)
|
||||
if path_has_open_fd(resolved):
|
||||
raise RuntimeError("refusing to remove local-path orphan with open fd/cwd reference: %s" % path)
|
||||
return resolved
|
||||
|
||||
def local_path_orphan_rows():
|
||||
if not config_bool(LOCAL_PATH_CONFIG, "enabled", False):
|
||||
return [], {"ok": False, "reason": "local-path-orphan-cleanup-disabled"}
|
||||
root = local_path_storage_root()
|
||||
prefixes = local_path_orphan_prefixes()
|
||||
if not root or not os.path.isdir(root) or os.path.islink(root):
|
||||
return [], {"ok": False, "reason": "local-path-root-unavailable", "root": root}
|
||||
if not prefixes:
|
||||
return [], {"ok": False, "reason": "local-path-prefix-allowlist-empty", "root": root}
|
||||
referenced = local_path_referenced_paths(root)
|
||||
min_age_minutes = config_float(LOCAL_PATH_CONFIG, "orphanMinAgeMinutes", 0.0, minimum=0.0)
|
||||
cutoff = time.time() - min_age_minutes * 60.0
|
||||
rows = []
|
||||
protected = []
|
||||
for name in sorted(os.listdir(root)):
|
||||
path = os.path.join(root, name)
|
||||
resolved = os.path.realpath(os.path.abspath(path))
|
||||
try:
|
||||
stat = os.lstat(path)
|
||||
except OSError:
|
||||
continue
|
||||
if not os.path.isdir(path) or os.path.islink(path) or not any(name.startswith(prefix) for prefix in prefixes):
|
||||
continue
|
||||
row = {"path": resolved, "name": name, "sizeBytes": 0, "estimatedReclaimBytes": 0}
|
||||
if not is_direct_local_path_child(root, resolved):
|
||||
protected.append({**row, "reason": "not-direct-child"})
|
||||
continue
|
||||
if stat.st_mtime >= cutoff:
|
||||
protected.append({**row, "reason": "younger-than-min-age"})
|
||||
continue
|
||||
referenced_by = [ref for ref in referenced if resolved == ref or ref.startswith(resolved.rstrip("/") + "/") or resolved.startswith(ref.rstrip("/") + "/")]
|
||||
if referenced_by:
|
||||
protected.append({**row, "reason": "pv-referenced", "referencedCount": len(referenced_by)})
|
||||
continue
|
||||
if path_has_open_fd(resolved):
|
||||
protected.append({**row, "reason": "open-fd"})
|
||||
continue
|
||||
size = du_size(resolved, 10) or path_size(resolved)
|
||||
rows.append({**row, "sizeBytes": size, "estimatedReclaimBytes": size})
|
||||
rows.sort(key=lambda item: safe_int(item.get("estimatedReclaimBytes")), reverse=True)
|
||||
return rows, {
|
||||
"ok": True,
|
||||
"root": root,
|
||||
"prefixes": prefixes,
|
||||
"referencedPathCount": len(referenced),
|
||||
"protectedCount": len(protected),
|
||||
"protectedPreview": protected[:8],
|
||||
"minAgeMinutes": min_age_minutes,
|
||||
}
|
||||
|
||||
def local_path_orphan_candidate():
|
||||
rows, meta = local_path_orphan_rows()
|
||||
if not meta.get("ok"):
|
||||
return {
|
||||
"id": "k3s-local-path-orphans:unavailable",
|
||||
"kind": "k3s-local-path-orphans-unavailable",
|
||||
"risk": "blocked",
|
||||
"description": "K3s local-path orphan cleanup is unavailable or disabled by YAML",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"diagnostic": meta,
|
||||
}
|
||||
limit = int(OPTIONS.get("limit") or 50)
|
||||
selected = rows[:limit]
|
||||
estimated = sum(safe_int(row.get("estimatedReclaimBytes")) for row in selected)
|
||||
if estimated <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "k3s-local-path-orphans:delete",
|
||||
"kind": "k3s-local-path-orphans-delete",
|
||||
"risk": "medium",
|
||||
"description": "Delete YAML-allowlisted k3s local-path storage directories that no PV references and no process has open",
|
||||
"path": meta.get("root"),
|
||||
"sizeBytes": estimated,
|
||||
"estimatedReclaimBytes": estimated,
|
||||
"orphanCount": len(rows),
|
||||
"selectedOrphanCount": len(selected),
|
||||
"protectedCount": meta.get("protectedCount"),
|
||||
"referencedPathCount": meta.get("referencedPathCount"),
|
||||
"selectedPreview": [{"name": row.get("name"), "path": row.get("path"), "estimatedReclaimBytes": row.get("estimatedReclaimBytes")} for row in selected[:8]],
|
||||
"protectedPreview": meta.get("protectedPreview"),
|
||||
"action": {"op": "rm-recursive", "allowlist": "yaml-local-path-orphan", "root": meta.get("root"), "limit": limit},
|
||||
}
|
||||
|
||||
def execute_local_path_orphan_cleanup():
|
||||
rows, meta = local_path_orphan_rows()
|
||||
if not meta.get("ok"):
|
||||
raise RuntimeError("local-path orphan cleanup unavailable: %s" % meta.get("reason"))
|
||||
limit = int(OPTIONS.get("limit") or 50)
|
||||
selected = rows[:limit]
|
||||
referenced = local_path_referenced_paths(local_path_storage_root())
|
||||
reclaimed = 0
|
||||
deleted = []
|
||||
for row in selected:
|
||||
path = assert_local_path_orphan(row.get("path"), referenced)
|
||||
before = du_size(path, 10) or path_size(path)
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
reclaimed += before
|
||||
deleted.append({"name": row.get("name"), "path": path, "reclaimedBytes": before})
|
||||
return {
|
||||
"reclaimedBytes": reclaimed,
|
||||
"deletedOrphanCount": len(deleted),
|
||||
"deletedPreview": deleted[:12],
|
||||
"root": meta.get("root"),
|
||||
"protectedCount": meta.get("protectedCount"),
|
||||
}
|
||||
@@ -0,0 +1,677 @@
|
||||
def active_hwlab_ci_writes():
|
||||
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" {print}' | head -40"], 15)
|
||||
lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()]
|
||||
return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)}
|
||||
|
||||
def active_hwlab_ci_jobs():
|
||||
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get jobs -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"Complete\" && $2 != \"Failed\" {print}' | head -40"], 15)
|
||||
lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()]
|
||||
return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)}
|
||||
|
||||
def wait_no_active_hwlab_ci(timeout=180):
|
||||
deadline = time.time() + timeout
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
writes = active_hwlab_ci_writes()
|
||||
jobs = active_hwlab_ci_jobs()
|
||||
last = {"writes": writes, "jobs": jobs}
|
||||
if writes.get("ok") and jobs.get("ok") and int(writes.get("activeCount") or 0) == 0 and int(jobs.get("activeCount") or 0) == 0:
|
||||
return {"ok": True, "last": last}
|
||||
time.sleep(5)
|
||||
return {"ok": False, "last": last}
|
||||
|
||||
def kubectl_json(args, timeout=20):
|
||||
result = command(["env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "kubectl"] + args + ["-o", "json"], timeout)
|
||||
if result["exitCode"] != 0:
|
||||
return None
|
||||
try:
|
||||
return json.loads(result["stdout"] or "{}")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def kctl(args, timeout=30):
|
||||
return command(["env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "kubectl"] + args, timeout)
|
||||
|
||||
def workload_image_refs():
|
||||
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get deploy,sts,ds,pod -A -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null | sort -u"], 30)
|
||||
refs = set()
|
||||
digests = set()
|
||||
for image in (result.get("stdout") or "").splitlines():
|
||||
image = image.strip()
|
||||
if not image.startswith("127.0.0.1:5000/"):
|
||||
continue
|
||||
ref = image.split("127.0.0.1:5000/", 1)[1]
|
||||
if "@sha256:" in ref:
|
||||
repo, digest = ref.split("@", 1)
|
||||
refs.add((repo, "@" + digest))
|
||||
digests.add("sha256:" + digest.split(":", 1)[1])
|
||||
elif ":" in ref:
|
||||
repo, tag = ref.rsplit(":", 1)
|
||||
refs.add((repo, tag))
|
||||
return refs, digests, bounded(result)
|
||||
|
||||
def registry_request(method, path, headers=None, timeout=20):
|
||||
url = "http://127.0.0.1:5000" + path
|
||||
req = urllib.request.Request(url, method=method, headers=headers or {})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
body = response.read()
|
||||
return {"status": response.status, "headers": dict(response.headers), "body": body.decode("utf-8", errors="replace")}
|
||||
|
||||
def registry_tag_rows():
|
||||
rows = []
|
||||
root = REGISTRY_REPOSITORY_ROOT
|
||||
if not os.path.isdir(root):
|
||||
return rows
|
||||
for repo_root, dirs, files in os.walk(root):
|
||||
if os.path.basename(repo_root) != "tags":
|
||||
continue
|
||||
rel = os.path.relpath(repo_root, root)
|
||||
suffix = "/_manifests/tags"
|
||||
if not rel.endswith(suffix):
|
||||
continue
|
||||
repo = rel[:-len(suffix)]
|
||||
try:
|
||||
tags = os.listdir(repo_root)
|
||||
except OSError:
|
||||
continue
|
||||
for tag in sorted(tags):
|
||||
link = os.path.join(repo_root, tag, "current", "link")
|
||||
if not os.path.isfile(link):
|
||||
continue
|
||||
try:
|
||||
with open(link, "r", encoding="utf-8") as handle:
|
||||
digest = handle.read().strip()
|
||||
stat = os.stat(link)
|
||||
except OSError:
|
||||
continue
|
||||
rows.append({
|
||||
"repo": repo,
|
||||
"tag": tag,
|
||||
"digest": digest,
|
||||
"mtime": stat.st_mtime,
|
||||
"mtimeIso": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(stat.st_mtime)),
|
||||
"path": os.path.join(repo_root, tag),
|
||||
})
|
||||
return rows
|
||||
|
||||
def registry_revision_rows():
|
||||
rows = []
|
||||
root = REGISTRY_REPOSITORY_ROOT
|
||||
if not os.path.isdir(root):
|
||||
return rows
|
||||
for repo_root, dirs, files in os.walk(root):
|
||||
if os.path.basename(repo_root) != "sha256":
|
||||
continue
|
||||
rel = os.path.relpath(repo_root, root)
|
||||
suffix = "/_manifests/revisions/sha256"
|
||||
if not rel.endswith(suffix):
|
||||
continue
|
||||
repo = rel[:-len(suffix)]
|
||||
try:
|
||||
revisions = os.listdir(repo_root)
|
||||
except OSError:
|
||||
continue
|
||||
for digest_hex in sorted(revisions):
|
||||
path = os.path.join(repo_root, digest_hex)
|
||||
link = os.path.join(path, "link")
|
||||
if not os.path.isfile(link):
|
||||
continue
|
||||
try:
|
||||
with open(link, "r", encoding="utf-8") as handle:
|
||||
digest = handle.read().strip()
|
||||
stat = os.stat(link)
|
||||
except OSError:
|
||||
continue
|
||||
rows.append({
|
||||
"repo": repo,
|
||||
"digest": digest,
|
||||
"mtime": stat.st_mtime,
|
||||
"mtimeIso": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(stat.st_mtime)),
|
||||
"path": path,
|
||||
})
|
||||
return rows
|
||||
|
||||
def registry_retention_repo(repo):
|
||||
return repo.startswith("hwlab/hwlab-") or repo.startswith("hwlab/cache/hwlab-")
|
||||
|
||||
def registry_digest_hex(digest):
|
||||
if not isinstance(digest, str) or not digest.startswith("sha256:"):
|
||||
return None
|
||||
value = digest.split(":", 1)[1]
|
||||
if re.match(r"^[0-9a-f]{64}$", value) is None:
|
||||
return None
|
||||
return value
|
||||
|
||||
def registry_blob_data_path(digest):
|
||||
value = registry_digest_hex(digest)
|
||||
if value is None:
|
||||
return None
|
||||
return os.path.join(REGISTRY_ROOT, "docker/registry/v2/blobs/sha256", value[:2], value, "data")
|
||||
|
||||
_manifest_cache = {}
|
||||
def registry_manifest_json(digest):
|
||||
if digest in _manifest_cache:
|
||||
return _manifest_cache[digest]
|
||||
path = registry_blob_data_path(digest)
|
||||
if path is None or not os.path.isfile(path):
|
||||
_manifest_cache[digest] = None
|
||||
return None
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
data = handle.read(8 * 1024 * 1024)
|
||||
value = json.loads(data.decode("utf-8"))
|
||||
except Exception:
|
||||
value = None
|
||||
_manifest_cache[digest] = value
|
||||
return value
|
||||
|
||||
def registry_manifest_refs(digest):
|
||||
manifest = registry_manifest_json(digest)
|
||||
if not isinstance(manifest, dict):
|
||||
return set()
|
||||
refs = set()
|
||||
config = manifest.get("config") or {}
|
||||
config_digest = config.get("digest")
|
||||
if isinstance(config_digest, str) and registry_digest_hex(config_digest) is not None:
|
||||
refs.add(config_digest)
|
||||
for item in manifest.get("layers") or []:
|
||||
item_digest = (item or {}).get("digest")
|
||||
if isinstance(item_digest, str) and registry_digest_hex(item_digest) is not None:
|
||||
refs.add(item_digest)
|
||||
for item in manifest.get("manifests") or []:
|
||||
item_digest = (item or {}).get("digest")
|
||||
if isinstance(item_digest, str) and registry_digest_hex(item_digest) is not None:
|
||||
refs.add(item_digest)
|
||||
return refs
|
||||
|
||||
def registry_digest_closure(seed):
|
||||
seen = set()
|
||||
stack = list(seed)
|
||||
while stack:
|
||||
digest = stack.pop()
|
||||
if digest in seen or registry_digest_hex(digest) is None:
|
||||
continue
|
||||
seen.add(digest)
|
||||
for child in registry_manifest_refs(digest):
|
||||
if child not in seen:
|
||||
stack.append(child)
|
||||
return seen
|
||||
|
||||
def registry_blob_size(digest):
|
||||
path = registry_blob_data_path(digest)
|
||||
if path is None or not os.path.isfile(path):
|
||||
return 0
|
||||
try:
|
||||
return int(os.lstat(path).st_blocks) * 512
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
def estimate_registry_reclaim(delete_manifest_digests, kept_manifest_digests):
|
||||
deleted = registry_digest_closure(delete_manifest_digests)
|
||||
kept = registry_digest_closure(kept_manifest_digests)
|
||||
reclaim = deleted - kept
|
||||
return sum(registry_blob_size(digest) for digest in reclaim)
|
||||
|
||||
def plan_registry_retention():
|
||||
keep_per_repo = int(OPTIONS.get("registryKeepPerRepo") if OPTIONS.get("registryKeepPerRepo") is not None else 5)
|
||||
min_age_hours = float(OPTIONS.get("registryMinAgeHours") if OPTIONS.get("registryMinAgeHours") is not None else 48)
|
||||
cutoff = time.time() - min_age_hours * 3600
|
||||
refs, digests, refs_command = workload_image_refs()
|
||||
rows = registry_tag_rows()
|
||||
revision_rows = registry_revision_rows()
|
||||
by_repo = {}
|
||||
for row in rows:
|
||||
by_repo.setdefault(row["repo"], []).append(row)
|
||||
keep = set()
|
||||
keep_reasons = {}
|
||||
for repo, items in by_repo.items():
|
||||
items.sort(key=lambda item: item["mtime"], reverse=True)
|
||||
for row in items[:keep_per_repo]:
|
||||
key = (row["repo"], row["tag"])
|
||||
keep.add(key)
|
||||
keep_reasons[key] = "latest-per-repo"
|
||||
for row in items:
|
||||
key = (row["repo"], row["tag"])
|
||||
if row["tag"] in REGISTRY_PROTECTED_TAGS:
|
||||
keep.add(key)
|
||||
keep_reasons[key] = "protected-tag"
|
||||
if key in refs:
|
||||
keep.add(key)
|
||||
keep_reasons[key] = "workload-tag-ref"
|
||||
if row["digest"] in digests:
|
||||
keep.add(key)
|
||||
keep_reasons[key] = "workload-digest-ref"
|
||||
if row["repo"].startswith("hwlab/cache/"):
|
||||
keep.add(key)
|
||||
keep_reasons[key] = "cache-repo"
|
||||
if row["mtime"] >= cutoff:
|
||||
keep.add(key)
|
||||
keep_reasons[key] = "recent-tag"
|
||||
delete_rows = []
|
||||
kept_count = 0
|
||||
delete_by_repo = {}
|
||||
keep_by_repo = {}
|
||||
kept_digests = set()
|
||||
for row in rows:
|
||||
key = (row["repo"], row["tag"])
|
||||
should_delete = (
|
||||
key not in keep
|
||||
and row["repo"].startswith("hwlab/hwlab-")
|
||||
and re.match(r"^[0-9a-f]{7,40}$", row["tag"]) is not None
|
||||
)
|
||||
if should_delete:
|
||||
delete_rows.append(row)
|
||||
delete_by_repo[row["repo"]] = delete_by_repo.get(row["repo"], 0) + 1
|
||||
else:
|
||||
kept_count += 1
|
||||
kept_digests.add(row["digest"])
|
||||
keep_by_repo[row["repo"]] = keep_by_repo.get(row["repo"], 0) + 1
|
||||
protected_digests = kept_digests | digests
|
||||
protected_digests.update(row["digest"] for row in revision_rows if not registry_retention_repo(row["repo"]))
|
||||
protected_digests = registry_digest_closure(protected_digests)
|
||||
delete_revision_rows = []
|
||||
revision_delete_by_repo = {}
|
||||
for row in revision_rows:
|
||||
if not registry_retention_repo(row["repo"]):
|
||||
continue
|
||||
if row["digest"] in protected_digests:
|
||||
continue
|
||||
delete_revision_rows.append(row)
|
||||
revision_delete_by_repo[row["repo"]] = revision_delete_by_repo.get(row["repo"], 0) + 1
|
||||
kept_revision_digests = set(row["digest"] for row in revision_rows if row not in delete_revision_rows)
|
||||
delete_revision_digests = set(row["digest"] for row in delete_revision_rows)
|
||||
deletable_manifests = {}
|
||||
for row in delete_rows:
|
||||
if row["digest"] in kept_digests:
|
||||
continue
|
||||
deletable_manifests.setdefault(row["repo"], set()).add(row["digest"])
|
||||
for row in delete_revision_rows:
|
||||
deletable_manifests.setdefault(row["repo"], set()).add(row["digest"])
|
||||
deletable_manifest_count = sum(len(items) for items in deletable_manifests.values())
|
||||
registry_size = du_size(REGISTRY_ROOT, 30) or 0
|
||||
estimate = estimate_registry_reclaim(delete_revision_digests, kept_revision_digests)
|
||||
return {
|
||||
"tagRows": rows,
|
||||
"revisionRows": revision_rows,
|
||||
"deleteRows": delete_rows,
|
||||
"deleteRevisionRows": delete_revision_rows,
|
||||
"summary": {
|
||||
"totalTags": len(rows),
|
||||
"totalRevisions": len(revision_rows),
|
||||
"repoCount": len(by_repo),
|
||||
"keepPerRepo": keep_per_repo,
|
||||
"minAgeHours": min_age_hours,
|
||||
"protectedWorkloadRefs": len(refs),
|
||||
"protectedDigestRefs": len(digests),
|
||||
"protectedDigestClosure": len(protected_digests),
|
||||
"keptTags": kept_count,
|
||||
"deleteTags": len(delete_rows),
|
||||
"deleteManifests": deletable_manifest_count,
|
||||
"deleteRevisions": len(delete_revision_rows),
|
||||
"deleteByRepo": delete_by_repo,
|
||||
"revisionDeleteByRepo": revision_delete_by_repo,
|
||||
"keepByRepo": keep_by_repo,
|
||||
"registrySizeBytes": registry_size,
|
||||
"estimatedReclaimBytes": estimate,
|
||||
},
|
||||
"deleteManifestsByRepo": {repo: sorted(list(digests)) for repo, digests in deletable_manifests.items()},
|
||||
"refsCommand": refs_command,
|
||||
}
|
||||
|
||||
def registry_deployment_preflight():
|
||||
dep = kubectl_json(["-n", "hwlab-ci", "get", "deploy", "hwlab-registry"], 20)
|
||||
if not dep:
|
||||
return {"ok": False, "reason": "registry-deployment-missing"}
|
||||
spec = ((dep.get("spec") or {}).get("template") or {}).get("spec") or {}
|
||||
containers = spec.get("containers") or []
|
||||
volumes = spec.get("volumes") or []
|
||||
registry_container = next((item for item in containers if item.get("name") == "registry"), containers[0] if containers else {})
|
||||
mounts = registry_container.get("volumeMounts") or []
|
||||
has_host_path = any(((vol.get("hostPath") or {}).get("path") == REGISTRY_ROOT and vol.get("name") == "storage") for vol in volumes)
|
||||
has_mount = any((mount.get("name") == "storage" and mount.get("mountPath") == "/var/lib/registry") for mount in mounts)
|
||||
image = str(registry_container.get("image") or "")
|
||||
ok = bool(has_host_path and has_mount and image.startswith("registry:") and spec.get("hostNetwork") is True)
|
||||
return {
|
||||
"ok": ok,
|
||||
"reason": "ok" if ok else "unexpected-registry-deployment-shape",
|
||||
"image": image,
|
||||
"hostNetwork": spec.get("hostNetwork"),
|
||||
"hasExpectedHostPath": has_host_path,
|
||||
"hasExpectedMount": has_mount,
|
||||
"replicas": (dep.get("spec") or {}).get("replicas"),
|
||||
"readyReplicas": (dep.get("status") or {}).get("readyReplicas"),
|
||||
}
|
||||
|
||||
def cronjob_suspend_states(names):
|
||||
states = {}
|
||||
for name in names:
|
||||
data = kubectl_json(["-n", "hwlab-ci", "get", "cronjob", name], 15)
|
||||
if data:
|
||||
states[name] = bool(((data.get("spec") or {}).get("suspend")) is True)
|
||||
return states
|
||||
|
||||
def patch_cronjob_suspend(name, suspend):
|
||||
payload = json.dumps({"spec": {"suspend": bool(suspend)}})
|
||||
return kctl(["-n", "hwlab-ci", "patch", "cronjob", name, "--type=merge", "-p", payload], 30)
|
||||
|
||||
def wait_registry_pod_count(target, timeout=90):
|
||||
deadline = time.time() + timeout
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
result = kctl(["-n", "hwlab-ci", "get", "pods", "-l", "app.kubernetes.io/name=hwlab-registry", "--no-headers"], 20)
|
||||
last = bounded(result)
|
||||
lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()]
|
||||
active = []
|
||||
for line in lines:
|
||||
parts = line.split()
|
||||
status = parts[2] if len(parts) >= 3 else ""
|
||||
if status in set(["Completed", "Error", "Failed", "Succeeded"]):
|
||||
continue
|
||||
active.append(line)
|
||||
if len(active) == target:
|
||||
return {"ok": True, "lines": active, "allLines": lines, "last": last}
|
||||
time.sleep(2)
|
||||
return {"ok": False, "lines": [], "last": last}
|
||||
|
||||
def wait_pod_terminal(name, timeout=900):
|
||||
deadline = time.time() + timeout
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
data = kubectl_json(["-n", "hwlab-ci", "get", "pod", name], 20)
|
||||
if data:
|
||||
phase = ((data.get("status") or {}).get("phase")) or ""
|
||||
last = {"phase": phase}
|
||||
if phase == "Succeeded":
|
||||
return {"ok": True, "phase": phase}
|
||||
if phase == "Failed":
|
||||
return {"ok": False, "phase": phase}
|
||||
time.sleep(3)
|
||||
return {"ok": False, "phase": "Timeout", "last": last}
|
||||
|
||||
def execute_registry_retention():
|
||||
if PROVIDER_ID.upper() != "G14":
|
||||
raise RuntimeError("HWLAB registry retention is only supported on G14")
|
||||
deployment = registry_deployment_preflight()
|
||||
if not deployment.get("ok"):
|
||||
raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason"))
|
||||
plan = plan_registry_retention()
|
||||
delete_rows = plan.get("deleteRows") or []
|
||||
delete_revision_rows = plan.get("deleteRevisionRows") or []
|
||||
delete_manifests = plan.get("deleteManifestsByRepo") or {}
|
||||
if not delete_rows and not delete_revision_rows:
|
||||
return {"reclaimedBytes": 0, "commandOutput": {"message": "no registry tags or revisions matched conservative retention", "registryPlan": plan.get("summary")}}
|
||||
if not delete_manifests:
|
||||
return {"reclaimedBytes": 0, "commandOutput": {"message": "matched manifests are still referenced by retained manifests; registry GC would not reclaim blobs", "registryPlan": plan.get("summary")}}
|
||||
cronjobs = ["hwlab-g14-branch-poller", "hwlab-v02-branch-poller"]
|
||||
original_crons = cronjob_suspend_states(cronjobs)
|
||||
before = du_size(REGISTRY_ROOT, 60) or 0
|
||||
gc_name = "hwlab-registry-gc-%s" % int(time.time())
|
||||
steps = []
|
||||
try:
|
||||
for name in original_crons:
|
||||
result = patch_cronjob_suspend(name, True)
|
||||
steps.append({"step": "suspend-cronjob", "name": name, "result": bounded(result)})
|
||||
if result["exitCode"] != 0:
|
||||
raise RuntimeError("failed to suspend cronjob %s" % name)
|
||||
idle_after_suspend = wait_no_active_hwlab_ci(180)
|
||||
steps.append({"step": "idle-after-suspend", "result": idle_after_suspend})
|
||||
if not idle_after_suspend.get("ok"):
|
||||
raise RuntimeError("refusing registry maintenance because hwlab-ci did not become idle after suspend")
|
||||
|
||||
deleted_manifests = []
|
||||
for repo, digests in delete_manifests.items():
|
||||
encoded_repo = "/".join(urllib.parse.quote(part, safe="") for part in repo.split("/"))
|
||||
for digest in digests:
|
||||
try:
|
||||
result = registry_request("DELETE", "/v2/%s/manifests/%s" % (encoded_repo, urllib.parse.quote(digest, safe=":")), {"Accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"})
|
||||
deleted_manifests.append({"repo": repo, "digest": digest, "status": result.get("status")})
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
deleted_manifests.append({"repo": repo, "digest": digest, "status": 404})
|
||||
else:
|
||||
raise
|
||||
steps.append({"step": "registry-api-delete-manifests", "count": len(deleted_manifests), "preview": deleted_manifests[:20]})
|
||||
|
||||
scale_down = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=0"], 60)
|
||||
steps.append({"step": "scale-registry-down", "result": bounded(scale_down)})
|
||||
if scale_down["exitCode"] != 0:
|
||||
raise RuntimeError("failed to scale registry down")
|
||||
waited_down = wait_registry_pod_count(0, 120)
|
||||
steps.append({"step": "wait-registry-down", "result": waited_down})
|
||||
if not waited_down.get("ok"):
|
||||
raise RuntimeError("registry pod did not scale down")
|
||||
|
||||
deleted = []
|
||||
for row in delete_rows:
|
||||
path = os.path.abspath(str(row.get("path") or ""))
|
||||
if not path.startswith(REGISTRY_REPOSITORY_ROOT + "/") or "/_manifests/tags/" not in path:
|
||||
raise RuntimeError("refusing unexpected registry tag path: %s" % path)
|
||||
if not re.match(r"^[0-9a-f]{7,40}$", str(row.get("tag") or "")):
|
||||
raise RuntimeError("refusing unexpected registry tag name: %s" % row.get("tag"))
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
shutil.rmtree(path)
|
||||
deleted.append({"repo": row.get("repo"), "tag": row.get("tag"), "digest": row.get("digest")})
|
||||
steps.append({"step": "delete-tag-directories", "count": len(deleted)})
|
||||
|
||||
deleted_revisions = []
|
||||
for row in delete_revision_rows:
|
||||
path = os.path.abspath(str(row.get("path") or ""))
|
||||
digest_hex = registry_digest_hex(str(row.get("digest") or ""))
|
||||
if digest_hex is None:
|
||||
raise RuntimeError("refusing unexpected registry revision digest: %s" % row.get("digest"))
|
||||
if not path.startswith(REGISTRY_REPOSITORY_ROOT + "/") or "/_manifests/revisions/sha256/" not in path:
|
||||
raise RuntimeError("refusing unexpected registry revision path: %s" % path)
|
||||
if os.path.basename(path) != digest_hex:
|
||||
raise RuntimeError("refusing registry revision path/digest mismatch: %s" % path)
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
shutil.rmtree(path)
|
||||
deleted_revisions.append({"repo": row.get("repo"), "digest": row.get("digest")})
|
||||
steps.append({"step": "delete-revision-directories", "count": len(deleted_revisions)})
|
||||
|
||||
overrides = {
|
||||
"apiVersion": "v1",
|
||||
"spec": {
|
||||
"restartPolicy": "Never",
|
||||
"containers": [{
|
||||
"name": "registry-gc",
|
||||
"image": "registry:2.8.3",
|
||||
"command": ["registry", "garbage-collect", "/etc/docker/registry/config.yml"],
|
||||
"volumeMounts": [{"name": "storage", "mountPath": "/var/lib/registry"}],
|
||||
}],
|
||||
"volumes": [{"name": "storage", "hostPath": {"path": REGISTRY_ROOT, "type": "DirectoryOrCreate"}}],
|
||||
},
|
||||
}
|
||||
run_gc = kctl(["-n", "hwlab-ci", "run", gc_name, "--restart=Never", "--image=registry:2.8.3", "--overrides=%s" % json.dumps(overrides)], 60)
|
||||
steps.append({"step": "start-registry-gc-pod", "result": bounded(run_gc), "pod": gc_name})
|
||||
if run_gc["exitCode"] != 0:
|
||||
raise RuntimeError("failed to start registry GC pod")
|
||||
waited_gc = wait_pod_terminal(gc_name, 900)
|
||||
steps.append({"step": "wait-registry-gc", "result": waited_gc})
|
||||
logs = kctl(["-n", "hwlab-ci", "logs", gc_name], 120)
|
||||
steps.append({"step": "registry-gc-logs", "result": bounded(logs)})
|
||||
if not waited_gc.get("ok"):
|
||||
raise RuntimeError("registry GC pod did not complete successfully")
|
||||
finally:
|
||||
cleanup_gc = kctl(["-n", "hwlab-ci", "delete", "pod", gc_name, "--ignore-not-found=true"], 60)
|
||||
steps.append({"step": "delete-registry-gc-pod", "result": bounded(cleanup_gc)})
|
||||
scale_up = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=%s" % int(deployment.get("replicas") or 1)], 60)
|
||||
steps.append({"step": "scale-registry-up", "result": bounded(scale_up)})
|
||||
rollout = kctl(["-n", "hwlab-ci", "rollout", "status", "deploy/hwlab-registry", "--timeout=180s"], 200)
|
||||
steps.append({"step": "wait-registry-rollout", "result": bounded(rollout)})
|
||||
for name, was_suspended in original_crons.items():
|
||||
restore = patch_cronjob_suspend(name, was_suspended)
|
||||
steps.append({"step": "restore-cronjob", "name": name, "suspend": was_suspended, "result": bounded(restore)})
|
||||
after = du_size(REGISTRY_ROOT, 60) or 0
|
||||
return {
|
||||
"reclaimedBytes": max(0, before - after),
|
||||
"commandOutput": {
|
||||
"registryPlan": plan.get("summary"),
|
||||
"deletedTagCount": len(delete_rows),
|
||||
"deletedRevisionCount": len(delete_revision_rows),
|
||||
"deletedManifestCount": sum(len(items) for items in delete_manifests.values()),
|
||||
"diskBeforeBytes": before,
|
||||
"diskAfterBytes": after,
|
||||
"steps": steps[-12:],
|
||||
},
|
||||
}
|
||||
|
||||
def execute_registry_garbage_collect_only():
|
||||
if PROVIDER_ID.upper() != "G14":
|
||||
raise RuntimeError("HWLAB registry garbage-collect is only supported on G14")
|
||||
deployment = registry_deployment_preflight()
|
||||
if not deployment.get("ok"):
|
||||
raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason"))
|
||||
cronjobs = ["hwlab-g14-branch-poller", "hwlab-v02-branch-poller"]
|
||||
original_crons = cronjob_suspend_states(cronjobs)
|
||||
before = du_size(REGISTRY_ROOT, 60) or 0
|
||||
gc_name = "hwlab-registry-gc-%s" % int(time.time())
|
||||
steps = []
|
||||
try:
|
||||
for name in original_crons:
|
||||
result = patch_cronjob_suspend(name, True)
|
||||
steps.append({"step": "suspend-cronjob", "name": name, "result": bounded(result)})
|
||||
if result["exitCode"] != 0:
|
||||
raise RuntimeError("failed to suspend cronjob %s" % name)
|
||||
idle_after_suspend = wait_no_active_hwlab_ci(180)
|
||||
steps.append({"step": "idle-after-suspend", "result": idle_after_suspend})
|
||||
if not idle_after_suspend.get("ok"):
|
||||
raise RuntimeError("refusing registry maintenance because hwlab-ci did not become idle after suspend")
|
||||
|
||||
scale_down = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=0"], 60)
|
||||
steps.append({"step": "scale-registry-down", "result": bounded(scale_down)})
|
||||
if scale_down["exitCode"] != 0:
|
||||
raise RuntimeError("failed to scale registry down")
|
||||
waited_down = wait_registry_pod_count(0, 120)
|
||||
steps.append({"step": "wait-registry-down", "result": waited_down})
|
||||
if not waited_down.get("ok"):
|
||||
raise RuntimeError("registry pod did not scale down")
|
||||
|
||||
overrides = {
|
||||
"apiVersion": "v1",
|
||||
"spec": {
|
||||
"restartPolicy": "Never",
|
||||
"containers": [{
|
||||
"name": "registry-gc",
|
||||
"image": "registry:2.8.3",
|
||||
"command": ["registry", "garbage-collect", "/etc/docker/registry/config.yml"],
|
||||
"volumeMounts": [{"name": "storage", "mountPath": "/var/lib/registry"}],
|
||||
}],
|
||||
"volumes": [{"name": "storage", "hostPath": {"path": REGISTRY_ROOT, "type": "DirectoryOrCreate"}}],
|
||||
},
|
||||
}
|
||||
run_gc = kctl(["-n", "hwlab-ci", "run", gc_name, "--restart=Never", "--image=registry:2.8.3", "--overrides=%s" % json.dumps(overrides)], 60)
|
||||
steps.append({"step": "start-registry-gc-pod", "result": bounded(run_gc), "pod": gc_name})
|
||||
if run_gc["exitCode"] != 0:
|
||||
raise RuntimeError("failed to start registry GC pod")
|
||||
waited_gc = wait_pod_terminal(gc_name, 900)
|
||||
steps.append({"step": "wait-registry-gc", "result": waited_gc})
|
||||
logs = kctl(["-n", "hwlab-ci", "logs", gc_name], 120)
|
||||
steps.append({"step": "registry-gc-logs", "result": bounded(logs)})
|
||||
if not waited_gc.get("ok"):
|
||||
raise RuntimeError("registry GC pod did not complete successfully")
|
||||
finally:
|
||||
cleanup_gc = kctl(["-n", "hwlab-ci", "delete", "pod", gc_name, "--ignore-not-found=true"], 60)
|
||||
steps.append({"step": "delete-registry-gc-pod", "result": bounded(cleanup_gc)})
|
||||
scale_up = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=%s" % int(deployment.get("replicas") or 1)], 60)
|
||||
steps.append({"step": "scale-registry-up", "result": bounded(scale_up)})
|
||||
rollout = kctl(["-n", "hwlab-ci", "rollout", "status", "deploy/hwlab-registry", "--timeout=180s"], 200)
|
||||
steps.append({"step": "wait-registry-rollout", "result": bounded(rollout)})
|
||||
for name, was_suspended in original_crons.items():
|
||||
restore = patch_cronjob_suspend(name, was_suspended)
|
||||
steps.append({"step": "restore-cronjob", "name": name, "suspend": was_suspended, "result": bounded(restore)})
|
||||
after = du_size(REGISTRY_ROOT, 60) or 0
|
||||
return {
|
||||
"reclaimedBytes": max(0, before - after),
|
||||
"commandOutput": {
|
||||
"message": "official registry garbage-collect only; no additional tag deletion",
|
||||
"diskBeforeBytes": before,
|
||||
"diskAfterBytes": after,
|
||||
"steps": steps[-12:],
|
||||
},
|
||||
}
|
||||
|
||||
def start_registry_retention_job(mode):
|
||||
job_id = "g14-registry-%s-%s" % (int(time.time()), os.getpid())
|
||||
paths = job_paths(job_id)
|
||||
started_at = now_iso()
|
||||
initial = {
|
||||
"ok": True,
|
||||
"action": "gc remote status",
|
||||
"providerId": PROVIDER_ID,
|
||||
"jobId": job_id,
|
||||
"status": "running",
|
||||
"kind": "hwlab-registry-retention-gc" if mode == "retention" else "hwlab-registry-garbage-collect",
|
||||
"mode": mode,
|
||||
"startedAt": started_at,
|
||||
"statePath": paths["state"],
|
||||
"logPath": paths["log"],
|
||||
"options": OPTIONS,
|
||||
}
|
||||
write_json_atomic(paths["state"], initial)
|
||||
pid = os.fork()
|
||||
if pid != 0:
|
||||
return {
|
||||
"status": "started",
|
||||
"reclaimedBytes": None,
|
||||
"commandOutput": {
|
||||
"jobId": job_id,
|
||||
"pid": pid,
|
||||
"statePath": paths["state"],
|
||||
"logPath": paths["log"],
|
||||
"statusCommand": "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id),
|
||||
"message": "registry retention GC is running as a detached remote job",
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
os.setsid()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
devnull = os.open(os.devnull, os.O_RDONLY)
|
||||
os.dup2(devnull, 0)
|
||||
os.close(devnull)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
log_handle = open(paths["log"], "a", encoding="utf-8", buffering=1)
|
||||
os.dup2(log_handle.fileno(), 1)
|
||||
os.dup2(log_handle.fileno(), 2)
|
||||
except Exception:
|
||||
log_handle = None
|
||||
try:
|
||||
print("[%s] starting HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True)
|
||||
result = execute_registry_retention() if mode == "retention" else execute_registry_garbage_collect_only()
|
||||
payload = dict(initial)
|
||||
payload.update({
|
||||
"status": "succeeded",
|
||||
"finishedAt": now_iso(),
|
||||
"result": result,
|
||||
"diskAfter": df_snapshot(),
|
||||
"clusterAfter": cluster_preflight(),
|
||||
})
|
||||
write_json_atomic(paths["state"], payload)
|
||||
print("[%s] completed HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True)
|
||||
os._exit(0)
|
||||
except Exception as exc:
|
||||
payload = dict(initial)
|
||||
payload.update({
|
||||
"ok": False,
|
||||
"status": "failed",
|
||||
"finishedAt": now_iso(),
|
||||
"error": str(exc),
|
||||
"diskAfter": df_snapshot(),
|
||||
"clusterAfter": cluster_preflight(),
|
||||
})
|
||||
try:
|
||||
write_json_atomic(paths["state"], payload)
|
||||
except Exception:
|
||||
pass
|
||||
print("[%s] failed HWLAB registry %s job %s: %s" % (now_iso(), mode, job_id, exc), flush=True)
|
||||
os._exit(1)
|
||||
finally:
|
||||
try:
|
||||
if log_handle:
|
||||
log_handle.close()
|
||||
except Exception:
|
||||
pass
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,57 @@
|
||||
def configured_observe_roots():
|
||||
roots = config_list(MEMORY_CONFIG, "observeStateRoots", config_list(MEMORY_CONFIG, "webObserveRoots", []))
|
||||
return [os.path.abspath(item) for item in roots if isinstance(item, str) and item.startswith("/")]
|
||||
|
||||
def is_direct_observe_run_path(path):
|
||||
resolved = os.path.abspath(path)
|
||||
for root in configured_observe_roots():
|
||||
if os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def path_has_open_fd(path):
|
||||
resolved = os.path.realpath(path)
|
||||
prefix = resolved.rstrip("/") + "/"
|
||||
proc_root = "/proc"
|
||||
try:
|
||||
pids = [name for name in os.listdir(proc_root) if name.isdigit()]
|
||||
except OSError:
|
||||
return True
|
||||
for pid in pids:
|
||||
base = os.path.join(proc_root, pid)
|
||||
for name in ["cwd", "root"]:
|
||||
try:
|
||||
target = os.path.realpath(os.readlink(os.path.join(base, name)))
|
||||
except OSError:
|
||||
continue
|
||||
if target == resolved or target.startswith(prefix):
|
||||
return True
|
||||
fd_dir = os.path.join(base, "fd")
|
||||
try:
|
||||
fds = os.listdir(fd_dir)
|
||||
except OSError:
|
||||
continue
|
||||
for fd in fds:
|
||||
try:
|
||||
target = os.path.realpath(os.readlink(os.path.join(fd_dir, fd)))
|
||||
except OSError:
|
||||
continue
|
||||
if target == resolved or target.startswith(prefix):
|
||||
return True
|
||||
return False
|
||||
|
||||
def assert_web_observe_candidate(path):
|
||||
resolved = os.path.abspath(path)
|
||||
if not is_direct_observe_run_path(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe path outside configured direct run roots: %s" % path)
|
||||
if os.path.islink(resolved) or not os.path.isdir(resolved):
|
||||
raise RuntimeError("refusing to remove non-directory or symlink web-observe path: %s" % path)
|
||||
stale_hours = config_float(MEMORY_CONFIG, "staleRunMaxAgeHours", 6.0, minimum=0.0)
|
||||
record = observe_run_record(resolved, stale_hours)
|
||||
if record.get("pidAlive"):
|
||||
raise RuntimeError("refusing to remove active web-observe run with live pid: %s" % path)
|
||||
if not record.get("staleSignal"):
|
||||
raise RuntimeError("refusing to remove web-observe run without stale signal: %s" % path)
|
||||
if path_has_open_fd(resolved):
|
||||
raise RuntimeError("refusing to remove web-observe run with open fd/cwd reference: %s" % path)
|
||||
return record
|
||||
+88
-2945
File diff suppressed because it is too large
Load Diff
+12
-7
@@ -58,7 +58,8 @@ export function rootHelp(): unknown {
|
||||
{ command: "decision requirement list|create|show|update|upsert [id|docNo] [--title text] [--body-file path] [--type external_goal|internal_goal|goal|decision|blocker|debt|experiment] [--doc-no DC-...] [--doc-type ...] [--doc-priority P0|P1|P2|P3] [--signer text] [--issued-at ISO]", description: "Manage productized requirement records over the PostgreSQL records model, excluding meeting records." },
|
||||
{ command: "decision show <id|docNo>", description: "Show one Decision Center record." },
|
||||
{ command: "deploy check|plan|apply [--file deploy.json|--env dev|prod] [--service id] [--commit full-sha] [--dry-run] [--force]", description: "Reconcile services from origin/master:deploy.json environments; --commit overrides one reviewed artifact consumer such as frontend for release/v1 validation or rollback. code-queue artifact consumption is dev-only." },
|
||||
{ command: "cicd branch-follower plan|apply|status|run-once|events|logs", description: "Deploy and inspect the YAML-first Kubernetes branch follower for HWLAB v0.3, AgentRun v0.2, and web-probe sentinel master without using host worktrees as source authority." },
|
||||
{ command: "cicd gitea-actions-poc plan|status", description: "Recommended CI/CD migration path for GH-1548/GH-1549: Gitea mirror and Gitea Actions driven orchestration with controlled Docker/BuildKit builder-plane, runtime plane 0 Docker and env reuse as a P0 no-regression contract." },
|
||||
{ command: "cicd branch-follower plan|apply|status|run-once|events|logs", description: "Deprecated migration-only Kubernetes branch follower controller; keep existing production status/debug during cutover, but do not add new self-maintained branch-following features." },
|
||||
{ command: "dev-env validate|prewarm-images", description: "Validate D601 unidesk-dev guardrails or prewarm dev foundation images into native k3s containerd through a bounded async job." },
|
||||
{ command: "artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service", description: "Manage the D601 host-managed CNCF Distribution registry and run pull-only artifact CD for supported services, including D601 direct, k3s-managed, and code-queue dev-only consumers." },
|
||||
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
|
||||
@@ -660,7 +661,7 @@ function agentRunHelpSummary(): unknown {
|
||||
|
||||
function platformInfraHelpSummary(): unknown {
|
||||
return {
|
||||
command: "platform-infra sub2api|egress-proxy|langbot|n8n|wechat-archive ...",
|
||||
command: "platform-infra sub2api|egress-proxy|langbot|n8n|wechat-archive|gitea ...",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api plan",
|
||||
@@ -688,8 +689,12 @@ function platformInfraHelpSummary(): unknown {
|
||||
"bun scripts/cli.ts platform-infra wechat-archive pull --remote-path /UniDesk/WeChatArchive/...",
|
||||
"bun scripts/cli.ts platform-infra wechat-archive wcf-host-status --full",
|
||||
"bun scripts/cli.ts platform-infra wechat-archive collector-status --full",
|
||||
"bun scripts/cli.ts platform-infra gitea plan --target JD01",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --confirm",
|
||||
"bun scripts/cli.ts platform-infra gitea status --target JD01",
|
||||
"bun scripts/cli.ts platform-infra gitea validate --target JD01",
|
||||
],
|
||||
description: "Operate G14 platform-infra services such as Sub2API, shared egress-proxy benchmarks, LangBot, n8n, WeChat archive workflows, and the YAML-controlled Codex pool.",
|
||||
description: "Operate platform-infra services such as Sub2API, shared egress-proxy benchmarks, LangBot, n8n, WeChat archive workflows, the YAML-controlled Codex pool, and internal Gitea for the GH-1548/GH-1549 CI/CD migration.",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -734,15 +739,15 @@ function webProbeHelpSummary(): unknown {
|
||||
|
||||
function cicdHelpSummary(): unknown {
|
||||
return {
|
||||
command: "cicd branch-follower plan|apply|status|run-once|events|logs",
|
||||
command: "cicd gitea-actions-poc plan|status | branch-follower ...",
|
||||
output: "text by default; use --json, --raw, or -o json|yaml for machine output",
|
||||
usage: [
|
||||
"bun scripts/cli.ts cicd gitea-actions-poc plan",
|
||||
"bun scripts/cli.ts cicd gitea-actions-poc status",
|
||||
"bun scripts/cli.ts cicd branch-follower plan",
|
||||
"bun scripts/cli.ts cicd branch-follower apply --confirm --wait",
|
||||
"bun scripts/cli.ts cicd branch-follower status",
|
||||
"bun scripts/cli.ts cicd branch-follower run-once --all --dry-run",
|
||||
],
|
||||
description: "YAML-first Kubernetes branch follower for three CI/CD running planes, with K8s state and adapter drill-down visibility.",
|
||||
description: "Gitea Actions driven CI/CD migration path; branch-follower remains deprecated migration-only status/debug during cutover.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
|
||||
json_tail() {
|
||||
name="$1"
|
||||
limit="${2:-2000}"
|
||||
python3 - "$tmp/$name" "$limit" <<'PY'
|
||||
import json, sys
|
||||
path, limit = sys.argv[1], int(sys.argv[2])
|
||||
try:
|
||||
data = open(path, encoding="utf-8", errors="replace").read()[-limit:]
|
||||
except FileNotFoundError:
|
||||
data = ""
|
||||
print(json.dumps(data))
|
||||
PY
|
||||
}
|
||||
|
||||
capture_json() {
|
||||
name="$1"
|
||||
shift
|
||||
"$@" -o json >"$tmp/$name.json" 2>"$tmp/$name.err"
|
||||
rc=$?
|
||||
printf '%s' "$rc" >"$tmp/$name.rc"
|
||||
}
|
||||
|
||||
run_apply() {
|
||||
manifest="$tmp/gitea.k8s.yaml"
|
||||
printf '%s' "$UNIDESK_GITEA_MANIFEST_B64" | base64 -d >"$manifest"
|
||||
dry_arg=""
|
||||
if [ "$UNIDESK_GITEA_DRY_RUN" = "1" ]; then
|
||||
dry_arg="--dry-run=server"
|
||||
fi
|
||||
timeout "$UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS" kubectl apply --server-side $dry_arg --force-conflicts --field-manager="$UNIDESK_GITEA_FIELD_MANAGER" -f "$manifest" >"$tmp/apply.out" 2>"$tmp/apply.err"
|
||||
apply_rc=$?
|
||||
if [ "$apply_rc" -eq 0 ] && [ "$UNIDESK_GITEA_WAIT" = "1" ] && [ "$UNIDESK_GITEA_DRY_RUN" != "1" ]; then
|
||||
timeout "$UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS" kubectl -n "$UNIDESK_GITEA_NAMESPACE" rollout status "statefulset/$UNIDESK_GITEA_STATEFULSET_NAME" --timeout="${UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS}s" >"$tmp/rollout.out" 2>"$tmp/rollout.err"
|
||||
rollout_rc=$?
|
||||
else
|
||||
rollout_rc=0
|
||||
: >"$tmp/rollout.out"
|
||||
if [ "$UNIDESK_GITEA_DRY_RUN" = "1" ]; then
|
||||
printf '%s\n' "dry-run: rollout wait skipped" >"$tmp/rollout.err"
|
||||
else
|
||||
printf '%s\n' "rollout wait not requested" >"$tmp/rollout.err"
|
||||
fi
|
||||
fi
|
||||
python3 - "$apply_rc" "$rollout_rc" "$tmp/apply.out" "$tmp/apply.err" "$tmp/rollout.out" "$tmp/rollout.err" <<'PY'
|
||||
import json, os, sys
|
||||
apply_rc, rollout_rc = int(sys.argv[1]), int(sys.argv[2])
|
||||
def text(path, limit=3000):
|
||||
try:
|
||||
return open(path, encoding="utf-8", errors="replace").read()[-limit:]
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
dry = os.environ.get("UNIDESK_GITEA_DRY_RUN") == "1"
|
||||
payload = {
|
||||
"ok": apply_rc == 0 and rollout_rc == 0,
|
||||
"target": os.environ.get("UNIDESK_GITEA_TARGET_ID"),
|
||||
"route": os.environ.get("UNIDESK_GITEA_ROUTE"),
|
||||
"namespace": os.environ.get("UNIDESK_GITEA_NAMESPACE"),
|
||||
"mode": "dry-run" if dry else "confirmed",
|
||||
"mutation": not dry,
|
||||
"image": os.environ.get("UNIDESK_GITEA_IMAGE"),
|
||||
"objects": {
|
||||
"statefulSet": os.environ.get("UNIDESK_GITEA_STATEFULSET_NAME"),
|
||||
"service": os.environ.get("UNIDESK_GITEA_SERVICE_NAME"),
|
||||
"networkPolicy": "allow-all",
|
||||
},
|
||||
"steps": {
|
||||
"apply": {"exitCode": apply_rc, "stdoutTail": text(sys.argv[3]), "stderrTail": text(sys.argv[4])},
|
||||
"rollout": {"exitCode": rollout_rc, "stdoutTail": text(sys.argv[5]), "stderrTail": text(sys.argv[6])},
|
||||
},
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if payload["ok"] else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
run_status() {
|
||||
selector="app.kubernetes.io/name=$UNIDESK_GITEA_APP_NAME,app.kubernetes.io/component=gitea"
|
||||
capture_json namespace kubectl get namespace "$UNIDESK_GITEA_NAMESPACE"
|
||||
capture_json networkpolicy kubectl -n "$UNIDESK_GITEA_NAMESPACE" get networkpolicy allow-all
|
||||
capture_json statefulset kubectl -n "$UNIDESK_GITEA_NAMESPACE" get statefulset "$UNIDESK_GITEA_STATEFULSET_NAME"
|
||||
capture_json service kubectl -n "$UNIDESK_GITEA_NAMESPACE" get service "$UNIDESK_GITEA_SERVICE_NAME"
|
||||
capture_json endpoints kubectl -n "$UNIDESK_GITEA_NAMESPACE" get endpoints "$UNIDESK_GITEA_SERVICE_NAME"
|
||||
capture_json pods kubectl -n "$UNIDESK_GITEA_NAMESPACE" get pods -l "$selector"
|
||||
capture_json pvc kubectl -n "$UNIDESK_GITEA_NAMESPACE" get pvc -l "$selector"
|
||||
capture_json events kubectl -n "$UNIDESK_GITEA_NAMESPACE" get events --sort-by=.lastTimestamp
|
||||
python3 - "$tmp" <<'PY'
|
||||
import json, os, sys
|
||||
tmp = sys.argv[1]
|
||||
def rc(name):
|
||||
try:
|
||||
return int(open(f"{tmp}/{name}.rc", encoding="utf-8").read() or "1")
|
||||
except FileNotFoundError:
|
||||
return 1
|
||||
def load(name):
|
||||
try:
|
||||
return json.load(open(f"{tmp}/{name}.json", encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
def items(name):
|
||||
data = load(name)
|
||||
if isinstance(data, dict) and isinstance(data.get("items"), list):
|
||||
return data["items"]
|
||||
if isinstance(data, dict) and data.get("kind") != "Status":
|
||||
return [data]
|
||||
return []
|
||||
def meta_name(item):
|
||||
return ((item or {}).get("metadata") or {}).get("name")
|
||||
def pod_ready(item):
|
||||
status = (item or {}).get("status") or {}
|
||||
return any(c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions", []))
|
||||
def pod_summary(item):
|
||||
status = (item or {}).get("status") or {}
|
||||
containers = status.get("containerStatuses") or []
|
||||
return {
|
||||
"name": meta_name(item),
|
||||
"phase": status.get("phase"),
|
||||
"ready": pod_ready(item),
|
||||
"restarts": sum(int(c.get("restartCount") or 0) for c in containers if isinstance(c, dict)),
|
||||
}
|
||||
def service_summary(item):
|
||||
spec = (item or {}).get("spec") or {}
|
||||
return {
|
||||
"name": meta_name(item),
|
||||
"type": spec.get("type"),
|
||||
"clusterIP": spec.get("clusterIP"),
|
||||
"ports": [{"name": p.get("name"), "port": p.get("port"), "targetPort": p.get("targetPort")} for p in spec.get("ports", [])],
|
||||
}
|
||||
def endpoint_ready(item):
|
||||
subsets = ((item or {}).get("subsets") or [])
|
||||
return any(s.get("addresses") for s in subsets if isinstance(s, dict))
|
||||
def pvc_summary(item):
|
||||
spec = (item or {}).get("spec") or {}
|
||||
status = (item or {}).get("status") or {}
|
||||
return {
|
||||
"name": meta_name(item),
|
||||
"phase": status.get("phase"),
|
||||
"storageClassName": spec.get("storageClassName"),
|
||||
"capacity": (status.get("capacity") or {}).get("storage"),
|
||||
}
|
||||
def event_tail(limit=8):
|
||||
result = []
|
||||
for item in items("events")[-limit:]:
|
||||
result.append({
|
||||
"type": item.get("type"),
|
||||
"reason": item.get("reason"),
|
||||
"object": f"{((item.get('involvedObject') or {}).get('kind') or '-')}/{((item.get('involvedObject') or {}).get('name') or '-')}",
|
||||
"message": (item.get("message") or "")[:300],
|
||||
"lastTimestamp": item.get("lastTimestamp") or item.get("eventTime"),
|
||||
})
|
||||
return result
|
||||
sts = load("statefulset") or {}
|
||||
sts_spec = sts.get("spec") or {}
|
||||
sts_status = sts.get("status") or {}
|
||||
desired = int(sts_spec.get("replicas") or 0)
|
||||
ready_replicas = int(sts_status.get("readyReplicas") or 0)
|
||||
pods = [pod_summary(item) for item in items("pods")]
|
||||
service = service_summary(load("service") or {})
|
||||
endpoint_ok = endpoint_ready(load("endpoints") or {})
|
||||
ready = rc("namespace") == 0 and rc("networkpolicy") == 0 and rc("service") == 0 and desired > 0 and ready_replicas >= desired and endpoint_ok
|
||||
payload = {
|
||||
"ok": True,
|
||||
"ready": ready,
|
||||
"target": os.environ.get("UNIDESK_GITEA_TARGET_ID"),
|
||||
"route": os.environ.get("UNIDESK_GITEA_ROUTE"),
|
||||
"namespace": os.environ.get("UNIDESK_GITEA_NAMESPACE"),
|
||||
"image": os.environ.get("UNIDESK_GITEA_IMAGE"),
|
||||
"serviceDns": f"{os.environ.get('UNIDESK_GITEA_SERVICE_NAME')}.{os.environ.get('UNIDESK_GITEA_NAMESPACE')}.svc.cluster.local:{os.environ.get('UNIDESK_GITEA_HTTP_PORT')}",
|
||||
"networkPolicy": {"allowAllPresent": rc("networkpolicy") == 0},
|
||||
"statefulSet": {"name": meta_name(sts), "desired": desired, "readyReplicas": ready_replicas, "updatedReplicas": sts_status.get("updatedReplicas")},
|
||||
"service": service,
|
||||
"endpointsReady": endpoint_ok,
|
||||
"pods": pods,
|
||||
"pvcs": [pvc_summary(item) for item in items("pvc")],
|
||||
"eventsTail": event_tail(),
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
sys.exit(0)
|
||||
PY
|
||||
}
|
||||
|
||||
run_validate() {
|
||||
proxy_path="/api/v1/namespaces/$UNIDESK_GITEA_NAMESPACE/services/http:$UNIDESK_GITEA_SERVICE_NAME:$UNIDESK_GITEA_HTTP_PORT/proxy$UNIDESK_GITEA_HEALTH_PATH"
|
||||
kubectl get --raw "$proxy_path" >"$tmp/health.out" 2>"$tmp/health.err"
|
||||
health_rc=$?
|
||||
capture_json endpoints kubectl -n "$UNIDESK_GITEA_NAMESPACE" get endpoints "$UNIDESK_GITEA_SERVICE_NAME"
|
||||
python3 - "$health_rc" "$tmp/health.out" "$tmp/health.err" "$tmp/endpoints.json" "$tmp/endpoints.rc" <<'PY'
|
||||
import json, os, sys
|
||||
health_rc = int(sys.argv[1])
|
||||
def text(path, limit=2000):
|
||||
try:
|
||||
return open(path, encoding="utf-8", errors="replace").read()[-limit:]
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
body = text(sys.argv[2], 4000)
|
||||
try:
|
||||
parsed = json.loads(body)
|
||||
except Exception:
|
||||
parsed = None
|
||||
try:
|
||||
endpoint_rc = int(open(sys.argv[5], encoding="utf-8").read() or "1")
|
||||
except Exception:
|
||||
endpoint_rc = 1
|
||||
try:
|
||||
endpoints = json.load(open(sys.argv[4], encoding="utf-8"))
|
||||
except Exception:
|
||||
endpoints = {}
|
||||
subsets = endpoints.get("subsets") if isinstance(endpoints, dict) else []
|
||||
endpoints_ready = any(isinstance(s, dict) and s.get("addresses") for s in (subsets or []))
|
||||
health_status = parsed.get("status") if isinstance(parsed, dict) else None
|
||||
ok = health_rc == 0 and endpoint_rc == 0 and endpoints_ready and health_status == "pass"
|
||||
payload = {
|
||||
"ok": ok,
|
||||
"target": os.environ.get("UNIDESK_GITEA_TARGET_ID"),
|
||||
"route": os.environ.get("UNIDESK_GITEA_ROUTE"),
|
||||
"namespace": os.environ.get("UNIDESK_GITEA_NAMESPACE"),
|
||||
"service": os.environ.get("UNIDESK_GITEA_SERVICE_NAME"),
|
||||
"serviceProxyPath": f"/api/v1/namespaces/{os.environ.get('UNIDESK_GITEA_NAMESPACE')}/services/http:{os.environ.get('UNIDESK_GITEA_SERVICE_NAME')}:{os.environ.get('UNIDESK_GITEA_HTTP_PORT')}/proxy{os.environ.get('UNIDESK_GITEA_HEALTH_PATH')}",
|
||||
"health": {
|
||||
"exitCode": health_rc,
|
||||
"status": health_status,
|
||||
"description": parsed.get("description") if isinstance(parsed, dict) else None,
|
||||
"bodyBytes": len(body.encode("utf-8")),
|
||||
"stderrTail": text(sys.argv[3]),
|
||||
},
|
||||
"endpointsReady": endpoints_ready,
|
||||
"valuesPrinted": False,
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
case "$UNIDESK_GITEA_ACTION" in
|
||||
apply) run_apply ;;
|
||||
status) run_status ;;
|
||||
validate) run_validate ;;
|
||||
*) printf '{"ok":false,"error":"unsupported-gitea-remote-action"}\n'; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,898 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { rootPath } from "./config";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import {
|
||||
capture,
|
||||
compactCapture,
|
||||
createYamlFieldReader,
|
||||
parseJsonOutput,
|
||||
readYamlRecord,
|
||||
shQuote,
|
||||
} from "./platform-infra-ops-library";
|
||||
|
||||
const configFile = rootPath("config", "platform-infra", "gitea.yaml");
|
||||
const configLabel = "config/platform-infra/gitea.yaml";
|
||||
const remoteScriptFile = rootPath("scripts", "src", "platform-infra-gitea-remote.sh");
|
||||
const fieldManager = "unidesk-platform-infra-gitea";
|
||||
const y = createYamlFieldReader(configLabel);
|
||||
|
||||
interface GiteaTarget {
|
||||
id: string;
|
||||
route: string;
|
||||
namespace: string;
|
||||
role: string;
|
||||
enabled: boolean;
|
||||
createNamespace: boolean;
|
||||
storageClassName: string;
|
||||
}
|
||||
|
||||
interface GiteaConfig {
|
||||
version: number;
|
||||
kind: "platform-infra-gitea";
|
||||
metadata: {
|
||||
id: string;
|
||||
owner: string;
|
||||
spec: string;
|
||||
relatedIssues: number[];
|
||||
};
|
||||
defaults: {
|
||||
targetId: string;
|
||||
};
|
||||
migration: {
|
||||
role: string;
|
||||
replaces: string;
|
||||
parentConfigRef: string;
|
||||
envReusePolicy: string;
|
||||
buildPlane: string;
|
||||
runtimePlane: string;
|
||||
};
|
||||
targets: GiteaTarget[];
|
||||
app: {
|
||||
name: string;
|
||||
statefulSetName: string;
|
||||
serviceName: string;
|
||||
replicas: number;
|
||||
image: {
|
||||
repository: string;
|
||||
tag: string;
|
||||
pullPolicy: "Always" | "IfNotPresent" | "Never";
|
||||
};
|
||||
service: {
|
||||
type: "ClusterIP";
|
||||
httpPort: number;
|
||||
sshPort: number;
|
||||
};
|
||||
server: {
|
||||
domain: string;
|
||||
rootUrl: string;
|
||||
sshDomain: string;
|
||||
protocol: "http" | "https";
|
||||
startSshServer: boolean;
|
||||
};
|
||||
database: {
|
||||
type: "sqlite3";
|
||||
path: string;
|
||||
};
|
||||
actions: {
|
||||
enabled: boolean;
|
||||
};
|
||||
registration: {
|
||||
disabled: boolean;
|
||||
};
|
||||
storage: {
|
||||
data: { size: string; mountPath: string };
|
||||
config: { size: string; mountPath: string };
|
||||
};
|
||||
securityContext: {
|
||||
runAsUser: number;
|
||||
runAsGroup: number;
|
||||
fsGroup: number;
|
||||
};
|
||||
resources: {
|
||||
requests: { cpu: string; memory: string };
|
||||
limits: { cpu: string; memory: string };
|
||||
};
|
||||
probes: {
|
||||
healthPath: string;
|
||||
initialDelaySeconds: number;
|
||||
periodSeconds: number;
|
||||
timeoutSeconds: number;
|
||||
failureThreshold: number;
|
||||
};
|
||||
};
|
||||
validation: {
|
||||
waitTimeoutSeconds: number;
|
||||
healthPath: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface CommonOptions {
|
||||
targetId: string | null;
|
||||
full: boolean;
|
||||
raw: boolean;
|
||||
}
|
||||
|
||||
interface ApplyOptions extends CommonOptions {
|
||||
confirm: boolean;
|
||||
dryRun: boolean;
|
||||
wait: boolean;
|
||||
}
|
||||
|
||||
export async function runPlatformInfraGiteaCommand(config: UniDeskConfig, args: string[]): Promise<Record<string, unknown> | RenderedCliResult> {
|
||||
const [action = "plan"] = args;
|
||||
if (action === "help" || action === "--help") return giteaHelp();
|
||||
if (action === "plan") {
|
||||
const options = parseCommonOptions(args.slice(1));
|
||||
const result = plan(options);
|
||||
return options.full || options.raw ? result : renderPlan(result);
|
||||
}
|
||||
if (action === "apply") {
|
||||
const options = parseApplyOptions(args.slice(1));
|
||||
const result = await apply(config, options);
|
||||
return options.full || options.raw ? result : renderApply(result);
|
||||
}
|
||||
if (action === "status") {
|
||||
const options = parseCommonOptions(args.slice(1));
|
||||
const result = await status(config, options);
|
||||
return options.full || options.raw ? result : renderStatus(result);
|
||||
}
|
||||
if (action === "validate") return await validate(config, parseCommonOptions(args.slice(1)));
|
||||
return {
|
||||
ok: false,
|
||||
error: "unsupported-platform-infra-gitea-command",
|
||||
args,
|
||||
help: giteaHelp(),
|
||||
};
|
||||
}
|
||||
|
||||
export function giteaHelp(): Record<string, unknown> {
|
||||
return {
|
||||
command: "platform-infra gitea plan|apply|status|validate",
|
||||
configTruth: configLabel,
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra gitea plan --target JD01",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --confirm",
|
||||
"bun scripts/cli.ts platform-infra gitea status --target JD01 [--full|--raw]",
|
||||
"bun scripts/cli.ts platform-infra gitea validate --target JD01 [--full|--raw]",
|
||||
],
|
||||
boundary: "Gitea is installed as an internal ClusterIP source-authority service for GH-1548/GH-1549; runner registration and repository mirror bootstrap are later controlled stages.",
|
||||
};
|
||||
}
|
||||
|
||||
function readGiteaConfig(): GiteaConfig {
|
||||
const root = readYamlRecord<Record<string, unknown>>(configFile, "platform-infra-gitea");
|
||||
const version = y.integerField(root, "version", "");
|
||||
if (version !== 1) throw new Error(`${configLabel}.version must be 1`);
|
||||
const metadata = y.objectField(root, "metadata", "");
|
||||
const defaults = y.objectField(root, "defaults", "");
|
||||
const migration = y.objectField(root, "migration", "");
|
||||
const app = y.objectField(root, "app", "");
|
||||
const image = y.objectField(app, "image", "app");
|
||||
const service = y.objectField(app, "service", "app");
|
||||
const server = y.objectField(app, "server", "app");
|
||||
const database = y.objectField(app, "database", "app");
|
||||
const actions = y.objectField(app, "actions", "app");
|
||||
const registration = y.objectField(app, "registration", "app");
|
||||
const storage = y.objectField(app, "storage", "app");
|
||||
const dataStorage = y.objectField(storage, "data", "app.storage");
|
||||
const configStorage = y.objectField(storage, "config", "app.storage");
|
||||
const securityContext = y.objectField(app, "securityContext", "app");
|
||||
const resources = y.objectField(app, "resources", "app");
|
||||
const requests = y.objectField(resources, "requests", "app.resources");
|
||||
const limits = y.objectField(resources, "limits", "app.resources");
|
||||
const probes = y.objectField(app, "probes", "app");
|
||||
const validation = y.objectField(root, "validation", "");
|
||||
const parsed: GiteaConfig = {
|
||||
version,
|
||||
kind: "platform-infra-gitea",
|
||||
metadata: {
|
||||
id: y.stringField(metadata, "id", "metadata"),
|
||||
owner: y.stringField(metadata, "owner", "metadata"),
|
||||
spec: y.stringField(metadata, "spec", "metadata"),
|
||||
relatedIssues: y.numberArrayField(metadata, "relatedIssues", "metadata"),
|
||||
},
|
||||
defaults: {
|
||||
targetId: y.stringField(defaults, "targetId", "defaults"),
|
||||
},
|
||||
migration: {
|
||||
role: y.stringField(migration, "role", "migration"),
|
||||
replaces: y.stringField(migration, "replaces", "migration"),
|
||||
parentConfigRef: y.stringField(migration, "parentConfigRef", "migration"),
|
||||
envReusePolicy: y.stringField(migration, "envReusePolicy", "migration"),
|
||||
buildPlane: y.stringField(migration, "buildPlane", "migration"),
|
||||
runtimePlane: y.stringField(migration, "runtimePlane", "migration"),
|
||||
},
|
||||
targets: y.arrayOfRecords(root.targets, "targets").map(parseTarget),
|
||||
app: {
|
||||
name: y.kubernetesNameField(app, "name", "app"),
|
||||
statefulSetName: y.kubernetesNameField(app, "statefulSetName", "app"),
|
||||
serviceName: y.kubernetesNameField(app, "serviceName", "app"),
|
||||
replicas: positiveInteger(app, "replicas", "app"),
|
||||
image: {
|
||||
repository: y.stringField(image, "repository", "app.image"),
|
||||
tag: y.stringField(image, "tag", "app.image"),
|
||||
pullPolicy: y.enumField(image, "pullPolicy", "app.image", ["Always", "IfNotPresent", "Never"] as const),
|
||||
},
|
||||
service: {
|
||||
type: y.enumField(service, "type", "app.service", ["ClusterIP"] as const),
|
||||
httpPort: y.portField(service, "httpPort", "app.service"),
|
||||
sshPort: y.portField(service, "sshPort", "app.service"),
|
||||
},
|
||||
server: {
|
||||
domain: y.hostField(server, "domain", "app.server"),
|
||||
rootUrl: urlField(server, "rootUrl", "app.server"),
|
||||
sshDomain: y.hostField(server, "sshDomain", "app.server"),
|
||||
protocol: y.enumField(server, "protocol", "app.server", ["http", "https"] as const),
|
||||
startSshServer: y.booleanField(server, "startSshServer", "app.server"),
|
||||
},
|
||||
database: {
|
||||
type: y.enumField(database, "type", "app.database", ["sqlite3"] as const),
|
||||
path: y.absolutePathField(database, "path", "app.database"),
|
||||
},
|
||||
actions: {
|
||||
enabled: y.booleanField(actions, "enabled", "app.actions"),
|
||||
},
|
||||
registration: {
|
||||
disabled: y.booleanField(registration, "disabled", "app.registration"),
|
||||
},
|
||||
storage: {
|
||||
data: { size: quantity(dataStorage, "size", "app.storage.data"), mountPath: y.absolutePathField(dataStorage, "mountPath", "app.storage.data") },
|
||||
config: { size: quantity(configStorage, "size", "app.storage.config"), mountPath: y.absolutePathField(configStorage, "mountPath", "app.storage.config") },
|
||||
},
|
||||
securityContext: {
|
||||
runAsUser: positiveInteger(securityContext, "runAsUser", "app.securityContext"),
|
||||
runAsGroup: positiveInteger(securityContext, "runAsGroup", "app.securityContext"),
|
||||
fsGroup: positiveInteger(securityContext, "fsGroup", "app.securityContext"),
|
||||
},
|
||||
resources: {
|
||||
requests: { cpu: y.stringField(requests, "cpu", "app.resources.requests"), memory: quantity(requests, "memory", "app.resources.requests") },
|
||||
limits: { cpu: y.stringField(limits, "cpu", "app.resources.limits"), memory: quantity(limits, "memory", "app.resources.limits") },
|
||||
},
|
||||
probes: {
|
||||
healthPath: y.apiPathField(probes, "healthPath", "app.probes"),
|
||||
initialDelaySeconds: positiveInteger(probes, "initialDelaySeconds", "app.probes"),
|
||||
periodSeconds: positiveInteger(probes, "periodSeconds", "app.probes"),
|
||||
timeoutSeconds: positiveInteger(probes, "timeoutSeconds", "app.probes"),
|
||||
failureThreshold: positiveInteger(probes, "failureThreshold", "app.probes"),
|
||||
},
|
||||
},
|
||||
validation: {
|
||||
waitTimeoutSeconds: boundedTimeout(validation, "waitTimeoutSeconds", "validation"),
|
||||
healthPath: y.apiPathField(validation, "healthPath", "validation"),
|
||||
},
|
||||
};
|
||||
validateConfig(parsed);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseTarget(record: Record<string, unknown>, index: number): GiteaTarget {
|
||||
const path = `targets[${index}]`;
|
||||
return {
|
||||
id: y.stringField(record, "id", path),
|
||||
route: y.stringField(record, "route", path),
|
||||
namespace: y.kubernetesNameField(record, "namespace", path),
|
||||
role: y.stringField(record, "role", path),
|
||||
enabled: y.booleanField(record, "enabled", path),
|
||||
createNamespace: y.booleanField(record, "createNamespace", path),
|
||||
storageClassName: y.stringField(record, "storageClassName", path),
|
||||
};
|
||||
}
|
||||
|
||||
function validateConfig(gitea: GiteaConfig): void {
|
||||
resolveTarget(gitea, gitea.defaults.targetId);
|
||||
if (!/^docker\.gitea\.com\/gitea$/u.test(gitea.app.image.repository)) throw new Error(`${configLabel}.app.image.repository must use the official Gitea image registry`);
|
||||
if (!/-rootless$/u.test(gitea.app.image.tag)) throw new Error(`${configLabel}.app.image.tag must use a rootless Gitea image`);
|
||||
if (gitea.app.service.type !== "ClusterIP") throw new Error(`${configLabel}.app.service.type must stay ClusterIP`);
|
||||
if (!gitea.app.actions.enabled) throw new Error(`${configLabel}.app.actions.enabled must be true for GH-1548/GH-1549`);
|
||||
if (!gitea.app.registration.disabled) throw new Error(`${configLabel}.app.registration.disabled must be true for the internal POC service`);
|
||||
if (gitea.app.probes.healthPath !== gitea.validation.healthPath) throw new Error(`${configLabel}.app.probes.healthPath must match validation.healthPath`);
|
||||
}
|
||||
|
||||
function resolveTarget(gitea: GiteaConfig, targetId: string | null): GiteaTarget {
|
||||
const resolved = targetId ?? gitea.defaults.targetId;
|
||||
const target = gitea.targets.find((item) => item.id.toLowerCase() === resolved.toLowerCase());
|
||||
if (target === undefined) throw new Error(`unknown gitea target ${resolved}; known targets: ${gitea.targets.map((item) => item.id).join(", ")}`);
|
||||
if (!target.enabled) throw new Error(`gitea target ${target.id} is disabled in ${configLabel}`);
|
||||
return target;
|
||||
}
|
||||
|
||||
function plan(options: CommonOptions): Record<string, unknown> {
|
||||
const gitea = readGiteaConfig();
|
||||
const target = resolveTarget(gitea, options.targetId);
|
||||
const manifest = renderManifest(gitea, target);
|
||||
const policy = policyChecks(gitea, target, manifest);
|
||||
return {
|
||||
ok: policy.every((check) => check.ok),
|
||||
action: "platform-infra-gitea-plan",
|
||||
mutation: false,
|
||||
config: configSummary(gitea, target),
|
||||
renderPlan: {
|
||||
target: targetSummary(target),
|
||||
objects: manifestObjectSummary(manifest),
|
||||
},
|
||||
policy,
|
||||
next: nextCommands(target.id),
|
||||
};
|
||||
}
|
||||
|
||||
async function apply(config: UniDeskConfig, options: ApplyOptions): Promise<Record<string, unknown>> {
|
||||
const gitea = readGiteaConfig();
|
||||
const target = resolveTarget(gitea, options.targetId);
|
||||
const manifest = renderManifest(gitea, target);
|
||||
const policy = policyChecks(gitea, target, manifest);
|
||||
if (!policy.every((check) => check.ok)) return { ok: false, action: "platform-infra-gitea-apply", mode: "policy-blocked", mutation: false, policy };
|
||||
const result = await capture(config, target.route, ["sh"], remoteScript("apply", gitea, target, manifest, options));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||
action: "platform-infra-gitea-apply",
|
||||
mode: options.dryRun ? "dry-run" : "confirmed",
|
||||
mutation: !options.dryRun,
|
||||
target: targetSummary(target),
|
||||
config: compactConfigSummary(gitea, target),
|
||||
policy,
|
||||
remote: parsed ?? compactCapture(result, { full: true }),
|
||||
next: nextCommands(target.id),
|
||||
};
|
||||
}
|
||||
|
||||
async function status(config: UniDeskConfig, options: CommonOptions): Promise<Record<string, unknown>> {
|
||||
const gitea = readGiteaConfig();
|
||||
const target = resolveTarget(gitea, options.targetId);
|
||||
const result = await capture(config, target.route, ["sh"], remoteScript("status", gitea, target, "", { ...options, confirm: false, dryRun: true, wait: false }));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
const summary = parsed === null ? null : statusSummary(parsed);
|
||||
return {
|
||||
ok: result.exitCode === 0 && summary?.ready === true,
|
||||
action: "platform-infra-gitea-status",
|
||||
mutation: false,
|
||||
target: targetSummary(target),
|
||||
config: configSummary(gitea, target),
|
||||
summary,
|
||||
remote: options.raw ? parsed : options.full ? parsed : summary ?? compactCapture(result, { full: true }),
|
||||
next: {
|
||||
apply: `bun scripts/cli.ts platform-infra gitea apply --target ${target.id} --confirm`,
|
||||
validate: `bun scripts/cli.ts platform-infra gitea validate --target ${target.id}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function validate(config: UniDeskConfig, options: CommonOptions): Promise<Record<string, unknown>> {
|
||||
const gitea = readGiteaConfig();
|
||||
const target = resolveTarget(gitea, options.targetId);
|
||||
const result = await capture(config, target.route, ["sh"], remoteScript("validate", gitea, target, "", { ...options, confirm: false, dryRun: true, wait: false }));
|
||||
const parsed = parseJsonOutput(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||
action: "platform-infra-gitea-validate",
|
||||
mutation: false,
|
||||
target: targetSummary(target),
|
||||
config: compactConfigSummary(gitea, target),
|
||||
validation: parsed ?? null,
|
||||
remote: options.raw && parsed !== null ? parsed : compactCapture(result, { full: options.full || result.exitCode !== 0 }),
|
||||
};
|
||||
}
|
||||
|
||||
function renderManifest(gitea: GiteaConfig, target: GiteaTarget): string {
|
||||
const app = gitea.app;
|
||||
const image = `${app.image.repository}:${app.image.tag}`;
|
||||
const labels = ` app.kubernetes.io/name: ${app.name}
|
||||
app.kubernetes.io/component: gitea
|
||||
app.kubernetes.io/part-of: devops-infra
|
||||
app.kubernetes.io/managed-by: unidesk`;
|
||||
return `apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: ${target.namespace}
|
||||
labels:
|
||||
app.kubernetes.io/name: devops-infra
|
||||
app.kubernetes.io/managed-by: unidesk
|
||||
unidesk.ai/runtime-node: ${target.id}
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: allow-all
|
||||
namespace: ${target.namespace}
|
||||
labels:
|
||||
${labels}
|
||||
spec:
|
||||
podSelector: {}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
- {}
|
||||
egress:
|
||||
- {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ${app.serviceName}
|
||||
namespace: ${target.namespace}
|
||||
labels:
|
||||
${labels}
|
||||
annotations:
|
||||
unidesk.ai/spec: ${yamlQuote(gitea.metadata.spec)}
|
||||
unidesk.ai/parent-config-ref: ${yamlQuote(gitea.migration.parentConfigRef)}
|
||||
spec:
|
||||
type: ${app.service.type}
|
||||
selector:
|
||||
app.kubernetes.io/name: ${app.name}
|
||||
app.kubernetes.io/component: gitea
|
||||
ports:
|
||||
- name: http
|
||||
port: ${app.service.httpPort}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
- name: ssh
|
||||
port: ${app.service.sshPort}
|
||||
targetPort: ssh
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: ${app.statefulSetName}
|
||||
namespace: ${target.namespace}
|
||||
labels:
|
||||
${labels}
|
||||
annotations:
|
||||
unidesk.ai/spec: ${yamlQuote(gitea.metadata.spec)}
|
||||
unidesk.ai/env-reuse-policy: ${yamlQuote(gitea.migration.envReusePolicy)}
|
||||
spec:
|
||||
serviceName: ${app.serviceName}
|
||||
replicas: ${app.replicas}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ${app.name}
|
||||
app.kubernetes.io/component: gitea
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: ${app.name}
|
||||
app.kubernetes.io/component: gitea
|
||||
app.kubernetes.io/part-of: devops-infra
|
||||
app.kubernetes.io/managed-by: unidesk
|
||||
annotations:
|
||||
unidesk.ai/runtime-plane: ${yamlQuote(gitea.migration.runtimePlane)}
|
||||
unidesk.ai/build-plane: ${yamlQuote(gitea.migration.buildPlane)}
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: ${app.securityContext.runAsUser}
|
||||
runAsGroup: ${app.securityContext.runAsGroup}
|
||||
fsGroup: ${app.securityContext.fsGroup}
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
containers:
|
||||
- name: gitea
|
||||
image: ${image}
|
||||
imagePullPolicy: ${app.image.pullPolicy}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: ${app.service.httpPort}
|
||||
- name: ssh
|
||||
containerPort: ${app.service.sshPort}
|
||||
env:
|
||||
${envVars(gitea, target)}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: ${app.probes.healthPath}
|
||||
port: http
|
||||
initialDelaySeconds: ${app.probes.initialDelaySeconds}
|
||||
periodSeconds: ${app.probes.periodSeconds}
|
||||
timeoutSeconds: ${app.probes.timeoutSeconds}
|
||||
failureThreshold: ${app.probes.failureThreshold}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: ${app.probes.healthPath}
|
||||
port: http
|
||||
initialDelaySeconds: ${app.probes.initialDelaySeconds}
|
||||
periodSeconds: ${app.probes.periodSeconds}
|
||||
timeoutSeconds: ${app.probes.timeoutSeconds}
|
||||
failureThreshold: ${app.probes.failureThreshold}
|
||||
resources:
|
||||
requests:
|
||||
cpu: ${yamlQuote(app.resources.requests.cpu)}
|
||||
memory: ${yamlQuote(app.resources.requests.memory)}
|
||||
limits:
|
||||
cpu: ${yamlQuote(app.resources.limits.cpu)}
|
||||
memory: ${yamlQuote(app.resources.limits.memory)}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: ${app.storage.data.mountPath}
|
||||
- name: config
|
||||
mountPath: ${app.storage.config.mountPath}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: data
|
||||
labels:
|
||||
app.kubernetes.io/name: ${app.name}
|
||||
app.kubernetes.io/component: gitea
|
||||
app.kubernetes.io/part-of: devops-infra
|
||||
app.kubernetes.io/managed-by: unidesk
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ${target.storageClassName}
|
||||
resources:
|
||||
requests:
|
||||
storage: ${app.storage.data.size}
|
||||
- metadata:
|
||||
name: config
|
||||
labels:
|
||||
app.kubernetes.io/name: ${app.name}
|
||||
app.kubernetes.io/component: gitea
|
||||
app.kubernetes.io/part-of: devops-infra
|
||||
app.kubernetes.io/managed-by: unidesk
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: ${target.storageClassName}
|
||||
resources:
|
||||
requests:
|
||||
storage: ${app.storage.config.size}
|
||||
`;
|
||||
}
|
||||
|
||||
function envVars(gitea: GiteaConfig, target: GiteaTarget): string {
|
||||
const app = gitea.app;
|
||||
const values: Record<string, string> = {
|
||||
GITEA_WORK_DIR: app.storage.data.mountPath,
|
||||
GITEA__security__INSTALL_LOCK: "true",
|
||||
GITEA__server__PROTOCOL: app.server.protocol,
|
||||
GITEA__server__DOMAIN: app.server.domain,
|
||||
GITEA__server__ROOT_URL: app.server.rootUrl,
|
||||
GITEA__server__HTTP_ADDR: "0.0.0.0",
|
||||
GITEA__server__HTTP_PORT: String(app.service.httpPort),
|
||||
GITEA__server__SSH_DOMAIN: app.server.sshDomain,
|
||||
GITEA__server__SSH_PORT: String(app.service.sshPort),
|
||||
GITEA__server__START_SSH_SERVER: app.server.startSshServer ? "true" : "false",
|
||||
GITEA__database__DB_TYPE: app.database.type,
|
||||
GITEA__database__PATH: app.database.path,
|
||||
GITEA__repository__ROOT: `${app.storage.data.mountPath}/git/repositories`,
|
||||
GITEA__actions__ENABLED: app.actions.enabled ? "true" : "false",
|
||||
GITEA__service__DISABLE_REGISTRATION: app.registration.disabled ? "true" : "false",
|
||||
GITEA__log__LEVEL: "Info",
|
||||
UNIDESK_GITEA_TARGET: target.id,
|
||||
};
|
||||
return Object.entries(values).map(([name, value]) => ` - name: ${name}
|
||||
value: ${yamlQuote(value)}`).join("\n");
|
||||
}
|
||||
|
||||
function remoteScript(action: "apply" | "status" | "validate", gitea: GiteaConfig, target: GiteaTarget, manifest: string, options: ApplyOptions): string {
|
||||
const env: Record<string, string> = {
|
||||
UNIDESK_GITEA_ACTION: action,
|
||||
UNIDESK_GITEA_TARGET_ID: target.id,
|
||||
UNIDESK_GITEA_ROUTE: target.route,
|
||||
UNIDESK_GITEA_NAMESPACE: target.namespace,
|
||||
UNIDESK_GITEA_APP_NAME: gitea.app.name,
|
||||
UNIDESK_GITEA_STATEFULSET_NAME: gitea.app.statefulSetName,
|
||||
UNIDESK_GITEA_SERVICE_NAME: gitea.app.serviceName,
|
||||
UNIDESK_GITEA_HTTP_PORT: String(gitea.app.service.httpPort),
|
||||
UNIDESK_GITEA_HEALTH_PATH: gitea.validation.healthPath,
|
||||
UNIDESK_GITEA_IMAGE: `${gitea.app.image.repository}:${gitea.app.image.tag}`,
|
||||
UNIDESK_GITEA_FIELD_MANAGER: fieldManager,
|
||||
UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS: String(gitea.validation.waitTimeoutSeconds),
|
||||
UNIDESK_GITEA_DRY_RUN: options.dryRun ? "1" : "0",
|
||||
UNIDESK_GITEA_WAIT: options.wait ? "1" : "0",
|
||||
UNIDESK_GITEA_FULL: options.full ? "1" : "0",
|
||||
UNIDESK_GITEA_MANIFEST_B64: Buffer.from(manifest, "utf8").toString("base64"),
|
||||
};
|
||||
const exports = Object.entries(env).map(([key, value]) => `export ${key}=${shQuote(value)}`).join("\n");
|
||||
return `${exports}\n${readFileSync(remoteScriptFile, "utf8")}`;
|
||||
}
|
||||
|
||||
function policyChecks(gitea: GiteaConfig, target: GiteaTarget, manifest: string): Array<Record<string, unknown>> {
|
||||
return [
|
||||
{ name: "yaml-source-of-truth", ok: true, detail: "Gitea target, namespace, image, storage, ports and probes are read from config/platform-infra/gitea.yaml." },
|
||||
{ name: "gh-1548-service-contract", ok: target.namespace === "devops-infra" && gitea.app.serviceName === "gitea-http", detail: "The service matches config/cicd-gitea-actions-poc.yaml sourceAuthority.giteaMirror." },
|
||||
{ name: "cluster-internal-only", ok: !/^\s*type:\s*(NodePort|LoadBalancer)\s*$/mu.test(manifest) && !/^\s*kind:\s*Ingress\s*$/mu.test(manifest), detail: "Gitea is ClusterIP-only for the POC." },
|
||||
{ name: "runtime-zero-docker", ok: !manifest.includes("/var/run/docker.sock") && !/^\s*hostPath:\s*$/mu.test(manifest), detail: "Runtime Gitea does not mount Docker socket or hostPath." },
|
||||
{ name: "rootless-image", ok: /-rootless$/u.test(gitea.app.image.tag), detail: "The runtime image is Gitea rootless." },
|
||||
{ name: "actions-enabled", ok: gitea.app.actions.enabled, detail: "Gitea Actions is enabled; runner registration is a later controlled stage." },
|
||||
{ name: "env-reuse-preserved", ok: gitea.migration.envReusePolicy === "preserve-existing-runtime-env-reuse", detail: "This install does not replace the existing env reuse path or runtime deployment." },
|
||||
];
|
||||
}
|
||||
|
||||
function configSummary(gitea: GiteaConfig, target: GiteaTarget): Record<string, unknown> {
|
||||
return {
|
||||
path: configLabel,
|
||||
metadata: gitea.metadata,
|
||||
migration: gitea.migration,
|
||||
target: targetSummary(target),
|
||||
app: appSummary(gitea),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactConfigSummary(gitea: GiteaConfig, target: GiteaTarget): Record<string, unknown> {
|
||||
return {
|
||||
path: configLabel,
|
||||
target: targetSummary(target),
|
||||
app: {
|
||||
image: `${gitea.app.image.repository}:${gitea.app.image.tag}`,
|
||||
serviceDns: serviceDns(gitea, target),
|
||||
rootUrl: gitea.app.server.rootUrl,
|
||||
actionsEnabled: gitea.app.actions.enabled,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function targetSummary(target: GiteaTarget): Record<string, unknown> {
|
||||
return {
|
||||
id: target.id,
|
||||
route: target.route,
|
||||
namespace: target.namespace,
|
||||
role: target.role,
|
||||
createNamespace: target.createNamespace,
|
||||
storageClassName: target.storageClassName,
|
||||
};
|
||||
}
|
||||
|
||||
function appSummary(gitea: GiteaConfig): Record<string, unknown> {
|
||||
return {
|
||||
name: gitea.app.name,
|
||||
statefulSetName: gitea.app.statefulSetName,
|
||||
serviceName: gitea.app.serviceName,
|
||||
image: `${gitea.app.image.repository}:${gitea.app.image.tag}`,
|
||||
replicas: gitea.app.replicas,
|
||||
service: gitea.app.service,
|
||||
rootUrl: gitea.app.server.rootUrl,
|
||||
actionsEnabled: gitea.app.actions.enabled,
|
||||
registrationDisabled: gitea.app.registration.disabled,
|
||||
storage: gitea.app.storage,
|
||||
healthPath: gitea.validation.healthPath,
|
||||
};
|
||||
}
|
||||
|
||||
function statusSummary(payload: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
ready: payload.ready === true,
|
||||
target: payload.target,
|
||||
route: payload.route,
|
||||
namespace: payload.namespace,
|
||||
image: payload.image,
|
||||
serviceDns: payload.serviceDns,
|
||||
networkPolicy: payload.networkPolicy,
|
||||
statefulSet: payload.statefulSet,
|
||||
service: payload.service,
|
||||
endpointsReady: payload.endpointsReady === true,
|
||||
pods: Array.isArray(payload.pods) ? payload.pods : [],
|
||||
pvcs: Array.isArray(payload.pvcs) ? payload.pvcs : [],
|
||||
eventsTail: Array.isArray(payload.eventsTail) ? payload.eventsTail : [],
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function renderPlan(result: Record<string, unknown>): RenderedCliResult {
|
||||
const config = record(result.config);
|
||||
const target = record(config.target);
|
||||
const app = record(config.app);
|
||||
const migration = record(config.migration);
|
||||
const policy = arrayRecords(result.policy);
|
||||
const failed = policy.filter((item) => item.ok === false);
|
||||
const next = record(result.next);
|
||||
return rendered(result, "platform-infra gitea plan", [
|
||||
"PLATFORM-INFRA GITEA PLAN",
|
||||
...table(["FIELD", "VALUE", "DETAIL", "VALUE"], [
|
||||
["TARGET", stringValue(target.id), "route", stringValue(target.route)],
|
||||
["NAMESPACE", stringValue(target.namespace), "role", stringValue(target.role)],
|
||||
["IMAGE", stringValue(app.image), "replicas", stringValue(app.replicas)],
|
||||
["SERVICE", `${stringValue(app.serviceName)}:${stringValue(record(app.service).httpPort)}`, "dns", serviceDnsFromObjects(app, target)],
|
||||
["ACTIONS", boolText(app.actionsEnabled), "registrationDisabled", boolText(app.registrationDisabled)],
|
||||
["MIGRATION", stringValue(migration.role), "replaces", stringValue(migration.replaces)],
|
||||
["POLICY", failed.length === 0 ? "ok" : `failed=${failed.length}`, "valuesPrinted", "false"],
|
||||
]),
|
||||
"",
|
||||
"NEXT",
|
||||
` dry-run: ${stringValue(next.dryRun)}`,
|
||||
` apply: ${stringValue(next.apply)}`,
|
||||
` status: ${stringValue(next.status)}`,
|
||||
` validate: ${stringValue(next.validate)}`,
|
||||
"",
|
||||
"Boundary: Gitea is internal ClusterIP source authority for GH-1548/GH-1549; runner and mirror repo bootstrap are separate controlled stages.",
|
||||
"Disclosure: Secret values are not printed; this stage does not create runner credentials.",
|
||||
]);
|
||||
}
|
||||
|
||||
function renderApply(result: Record<string, unknown>): RenderedCliResult {
|
||||
const target = record(result.target);
|
||||
const remote = record(result.remote);
|
||||
const steps = record(remote.steps);
|
||||
const applyStep = record(steps.apply);
|
||||
const rolloutStep = record(steps.rollout);
|
||||
return rendered(result, "platform-infra gitea apply", [
|
||||
"PLATFORM-INFRA GITEA APPLY",
|
||||
...table(["TARGET", "NAMESPACE", "MODE", "OK"], [[stringValue(target.id), stringValue(target.namespace), stringValue(result.mode), boolText(result.ok)]]),
|
||||
"",
|
||||
"STEPS",
|
||||
...table(["STEP", "EXIT", "DETAIL"], [
|
||||
["apply", stringValue(applyStep.exitCode), compactLine(stringValue(applyStep.stderrTail, stringValue(applyStep.stdoutTail)))],
|
||||
["rollout", stringValue(rolloutStep.exitCode), compactLine(stringValue(rolloutStep.stderrTail, stringValue(rolloutStep.stdoutTail)))],
|
||||
]),
|
||||
...remoteErrorLines(result),
|
||||
"",
|
||||
`NEXT bun scripts/cli.ts platform-infra gitea status --target ${stringValue(target.id)}`,
|
||||
]);
|
||||
}
|
||||
|
||||
function renderStatus(result: Record<string, unknown>): RenderedCliResult {
|
||||
const summary = record(result.summary);
|
||||
const statefulSet = record(summary.statefulSet);
|
||||
const service = record(summary.service);
|
||||
const networkPolicy = record(summary.networkPolicy);
|
||||
const pods = arrayRecords(summary.pods).map((pod) => [stringValue(pod.name), stringValue(pod.phase), boolText(pod.ready), stringValue(pod.restarts)]);
|
||||
const pvcs = arrayRecords(summary.pvcs).map((pvc) => [stringValue(pvc.name), stringValue(pvc.phase), stringValue(pvc.capacity)]);
|
||||
return rendered(result, "platform-infra gitea status", [
|
||||
"PLATFORM-INFRA GITEA STATUS",
|
||||
...table(["TARGET", "ROUTE", "NAMESPACE", "READY"], [[stringValue(summary.target), stringValue(summary.route), stringValue(summary.namespace), boolText(summary.ready)]]),
|
||||
"",
|
||||
"CONTROL",
|
||||
...table(["CHECK", "VALUE", "DETAIL"], [
|
||||
["statefulSet", `${stringValue(statefulSet.readyReplicas)}/${stringValue(statefulSet.desired)}`, stringValue(statefulSet.name)],
|
||||
["service", stringValue(service.type), stringValue(summary.serviceDns)],
|
||||
["endpoints", boolText(summary.endpointsReady), stringValue(service.clusterIP)],
|
||||
["allow-all", boolText(networkPolicy.allowAllPresent), "NetworkPolicy"],
|
||||
]),
|
||||
"",
|
||||
"PODS",
|
||||
...(pods.length === 0 ? ["-"] : table(["POD", "PHASE", "READY", "RESTARTS"], pods)),
|
||||
"",
|
||||
"PVCS",
|
||||
...(pvcs.length === 0 ? ["-"] : table(["PVC", "PHASE", "CAPACITY"], pvcs)),
|
||||
...remoteErrorLines(result),
|
||||
"",
|
||||
`NEXT bun scripts/cli.ts platform-infra gitea validate --target ${stringValue(summary.target)}`,
|
||||
]);
|
||||
}
|
||||
|
||||
function remoteErrorLines(result: Record<string, unknown>): string[] {
|
||||
if (result.ok !== false) return [];
|
||||
const remote = record(result.remote);
|
||||
const exitCode = stringValue(remote.exitCode);
|
||||
const stderr = compactLine(stringValue(remote.stderrTail));
|
||||
const stdout = compactLine(stringValue(remote.stdoutTail));
|
||||
const detail = stderr !== "-" ? stderr : stdout;
|
||||
return ["", "ERROR", ...table(["EXIT", "DETAIL"], [[exitCode, detail]])];
|
||||
}
|
||||
|
||||
function rendered(result: Record<string, unknown>, command: string, lines: string[]): RenderedCliResult {
|
||||
return { ok: result.ok !== false, command, renderedText: lines.join("\n"), contentType: "text/plain" };
|
||||
}
|
||||
|
||||
function parseApplyOptions(args: string[]): ApplyOptions {
|
||||
const commonArgs: string[] = [];
|
||||
let confirm = false;
|
||||
let dryRun = false;
|
||||
let wait = false;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--confirm") confirm = true;
|
||||
else if (arg === "--dry-run") dryRun = true;
|
||||
else if (arg === "--wait") wait = true;
|
||||
else {
|
||||
commonArgs.push(arg);
|
||||
if (arg === "--target" || arg === "--node") {
|
||||
commonArgs.push(args[index + 1] ?? "");
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (confirm && dryRun) throw new Error("gitea apply accepts only one of --confirm or --dry-run");
|
||||
return { ...parseCommonOptions(commonArgs), confirm, dryRun: dryRun || !confirm, wait };
|
||||
}
|
||||
|
||||
function parseCommonOptions(args: string[]): CommonOptions {
|
||||
let targetId: string | null = null;
|
||||
let full = false;
|
||||
let raw = false;
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (arg === "--target" || arg === "--node") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
||||
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${arg} must be a simple target id`);
|
||||
targetId = value;
|
||||
index += 1;
|
||||
} else if (arg === "--full") {
|
||||
full = true;
|
||||
} else if (arg === "--raw") {
|
||||
raw = true;
|
||||
full = true;
|
||||
} else {
|
||||
throw new Error(`unsupported gitea option: ${arg}`);
|
||||
}
|
||||
}
|
||||
return { targetId, full, raw };
|
||||
}
|
||||
|
||||
function nextCommands(targetId: string): Record<string, string> {
|
||||
return {
|
||||
dryRun: `bun scripts/cli.ts platform-infra gitea apply --target ${targetId} --dry-run`,
|
||||
apply: `bun scripts/cli.ts platform-infra gitea apply --target ${targetId} --confirm`,
|
||||
status: `bun scripts/cli.ts platform-infra gitea status --target ${targetId}`,
|
||||
validate: `bun scripts/cli.ts platform-infra gitea validate --target ${targetId}`,
|
||||
};
|
||||
}
|
||||
|
||||
function manifestObjectSummary(yaml: string): Array<Record<string, unknown>> {
|
||||
const objects: Array<Record<string, unknown>> = [];
|
||||
for (const doc of yaml.split(/^---$/mu)) {
|
||||
const kind = doc.match(/^\s*kind:\s*([A-Za-z0-9._-]+)\s*$/mu)?.[1];
|
||||
const name = doc.match(/^\s*name:\s*([A-Za-z0-9._-]+)\s*$/mu)?.[1];
|
||||
const namespace = doc.match(/^\s*namespace:\s*([A-Za-z0-9._-]+)\s*$/mu)?.[1] ?? null;
|
||||
if (kind !== undefined && name !== undefined) objects.push({ kind, name, namespace });
|
||||
}
|
||||
return objects;
|
||||
}
|
||||
|
||||
function serviceDns(gitea: GiteaConfig, target: GiteaTarget): string {
|
||||
return `${gitea.app.serviceName}.${target.namespace}.svc.cluster.local:${gitea.app.service.httpPort}`;
|
||||
}
|
||||
|
||||
function serviceDnsFromObjects(app: Record<string, unknown>, target: Record<string, unknown>): string {
|
||||
const service = record(app.service);
|
||||
return `${stringValue(app.serviceName)}.${stringValue(target.namespace)}.svc.cluster.local:${stringValue(service.httpPort)}`;
|
||||
}
|
||||
|
||||
function positiveInteger(obj: Record<string, unknown>, key: string, path: string): number {
|
||||
const value = y.integerField(obj, key, path);
|
||||
if (value < 1) throw new Error(`${configLabel}.${path}.${key} must be positive`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function boundedTimeout(obj: Record<string, unknown>, key: string, path: string): number {
|
||||
const value = positiveInteger(obj, key, path);
|
||||
if (value > 55) throw new Error(`${configLabel}.${path}.${key} must fit the 60s trans short-connection budget`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function quantity(obj: Record<string, unknown>, key: string, path: string): string {
|
||||
const value = y.stringField(obj, key, path);
|
||||
if (!/^[0-9]+(?:m|Ki|Mi|Gi|Ti)?$/u.test(value)) throw new Error(`${configLabel}.${path}.${key} must be a Kubernetes quantity`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function urlField(obj: Record<string, unknown>, key: string, path: string): string {
|
||||
const value = y.stringField(obj, key, path);
|
||||
const parsed = new URL(value);
|
||||
if (!["http:", "https:"].includes(parsed.protocol) || parsed.search || parsed.hash) throw new Error(`${configLabel}.${path}.${key} must be an http(s) URL without query or hash`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function yamlQuote(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item) => typeof item === "object" && item !== null && !Array.isArray(item)) as Record<string, unknown>[] : [];
|
||||
}
|
||||
|
||||
function stringValue(value: unknown, fallback = "-"): string {
|
||||
if (typeof value === "string" && value.length > 0) return value;
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function boolText(value: unknown): string {
|
||||
return value === true ? "true" : "false";
|
||||
}
|
||||
|
||||
function compactLine(value: string): string {
|
||||
const trimmed = value.replace(/\s+/gu, " ").trim();
|
||||
return trimmed.length > 0 ? trimmed.slice(0, 220) : "-";
|
||||
}
|
||||
|
||||
function table(headers: string[], rows: string[][]): string[] {
|
||||
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => (row[index] ?? "").length)));
|
||||
const format = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index])).join(" ");
|
||||
return [format(headers), format(headers.map((header, index) => "-".repeat(Math.max(header.length, widths[index])))), ...rows.map(format)];
|
||||
}
|
||||
@@ -369,7 +369,7 @@ export interface ManagedResourceCleanupPlan {
|
||||
export function platformInfraHelp(): unknown {
|
||||
const target = sub2ApiHelpTargetSummary();
|
||||
return {
|
||||
command: "platform-infra sub2api|langbot|n8n|wechat-archive|observability|secret-plane|kafka ...",
|
||||
command: "platform-infra sub2api|langbot|n8n|wechat-archive|observability|secret-plane|kafka|gitea ...",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts platform-infra sub2api plan [--target G14|D601]",
|
||||
@@ -430,8 +430,13 @@ export function platformInfraHelp(): unknown {
|
||||
"bun scripts/cli.ts platform-infra kafka offsets --node D518 --topic hwlab.agentrun.command.v1",
|
||||
"bun scripts/cli.ts platform-infra kafka tail --node D518 --topic hwlab.agentrun.command.v1 --limit 5",
|
||||
"bun scripts/cli.ts platform-infra kafka produce --node D518 --topic hwlab.agentrun.command.v1 --key <trace-or-session>",
|
||||
"bun scripts/cli.ts platform-infra gitea plan --target JD01",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --dry-run",
|
||||
"bun scripts/cli.ts platform-infra gitea apply --target JD01 --confirm",
|
||||
"bun scripts/cli.ts platform-infra gitea status --target JD01",
|
||||
"bun scripts/cli.ts platform-infra gitea validate --target JD01",
|
||||
],
|
||||
description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, WeChat archive workflows, OpenTelemetry tracing, the independent target-scoped secret plane, and the D518 Kafka event bus. Public services use PK01 Caddy+FRP rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
|
||||
description: "Operate YAML-controlled platform-infra services such as Sub2API, LangBot, n8n, WeChat archive workflows, OpenTelemetry tracing, the independent target-scoped secret plane, the D518 Kafka event bus, and the internal Gitea source-authority service for GH-1548/GH-1549. Public services use PK01 Caddy+FRP rather than Kubernetes Ingress, NodePort, or LoadBalancer.",
|
||||
target,
|
||||
codexPool: {
|
||||
usage: [
|
||||
|
||||
@@ -65,6 +65,10 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin
|
||||
const { runPlatformInfraKafkaCommand } = await import("../platform-infra-kafka");
|
||||
return await runPlatformInfraKafkaCommand(config, args.slice(1));
|
||||
}
|
||||
if (target === "gitea") {
|
||||
const { runPlatformInfraGiteaCommand } = await import("../platform-infra-gitea");
|
||||
return await runPlatformInfraGiteaCommand(config, args.slice(1));
|
||||
}
|
||||
if (target !== "sub2api") return unsupported(args);
|
||||
if (action === "plan" || action === undefined) {
|
||||
const planArgs = args.slice(2);
|
||||
|
||||
Reference in New Issue
Block a user