fix(forgejo-runner-prune): actually reach buildx cache, bring under gitops management
All checks were successful
sync-to-github / sync (push) Successful in 42s
All checks were successful
sync-to-github / sync (push) Successful in 42s
docker system prune --volumes can never touch the buildx builder's
cache volume - it's attached to a running container, and 'volumes'
prune only removes *unattached* ones. This CronJob ran every 30
minutes for 59 days reporting 'Total reclaimed space: 0B' every
single time while each runner's buildx cache silently grew to
57-98GB (discovered chasing a disk-pressure report on nxtgauge-2/3/4).
Fix: attach the real named builder ('nxtgauge-builder', matching
what CI actually uses) first, then call 'docker buildx prune'
directly, capped at --keep-storage 20GB so it doesn't just regrow
unbounded. Verified live: manually pruned gwsh7 (80%->16% node disk)
and ktst5 (84%->47%), then confirmed the patched CronJob is a true
no-op on an already-clean cache.
Also found and cleaned up (host-level, not gitops - out of band from
k8s): 18 orphaned buildx_buildkit_* containers on nxtgauge-1 from 2
months of ad-hoc 'docker buildx create' calls with no --name reuse,
totally invisible to buildx CLI and unrelated to CI (39.63GB, node
went 76%->50%). Added a daily cron job on that host
(~/.local/bin/docker-cleanup.sh) to keep it from reaccumulating,
since that's the host's standalone Docker daemon, outside k8s/gitops
entirely.
This CronJob + its RBAC (docker-prune-sa/-role/-binding) previously
existed only as manually-applied live objects, not tracked in git -
same pattern as nxtgauge-containerd-cleanup. Adding manifests and
wiring into clusters/production so Flux manages it going forward.
This commit is contained in:
parent
c9fd51d5bc
commit
bc13ea25d1
4 changed files with 98 additions and 0 deletions
|
|
@ -11,5 +11,6 @@ resources:
|
||||||
- ../../apps/litellm/base
|
- ../../apps/litellm/base
|
||||||
- ../../ops/openobserve-alerts
|
- ../../ops/openobserve-alerts
|
||||||
- ../../ops/nxtgauge-containerd-cleanup
|
- ../../ops/nxtgauge-containerd-cleanup
|
||||||
|
- ../../ops/nxtgauge-forgejo-runner-prune
|
||||||
- flux-system/traceworks2026-image-automation.yaml
|
- flux-system/traceworks2026-image-automation.yaml
|
||||||
- ../../apps/traceworks2026/overlays/prod
|
- ../../apps/traceworks2026/overlays/prod
|
||||||
|
|
|
||||||
60
ops/nxtgauge-forgejo-runner-prune/cronjob.yaml
Normal file
60
ops/nxtgauge-forgejo-runner-prune/cronjob.yaml
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: nxtgauge-forgejo-runner-prune
|
||||||
|
namespace: forgejo
|
||||||
|
labels:
|
||||||
|
app: forgejo-runner
|
||||||
|
tier: ops
|
||||||
|
spec:
|
||||||
|
schedule: "*/30 * * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
startingDeadlineSeconds: 300
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
activeDeadlineSeconds: 300
|
||||||
|
backoffLimit: 1
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: forgejo-runner-prune
|
||||||
|
spec:
|
||||||
|
serviceAccountName: docker-prune-sa
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: prune
|
||||||
|
image: ci.nxtgauge.com/admin/bitnami-kubectl:latest
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 32Mi
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 128Mi
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- |
|
||||||
|
echo "Pruning Docker + buildx cache on each forgejo-runner pod..."
|
||||||
|
# docker system prune alone can NEVER reach the buildx builder's
|
||||||
|
# cache - it lives in a named volume attached to a running
|
||||||
|
# container, which "system prune --volumes" only removes when
|
||||||
|
# *unattached*. This ran every 30 minutes for 59 days reporting
|
||||||
|
# "Total reclaimed space: 0B" while the builder cache silently
|
||||||
|
# grew to 60-98GB per runner (fixed 2026-08-16). Attaching the
|
||||||
|
# real named builder first and calling `buildx prune` directly
|
||||||
|
# is what actually reaches it; `system prune` after that still
|
||||||
|
# helps with unrelated dangling images/containers.
|
||||||
|
for pod in $(kubectl get pods -n forgejo -l app=forgejo-runner -o name); do
|
||||||
|
echo "Pruning $pod..."
|
||||||
|
kubectl exec -n forgejo "$pod" -c dind -- sh -c '
|
||||||
|
docker buildx create --use --name nxtgauge-builder >/dev/null 2>&1 || docker buildx use nxtgauge-builder >/dev/null 2>&1
|
||||||
|
echo " buildx cache before:"; docker buildx du 2>/dev/null | tail -1
|
||||||
|
docker buildx prune -af --keep-storage 20GB 2>&1 | tail -3
|
||||||
|
docker system prune -af 2>&1 | tail -2
|
||||||
|
' 2>/dev/null || true
|
||||||
|
done
|
||||||
|
echo "Done."
|
||||||
6
ops/nxtgauge-forgejo-runner-prune/kustomization.yaml
Normal file
6
ops/nxtgauge-forgejo-runner-prune/kustomization.yaml
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- rbac.yaml
|
||||||
|
- cronjob.yaml
|
||||||
31
ops/nxtgauge-forgejo-runner-prune/rbac.yaml
Normal file
31
ops/nxtgauge-forgejo-runner-prune/rbac.yaml
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: docker-prune-sa
|
||||||
|
namespace: forgejo
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: docker-prune-role
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods/exec"]
|
||||||
|
verbs: ["create"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods"]
|
||||||
|
verbs: ["get", "list"]
|
||||||
|
---
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: RoleBinding
|
||||||
|
metadata:
|
||||||
|
name: docker-prune-binding
|
||||||
|
namespace: forgejo
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: docker-prune-role
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: docker-prune-sa
|
||||||
|
namespace: forgejo
|
||||||
Loading…
Add table
Reference in a new issue