From bc13ea25d1b192c4bfef961e4d424b93cca3891d Mon Sep 17 00:00:00 2001 From: sync-test Date: Sun, 16 Aug 2026 04:11:21 +0530 Subject: [PATCH] fix(forgejo-runner-prune): actually reach buildx cache, bring under gitops management docker system prune --volumes can never touch the buildx builder's cache volume - it's attached to a running container, and 'volumes' prune only removes *unattached* ones. This CronJob ran every 30 minutes for 59 days reporting 'Total reclaimed space: 0B' every single time while each runner's buildx cache silently grew to 57-98GB (discovered chasing a disk-pressure report on nxtgauge-2/3/4). Fix: attach the real named builder ('nxtgauge-builder', matching what CI actually uses) first, then call 'docker buildx prune' directly, capped at --keep-storage 20GB so it doesn't just regrow unbounded. Verified live: manually pruned gwsh7 (80%->16% node disk) and ktst5 (84%->47%), then confirmed the patched CronJob is a true no-op on an already-clean cache. Also found and cleaned up (host-level, not gitops - out of band from k8s): 18 orphaned buildx_buildkit_* containers on nxtgauge-1 from 2 months of ad-hoc 'docker buildx create' calls with no --name reuse, totally invisible to buildx CLI and unrelated to CI (39.63GB, node went 76%->50%). Added a daily cron job on that host (~/.local/bin/docker-cleanup.sh) to keep it from reaccumulating, since that's the host's standalone Docker daemon, outside k8s/gitops entirely. This CronJob + its RBAC (docker-prune-sa/-role/-binding) previously existed only as manually-applied live objects, not tracked in git - same pattern as nxtgauge-containerd-cleanup. Adding manifests and wiring into clusters/production so Flux manages it going forward. --- clusters/production/kustomization.yaml | 1 + .../cronjob.yaml | 60 +++++++++++++++++++ .../kustomization.yaml | 6 ++ ops/nxtgauge-forgejo-runner-prune/rbac.yaml | 31 ++++++++++ 4 files changed, 98 insertions(+) create mode 100644 ops/nxtgauge-forgejo-runner-prune/cronjob.yaml create mode 100644 ops/nxtgauge-forgejo-runner-prune/kustomization.yaml create mode 100644 ops/nxtgauge-forgejo-runner-prune/rbac.yaml diff --git a/clusters/production/kustomization.yaml b/clusters/production/kustomization.yaml index 6183d7a..d8a7bbc 100644 --- a/clusters/production/kustomization.yaml +++ b/clusters/production/kustomization.yaml @@ -11,5 +11,6 @@ resources: - ../../apps/litellm/base - ../../ops/openobserve-alerts - ../../ops/nxtgauge-containerd-cleanup + - ../../ops/nxtgauge-forgejo-runner-prune - flux-system/traceworks2026-image-automation.yaml - ../../apps/traceworks2026/overlays/prod diff --git a/ops/nxtgauge-forgejo-runner-prune/cronjob.yaml b/ops/nxtgauge-forgejo-runner-prune/cronjob.yaml new file mode 100644 index 0000000..10700ba --- /dev/null +++ b/ops/nxtgauge-forgejo-runner-prune/cronjob.yaml @@ -0,0 +1,60 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nxtgauge-forgejo-runner-prune + namespace: forgejo + labels: + app: forgejo-runner + tier: ops +spec: + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + activeDeadlineSeconds: 300 + backoffLimit: 1 + template: + metadata: + labels: + app: forgejo-runner-prune + spec: + serviceAccountName: docker-prune-sa + restartPolicy: OnFailure + containers: + - name: prune + image: ci.nxtgauge.com/admin/bitnami-kubectl:latest + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + command: + - /bin/sh + - -ec + - | + echo "Pruning Docker + buildx cache on each forgejo-runner pod..." + # docker system prune alone can NEVER reach the buildx builder's + # cache - it lives in a named volume attached to a running + # container, which "system prune --volumes" only removes when + # *unattached*. This ran every 30 minutes for 59 days reporting + # "Total reclaimed space: 0B" while the builder cache silently + # grew to 60-98GB per runner (fixed 2026-08-16). Attaching the + # real named builder first and calling `buildx prune` directly + # is what actually reaches it; `system prune` after that still + # helps with unrelated dangling images/containers. + for pod in $(kubectl get pods -n forgejo -l app=forgejo-runner -o name); do + echo "Pruning $pod..." + kubectl exec -n forgejo "$pod" -c dind -- sh -c ' + docker buildx create --use --name nxtgauge-builder >/dev/null 2>&1 || docker buildx use nxtgauge-builder >/dev/null 2>&1 + echo " buildx cache before:"; docker buildx du 2>/dev/null | tail -1 + docker buildx prune -af --keep-storage 20GB 2>&1 | tail -3 + docker system prune -af 2>&1 | tail -2 + ' 2>/dev/null || true + done + echo "Done." diff --git a/ops/nxtgauge-forgejo-runner-prune/kustomization.yaml b/ops/nxtgauge-forgejo-runner-prune/kustomization.yaml new file mode 100644 index 0000000..2aa3f7d --- /dev/null +++ b/ops/nxtgauge-forgejo-runner-prune/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - cronjob.yaml diff --git a/ops/nxtgauge-forgejo-runner-prune/rbac.yaml b/ops/nxtgauge-forgejo-runner-prune/rbac.yaml new file mode 100644 index 0000000..1e11230 --- /dev/null +++ b/ops/nxtgauge-forgejo-runner-prune/rbac.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: docker-prune-sa + namespace: forgejo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: docker-prune-role +rules: + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: docker-prune-binding + namespace: forgejo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: docker-prune-role +subjects: + - kind: ServiceAccount + name: docker-prune-sa + namespace: forgejo