diff --git a/clusters/production/kustomization.yaml b/clusters/production/kustomization.yaml index 6183d7a..d8a7bbc 100644 --- a/clusters/production/kustomization.yaml +++ b/clusters/production/kustomization.yaml @@ -11,5 +11,6 @@ resources: - ../../apps/litellm/base - ../../ops/openobserve-alerts - ../../ops/nxtgauge-containerd-cleanup + - ../../ops/nxtgauge-forgejo-runner-prune - flux-system/traceworks2026-image-automation.yaml - ../../apps/traceworks2026/overlays/prod diff --git a/ops/nxtgauge-forgejo-runner-prune/cronjob.yaml b/ops/nxtgauge-forgejo-runner-prune/cronjob.yaml new file mode 100644 index 0000000..10700ba --- /dev/null +++ b/ops/nxtgauge-forgejo-runner-prune/cronjob.yaml @@ -0,0 +1,60 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nxtgauge-forgejo-runner-prune + namespace: forgejo + labels: + app: forgejo-runner + tier: ops +spec: + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + activeDeadlineSeconds: 300 + backoffLimit: 1 + template: + metadata: + labels: + app: forgejo-runner-prune + spec: + serviceAccountName: docker-prune-sa + restartPolicy: OnFailure + containers: + - name: prune + image: ci.nxtgauge.com/admin/bitnami-kubectl:latest + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + command: + - /bin/sh + - -ec + - | + echo "Pruning Docker + buildx cache on each forgejo-runner pod..." + # docker system prune alone can NEVER reach the buildx builder's + # cache - it lives in a named volume attached to a running + # container, which "system prune --volumes" only removes when + # *unattached*. This ran every 30 minutes for 59 days reporting + # "Total reclaimed space: 0B" while the builder cache silently + # grew to 60-98GB per runner (fixed 2026-08-16). Attaching the + # real named builder first and calling `buildx prune` directly + # is what actually reaches it; `system prune` after that still + # helps with unrelated dangling images/containers. + for pod in $(kubectl get pods -n forgejo -l app=forgejo-runner -o name); do + echo "Pruning $pod..." + kubectl exec -n forgejo "$pod" -c dind -- sh -c ' + docker buildx create --use --name nxtgauge-builder >/dev/null 2>&1 || docker buildx use nxtgauge-builder >/dev/null 2>&1 + echo " buildx cache before:"; docker buildx du 2>/dev/null | tail -1 + docker buildx prune -af --keep-storage 20GB 2>&1 | tail -3 + docker system prune -af 2>&1 | tail -2 + ' 2>/dev/null || true + done + echo "Done." diff --git a/ops/nxtgauge-forgejo-runner-prune/kustomization.yaml b/ops/nxtgauge-forgejo-runner-prune/kustomization.yaml new file mode 100644 index 0000000..2aa3f7d --- /dev/null +++ b/ops/nxtgauge-forgejo-runner-prune/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - cronjob.yaml diff --git a/ops/nxtgauge-forgejo-runner-prune/rbac.yaml b/ops/nxtgauge-forgejo-runner-prune/rbac.yaml new file mode 100644 index 0000000..1e11230 --- /dev/null +++ b/ops/nxtgauge-forgejo-runner-prune/rbac.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: docker-prune-sa + namespace: forgejo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: docker-prune-role +rules: + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: docker-prune-binding + namespace: forgejo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: docker-prune-role +subjects: + - kind: ServiceAccount + name: docker-prune-sa + namespace: forgejo