diff --git a/clusters/production/kustomization.yaml b/clusters/production/kustomization.yaml index d70e560..6183d7a 100644 --- a/clusters/production/kustomization.yaml +++ b/clusters/production/kustomization.yaml @@ -10,5 +10,6 @@ resources: - ../../apps/ollama/base - ../../apps/litellm/base - ../../ops/openobserve-alerts + - ../../ops/nxtgauge-containerd-cleanup - flux-system/traceworks2026-image-automation.yaml - ../../apps/traceworks2026/overlays/prod diff --git a/ops/nxtgauge-containerd-cleanup/cronjob.yaml b/ops/nxtgauge-containerd-cleanup/cronjob.yaml new file mode 100644 index 0000000..407dff9 --- /dev/null +++ b/ops/nxtgauge-containerd-cleanup/cronjob.yaml @@ -0,0 +1,77 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nxtgauge-containerd-cleanup + namespace: kube-system +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + activeDeadlineSeconds: 540 + template: + spec: + serviceAccountName: nxtgauge-containerd-cleanup + hostNetwork: true + restartPolicy: OnFailure + nodeSelector: + cleanup-target: "true" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: cleanup + image: ci.nxtgauge.com/admin/alpine:3.19 + securityContext: + privileged: true + command: + - /bin/sh + - -ec + - | + echo "=== before ===" + df -h /var/lib/rancher/k3s/agent/containerd || true + du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs 2>/dev/null || true + du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content 2>/dev/null || true + echo "=== prune ===" + # No standalone ctr/crictl binary is installed on these k3s nodes; + # only the k3s binary (bundling "k3s ctr"/"k3s crictl" subcommands) is + # present at /usr/local/bin. Bare "ctr" always failed with "not found" + # and was silently swallowed by "|| true", making this a 68-day no-op + # (fixed 2026-08-15). + /usr/local/bin/k3s ctr -n k8s.io images prune --all || true + echo "=== after ===" + df -h /var/lib/rancher/k3s/agent/containerd || true + du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs 2>/dev/null || true + du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content 2>/dev/null || true + volumeMounts: + - name: containerd-root + mountPath: /var/lib/rancher/k3s/agent/containerd + - name: run + mountPath: /run + - name: sys + mountPath: /sys + readOnly: true + - name: bin + mountPath: /usr/local/bin + readOnly: true + volumes: + - name: containerd-root + hostPath: + path: /var/lib/rancher/k3s/agent/containerd + type: Directory + - name: run + hostPath: + path: /run + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + - name: bin + hostPath: + path: /usr/local/bin + type: Directory diff --git a/ops/nxtgauge-containerd-cleanup/kustomization.yaml b/ops/nxtgauge-containerd-cleanup/kustomization.yaml new file mode 100644 index 0000000..ab42a78 --- /dev/null +++ b/ops/nxtgauge-containerd-cleanup/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceaccount.yaml + - cronjob.yaml diff --git a/ops/nxtgauge-containerd-cleanup/serviceaccount.yaml b/ops/nxtgauge-containerd-cleanup/serviceaccount.yaml new file mode 100644 index 0000000..bbcc35c --- /dev/null +++ b/ops/nxtgauge-containerd-cleanup/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nxtgauge-containerd-cleanup + namespace: kube-system