fix(containerd-cleanup): use k3s-bundled ctr, bring under gitops management
All checks were successful
sync-to-github / sync (push) Successful in 39s
All checks were successful
sync-to-github / sync (push) Successful in 39s
The prune step called bare 'ctr', which has never existed as a standalone binary on these k3s nodes - only the k3s binary itself (bundling 'k3s ctr'/'k3s crictl' subcommands) lives at /usr/local/bin. The failure was silently swallowed by '|| true', so the CronJob has been reporting Complete every 10 minutes for 68 days while doing nothing; node disk sat at 83% used. Verified live: patched the CronJob's command to invoke /usr/local/bin/k3s ctr directly, manually triggered a run, confirmed it actually prunes now (deleted ~75 dangling images). This CronJob and its ServiceAccount previously existed only as a manually-applied live object, not tracked in git. Adding the manifests here and wiring them into clusters/production so Flux manages it going forward instead of it silently drifting again.
This commit is contained in:
parent
219990bd88
commit
c9fd51d5bc
4 changed files with 89 additions and 0 deletions
|
|
@ -10,5 +10,6 @@ resources:
|
||||||
- ../../apps/ollama/base
|
- ../../apps/ollama/base
|
||||||
- ../../apps/litellm/base
|
- ../../apps/litellm/base
|
||||||
- ../../ops/openobserve-alerts
|
- ../../ops/openobserve-alerts
|
||||||
|
- ../../ops/nxtgauge-containerd-cleanup
|
||||||
- flux-system/traceworks2026-image-automation.yaml
|
- flux-system/traceworks2026-image-automation.yaml
|
||||||
- ../../apps/traceworks2026/overlays/prod
|
- ../../apps/traceworks2026/overlays/prod
|
||||||
|
|
|
||||||
77
ops/nxtgauge-containerd-cleanup/cronjob.yaml
Normal file
77
ops/nxtgauge-containerd-cleanup/cronjob.yaml
Normal file
|
|
@ -0,0 +1,77 @@
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: nxtgauge-containerd-cleanup
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
schedule: "*/10 * * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
startingDeadlineSeconds: 300
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
activeDeadlineSeconds: 540
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
serviceAccountName: nxtgauge-containerd-cleanup
|
||||||
|
hostNetwork: true
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
nodeSelector:
|
||||||
|
cleanup-target: "true"
|
||||||
|
tolerations:
|
||||||
|
- key: node-role.kubernetes.io/control-plane
|
||||||
|
operator: Exists
|
||||||
|
effect: NoSchedule
|
||||||
|
containers:
|
||||||
|
- name: cleanup
|
||||||
|
image: ci.nxtgauge.com/admin/alpine:3.19
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -ec
|
||||||
|
- |
|
||||||
|
echo "=== before ==="
|
||||||
|
df -h /var/lib/rancher/k3s/agent/containerd || true
|
||||||
|
du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs 2>/dev/null || true
|
||||||
|
du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content 2>/dev/null || true
|
||||||
|
echo "=== prune ==="
|
||||||
|
# No standalone ctr/crictl binary is installed on these k3s nodes;
|
||||||
|
# only the k3s binary (bundling "k3s ctr"/"k3s crictl" subcommands) is
|
||||||
|
# present at /usr/local/bin. Bare "ctr" always failed with "not found"
|
||||||
|
# and was silently swallowed by "|| true", making this a 68-day no-op
|
||||||
|
# (fixed 2026-08-15).
|
||||||
|
/usr/local/bin/k3s ctr -n k8s.io images prune --all || true
|
||||||
|
echo "=== after ==="
|
||||||
|
df -h /var/lib/rancher/k3s/agent/containerd || true
|
||||||
|
du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs 2>/dev/null || true
|
||||||
|
du -sh /var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content 2>/dev/null || true
|
||||||
|
volumeMounts:
|
||||||
|
- name: containerd-root
|
||||||
|
mountPath: /var/lib/rancher/k3s/agent/containerd
|
||||||
|
- name: run
|
||||||
|
mountPath: /run
|
||||||
|
- name: sys
|
||||||
|
mountPath: /sys
|
||||||
|
readOnly: true
|
||||||
|
- name: bin
|
||||||
|
mountPath: /usr/local/bin
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: containerd-root
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/rancher/k3s/agent/containerd
|
||||||
|
type: Directory
|
||||||
|
- name: run
|
||||||
|
hostPath:
|
||||||
|
path: /run
|
||||||
|
type: Directory
|
||||||
|
- name: sys
|
||||||
|
hostPath:
|
||||||
|
path: /sys
|
||||||
|
type: Directory
|
||||||
|
- name: bin
|
||||||
|
hostPath:
|
||||||
|
path: /usr/local/bin
|
||||||
|
type: Directory
|
||||||
6
ops/nxtgauge-containerd-cleanup/kustomization.yaml
Normal file
6
ops/nxtgauge-containerd-cleanup/kustomization.yaml
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- serviceaccount.yaml
|
||||||
|
- cronjob.yaml
|
||||||
5
ops/nxtgauge-containerd-cleanup/serviceaccount.yaml
Normal file
5
ops/nxtgauge-containerd-cleanup/serviceaccount.yaml
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: nxtgauge-containerd-cleanup
|
||||||
|
namespace: kube-system
|
||||||
Loading…
Add table
Reference in a new issue