feat(backup): add postgres backup CronJobs for litellm and main data namespace

- pg_dump + B2 (S3-compatible) upload with retention-based pruning
- litellm backup job is suspended: the nxtgauge-ai postgres instance
  currently has a corrupt litellm database catalog file, un-suspend
  once that's fixed/reinitialized
- secrets encrypted with sops per repo convention (.sops.yaml)
- not yet wired into kustomizations; apps/postgresql is not part of
  the Flux-applied tree today (statefulset.yaml predates this and is
  also unwired) - needs a deliberate decision on how postgres is
  meant to be deployed/applied before enabling

apps/litellm/base/db-backup-cronjob.yaml

@@ -0,0 +1,91 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: litellm-postgres-backup
+  namespace: nxtgauge-ai
+spec:
+  schedule: "45 2 * * *"
+  # Suspended: the nxtgauge-ai postgres instance has no working "litellm" database today
+  # (CREATE DATABASE fails on a corrupt system catalog file, base/<oid>/2617 "File exists").
+  # Un-suspend once that instance is fixed/reinitialized.
+  suspend: true
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 5
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: pg-backup
+              image: postgres:15-alpine
+              envFrom:
+                - secretRef:
+                    name: litellm-postgres-backup-secret
+              env:
+                - name: PGHOST
+                  value: postgres
+                - name: PGPORT
+                  value: "5432"
+                - name: PGUSER
+                  valueFrom:
+                    secretKeyRef:
+                      name: postgres-credentials
+                      key: POSTGRES_USER
+                - name: PGPASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: postgres-credentials
+                      key: POSTGRES_PASSWORD
+                - name: PGDATABASE
+                  valueFrom:
+                    secretKeyRef:
+                      name: postgres-credentials
+                      key: POSTGRES_DB
+              command: ["/bin/sh", "-ec"]
+              args:
+                - |
+                  apk add --no-cache aws-cli python3 >/dev/null
+
+                  TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+                  RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql"
+                  DUMP_FILE="${RAW_FILE}.gz"
+
+                  pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE"
+                  gzip -9 "$RAW_FILE"
+
+                  export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID"
+                  export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY"
+                  export AWS_DEFAULT_REGION="$B2_REGION"
+
+                  aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")"
+                  rm -f "$DUMP_FILE"
+                  echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/"
+
+                  python3 - <<'PYEOF'
+                  import subprocess, json, datetime, os
+
+                  bucket = os.environ["B2_BUCKET_NAME"]
+                  prefix = os.environ["BACKUP_PREFIX"]
+                  endpoint = os.environ["B2_ENDPOINT"]
+                  retention_days = int(os.environ.get("RETENTION_DAYS", "30"))
+                  cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
+
+                  out = subprocess.run(
+                      ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
+                       "--bucket", bucket, "--prefix", prefix],
+                      capture_output=True, text=True, check=True,
+                  )
+                  listing = json.loads(out.stdout or "{}")
+                  for obj in listing.get("Contents", []):
+                      last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00"))
+                      if last_modified < cutoff:
+                          subprocess.run(
+                              ["aws", "--endpoint-url", endpoint, "s3api", "delete-object",
+                               "--bucket", bucket, "--key", obj["Key"]],
+                              check=True,
+                          )
+                          print("deleted expired backup:", obj["Key"])
+                  PYEOF

apps/litellm/base/db-backup-secret.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: litellm-postgres-backup-secret
+    namespace: nxtgauge-ai
+type: Opaque
+stringData:
+    B2_ACCESS_KEY_ID: ENC[AES256_GCM,data:yN9jyY/mHskyNY9kuFOBxQ0BUINQxnE2fw==,iv:2sfOXDoDKdDA3fTd/9rRQMp4HkE3e7uX58Aq1RWMjuw=,tag:xX98THKq5vkcgCVHO083CQ==,type:str]
+    B2_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:LZV5e3XQGi3Pgg8o7ffqIUP0MH5hayT3sYqkjg8dkw==,iv:9bGRz5ITzhtXu+j+02g4gpUnbyOhs0UrP/R/xJJlueU=,tag:K1LcWKHX5h9EOSBj5Q2Cng==,type:str]
+    B2_BUCKET_NAME: ENC[AES256_GCM,data:yP03INPoykOW3OunfHsF,iv:dEJ4okMMmh/EQXBRJeVjD1SjmL5irEQRy+3nUlFYhDY=,tag:fBJgeWexd/aBLVPFIIgRjg==,type:str]
+    B2_ENDPOINT: ENC[AES256_GCM,data:sMZy7lZGhlKBgeXfxgjTynpjzsLSHtWcwKjyRGhSI4l743kBnM5yCBU=,iv:tZeFK0RxNFCAbpSEW8ICQT2rgdc7ZI98vEaIHszaHr8=,tag:KcAVpgFWCorKJ9sWOucliQ==,type:str]
+    B2_REGION: ENC[AES256_GCM,data:t9ExTVXacRQXvWTGbRM=,iv:KMw95/jQyqpYg73NVWEomxehMa2F05R2SlVJh4V/xPQ=,tag:1L+0ykYFF7LK+9QD+os4Kg==,type:str]
+    BACKUP_PREFIX: ENC[AES256_GCM,data:8IsLsR5ls+dW4MvQ7KwveA==,iv:TmESoyByMxEiFZ8pi85T+Gp3P7Do3Zg7BzhUTjBr8Ik=,tag:/FUuSfg7kI6EK+WTvvE5BQ==,type:str]
+    RETENTION_DAYS: ENC[AES256_GCM,data:mAY=,iv:ZCld9d9vQD3goHcZ9yH0pFmvnGAGLGm1dUz+YPdptl4=,tag:N6TRLOFwPbE/ZF3/a0Y97A==,type:str]
+sops:
+    age:
+        - recipient: age1sqxc53yvap8763eup99g6eza6fa5u65qetdymlh5ch4ugvvv5q0sf3pm4l
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvdjRoMzdlWlhtck9EM2FE
+            SXZFdG5GMDJkM2xUQjJJeHRRbWZsdjlhblZVClpPZTU1YlA2TkJ3elRQdWY3a0ds
+            cHNZUllxSnJDVWxBaTMzMENtNG4rZGcKLS0tIDFtUW5EZk91NFFwa2svelF4Z0dN
+            clZmdmtkdWJaOHVLOHlBOGIzSUtWcjQKt3I7Zar8UCsS8UfY+SGbc8nxX+PUjp0f
+            8PJKBFr9VahsZp7XsN6iq1qbWWdSpWlo8mlkyhxF6ZfiP/LI+N/Okg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-15T17:03:21Z"
+    mac: ENC[AES256_GCM,data:tX/IdisbNhepZ4fuaS39EJ2wEPUEpVMsTJvuKqejYLkm4fQ4TlAhyW/IE6qZIqg/PlKVE5316jFM95oW6KK4Ymeth3jglVedV0wfx7rrvkHoiJL+RtzFG4leD7UHnkHDBrhtAJzNLlQhm+uy8BKpxGU4XsAXWxCU2xqNmsuhaeE=,iv:WLdixhlKx5KmNo7xKfZc/P0ks+aS9kq55IcO/zevJTc=,tag:O3nT3OPocIn/ijhGWJaIvA==,type:str]
+    encrypted_regex: ^(data|stringData)$
+    version: 3.10.2

apps/postgresql/backup-cronjob.yaml

@@ -0,0 +1,67 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: pg-postgresql-backup
+  namespace: data
+spec:
+  schedule: "30 2 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 5
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: pg-backup
+              image: postgres:16-alpine
+              envFrom:
+                - secretRef:
+                    name: pg-postgresql-backup-secret
+              command: ["/bin/sh", "-ec"]
+              args:
+                - |
+                  apk add --no-cache aws-cli python3 >/dev/null
+
+                  TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+                  RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql"
+                  DUMP_FILE="${RAW_FILE}.gz"
+
+                  pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE"
+                  gzip -9 "$RAW_FILE"
+
+                  export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID"
+                  export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY"
+                  export AWS_DEFAULT_REGION="$B2_REGION"
+
+                  aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")"
+                  rm -f "$DUMP_FILE"
+                  echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/"
+
+                  python3 - <<'PYEOF'
+                  import subprocess, json, datetime, os
+
+                  bucket = os.environ["B2_BUCKET_NAME"]
+                  prefix = os.environ["BACKUP_PREFIX"]
+                  endpoint = os.environ["B2_ENDPOINT"]
+                  retention_days = int(os.environ.get("RETENTION_DAYS", "30"))
+                  cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
+
+                  out = subprocess.run(
+                      ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
+                       "--bucket", bucket, "--prefix", prefix],
+                      capture_output=True, text=True, check=True,
+                  )
+                  listing = json.loads(out.stdout or "{}")
+                  for obj in listing.get("Contents", []):
+                      last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00"))
+                      if last_modified < cutoff:
+                          subprocess.run(
+                              ["aws", "--endpoint-url", endpoint, "s3api", "delete-object",
+                               "--bucket", bucket, "--key", obj["Key"]],
+                              check=True,
+                          )
+                          print("deleted expired backup:", obj["Key"])
+                  PYEOF

apps/postgresql/backup-secret.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: pg-postgresql-backup-secret
+    namespace: data
+type: Opaque
+stringData:
+    PGHOST: ENC[AES256_GCM,data:q9CYJ8BfUmV34FarbQ==,iv:0c4EK546jGjc7XWejS/lqC1VKuEGWCcDikHQmCLzaik=,tag:wkEODY8ZYjE6E1sGoIqDiw==,type:str]
+    PGPORT: ENC[AES256_GCM,data:EmsjAQ==,iv:YLvsGmVgHm9k0tDheWgMZufCW2Im/VOxWa4hhntOFNU=,tag:m8KBwpVUXX/RAjz/Chy0nw==,type:str]
+    PGUSER: ENC[AES256_GCM,data:99mVYPGLdmM=,iv:FW/6THa6vkzUDEKqBqNfgYBhMFTIzUiMgelYzNBgNiM=,tag:xFryuyqEdXv+oJ969Q+3Cw==,type:str]
+    PGPASSWORD: ENC[AES256_GCM,data:3YfO5dd7TE1Sh/ZwoihvYrnqkJq0R/C9702dm+ZH35B7wytfLA5iU/Hmxw==,iv:iiXpitEOSUtes7j3zPQOIgrkDIVBtgQNpQQ2JetGRlc=,tag:dBGVlRzl9edCWGCdEsE7JA==,type:str]
+    PGDATABASE: ENC[AES256_GCM,data:VQ3sBkUwgQo=,iv:BA20my9o+1KNNohx1EBD8E281rqoz6Z4lJZiGCyvP+A=,tag:+7D/KGTMMN8cB5TYnvDdWA==,type:str]
+    B2_ACCESS_KEY_ID: ENC[AES256_GCM,data:GIzq8wdTtvgYEI61nmPVdaKdNePY5bWUsA==,iv:gxvexXRO4hp/+0trsI77sbTzGD5dbJqdAqfvD8Vyd3Y=,tag:TKdpqdg7iSM3mE7EWmJ4IA==,type:str]
+    B2_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:acJclcstPO3tr2RtfHuBZahZvCRnlNNfbAjl6G7oCg==,iv:vOTGiQkO8nObvRSiikHf1+UMpqcJILgpixvehC4rSoo=,tag:vct0vuY2vvI2Iy/tTmwFSw==,type:str]
+    B2_BUCKET_NAME: ENC[AES256_GCM,data:vlArED2zywHcFb1P28ko,iv:ehOZiT6D5JuqyIprdwCuGKBO0j+tfcYbJXusay4X9Dk=,tag:HTmhgLPtQ+hRMHnPDxm+EQ==,type:str]
+    B2_ENDPOINT: ENC[AES256_GCM,data:T5VP8p99RxswSf+izejtzFlG/PWTqCj/R0fUqpUgmI9KaXI6K0eN8HY=,iv:CJJ0qa2Snfi6hz0/yr1A/yjcoEq4UtZMig/I8vRPTWg=,tag:7tbKKZPFeB2Yqnogw7/tsg==,type:str]
+    B2_REGION: ENC[AES256_GCM,data:Ca3TwWstC1kvZJ+W3VI=,iv:0EkdXRPxKs7WKle0htL9LSMc9EJwMaRn+IKu9tEXrPY=,tag:ZJC6Izx6M9ziHapyBMelcQ==,type:str]
+    BACKUP_PREFIX: ENC[AES256_GCM,data:qMI/Fj4ODTM3OhxocjXBo3E=,iv:FEljbqYmnclrDuVJXej3lS/q4HhxaFDu+DZma13QJb8=,tag:RXMek4SgCbi5JUyFMeYR9w==,type:str]
+    RETENTION_DAYS: ENC[AES256_GCM,data:NJ4=,iv:CPMogSsUKz2m2ubDZ11fSkbbGLI03loBgS7/fY3+iXo=,tag:kH/hy+5K06YEZuK4BeY/6g==,type:str]
+sops:
+    age:
+        - recipient: age1sqxc53yvap8763eup99g6eza6fa5u65qetdymlh5ch4ugvvv5q0sf3pm4l
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6cmgwSHJtYWk3QzNKTWJz
+            L2dBZFRUS3VZVlFBWFc3VDk4N1lIa2g5RW5zCi9yQmFjQ3JRMHVhaEhwWDQzZEEz
+            bVRidUF0QU5xK3ArellHVDhaRmZ5REUKLS0tIGdWdS9JSDh1aDUvaEdJUDBzOHdj
+            a3MwSFVtZ3RpdzVCN1FBOGxtekRmYVUKJRIo/Bavm9OisrwrqKNNSYc6NquVkBrp
+            +d3hHuC1JYsBGndjxSzSyiP6YYbkuC1Y303vVac+To6fXdFLZW63eQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-08-15T17:03:21Z"
+    mac: ENC[AES256_GCM,data:mDx+iKI92dZTrGcyepYyO4ZdpPgpe+KLm7oQauZzRWeG5dbLc0jAwf4oRXlYFBJ5U5BVvVEL1bButrShzW/uBAbHig0zJS4x7iJfFmZv1uuWQtzsawL0E59NQD97Tz4dktgRrRwqu0C2+qy5+IsHpo3msSfVwhhSnkbIHPZ76Sg=,iv:YJjrcL29jjHF2mVZ7BE8g33wEB/FqcCaq9OoU9I3Lp4=,tag:zJ3JbX/W/4b01bOntWdlrg==,type:str]
+    encrypted_regex: ^(data|stringData)$
+    version: 3.10.2