diff --git a/apps/litellm/base/db-backup-cronjob.yaml b/apps/litellm/base/db-backup-cronjob.yaml new file mode 100644 index 0000000..ce7a732 --- /dev/null +++ b/apps/litellm/base/db-backup-cronjob.yaml @@ -0,0 +1,91 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: litellm-postgres-backup + namespace: nxtgauge-ai +spec: + schedule: "45 2 * * *" + # Suspended: the nxtgauge-ai postgres instance has no working "litellm" database today + # (CREATE DATABASE fails on a corrupt system catalog file, base//2617 "File exists"). + # Un-suspend once that instance is fixed/reinitialized. + suspend: true + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + containers: + - name: pg-backup + image: postgres:15-alpine + envFrom: + - secretRef: + name: litellm-postgres-backup-secret + env: + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + command: ["/bin/sh", "-ec"] + args: + - | + apk add --no-cache aws-cli python3 >/dev/null + + TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" + RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql" + DUMP_FILE="${RAW_FILE}.gz" + + pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE" + gzip -9 "$RAW_FILE" + + export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID" + export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY" + export AWS_DEFAULT_REGION="$B2_REGION" + + aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")" + rm -f "$DUMP_FILE" + echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/" + + python3 - <<'PYEOF' + import subprocess, json, datetime, os + + bucket = os.environ["B2_BUCKET_NAME"] + prefix = os.environ["BACKUP_PREFIX"] + endpoint = os.environ["B2_ENDPOINT"] + retention_days = int(os.environ.get("RETENTION_DAYS", "30")) + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days) + + out = subprocess.run( + ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2", + "--bucket", bucket, "--prefix", prefix], + capture_output=True, text=True, check=True, + ) + listing = json.loads(out.stdout or "{}") + for obj in listing.get("Contents", []): + last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00")) + if last_modified < cutoff: + subprocess.run( + ["aws", "--endpoint-url", endpoint, "s3api", "delete-object", + "--bucket", bucket, "--key", obj["Key"]], + check=True, + ) + print("deleted expired backup:", obj["Key"]) + PYEOF diff --git a/apps/litellm/base/db-backup-secret.yaml b/apps/litellm/base/db-backup-secret.yaml new file mode 100644 index 0000000..47a4c92 --- /dev/null +++ b/apps/litellm/base/db-backup-secret.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Secret +metadata: + name: litellm-postgres-backup-secret + namespace: nxtgauge-ai +type: Opaque +stringData: + B2_ACCESS_KEY_ID: ENC[AES256_GCM,data:yN9jyY/mHskyNY9kuFOBxQ0BUINQxnE2fw==,iv:2sfOXDoDKdDA3fTd/9rRQMp4HkE3e7uX58Aq1RWMjuw=,tag:xX98THKq5vkcgCVHO083CQ==,type:str] + B2_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:LZV5e3XQGi3Pgg8o7ffqIUP0MH5hayT3sYqkjg8dkw==,iv:9bGRz5ITzhtXu+j+02g4gpUnbyOhs0UrP/R/xJJlueU=,tag:K1LcWKHX5h9EOSBj5Q2Cng==,type:str] + B2_BUCKET_NAME: ENC[AES256_GCM,data:yP03INPoykOW3OunfHsF,iv:dEJ4okMMmh/EQXBRJeVjD1SjmL5irEQRy+3nUlFYhDY=,tag:fBJgeWexd/aBLVPFIIgRjg==,type:str] + B2_ENDPOINT: ENC[AES256_GCM,data:sMZy7lZGhlKBgeXfxgjTynpjzsLSHtWcwKjyRGhSI4l743kBnM5yCBU=,iv:tZeFK0RxNFCAbpSEW8ICQT2rgdc7ZI98vEaIHszaHr8=,tag:KcAVpgFWCorKJ9sWOucliQ==,type:str] + B2_REGION: ENC[AES256_GCM,data:t9ExTVXacRQXvWTGbRM=,iv:KMw95/jQyqpYg73NVWEomxehMa2F05R2SlVJh4V/xPQ=,tag:1L+0ykYFF7LK+9QD+os4Kg==,type:str] + BACKUP_PREFIX: ENC[AES256_GCM,data:8IsLsR5ls+dW4MvQ7KwveA==,iv:TmESoyByMxEiFZ8pi85T+Gp3P7Do3Zg7BzhUTjBr8Ik=,tag:/FUuSfg7kI6EK+WTvvE5BQ==,type:str] + RETENTION_DAYS: ENC[AES256_GCM,data:mAY=,iv:ZCld9d9vQD3goHcZ9yH0pFmvnGAGLGm1dUz+YPdptl4=,tag:N6TRLOFwPbE/ZF3/a0Y97A==,type:str] +sops: + age: + - recipient: age1sqxc53yvap8763eup99g6eza6fa5u65qetdymlh5ch4ugvvv5q0sf3pm4l + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvdjRoMzdlWlhtck9EM2FE + SXZFdG5GMDJkM2xUQjJJeHRRbWZsdjlhblZVClpPZTU1YlA2TkJ3elRQdWY3a0ds + cHNZUllxSnJDVWxBaTMzMENtNG4rZGcKLS0tIDFtUW5EZk91NFFwa2svelF4Z0dN + clZmdmtkdWJaOHVLOHlBOGIzSUtWcjQKt3I7Zar8UCsS8UfY+SGbc8nxX+PUjp0f + 8PJKBFr9VahsZp7XsN6iq1qbWWdSpWlo8mlkyhxF6ZfiP/LI+N/Okg== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-08-15T17:03:21Z" + mac: ENC[AES256_GCM,data:tX/IdisbNhepZ4fuaS39EJ2wEPUEpVMsTJvuKqejYLkm4fQ4TlAhyW/IE6qZIqg/PlKVE5316jFM95oW6KK4Ymeth3jglVedV0wfx7rrvkHoiJL+RtzFG4leD7UHnkHDBrhtAJzNLlQhm+uy8BKpxGU4XsAXWxCU2xqNmsuhaeE=,iv:WLdixhlKx5KmNo7xKfZc/P0ks+aS9kq55IcO/zevJTc=,tag:O3nT3OPocIn/ijhGWJaIvA==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.10.2 diff --git a/apps/postgresql/backup-cronjob.yaml b/apps/postgresql/backup-cronjob.yaml new file mode 100644 index 0000000..b599179 --- /dev/null +++ b/apps/postgresql/backup-cronjob.yaml @@ -0,0 +1,67 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pg-postgresql-backup + namespace: data +spec: + schedule: "30 2 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + containers: + - name: pg-backup + image: postgres:16-alpine + envFrom: + - secretRef: + name: pg-postgresql-backup-secret + command: ["/bin/sh", "-ec"] + args: + - | + apk add --no-cache aws-cli python3 >/dev/null + + TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" + RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql" + DUMP_FILE="${RAW_FILE}.gz" + + pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE" + gzip -9 "$RAW_FILE" + + export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID" + export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY" + export AWS_DEFAULT_REGION="$B2_REGION" + + aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")" + rm -f "$DUMP_FILE" + echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/" + + python3 - <<'PYEOF' + import subprocess, json, datetime, os + + bucket = os.environ["B2_BUCKET_NAME"] + prefix = os.environ["BACKUP_PREFIX"] + endpoint = os.environ["B2_ENDPOINT"] + retention_days = int(os.environ.get("RETENTION_DAYS", "30")) + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days) + + out = subprocess.run( + ["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2", + "--bucket", bucket, "--prefix", prefix], + capture_output=True, text=True, check=True, + ) + listing = json.loads(out.stdout or "{}") + for obj in listing.get("Contents", []): + last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00")) + if last_modified < cutoff: + subprocess.run( + ["aws", "--endpoint-url", endpoint, "s3api", "delete-object", + "--bucket", bucket, "--key", obj["Key"]], + check=True, + ) + print("deleted expired backup:", obj["Key"]) + PYEOF diff --git a/apps/postgresql/backup-secret.yaml b/apps/postgresql/backup-secret.yaml new file mode 100644 index 0000000..019080b --- /dev/null +++ b/apps/postgresql/backup-secret.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Secret +metadata: + name: pg-postgresql-backup-secret + namespace: data +type: Opaque +stringData: + PGHOST: ENC[AES256_GCM,data:q9CYJ8BfUmV34FarbQ==,iv:0c4EK546jGjc7XWejS/lqC1VKuEGWCcDikHQmCLzaik=,tag:wkEODY8ZYjE6E1sGoIqDiw==,type:str] + PGPORT: ENC[AES256_GCM,data:EmsjAQ==,iv:YLvsGmVgHm9k0tDheWgMZufCW2Im/VOxWa4hhntOFNU=,tag:m8KBwpVUXX/RAjz/Chy0nw==,type:str] + PGUSER: ENC[AES256_GCM,data:99mVYPGLdmM=,iv:FW/6THa6vkzUDEKqBqNfgYBhMFTIzUiMgelYzNBgNiM=,tag:xFryuyqEdXv+oJ969Q+3Cw==,type:str] + PGPASSWORD: ENC[AES256_GCM,data:3YfO5dd7TE1Sh/ZwoihvYrnqkJq0R/C9702dm+ZH35B7wytfLA5iU/Hmxw==,iv:iiXpitEOSUtes7j3zPQOIgrkDIVBtgQNpQQ2JetGRlc=,tag:dBGVlRzl9edCWGCdEsE7JA==,type:str] + PGDATABASE: ENC[AES256_GCM,data:VQ3sBkUwgQo=,iv:BA20my9o+1KNNohx1EBD8E281rqoz6Z4lJZiGCyvP+A=,tag:+7D/KGTMMN8cB5TYnvDdWA==,type:str] + B2_ACCESS_KEY_ID: ENC[AES256_GCM,data:GIzq8wdTtvgYEI61nmPVdaKdNePY5bWUsA==,iv:gxvexXRO4hp/+0trsI77sbTzGD5dbJqdAqfvD8Vyd3Y=,tag:TKdpqdg7iSM3mE7EWmJ4IA==,type:str] + B2_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:acJclcstPO3tr2RtfHuBZahZvCRnlNNfbAjl6G7oCg==,iv:vOTGiQkO8nObvRSiikHf1+UMpqcJILgpixvehC4rSoo=,tag:vct0vuY2vvI2Iy/tTmwFSw==,type:str] + B2_BUCKET_NAME: ENC[AES256_GCM,data:vlArED2zywHcFb1P28ko,iv:ehOZiT6D5JuqyIprdwCuGKBO0j+tfcYbJXusay4X9Dk=,tag:HTmhgLPtQ+hRMHnPDxm+EQ==,type:str] + B2_ENDPOINT: ENC[AES256_GCM,data:T5VP8p99RxswSf+izejtzFlG/PWTqCj/R0fUqpUgmI9KaXI6K0eN8HY=,iv:CJJ0qa2Snfi6hz0/yr1A/yjcoEq4UtZMig/I8vRPTWg=,tag:7tbKKZPFeB2Yqnogw7/tsg==,type:str] + B2_REGION: ENC[AES256_GCM,data:Ca3TwWstC1kvZJ+W3VI=,iv:0EkdXRPxKs7WKle0htL9LSMc9EJwMaRn+IKu9tEXrPY=,tag:ZJC6Izx6M9ziHapyBMelcQ==,type:str] + BACKUP_PREFIX: ENC[AES256_GCM,data:qMI/Fj4ODTM3OhxocjXBo3E=,iv:FEljbqYmnclrDuVJXej3lS/q4HhxaFDu+DZma13QJb8=,tag:RXMek4SgCbi5JUyFMeYR9w==,type:str] + RETENTION_DAYS: ENC[AES256_GCM,data:NJ4=,iv:CPMogSsUKz2m2ubDZ11fSkbbGLI03loBgS7/fY3+iXo=,tag:kH/hy+5K06YEZuK4BeY/6g==,type:str] +sops: + age: + - recipient: age1sqxc53yvap8763eup99g6eza6fa5u65qetdymlh5ch4ugvvv5q0sf3pm4l + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6cmgwSHJtYWk3QzNKTWJz + L2dBZFRUS3VZVlFBWFc3VDk4N1lIa2g5RW5zCi9yQmFjQ3JRMHVhaEhwWDQzZEEz + bVRidUF0QU5xK3ArellHVDhaRmZ5REUKLS0tIGdWdS9JSDh1aDUvaEdJUDBzOHdj + a3MwSFVtZ3RpdzVCN1FBOGxtekRmYVUKJRIo/Bavm9OisrwrqKNNSYc6NquVkBrp + +d3hHuC1JYsBGndjxSzSyiP6YYbkuC1Y303vVac+To6fXdFLZW63eQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-08-15T17:03:21Z" + mac: ENC[AES256_GCM,data:mDx+iKI92dZTrGcyepYyO4ZdpPgpe+KLm7oQauZzRWeG5dbLc0jAwf4oRXlYFBJ5U5BVvVEL1bButrShzW/uBAbHig0zJS4x7iJfFmZv1uuWQtzsawL0E59NQD97Tz4dktgRrRwqu0C2+qy5+IsHpo3msSfVwhhSnkbIHPZ76Sg=,iv:YJjrcL29jjHF2mVZ7BE8g33wEB/FqcCaq9OoU9I3Lp4=,tag:zJ3JbX/W/4b01bOntWdlrg==,type:str] + encrypted_regex: ^(data|stringData)$ + version: 3.10.2