feat(backup): add postgres backup CronJobs for litellm and main data namespace
- pg_dump + B2 (S3-compatible) upload with retention-based pruning - litellm backup job is suspended: the nxtgauge-ai postgres instance currently has a corrupt litellm database catalog file, un-suspend once that's fixed/reinitialized - secrets encrypted with sops per repo convention (.sops.yaml) - not yet wired into kustomizations; apps/postgresql is not part of the Flux-applied tree today (statefulset.yaml predates this and is also unwired) - needs a deliberate decision on how postgres is meant to be deployed/applied before enabling
This commit is contained in:
sync-test
parent
359f7bb159
commit
798b336419
4 changed files with 221 additions and 0 deletions
91
apps/litellm/base/db-backup-cronjob.yaml
Normal file
91
apps/litellm/base/db-backup-cronjob.yaml
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: litellm-postgres-backup
|
||||
namespace: nxtgauge-ai
|
||||
spec:
|
||||
schedule: "45 2 * * *"
|
||||
# Suspended: the nxtgauge-ai postgres instance has no working "litellm" database today
|
||||
# (CREATE DATABASE fails on a corrupt system catalog file, base/<oid>/2617 "File exists").
|
||||
# Un-suspend once that instance is fixed/reinitialized.
|
||||
suspend: true
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 5
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: pg-backup
|
||||
image: postgres:15-alpine
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: litellm-postgres-backup-secret
|
||||
env:
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-credentials
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-credentials
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: postgres-credentials
|
||||
key: POSTGRES_DB
|
||||
command: ["/bin/sh", "-ec"]
|
||||
args:
|
||||
- |
|
||||
apk add --no-cache aws-cli python3 >/dev/null
|
||||
|
||||
TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql"
|
||||
DUMP_FILE="${RAW_FILE}.gz"
|
||||
|
||||
pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE"
|
||||
gzip -9 "$RAW_FILE"
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="$B2_REGION"
|
||||
|
||||
aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")"
|
||||
rm -f "$DUMP_FILE"
|
||||
echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/"
|
||||
|
||||
python3 - <<'PYEOF'
|
||||
import subprocess, json, datetime, os
|
||||
|
||||
bucket = os.environ["B2_BUCKET_NAME"]
|
||||
prefix = os.environ["BACKUP_PREFIX"]
|
||||
endpoint = os.environ["B2_ENDPOINT"]
|
||||
retention_days = int(os.environ.get("RETENTION_DAYS", "30"))
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
|
||||
out = subprocess.run(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
|
||||
"--bucket", bucket, "--prefix", prefix],
|
||||
capture_output=True, text=True, check=True,
|
||||
)
|
||||
listing = json.loads(out.stdout or "{}")
|
||||
for obj in listing.get("Contents", []):
|
||||
last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00"))
|
||||
if last_modified < cutoff:
|
||||
subprocess.run(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "delete-object",
|
||||
"--bucket", bucket, "--key", obj["Key"]],
|
||||
check=True,
|
||||
)
|
||||
print("deleted expired backup:", obj["Key"])
|
||||
PYEOF
|
||||
29
apps/litellm/base/db-backup-secret.yaml
Normal file
29
apps/litellm/base/db-backup-secret.yaml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: litellm-postgres-backup-secret
|
||||
namespace: nxtgauge-ai
|
||||
type: Opaque
|
||||
stringData:
|
||||
B2_ACCESS_KEY_ID: ENC[AES256_GCM,data:yN9jyY/mHskyNY9kuFOBxQ0BUINQxnE2fw==,iv:2sfOXDoDKdDA3fTd/9rRQMp4HkE3e7uX58Aq1RWMjuw=,tag:xX98THKq5vkcgCVHO083CQ==,type:str]
|
||||
B2_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:LZV5e3XQGi3Pgg8o7ffqIUP0MH5hayT3sYqkjg8dkw==,iv:9bGRz5ITzhtXu+j+02g4gpUnbyOhs0UrP/R/xJJlueU=,tag:K1LcWKHX5h9EOSBj5Q2Cng==,type:str]
|
||||
B2_BUCKET_NAME: ENC[AES256_GCM,data:yP03INPoykOW3OunfHsF,iv:dEJ4okMMmh/EQXBRJeVjD1SjmL5irEQRy+3nUlFYhDY=,tag:fBJgeWexd/aBLVPFIIgRjg==,type:str]
|
||||
B2_ENDPOINT: ENC[AES256_GCM,data:sMZy7lZGhlKBgeXfxgjTynpjzsLSHtWcwKjyRGhSI4l743kBnM5yCBU=,iv:tZeFK0RxNFCAbpSEW8ICQT2rgdc7ZI98vEaIHszaHr8=,tag:KcAVpgFWCorKJ9sWOucliQ==,type:str]
|
||||
B2_REGION: ENC[AES256_GCM,data:t9ExTVXacRQXvWTGbRM=,iv:KMw95/jQyqpYg73NVWEomxehMa2F05R2SlVJh4V/xPQ=,tag:1L+0ykYFF7LK+9QD+os4Kg==,type:str]
|
||||
BACKUP_PREFIX: ENC[AES256_GCM,data:8IsLsR5ls+dW4MvQ7KwveA==,iv:TmESoyByMxEiFZ8pi85T+Gp3P7Do3Zg7BzhUTjBr8Ik=,tag:/FUuSfg7kI6EK+WTvvE5BQ==,type:str]
|
||||
RETENTION_DAYS: ENC[AES256_GCM,data:mAY=,iv:ZCld9d9vQD3goHcZ9yH0pFmvnGAGLGm1dUz+YPdptl4=,tag:N6TRLOFwPbE/ZF3/a0Y97A==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- recipient: age1sqxc53yvap8763eup99g6eza6fa5u65qetdymlh5ch4ugvvv5q0sf3pm4l
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvdjRoMzdlWlhtck9EM2FE
|
||||
SXZFdG5GMDJkM2xUQjJJeHRRbWZsdjlhblZVClpPZTU1YlA2TkJ3elRQdWY3a0ds
|
||||
cHNZUllxSnJDVWxBaTMzMENtNG4rZGcKLS0tIDFtUW5EZk91NFFwa2svelF4Z0dN
|
||||
clZmdmtkdWJaOHVLOHlBOGIzSUtWcjQKt3I7Zar8UCsS8UfY+SGbc8nxX+PUjp0f
|
||||
8PJKBFr9VahsZp7XsN6iq1qbWWdSpWlo8mlkyhxF6ZfiP/LI+N/Okg==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-08-15T17:03:21Z"
|
||||
mac: ENC[AES256_GCM,data:tX/IdisbNhepZ4fuaS39EJ2wEPUEpVMsTJvuKqejYLkm4fQ4TlAhyW/IE6qZIqg/PlKVE5316jFM95oW6KK4Ymeth3jglVedV0wfx7rrvkHoiJL+RtzFG4leD7UHnkHDBrhtAJzNLlQhm+uy8BKpxGU4XsAXWxCU2xqNmsuhaeE=,iv:WLdixhlKx5KmNo7xKfZc/P0ks+aS9kq55IcO/zevJTc=,tag:O3nT3OPocIn/ijhGWJaIvA==,type:str]
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.10.2
|
||||
67
apps/postgresql/backup-cronjob.yaml
Normal file
67
apps/postgresql/backup-cronjob.yaml
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: pg-postgresql-backup
|
||||
namespace: data
|
||||
spec:
|
||||
schedule: "30 2 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 5
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: pg-backup
|
||||
image: postgres:16-alpine
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: pg-postgresql-backup-secret
|
||||
command: ["/bin/sh", "-ec"]
|
||||
args:
|
||||
- |
|
||||
apk add --no-cache aws-cli python3 >/dev/null
|
||||
|
||||
TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql"
|
||||
DUMP_FILE="${RAW_FILE}.gz"
|
||||
|
||||
pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE"
|
||||
gzip -9 "$RAW_FILE"
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="$B2_REGION"
|
||||
|
||||
aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")"
|
||||
rm -f "$DUMP_FILE"
|
||||
echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/"
|
||||
|
||||
python3 - <<'PYEOF'
|
||||
import subprocess, json, datetime, os
|
||||
|
||||
bucket = os.environ["B2_BUCKET_NAME"]
|
||||
prefix = os.environ["BACKUP_PREFIX"]
|
||||
endpoint = os.environ["B2_ENDPOINT"]
|
||||
retention_days = int(os.environ.get("RETENTION_DAYS", "30"))
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
|
||||
out = subprocess.run(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
|
||||
"--bucket", bucket, "--prefix", prefix],
|
||||
capture_output=True, text=True, check=True,
|
||||
)
|
||||
listing = json.loads(out.stdout or "{}")
|
||||
for obj in listing.get("Contents", []):
|
||||
last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00"))
|
||||
if last_modified < cutoff:
|
||||
subprocess.run(
|
||||
["aws", "--endpoint-url", endpoint, "s3api", "delete-object",
|
||||
"--bucket", bucket, "--key", obj["Key"]],
|
||||
check=True,
|
||||
)
|
||||
print("deleted expired backup:", obj["Key"])
|
||||
PYEOF
|
||||
34
apps/postgresql/backup-secret.yaml
Normal file
34
apps/postgresql/backup-secret.yaml
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: pg-postgresql-backup-secret
|
||||
namespace: data
|
||||
type: Opaque
|
||||
stringData:
|
||||
PGHOST: ENC[AES256_GCM,data:q9CYJ8BfUmV34FarbQ==,iv:0c4EK546jGjc7XWejS/lqC1VKuEGWCcDikHQmCLzaik=,tag:wkEODY8ZYjE6E1sGoIqDiw==,type:str]
|
||||
PGPORT: ENC[AES256_GCM,data:EmsjAQ==,iv:YLvsGmVgHm9k0tDheWgMZufCW2Im/VOxWa4hhntOFNU=,tag:m8KBwpVUXX/RAjz/Chy0nw==,type:str]
|
||||
PGUSER: ENC[AES256_GCM,data:99mVYPGLdmM=,iv:FW/6THa6vkzUDEKqBqNfgYBhMFTIzUiMgelYzNBgNiM=,tag:xFryuyqEdXv+oJ969Q+3Cw==,type:str]
|
||||
PGPASSWORD: ENC[AES256_GCM,data:3YfO5dd7TE1Sh/ZwoihvYrnqkJq0R/C9702dm+ZH35B7wytfLA5iU/Hmxw==,iv:iiXpitEOSUtes7j3zPQOIgrkDIVBtgQNpQQ2JetGRlc=,tag:dBGVlRzl9edCWGCdEsE7JA==,type:str]
|
||||
PGDATABASE: ENC[AES256_GCM,data:VQ3sBkUwgQo=,iv:BA20my9o+1KNNohx1EBD8E281rqoz6Z4lJZiGCyvP+A=,tag:+7D/KGTMMN8cB5TYnvDdWA==,type:str]
|
||||
B2_ACCESS_KEY_ID: ENC[AES256_GCM,data:GIzq8wdTtvgYEI61nmPVdaKdNePY5bWUsA==,iv:gxvexXRO4hp/+0trsI77sbTzGD5dbJqdAqfvD8Vyd3Y=,tag:TKdpqdg7iSM3mE7EWmJ4IA==,type:str]
|
||||
B2_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:acJclcstPO3tr2RtfHuBZahZvCRnlNNfbAjl6G7oCg==,iv:vOTGiQkO8nObvRSiikHf1+UMpqcJILgpixvehC4rSoo=,tag:vct0vuY2vvI2Iy/tTmwFSw==,type:str]
|
||||
B2_BUCKET_NAME: ENC[AES256_GCM,data:vlArED2zywHcFb1P28ko,iv:ehOZiT6D5JuqyIprdwCuGKBO0j+tfcYbJXusay4X9Dk=,tag:HTmhgLPtQ+hRMHnPDxm+EQ==,type:str]
|
||||
B2_ENDPOINT: ENC[AES256_GCM,data:T5VP8p99RxswSf+izejtzFlG/PWTqCj/R0fUqpUgmI9KaXI6K0eN8HY=,iv:CJJ0qa2Snfi6hz0/yr1A/yjcoEq4UtZMig/I8vRPTWg=,tag:7tbKKZPFeB2Yqnogw7/tsg==,type:str]
|
||||
B2_REGION: ENC[AES256_GCM,data:Ca3TwWstC1kvZJ+W3VI=,iv:0EkdXRPxKs7WKle0htL9LSMc9EJwMaRn+IKu9tEXrPY=,tag:ZJC6Izx6M9ziHapyBMelcQ==,type:str]
|
||||
BACKUP_PREFIX: ENC[AES256_GCM,data:qMI/Fj4ODTM3OhxocjXBo3E=,iv:FEljbqYmnclrDuVJXej3lS/q4HhxaFDu+DZma13QJb8=,tag:RXMek4SgCbi5JUyFMeYR9w==,type:str]
|
||||
RETENTION_DAYS: ENC[AES256_GCM,data:NJ4=,iv:CPMogSsUKz2m2ubDZ11fSkbbGLI03loBgS7/fY3+iXo=,tag:kH/hy+5K06YEZuK4BeY/6g==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- recipient: age1sqxc53yvap8763eup99g6eza6fa5u65qetdymlh5ch4ugvvv5q0sf3pm4l
|
||||
enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6cmgwSHJtYWk3QzNKTWJz
|
||||
L2dBZFRUS3VZVlFBWFc3VDk4N1lIa2g5RW5zCi9yQmFjQ3JRMHVhaEhwWDQzZEEz
|
||||
bVRidUF0QU5xK3ArellHVDhaRmZ5REUKLS0tIGdWdS9JSDh1aDUvaEdJUDBzOHdj
|
||||
a3MwSFVtZ3RpdzVCN1FBOGxtekRmYVUKJRIo/Bavm9OisrwrqKNNSYc6NquVkBrp
|
||||
+d3hHuC1JYsBGndjxSzSyiP6YYbkuC1Y303vVac+To6fXdFLZW63eQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2026-08-15T17:03:21Z"
|
||||
mac: ENC[AES256_GCM,data:mDx+iKI92dZTrGcyepYyO4ZdpPgpe+KLm7oQauZzRWeG5dbLc0jAwf4oRXlYFBJ5U5BVvVEL1bButrShzW/uBAbHig0zJS4x7iJfFmZv1uuWQtzsawL0E59NQD97Tz4dktgRrRwqu0C2+qy5+IsHpo3msSfVwhhSnkbIHPZ76Sg=,iv:YJjrcL29jjHF2mVZ7BE8g33wEB/FqcCaq9OoU9I3Lp4=,tag:zJ3JbX/W/4b01bOntWdlrg==,type:str]
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
version: 3.10.2
|
||||
Loading…
Add table
Reference in a new issue