798b336419
- pg_dump + B2 (S3-compatible) upload with retention-based pruning - litellm backup job is suspended: the nxtgauge-ai postgres instance currently has a corrupt litellm database catalog file, un-suspend once that's fixed/reinitialized - secrets encrypted with sops per repo convention (.sops.yaml) - not yet wired into kustomizations; apps/postgresql is not part of the Flux-applied tree today (statefulset.yaml predates this and is also unwired) - needs a deliberate decision on how postgres is meant to be deployed/applied before enabling
91 lines
3.7 KiB
YAML
91 lines
3.7 KiB
YAML
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: litellm-postgres-backup
|
|
namespace: nxtgauge-ai
|
|
spec:
|
|
schedule: "45 2 * * *"
|
|
# Suspended: the nxtgauge-ai postgres instance has no working "litellm" database today
|
|
# (CREATE DATABASE fails on a corrupt system catalog file, base/<oid>/2617 "File exists").
|
|
# Un-suspend once that instance is fixed/reinitialized.
|
|
suspend: true
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 5
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: pg-backup
|
|
image: postgres:15-alpine
|
|
envFrom:
|
|
- secretRef:
|
|
name: litellm-postgres-backup-secret
|
|
env:
|
|
- name: PGHOST
|
|
value: postgres
|
|
- name: PGPORT
|
|
value: "5432"
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-credentials
|
|
key: POSTGRES_USER
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-credentials
|
|
key: POSTGRES_PASSWORD
|
|
- name: PGDATABASE
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: postgres-credentials
|
|
key: POSTGRES_DB
|
|
command: ["/bin/sh", "-ec"]
|
|
args:
|
|
- |
|
|
apk add --no-cache aws-cli python3 >/dev/null
|
|
|
|
TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
RAW_FILE="/tmp/${PGDATABASE}-${TIMESTAMP}.sql"
|
|
DUMP_FILE="${RAW_FILE}.gz"
|
|
|
|
pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -f "$RAW_FILE"
|
|
gzip -9 "$RAW_FILE"
|
|
|
|
export AWS_ACCESS_KEY_ID="$B2_ACCESS_KEY_ID"
|
|
export AWS_SECRET_ACCESS_KEY="$B2_SECRET_ACCESS_KEY"
|
|
export AWS_DEFAULT_REGION="$B2_REGION"
|
|
|
|
aws --endpoint-url "$B2_ENDPOINT" s3 cp "$DUMP_FILE" "s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/$(basename "$DUMP_FILE")"
|
|
rm -f "$DUMP_FILE"
|
|
echo "Uploaded $(basename "$DUMP_FILE") to s3://${B2_BUCKET_NAME}/${BACKUP_PREFIX}/"
|
|
|
|
python3 - <<'PYEOF'
|
|
import subprocess, json, datetime, os
|
|
|
|
bucket = os.environ["B2_BUCKET_NAME"]
|
|
prefix = os.environ["BACKUP_PREFIX"]
|
|
endpoint = os.environ["B2_ENDPOINT"]
|
|
retention_days = int(os.environ.get("RETENTION_DAYS", "30"))
|
|
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
|
|
|
out = subprocess.run(
|
|
["aws", "--endpoint-url", endpoint, "s3api", "list-objects-v2",
|
|
"--bucket", bucket, "--prefix", prefix],
|
|
capture_output=True, text=True, check=True,
|
|
)
|
|
listing = json.loads(out.stdout or "{}")
|
|
for obj in listing.get("Contents", []):
|
|
last_modified = datetime.datetime.fromisoformat(obj["LastModified"].replace("Z", "+00:00"))
|
|
if last_modified < cutoff:
|
|
subprocess.run(
|
|
["aws", "--endpoint-url", endpoint, "s3api", "delete-object",
|
|
"--bucket", bucket, "--key", obj["Key"]],
|
|
check=True,
|
|
)
|
|
print("deleted expired backup:", obj["Key"])
|
|
PYEOF
|