fix: restore all services after registry wipe

- Fix PostgreSQL endpoints (now auto-creating correctly) - Fix Forgejo DB connection - Update retention to keep 10 SHA tags (was 2, too aggressive) - Update all deployments to use available tags - Add missing base images to registry (alpine, node, rust, python) - Protect base images from retention deletion
2026-06-13 00:25:51 +05:30 · 2026-06-13 00:25:51 +05:30 · cbc7fb42e6
commit cbc7fb42e6
parent 60858814b8
32 changed files with 598 additions and 89 deletions
--- a/.github/workflows/sync-to-forgejo.yml
+++ b/.github/workflows/sync-to-forgejo.yml
@ -4,6 +4,7 @@ on:
  push:
    branches:
      - main
+      - high-performance

 jobs:
  sync:
@ -16,7 +17,7 @@ jobs:
        with:
          fetch-depth: 0

-      - name: Sync to Forgejo
+      - name: Push branch to Forgejo
        env:
          FORGEJO_SECRET: ${{ secrets.FORGEJO_SECRET || secrets.GITEA_SECRET }}
          FORGEJO_OWNER: ${{ secrets.FORGEJO_OWNER || 'ashwin' }}
@ -24,16 +25,16 @@ jobs:
          REPO: ${{ github.event.repository.name }}
          BRANCH: ${{ github.ref_name }}
        run: |
-          set -euxo pipefail
-          export GIT_TERMINAL_PROMPT=0
-          export GIT_TRACE=1
-          export GIT_CURL_VERBOSE=1
+          set -euo pipefail
+          test -n "${FORGEJO_SECRET:-}" || { echo "FORGEJO_SECRET is empty"; exit 1; }

-          USER="${FORGEJO_USERNAME}"
+          AUTH="$(printf '%s' "${FORGEJO_USERNAME}:${FORGEJO_SECRET}" | base64 -w0)"
          TARGET="https://ci.nxtgauge.com/${FORGEJO_OWNER}/${REPO}.git"
-          AUTH="$(printf '%s' "${USER}:${FORGEJO_SECRET}" | base64 -w0)"

-          test -n "${FORGEJO_SECRET:-}" || (echo "FORGEJO_SECRET empty" && exit 1)
-          curl -fsS -H "Authorization: Basic ${AUTH}" https://ci.nxtgauge.com/api/v1/user >/dev/null
-          curl -fsS -X POST             -H "Authorization: Basic ${AUTH}"             "https://ci.nxtgauge.com/api/v1/repos/${FORGEJO_OWNER}/${REPO}/mirror-sync" >/dev/null
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git remote remove forgejo 2>/dev/null || true
+          git remote add forgejo "${TARGET}"

+          git -c http.extraHeader="AUTHORIZATION: basic ${AUTH}" push forgejo "HEAD:${BRANCH}" --force
+          git -c http.extraHeader="AUTHORIZATION: basic ${AUTH}" push forgejo --tags --force
--- a/apps/forgejo/deployment.yaml
+++ b/apps/forgejo/deployment.yaml
@ -0,0 +1,168 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: forgejo
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: forgejo-config
+  namespace: forgejo
+data:
+  app.ini: |
+    RUN_MODE = prod
+    RUN_USER = forge
+    [server]
+    DOMAIN = ci.nxtgauge.com
+    HTTP_PORT = 3000
+    ROOT_URL = https://ci.nxtgauge.com/
+    DISABLE_SSH = false
+    SSH_PORT = 22
+    LFS_OBJECTS_PATH = /data/gitea/lfs
+    [database]
+    DB_TYPE = postgres
+    HOST = pg-postgresql.data.svc.cluster.local:5432
+    NAME = forgejo
+    USER = nxtgauge
+    PASSWD = chandan2026@1
+    SSL_MODE = disable
+    [security]
+    INSTALL_LOCK = true
+    SECRET_KEY = eF4nC8wQ3rT2yU9iO5pA1sD6fG7hJ8kL0zXcVbNmMqWeRtY
+    [service]
+    DISABLE_REGISTRATION = false
+    [log]
+    MODE = console
+    LEVEL = info
+    [packages]
+    ENABLED = true
+    [packages.container]
+    ENABLED = true
+    REGISTRY_TYPE = docker-registry
+    REGISTRY_URL = https://registry.nxtgauge.com
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: forgejo-data
+  namespace: forgejo
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: forgejo
+  namespace: forgejo
+  labels:
+    app: forgejo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: forgejo
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: forgejo
+    spec:
+      containers:
+        - name: forgejo
+          image: codeberg.org/forgejo/forgejo:10
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3000
+              name: http
+          env:
+            - name: FORGEJO__SERVER__ROOT_URL
+              value: "https://ci.nxtgauge.com/"
+            - name: FORGEJO__DATABASE__HOST
+              value: "pg-postgresql.data.svc.cluster.local"
+            - name: FORGEJO__DATABASE__PORT
+              value: "5432"
+            - name: FORGEJO__DATABASE__NAME
+              value: "forgejo"
+            - name: FORGEJO__DATABASE__USER
+              value: "nxtgauge"
+            - name: FORGEJO__DATABASE__PASSWD
+              value: "chandan2026@1"
+            - name: FORGEJO__PACKAGES__ENABLED
+              value: "true"
+            - name: FORGEJO__PACKAGES__CONTAINER__ENABLED
+              value: "true"
+            - name: FORGEJO__PACKAGES__CONTAINER__REGISTRY_TYPE
+              value: "docker-registry"
+            - name: FORGEJO__PACKAGES__CONTAINER__REGISTRY_URL
+              value: "https://registry.nxtgauge.com"
+          volumeMounts:
+            - mountPath: /data
+              name: data
+          resources:
+            limits:
+              cpu: "1"
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 256Mi
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 3000
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 3000
+            initialDelaySeconds: 60
+            periodSeconds: 30
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: forgejo-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: forgejo-http
+  namespace: forgejo
+spec:
+  selector:
+    app: forgejo
+  ports:
+    - port: 3000
+      targetPort: 3000
+      name: http
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: forgejo
+  namespace: forgejo
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    traefik.ingress.kubernetes.io/router.entrypoints: web,websecure
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - ci.nxtgauge.com
+      secretName: forgejo-tls
+  rules:
+    - host: ci.nxtgauge.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: forgejo-http
+                port:
+                  number: 3000
--- a/apps/forgejo/runner-deployment.yaml
+++ b/apps/forgejo/runner-deployment.yaml
@ -1,12 +1,11 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
  name: forgejo-runner
  namespace: forgejo
  labels:
    app: forgejo-runner
 spec:
-  replicas: 2
  selector:
    matchLabels:
      app: forgejo-runner
@ -15,6 +14,22 @@ spec:
      labels:
        app: forgejo-runner
    spec:
+      initContainers:
+      - name: init-runner-permissions
+        image: busybox:1.36
+        command: ["/bin/sh", "-ec"]
+        args:
+        - |
+          mkdir -p /data /cache
+          chown -R 1000:0 /data /cache
+          chmod -R g=u /data /cache
+        securityContext:
+          runAsUser: 0
+        volumeMounts:
+        - name: runner-config
+          mountPath: /data
+        - name: runner-cache
+          mountPath: /cache
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
@ -24,38 +39,75 @@ spec:
                    operator: DoesNotExist
                  - key: node-role.kubernetes.io/master
                    operator: DoesNotExist
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app: forgejo-runner
-              topologyKey: kubernetes.io/hostname
      containers:
+      - name: dind
+        image: docker:27-dind
+        args:
+        - --host=tcp://0.0.0.0:2375
+        - --tls=false
+        securityContext:
+          privileged: true
+        env:
+        - name: DOCKER_TLS_CERTDIR
+          value: ""
+        volumeMounts:
+        - name: dind-storage
+          mountPath: /var/lib/docker
+        resources:
+          requests:
+            cpu: 250m
+            memory: 512Mi
+          limits:
+            cpu: 2
+            memory: 4Gi
      - name: runner
        image: code.forgejo.org/forgejo/runner:6
        env:
        - name: DOCKER_HOST
-          value: unix:///var/run/docker.sock
+          value: tcp://127.0.0.1:2375
        - name: FORGEJO_INSTANCE_URL
-          value: http://forgejo.forgejo.svc.cluster.local:3000
+          value: http://forgejo-http.forgejo.svc.cluster.local:3000
        - name: FORGEJO_RUNNER_REGISTRATION_TOKEN
          valueFrom:
            secretKeyRef:
              name: forgejo-runner-secret
-              key: token
-        - name: FORGEJO_RUNNER_NAME
+              key: FORGEJO_RUNNER_REGISTRATION_TOKEN
+        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
-              fieldPath: metadata.name
+              fieldPath: spec.nodeName
        - name: FORGEJO_RUNNER_LABELS
-          value: "ubuntu-latest:docker://node:20-bookworm,self-hosted:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,ubuntu-24.04:docker://node:20-bookworm,debian-12:docker://node:20-bookworm"
+          value: "self-hosted:docker://ghcr.io/catthehacker/ubuntu:act-latest,linux:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-latest,ubuntu-24.04:docker://ghcr.io/catthehacker/ubuntu:act-latest,debian-12:docker://ghcr.io/catthehacker/ubuntu:act-latest"
        volumeMounts:
-        - name: docker-sock
-          mountPath: /var/run/docker.sock
        - name: runner-config
          mountPath: /data
        - name: runner-cache
          mountPath: /cache
+        command: ["/bin/sh"]
+        args:
+        - -ec
+        - |
+          cd /data
+          RUNNER_NAME="${K8S_NODE_NAME}"
+
+          echo "Waiting for Docker sidecar on ${K8S_NODE_NAME}..."
+          sleep 8
+
+
+          if [ ! -f .runner ]; then
+            echo "Registering runner ${RUNNER_NAME} on node ${K8S_NODE_NAME}..."
+            forgejo-runner register \
+              --no-interactive \
+              --instance "$FORGEJO_INSTANCE_URL" \
+              --token "$FORGEJO_RUNNER_REGISTRATION_TOKEN" \
+              --name "$RUNNER_NAME" \
+              --labels "$FORGEJO_RUNNER_LABELS"
+          else
+            echo "Reusing existing runner state for ${RUNNER_NAME} on node ${K8S_NODE_NAME}."
+          fi
+
+          echo "Starting daemon for ${RUNNER_NAME}..."
+          exec forgejo-runner daemon
        resources:
          requests:
            cpu: 200m
@ -64,10 +116,6 @@ spec:
            cpu: 4
            memory: 8Gi
      volumes:
-      - name: docker-sock
-        hostPath:
-          path: /var/run/docker.sock
-          type: Socket
      - name: runner-config
        hostPath:
          path: /var/lib/forgejo-runner
@ -76,6 +124,8 @@ spec:
        hostPath:
          path: /var/cache/forgejo-runner
          type: DirectoryOrCreate
+      - name: dind-storage
+        emptyDir: {}
 ---
 apiVersion: v1
 kind: Secret
@ -84,4 +134,4 @@ metadata:
  namespace: forgejo
 type: Opaque
 stringData:
-  token: "od2pOx...k7MT"
+  FORGEJO_RUNNER_REGISTRATION_TOKEN: "it63i9ZDM3NwSkSUMOYQSNTxpm3R8njC9SlIoRlq"
--- a/apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml
+++ b/apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml
@ -7,4 +7,4 @@ patchesStrategicMerge:
  - replicas-patch.yaml
 images:
  - name: registry.nxtgauge.com/nxtgauge-admin-solid
-    newTag: high-performance-latest
+    newTag: latest
--- a/apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml
+++ b/apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml
@ -5,4 +5,4 @@ resources:
  - ../../base
 images:
  - name: registry.nxtgauge.com/nxtgauge-ai-assistant
-    newTag: 2f999dfe95a48ea4090a90519dc3950f1e729924
+    newTag: latest
--- a/apps/nxtgauge-backend-rust/base/catering-services-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/catering-services-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: catering-services
-          image: registry.nxtgauge.com/nxtgauge-rust-catering-services:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-catering-services:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9115
--- a/apps/nxtgauge-backend-rust/base/companies-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/companies-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: companies
-          image: registry.nxtgauge.com/nxtgauge-rust-companies:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-companies:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9102
--- a/apps/nxtgauge-backend-rust/base/cron-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/cron-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: cron
-          image: registry.nxtgauge.com/nxtgauge-rust-cron:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-cron:latest
          imagePullPolicy: Always
          envFrom:
            - configMapRef:
--- a/apps/nxtgauge-backend-rust/base/customers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/customers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: customers
-          image: registry.nxtgauge.com/nxtgauge-rust-customers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-customers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9105
--- a/apps/nxtgauge-backend-rust/base/developers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/developers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: developers
-          image: registry.nxtgauge.com/nxtgauge-rust-developers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-developers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9110
--- a/apps/nxtgauge-backend-rust/base/employees-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/employees-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: employees
-          image: registry.nxtgauge.com/nxtgauge-rust-employees:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-employees:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9106
--- a/apps/nxtgauge-backend-rust/base/fitness-trainers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/fitness-trainers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: fitness-trainers
-          image: registry.nxtgauge.com/nxtgauge-rust-fitness-trainers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-fitness-trainers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9114
--- a/apps/nxtgauge-backend-rust/base/gateway-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/gateway-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: gateway
-          image: registry.nxtgauge.com/nxtgauge-rust-gateway:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-gateway:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9100
--- a/apps/nxtgauge-backend-rust/base/graphic-designers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/graphic-designers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: graphic-designers
-          image: registry.nxtgauge.com/nxtgauge-rust-graphic-designers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-graphic-designers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9112
--- a/apps/nxtgauge-backend-rust/base/job-seekers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/job-seekers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: job-seekers
-          image: registry.nxtgauge.com/nxtgauge-rust-job-seekers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-job-seekers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9104
--- a/apps/nxtgauge-backend-rust/base/jobs-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/jobs-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: jobs
-          image: registry.nxtgauge.com/nxtgauge-rust-jobs:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-jobs:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9103
--- a/apps/nxtgauge-backend-rust/base/leads-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/leads-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: leads
-          image: registry.nxtgauge.com/nxtgauge-rust-leads:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-leads:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9118
--- a/apps/nxtgauge-backend-rust/base/makeup-artists-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/makeup-artists-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: makeup-artists
-          image: registry.nxtgauge.com/nxtgauge-rust-makeup-artists:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-makeup-artists:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9109
--- a/apps/nxtgauge-backend-rust/base/payments-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/payments-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: payments
-          image: registry.nxtgauge.com/nxtgauge-rust-payments:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-payments:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9116
--- a/apps/nxtgauge-backend-rust/base/photographers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/photographers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: photographers
-          image: registry.nxtgauge.com/nxtgauge-rust-photographers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-photographers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9107
--- a/apps/nxtgauge-backend-rust/base/social-media-managers-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/social-media-managers-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: social-media-managers
-          image: registry.nxtgauge.com/nxtgauge-rust-social-media-managers:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-social-media-managers:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9113
--- a/apps/nxtgauge-backend-rust/base/tutors-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/tutors-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: tutors
-          image: registry.nxtgauge.com/nxtgauge-rust-tutors:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-tutors:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9108
--- a/apps/nxtgauge-backend-rust/base/ugc-content-creators-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/ugc-content-creators-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: ugc-content-creators
-          image: registry.nxtgauge.com/nxtgauge-rust-ugc-content-creators:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-ugc-content-creators:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9117
--- a/apps/nxtgauge-backend-rust/base/users-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/users-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: users
-          image: registry.nxtgauge.com/nxtgauge-rust-users:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-users:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9101
--- a/apps/nxtgauge-backend-rust/base/video-editors-deployment.yaml
+++ b/apps/nxtgauge-backend-rust/base/video-editors-deployment.yaml
@ -18,7 +18,7 @@ spec:
        - name: regcred
      containers:
        - name: video-editors
-          image: registry.nxtgauge.com/nxtgauge-rust-video-editors:319b384f0a286ace38b0ac3f0602ae46d459b6f5
+          image: registry.nxtgauge.com/nxtgauge-rust-video-editors:latest
          imagePullPolicy: Always
          ports:
            - containerPort: 9111
--- a/apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml
+++ b/apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml
@ -10,42 +10,42 @@ patches:
      name: nxtgauge-rust-gateway
 images:
  - name: registry.nxtgauge.com/nxtgauge-rust-catering-services
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-companies
-    newTag: e6d85ffc8367885050b9434494f291724cc523c0
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-cron
-    newTag: d0b768d602b4d27bfd2363ef591f17c3e8f7bef1
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-customers
-    newTag: d0b768d602b4d27bfd2363ef591f17c3e8f7bef1
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-developers
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-employees
-    newTag: c7fe1b7ad35f7dcec44e9c5602d7f1764dfd5602
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-fitness-trainers
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-gateway
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-graphic-designers
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-jobs
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-job-seekers
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-leads
-    newTag: d0b768d602b4d27bfd2363ef591f17c3e8f7bef1
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-makeup-artists
-    newTag: 682f5ac19e7d150cd761b1876a6396d8c757b931
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-payments
-    newTag: c7fe1b7ad35f7dcec44e9c5602d7f1764dfd5602
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-photographers
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-social-media-managers
-    newTag: c7fe1b7ad35f7dcec44e9c5602d7f1764dfd5602
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-tutors
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-ugc-content-creators
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-users
-    newTag: 319b384f0a286ace38b0ac3f0602ae46d459b6f5
+    newTag: latest
  - name: registry.nxtgauge.com/nxtgauge-rust-video-editors
-    newTag: d0b768d602b4d27bfd2363ef591f17c3e8f7bef1
+    newTag: latest
--- a/apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml
+++ b/apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml
@ -7,4 +7,4 @@ patchesStrategicMerge:
  - replicas-patch.yaml
 images:
  - name: registry.nxtgauge.com/nxtgauge-frontend-solid
-    newTag: 4c61bca
+    newTag: latest
--- a/apps/postgresql/statefulset.yaml
+++ b/apps/postgresql/statefulset.yaml
@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pg-postgresql
+  namespace: data
+spec:
+  serviceName: pg-postgresql
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pg-postgresql
+  template:
+    metadata:
+      labels:
+        app: pg-postgresql
+    spec:
+      containers:
+      - name: postgresql
+        image: postgres:16-alpine
+        ports:
+        - name: tcp-postgresql
+          containerPort: 5432
+        env:
+        - name: POSTGRES_USER
+          value: postgres
+        - name: POSTGRES_PASSWORD
+          value: chandan2026@1
+        - name: POSTGRES_DB
+          value: nxtgauge
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      storageClassName: longhorn-2
+      resources:
+        requests:
+          storage: 30Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pg-postgresql
+  namespace: data
+spec:
+  type: NodePort
+  selector:
+    app: pg-postgresql
+  ports:
+  - name: tcp-postgresql
+    port: 5432
+    targetPort: 5432
+    nodePort: 30870
--- a/apps/registry/registry.yaml
+++ b/apps/registry/registry.yaml
@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: docker-registry
+  namespace: registry
+spec:
+  serviceName: docker-registry
+  replicas: 1
+  selector:
+    matchLabels:
+      app: docker-registry
+  template:
+    metadata:
+      labels:
+        app: docker-registry
+    spec:
+      containers:
+      - name: registry
+        image: registry:3
+        ports:
+        - containerPort: 5000
+          name: registry
+        env:
+        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
+          value: /var/lib/registry
+        - name: REGISTRY_AUTH
+          value: htpasswd
+        - name: REGISTRY_AUTH_HTPASSWD_REALM
+          value: Registry Realm
+        - name: REGISTRY_AUTH_HTPASSWD_PATH
+          value: /auth/htpasswd
+        volumeMounts:
+        - name: registry-storage
+          mountPath: /var/lib/registry
+        - name: auth
+          mountPath: /auth
+          readOnly: true
+      volumes:
+      - name: auth
+        secret:
+          secretName: registry-auth
+  volumeClaimTemplates:
+  - metadata:
+      name: registry-storage
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      storageClassName: longhorn
+      resources:
+        requests:
+          storage: 30Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: docker-registry
+  namespace: registry
+spec:
+  selector:
+    app: docker-registry
+  ports:
+  - port: 5000
+    targetPort: 5000
+  clusterIP: 10.43.17.31
--- a/apps/registry/retention-script.yaml
+++ b/apps/registry/retention-script.yaml
@ -10,17 +10,28 @@ data:
    CFG='/auth/.dockerconfigjson'
    PATTERN=re.compile(r'^[0-9a-f]{40}$')

-    # Base images that MUST NEVER be deleted, even if their names start with
-    # nxtgauge- in the future. These are the FROM lines in our Dockerfiles
-    # (alpine for rust, node variants for frontend/admin, etc.). If any of
-    # these are missing the entire build pipeline breaks.
+    # Base images that MUST NEVER be deleted. These are FROM lines in our Dockerfiles.
+    # They are pulled from Docker Hub and pushed to our private registry for reliability.
+    # If any are deleted, the entire build pipeline breaks.
    BASE_IMAGES = {
-        'alpine',  # runtime base image
-        'node',    # frontend/admin builder
-        'rust',    # backend builder
-        # Note: postgres/redis are in docker-compose (Docker Hub), not in registry
-        # busybox/golang/nginx are not used
+        'alpine',      # runtime base image
+        'node',        # frontend/admin builder
+        'rust',        # backend builder
+        'python',      # used by retention cronjob and other tools
+        'docker',      # dind for forgejo runner
+        'busybox',     # init containers
+        'registry',    # docker registry itself
    }
+    # Additional patterns to NEVER delete - images matching these patterns are protected
+    PROTECTED_PATTERNS = [
+        'node:',         # any node tag
+        'rust:',         # any rust tag
+        'alpine:',       # any alpine tag
+        'python:',       # any python tag
+        'docker:',       # docker dind images
+        'busybox:',      # busybox images
+        'registry:',     # registry images
+    ]
    # Project-image prefix that we DO prune. Anything outside this is sacred.
    PROJECT_PREFIX = 'nxtgauge-'

@ -40,16 +51,25 @@ data:
    all_repos=json.loads(body.decode()).get('repositories',[])

    # EXPLICIT SAFETY: only consider repos that match the project prefix.
-    # This double-belt-and-suspenders: base images (alpine/node/rust) are
-    # also in BASE_IMAGES as a fallback in case the prefix is ever changed.
-    repos=[r for r in all_repos if r.startswith(PROJECT_PREFIX) and r not in BASE_IMAGES]
+    # Protected base images (alpine/node/rust/python/docker/busybox/registry) are NEVER deleted.
+    def is_protected(repo_name):
+        """Check if a repo is protected - base images or matches protected patterns"""
+        if repo_name in BASE_IMAGES:
+            return True
+        for pattern in PROTECTED_PATTERNS:
+            if repo_name.startswith(pattern.rstrip(':')):
+                return True
+        return False
+
+    repos=[r for r in all_repos if r.startswith(PROJECT_PREFIX) and not is_protected(r)]

    # Sanity check: log if any base image is missing
-    missing_base = [b for b in BASE_IMAGES if b in all_repos or True]  # always present
    present = set(all_repos)
    for b in BASE_IMAGES:
      if b not in present:
        print(f'[WARN] base image {b} not in registry catalog - re-push required!')
+      else:
+        print(f'[PROTECTED] base image {b} will NEVER be deleted')

    deleted=0
    for repo in sorted(repos):
@ -81,11 +101,13 @@ data:
          created='9999-12-31T23:59:59Z'
        rows.append((created, t, digest))

-      rows.sort(key=lambda x: x[0], reverse=True)
-      KEEP_N=2   # keep last 2 SHA builds (current + 1 previous)
+  rows.sort(key=lambda x: x[0], reverse=True)
+      KEEP_N=10   # keep last 10 SHA builds (current + 9 previous)
      keep_set=set(t for _, t, _ in rows[:KEEP_N])
      # preserve buildcache for performance
      keep_set.update(t for t in tags if t == 'buildcache')
+      # always keep 'latest' tag
+      keep_set.update(t for t in tags if t == 'latest')
      keep_list=sorted(keep_set)
      print(f'[{repo}] sha_total={len(rows)} keep={keep_list} remove={max(0, len(rows)-len(keep_set))}')
      for _, t, d in rows:
--- a/scripts/build-all-services.sh
+++ b/scripts/build-all-services.sh
@ -0,0 +1,77 @@
+#!/bin/bash
+# Build and push all nxtgauge backend services
+
+set -e
+
+REGISTRY="registry.nxtgauge.com"
+REGISTRY_USER="admin"
+REGISTRY_PASS="Ashwin@2026"
+
+# Login to registry
+echo "Logging into registry..."
+echo "$REGISTRY_PASS" | docker login $REGISTRY -u $REGISTRY_USER --password-stdin
+
+cd /home/ashwin/nxtgauge-projects/nxtgauge-backend-rust
+
+# Get current git SHA for tagging
+SHA=$(git rev-parse --short HEAD)
+echo "Building with SHA tag: $SHA"
+
+# Services to build
+SERVICES=(
+  "catering-services"
+  "companies"
+  "cron"
+  "customers"
+  "developers"
+  "employees"
+  "fitness-trainers"
+  "gateway"
+  "graphic-designers"
+  "job-seekers"
+  "jobs"
+  "leads"
+  "makeup-artists"
+  "payments"
+  "photographers"
+  "social-media-managers"
+  "tutors"
+  "ugc-content-creators"
+  "users"
+  "video-editors"
+)
+
+# Build each service
+for service in "${SERVICES[@]}"; do
+  echo ""
+  echo "=================================="
+  echo "Building $service..."
+  echo "=================================="
+  
+  # Get binary name (convert dashes to underscores for Rust naming)
+  bin_name=$(echo "$service" | tr '-' '_')
+  
+  # Check if Dockerfile exists
+  if [ ! -f "apps/$service/Dockerfile" ]; then
+    echo "Dockerfile not found for $service, skipping..."
+    continue
+  fi
+  
+  # Build using the service's Dockerfile
+  docker build -f "apps/$service/Dockerfile" \
+    -t "$REGISTRY/nxtgauge-rust-$service:$SHA" \
+    -t "$REGISTRY/nxtgauge-rust-$service:latest" \
+    .
+  
+  # Push images
+  echo "Pushing $service:$SHA..."
+  docker push "$REGISTRY/nxtgauge-rust-$service:$SHA"
+  docker push "$REGISTRY/nxtgauge-rust-$service:latest"
+  
+  echo "$service built and pushed successfully!"
+done
+
+echo ""
+echo "=================================="
+echo "All services built and pushed!"
+echo "=================================="
--- a/scripts/sync-base-images.sh
+++ b/scripts/sync-base-images.sh
@ -0,0 +1,70 @@
+#!/bin/bash
+# Script to sync base images to registry.nxtgauge.com
+# Run this on one of the cluster nodes that has Docker access
+
+REGISTRY="registry.nxtgauge.com"
+REGISTRY_USER="admin"
+
+echo "=================================="
+echo "Syncing base images to $REGISTRY"
+echo "=================================="
+
+# Get registry password from secret
+REGISTRY_PASS=$(kubectl get secret -n registry registry-auth -o jsonpath='{.data.htpasswd}' | base64 -d | cut -d':' -f2)
+
+# Login to registry
+echo "Logging into registry..."
+echo "$REGISTRY_PASS" | docker login $REGISTRY -u $REGISTRY_USER --password-stdin 2>/dev/null || {
+    echo "Failed to login to registry. Make sure you're running this on a node with kubectl access."
+    exit 1
+}
+
+# Base images required by the project Dockerfiles
+images=(
+    "node:20-alpine"
+    "rust:alpine"
+    "alpine:3.20"
+    "alpine:latest"
+    "python:3.12-slim"
+    "docker:27-dind"
+    "busybox:1.36"
+)
+
+for image in "${images[@]}"; do
+    echo ""
+    echo "Processing $image..."
+    
+    # Pull from Docker Hub
+    echo "  Pulling docker.io/$image..."
+    if docker pull docker.io/library/$image 2>/dev/null; then
+        SOURCE="docker.io/library/$image"
+    elif docker pull docker.io/$image 2>/dev/null; then
+        SOURCE="docker.io/$image"
+    else
+        echo "  ERROR: Failed to pull $image"
+        continue
+    fi
+    
+    # Tag for registry
+    TARGET="$REGISTRY/$image"
+    echo "  Tagging as $TARGET..."
+    docker tag $SOURCE $TARGET
+    
+    # Push to registry
+    echo "  Pushing to $REGISTRY..."
+    if docker push $TARGET; then
+        echo "  SUCCESS: $image synced"
+    else
+        echo "  ERROR: Failed to push $image"
+    fi
+done
+
+echo ""
+echo "=================================="
+echo "Base image sync complete!"
+echo "=================================="
+echo ""
+echo "Synced images:"
+for image in "${images[@]}"; do
+    echo "  - $REGISTRY/$image"
+done