fix(registry): use node-resolvable backend registry endpoint and add k3s registries runbook

apps/nxtgauge-backend-rust/base/catering-services-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: catering-services
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-catering-services
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-catering-services
           imagePullPolicy: Always
           ports:
             - containerPort: 9115

apps/nxtgauge-backend-rust/base/companies-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: companies
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-companies
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-companies
           imagePullPolicy: Always
           ports:
             - containerPort: 9102

apps/nxtgauge-backend-rust/base/cron-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: cron
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-cron
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-cron
           imagePullPolicy: Always
           envFrom:
             - configMapRef:

apps/nxtgauge-backend-rust/base/customers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: customers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-customers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-customers
           imagePullPolicy: Always
           ports:
             - containerPort: 9105

apps/nxtgauge-backend-rust/base/developers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: developers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-developers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-developers
           imagePullPolicy: Always
           ports:
             - containerPort: 9110

apps/nxtgauge-backend-rust/base/employees-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: employees
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-employees
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-employees
           imagePullPolicy: Always
           ports:
             - containerPort: 9106

apps/nxtgauge-backend-rust/base/fitness-trainers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: fitness-trainers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-fitness-trainers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-fitness-trainers
           imagePullPolicy: Always
           ports:
             - containerPort: 9114

apps/nxtgauge-backend-rust/base/gateway-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: gateway
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-gateway
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-gateway
           imagePullPolicy: Always
           ports:
             - containerPort: 9100

apps/nxtgauge-backend-rust/base/graphic-designers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: graphic-designers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-graphic-designers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-graphic-designers
           imagePullPolicy: Always
           ports:
             - containerPort: 9112

apps/nxtgauge-backend-rust/base/job-seekers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: job-seekers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-job-seekers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-job-seekers
           imagePullPolicy: Always
           ports:
             - containerPort: 9104

apps/nxtgauge-backend-rust/base/makeup-artists-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: makeup-artists
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-makeup-artists
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-makeup-artists
           imagePullPolicy: Always
           ports:
             - containerPort: 9109

apps/nxtgauge-backend-rust/base/payments-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: payments
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-payments
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-payments
           imagePullPolicy: Always
           ports:
             - containerPort: 9116

apps/nxtgauge-backend-rust/base/photographers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: photographers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-photographers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-photographers
           imagePullPolicy: Always
           ports:
             - containerPort: 9107

apps/nxtgauge-backend-rust/base/social-media-managers-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: social-media-managers
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-social-media-managers
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-social-media-managers
           imagePullPolicy: Always
           ports:
             - containerPort: 9113

apps/nxtgauge-backend-rust/base/tutors-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: tutors
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-tutors
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-tutors
           imagePullPolicy: Always
           ports:
             - containerPort: 9108

apps/nxtgauge-backend-rust/base/ugc-content-creators-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: ugc-content-creators
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-ugc-content-creators
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-ugc-content-creators
           imagePullPolicy: Always
           ports:
             - containerPort: 9117

apps/nxtgauge-backend-rust/base/users-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: users
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-users
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-users
           imagePullPolicy: Always
           ports:
             - containerPort: 9101

apps/nxtgauge-backend-rust/base/video-editors-deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: video-editors
-          image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-video-editors
+          image: registry.nxtgauge.internal:5000/nxtgauge-rust-video-editors
           imagePullPolicy: Always
           ports:
             - containerPort: 9111

apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml

@@ -5,39 +5,39 @@ resources:
 patchesStrategicMerge:
   - replicas-patch.yaml
 images:
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-gateway
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-gateway
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-users
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-users
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-companies
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-companies
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-job-seekers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-job-seekers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-customers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-customers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-payments
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-payments
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-employees
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-employees
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-photographers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-photographers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-makeup-artists
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-makeup-artists
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-tutors
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-tutors
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-developers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-developers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-video-editors
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-video-editors
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-graphic-designers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-graphic-designers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-social-media-managers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-social-media-managers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-fitness-trainers
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-fitness-trainers
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-catering-services
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-catering-services
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-ugc-content-creators
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-ugc-content-creators
     newTag: high-performance-latest
-  - name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-cron
+  - name: registry.nxtgauge.internal:5000/nxtgauge-rust-cron
     newTag: high-performance-latest

argocd/nxtgauge-backend-rust-application.yaml

@@ -5,24 +5,24 @@ metadata:
   namespace: argocd
   annotations:
     argocd-image-updater.argoproj.io/image-list: >-
-      gateway=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-gateway:high-performance-latest,
+      gateway=registry.nxtgauge.internal:5000/nxtgauge-rust-gateway:high-performance-latest,
-      users=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-users:high-performance-latest,
+      users=registry.nxtgauge.internal:5000/nxtgauge-rust-users:high-performance-latest,
-      companies=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-companies:high-performance-latest,
+      companies=registry.nxtgauge.internal:5000/nxtgauge-rust-companies:high-performance-latest,
-      job-seekers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-job-seekers:high-performance-latest,
+      job-seekers=registry.nxtgauge.internal:5000/nxtgauge-rust-job-seekers:high-performance-latest,
-      customers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-customers:high-performance-latest,
+      customers=registry.nxtgauge.internal:5000/nxtgauge-rust-customers:high-performance-latest,
-      payments=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-payments:high-performance-latest,
+      payments=registry.nxtgauge.internal:5000/nxtgauge-rust-payments:high-performance-latest,
-      employees=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-employees:high-performance-latest,
+      employees=registry.nxtgauge.internal:5000/nxtgauge-rust-employees:high-performance-latest,
-      photographers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-photographers:high-performance-latest,
+      photographers=registry.nxtgauge.internal:5000/nxtgauge-rust-photographers:high-performance-latest,
-      makeup-artists=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-makeup-artists:high-performance-latest,
+      makeup-artists=registry.nxtgauge.internal:5000/nxtgauge-rust-makeup-artists:high-performance-latest,
-      tutors=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-tutors:high-performance-latest,
+      tutors=registry.nxtgauge.internal:5000/nxtgauge-rust-tutors:high-performance-latest,
-      developers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-developers:high-performance-latest,
+      developers=registry.nxtgauge.internal:5000/nxtgauge-rust-developers:high-performance-latest,
-      video-editors=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-video-editors:high-performance-latest,
+      video-editors=registry.nxtgauge.internal:5000/nxtgauge-rust-video-editors:high-performance-latest,
-      graphic-designers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-graphic-designers:high-performance-latest,
+      graphic-designers=registry.nxtgauge.internal:5000/nxtgauge-rust-graphic-designers:high-performance-latest,
-      social-media-managers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-social-media-managers:high-performance-latest,
+      social-media-managers=registry.nxtgauge.internal:5000/nxtgauge-rust-social-media-managers:high-performance-latest,
-      fitness-trainers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-fitness-trainers:high-performance-latest,
+      fitness-trainers=registry.nxtgauge.internal:5000/nxtgauge-rust-fitness-trainers:high-performance-latest,
-      catering-services=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-catering-services:high-performance-latest,
+      catering-services=registry.nxtgauge.internal:5000/nxtgauge-rust-catering-services:high-performance-latest,
-      ugc-content-creators=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-ugc-content-creators:high-performance-latest,
+      ugc-content-creators=registry.nxtgauge.internal:5000/nxtgauge-rust-ugc-content-creators:high-performance-latest,
-      cron=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-cron:high-performance-latest
+      cron=registry.nxtgauge.internal:5000/nxtgauge-rust-cron:high-performance-latest
     argocd-image-updater.argoproj.io/gateway.update-strategy: digest
     argocd-image-updater.argoproj.io/gateway.allow-tags: regexp:^high-performance-latest$
     argocd-image-updater.argoproj.io/users.update-strategy: digest

ops/k3s/README.md

@@ -0,0 +1,55 @@
+# k3s Local Registry Node Configuration
+
+This repo now uses `registry.nxtgauge.internal:5000` for backend images.
+
+## Why
+Image pulls happen on k3s nodes via containerd, not inside cluster DNS context.
+Using `*.svc.cluster.local` for image pulls can fail with DNS lookup errors from node runtime.
+
+## Required node config
+Each node must have `/etc/rancher/k3s/registries.yaml` configured to trust and use the registry.
+
+Template file:
+- `ops/k3s/registries.yaml`
+
+## Apply to all nodes
+
+1. Export required env vars:
+
+```bash
+export K3S_NODES="node1 node2 node3"
+export REGISTRY_USERNAME="<registry-user>"
+export REGISTRY_PASSWORD="<registry-pass>"
+```
+
+2. Apply config and restart k3s on each node:
+
+```bash
+./ops/k3s/apply-registries.sh
+```
+
+## Manual steps (if needed)
+On each node:
+
+1. Copy `registries.yaml` to `/etc/rancher/k3s/registries.yaml`
+2. Restart runtime:
+
+```bash
+sudo systemctl restart k3s
+# or for agents
+sudo systemctl restart k3s-agent
+```
+
+3. Verify pod pulls:
+
+```bash
+kubectl -n nxtgauge get pods
+kubectl -n nxtgauge describe pod <failing-pod>
+```
+
+## Notes
+- Ensure DNS for `registry.nxtgauge.internal` resolves from every k3s node.
+- If DNS is not available, use a stable node-reachable IP:port and update:
+  - backend GitOps manifests
+  - backend Woodpecker registry push target
+  - `ops/k3s/registries.yaml`

ops/k3s/apply-registries.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage:
+#   export K3S_NODES="node1 node2 node3"
+#   export REGISTRY_USERNAME="..."
+#   export REGISTRY_PASSWORD="..."
+#   ./ops/k3s/apply-registries.sh
+
+if [[ -z "${K3S_NODES:-}" ]]; then
+  echo "K3S_NODES is required (space-separated ssh targets)"
+  exit 1
+fi
+
+if [[ -z "${REGISTRY_USERNAME:-}" || -z "${REGISTRY_PASSWORD:-}" ]]; then
+  echo "REGISTRY_USERNAME and REGISTRY_PASSWORD are required"
+  exit 1
+fi
+
+TMP_FILE="$(mktemp)"
+
+sed \
+  -e "s#\${REGISTRY_USERNAME}#${REGISTRY_USERNAME}#g" \
+  -e "s#\${REGISTRY_PASSWORD}#${REGISTRY_PASSWORD}#g" \
+  "$(dirname "$0")/registries.yaml" > "$TMP_FILE"
+
+for node in ${K3S_NODES}; do
+  echo "Applying registry config on ${node}"
+  scp "$TMP_FILE" "${node}:/tmp/registries.yaml"
+  ssh "$node" "sudo mkdir -p /etc/rancher/k3s && sudo mv /tmp/registries.yaml /etc/rancher/k3s/registries.yaml && sudo systemctl restart k3s || sudo systemctl restart k3s-agent"
+  echo "Waiting for ${node} to recover..."
+  sleep 8
+done
+
+rm -f "$TMP_FILE"
+
+echo "Done: registries.yaml applied and k3s restarted on all nodes."

ops/k3s/registries.yaml

@@ -0,0 +1,12 @@
+mirrors:
+  "registry.nxtgauge.internal:5000":
+    endpoint:
+      - "http://registry.nxtgauge.internal:5000"
+
+configs:
+  "registry.nxtgauge.internal:5000":
+    tls:
+      insecure_skip_verify: true
+    auth:
+      username: "${REGISTRY_USERNAME}"
+      password: "${REGISTRY_PASSWORD}"