ashwin/nxtgauge-gitops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(registry): use node-resolvable backend registry endpoint and add k3s registries runbook

Browse source
This commit is contained in:
Ashwin Kumar 2026-04-11 21:59:50 +02:00
parent d2e0a11ed7
commit 96bc5aa42a
23 changed files with 158 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/nxtgauge-backend-rust/base/catering-services-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: catering-services
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-catering-services
image: registry.nxtgauge.internal:5000/nxtgauge-rust-catering-services
imagePullPolicy: Always
ports:
- containerPort: 9115

2
apps/nxtgauge-backend-rust/base/companies-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: companies
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-companies
image: registry.nxtgauge.internal:5000/nxtgauge-rust-companies
imagePullPolicy: Always
ports:
- containerPort: 9102

2
apps/nxtgauge-backend-rust/base/cron-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: cron
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-cron
image: registry.nxtgauge.internal:5000/nxtgauge-rust-cron
imagePullPolicy: Always
envFrom:
- configMapRef:

2
apps/nxtgauge-backend-rust/base/customers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: customers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-customers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-customers
imagePullPolicy: Always
ports:
- containerPort: 9105

2
apps/nxtgauge-backend-rust/base/developers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: developers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-developers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-developers
imagePullPolicy: Always
ports:
- containerPort: 9110

2
apps/nxtgauge-backend-rust/base/employees-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: employees
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-employees
image: registry.nxtgauge.internal:5000/nxtgauge-rust-employees
imagePullPolicy: Always
ports:
- containerPort: 9106

2
apps/nxtgauge-backend-rust/base/fitness-trainers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: fitness-trainers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-fitness-trainers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-fitness-trainers
imagePullPolicy: Always
ports:
- containerPort: 9114

2
apps/nxtgauge-backend-rust/base/gateway-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: gateway
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-gateway
image: registry.nxtgauge.internal:5000/nxtgauge-rust-gateway
imagePullPolicy: Always
ports:
- containerPort: 9100

2
apps/nxtgauge-backend-rust/base/graphic-designers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: graphic-designers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-graphic-designers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-graphic-designers
imagePullPolicy: Always
ports:
- containerPort: 9112

2
apps/nxtgauge-backend-rust/base/job-seekers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: job-seekers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-job-seekers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-job-seekers
imagePullPolicy: Always
ports:
- containerPort: 9104

2
apps/nxtgauge-backend-rust/base/makeup-artists-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: makeup-artists
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-makeup-artists
image: registry.nxtgauge.internal:5000/nxtgauge-rust-makeup-artists
imagePullPolicy: Always
ports:
- containerPort: 9109

2
apps/nxtgauge-backend-rust/base/payments-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: payments
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-payments
image: registry.nxtgauge.internal:5000/nxtgauge-rust-payments
imagePullPolicy: Always
ports:
- containerPort: 9116

2
apps/nxtgauge-backend-rust/base/photographers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: photographers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-photographers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-photographers
imagePullPolicy: Always
ports:
- containerPort: 9107

2
apps/nxtgauge-backend-rust/base/social-media-managers-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: social-media-managers
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-social-media-managers
image: registry.nxtgauge.internal:5000/nxtgauge-rust-social-media-managers
imagePullPolicy: Always
ports:
- containerPort: 9113

2
apps/nxtgauge-backend-rust/base/tutors-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: tutors
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-tutors
image: registry.nxtgauge.internal:5000/nxtgauge-rust-tutors
imagePullPolicy: Always
ports:
- containerPort: 9108

2
apps/nxtgauge-backend-rust/base/ugc-content-creators-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: ugc-content-creators
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-ugc-content-creators
image: registry.nxtgauge.internal:5000/nxtgauge-rust-ugc-content-creators
imagePullPolicy: Always
ports:
- containerPort: 9117

2
apps/nxtgauge-backend-rust/base/users-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: users
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-users
image: registry.nxtgauge.internal:5000/nxtgauge-rust-users
imagePullPolicy: Always
ports:
- containerPort: 9101

2
apps/nxtgauge-backend-rust/base/video-editors-deployment.yaml
View file

@ -17,7 +17,7 @@ spec:
spec:
containers:
- name: video-editors
image: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-video-editors
image: registry.nxtgauge.internal:5000/nxtgauge-rust-video-editors
imagePullPolicy: Always
ports:
- containerPort: 9111

36
apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml
View file

@ -5,39 +5,39 @@ resources:
patchesStrategicMerge:
- replicas-patch.yaml
images:
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-gateway
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-gateway
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-users
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-users
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-companies
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-companies
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-job-seekers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-job-seekers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-customers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-customers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-payments
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-payments
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-employees
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-employees
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-photographers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-photographers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-makeup-artists
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-makeup-artists
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-tutors
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-tutors
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-developers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-developers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-video-editors
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-video-editors
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-graphic-designers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-graphic-designers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-social-media-managers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-social-media-managers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-fitness-trainers
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-fitness-trainers
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-catering-services
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-catering-services
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-ugc-content-creators
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-ugc-content-creators
newTag: high-performance-latest
- name: docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-cron
- name: registry.nxtgauge.internal:5000/nxtgauge-rust-cron
newTag: high-performance-latest

36
argocd/nxtgauge-backend-rust-application.yaml
View file

@ -5,24 +5,24 @@ metadata:
namespace: argocd
annotations:
argocd-image-updater.argoproj.io/image-list: >-
gateway=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-gateway:high-performance-latest,
users=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-users:high-performance-latest,
companies=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-companies:high-performance-latest,
job-seekers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-job-seekers:high-performance-latest,
customers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-customers:high-performance-latest,
payments=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-payments:high-performance-latest,
employees=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-employees:high-performance-latest,
photographers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-photographers:high-performance-latest,
makeup-artists=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-makeup-artists:high-performance-latest,
tutors=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-tutors:high-performance-latest,
developers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-developers:high-performance-latest,
video-editors=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-video-editors:high-performance-latest,
graphic-designers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-graphic-designers:high-performance-latest,
social-media-managers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-social-media-managers:high-performance-latest,
fitness-trainers=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-fitness-trainers:high-performance-latest,
catering-services=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-catering-services:high-performance-latest,
ugc-content-creators=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-ugc-content-creators:high-performance-latest,
cron=docker-registry.registry.svc.cluster.local:5000/nxtgauge-rust-cron:high-performance-latest
gateway=registry.nxtgauge.internal:5000/nxtgauge-rust-gateway:high-performance-latest,
users=registry.nxtgauge.internal:5000/nxtgauge-rust-users:high-performance-latest,
companies=registry.nxtgauge.internal:5000/nxtgauge-rust-companies:high-performance-latest,
job-seekers=registry.nxtgauge.internal:5000/nxtgauge-rust-job-seekers:high-performance-latest,
customers=registry.nxtgauge.internal:5000/nxtgauge-rust-customers:high-performance-latest,
payments=registry.nxtgauge.internal:5000/nxtgauge-rust-payments:high-performance-latest,
employees=registry.nxtgauge.internal:5000/nxtgauge-rust-employees:high-performance-latest,
photographers=registry.nxtgauge.internal:5000/nxtgauge-rust-photographers:high-performance-latest,
makeup-artists=registry.nxtgauge.internal:5000/nxtgauge-rust-makeup-artists:high-performance-latest,
tutors=registry.nxtgauge.internal:5000/nxtgauge-rust-tutors:high-performance-latest,
developers=registry.nxtgauge.internal:5000/nxtgauge-rust-developers:high-performance-latest,
video-editors=registry.nxtgauge.internal:5000/nxtgauge-rust-video-editors:high-performance-latest,
graphic-designers=registry.nxtgauge.internal:5000/nxtgauge-rust-graphic-designers:high-performance-latest,
social-media-managers=registry.nxtgauge.internal:5000/nxtgauge-rust-social-media-managers:high-performance-latest,
fitness-trainers=registry.nxtgauge.internal:5000/nxtgauge-rust-fitness-trainers:high-performance-latest,
catering-services=registry.nxtgauge.internal:5000/nxtgauge-rust-catering-services:high-performance-latest,
ugc-content-creators=registry.nxtgauge.internal:5000/nxtgauge-rust-ugc-content-creators:high-performance-latest,
cron=registry.nxtgauge.internal:5000/nxtgauge-rust-cron:high-performance-latest
argocd-image-updater.argoproj.io/gateway.update-strategy: digest
argocd-image-updater.argoproj.io/gateway.allow-tags: regexp:^high-performance-latest$
argocd-image-updater.argoproj.io/users.update-strategy: digest

55
ops/k3s/README.md Normal file
View file

@ -0,0 +1,55 @@
# k3s Local Registry Node Configuration
This repo now uses `registry.nxtgauge.internal:5000` for backend images.
## Why
Image pulls happen on k3s nodes via containerd, not inside cluster DNS context.
Using `*.svc.cluster.local` for image pulls can fail with DNS lookup errors from node runtime.
## Required node config
Each node must have `/etc/rancher/k3s/registries.yaml` configured to trust and use the registry.
Template file:
- `ops/k3s/registries.yaml`
## Apply to all nodes
1. Export required env vars:
```bash
export K3S_NODES="node1 node2 node3"
export REGISTRY_USERNAME="<registry-user>"
export REGISTRY_PASSWORD="<registry-pass>"
```
2. Apply config and restart k3s on each node:
```bash
./ops/k3s/apply-registries.sh
```
## Manual steps (if needed)
On each node:
1. Copy `registries.yaml` to `/etc/rancher/k3s/registries.yaml`
2. Restart runtime:
```bash
sudo systemctl restart k3s
# or for agents
sudo systemctl restart k3s-agent
```
3. Verify pod pulls:
```bash
kubectl -n nxtgauge get pods
kubectl -n nxtgauge describe pod <failing-pod>
```
## Notes
- Ensure DNS for `registry.nxtgauge.internal` resolves from every k3s node.
- If DNS is not available, use a stable node-reachable IP:port and update:
- backend GitOps manifests
- backend Woodpecker registry push target
- `ops/k3s/registries.yaml`

37
ops/k3s/apply-registries.sh Executable file
View file

@ -0,0 +1,37 @@
#!/usr/bin/env bash
set -euo pipefail
# Usage:
# export K3S_NODES="node1 node2 node3"
# export REGISTRY_USERNAME="..."
# export REGISTRY_PASSWORD="..."
# ./ops/k3s/apply-registries.sh
if [[ -z "${K3S_NODES:-}" ]]; then
echo "K3S_NODES is required (space-separated ssh targets)"
exit 1
fi
if [[ -z "${REGISTRY_USERNAME:-}" || -z "${REGISTRY_PASSWORD:-}" ]]; then
echo "REGISTRY_USERNAME and REGISTRY_PASSWORD are required"
exit 1
fi
TMP_FILE="$(mktemp)"
sed \
-e "s#\${REGISTRY_USERNAME}#${REGISTRY_USERNAME}#g" \
-e "s#\${REGISTRY_PASSWORD}#${REGISTRY_PASSWORD}#g" \
"$(dirname "$0")/registries.yaml" > "$TMP_FILE"
for node in ${K3S_NODES}; do
echo "Applying registry config on ${node}"
scp "$TMP_FILE" "${node}:/tmp/registries.yaml"
ssh "$node" "sudo mkdir -p /etc/rancher/k3s && sudo mv /tmp/registries.yaml /etc/rancher/k3s/registries.yaml && sudo systemctl restart k3s || sudo systemctl restart k3s-agent"
echo "Waiting for ${node} to recover..."
sleep 8
done
rm -f "$TMP_FILE"
echo "Done: registries.yaml applied and k3s restarted on all nodes."

12
ops/k3s/registries.yaml Normal file
View file

@ -0,0 +1,12 @@
mirrors:
"registry.nxtgauge.internal:5000":
endpoint:
- "http://registry.nxtgauge.internal:5000"
configs:
"registry.nxtgauge.internal:5000":
tls:
insecure_skip_verify: true
auth:
username: "${REGISTRY_USERNAME}"
password: "${REGISTRY_PASSWORD}"