From 75acea11eb5c317359e63fdf835d088e47304d03 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Sivakumar Date: Fri, 17 Apr 2026 05:25:04 +0530 Subject: [PATCH] fix: registry ingress + woodpecker pulls + registry dns overrides --- .../overlays/prod/kustomization.yaml | 2 +- .../overlays/prod/kustomization.yaml | 6 ++-- .../overlays/prod/kustomization.yaml | 2 +- argocd/coredns-nodehosts-application.yaml | 21 +++++++++++ argocd/registry-ingress-application.yaml | 21 +++++++++++ .../woodpecker-registry-pull-application.yaml | 21 +++++++++++ ops/coredns-nodehosts/kustomization.yaml | 6 ++++ .../patch-coredns-nodehosts.yaml | 12 +++++++ ops/k3s/README.md | 19 +++++++--- ops/k3s/apply-registries.sh | 10 +++++- ops/k3s/registries.yaml | 8 ++--- ops/registry-ingress/kustomization.yaml | 6 ++++ ops/registry-ingress/registry-ingress.yaml | 27 ++++++++++++++ ops/woodpecker-registry-pull/README.md | 35 +++++++++++++++++++ .../kustomization.yaml | 6 ++++ .../serviceaccount-default.yaml | 8 +++++ 16 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 argocd/coredns-nodehosts-application.yaml create mode 100644 argocd/registry-ingress-application.yaml create mode 100644 argocd/woodpecker-registry-pull-application.yaml create mode 100644 ops/coredns-nodehosts/kustomization.yaml create mode 100644 ops/coredns-nodehosts/patch-coredns-nodehosts.yaml create mode 100644 ops/registry-ingress/kustomization.yaml create mode 100644 ops/registry-ingress/registry-ingress.yaml create mode 100644 ops/woodpecker-registry-pull/README.md create mode 100644 ops/woodpecker-registry-pull/kustomization.yaml create mode 100644 ops/woodpecker-registry-pull/serviceaccount-default.yaml diff --git a/apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml b/apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml index 53dd576..2e23396 100644 --- a/apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml +++ b/apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml @@ -6,4 +6,4 @@ patchesStrategicMerge: - replicas-patch.yaml images: - name: registry.nxtgauge.com/nxtgauge-admin-solid - newTag: e044d4c + newTag: high-performance-latest diff --git a/apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml b/apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml index 96a3edd..0f5ba1c 100644 --- a/apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml +++ b/apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml @@ -9,11 +9,11 @@ patches: name: nxtgauge-rust-gateway images: - name: registry.nxtgauge.com/nxtgauge-rust-gateway - newTag: d084491 + newTag: high-performance-latest - name: registry.nxtgauge.com/nxtgauge-rust-users - newTag: 9444056 + newTag: high-performance-latest - name: registry.nxtgauge.com/nxtgauge-frontend-solid - newTag: 152f918 + newTag: high-performance-latest - name: registry.nxtgauge.com/nxtgauge-rust-companies newTag: high-performance-latest - name: registry.nxtgauge.com/nxtgauge-rust-job-seekers diff --git a/apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml b/apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml index 937194c..f3559fd 100644 --- a/apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml +++ b/apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml @@ -6,4 +6,4 @@ patchesStrategicMerge: - replicas-patch.yaml images: - name: registry.nxtgauge.com/nxtgauge-frontend-solid - newTag: d26f0bf + newTag: high-performance-latest diff --git a/argocd/coredns-nodehosts-application.yaml b/argocd/coredns-nodehosts-application.yaml new file mode 100644 index 0000000..2d38ac1 --- /dev/null +++ b/argocd/coredns-nodehosts-application.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: coredns-nodehosts + namespace: argocd +spec: + destination: + namespace: kube-system + server: https://kubernetes.default.svc + project: default + source: + path: ops/coredns-nodehosts + repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + diff --git a/argocd/registry-ingress-application.yaml b/argocd/registry-ingress-application.yaml new file mode 100644 index 0000000..2436af1 --- /dev/null +++ b/argocd/registry-ingress-application.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: registry-ingress + namespace: argocd +spec: + destination: + namespace: registry + server: https://kubernetes.default.svc + project: default + source: + path: ops/registry-ingress + repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + diff --git a/argocd/woodpecker-registry-pull-application.yaml b/argocd/woodpecker-registry-pull-application.yaml new file mode 100644 index 0000000..bcc3ad1 --- /dev/null +++ b/argocd/woodpecker-registry-pull-application.yaml @@ -0,0 +1,21 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: woodpecker-registry-pull + namespace: argocd +spec: + destination: + namespace: woodpecker + server: https://kubernetes.default.svc + project: default + source: + path: ops/woodpecker-registry-pull + repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git + targetRevision: main + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + diff --git a/ops/coredns-nodehosts/kustomization.yaml b/ops/coredns-nodehosts/kustomization.yaml new file mode 100644 index 0000000..b4cf147 --- /dev/null +++ b/ops/coredns-nodehosts/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - patch-coredns-nodehosts.yaml + diff --git a/ops/coredns-nodehosts/patch-coredns-nodehosts.yaml b/ops/coredns-nodehosts/patch-coredns-nodehosts.yaml new file mode 100644 index 0000000..b2731a2 --- /dev/null +++ b/ops/coredns-nodehosts/patch-coredns-nodehosts.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + NodeHosts: | + 10.0.0.2 nxtgauge-1 + 10.0.0.3 nxtgauge-2 + 10.0.0.5 nxtgauge-3 + 10.0.0.2 registry.nxtgauge.com + diff --git a/ops/k3s/README.md b/ops/k3s/README.md index 0c5c676..eb5aa33 100644 --- a/ops/k3s/README.md +++ b/ops/k3s/README.md @@ -1,17 +1,25 @@ -# k3s Local Registry Node Configuration +# k3s Registry Node Configuration -This repo now uses `registry.nxtgauge.internal:5000` for backend images. +This repo uses `registry.nxtgauge.com` for backend images. ## Why Image pulls happen on k3s nodes via containerd, not inside cluster DNS context. Using `*.svc.cluster.local` for image pulls can fail with DNS lookup errors from node runtime. ## Required node config -Each node must have `/etc/rancher/k3s/registries.yaml` configured to trust and use the registry. +Each node must have `/etc/rancher/k3s/registries.yaml` configured with auth for the registry. Template file: - `ops/k3s/registries.yaml` +## Recommended node DNS/hosts override (prevents Cloudflare/proxy path) +Even if `registry.nxtgauge.com` is set to "DNS only" in Cloudflare, k3s nodes can still end up resolving to public/IPv6 records depending on upstream DNS/caches. + +For reliable large image pulls/pushes (avoids `413 Payload Too Large` from proxies), point nodes directly at the in-cluster ingress VIP: + +- Traefik VIPs: `10.0.0.2`, `10.0.0.3`, `10.0.0.5` +- Recommended: pick one stable VIP (example `10.0.0.2`) and map `registry.nxtgauge.com` to it on every node. + ## Apply to all nodes 1. Export required env vars: @@ -20,6 +28,7 @@ Template file: export K3S_NODES="node1 node2 node3" export REGISTRY_USERNAME="" export REGISTRY_PASSWORD="" +export REGISTRY_VIP_IP="10.0.0.2" # optional but recommended ``` 2. Apply config and restart k3s on each node: @@ -48,8 +57,8 @@ kubectl -n nxtgauge describe pod ``` ## Notes -- Ensure DNS for `registry.nxtgauge.internal` resolves from every k3s node. -- If DNS is not available, use a stable node-reachable IP:port and update: +- Ensure DNS for `registry.nxtgauge.com` resolves from every k3s node. +- If DNS is not available, use a stable node-reachable IP and update: - backend GitOps manifests - backend Woodpecker registry push target - `ops/k3s/registries.yaml` diff --git a/ops/k3s/apply-registries.sh b/ops/k3s/apply-registries.sh index e4246a0..ac70a8b 100755 --- a/ops/k3s/apply-registries.sh +++ b/ops/k3s/apply-registries.sh @@ -5,6 +5,7 @@ set -euo pipefail # export K3S_NODES="node1 node2 node3" # export REGISTRY_USERNAME="..." # export REGISTRY_PASSWORD="..." +# export REGISTRY_VIP_IP="10.0.0.2" # optional (recommended) # ./ops/k3s/apply-registries.sh if [[ -z "${K3S_NODES:-}" ]]; then @@ -27,7 +28,14 @@ sed \ for node in ${K3S_NODES}; do echo "Applying registry config on ${node}" scp "$TMP_FILE" "${node}:/tmp/registries.yaml" - ssh "$node" "sudo mkdir -p /etc/rancher/k3s && sudo mv /tmp/registries.yaml /etc/rancher/k3s/registries.yaml && sudo systemctl restart k3s || sudo systemctl restart k3s-agent" + ssh "$node" "sudo mkdir -p /etc/rancher/k3s && sudo mv /tmp/registries.yaml /etc/rancher/k3s/registries.yaml" + + if [[ -n "${REGISTRY_VIP_IP:-}" ]]; then + echo "Ensuring /etc/hosts contains registry.nxtgauge.com -> ${REGISTRY_VIP_IP} on ${node}" + ssh "$node" "sudo sh -lc 'grep -q \"\\sregistry\\.nxtgauge\\.com\\b\" /etc/hosts && sed -i \"s/^.*\\sregistry\\.nxtgauge\\.com\\b.*/${REGISTRY_VIP_IP} registry.nxtgauge.com/\" /etc/hosts || echo \"${REGISTRY_VIP_IP} registry.nxtgauge.com\" >> /etc/hosts'" + fi + + ssh "$node" "sudo systemctl restart k3s || sudo systemctl restart k3s-agent" echo "Waiting for ${node} to recover..." sleep 8 done diff --git a/ops/k3s/registries.yaml b/ops/k3s/registries.yaml index 23b2bf4..0b38cbc 100644 --- a/ops/k3s/registries.yaml +++ b/ops/k3s/registries.yaml @@ -1,12 +1,10 @@ mirrors: - "registry.nxtgauge.internal:5000": + "registry.nxtgauge.com": endpoint: - - "http://registry.nxtgauge.internal:5000" + - "https://registry.nxtgauge.com" configs: - "registry.nxtgauge.internal:5000": - tls: - insecure_skip_verify: true + "registry.nxtgauge.com": auth: username: "${REGISTRY_USERNAME}" password: "${REGISTRY_PASSWORD}" diff --git a/ops/registry-ingress/kustomization.yaml b/ops/registry-ingress/kustomization.yaml new file mode 100644 index 0000000..6a10161 --- /dev/null +++ b/ops/registry-ingress/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - registry-ingress.yaml + diff --git a/ops/registry-ingress/registry-ingress.yaml b/ops/registry-ingress/registry-ingress.yaml new file mode 100644 index 0000000..fca3c3a --- /dev/null +++ b/ops/registry-ingress/registry-ingress.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: docker-registry + namespace: registry + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.entrypoints: web,websecure + traefik.ingress.kubernetes.io/router.priority: "100" +spec: + ingressClassName: traefik + tls: + - hosts: + - registry.nxtgauge.com + secretName: registry-tls + rules: + - host: registry.nxtgauge.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: docker-registry + port: + number: 5000 + diff --git a/ops/woodpecker-registry-pull/README.md b/ops/woodpecker-registry-pull/README.md new file mode 100644 index 0000000..5bf1808 --- /dev/null +++ b/ops/woodpecker-registry-pull/README.md @@ -0,0 +1,35 @@ +# Woodpecker: allow pulling from private registry + +Woodpecker pipelines run as Kubernetes pods in the `woodpecker` namespace. If pipeline step images use `registry.nxtgauge.com/...` (private, Basic auth), kubelet needs an `imagePullSecret`. + +This is required for base images (example `registry.nxtgauge.com/rust:alpine`) and also for any mirrored plugin images (example `registry.nxtgauge.com/kaniko:2.1.1`). + +## Required secret + +Create this once: + +```bash +kubectl -n woodpecker create secret docker-registry registry-nxtgauge-pull \ + --docker-server=registry.nxtgauge.com \ + --docker-username="" \ + --docker-password="" \ + --docker-email="ci@nxtgauge.com" +``` + +## Mirroring common plugin images (optional) + +If your pipelines reference plugin images from the internal registry (example `registry.nxtgauge.com/kaniko:2.1.1`) make sure those images exist in the registry. + +Example mirror from Docker Hub to internal: + +```bash +docker pull woodpeckerci/plugin-kaniko:2.1.1 +docker tag woodpeckerci/plugin-kaniko:2.1.1 registry.nxtgauge.com/kaniko:2.1.1 +docker push registry.nxtgauge.com/kaniko:2.1.1 +``` + +## What this kustomize applies + +It patches/ensures the `default` ServiceAccount in `woodpecker` includes: + +- `imagePullSecrets: [registry-nxtgauge-pull]` diff --git a/ops/woodpecker-registry-pull/kustomization.yaml b/ops/woodpecker-registry-pull/kustomization.yaml new file mode 100644 index 0000000..b53baee --- /dev/null +++ b/ops/woodpecker-registry-pull/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceaccount-default.yaml + diff --git a/ops/woodpecker-registry-pull/serviceaccount-default.yaml b/ops/woodpecker-registry-pull/serviceaccount-default.yaml new file mode 100644 index 0000000..eedc19e --- /dev/null +++ b/ops/woodpecker-registry-pull/serviceaccount-default.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default + namespace: woodpecker +imagePullSecrets: + - name: registry-nxtgauge-pull +