perf(ci): parallelize service builds across all 3 runners via matrix strategy

The build was structured as a single job looping through all ~20
services sequentially, so only 1 of the 3 deployed runner pods (one
per worker node) was ever used - the other 2 sat idle for the entire
build. Switched to a static per-service matrix (max-parallel: 3,
matching the 3 runners at capacity 1 each) so independent services
build concurrently. Each matrix job does its own quick "does this
service need building" check up front (same change-detection logic,
now per-job) rather than relying on a shared job output, to avoid
needing cross-job artifact/output passing.

GitOps updates also move into each matrix job (previously a single
step at the end) since there's no longer one job aggregating all
results - added a fetch/reset/retry loop since multiple jobs can now
push to the same gitops branch concurrently.

Tried adding cargo registry/target cache mounts to Dockerfile.simple
for a bigger per-build win too, but measured it directly (local A/B:
cold build 2m37s vs a second, supposedly-cached build 7m18s) and it
made things slower here, likely cargo's own cache-verification pass
outweighing the benefit for this dependency set - reverted that part,
Dockerfile.simple is unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

.forgejo/workflows/build.yaml

@@ -13,6 +13,31 @@ concurrency:
 jobs:
   build:
     runs-on: docker-ready
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        service:
+          - gateway
+          - users
+          - companies
+          - jobs
+          - leads
+          - job-seekers
+          - customers
+          - payments
+          - employees
+          - photographers
+          - makeup-artists
+          - tutors
+          - developers
+          - video-editors
+          - graphic-designers
+          - social-media-managers
+          - fitness-trainers
+          - catering-services
+          - ugc-content-creators
+          - cron
     env:
       DOCKER_BUILDKIT: "1"
     steps:
@@ -21,7 +46,41 @@ jobs:
         with:
           fetch-depth: 0
 
+      # Static matrix (all 20 services every push) instead of a single job
+      # looping sequentially - up to 3 run concurrently (one per runner pod;
+      # each runner's capacity is 1). This step is each job's own quick
+      # "do I actually need to do anything" check, so unaffected services
+      # skip in a couple of seconds rather than sitting in a shared queue.
+      - name: Check if this service needs building
+        id: check
+        run: |
+          set -euo pipefail
+          service="${{ matrix.service }}"
+          svc_dir="$(echo "$service" | tr '-' '_')"
+
+          if git rev-parse --verify HEAD^ >/dev/null 2>&1; then
+            CHANGED_FILES="$(git diff --name-only HEAD^ HEAD)"
+          else
+            CHANGED_FILES="$(git ls-files)"
+          fi
+          LAST_COMMIT_MSG="$(git log -1 --pretty=%B | tr '\n' ' ')"
+
+          build=false
+          if echo "$LAST_COMMIT_MSG" | grep -Eiq 'trigger build|force build|rebuild all'; then
+            build=true
+          elif echo "$CHANGED_FILES" | grep -Eq '^(\.forgejo/workflows/|Dockerfile|Cargo\.toml|Cargo\.lock|crates/|scripts/)'; then
+            build=true
+          elif echo "$CHANGED_FILES" | grep -q "^apps/${svc_dir}/"; then
+            build=true
+          fi
+
+          echo "build=$build" >> "$GITHUB_OUTPUT"
+          if [ "$build" = "false" ]; then
+            echo "No changes relevant to $service - skipping."
+          fi
+
       - name: Point DOCKER_HOST at this container's own gateway
+        if: steps.check.outputs.build == 'true'
         run: |
           set -euo pipefail
           # 127.0.0.1 doesn't work: the job container is nested one level
@@ -39,97 +98,37 @@ jobs:
           echo "Detected docker host gateway: $GATEWAY"
           echo "DOCKER_HOST=tcp://$GATEWAY:2375" >> "$GITHUB_ENV"
 
-      - name: Detect changed services
-        run: |
-          set -euo pipefail
-          ALL_SERVICES='gateway users companies jobs leads job-seekers customers payments employees photographers makeup-artists tutors developers video-editors graphic-designers social-media-managers fitness-trainers catering-services ugc-content-creators cron'
-
-          if git rev-parse --verify HEAD^ >/dev/null 2>&1; then
-            CHANGED_FILES="$(git diff --name-only HEAD^ HEAD)"
-          else
-            CHANGED_FILES="$(git ls-files)"
-          fi
-
-          LAST_COMMIT_MSG="$(git log -1 --pretty=%B | tr '\n' ' ')"
-
-          force_full_build=false
-          if echo "$LAST_COMMIT_MSG" | grep -Eiq 'trigger build|force build|rebuild all'; then
-            force_full_build=true
-          elif echo "$CHANGED_FILES" | grep -Eq '^(\.forgejo/workflows/|Dockerfile|Cargo\.toml|Cargo\.lock|crates/|scripts/)'; then
-            force_full_build=true
-          fi
-
-          if [ "$force_full_build" = true ]; then
-            printf '%s\n' $ALL_SERVICES > /tmp/changed-services.txt
-            exit 0
-          fi
-
-          : > /tmp/changed-services.txt
-          while IFS= read -r f; do
-            case "$f" in
-              apps/gateway/*) echo gateway >> /tmp/changed-services.txt ;;
-              apps/users/*) echo users >> /tmp/changed-services.txt ;;
-              apps/companies/*) echo companies >> /tmp/changed-services.txt ;;
-              apps/jobs/*) echo jobs >> /tmp/changed-services.txt ;;
-              apps/leads/*) echo leads >> /tmp/changed-services.txt ;;
-              apps/job_seekers/*) echo job-seekers >> /tmp/changed-services.txt ;;
-              apps/customers/*) echo customers >> /tmp/changed-services.txt ;;
-              apps/payments/*) echo payments >> /tmp/changed-services.txt ;;
-              apps/employees/*) echo employees >> /tmp/changed-services.txt ;;
-              apps/photographers/*) echo photographers >> /tmp/changed-services.txt ;;
-              apps/makeup_artists/*) echo makeup-artists >> /tmp/changed-services.txt ;;
-              apps/tutors/*) echo tutors >> /tmp/changed-services.txt ;;
-              apps/developers/*) echo developers >> /tmp/changed-services.txt ;;
-              apps/video_editors/*) echo video-editors >> /tmp/changed-services.txt ;;
-              apps/graphic_designers/*) echo graphic-designers >> /tmp/changed-services.txt ;;
-              apps/social_media_managers/*) echo social-media-managers >> /tmp/changed-services.txt ;;
-              apps/fitness_trainers/*) echo fitness-trainers >> /tmp/changed-services.txt ;;
-              apps/catering_services/*) echo catering-services >> /tmp/changed-services.txt ;;
-              apps/ugc_content_creators/*) echo ugc-content-creators >> /tmp/changed-services.txt ;;
-              apps/cron/*) echo cron >> /tmp/changed-services.txt ;;
-            esac
-          done <<<"$CHANGED_FILES"
-
-          sort -u /tmp/changed-services.txt -o /tmp/changed-services.txt
-
-      - name: Stop if nothing changed
-        run: |
-          set -euo pipefail
-          if [ ! -s /tmp/changed-services.txt ]; then
-            echo "No backend service changes detected."
-            exit 0
-          fi
-
       - name: Set up Docker Buildx
+        if: steps.check.outputs.build == 'true'
         run: |
           set -euo pipefail
-          [ -s /tmp/changed-services.txt ] || exit 0
           docker version
+          # Same builder name on every node is safe: each runner has
+          # capacity 1, so only one job ever touches a given node's dind
+          # engine at a time - reusing the name lets its build cache persist
+          # (and warm up) across services scheduled on that node over time.
           docker buildx create --use --name nxtgauge-builder || docker buildx use nxtgauge-builder
           docker buildx inspect --bootstrap
 
       - name: Login to registry
+        if: steps.check.outputs.build == 'true'
         env:
           REGISTRY_HOST: ${{ secrets.REGISTRY_HOST || 'ci.nxtgauge.com' }}
-          REGISTRY_NAMESPACE: ${{ secrets.REGISTRY_NAMESPACE || 'ashwin' }}
           REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
           REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
         run: |
           set -euo pipefail
-          [ -s /tmp/changed-services.txt ] || exit 0
           printf '%s' "$REGISTRY_PASSWORD" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USERNAME" --password-stdin
 
-      - name: Build changed services
+      - name: Build and push
+        if: steps.check.outputs.build == 'true'
         env:
           REGISTRY_HOST: ${{ secrets.REGISTRY_HOST || 'ci.nxtgauge.com' }}
           REGISTRY_NAMESPACE: ${{ secrets.REGISTRY_NAMESPACE || 'ashwin' }}
           SHA: ${{ github.sha }}
         run: |
           set -euo pipefail
-          [ -s /tmp/changed-services.txt ] || exit 0
-          : > /tmp/built-services.tsv
-          while IFS= read -r service; do
-            [ -n "$service" ] || continue
+          service="${{ matrix.service }}"
           metadata_file="/tmp/${service}-metadata.json"
           image_ref="$REGISTRY_HOST/$REGISTRY_NAMESPACE/nxtgauge-rust-${service}:${SHA}"
 
@@ -148,11 +147,10 @@ jobs:
             echo "Failed to determine digest for $service" >&2
             exit 1
           fi
-
-            printf '%s\t%s\n' "$service" "$digest" >> /tmp/built-services.tsv
-          done < /tmp/changed-services.txt
+          echo "DIGEST=$digest" >> "$GITHUB_ENV"
 
       - name: Update GitOps release state
+        if: steps.check.outputs.build == 'true'
         env:
           GITOPS_SERVER: ${{ secrets.GITOPS_SERVER || 'ci.nxtgauge.com' }}
           GITOPS_OWNER: ${{ secrets.GITOPS_OWNER || 'ashwin' }}
@@ -163,21 +161,22 @@ jobs:
           SHA: ${{ github.sha }}
         run: |
           set -euo pipefail
-          [ -s /tmp/built-services.tsv ] || exit 0
           test -n "${GITOPS_PUSH_TOKEN:-}" || { echo "GITOPS_PUSH_TOKEN is empty"; exit 1; }
+          service="${{ matrix.service }}"
 
           git clone "https://${GITOPS_PUSH_USERNAME}:${GITOPS_PUSH_TOKEN}@${GITOPS_SERVER}/${GITOPS_OWNER}/${GITOPS_REPO}.git" /tmp/nxtgauge-gitops
           cd /tmp/nxtgauge-gitops
-          git checkout "$GITOPS_BRANCH"
 
-          while IFS=$'\t' read -r service digest; do
-            [ -n "$service" ] || continue
-            ./scripts/set-backend-rust-release.sh "$service" "$digest"
-          done < /tmp/built-services.tsv
+          # Up to 3 of these jobs can be pushing to the same gitops branch at
+          # once now - retry with a fresh pull+rebase on a non-fast-forward
+          # rejection instead of assuming we're the only writer.
+          for attempt in 1 2 3 4 5; do
+            git checkout "$GITOPS_BRANCH"
+            ./scripts/set-backend-rust-release.sh "$service" "$DIGEST"
 
             if git diff --quiet; then
-            echo "GitOps repo already up to date."
-            exit 0
+              echo "GitOps repo already up to date for $service."
+              break
             fi
 
             git config user.name "forgejo-actions[bot]"
@@ -186,5 +185,14 @@ jobs:
               apps/nxtgauge-backend-rust/overlays/prod/backend-release-state.tsv \
               apps/nxtgauge-backend-rust/overlays/prod/release-patches.yaml \
               apps/nxtgauge-backend-rust/overlays/prod/disabled-deployments.yaml
-          git commit -m "chore(gitops): update backend rust images for ${SHA}"
-          git push origin "HEAD:${GITOPS_BRANCH}"
+            git commit -m "chore(gitops): update ${service} image for ${SHA}"
+
+            if git push origin "HEAD:${GITOPS_BRANCH}"; then
+              break
+            fi
+
+            echo "Push rejected (attempt $attempt/5), pulling latest and retrying..."
+            git fetch origin "$GITOPS_BRANCH"
+            git reset --hard "origin/${GITOPS_BRANCH}"
+            sleep $((attempt * 2))
+          done