diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 5f9145f..9d45ac5 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -13,6 +13,31 @@ concurrency: jobs: build: runs-on: docker-ready + strategy: + fail-fast: false + max-parallel: 3 + matrix: + service: + - gateway + - users + - companies + - jobs + - leads + - job-seekers + - customers + - payments + - employees + - photographers + - makeup-artists + - tutors + - developers + - video-editors + - graphic-designers + - social-media-managers + - fitness-trainers + - catering-services + - ugc-content-creators + - cron env: DOCKER_BUILDKIT: "1" steps: @@ -21,7 +46,41 @@ jobs: with: fetch-depth: 0 + # Static matrix (all 20 services every push) instead of a single job + # looping sequentially - up to 3 run concurrently (one per runner pod; + # each runner's capacity is 1). This step is each job's own quick + # "do I actually need to do anything" check, so unaffected services + # skip in a couple of seconds rather than sitting in a shared queue. + - name: Check if this service needs building + id: check + run: | + set -euo pipefail + service="${{ matrix.service }}" + svc_dir="$(echo "$service" | tr '-' '_')" + + if git rev-parse --verify HEAD^ >/dev/null 2>&1; then + CHANGED_FILES="$(git diff --name-only HEAD^ HEAD)" + else + CHANGED_FILES="$(git ls-files)" + fi + LAST_COMMIT_MSG="$(git log -1 --pretty=%B | tr '\n' ' ')" + + build=false + if echo "$LAST_COMMIT_MSG" | grep -Eiq 'trigger build|force build|rebuild all'; then + build=true + elif echo "$CHANGED_FILES" | grep -Eq '^(\.forgejo/workflows/|Dockerfile|Cargo\.toml|Cargo\.lock|crates/|scripts/)'; then + build=true + elif echo "$CHANGED_FILES" | grep -q "^apps/${svc_dir}/"; then + build=true + fi + + echo "build=$build" >> "$GITHUB_OUTPUT" + if [ "$build" = "false" ]; then + echo "No changes relevant to $service - skipping." + fi + - name: Point DOCKER_HOST at this container's own gateway + if: steps.check.outputs.build == 'true' run: | set -euo pipefail # 127.0.0.1 doesn't work: the job container is nested one level @@ -39,120 +98,59 @@ jobs: echo "Detected docker host gateway: $GATEWAY" echo "DOCKER_HOST=tcp://$GATEWAY:2375" >> "$GITHUB_ENV" - - name: Detect changed services - run: | - set -euo pipefail - ALL_SERVICES='gateway users companies jobs leads job-seekers customers payments employees photographers makeup-artists tutors developers video-editors graphic-designers social-media-managers fitness-trainers catering-services ugc-content-creators cron' - - if git rev-parse --verify HEAD^ >/dev/null 2>&1; then - CHANGED_FILES="$(git diff --name-only HEAD^ HEAD)" - else - CHANGED_FILES="$(git ls-files)" - fi - - LAST_COMMIT_MSG="$(git log -1 --pretty=%B | tr '\n' ' ')" - - force_full_build=false - if echo "$LAST_COMMIT_MSG" | grep -Eiq 'trigger build|force build|rebuild all'; then - force_full_build=true - elif echo "$CHANGED_FILES" | grep -Eq '^(\.forgejo/workflows/|Dockerfile|Cargo\.toml|Cargo\.lock|crates/|scripts/)'; then - force_full_build=true - fi - - if [ "$force_full_build" = true ]; then - printf '%s\n' $ALL_SERVICES > /tmp/changed-services.txt - exit 0 - fi - - : > /tmp/changed-services.txt - while IFS= read -r f; do - case "$f" in - apps/gateway/*) echo gateway >> /tmp/changed-services.txt ;; - apps/users/*) echo users >> /tmp/changed-services.txt ;; - apps/companies/*) echo companies >> /tmp/changed-services.txt ;; - apps/jobs/*) echo jobs >> /tmp/changed-services.txt ;; - apps/leads/*) echo leads >> /tmp/changed-services.txt ;; - apps/job_seekers/*) echo job-seekers >> /tmp/changed-services.txt ;; - apps/customers/*) echo customers >> /tmp/changed-services.txt ;; - apps/payments/*) echo payments >> /tmp/changed-services.txt ;; - apps/employees/*) echo employees >> /tmp/changed-services.txt ;; - apps/photographers/*) echo photographers >> /tmp/changed-services.txt ;; - apps/makeup_artists/*) echo makeup-artists >> /tmp/changed-services.txt ;; - apps/tutors/*) echo tutors >> /tmp/changed-services.txt ;; - apps/developers/*) echo developers >> /tmp/changed-services.txt ;; - apps/video_editors/*) echo video-editors >> /tmp/changed-services.txt ;; - apps/graphic_designers/*) echo graphic-designers >> /tmp/changed-services.txt ;; - apps/social_media_managers/*) echo social-media-managers >> /tmp/changed-services.txt ;; - apps/fitness_trainers/*) echo fitness-trainers >> /tmp/changed-services.txt ;; - apps/catering_services/*) echo catering-services >> /tmp/changed-services.txt ;; - apps/ugc_content_creators/*) echo ugc-content-creators >> /tmp/changed-services.txt ;; - apps/cron/*) echo cron >> /tmp/changed-services.txt ;; - esac - done <<<"$CHANGED_FILES" - - sort -u /tmp/changed-services.txt -o /tmp/changed-services.txt - - - name: Stop if nothing changed - run: | - set -euo pipefail - if [ ! -s /tmp/changed-services.txt ]; then - echo "No backend service changes detected." - exit 0 - fi - - name: Set up Docker Buildx + if: steps.check.outputs.build == 'true' run: | set -euo pipefail - [ -s /tmp/changed-services.txt ] || exit 0 docker version + # Same builder name on every node is safe: each runner has + # capacity 1, so only one job ever touches a given node's dind + # engine at a time - reusing the name lets its build cache persist + # (and warm up) across services scheduled on that node over time. docker buildx create --use --name nxtgauge-builder || docker buildx use nxtgauge-builder docker buildx inspect --bootstrap - name: Login to registry + if: steps.check.outputs.build == 'true' env: REGISTRY_HOST: ${{ secrets.REGISTRY_HOST || 'ci.nxtgauge.com' }} - REGISTRY_NAMESPACE: ${{ secrets.REGISTRY_NAMESPACE || 'ashwin' }} REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} run: | set -euo pipefail - [ -s /tmp/changed-services.txt ] || exit 0 printf '%s' "$REGISTRY_PASSWORD" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USERNAME" --password-stdin - - name: Build changed services + - name: Build and push + if: steps.check.outputs.build == 'true' env: REGISTRY_HOST: ${{ secrets.REGISTRY_HOST || 'ci.nxtgauge.com' }} REGISTRY_NAMESPACE: ${{ secrets.REGISTRY_NAMESPACE || 'ashwin' }} SHA: ${{ github.sha }} run: | set -euo pipefail - [ -s /tmp/changed-services.txt ] || exit 0 - : > /tmp/built-services.tsv - while IFS= read -r service; do - [ -n "$service" ] || continue - metadata_file="/tmp/${service}-metadata.json" - image_ref="$REGISTRY_HOST/$REGISTRY_NAMESPACE/nxtgauge-rust-${service}:${SHA}" + service="${{ matrix.service }}" + metadata_file="/tmp/${service}-metadata.json" + image_ref="$REGISTRY_HOST/$REGISTRY_NAMESPACE/nxtgauge-rust-${service}:${SHA}" - docker buildx build --push \ - --metadata-file "$metadata_file" \ - -f Dockerfile.simple \ - --build-arg SERVICE_NAME="$service" \ - -t "$image_ref" \ - . + docker buildx build --push \ + --metadata-file "$metadata_file" \ + -f Dockerfile.simple \ + --build-arg SERVICE_NAME="$service" \ + -t "$image_ref" \ + . - # buildx writes the metadata file pretty-printed (space after the - # colon), which the old compact-JSON-only pattern never matched - - # tolerate optional whitespace and extract the digest directly. - digest="$(grep -o '"containerimage\.digest"[[:space:]]*:[[:space:]]*"sha256:[^"]*"' "$metadata_file" | grep -o 'sha256:[^"]*')" - if [ -z "$digest" ]; then - echo "Failed to determine digest for $service" >&2 - exit 1 - fi - - printf '%s\t%s\n' "$service" "$digest" >> /tmp/built-services.tsv - done < /tmp/changed-services.txt + # buildx writes the metadata file pretty-printed (space after the + # colon), which the old compact-JSON-only pattern never matched - + # tolerate optional whitespace and extract the digest directly. + digest="$(grep -o '"containerimage\.digest"[[:space:]]*:[[:space:]]*"sha256:[^"]*"' "$metadata_file" | grep -o 'sha256:[^"]*')" + if [ -z "$digest" ]; then + echo "Failed to determine digest for $service" >&2 + exit 1 + fi + echo "DIGEST=$digest" >> "$GITHUB_ENV" - name: Update GitOps release state + if: steps.check.outputs.build == 'true' env: GITOPS_SERVER: ${{ secrets.GITOPS_SERVER || 'ci.nxtgauge.com' }} GITOPS_OWNER: ${{ secrets.GITOPS_OWNER || 'ashwin' }} @@ -163,28 +161,38 @@ jobs: SHA: ${{ github.sha }} run: | set -euo pipefail - [ -s /tmp/built-services.tsv ] || exit 0 test -n "${GITOPS_PUSH_TOKEN:-}" || { echo "GITOPS_PUSH_TOKEN is empty"; exit 1; } + service="${{ matrix.service }}" git clone "https://${GITOPS_PUSH_USERNAME}:${GITOPS_PUSH_TOKEN}@${GITOPS_SERVER}/${GITOPS_OWNER}/${GITOPS_REPO}.git" /tmp/nxtgauge-gitops cd /tmp/nxtgauge-gitops - git checkout "$GITOPS_BRANCH" - while IFS=$'\t' read -r service digest; do - [ -n "$service" ] || continue - ./scripts/set-backend-rust-release.sh "$service" "$digest" - done < /tmp/built-services.tsv + # Up to 3 of these jobs can be pushing to the same gitops branch at + # once now - retry with a fresh pull+rebase on a non-fast-forward + # rejection instead of assuming we're the only writer. + for attempt in 1 2 3 4 5; do + git checkout "$GITOPS_BRANCH" + ./scripts/set-backend-rust-release.sh "$service" "$DIGEST" - if git diff --quiet; then - echo "GitOps repo already up to date." - exit 0 - fi + if git diff --quiet; then + echo "GitOps repo already up to date for $service." + break + fi - git config user.name "forgejo-actions[bot]" - git config user.email "forgejo-actions@ci.nxtgauge.com" - git add \ - apps/nxtgauge-backend-rust/overlays/prod/backend-release-state.tsv \ - apps/nxtgauge-backend-rust/overlays/prod/release-patches.yaml \ - apps/nxtgauge-backend-rust/overlays/prod/disabled-deployments.yaml - git commit -m "chore(gitops): update backend rust images for ${SHA}" - git push origin "HEAD:${GITOPS_BRANCH}" + git config user.name "forgejo-actions[bot]" + git config user.email "forgejo-actions@ci.nxtgauge.com" + git add \ + apps/nxtgauge-backend-rust/overlays/prod/backend-release-state.tsv \ + apps/nxtgauge-backend-rust/overlays/prod/release-patches.yaml \ + apps/nxtgauge-backend-rust/overlays/prod/disabled-deployments.yaml + git commit -m "chore(gitops): update ${service} image for ${SHA}" + + if git push origin "HEAD:${GITOPS_BRANCH}"; then + break + fi + + echo "Push rejected (attempt $attempt/5), pulling latest and retrying..." + git fetch origin "$GITOPS_BRANCH" + git reset --hard "origin/${GITOPS_BRANCH}" + sleep $((attempt * 2)) + done