From 49fd4fd981a851c4a3697698b5fcd8eb30c1eca6 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Sivakumar Date: Mon, 6 Jul 2026 04:27:57 +0530 Subject: [PATCH] ci: add Forgejo workflows to main branch --- .forgejo/scripts/registry_prune.py | 277 +++++++++++++++++++++++++++++ .forgejo/scripts/update-gitops.py | 149 ++++++++++++++++ .forgejo/workflows/build.yaml | 172 ++++++++++++++++++ 3 files changed, 598 insertions(+) create mode 100644 .forgejo/scripts/registry_prune.py create mode 100644 .forgejo/scripts/update-gitops.py create mode 100644 .forgejo/workflows/build.yaml diff --git a/.forgejo/scripts/registry_prune.py b/.forgejo/scripts/registry_prune.py new file mode 100644 index 0000000..12c40ec --- /dev/null +++ b/.forgejo/scripts/registry_prune.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +""" +Registry Image Tag Pruner - Keeps only the latest 1 SHA-tag per repository. + +Usage: + python3 registry_prune.py \ + --registry registry.nxtgauge.com \ + --repo nxtgauge-rust-gateway \ + --username "$REGISTRY_USERNAME" \ + --password "$REGISTRY_PASSWORD" + +Environment variables can also be used: + REGISTRY_HOST, REGISTRY_REPO, REGISTRY_USERNAME, REGISTRY_PASSWORD + +SHA-like tags are identified by pattern: ^[a-f0-9]{40}$ +Non-SHA tags (e.g., high-performance-latest, main-latest, latest) are NEVER deleted. + +Exit code: 0 on success (or if prune fails gracefully), non-zero only on critical error. +""" + +import argparse +import base64 +import json +import os +import sys +import time +from urllib.request import Request, urlopen +from urllib.error import URLError, HTTPError + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Prune Docker registry tags, keeping only the latest SHA tag." + ) + parser.add_argument("--registry", default=os.environ.get("REGISTRY_HOST")) + parser.add_argument("--repo", default=os.environ.get("REGISTRY_REPO")) + parser.add_argument("--username", default=os.environ.get("REGISTRY_USERNAME")) + parser.add_argument("--password", default=os.environ.get("REGISTRY_PASSWORD")) + parser.add_argument("--keep", type=int, default=1, help="Number of SHA tags to keep (default: 1)") + return parser.parse_args() + + +def api_request(url: str, method: str, username: str, password: str, data=None, retries: int = 3) -> dict | None: + """Make an authenticated API request with retry logic.""" + auth = base64.b64encode(f"{username}:{password}".encode()).decode() + headers = { + "Authorization": f"Basic {auth}", + "Content-Type": "application/json", + } + + for attempt in range(1, retries + 1): + try: + req = Request(url, method=method, headers=headers, data=data) + with urlopen(req, timeout=30) as response: + content = response.read() + if content: + return json.loads(content) + return {} + except HTTPError as e: + if e.code == 401: + print(f" [ERROR] Authentication failed (401)") + return None + if e.code == 404: + print(f" [WARN] Resource not found: {url}") + return None + print(f" [RETRY {attempt}/{retries}] HTTP {e.code} for {url}") + except URLError as e: + print(f" [RETRY {attempt}/{retries}] URL error: {e.reason}") + except Exception as e: + print(f" [RETRY {attempt}/{retries}] Error: {e}") + + if attempt < retries: + time.sleep(attempt * 2) + + print(f" [ERROR] Failed after {retries} attempts for {url}") + return None + + +def get_tag_digest(registry: str, repo: str, tag: str, username: str, password: str) -> tuple[str, str] | None: + """Get the digest (sha256:...) and created time for a tag.""" + url = f"https://{registry}/v2/{repo}/manifests/{tag}" + auth = base64.b64encode(f"{username}:{password}".encode()).decode() + + for attempt in range(1, 4): + try: + req = Request(url, method="GET", headers={ + "Authorization": f"Basic {auth}", + "Accept": "application/vnd.docker.distribution.manifest.v2+json", + }) + with urlopen(req, timeout=30) as response: + digest = response.headers.get("Docker-Content-Digest", "") + created = response.headers.get("Date", "") + return digest, created + except Exception as e: + print(f" [RETRY {attempt}/3] Getting digest for {tag}: {e}") + time.sleep(attempt) + + return None + + +def delete_tag(registry: str, repo: str, digest: str, username: str, password: str) -> bool: + """Delete a tag by its digest.""" + url = f"https://{registry}/v2/{repo}/manifests/{digest}" + auth = base64.b64encode(f"{username}:{password}".encode()).decode() + + for attempt in range(1, 4): + try: + req = Request(url, method="DELETE", headers={ + "Authorization": f"Basic {auth}", + }) + with urlopen(req, timeout=30) as response: + if response.status in (200, 202, 404): + return True + except HTTPError as e: + if e.code == 404: + return True # Already deleted + print(f" [RETRY {attempt}/3] Deleting {digest[:20]}...: {e}") + except Exception as e: + print(f" [RETRY {attempt}/3] Deleting {digest[:20]}...: {e}") + + time.sleep(attempt) + + return False + + +def is_sha_tag(tag: str) -> bool: + """Check if tag looks like a SHA (40 hex chars).""" + import re + return bool(re.match(r"^[a-f0-9]{40}$", tag)) + + +def prune_tags(registry: str, repo: str, username: str, password: str, keep: int = 1) -> bool: + """ + Main prune logic: + - List all tags for the repo + - Filter SHA-like tags + - Sort by created date (newest first) + - Keep newest `keep` tags + - Delete older SHA tags by digest + - Never delete non-SHA tags + """ + print(f"\n=== Pruning {registry}/{repo} ===") + print(f"Strategy: Keep {keep} newest SHA tag(s), delete older SHA tags") + print(f"Non-SHA tags (e.g., high-performance-latest, main-latest, latest) are preserved\n") + + # Get catalog (list of repos) + catalog_url = f"https://{registry}/v2/_catalog" + catalog = api_request(catalog_url, "GET", username, password) + if catalog is None: + print("[ERROR] Failed to get repository catalog") + return False + + if repo not in catalog.get("repositories", []): + print(f"[INFO] Repository {repo} not found in catalog") + return True + + # Get tags for repo + tags_url = f"https://{registry}/v2/{repo}/tags/list" + tags_data = api_request(tags_url, "GET", username, password) + if tags_data is None: + print(f"[ERROR] Failed to get tags for {repo}") + return False + + all_tags = tags_data.get("tags", []) + if not all_tags: + print("[INFO] No tags found") + return True + + # Separate SHA tags from non-SHA tags + sha_tags = [t for t in all_tags if is_sha_tag(t)] + non_sha_tags = [t for t in all_tags if not is_sha_tag(t)] + + print(f"Total tags: {len(all_tags)}") + print(f" SHA tags (candidates for pruning): {len(sha_tags)}") + print(f" Non-SHA tags (protected): {len(non_sha_tags)}") + if non_sha_tags: + print(f" Protected tags: {', '.join(sorted(non_sha_tags))}") + + if not sha_tags: + print("\n[INFO] No SHA tags to prune") + return True + + # Get digest and created time for each SHA tag + tag_info = [] + for tag in sha_tags: + result = get_tag_digest(registry, repo, tag, username, password) + if result: + digest, created = result + tag_info.append({ + "tag": tag, + "digest": digest, + "created": created, + "timestamp": parse_http_date(created) if created else 0, + }) + time.sleep(0.1) # Be nice to the registry + + if not tag_info: + print("\n[ERROR] Could not get info for any SHA tags") + return False + + # Sort by timestamp (newest first) + tag_info.sort(key=lambda x: x["timestamp"], reverse=True) + + print(f"\nSHA tags sorted by age (newest first):") + for i, info in enumerate(tag_info): + marker = " [KEEP]" if i < keep else " [DELETE]" + print(f" {i+1}. {info['tag']} ({info['created'] or 'unknown date'}){marker}") + + # Delete older SHA tags + deleted_count = 0 + kept_count = 0 + + for i, info in enumerate(tag_info): + if i < keep: + print(f"\n[KEEP] {info['tag']}") + kept_count += 1 + continue + + print(f"\n[DELETE] {info['tag']} (digest: {info['digest'][:20]}...)") + if delete_tag(registry, repo, info["digest"], username, password): + print(f" [OK] Deleted {info['tag']}") + deleted_count += 1 + else: + print(f" [WARN] Failed to delete {info['tag']} (will retry next run)") + + time.sleep(0.2) # Be nice to the registry + + print(f"\n=== Prune Summary ===") + print(f"Tags kept: {kept_count}") + print(f"Tags deleted: {deleted_count}") + print(f"Tags protected (non-SHA): {len(non_sha_tags)}") + + return True + + +def parse_http_date(date_str: str) -> float: + """Parse HTTP Date header to timestamp.""" + from email.utils import parsedate_to_datetime + try: + return parsedate_to_datetime(date_str).timestamp() + except Exception: + return 0 + + +def main(): + args = parse_args() + + # Validate required args + registry = args.registry or os.environ.get("REGISTRY_HOST") + repo = args.repo or os.environ.get("REGISTRY_REPO") + username = args.username or os.environ.get("REGISTRY_USERNAME") + password = args.password or os.environ.get("REGISTRY_PASSWORD") + + if not all([registry, repo, username, password]): + print("[ERROR] Missing required arguments. Need: --registry, --repo, --username, --password") + print("Or set environment variables: REGISTRY_HOST, REGISTRY_REPO, REGISTRY_USERNAME, REGISTRY_PASSWORD") + sys.exit(1) + + print(f"Registry: {registry}") + print(f"Repository: {repo}") + print(f"Username: {username}") + + try: + success = prune_tags(registry, repo, username, password, args.keep) + if success: + print("\n[OK] Prune completed successfully") + sys.exit(0) + else: + print("\n[WARN] Prune completed with some errors") + sys.exit(0) # Exit 0 per requirement - never fail workflow + except Exception as e: + print(f"\n[ERROR] Prune failed with exception: {e}") + sys.exit(0) # Exit 0 per requirement - never fail workflow + + +if __name__ == "__main__": + main() diff --git a/.forgejo/scripts/update-gitops.py b/.forgejo/scripts/update-gitops.py new file mode 100644 index 0000000..317d66e --- /dev/null +++ b/.forgejo/scripts/update-gitops.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Update GitOps kustomization.yaml with new image SHA tags. + +Usage: + python3 update-gitops.py \ + --repo /path/to/nxtgauge-gitops \ + --service gateway \ + --sha abc123def456... + +This script: +1. Updates the newTag for the specified service to the SHA +2. Commits and pushes to the gitops repo +3. Flux detects the change and deploys +""" + +import argparse +import os +import re +import subprocess +import sys + + +def run(cmd: list[str], cwd: str = None) -> tuple[int, str, str]: + """Run a command and return (returncode, stdout, stderr).""" + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + return result.returncode, result.stdout, result.stderr + + +def update_kustomization(kustomization_path: str, service: str, sha: str) -> bool: + """Update the newTag for a service in kustomization.yaml.""" + with open(kustomization_path, "r") as f: + content = f.read() + + # Pattern to find image entry for the service + # Matches: - name: registry.nxtgauge.com/nxtgauge-rust-{service} + # newTag: something + pattern = rf'(\s+-\s+name:\s+registry\.nxtgauge\.com/nxtgauge-rust-{re.escape(service)}\n\s+newTag:\s+)[^\n]+' + + replacement = rf'\g<1>{sha}' + + new_content, count = re.subn(pattern, replacement, content) + + if count == 0: + # Try without the nxtgauge-rust- prefix (for frontend, admin, etc) + pattern = rf'(\s+-\s+name:\s+registry\.nxtgauge\.com/nxtgauge-{re.escape(service)}\n\s+newTag:\s+)[^\n]+' + new_content, count = re.subn(pattern, replacement, content) + + if count == 0: + print(f"[ERROR] Could not find image entry for service: {service}") + return False + + with open(kustomization_path, "w") as f: + f.write(new_content) + + print(f"[OK] Updated {service} to SHA {sha}") + return True + + +def main(): + parser = argparse.ArgumentParser(description="Update GitOps with new image SHA") + parser.add_argument("--repo", required=True, help="Path to gitops repo") + parser.add_argument("--service", required=True, help="Service name (e.g., gateway, users, frontend-solid)") + parser.add_argument("--sha", required=True, help="Git SHA to deploy") + parser.add_argument("--message", default=None, help="Commit message") + args = parser.parse_args() + + service_image_map = { + "gateway": "nxtgauge-rust-gateway", + "users": "nxtgauge-rust-users", + "companies": "nxtgauge-rust-companies", + "jobs": "nxtgauge-rust-jobs", + "leads": "nxtgauge-rust-leads", + "job-seekers": "nxtgauge-rust-job-seekers", + "customers": "nxtgauge-rust-customers", + "payments": "nxtgauge-rust-payments", + "employees": "nxtgauge-rust-employees", + "photographers": "nxtgauge-rust-photographers", + "makeup-artists": "nxtgauge-rust-makeup-artists", + "tutors": "nxtgauge-rust-tutors", + "developers": "nxtgauge-rust-developers", + "video-editors": "nxtgauge-rust-video-editors", + "graphic-designers": "nxtgauge-rust-graphic-designers", + "social-media-managers": "nxtgauge-rust-social-media-managers", + "fitness-trainers": "nxtgauge-rust-fitness-trainers", + "catering-services": "nxtgauge-rust-catering-services", + "ugc-content-creators": "nxtgauge-rust-ugc-content-creators", + "cron": "nxtgauge-rust-cron", + "frontend-solid": "nxtgauge-frontend-solid", + "admin-solid": "nxtgauge-admin-solid", + "ai-assistant": "nxtgauge-ai-assistant", + } + + # Determine which kustomization file to update + if service_image_map.get(args.service): + image_name = service_image_map[args.service] + else: + image_name = f"nxtgauge-{args.service}" + + # Find the right kustomization file based on service + if "frontend" in args.service: + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml") + if not os.path.exists(kustomization_path): + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-frontend-solid/base/kustomization.yaml") + elif "admin" in args.service: + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml") + if not os.path.exists(kustomization_path): + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-admin-solid/base/kustomization.yaml") + elif "ai-assistant" in args.service: + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml") + if not os.path.exists(kustomization_path): + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-ai-assistant/base/kustomization.yaml") + else: + kustomization_path = os.path.join(args.repo, "apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml") + + if not os.path.exists(kustomization_path): + print(f"[ERROR] Kustomization file not found: {kustomization_path}") + sys.exit(0) # Exit 0 per workflow requirement + + print(f"Updating {kustomization_path} for service {args.service}") + + if not update_kustomization(kustomization_path, args.service, args.sha): + sys.exit(0) # Exit 0 per workflow requirement + + # Git add, commit, push + commit_msg = args.message or f"chore: deploy {args.service}@{args.sha}" + + run(["git", "add", "-A"], cwd=args.repo) + code, stdout, stderr = run(["git", "diff", "--cached", "--stat"], cwd=args.repo) + + if not stdout.strip(): + print("[INFO] No changes to commit") + sys.exit(0) + + print(f"Changes to commit:\n{stdout}") + + run(["git", "commit", "-m", commit_msg], cwd=args.repo) + code, stdout, stderr = run(["git", "push"], cwd=args.repo) + + if code != 0: + print(f"[ERROR] Push failed: {stderr}") + else: + print(f"[OK] Pushed update to gitops repo") + + sys.exit(0) # Always exit 0 per workflow requirement + + +if __name__ == "__main__": + main() diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml new file mode 100644 index 0000000..cd970f2 --- /dev/null +++ b/.forgejo/workflows/build.yaml @@ -0,0 +1,172 @@ +name: build-and-release + +on: + push: + branches: + - main + - high-performance + +concurrency: + group: backend-build-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: docker-ready + env: + DOCKER_HOST: tcp://127.0.0.1:2375 + DOCKER_BUILDKIT: "1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed services + run: | + set -euo pipefail + ALL_SERVICES='gateway users companies jobs leads job-seekers customers payments employees photographers makeup-artists tutors developers video-editors graphic-designers social-media-managers fitness-trainers catering-services ugc-content-creators cron' + + if git rev-parse --verify HEAD^ >/dev/null 2>&1; then + CHANGED_FILES="$(git diff --name-only HEAD^ HEAD)" + else + CHANGED_FILES="$(git ls-files)" + fi + + LAST_COMMIT_MSG="$(git log -1 --pretty=%B | tr '\n' ' ')" + + force_full_build=false + if echo "$LAST_COMMIT_MSG" | grep -Eiq 'trigger build|force build|rebuild all'; then + force_full_build=true + elif echo "$CHANGED_FILES" | grep -Eq '^(\.forgejo/workflows/|Dockerfile|Cargo\.toml|Cargo\.lock|crates/|scripts/)'; then + force_full_build=true + fi + + if [ "$force_full_build" = true ]; then + printf '%s\n' $ALL_SERVICES > /tmp/changed-services.txt + exit 0 + fi + + : > /tmp/changed-services.txt + while IFS= read -r f; do + case "$f" in + apps/gateway/*) echo gateway >> /tmp/changed-services.txt ;; + apps/users/*) echo users >> /tmp/changed-services.txt ;; + apps/companies/*) echo companies >> /tmp/changed-services.txt ;; + apps/jobs/*) echo jobs >> /tmp/changed-services.txt ;; + apps/leads/*) echo leads >> /tmp/changed-services.txt ;; + apps/job_seekers/*) echo job-seekers >> /tmp/changed-services.txt ;; + apps/customers/*) echo customers >> /tmp/changed-services.txt ;; + apps/payments/*) echo payments >> /tmp/changed-services.txt ;; + apps/employees/*) echo employees >> /tmp/changed-services.txt ;; + apps/photographers/*) echo photographers >> /tmp/changed-services.txt ;; + apps/makeup_artists/*) echo makeup-artists >> /tmp/changed-services.txt ;; + apps/tutors/*) echo tutors >> /tmp/changed-services.txt ;; + apps/developers/*) echo developers >> /tmp/changed-services.txt ;; + apps/video_editors/*) echo video-editors >> /tmp/changed-services.txt ;; + apps/graphic_designers/*) echo graphic-designers >> /tmp/changed-services.txt ;; + apps/social_media_managers/*) echo social-media-managers >> /tmp/changed-services.txt ;; + apps/fitness_trainers/*) echo fitness-trainers >> /tmp/changed-services.txt ;; + apps/catering_services/*) echo catering-services >> /tmp/changed-services.txt ;; + apps/ugc_content_creators/*) echo ugc-content-creators >> /tmp/changed-services.txt ;; + apps/cron/*) echo cron >> /tmp/changed-services.txt ;; + esac + done < /tmp/built-services.tsv + while IFS= read -r service; do + [ -n "$service" ] || continue + metadata_file="/tmp/${service}-metadata.json" + image_ref="$REGISTRY_HOST/$REGISTRY_NAMESPACE/nxtgauge-rust-${service}:${SHA}" + + docker buildx build --push \ + --metadata-file "$metadata_file" \ + -f Dockerfile.simple \ + --build-arg SERVICE_NAME="$service" \ + -t "$image_ref" \ + . + + digest="$(grep -o '"containerimage.digest":"sha256:[^"]*"' "$metadata_file" | cut -d'"' -f4)" + if [ -z "$digest" ]; then + echo "Failed to determine digest for $service" >&2 + exit 1 + fi + + printf '%s\t%s\n' "$service" "$digest" >> /tmp/built-services.tsv + done < /tmp/changed-services.txt + + - name: Update GitOps release state + env: + GITOPS_SERVER: ${{ secrets.GITOPS_SERVER || 'ci.nxtgauge.com' }} + GITOPS_OWNER: ${{ secrets.GITOPS_OWNER || 'ashwin' }} + GITOPS_REPO: ${{ secrets.GITOPS_REPO || 'nxtgauge-gitops' }} + GITOPS_BRANCH: ${{ secrets.GITOPS_BRANCH || 'main' }} + GITOPS_PUSH_USERNAME: ${{ secrets.GITOPS_PUSH_USERNAME }} + GITOPS_PUSH_TOKEN: ${{ secrets.GITOPS_PUSH_TOKEN }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + [ -s /tmp/built-services.tsv ] || exit 0 + test -n "${GITOPS_PUSH_TOKEN:-}" || { echo "GITOPS_PUSH_TOKEN is empty"; exit 1; } + + git clone "https://${GITOPS_PUSH_USERNAME}:${GITOPS_PUSH_TOKEN}@${GITOPS_SERVER}/${GITOPS_OWNER}/${GITOPS_REPO}.git" /tmp/nxtgauge-gitops + cd /tmp/nxtgauge-gitops + git checkout "$GITOPS_BRANCH" + + while IFS=$'\t' read -r service digest; do + [ -n "$service" ] || continue + ./scripts/set-backend-rust-release.sh "$service" "$digest" + done < /tmp/built-services.tsv + + if git diff --quiet; then + echo "GitOps repo already up to date." + exit 0 + fi + + git config user.name "forgejo-actions[bot]" + git config user.email "forgejo-actions@ci.nxtgauge.com" + git add \ + apps/nxtgauge-backend-rust/overlays/prod/backend-release-state.tsv \ + apps/nxtgauge-backend-rust/overlays/prod/release-patches.yaml \ + apps/nxtgauge-backend-rust/overlays/prod/disabled-deployments.yaml + git commit -m "chore(gitops): update backend rust images for ${SHA}" + git push origin "HEAD:${GITOPS_BRANCH}"