ci: add Forgejo workflows to main branch

This commit is contained in:
Ashwin Kumar Sivakumar 2026-07-06 04:27:57 +05:30
parent 3c0f45f1ee
commit 49fd4fd981
3 changed files with 598 additions and 0 deletions

View file

@ -0,0 +1,277 @@
#!/usr/bin/env python3
"""
Registry Image Tag Pruner - Keeps only the latest 1 SHA-tag per repository.
Usage:
python3 registry_prune.py \
--registry registry.nxtgauge.com \
--repo nxtgauge-rust-gateway \
--username "$REGISTRY_USERNAME" \
--password "$REGISTRY_PASSWORD"
Environment variables can also be used:
REGISTRY_HOST, REGISTRY_REPO, REGISTRY_USERNAME, REGISTRY_PASSWORD
SHA-like tags are identified by pattern: ^[a-f0-9]{40}$
Non-SHA tags (e.g., high-performance-latest, main-latest, latest) are NEVER deleted.
Exit code: 0 on success (or if prune fails gracefully), non-zero only on critical error.
"""
import argparse
import base64
import json
import os
import sys
import time
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
def parse_args():
parser = argparse.ArgumentParser(
description="Prune Docker registry tags, keeping only the latest SHA tag."
)
parser.add_argument("--registry", default=os.environ.get("REGISTRY_HOST"))
parser.add_argument("--repo", default=os.environ.get("REGISTRY_REPO"))
parser.add_argument("--username", default=os.environ.get("REGISTRY_USERNAME"))
parser.add_argument("--password", default=os.environ.get("REGISTRY_PASSWORD"))
parser.add_argument("--keep", type=int, default=1, help="Number of SHA tags to keep (default: 1)")
return parser.parse_args()
def api_request(url: str, method: str, username: str, password: str, data=None, retries: int = 3) -> dict | None:
"""Make an authenticated API request with retry logic."""
auth = base64.b64encode(f"{username}:{password}".encode()).decode()
headers = {
"Authorization": f"Basic {auth}",
"Content-Type": "application/json",
}
for attempt in range(1, retries + 1):
try:
req = Request(url, method=method, headers=headers, data=data)
with urlopen(req, timeout=30) as response:
content = response.read()
if content:
return json.loads(content)
return {}
except HTTPError as e:
if e.code == 401:
print(f" [ERROR] Authentication failed (401)")
return None
if e.code == 404:
print(f" [WARN] Resource not found: {url}")
return None
print(f" [RETRY {attempt}/{retries}] HTTP {e.code} for {url}")
except URLError as e:
print(f" [RETRY {attempt}/{retries}] URL error: {e.reason}")
except Exception as e:
print(f" [RETRY {attempt}/{retries}] Error: {e}")
if attempt < retries:
time.sleep(attempt * 2)
print(f" [ERROR] Failed after {retries} attempts for {url}")
return None
def get_tag_digest(registry: str, repo: str, tag: str, username: str, password: str) -> tuple[str, str] | None:
"""Get the digest (sha256:...) and created time for a tag."""
url = f"https://{registry}/v2/{repo}/manifests/{tag}"
auth = base64.b64encode(f"{username}:{password}".encode()).decode()
for attempt in range(1, 4):
try:
req = Request(url, method="GET", headers={
"Authorization": f"Basic {auth}",
"Accept": "application/vnd.docker.distribution.manifest.v2+json",
})
with urlopen(req, timeout=30) as response:
digest = response.headers.get("Docker-Content-Digest", "")
created = response.headers.get("Date", "")
return digest, created
except Exception as e:
print(f" [RETRY {attempt}/3] Getting digest for {tag}: {e}")
time.sleep(attempt)
return None
def delete_tag(registry: str, repo: str, digest: str, username: str, password: str) -> bool:
"""Delete a tag by its digest."""
url = f"https://{registry}/v2/{repo}/manifests/{digest}"
auth = base64.b64encode(f"{username}:{password}".encode()).decode()
for attempt in range(1, 4):
try:
req = Request(url, method="DELETE", headers={
"Authorization": f"Basic {auth}",
})
with urlopen(req, timeout=30) as response:
if response.status in (200, 202, 404):
return True
except HTTPError as e:
if e.code == 404:
return True # Already deleted
print(f" [RETRY {attempt}/3] Deleting {digest[:20]}...: {e}")
except Exception as e:
print(f" [RETRY {attempt}/3] Deleting {digest[:20]}...: {e}")
time.sleep(attempt)
return False
def is_sha_tag(tag: str) -> bool:
"""Check if tag looks like a SHA (40 hex chars)."""
import re
return bool(re.match(r"^[a-f0-9]{40}$", tag))
def prune_tags(registry: str, repo: str, username: str, password: str, keep: int = 1) -> bool:
"""
Main prune logic:
- List all tags for the repo
- Filter SHA-like tags
- Sort by created date (newest first)
- Keep newest `keep` tags
- Delete older SHA tags by digest
- Never delete non-SHA tags
"""
print(f"\n=== Pruning {registry}/{repo} ===")
print(f"Strategy: Keep {keep} newest SHA tag(s), delete older SHA tags")
print(f"Non-SHA tags (e.g., high-performance-latest, main-latest, latest) are preserved\n")
# Get catalog (list of repos)
catalog_url = f"https://{registry}/v2/_catalog"
catalog = api_request(catalog_url, "GET", username, password)
if catalog is None:
print("[ERROR] Failed to get repository catalog")
return False
if repo not in catalog.get("repositories", []):
print(f"[INFO] Repository {repo} not found in catalog")
return True
# Get tags for repo
tags_url = f"https://{registry}/v2/{repo}/tags/list"
tags_data = api_request(tags_url, "GET", username, password)
if tags_data is None:
print(f"[ERROR] Failed to get tags for {repo}")
return False
all_tags = tags_data.get("tags", [])
if not all_tags:
print("[INFO] No tags found")
return True
# Separate SHA tags from non-SHA tags
sha_tags = [t for t in all_tags if is_sha_tag(t)]
non_sha_tags = [t for t in all_tags if not is_sha_tag(t)]
print(f"Total tags: {len(all_tags)}")
print(f" SHA tags (candidates for pruning): {len(sha_tags)}")
print(f" Non-SHA tags (protected): {len(non_sha_tags)}")
if non_sha_tags:
print(f" Protected tags: {', '.join(sorted(non_sha_tags))}")
if not sha_tags:
print("\n[INFO] No SHA tags to prune")
return True
# Get digest and created time for each SHA tag
tag_info = []
for tag in sha_tags:
result = get_tag_digest(registry, repo, tag, username, password)
if result:
digest, created = result
tag_info.append({
"tag": tag,
"digest": digest,
"created": created,
"timestamp": parse_http_date(created) if created else 0,
})
time.sleep(0.1) # Be nice to the registry
if not tag_info:
print("\n[ERROR] Could not get info for any SHA tags")
return False
# Sort by timestamp (newest first)
tag_info.sort(key=lambda x: x["timestamp"], reverse=True)
print(f"\nSHA tags sorted by age (newest first):")
for i, info in enumerate(tag_info):
marker = " [KEEP]" if i < keep else " [DELETE]"
print(f" {i+1}. {info['tag']} ({info['created'] or 'unknown date'}){marker}")
# Delete older SHA tags
deleted_count = 0
kept_count = 0
for i, info in enumerate(tag_info):
if i < keep:
print(f"\n[KEEP] {info['tag']}")
kept_count += 1
continue
print(f"\n[DELETE] {info['tag']} (digest: {info['digest'][:20]}...)")
if delete_tag(registry, repo, info["digest"], username, password):
print(f" [OK] Deleted {info['tag']}")
deleted_count += 1
else:
print(f" [WARN] Failed to delete {info['tag']} (will retry next run)")
time.sleep(0.2) # Be nice to the registry
print(f"\n=== Prune Summary ===")
print(f"Tags kept: {kept_count}")
print(f"Tags deleted: {deleted_count}")
print(f"Tags protected (non-SHA): {len(non_sha_tags)}")
return True
def parse_http_date(date_str: str) -> float:
"""Parse HTTP Date header to timestamp."""
from email.utils import parsedate_to_datetime
try:
return parsedate_to_datetime(date_str).timestamp()
except Exception:
return 0
def main():
args = parse_args()
# Validate required args
registry = args.registry or os.environ.get("REGISTRY_HOST")
repo = args.repo or os.environ.get("REGISTRY_REPO")
username = args.username or os.environ.get("REGISTRY_USERNAME")
password = args.password or os.environ.get("REGISTRY_PASSWORD")
if not all([registry, repo, username, password]):
print("[ERROR] Missing required arguments. Need: --registry, --repo, --username, --password")
print("Or set environment variables: REGISTRY_HOST, REGISTRY_REPO, REGISTRY_USERNAME, REGISTRY_PASSWORD")
sys.exit(1)
print(f"Registry: {registry}")
print(f"Repository: {repo}")
print(f"Username: {username}")
try:
success = prune_tags(registry, repo, username, password, args.keep)
if success:
print("\n[OK] Prune completed successfully")
sys.exit(0)
else:
print("\n[WARN] Prune completed with some errors")
sys.exit(0) # Exit 0 per requirement - never fail workflow
except Exception as e:
print(f"\n[ERROR] Prune failed with exception: {e}")
sys.exit(0) # Exit 0 per requirement - never fail workflow
if __name__ == "__main__":
main()

View file

@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""
Update GitOps kustomization.yaml with new image SHA tags.
Usage:
python3 update-gitops.py \
--repo /path/to/nxtgauge-gitops \
--service gateway \
--sha abc123def456...
This script:
1. Updates the newTag for the specified service to the SHA
2. Commits and pushes to the gitops repo
3. Flux detects the change and deploys
"""
import argparse
import os
import re
import subprocess
import sys
def run(cmd: list[str], cwd: str = None) -> tuple[int, str, str]:
"""Run a command and return (returncode, stdout, stderr)."""
result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
return result.returncode, result.stdout, result.stderr
def update_kustomization(kustomization_path: str, service: str, sha: str) -> bool:
"""Update the newTag for a service in kustomization.yaml."""
with open(kustomization_path, "r") as f:
content = f.read()
# Pattern to find image entry for the service
# Matches: - name: registry.nxtgauge.com/nxtgauge-rust-{service}
# newTag: something
pattern = rf'(\s+-\s+name:\s+registry\.nxtgauge\.com/nxtgauge-rust-{re.escape(service)}\n\s+newTag:\s+)[^\n]+'
replacement = rf'\g<1>{sha}'
new_content, count = re.subn(pattern, replacement, content)
if count == 0:
# Try without the nxtgauge-rust- prefix (for frontend, admin, etc)
pattern = rf'(\s+-\s+name:\s+registry\.nxtgauge\.com/nxtgauge-{re.escape(service)}\n\s+newTag:\s+)[^\n]+'
new_content, count = re.subn(pattern, replacement, content)
if count == 0:
print(f"[ERROR] Could not find image entry for service: {service}")
return False
with open(kustomization_path, "w") as f:
f.write(new_content)
print(f"[OK] Updated {service} to SHA {sha}")
return True
def main():
parser = argparse.ArgumentParser(description="Update GitOps with new image SHA")
parser.add_argument("--repo", required=True, help="Path to gitops repo")
parser.add_argument("--service", required=True, help="Service name (e.g., gateway, users, frontend-solid)")
parser.add_argument("--sha", required=True, help="Git SHA to deploy")
parser.add_argument("--message", default=None, help="Commit message")
args = parser.parse_args()
service_image_map = {
"gateway": "nxtgauge-rust-gateway",
"users": "nxtgauge-rust-users",
"companies": "nxtgauge-rust-companies",
"jobs": "nxtgauge-rust-jobs",
"leads": "nxtgauge-rust-leads",
"job-seekers": "nxtgauge-rust-job-seekers",
"customers": "nxtgauge-rust-customers",
"payments": "nxtgauge-rust-payments",
"employees": "nxtgauge-rust-employees",
"photographers": "nxtgauge-rust-photographers",
"makeup-artists": "nxtgauge-rust-makeup-artists",
"tutors": "nxtgauge-rust-tutors",
"developers": "nxtgauge-rust-developers",
"video-editors": "nxtgauge-rust-video-editors",
"graphic-designers": "nxtgauge-rust-graphic-designers",
"social-media-managers": "nxtgauge-rust-social-media-managers",
"fitness-trainers": "nxtgauge-rust-fitness-trainers",
"catering-services": "nxtgauge-rust-catering-services",
"ugc-content-creators": "nxtgauge-rust-ugc-content-creators",
"cron": "nxtgauge-rust-cron",
"frontend-solid": "nxtgauge-frontend-solid",
"admin-solid": "nxtgauge-admin-solid",
"ai-assistant": "nxtgauge-ai-assistant",
}
# Determine which kustomization file to update
if service_image_map.get(args.service):
image_name = service_image_map[args.service]
else:
image_name = f"nxtgauge-{args.service}"
# Find the right kustomization file based on service
if "frontend" in args.service:
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-frontend-solid/overlays/prod/kustomization.yaml")
if not os.path.exists(kustomization_path):
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-frontend-solid/base/kustomization.yaml")
elif "admin" in args.service:
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-admin-solid/overlays/prod/kustomization.yaml")
if not os.path.exists(kustomization_path):
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-admin-solid/base/kustomization.yaml")
elif "ai-assistant" in args.service:
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml")
if not os.path.exists(kustomization_path):
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-ai-assistant/base/kustomization.yaml")
else:
kustomization_path = os.path.join(args.repo, "apps/nxtgauge-backend-rust/overlays/prod/kustomization.yaml")
if not os.path.exists(kustomization_path):
print(f"[ERROR] Kustomization file not found: {kustomization_path}")
sys.exit(0) # Exit 0 per workflow requirement
print(f"Updating {kustomization_path} for service {args.service}")
if not update_kustomization(kustomization_path, args.service, args.sha):
sys.exit(0) # Exit 0 per workflow requirement
# Git add, commit, push
commit_msg = args.message or f"chore: deploy {args.service}@{args.sha}"
run(["git", "add", "-A"], cwd=args.repo)
code, stdout, stderr = run(["git", "diff", "--cached", "--stat"], cwd=args.repo)
if not stdout.strip():
print("[INFO] No changes to commit")
sys.exit(0)
print(f"Changes to commit:\n{stdout}")
run(["git", "commit", "-m", commit_msg], cwd=args.repo)
code, stdout, stderr = run(["git", "push"], cwd=args.repo)
if code != 0:
print(f"[ERROR] Push failed: {stderr}")
else:
print(f"[OK] Pushed update to gitops repo")
sys.exit(0) # Always exit 0 per workflow requirement
if __name__ == "__main__":
main()

View file

@ -0,0 +1,172 @@
name: build-and-release
on:
push:
branches:
- main
- high-performance
concurrency:
group: backend-build-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: docker-ready
env:
DOCKER_HOST: tcp://127.0.0.1:2375
DOCKER_BUILDKIT: "1"
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Detect changed services
run: |
set -euo pipefail
ALL_SERVICES='gateway users companies jobs leads job-seekers customers payments employees photographers makeup-artists tutors developers video-editors graphic-designers social-media-managers fitness-trainers catering-services ugc-content-creators cron'
if git rev-parse --verify HEAD^ >/dev/null 2>&1; then
CHANGED_FILES="$(git diff --name-only HEAD^ HEAD)"
else
CHANGED_FILES="$(git ls-files)"
fi
LAST_COMMIT_MSG="$(git log -1 --pretty=%B | tr '\n' ' ')"
force_full_build=false
if echo "$LAST_COMMIT_MSG" | grep -Eiq 'trigger build|force build|rebuild all'; then
force_full_build=true
elif echo "$CHANGED_FILES" | grep -Eq '^(\.forgejo/workflows/|Dockerfile|Cargo\.toml|Cargo\.lock|crates/|scripts/)'; then
force_full_build=true
fi
if [ "$force_full_build" = true ]; then
printf '%s\n' $ALL_SERVICES > /tmp/changed-services.txt
exit 0
fi
: > /tmp/changed-services.txt
while IFS= read -r f; do
case "$f" in
apps/gateway/*) echo gateway >> /tmp/changed-services.txt ;;
apps/users/*) echo users >> /tmp/changed-services.txt ;;
apps/companies/*) echo companies >> /tmp/changed-services.txt ;;
apps/jobs/*) echo jobs >> /tmp/changed-services.txt ;;
apps/leads/*) echo leads >> /tmp/changed-services.txt ;;
apps/job_seekers/*) echo job-seekers >> /tmp/changed-services.txt ;;
apps/customers/*) echo customers >> /tmp/changed-services.txt ;;
apps/payments/*) echo payments >> /tmp/changed-services.txt ;;
apps/employees/*) echo employees >> /tmp/changed-services.txt ;;
apps/photographers/*) echo photographers >> /tmp/changed-services.txt ;;
apps/makeup_artists/*) echo makeup-artists >> /tmp/changed-services.txt ;;
apps/tutors/*) echo tutors >> /tmp/changed-services.txt ;;
apps/developers/*) echo developers >> /tmp/changed-services.txt ;;
apps/video_editors/*) echo video-editors >> /tmp/changed-services.txt ;;
apps/graphic_designers/*) echo graphic-designers >> /tmp/changed-services.txt ;;
apps/social_media_managers/*) echo social-media-managers >> /tmp/changed-services.txt ;;
apps/fitness_trainers/*) echo fitness-trainers >> /tmp/changed-services.txt ;;
apps/catering_services/*) echo catering-services >> /tmp/changed-services.txt ;;
apps/ugc_content_creators/*) echo ugc-content-creators >> /tmp/changed-services.txt ;;
apps/cron/*) echo cron >> /tmp/changed-services.txt ;;
esac
done <<EOF2
$CHANGED_FILES
EOF2
sort -u /tmp/changed-services.txt -o /tmp/changed-services.txt
- name: Stop if nothing changed
run: |
set -euo pipefail
if [ ! -s /tmp/changed-services.txt ]; then
echo "No backend service changes detected."
exit 0
fi
- name: Set up Docker Buildx
run: |
set -euo pipefail
[ -s /tmp/changed-services.txt ] || exit 0
docker version
docker buildx create --use --name nxtgauge-builder || docker buildx use nxtgauge-builder
docker buildx inspect --bootstrap
- name: Login to registry
env:
REGISTRY_HOST: ${{ secrets.REGISTRY_HOST || 'ci.nxtgauge.com' }}
REGISTRY_NAMESPACE: ${{ secrets.REGISTRY_NAMESPACE || 'ashwin' }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
run: |
set -euo pipefail
[ -s /tmp/changed-services.txt ] || exit 0
printf '%s' "$REGISTRY_PASSWORD" | docker login "$REGISTRY_HOST" -u "$REGISTRY_USERNAME" --password-stdin
- name: Build changed services
env:
REGISTRY_HOST: ${{ secrets.REGISTRY_HOST || 'ci.nxtgauge.com' }}
REGISTRY_NAMESPACE: ${{ secrets.REGISTRY_NAMESPACE || 'ashwin' }}
SHA: ${{ github.sha }}
run: |
set -euo pipefail
[ -s /tmp/changed-services.txt ] || exit 0
: > /tmp/built-services.tsv
while IFS= read -r service; do
[ -n "$service" ] || continue
metadata_file="/tmp/${service}-metadata.json"
image_ref="$REGISTRY_HOST/$REGISTRY_NAMESPACE/nxtgauge-rust-${service}:${SHA}"
docker buildx build --push \
--metadata-file "$metadata_file" \
-f Dockerfile.simple \
--build-arg SERVICE_NAME="$service" \
-t "$image_ref" \
.
digest="$(grep -o '"containerimage.digest":"sha256:[^"]*"' "$metadata_file" | cut -d'"' -f4)"
if [ -z "$digest" ]; then
echo "Failed to determine digest for $service" >&2
exit 1
fi
printf '%s\t%s\n' "$service" "$digest" >> /tmp/built-services.tsv
done < /tmp/changed-services.txt
- name: Update GitOps release state
env:
GITOPS_SERVER: ${{ secrets.GITOPS_SERVER || 'ci.nxtgauge.com' }}
GITOPS_OWNER: ${{ secrets.GITOPS_OWNER || 'ashwin' }}
GITOPS_REPO: ${{ secrets.GITOPS_REPO || 'nxtgauge-gitops' }}
GITOPS_BRANCH: ${{ secrets.GITOPS_BRANCH || 'main' }}
GITOPS_PUSH_USERNAME: ${{ secrets.GITOPS_PUSH_USERNAME }}
GITOPS_PUSH_TOKEN: ${{ secrets.GITOPS_PUSH_TOKEN }}
SHA: ${{ github.sha }}
run: |
set -euo pipefail
[ -s /tmp/built-services.tsv ] || exit 0
test -n "${GITOPS_PUSH_TOKEN:-}" || { echo "GITOPS_PUSH_TOKEN is empty"; exit 1; }
git clone "https://${GITOPS_PUSH_USERNAME}:${GITOPS_PUSH_TOKEN}@${GITOPS_SERVER}/${GITOPS_OWNER}/${GITOPS_REPO}.git" /tmp/nxtgauge-gitops
cd /tmp/nxtgauge-gitops
git checkout "$GITOPS_BRANCH"
while IFS=$'\t' read -r service digest; do
[ -n "$service" ] || continue
./scripts/set-backend-rust-release.sh "$service" "$digest"
done < /tmp/built-services.tsv
if git diff --quiet; then
echo "GitOps repo already up to date."
exit 0
fi
git config user.name "forgejo-actions[bot]"
git config user.email "forgejo-actions@ci.nxtgauge.com"
git add \
apps/nxtgauge-backend-rust/overlays/prod/backend-release-state.tsv \
apps/nxtgauge-backend-rust/overlays/prod/release-patches.yaml \
apps/nxtgauge-backend-rust/overlays/prod/disabled-deployments.yaml
git commit -m "chore(gitops): update backend rust images for ${SHA}"
git push origin "HEAD:${GITOPS_BRANCH}"