diff --git a/OTP_ISSUE_FIX_PROMPT.md b/OTP_ISSUE_FIX_PROMPT.md index 91ef71b..9b581f6 100644 --- a/OTP_ISSUE_FIX_PROMPT.md +++ b/OTP_ISSUE_FIX_PROMPT.md @@ -79,7 +79,7 @@ If there's a `high-performance-latest` branch in the respective repositories, en ## Verification Steps After applying the fix: -1. Trigger ArgoCD sync for both applications +1. Trigger Flux sync for both applications 2. Wait for pods to restart with new images 3. Test signup flow: enter email → receive OTP → verify OTP → account created 4. Check logs if signup still fails diff --git a/ROUTE_ISSUE_ANALYSIS.md b/ROUTE_ISSUE_ANALYSIS.md index f130f3f..571d44a 100644 --- a/ROUTE_ISSUE_ANALYSIS.md +++ b/ROUTE_ISSUE_ANALYSIS.md @@ -154,7 +154,7 @@ Ensure the route fixes from commits `152f918` and `d084491` are merged into the - Create commit with updated image tags - Push to main branch -3. **Trigger ArgoCD Sync:** +3. **Trigger Flux Sync:** - Sync `nxtgauge-frontend-solid` application - Sync `nxtgauge-backend-rust` application diff --git a/apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml b/apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml index c48dbf9..06a54a6 100644 --- a/apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml +++ b/apps/nxtgauge-ai-assistant/overlays/prod/kustomization.yaml @@ -5,4 +5,4 @@ resources: - ../../base images: - name: registry.nxtgauge.com/nxtgauge-ai-assistant - newTag: high-performance-latest + newTag: main-latest diff --git a/apps/nxtgauge-backend-rust/base/openobserve-endpoint-monitor-cronjob.yaml b/apps/nxtgauge-backend-rust/base/openobserve-endpoint-monitor-cronjob.yaml index 8bedb01..bc9c4c3 100644 --- a/apps/nxtgauge-backend-rust/base/openobserve-endpoint-monitor-cronjob.yaml +++ b/apps/nxtgauge-backend-rust/base/openobserve-endpoint-monitor-cronjob.yaml @@ -39,11 +39,13 @@ spec: ok="false" fi payload="$(printf '[{"endpoint":"%s","url":"%s","status_code":%s,"ok":%s,"latency_ms":%s,"checked_at":"%s"}]' "$name" "$url" "$code" "$ok" "$latency_ms" "$checked_at")" - curl -sS -X POST \ + if ! curl -sS -X POST \ "${OO_ENDPOINT}/api/${OO_ORG}/${OO_STREAM}/_json" \ -H "Authorization: ${OO_AUTH_HEADER}" \ -H "Content-Type: application/json" \ - -d "$payload" >/dev/null + -d "$payload" >/dev/null; then + echo "openobserve post failed for ${name}" >&2 + fi } post_result "frontend" "https://test111.nxtgauge.com/" diff --git a/apps/nxtgauge-backend-rust/base/openobserve-k8s-monitor-cronjob.yaml b/apps/nxtgauge-backend-rust/base/openobserve-k8s-monitor-cronjob.yaml index f38ad51..37a044e 100644 --- a/apps/nxtgauge-backend-rust/base/openobserve-k8s-monitor-cronjob.yaml +++ b/apps/nxtgauge-backend-rust/base/openobserve-k8s-monitor-cronjob.yaml @@ -77,7 +77,7 @@ spec: "error": err, } - now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" + now = datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") records = [] nodes = kube_get("/api/v1/nodes").get("items", []) @@ -136,8 +136,8 @@ spec: ("admin-svc", "http://nxtgauge-admin-solid.nxtgauge.svc.cluster.local/"), ("api-gateway-svc", "http://nxtgauge-rust-gateway.nxtgauge.svc.cluster.local:9100/health"), ("registry-svc", "http://docker-registry.registry.svc.cluster.local:5000/v2/"), - ("woodpecker-svc", "http://woodpecker-server.woodpecker.svc.cluster.local/"), - ("argocd-metrics", "http://argocd-server-metrics.argocd.svc.cluster.local:8083/metrics"), + ("forgejo-svc", "http://forgejo-http.forgejo.svc.cluster.local:3000/"), + ("flux-source-controller", "http://source-controller.flux-system.svc.cluster.local/metrics"), ("openobserve-svc", "http://o2-openobserve-standalone.openobserve.svc.cluster.local:5080/healthz"), ] for name, url in endpoints: @@ -172,5 +172,8 @@ spec: }, method="POST", ) - with urllib.request.urlopen(req, timeout=30) as resp: - _ = resp.read() + try: + with urllib.request.urlopen(req, timeout=30) as resp: + _ = resp.read() + except Exception as exc: + print(f"openobserve post failed: {exc}") diff --git a/argocd/ai-guard-application.yaml b/argocd/ai-guard-application.yaml deleted file mode 100644 index a0b8a2b..0000000 --- a/argocd/ai-guard-application.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: ai-guard - namespace: argocd - annotations: - argocd-image-updater.argoproj.io/image-list: guard=registry.nxtgauge.com/ai-guard:latest - argocd-image-updater.argoproj.io/guard.update-strategy: digest - argocd-image-updater.argoproj.io/write-back-method: argocd -spec: - project: default - source: - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - path: apps/ai-guard/overlays/prod - destination: - server: https://kubernetes.default.svc - namespace: nxtgauge-ai - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/argocd/coredns-nodehosts-application.yaml b/argocd/coredns-nodehosts-application.yaml deleted file mode 100644 index 2d38ac1..0000000 --- a/argocd/coredns-nodehosts-application.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: coredns-nodehosts - namespace: argocd -spec: - destination: - namespace: kube-system - server: https://kubernetes.default.svc - project: default - source: - path: ops/coredns-nodehosts - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - diff --git a/argocd/nxtgauge-admin-solid-application.yaml b/argocd/nxtgauge-admin-solid-application.yaml deleted file mode 100644 index 4e6b122..0000000 --- a/argocd/nxtgauge-admin-solid-application.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: nxtgauge-admin-solid - namespace: argocd - annotations: - argocd-image-updater.argoproj.io/image-list: admin=registry.nxtgauge.com/nxtgauge-admin-solid:high-performance-latest - argocd-image-updater.argoproj.io/admin.update-strategy: digest - argocd-image-updater.argoproj.io/admin.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/write-back-method: argocd -spec: - project: default - source: - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - path: apps/nxtgauge-admin-solid/overlays/prod - destination: - server: https://kubernetes.default.svc - namespace: nxtgauge - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/argocd/nxtgauge-ai-assistant-application.yaml b/argocd/nxtgauge-ai-assistant-application.yaml deleted file mode 100644 index 51948d5..0000000 --- a/argocd/nxtgauge-ai-assistant-application.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: nxtgauge-ai-assistant - namespace: argocd - annotations: - argocd-image-updater.argoproj.io/image-list: ai=registry.nxtgauge.com/nxtgauge-ai-assistant:high-performance-latest - argocd-image-updater.argoproj.io/ai.update-strategy: digest - argocd-image-updater.argoproj.io/ai.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/write-back-method: argocd -spec: - project: default - source: - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - path: apps/nxtgauge-ai-assistant/overlays/prod - destination: - server: https://kubernetes.default.svc - namespace: nxtgauge - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/argocd/nxtgauge-backend-rust-application.yaml b/argocd/nxtgauge-backend-rust-application.yaml deleted file mode 100644 index 7dba6a5..0000000 --- a/argocd/nxtgauge-backend-rust-application.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: nxtgauge-backend-rust - namespace: argocd - annotations: - argocd-image-updater.argoproj.io/image-list: >- - gateway=registry.nxtgauge.com/nxtgauge-rust-gateway:d084491, - users=registry.nxtgauge.com/nxtgauge-rust-users:d084491, - companies=registry.nxtgauge.com/nxtgauge-rust-companies:high-performance-latest, - job-seekers=registry.nxtgauge.com/nxtgauge-rust-job-seekers:high-performance-latest, - customers=registry.nxtgauge.com/nxtgauge-rust-customers:high-performance-latest, - payments=registry.nxtgauge.com/nxtgauge-rust-payments:high-performance-latest, - employees=registry.nxtgauge.com/nxtgauge-rust-employees:high-performance-latest, - photographers=registry.nxtgauge.com/nxtgauge-rust-photographers:high-performance-latest, - makeup-artists=registry.nxtgauge.com/nxtgauge-rust-makeup-artists:high-performance-latest, - tutors=registry.nxtgauge.com/nxtgauge-rust-tutors:high-performance-latest, - developers=registry.nxtgauge.com/nxtgauge-rust-developers:high-performance-latest, - video-editors=registry.nxtgauge.com/nxtgauge-rust-video-editors:high-performance-latest, - graphic-designers=registry.nxtgauge.com/nxtgauge-rust-graphic-designers:high-performance-latest, - social-media-managers=registry.nxtgauge.com/nxtgauge-rust-social-media-managers:high-performance-latest, - fitness-trainers=registry.nxtgauge.com/nxtgauge-rust-fitness-trainers:high-performance-latest, - catering-services=registry.nxtgauge.com/nxtgauge-rust-catering-services:high-performance-latest, - ugc-content-creators=registry.nxtgauge.com/nxtgauge-rust-ugc-content-creators:high-performance-latest, - cron=registry.nxtgauge.com/nxtgauge-rust-cron:high-performance-latest - argocd-image-updater.argoproj.io/gateway.update-strategy: digest - argocd-image-updater.argoproj.io/gateway.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/users.update-strategy: digest - argocd-image-updater.argoproj.io/users.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/companies.update-strategy: digest - argocd-image-updater.argoproj.io/companies.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/jobs.update-strategy: digest - argocd-image-updater.argoproj.io/jobs.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/leads.update-strategy: digest - argocd-image-updater.argoproj.io/leads.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/job-seekers.update-strategy: digest - argocd-image-updater.argoproj.io/job-seekers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/customers.update-strategy: digest - argocd-image-updater.argoproj.io/customers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/payments.update-strategy: digest - argocd-image-updater.argoproj.io/payments.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/employees.update-strategy: digest - argocd-image-updater.argoproj.io/employees.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/photographers.update-strategy: digest - argocd-image-updater.argoproj.io/photographers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/makeup-artists.update-strategy: digest - argocd-image-updater.argoproj.io/makeup-artists.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/tutors.update-strategy: digest - argocd-image-updater.argoproj.io/tutors.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/developers.update-strategy: digest - argocd-image-updater.argoproj.io/developers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/video-editors.update-strategy: digest - argocd-image-updater.argoproj.io/video-editors.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/graphic-designers.update-strategy: digest - argocd-image-updater.argoproj.io/graphic-designers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/social-media-managers.update-strategy: digest - argocd-image-updater.argoproj.io/social-media-managers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/fitness-trainers.update-strategy: digest - argocd-image-updater.argoproj.io/fitness-trainers.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/catering-services.update-strategy: digest - argocd-image-updater.argoproj.io/catering-services.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/ugc-content-creators.update-strategy: digest - argocd-image-updater.argoproj.io/ugc-content-creators.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/cron.update-strategy: digest - argocd-image-updater.argoproj.io/cron.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/write-back-method: argocd -spec: - destination: - namespace: nxtgauge - server: https://kubernetes.default.svc - project: default - source: - path: apps/nxtgauge-backend-rust/overlays/prod - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/argocd/nxtgauge-frontend-solid-application.yaml b/argocd/nxtgauge-frontend-solid-application.yaml deleted file mode 100644 index 9df2a31..0000000 --- a/argocd/nxtgauge-frontend-solid-application.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: nxtgauge-frontend-solid - namespace: argocd - annotations: - argocd-image-updater.argoproj.io/image-list: frontend=registry.nxtgauge.com/nxtgauge-frontend-solid:bb6db6c - argocd-image-updater.argoproj.io/frontend.update-strategy: digest - argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^high-performance-latest$ - argocd-image-updater.argoproj.io/write-back-method: argocd -spec: - project: default - source: - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - path: apps/nxtgauge-frontend-solid/overlays/prod - destination: - server: https://kubernetes.default.svc - namespace: nxtgauge - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true diff --git a/argocd/ollama-application.yaml b/argocd/ollama-application.yaml deleted file mode 100644 index 35805f4..0000000 --- a/argocd/ollama-application.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: ollama - namespace: argocd -spec: - destination: - namespace: nxtgauge-ai - server: https://kubernetes.default.svc - project: default - source: - path: apps/ollama/base - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true \ No newline at end of file diff --git a/argocd/openobserve-alerts-application.yaml b/argocd/openobserve-alerts-application.yaml deleted file mode 100644 index 675e133..0000000 --- a/argocd/openobserve-alerts-application.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: openobserve-alerts - namespace: argocd -spec: - project: default - source: - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - path: ops/openobserve-alerts - destination: - server: https://kubernetes.default.svc - namespace: openobserve - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - diff --git a/argocd/openobserve-otelcol-application.yaml b/argocd/openobserve-otelcol-application.yaml deleted file mode 100644 index 345a724..0000000 --- a/argocd/openobserve-otelcol-application.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: openobserve-otelcol - namespace: argocd -spec: - project: default - source: - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - path: ops/openobserve-otelcol - destination: - server: https://kubernetes.default.svc - namespace: openobserve - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - diff --git a/argocd/registry-ingress-application.yaml b/argocd/registry-ingress-application.yaml deleted file mode 100644 index 2436af1..0000000 --- a/argocd/registry-ingress-application.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: registry-ingress - namespace: argocd -spec: - destination: - namespace: registry - server: https://kubernetes.default.svc - project: default - source: - path: ops/registry-ingress - repoURL: https://github.com/Traceworks2023/nxtgauge-gitops.git - targetRevision: main - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - diff --git a/clusters/production/kustomization.yaml b/clusters/production/kustomization.yaml index 3f5ef7c..ecd0a7f 100644 --- a/clusters/production/kustomization.yaml +++ b/clusters/production/kustomization.yaml @@ -6,3 +6,4 @@ resources: - ../../apps/nxtgauge-admin-solid/overlays/prod - ../../apps/nxtgauge-ai-assistant/overlays/prod - ../../apps/ollama/base + - ../../apps/registry diff --git a/ops/openobserve-alerts/configmap-additional.yaml b/ops/openobserve-alerts/configmap-additional.yaml index 9875015..18a45c9 100644 --- a/ops/openobserve-alerts/configmap-additional.yaml +++ b/ops/openobserve-alerts/configmap-additional.yaml @@ -128,12 +128,12 @@ data: 10 2 30 \ "longhorn/{k8s_pod_name}: {msg}" - # ArgoCD Health + # Flux Health ensure_alert \ - "argocd-app-sync-failures" \ - "SELECT k8s_pod_name, k8s_container_name, substring(body, 1, 220) AS msg FROM \"default\" WHERE k8s_namespace_name = 'argocd' AND (body ILIKE '%SyncFailed%' OR body ILIKE '%ComparisonError%' OR body ILIKE '%ResourceQuota%') ORDER BY _timestamp DESC LIMIT 50" \ + "flux-reconcile-failures" \ + "SELECT k8s_pod_name, k8s_container_name, substring(body, 1, 220) AS msg FROM \"default\" WHERE k8s_namespace_name = 'flux-system' AND (body ILIKE '%SyncFailed%' OR body ILIKE '%ComparisonError%' OR body ILIKE '%ResourceQuota%') ORDER BY _timestamp DESC LIMIT 50" \ 10 2 30 \ - "argocd/{k8s_pod_name}: {msg}" + "flux/{k8s_pod_name}: {msg}" # Registry Health ensure_alert \ diff --git a/ops/openobserve-alerts/configmap.yaml b/ops/openobserve-alerts/configmap.yaml index 01a7042..4882cc1 100644 --- a/ops/openobserve-alerts/configmap.yaml +++ b/ops/openobserve-alerts/configmap.yaml @@ -148,16 +148,16 @@ data: "{k8s_namespace_name}/{k8s_pod_name} {body_object_reason}: {body_object_message}" ensure_alert \ - "argocd-errors" \ - "SELECT k8s_pod_name, k8s_container_name, substring(body, 1, 220) AS msg FROM \"default\" WHERE k8s_namespace_name = 'argocd' AND (body ILIKE '%level=error%' OR body ILIKE '%ERROR%' OR body ILIKE '%ComparisonError%' OR body ILIKE '%SyncFailed%') ORDER BY _timestamp DESC LIMIT 50" \ + "flux-errors" \ + "SELECT k8s_pod_name, k8s_container_name, substring(body, 1, 220) AS msg FROM \"default\" WHERE k8s_namespace_name = 'flux-system' AND (body ILIKE '%level=error%' OR body ILIKE '%ERROR%' OR body ILIKE '%ComparisonError%' OR body ILIKE '%SyncFailed%') ORDER BY _timestamp DESC LIMIT 50" \ 10 2 30 \ - "argocd/{k8s_pod_name} {k8s_container_name}: {msg}" + "flux/{k8s_pod_name} {k8s_container_name}: {msg}" ensure_alert \ - "gitea-runner-errors" \ - "SELECT k8s_pod_name, k8s_container_name, substring(body, 1, 220) AS msg FROM \"default\" WHERE k8s_namespace_name = 'gitea' AND (body ILIKE '%error%' OR body ILIKE '%ERROR%' OR body ILIKE '%failed%' OR body ILIKE '%job failed%') ORDER BY _timestamp DESC LIMIT 50" \ + "forgejo-runner-errors" \ + "SELECT k8s_pod_name, k8s_container_name, substring(body, 1, 220) AS msg FROM \"default\" WHERE k8s_namespace_name = 'forgejo' AND (body ILIKE '%error%' OR body ILIKE '%ERROR%' OR body ILIKE '%failed%' OR body ILIKE '%job failed%') ORDER BY _timestamp DESC LIMIT 50" \ 10 2 30 \ - "gitea/{k8s_pod_name} {k8s_container_name}: {msg}" + "forgejo/{k8s_pod_name} {k8s_container_name}: {msg}" ensure_alert \ "registry-errors" \