From d503a32de7b11d19b9695b2f424a731079bf7fc5 Mon Sep 17 00:00:00 2001 From: sync-test Date: Tue, 21 Jul 2026 06:30:13 +0530 Subject: [PATCH] feat(ai): fix Ollama resource limits and bring LiteLLM under GitOps management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the AI architecture doc ("Improve Generation Quality") — qwen3:4b and qwen3:8b were already pulled onto the Ollama PVC, and apps/litellm/base/configmap.yaml already had the correct model_list mapping every feature alias to them instead of gemma3:270m. Neither was actually in effect: 1. apps/litellm was never included in clusters/production/kustomization.yaml, so it was only ever deployed by a one-off manual `kubectl apply` and has been completely outside GitOps ever since (same root cause as the ai-guard registry drift found earlier). Added it to the root kustomization. Corrected its image reference from registry.nxtgauge.com/litellm:latest (doesn't appear to exist) to ghcr.io/berriai/litellm:latest, matching what's actually running live — adopting this file without that fix would have broken a working deployment the moment Flux started managing it. 2. apps/ollama/base/deployment.yaml's memory limit (1500Mi) was too small to ever load qwen3:4b (~2.5GB) or qwen3:8b (~5.2GB) — every model alias in the (also-never-applied) LiteLLM config was therefore unusable regardless of what it was named. Raised to 4 CPU / 8Gi limit (node has 16GB total, was at ~26% memory use) and added OLLAMA_KEEP_ALIVE=30m so a loaded model survives the gaps between bursty feature requests instead of reloading from disk on every first call after 5+ minutes idle. --- apps/litellm/base/deployment.yaml | 10 +++++++++- apps/ollama/base/deployment.yaml | 22 ++++++++++++++++++---- clusters/production/kustomization.yaml | 1 + 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/apps/litellm/base/deployment.yaml b/apps/litellm/base/deployment.yaml index e1a76ed..e0c69d4 100644 --- a/apps/litellm/base/deployment.yaml +++ b/apps/litellm/base/deployment.yaml @@ -17,7 +17,15 @@ spec: spec: containers: - name: litellm - image: registry.nxtgauge.com/litellm:latest + # This app was never included in Flux's root kustomization (see + # clusters/production/kustomization.yaml), so it was only ever + # deployed by a one-off manual `kubectl apply` and has since + # drifted from this file. Corrected to match what's actually + # running live (the real upstream image) rather than + # registry.nxtgauge.com/litellm:latest, which doesn't appear to + # exist/be maintained — using it would have broken a working + # deployment the moment this file was wired back into GitOps. + image: ghcr.io/berriai/litellm:latest command: - "/bin/bash" args: diff --git a/apps/ollama/base/deployment.yaml b/apps/ollama/base/deployment.yaml index c29b291..f1c8b37 100644 --- a/apps/ollama/base/deployment.yaml +++ b/apps/ollama/base/deployment.yaml @@ -24,16 +24,30 @@ spec: env: - name: OLLAMA_HOST value: "0.0.0.0:11434" + # Keep a loaded model resident for 30 min of inactivity instead + # of Ollama's 5-minute default — job-description/resume/cover- + # letter traffic is bursty, and reloading a 2.5-5GB model from + # disk on every request would add multi-second latency to each + # first call after a gap. + - name: OLLAMA_KEEP_ALIVE + value: "30m" volumeMounts: - name: ollama-models mountPath: /root/.ollama resources: requests: - cpu: 500m - memory: 700Mi - limits: cpu: 1000m - memory: 1500Mi + memory: 3Gi + limits: + # qwen3:4b (~2.5GB on disk) and qwen3:8b (~5.2GB) are already + # pulled onto the PVC, but the previous 1500Mi limit could + # only ever load gemma3:270m — which is why every LiteLLM + # model alias was mapped to gemma3:270m regardless of name + # (see apps/litellm/base/configmap.yaml). Sized to comfortably + # hold qwen3:8b plus KV cache/runtime overhead, with headroom; + # node has 16GB total and was at ~26% memory use. + cpu: 4000m + memory: 8Gi volumes: - name: ollama-models persistentVolumeClaim: diff --git a/clusters/production/kustomization.yaml b/clusters/production/kustomization.yaml index 5fea507..d70e560 100644 --- a/clusters/production/kustomization.yaml +++ b/clusters/production/kustomization.yaml @@ -8,6 +8,7 @@ resources: - ../../apps/nxtgauge-ai-assistant/overlays/prod - ../../apps/github-actions-runners/base - ../../apps/ollama/base + - ../../apps/litellm/base - ../../ops/openobserve-alerts - flux-system/traceworks2026-image-automation.yaml - ../../apps/traceworks2026/overlays/prod