diff --git a/apps/litellm/base/deployment.yaml b/apps/litellm/base/deployment.yaml index e1a76ed..e0c69d4 100644 --- a/apps/litellm/base/deployment.yaml +++ b/apps/litellm/base/deployment.yaml @@ -17,7 +17,15 @@ spec: spec: containers: - name: litellm - image: registry.nxtgauge.com/litellm:latest + # This app was never included in Flux's root kustomization (see + # clusters/production/kustomization.yaml), so it was only ever + # deployed by a one-off manual `kubectl apply` and has since + # drifted from this file. Corrected to match what's actually + # running live (the real upstream image) rather than + # registry.nxtgauge.com/litellm:latest, which doesn't appear to + # exist/be maintained — using it would have broken a working + # deployment the moment this file was wired back into GitOps. + image: ghcr.io/berriai/litellm:latest command: - "/bin/bash" args: diff --git a/apps/ollama/base/deployment.yaml b/apps/ollama/base/deployment.yaml index c29b291..f1c8b37 100644 --- a/apps/ollama/base/deployment.yaml +++ b/apps/ollama/base/deployment.yaml @@ -24,16 +24,30 @@ spec: env: - name: OLLAMA_HOST value: "0.0.0.0:11434" + # Keep a loaded model resident for 30 min of inactivity instead + # of Ollama's 5-minute default — job-description/resume/cover- + # letter traffic is bursty, and reloading a 2.5-5GB model from + # disk on every request would add multi-second latency to each + # first call after a gap. + - name: OLLAMA_KEEP_ALIVE + value: "30m" volumeMounts: - name: ollama-models mountPath: /root/.ollama resources: requests: - cpu: 500m - memory: 700Mi - limits: cpu: 1000m - memory: 1500Mi + memory: 3Gi + limits: + # qwen3:4b (~2.5GB on disk) and qwen3:8b (~5.2GB) are already + # pulled onto the PVC, but the previous 1500Mi limit could + # only ever load gemma3:270m — which is why every LiteLLM + # model alias was mapped to gemma3:270m regardless of name + # (see apps/litellm/base/configmap.yaml). Sized to comfortably + # hold qwen3:8b plus KV cache/runtime overhead, with headroom; + # node has 16GB total and was at ~26% memory use. + cpu: 4000m + memory: 8Gi volumes: - name: ollama-models persistentVolumeClaim: diff --git a/clusters/production/kustomization.yaml b/clusters/production/kustomization.yaml index 5fea507..d70e560 100644 --- a/clusters/production/kustomization.yaml +++ b/clusters/production/kustomization.yaml @@ -8,6 +8,7 @@ resources: - ../../apps/nxtgauge-ai-assistant/overlays/prod - ../../apps/github-actions-runners/base - ../../apps/ollama/base + - ../../apps/litellm/base - ../../ops/openobserve-alerts - flux-system/traceworks2026-image-automation.yaml - ../../apps/traceworks2026/overlays/prod