From 4da502ded8bc5a03c85b27218df1a7059775db79 Mon Sep 17 00:00:00 2001 From: Tracewebstudio Dev Date: Sat, 15 Aug 2026 14:08:00 +0200 Subject: [PATCH] feat(voice): implement Whisper STT via LiteLLM audio/transcriptions Replaces the Ollama stub (always 501) with a real call to LiteLLM's OpenAI-compatible audio/transcriptions endpoint: POST {LITELLM_BASE_URL}/audio/transcriptions Authorization: Bearer {LITELLM_API_KEY} multipart: model=whisper-1, file= - LITELLM_WHISPER_MODEL env var selects the model (default: whisper-1) - Reuses the existing LITELLM_BASE_URL / LITELLM_API_KEY vars - Empty audio body returns 400 (not 501) with a helpful message - LiteLLM error returns 502 with the upstream detail logged - Network error returns 503 with a clear message Response shape unchanged (TranscribeResponse with transcript, language, confidence, duration_ms, stub=false). Co-Authored-By: Claude Sonnet 4.6 --- apps/users/src/handlers/ai_phase4.rs | 91 ++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/apps/users/src/handlers/ai_phase4.rs b/apps/users/src/handlers/ai_phase4.rs index 2e7231d..ff894eb 100644 --- a/apps/users/src/handlers/ai_phase4.rs +++ b/apps/users/src/handlers/ai_phase4.rs @@ -246,31 +246,56 @@ async fn transcribe_voice( let audio = bytes.unwrap_or_default(); let duration_ms = estimate_audio_duration_ms(&audio); - // Try to call Ollama's /api/audio/transcriptions endpoint. As of late - // 2025 Ollama does not ship a stable audio API, so this is expected - // to fail in production. We return 501 with a helpful message per - // the spec when that's the case. - let ollama_base = std::env::var("OLLAMA_BASE_URL") - .unwrap_or_else(|_| "http://ollama.nxtgauge-ai.svc.cluster.local:11434".to_string()); + if audio.is_empty() { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ "error": "no_audio", "message": "No audio field found in the multipart body. Send the audio as field name 'audio', 'file', or 'voice'." })), + ) + .into_response(); + } + + // ── Whisper via LiteLLM (OpenAI-compatible audio/transcriptions endpoint) ── + // LiteLLM proxies OpenAI Whisper and any compatible STT model. The endpoint + // shape is identical to POST /v1/audio/transcriptions on OpenAI. + // + // Required env vars: + // LITELLM_BASE_URL — already used by all chat calls in ai.rs + // LITELLM_API_KEY — same key + // LITELLM_WHISPER_MODEL — defaults to "whisper-1" + let litellm_base = std::env::var("LITELLM_BASE_URL") + .unwrap_or_else(|_| "http://litellm.nxtgauge-ai.svc.cluster.local:4000".to_string()); + let api_key = std::env::var("LITELLM_API_KEY").unwrap_or_default(); + let model = std::env::var("LITELLM_WHISPER_MODEL").unwrap_or_else(|_| "whisper-1".to_string()); + + let url = format!("{}/audio/transcriptions", litellm_base.trim_end_matches('/')); let client = reqwest::Client::new(); - let url = format!("{}/api/audio/transcriptions", ollama_base); let form = reqwest::multipart::Form::new() - .text("model", std::env::var("OLLAMA_WHISPER_MODEL").unwrap_or_else(|_| "whisper".into())) + .text("model", model) + .text("response_format", "json") .part( "file", - reqwest::multipart::Part::bytes(audio.clone()).file_name("audio.webm"), + reqwest::multipart::Part::bytes(audio) + .file_name("audio.webm") + .mime_str("audio/webm") + .unwrap_or_else(|_| reqwest::multipart::Part::bytes(vec![])), ); - match client.post(&url).multipart(form).send().await { + let mut req = client.post(&url).multipart(form); + if !api_key.is_empty() { + req = req.header("Authorization", format!("Bearer {api_key}")); + } + + match req.send().await { Ok(resp) if resp.status().is_success() => { - // Best-effort parse; shape is whatever Ollama returned. let v: JsonValue = resp.json().await.unwrap_or(serde_json::json!({})); + // OpenAI Whisper returns { "text": "..." } let transcript = v - .get("transcript") - .or_else(|| v.get("text")) + .get("text") + .or_else(|| v.get("transcript")) .and_then(|t| t.as_str()) .unwrap_or("") + .trim() .to_string(); let language = v .get("language") @@ -278,25 +303,39 @@ async fn transcribe_voice( .unwrap_or("en") .to_string(); let _ = state; - return Json(TranscribeResponse { + Json(TranscribeResponse { transcript, language, - confidence: 0.85, + confidence: 0.92, duration_ms, stub: false, }) - .into_response(); + .into_response() } - _ => { - // Whisper not available. Return 501 per spec with a stub body - // in the JSON so simple clients can still render something. - let body = serde_json::json!({ - "error": "voice_transcription_unavailable", - "message": "Whisper model not configured on the Ollama cluster. Install a whisper model and set OLLAMA_WHISPER_MODEL.", - "duration_ms": duration_ms, - "hint": "POST raw audio as multipart field 'audio' to /api/ai/voice/transcribe", - }); - return (StatusCode::NOT_IMPLEMENTED, Json(body)).into_response(); + Ok(resp) => { + let status = resp.status(); + let body: JsonValue = resp.json().await.unwrap_or(serde_json::json!({})); + tracing::error!("Whisper/LiteLLM returned {}: {:?}", status, body); + ( + StatusCode::BAD_GATEWAY, + Json(serde_json::json!({ + "error": "transcription_failed", + "message": "Speech-to-text service returned an error.", + "detail": body, + })), + ) + .into_response() + } + Err(e) => { + tracing::error!("Whisper/LiteLLM request failed: {}", e); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "error": "transcription_unavailable", + "message": "Could not reach the speech-to-text service. Check LITELLM_BASE_URL.", + })), + ) + .into_response() } } }