feat(voice): implement Whisper STT via LiteLLM audio/transcriptions
All checks were successful
build-and-release / build (cron) (push) Successful in 13s
build-and-release / build (users) (push) Successful in 3m41s
build-and-release / build (employees) (push) Successful in 13s
build-and-release / build (fitness-trainers) (push) Successful in 7s
build-and-release / build (gateway) (push) Successful in 6s
build-and-release / build (job-seekers) (push) Successful in 6s
build-and-release / build (makeup-artists) (push) Successful in 8s
build-and-release / build (payments) (push) Successful in 5s
build-and-release / build (graphic-designers) (push) Successful in 12s
build-and-release / build (jobs) (push) Successful in 13s
build-and-release / build (photographers) (push) Successful in 5s
build-and-release / build (ugc-content-creators) (push) Successful in 4s
build-and-release / build (social-media-managers) (push) Successful in 9s
build-and-release / build (tutors) (push) Successful in 11s
build-and-release / build (catering-services) (push) Successful in 5s
build-and-release / build (video-editors) (push) Successful in 5s
backend-integration-tests / ai-credits (push) Successful in 10s
build-and-release / build (companies) (push) Successful in 8s
build-and-release / build (customers) (push) Successful in 7s
build-and-release / build (developers) (push) Successful in 9s
All checks were successful
build-and-release / build (cron) (push) Successful in 13s
build-and-release / build (users) (push) Successful in 3m41s
build-and-release / build (employees) (push) Successful in 13s
build-and-release / build (fitness-trainers) (push) Successful in 7s
build-and-release / build (gateway) (push) Successful in 6s
build-and-release / build (job-seekers) (push) Successful in 6s
build-and-release / build (makeup-artists) (push) Successful in 8s
build-and-release / build (payments) (push) Successful in 5s
build-and-release / build (graphic-designers) (push) Successful in 12s
build-and-release / build (jobs) (push) Successful in 13s
build-and-release / build (photographers) (push) Successful in 5s
build-and-release / build (ugc-content-creators) (push) Successful in 4s
build-and-release / build (social-media-managers) (push) Successful in 9s
build-and-release / build (tutors) (push) Successful in 11s
build-and-release / build (catering-services) (push) Successful in 5s
build-and-release / build (video-editors) (push) Successful in 5s
backend-integration-tests / ai-credits (push) Successful in 10s
build-and-release / build (companies) (push) Successful in 8s
build-and-release / build (customers) (push) Successful in 7s
build-and-release / build (developers) (push) Successful in 9s
Replaces the Ollama stub (always 501) with a real call to LiteLLM's
OpenAI-compatible audio/transcriptions endpoint:
POST {LITELLM_BASE_URL}/audio/transcriptions
Authorization: Bearer {LITELLM_API_KEY}
multipart: model=whisper-1, file=<audio.webm>
- LITELLM_WHISPER_MODEL env var selects the model (default: whisper-1)
- Reuses the existing LITELLM_BASE_URL / LITELLM_API_KEY vars
- Empty audio body returns 400 (not 501) with a helpful message
- LiteLLM error returns 502 with the upstream detail logged
- Network error returns 503 with a clear message
Response shape unchanged (TranscribeResponse with transcript, language,
confidence, duration_ms, stub=false).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
b039ed7342
commit
4da502ded8
1 changed files with 65 additions and 26 deletions
|
|
@ -246,31 +246,56 @@ async fn transcribe_voice(
|
|||
let audio = bytes.unwrap_or_default();
|
||||
let duration_ms = estimate_audio_duration_ms(&audio);
|
||||
|
||||
// Try to call Ollama's /api/audio/transcriptions endpoint. As of late
|
||||
// 2025 Ollama does not ship a stable audio API, so this is expected
|
||||
// to fail in production. We return 501 with a helpful message per
|
||||
// the spec when that's the case.
|
||||
let ollama_base = std::env::var("OLLAMA_BASE_URL")
|
||||
.unwrap_or_else(|_| "http://ollama.nxtgauge-ai.svc.cluster.local:11434".to_string());
|
||||
if audio.is_empty() {
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(serde_json::json!({ "error": "no_audio", "message": "No audio field found in the multipart body. Send the audio as field name 'audio', 'file', or 'voice'." })),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
// ── Whisper via LiteLLM (OpenAI-compatible audio/transcriptions endpoint) ──
|
||||
// LiteLLM proxies OpenAI Whisper and any compatible STT model. The endpoint
|
||||
// shape is identical to POST /v1/audio/transcriptions on OpenAI.
|
||||
//
|
||||
// Required env vars:
|
||||
// LITELLM_BASE_URL — already used by all chat calls in ai.rs
|
||||
// LITELLM_API_KEY — same key
|
||||
// LITELLM_WHISPER_MODEL — defaults to "whisper-1"
|
||||
let litellm_base = std::env::var("LITELLM_BASE_URL")
|
||||
.unwrap_or_else(|_| "http://litellm.nxtgauge-ai.svc.cluster.local:4000".to_string());
|
||||
let api_key = std::env::var("LITELLM_API_KEY").unwrap_or_default();
|
||||
let model = std::env::var("LITELLM_WHISPER_MODEL").unwrap_or_else(|_| "whisper-1".to_string());
|
||||
|
||||
let url = format!("{}/audio/transcriptions", litellm_base.trim_end_matches('/'));
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let url = format!("{}/api/audio/transcriptions", ollama_base);
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.text("model", std::env::var("OLLAMA_WHISPER_MODEL").unwrap_or_else(|_| "whisper".into()))
|
||||
.text("model", model)
|
||||
.text("response_format", "json")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(audio.clone()).file_name("audio.webm"),
|
||||
reqwest::multipart::Part::bytes(audio)
|
||||
.file_name("audio.webm")
|
||||
.mime_str("audio/webm")
|
||||
.unwrap_or_else(|_| reqwest::multipart::Part::bytes(vec![])),
|
||||
);
|
||||
|
||||
match client.post(&url).multipart(form).send().await {
|
||||
let mut req = client.post(&url).multipart(form);
|
||||
if !api_key.is_empty() {
|
||||
req = req.header("Authorization", format!("Bearer {api_key}"));
|
||||
}
|
||||
|
||||
match req.send().await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
// Best-effort parse; shape is whatever Ollama returned.
|
||||
let v: JsonValue = resp.json().await.unwrap_or(serde_json::json!({}));
|
||||
// OpenAI Whisper returns { "text": "..." }
|
||||
let transcript = v
|
||||
.get("transcript")
|
||||
.or_else(|| v.get("text"))
|
||||
.get("text")
|
||||
.or_else(|| v.get("transcript"))
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.to_string();
|
||||
let language = v
|
||||
.get("language")
|
||||
|
|
@ -278,25 +303,39 @@ async fn transcribe_voice(
|
|||
.unwrap_or("en")
|
||||
.to_string();
|
||||
let _ = state;
|
||||
return Json(TranscribeResponse {
|
||||
Json(TranscribeResponse {
|
||||
transcript,
|
||||
language,
|
||||
confidence: 0.85,
|
||||
confidence: 0.92,
|
||||
duration_ms,
|
||||
stub: false,
|
||||
})
|
||||
.into_response();
|
||||
.into_response()
|
||||
}
|
||||
_ => {
|
||||
// Whisper not available. Return 501 per spec with a stub body
|
||||
// in the JSON so simple clients can still render something.
|
||||
let body = serde_json::json!({
|
||||
"error": "voice_transcription_unavailable",
|
||||
"message": "Whisper model not configured on the Ollama cluster. Install a whisper model and set OLLAMA_WHISPER_MODEL.",
|
||||
"duration_ms": duration_ms,
|
||||
"hint": "POST raw audio as multipart field 'audio' to /api/ai/voice/transcribe",
|
||||
});
|
||||
return (StatusCode::NOT_IMPLEMENTED, Json(body)).into_response();
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
let body: JsonValue = resp.json().await.unwrap_or(serde_json::json!({}));
|
||||
tracing::error!("Whisper/LiteLLM returned {}: {:?}", status, body);
|
||||
(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Json(serde_json::json!({
|
||||
"error": "transcription_failed",
|
||||
"message": "Speech-to-text service returned an error.",
|
||||
"detail": body,
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Whisper/LiteLLM request failed: {}", e);
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(serde_json::json!({
|
||||
"error": "transcription_unavailable",
|
||||
"message": "Could not reach the speech-to-text service. Check LITELLM_BASE_URL.",
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue