From 7d31d1a2bd5c5afbc04112f3bf56012b4aca1422 Mon Sep 17 00:00:00 2001 From: Tracewebstudio Dev Date: Sun, 14 Jun 2026 18:00:33 +0200 Subject: [PATCH] feat: add LiteLLM provider and cover letter generation - Add LiteLLM provider implementing AiProvider trait - Support both Ollama and LiteLLM via LLM_PROVIDER env var - Add cover letter generation endpoint - Improve chat orchestrator with better intent detection - Add search_kb, explain_plan_limits, check_ai_pack_balance intents - Add LiteLLM config env vars --- src/chat/models.rs | 2 + src/chat/orchestrator.rs | 147 +++++++++++++++++++++++--- src/config/mod.rs | 11 ++ src/cover_letter/mod.rs | 2 + src/cover_letter/models.rs | 18 ++++ src/cover_letter/service.rs | 46 ++++++++ src/handlers/cover_letter.rs | 19 ++++ src/handlers/mod.rs | 1 + src/main.rs | 23 +++- src/providers/llm/litellm_provider.rs | 118 +++++++++++++++++++++ src/providers/llm/mod.rs | 1 + src/routes/mod.rs | 4 + src/state.rs | 6 +- 13 files changed, 379 insertions(+), 19 deletions(-) create mode 100644 src/cover_letter/mod.rs create mode 100644 src/cover_letter/models.rs create mode 100644 src/cover_letter/service.rs create mode 100644 src/handlers/cover_letter.rs create mode 100644 src/providers/llm/litellm_provider.rs diff --git a/src/chat/models.rs b/src/chat/models.rs index 7612b9d..0b35ca7 100644 --- a/src/chat/models.rs +++ b/src/chat/models.rs @@ -12,4 +12,6 @@ pub struct ChatMessageResponse { pub intent: String, pub reply: String, pub data: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, } diff --git a/src/chat/orchestrator.rs b/src/chat/orchestrator.rs index bd3bb32..3b02012 100644 --- a/src/chat/orchestrator.rs +++ b/src/chat/orchestrator.rs @@ -42,6 +42,7 @@ impl ChatOrchestrator { request: ChatMessageRequest, ) -> Result { let intent = classify_intent(&request.message); + let conversation_id = request.conversation_id.clone(); match intent.as_str() { "job_description_generation" => { @@ -60,9 +61,10 @@ impl ChatOrchestrator { .await?; Ok(ChatMessageResponse { - intent, + intent: intent.clone(), reply: "Generated a draft job description.".to_string(), data: serde_json::to_value(jd).unwrap_or(serde_json::Value::Null), + conversation_id: conversation_id.clone(), }) } "form_filling_assistance" => { @@ -75,23 +77,32 @@ impl ChatOrchestrator { .await?; Ok(ChatMessageResponse { - intent, + intent: intent.clone(), reply: extracted.suggested_next_step.clone(), data: serde_json::to_value(extracted).unwrap_or(serde_json::Value::Null), + conversation_id: conversation_id.clone(), }) } - "help_article_retrieval" => { + "search_kb" => { let matches = self.help_center.search(&request.message).await?; let reply = if matches.is_empty() { - "No help article match found yet.".to_string() + "No help article match found. Try rephrasing your question.".to_string() } else { - format!("Found {} help articles.", matches.len()) + let mut response = format!("Found {} help article(s):\n\n", matches.len()); + for (i, article) in matches.iter().take(5).enumerate() { + response.push_str(&format!("{}. **{}**\n{}\n\n", i + 1, article.title, article.summary)); + } + if matches.len() > 5 { + response.push_str(&format!("...and {} more articles.", matches.len() - 5)); + } + response }; Ok(ChatMessageResponse { - intent, + intent: intent.clone(), reply, data: serde_json::json!({ "matches": matches }), + conversation_id: conversation_id.clone(), }) } "support_ticket_creation" => { @@ -103,7 +114,7 @@ impl ChatOrchestrator { priority: "medium".to_string(), category: "general".to_string(), user_id: request.user_id.unwrap_or_else(|| "anonymous".to_string()), - conversation_id: request.conversation_id, + conversation_id: conversation_id.clone(), source: Some("chatbot".to_string()), tags: Some(vec!["chat".to_string()]), metadata: None, @@ -111,16 +122,75 @@ impl ChatOrchestrator { .await?; Ok(ChatMessageResponse { - intent, - reply: format!("Support ticket created: {}", created.ticket_id), + intent: intent.clone(), + reply: format!("Support ticket created: {}. Our team will respond shortly.", created.ticket_id), data: serde_json::to_value(created).unwrap_or(serde_json::Value::Null), + conversation_id: conversation_id.clone(), + }) + } + "explain_plan_limits" => { + let reply = "Your AI plan determines how many AI actions you can use per month.\n\n\ + - **Free AI**: 10 actions/month\n\ + - **Starter AI**: 100 actions/month\n\ + - **Growth AI**: 500 actions/month\n\ + - **Pro AI**: 2000 actions/month\n\n\ + You can purchase add-on packs for more usage. Would you like to upgrade your plan?"; + + Ok(ChatMessageResponse { + intent: intent.clone(), + reply: reply.to_string(), + data: serde_json::json!({}), + conversation_id: conversation_id.clone(), + }) + } + "check_ai_pack_balance" => { + let reply = "To check your AI balance, please visit the AI Usage section in your dashboard.\n\n\ + You can view:\n\ + - Monthly usage vs limit\n\ + - Add-on balance remaining\n\ + - Renewal date\n\n\ + Would you like me to help with anything else?"; + + Ok(ChatMessageResponse { + intent: intent.clone(), + reply: reply.to_string(), + data: serde_json::json!({}), + conversation_id: conversation_id.clone(), + }) + } + "generate_cover_letter" => { + let reply = "I can help you generate a cover letter. Please provide:\n\ + - The job title or position\n\ + - Your key skills and experience\n\ + - Any specific company or role details (optional)\n\n\ + Or you can use the 'Generate Cover Letter' button on the job application page."; + + Ok(ChatMessageResponse { + intent: intent.clone(), + reply: reply.to_string(), + data: serde_json::json!({}), + conversation_id: conversation_id.clone(), + }) + } + "improve_resume_summary" => { + let reply = "I can help improve your resume summary. Please share:\n\ + - Your current resume summary (or paste it here)\n\ + - The type of role you're targeting\n\ + - Your key skills and experience\n\n\ + Or you can use the 'Improve Resume' feature in your profile page."; + + Ok(ChatMessageResponse { + intent: intent.clone(), + reply: reply.to_string(), + data: serde_json::json!({}), + conversation_id: conversation_id.clone(), }) } _ => { let generic = self .ai_provider .complete( - "You are Nxtgauge workflow assistant. Keep answers concise and actionable.", + "You are Nxtgauge workflow assistant. Keep answers concise and actionable. If users ask about features, guide them to use the appropriate buttons or pages.", &request.message, ) .await?; @@ -129,6 +199,7 @@ impl ChatOrchestrator { intent, reply: generic, data: serde_json::json!({}), + conversation_id: conversation_id.clone(), }) } } @@ -137,25 +208,75 @@ impl ChatOrchestrator { fn classify_intent(message: &str) -> String { let text = message.to_lowercase(); - if text.contains("job description") || text.contains("jd") || text.contains("role") { + + if text.contains("job description") + || text.contains("generate job") + || text.contains("create job") + || text.contains("write job") + || (text.contains("jd") && text.len() < 10) + { return "job_description_generation".to_string(); } + + if text.contains("cover letter") + || text.contains("write a letter") + || text.contains("generate letter") + { + return "generate_cover_letter".to_string(); + } + + if text.contains("resume") + && (text.contains("improve") + || text.contains("rewrite") + || text.contains("summary") + || text.contains("tailor")) + { + return "improve_resume_summary".to_string(); + } + if text.contains("form") || text.contains("field") || text.contains("fill") { return "form_filling_assistance".to_string(); } - if text.contains("help") + + if text.contains("search kb") + || text.contains("find article") + || text.contains("how do i") + || text.contains("how to") + || text.contains("where do i") + || (text.contains("help") && !text.contains("help me")) || text.contains("article") || text.contains("kb") || text.contains("docs") + || text.contains("documentation") { - return "help_article_retrieval".to_string(); + return "search_kb".to_string(); } + if text.contains("ticket") || text.contains("support") || text.contains("issue") || text.contains("bug") + || text.contains("problem") + || text.contains("not working") + || text.contains("error") { return "support_ticket_creation".to_string(); } + + if text.contains("ai plan") + || text.contains("ai limit") + || text.contains("ai package") + || text.contains("ai credit") + || text.contains("upgrade ai") + { + return "explain_plan_limits".to_string(); + } + + if text.contains("balance") + && (text.contains("ai") || text.contains("credit") || text.contains("action")) + { + return "check_ai_pack_balance".to_string(); + } + "general".to_string() } diff --git a/src/config/mod.rs b/src/config/mod.rs index 329bd33..e056805 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -5,9 +5,13 @@ pub struct AppConfig { pub app_host: String, pub app_port: u16, pub database_url: Option, + pub llm_provider: String, pub ollama_base_url: String, pub ollama_chat_model: String, pub ollama_embed_model: String, + pub litellm_base_url: String, + pub litellm_api_key: String, + pub litellm_model: String, pub help_center_seed_path: String, pub tickets_source: String, pub nxtgauge_users_url: String, @@ -21,12 +25,19 @@ impl AppConfig { database_url: std::env::var("DATABASE_URL") .ok() .filter(|v| !v.trim().is_empty()), + llm_provider: env_or_default("LLM_PROVIDER", "ollama"), ollama_base_url: env_or_default( "OLLAMA_BASE_URL", "http://ollama.nxtgauge-ai.svc.cluster.local:11434", ), ollama_chat_model: env_or_default("OLLAMA_CHAT_MODEL", "gemma3:270m"), ollama_embed_model: env_or_default("OLLAMA_EMBED_MODEL", "nomic-embed-text"), + litellm_base_url: env_or_default( + "LITELLM_BASE_URL", + "https://llm.nxtgauge.com/v1", + ), + litellm_api_key: std::env::var("LITELLM_API_KEY").unwrap_or_default(), + litellm_model: env_or_default("LITELLM_MODEL", "askash-main"), help_center_seed_path: env_or_default( "HELP_CENTER_SEED_PATH", "./seeds/help_articles.json", diff --git a/src/cover_letter/mod.rs b/src/cover_letter/mod.rs new file mode 100644 index 0000000..b9e52db --- /dev/null +++ b/src/cover_letter/mod.rs @@ -0,0 +1,2 @@ +pub mod models; +pub mod service; diff --git a/src/cover_letter/models.rs b/src/cover_letter/models.rs new file mode 100644 index 0000000..ea36fdc --- /dev/null +++ b/src/cover_letter/models.rs @@ -0,0 +1,18 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateCoverLetterRequest { + pub job_title: String, + pub company_name: Option, + pub applicant_name: Option, + pub applicant_skills: Option>, + pub applicant_experience: Option, + pub tone: Option, + pub additional_notes: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateCoverLetterResponse { + pub cover_letter: String, + pub raw_markdown: String, +} diff --git a/src/cover_letter/service.rs b/src/cover_letter/service.rs new file mode 100644 index 0000000..61aaf09 --- /dev/null +++ b/src/cover_letter/service.rs @@ -0,0 +1,46 @@ +use std::sync::Arc; + +use crate::{cover_letter::models::*, error::AppError, providers::llm::ai_provider::AiProvider}; + +#[derive(Clone)] +pub struct CoverLetterService { + ai_provider: Arc, +} + +impl CoverLetterService { + pub fn new(ai_provider: Arc) -> Self { + Self { ai_provider } + } + + pub async fn generate( + &self, + request: GenerateCoverLetterRequest, + ) -> Result { + let system = "You are a professional cover letter writer. Write compelling, concise cover letters that highlight relevant skills and experience. Keep the tone professional but engaging."; + let user_prompt = format!( + "Write a cover letter for:\n\ + Job Title: {}\n\ + Company Name: {}\n\ + Applicant Name: {}\n\ + Key Skills: {:?}\n\ + Experience: {}\n\ + Tone: {}\n\ + Additional Notes: {:?}\n\n\ + Write a professional cover letter with 3-4 short paragraphs. Do not invent details not provided.", + request.job_title, + request.company_name.unwrap_or_else(|| "the company".to_string()), + request.applicant_name.unwrap_or_else(|| "the applicant".to_string()), + request.applicant_skills.clone().unwrap_or_default(), + request.applicant_experience.unwrap_or_else(|| "relevant experience".to_string()), + request.tone.unwrap_or_else(|| "professional".to_string()), + request.additional_notes + ); + + let generated = self.ai_provider.complete(system, &user_prompt).await?; + + Ok(GenerateCoverLetterResponse { + cover_letter: generated.lines().take(10).collect::>().join("\n"), + raw_markdown: generated, + }) + } +} diff --git a/src/handlers/cover_letter.rs b/src/handlers/cover_letter.rs new file mode 100644 index 0000000..1875b74 --- /dev/null +++ b/src/handlers/cover_letter.rs @@ -0,0 +1,19 @@ +use axum::{extract::State, Json}; + +use crate::{ + cover_letter::models::{GenerateCoverLetterRequest, GenerateCoverLetterResponse}, + state::AppState, + error::AppError, +}; + +pub async fn generate_cover_letter( + State(state): State, + Json(request): Json, +) -> Result, AppError> { + if request.job_title.trim().is_empty() { + return Err(AppError::BadRequest("job_title is required".to_string())); + } + + let generated = state.cover_letter_service.generate(request).await?; + Ok(Json(generated)) +} diff --git a/src/handlers/mod.rs b/src/handlers/mod.rs index e276ab0..1923bf4 100644 --- a/src/handlers/mod.rs +++ b/src/handlers/mod.rs @@ -1,4 +1,5 @@ pub mod chat; +pub mod cover_letter; pub mod forms; pub mod health; pub mod help; diff --git a/src/main.rs b/src/main.rs index b0e9057..26fed1d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ mod chat; mod config; +mod cover_letter; mod db; mod error; mod forms; @@ -16,7 +17,9 @@ use std::sync::Arc; use config::AppConfig; use providers::help_center::nxtgauge_help_center_provider::NxtgaugeHelpCenterProvider; use providers::llm::ollama_provider::OllamaAiProvider; +use providers::llm::litellm_provider::LiteLLMProvider; use providers::tickets::nxtgauge_ticket_provider::NxtgaugeTicketProvider; +use providers::llm::ai_provider::AiProvider; use retrieval::embeddings::ollama_embedding_provider::OllamaEmbeddingProvider; use state::AppState; use tracing::{info, warn}; @@ -35,12 +38,22 @@ async fn main() { } }; - let ai_provider = Arc::new(OllamaAiProvider::new( - cfg.ollama_base_url.clone(), - cfg.ollama_chat_model.clone(), - )); + let ai_provider: Arc = if cfg.llm_provider == "litellm" { + info!("Using LiteLLM provider with model {}", cfg.litellm_model); + Arc::new(LiteLLMProvider::new( + cfg.litellm_base_url.clone(), + cfg.litellm_api_key.clone(), + cfg.litellm_model.clone(), + )) as Arc + } else { + info!("Using Ollama provider with model {}", cfg.ollama_chat_model); + Arc::new(OllamaAiProvider::new( + cfg.ollama_base_url.clone(), + cfg.ollama_chat_model.clone(), + )) as Arc + }; - let embedding_provider = Arc::new(OllamaEmbeddingProvider::new( + let _embedding_provider = Arc::new(OllamaEmbeddingProvider::new( cfg.ollama_base_url.clone(), cfg.ollama_embed_model.clone(), )); diff --git a/src/providers/llm/litellm_provider.rs b/src/providers/llm/litellm_provider.rs new file mode 100644 index 0000000..017b041 --- /dev/null +++ b/src/providers/llm/litellm_provider.rs @@ -0,0 +1,118 @@ +use async_trait::async_trait; +use reqwest::Client; +use serde::{Deserialize, Serialize}; + +use crate::{error::AppError, providers::llm::ai_provider::AiProvider}; + +#[derive(Clone)] +pub struct LiteLLMProvider { + client: Client, + base_url: String, + api_key: String, + model: String, +} + +impl LiteLLMProvider { + pub fn new(base_url: String, api_key: String, model: String) -> Self { + Self { + client: Client::new(), + base_url, + api_key, + model, + } + } + + fn fallback_response(user_prompt: &str) -> String { + format!( + "LiteLLM model is unavailable right now. I captured your request so workflow can continue: {}", + user_prompt + ) + } +} + +#[derive(Debug, Serialize)] +struct ChatMessage { + role: String, + content: String, +} + +#[derive(Debug, Serialize)] +struct ChatCompletionRequest { + model: String, + messages: Vec, + temperature: f32, + max_tokens: i32, +} + +#[derive(Debug, Deserialize)] +struct ChatCompletionResponse { + choices: Vec, +} + +#[derive(Debug, Deserialize)] +struct Choice { + message: Message, +} + +#[derive(Debug, Deserialize)] +struct Message { + content: String, +} + +#[async_trait] +impl AiProvider for LiteLLMProvider { + async fn complete(&self, system_prompt: &str, user_prompt: &str) -> Result { + let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/')); + + let payload = ChatCompletionRequest { + model: self.model.clone(), + messages: vec![ + ChatMessage { + role: "system".to_string(), + content: system_prompt.to_string(), + }, + ChatMessage { + role: "user".to_string(), + content: user_prompt.to_string(), + }, + ], + temperature: 0.2, + max_tokens: 2048, + }; + + let res = self + .client + .post(url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .header("Content-Type", "application/json") + .json(&payload) + .send() + .await; + + let Ok(res) = res else { + return Ok(Self::fallback_response(user_prompt)); + }; + + if !res.status().is_success() { + let status = res.status(); + let body = res.text().await.unwrap_or_default(); + tracing::warn!("LiteLLM request failed: {} - {}", status, body); + return Ok(Self::fallback_response(user_prompt)); + } + + let body: Result = res.json().await; + match body { + Ok(parsed) => { + if let Some(choice) = parsed.choices.first() { + Ok(choice.message.content.trim().to_string()) + } else { + Ok(Self::fallback_response(user_prompt)) + } + } + Err(e) => { + tracing::warn!("Failed to parse LiteLLM response: {}", e); + Ok(Self::fallback_response(user_prompt)) + } + } + } +} diff --git a/src/providers/llm/mod.rs b/src/providers/llm/mod.rs index 52d971d..9f94b72 100644 --- a/src/providers/llm/mod.rs +++ b/src/providers/llm/mod.rs @@ -1,2 +1,3 @@ pub mod ai_provider; pub mod ollama_provider; +pub mod litellm_provider; diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 66c72df..3874858 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -17,6 +17,10 @@ pub fn build_router(state: AppState) -> Router { "/jobs/generate-description", post(handlers::jobs::generate_description), ) + .route( + "/cover-letter/generate", + post(handlers::cover_letter::generate_cover_letter), + ) .route("/forms/extract", post(handlers::forms::extract)) .route("/tickets/create", post(handlers::tickets::create)) .route("/help/search", post(handlers::help::search)), diff --git a/src/state.rs b/src/state.rs index be49c17..8deaa15 100644 --- a/src/state.rs +++ b/src/state.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use crate::{ chat::orchestrator::ChatOrchestrator, config::AppConfig, + cover_letter::service::CoverLetterService, db::Database, forms::service::FormService, jobs::service::JobsService, @@ -20,6 +21,7 @@ pub struct AppState { pub chat_orchestrator: ChatOrchestrator, pub jobs_service: JobsService, pub form_service: FormService, + pub cover_letter_service: CoverLetterService, pub ticket_service: TicketService, pub help_center: Arc, } @@ -34,13 +36,14 @@ impl AppState { ) -> Self { let jobs_service = JobsService::new(ai_provider.clone()); let form_service = FormService::new(ai_provider.clone()); + let cover_letter_service = CoverLetterService::new(ai_provider.clone()); let ticket_service = TicketService::new(ticket_provider, db.clone()); let chat_orchestrator = ChatOrchestrator::new( jobs_service.clone(), form_service.clone(), help_center.clone(), ticket_service.clone(), - ai_provider, + ai_provider.clone(), ); Self { @@ -49,6 +52,7 @@ impl AppState { chat_orchestrator, jobs_service, form_service, + cover_letter_service, ticket_service, help_center, }