Refactor LLM and voice provider settings, enhance configuration options, and update dependencies

.env.example

@@ -1,25 +1,18 @@
-
-LLM_PROVIDER=nvidia  # or "huggingface"
+# NVIDIA LLM Settings
 NVIDIA_API_KEY=your_nvidia_api_key_here
 NVIDIA_MODEL=meta/llama-3.1-8b-instruct
 
-# Uncomment the following block when using HuggingFace instead of NVIDIA
-# LLM_PROVIDER=huggingface
-# HF_MODEL=microsoft/DialoGPT-medium
-# HF_TOKEN=your_huggingface_token_here  # Get from: https://huggingface.co/settings/tokens
-# HF_USE_INFERENCE_API=false  # true to use the Hugging Face Inference API, false to run locally
-# HF_TRUST_REMOTE_CODE=false  # Enable when the repo requires custom model/tokenizer code
-# HF_USE_FAST_TOKENIZER=false  # Set to true when you need the fast tokenizer; disable to avoid legacy conversion issues
+# HuggingFace Settings
+HF_TOKEN=your_huggingface_token_here  # Get from: https://huggingface.co/settings/tokens
 
+# LLM Parameters
+LLM_TEMPERATURE=0.7
+LLM_MAX_TOKENS=1024
 
-# Voice Provider Options
-# NVIDIA API uses the same NVIDIA_API_KEY as the LLM provider
+# Voice Provider Settings
 VOICE_PROVIDER=nvidia
-
-# NVIDIA Voice Settings (default)
 NVIDIA_VOICE_LANGUAGE=en-US
 NVIDIA_VOICE_NAME=Magpie-Multilingual.EN-US.Aria
 
 # NVIDIA TTS requires an endpoint from build.nvidia.com
-# Get your TTS endpoint from: https://build.nvidia.com/
 NVIDIA_TTS_ENDPOINT=https://your-tts-endpoint-here

pyproject.toml

@@ -16,10 +16,10 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "python-multipart>=0.0.22",
     "streamlit>=1.53.1",
-    "transformers",
-    "torch",
     "uvicorn[standard]>=0.40.0",
     "websockets>=16.0",
+    "transformers>=4.32.0",
+    "torch>=2.1.1",
     "accelerate>=1.12.0",
 ]
 

src/agent/llm_factory.py

@@ -1,7 +1,5 @@
-from typing import Dict, Optional
-
-from langchain_core.language_models import BaseLanguageModel
-from langchain_huggingface import HuggingFaceEndpoint, HuggingFacePipeline
+from huggingface_hub import InferenceClient
+from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
 from langchain_nvidia_ai_endpoints import ChatNVIDIA
 
 from src.core.logger import logger
@@ -9,83 +7,62 @@ from src.core.settings import settings
 
 
 class LLMFactory:
-    _instances: Dict[str, BaseLanguageModel] = {}
-
-    @classmethod
-    def create_llm(cls, provider: Optional[str] = None) -> BaseLanguageModel:
-        provider = (provider or settings.llm.LLM_PROVIDER).lower()
-
-        if provider in cls._instances:
-            return cls._instances[provider]
-
-        if provider == "nvidia":
-            llm = cls._create_nvidia_llm()
-        elif provider == "huggingface":
-            llm = cls._create_huggingface_llm()
-        else:
-            raise ValueError(f"Unknown LLM provider: {provider}")
-
-        cls._instances[provider] = llm
-        return llm
-
-    @classmethod
-    def reset_cache(cls, provider: Optional[str] = None) -> None:
-        if provider:
-            cls._instances.pop(provider.lower(), None)
-        else:
-            cls._instances.clear()
-
     @staticmethod
-    def _create_nvidia_llm() -> BaseLanguageModel:
-        logger.info(f"Initializing NVIDIA LLM: {settings.llm.NVIDIA_MODEL}")
+    def create_nvidia_llm(
+        model: str = settings.llm.NVIDIA_MODEL,
+        temperature: float = settings.llm.LLM_TEMPERATURE,
+        max_tokens: int = settings.llm.LLM_MAX_TOKENS,
+    ) -> ChatNVIDIA:
+        logger.info(f"Initializing NVIDIA LLM: {model}")
 
         if not settings.llm.NVIDIA_API_KEY:
             raise ValueError("NVIDIA_API_KEY must be set to use the NVIDIA LLM provider.")
 
         return ChatNVIDIA(
-            model=settings.llm.NVIDIA_MODEL,
+            model=model,
             api_key=settings.llm.NVIDIA_API_KEY,
-            temperature=settings.llm.LLM_TEMPERATURE,
-            max_completion_tokens=settings.llm.LLM_MAX_TOKENS,
+            temperature=temperature,
+            max_completion_tokens=max_tokens,
         )
 
     @staticmethod
-    def _create_huggingface_llm() -> BaseLanguageModel:
-        model_id = settings.llm.HF_MODEL
-        if not model_id:
-            raise ValueError("HF_MODEL must be set when using the HuggingFace LLM provider.")
+    def create_huggingface_llm(
+        model_id: str,
+        provider: str = "auto",
+        temperature: float = settings.llm.LLM_TEMPERATURE,
+        max_tokens: int = settings.llm.LLM_MAX_TOKENS,
+    ) -> ChatHuggingFace:
+        token = (settings.llm.HF_TOKEN or "").strip()
+        if not token:
+            raise ValueError("HF_TOKEN must be set to use the HuggingFace LLM provider.")
+
+        logger.info(f"Initializing HuggingFace LLM: {model_id} via provider={provider}")
+
+        llm = HuggingFaceEndpoint(
+            repo_id=model_id,
+            provider=provider,
+            huggingfacehub_api_token=token,
+            temperature=temperature,
+            max_new_tokens=max_tokens,
+        )
+        return ChatHuggingFace(llm=llm)
+
+    @staticmethod
+    def create_huggingface_stt(model_id: str | None = None) -> InferenceClient:
+        token = (settings.llm.HF_TOKEN or "").strip()
+        if not token:
+            raise ValueError("HF_TOKEN must be set to use the HuggingFace STT provider.")
 
-        if settings.llm.HF_USE_INFERENCE_API:
-            if not settings.llm.HF_TOKEN or not settings.llm.HF_TOKEN.strip():
-                raise ValueError(
-                    "HF_TOKEN must be provided when HF_USE_INFERENCE_API is true."
-                )
+        logger.info(f"Initializing HuggingFace STT: {model_id or 'default'}")
 
-            logger.info(f"Initializing Hugging Face Inference API LLM: {model_id}")
-            return HuggingFaceEndpoint(
-                repo_id=model_id,
-                huggingfacehub_api_token=settings.llm.HF_TOKEN,
-                temperature=settings.llm.LLM_TEMPERATURE,
-                max_new_tokens=settings.llm.LLM_MAX_TOKENS,
-            )
+        return InferenceClient(model=model_id, token=token)
 
-        logger.info(f"Initializing local Hugging Face LLM: {model_id}")
-        logger.info("Downloading model if not already cached...")
-        return HuggingFacePipeline.from_model_id(
-            model_id=model_id,
-            task="text-generation",
-            trust_remote_code=settings.llm.HF_TRUST_REMOTE_CODE,
-            device_map="auto",
-            model_kwargs={
-                "temperature": settings.llm.LLM_TEMPERATURE,
-                "do_sample": True,
-            },
-            pipeline_kwargs={
-                "max_new_tokens": settings.llm.LLM_MAX_TOKENS,
-                "temperature": settings.llm.LLM_TEMPERATURE,
-                "do_sample": True,
-                "tokenizer_kwargs": {
-                    "use_fast": settings.llm.HF_USE_FAST_TOKENIZER,
-                },
-            },
-        )
+    @staticmethod
+    def create_huggingface_tts(model_id: str | None = None) -> InferenceClient:
+        token = (settings.llm.HF_TOKEN or "").strip()
+        if not token:
+            raise ValueError("HF_TOKEN must be set to use the HuggingFace TTS provider.")
+
+        logger.info(f"Initializing HuggingFace TTS: {model_id or 'default'}")
+
+        return InferenceClient(model=model_id, token=token)

src/core/settings.py

@@ -2,7 +2,7 @@ import json
 from pathlib import Path
 from typing import Any, Optional
 
-from pydantic import Field, ValidationError, field_validator
+from pydantic import Field, ValidationError
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from dotenv import load_dotenv
 
@@ -61,28 +61,14 @@ class VoiceSettings(CoreSettings):
 
 
 class LLMSettings(CoreSettings):
-    LLM_PROVIDER: str = Field(default="nvidia")
-    
     NVIDIA_API_KEY: Optional[str] = Field(default=None)
     NVIDIA_MODEL: str = Field(default="meta/llama-3.1-8b-instruct")
     NVIDIA_BASE_URL: str = Field(default="https://integrate.api.nvidia.com/v1")
-    
+
     HF_TOKEN: Optional[str] = Field(default=None)
-    HF_MODEL: Optional[str] = Field(default="TheBloke/Llama-2-7B-Chat-GGUF")
-    HF_USE_INFERENCE_API: bool = Field(default=False)
-    HF_TRUST_REMOTE_CODE: bool = Field(default=False)
-    HF_USE_FAST_TOKENIZER: bool = Field(default=False)
-    
+
     LLM_TEMPERATURE: float = Field(default=0.7, ge=0.0, le=2.0)
     LLM_MAX_TOKENS: int = Field(default=1024, gt=0)
-    LLM_STREAMING: bool = Field(default=True)
-    
-    @field_validator("LLM_PROVIDER")
-    @classmethod
-    def validate_provider(cls, v: str) -> str:
-        if v.lower() not in ["nvidia", "huggingface"]:
-            raise ValueError("LLM_PROVIDER must be 'nvidia' or 'huggingface'")
-        return v.lower()
 
 
 class APISettings(CoreSettings):

src/models/voice/factory.py

@@ -1,41 +0,0 @@
-from typing import Dict, Type, Callable
-
-from src.core.logger import logger
-from src.core.settings import settings
-from src.models.voice.base import BaseVoiceProvider, VoiceProviderConfig
-from src.models.voice.nvidia import NvidiaVoiceProvider, NvidiaConfig
-
-
-class VoiceProviderFactory:
-    _registry: Dict[str, Callable[[VoiceProviderConfig], BaseVoiceProvider]] = {}
-    
-    @classmethod
-    def register(cls, name: str, provider_class: Type[BaseVoiceProvider]) -> None:
-        cls._registry[name.lower()] = provider_class
-        logger.debug(f"Registered voice provider: {name}")
-    
-    @classmethod
-    def create_provider(cls, provider_name: str = None) -> BaseVoiceProvider:
-        provider_name = provider_name or settings.voice.VOICE_PROVIDER
-        provider_name = provider_name.lower()
-        
-        if provider_name not in cls._registry:
-            raise ValueError(f"Unknown voice provider: {provider_name}. Available: {list(cls._registry.keys())}")
-
-        logger.info(f"Creating voice provider: {provider_name}")
-
-        if provider_name == "nvidia":
-            config = NvidiaConfig(
-                api_key=settings.llm.NVIDIA_API_KEY,
-                language=settings.voice.NVIDIA_VOICE_LANGUAGE,
-                voice_name=settings.voice.NVIDIA_VOICE_NAME,
-                tts_model=settings.voice.NVIDIA_TTS_MODEL,
-                tts_endpoint=settings.voice.NVIDIA_TTS_ENDPOINT,
-                sample_rate_output=settings.voice.SAMPLE_RATE_OUTPUT,
-            )
-            return cls._registry[provider_name](config)
-
-        raise NotImplementedError(f"Configuration for {provider_name} not yet implemented")
-
-
-VoiceProviderFactory.register("nvidia", NvidiaVoiceProvider)

src/models/voice/nvidia.py

@@ -1,91 +0,0 @@
-import asyncio
-from typing import AsyncIterator, Optional
-
-import httpx
-
-from src.core.logger import logger
-from src.models.voice.base import BaseVoiceProvider, VoiceProviderConfig
-from src.models.voice.types import TranscriptionResult, VADInfo
-
-
-class NvidiaConfig(VoiceProviderConfig):
-    provider_name: str = "nvidia"
-    api_key: str
-    language: str = "en-US"
-    voice_name: str = "Magpie-Multilingual.EN-US.Aria"
-    tts_model: str = "magpie-tts-multilingual"
-    tts_endpoint: str = ""
-    sample_rate_output: int = 48000
-
-
-class NvidiaVoiceProvider(BaseVoiceProvider):
-    def __init__(self, config: NvidiaConfig):
-        super().__init__(config)
-        self.config: NvidiaConfig = config
-        self._current_vad: Optional[VADInfo] = None
-
-    async def connect(self) -> None:
-        # No connection needed for HTTP API
-        self._connected = True
-        logger.info("NVIDIA API TTS provider ready")
-
-    async def disconnect(self) -> None:
-        self._connected = False
-        logger.info("NVIDIA API TTS provider disconnected")
-
-    async def text_to_speech(
-        self, text: str, stream: bool = True
-    ) -> AsyncIterator[bytes]:
-        if not self.is_connected:
-            raise RuntimeError("NVIDIA API provider not connected")
-
-        if not self.config.tts_endpoint:
-            raise RuntimeError(
-                "TTS requires NVIDIA_TTS_ENDPOINT to be set. "
-                "Get a TTS endpoint from: https://build.nvidia.com/"
-            )
-
-        async for chunk in self._text_to_speech_http(text, stream):
-            yield chunk
-
-    async def _text_to_speech_http(
-        self, text: str, stream: bool = True
-    ) -> AsyncIterator[bytes]:
-        endpoint = self.config.tts_endpoint.rstrip("/")
-        url = f"{endpoint}/v1/audio/synthesize"
-
-        try:
-            logger.debug(f"Generating speech via HTTP API for text: {text[:50]}...")
-
-            headers = {
-                "Authorization": f"Bearer {self.config.api_key}",
-                "Content-Type": "application/json"
-            }
-
-            payload = {
-                "language": self.config.language,
-                "text": text,
-                "voice": self.config.voice_name,
-                "sample_rate_hz": self.config.sample_rate_output,
-            }
-
-            async with httpx.AsyncClient(timeout=60.0) as client:
-                response = await client.post(url, json=payload, headers=headers)
-                response.raise_for_status()
-
-                # For streaming, we need to handle the response appropriately
-                # For now, return the full content
-                yield response.content
-
-            logger.debug("HTTP TTS generation complete")
-
-        except httpx.HTTPStatusError as e:
-            logger.error(f"HTTP error in NVIDIA TTS API: {e.response.status_code} - {e.response.text}")
-            raise RuntimeError(f"NVIDIA TTS API error: {e.response.status_code}") from e
-        except Exception as e:
-            logger.error(f"Error in NVIDIA HTTP TTS API: {e}")
-            raise
-
-
-    async def get_vad_info(self) -> Optional[VADInfo]:
-        return self._current_vad

uv.lock

@@ -1099,8 +1099,8 @@ requires-dist = [
     { name = "python-dotenv", specifier = ">=1.0.0" },
     { name = "python-multipart", specifier = ">=0.0.22" },
     { name = "streamlit", specifier = ">=1.53.1" },
-    { name = "torch" },
-    { name = "transformers" },
+    { name = "torch", specifier = ">=2.1.1" },
+    { name = "transformers", specifier = ">=4.32.0" },
     { name = "uvicorn", extras = ["standard"], specifier = ">=0.40.0" },
     { name = "websockets", specifier = ">=16.0" },
 ]