init commit

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+RUN apt-get update && \
+apt-get install -y --no-install-recommends git g++ make wget && \
+apt-get clean && \
+rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+EXPOSE 7860
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,11 @@
 ---
 title: General Agent
-emoji: 🔥
-colorFrom: green
-colorTo: pink
+emoji: 🏃
+colorFrom: red
+colorTo: yellow
 sdk: docker
 pinned: false
+short_description: First attempt to build and expose agent
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

main.py

@@ -0,0 +1,94 @@
+import typing as t
+import os
+# from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import logging
+
+from src.utils import (
+    OpenAIClient,
+    TogetherAIClient,
+    GeminiClient,
+    GroqClient,
+    MistralClient,
+)
+from src.models_enums import ModelProvider
+
+# load_dotenv()
+
+assert os.environ['TOGETHER_API_KEY'] is not None
+
+# Configure basic logging to see messages in stdout (and thus in HF Space logs)
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class RequestData(BaseModel):
+    prompt: str
+    max_tokens: int = 50
+    system_prompt: t.Optional[str] = None
+
+MODEL_PROVIDER2CLIENT = {
+    ModelProvider.OPENAI.value: OpenAIClient,
+    ModelProvider.GEMINI.value: GeminiClient,
+    ModelProvider.TOGETHERAI.value: TogetherAIClient,
+    ModelProvider.GROQ.value: GroqClient,
+    ModelProvider.MISTRAL.value: MistralClient,
+}
+
+
+app = FastAPI()
+logger.info("FastAPI app initialized.")
+
+
+# The application now starts without initializing a specific LLM,
+# which makes it more flexible.
+
+@app.post("/generate/{model_provider}/{model_name:path}")
+async def generate_text(
+    model_provider: str,
+    model_name: str,
+    request: RequestData
+):
+    """
+    Generates text using a specified LLM provider and model.
+
+    Example:
+    POST /generate/togetherai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free
+    with body: {"prompt": "...", "max_tokens": 100}
+    """
+    logger.info(f"Received POST request to /generate/{model_provider}/{model_name}.")
+
+    # Check if the requested model provider exists
+    if model_provider not in MODEL_PROVIDER2CLIENT:
+        logger.error(f"Invalid model provider: {model_provider}")
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid model provider: {model_provider}. "
+                   f"Available providers: {[p.value for p in ModelProvider]}"
+        )
+
+    try:
+        # Get the correct client class and instantiate it dynamically
+        llm_client_class = MODEL_PROVIDER2CLIENT[model_provider]
+        llm_client = llm_client_class(model=model_name)
+
+        # Call the client's async method
+        output = await llm_client(
+            prompt=request.prompt, 
+            system_prompt=request.system_prompt,
+            max_tokens=request.max_tokens
+        )
+        
+        return output
+    
+    except Exception as e:
+        logger.error(
+            f"Error during text generation for {model_provider}/{model_name}: {str(e)}", 
+            exc_info=True
+        )
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/health")
+async def health_check():
+    logger.info("Received GET request to /health.")
+    return {"status": "ok"}

requirements.txt

@@ -0,0 +1,11 @@
+openai
+fastapi==0.109.0
+uvicorn==0.27.0
+torch==2.2.1 --index-url https://download.pytorch.org/whl/cpu
+transformers==4.40.2
+accelerate==0.29.3
+sentencepiece==0.2.0
+numpy==1.26.4
+protobuf==3.20.3
+python-dotenv
+together

src/models_enums.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+class ModelProvider(Enum):
+    OPENAI = 'openai'
+    GEMINI = 'gemini'
+    MISTRAL = 'mistral'
+    TOGETHERAI = 'togetherai'
+    GROQ = 'groq'
+    

src/utils.py

@@ -0,0 +1,260 @@
+import typing as t
+import asyncio
+from abc import ABC, abstractmethod
+import os
+
+# External Libraries
+import requests
+import json
+from together import Together
+from openai import AsyncOpenAI
+import aiohttp # Using aiohttp for async HTTP requests
+
+# A standard response type to avoid type errors with Together.
+try:
+    from together.types.chat_completions import ChatCompletionResponse
+except ImportError:
+    ChatCompletionResponse = t.Any
+
+
+# --- ABSTRACT BASE CLASS (The Core Abstraction) ---
+class BaseLLMClient(ABC):
+    """
+    Abstract base class for all LLM clients.
+    Defines a common, asynchronous interface for calling LLMs.
+    """
+    def __init__(self, model: str, **kwargs):
+        self.client = None
+        self.model = model
+        self.kwargs = kwargs
+        
+    @abstractmethod
+    async def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = 1_000,
+        system_prompt: t.Optional[str] = None,
+        **kwargs
+    ) -> str:
+        """
+        Defines the async call method for all concrete clients.
+        """
+        ...
+        
+    def _create_messages(
+        self,
+        prompt: str,
+        system_prompt: t.Optional[str] = None
+    ) -> t.List[t.Dict[str, str]]:
+        """
+        Helper to create a standard message dictionary with an optional system prompt.
+        The system prompt is always added as the first message to set the model's context.
+        """
+        messages = []
+        if system_prompt:
+            messages.append({"role": "system", "content": system_prompt})
+        messages.append({"role": "user", "content": prompt})
+        return messages
+
+
+# --- CONCRETE IMPLEMENTATIONS (The Adapters) ---
+
+class OpenAIClient(BaseLLMClient):
+    """
+    Adapter for the OpenAI (and OpenAI-compatible) Async API client.
+    """
+    def __init__(self, model: str, **kwargs):
+        super().__init__(model, **kwargs)
+        self.client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+    async def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = 1_000,
+        system_prompt: t.Optional[str] = None,
+        **kwargs
+    ) -> str:
+        try:
+            messages = self._create_messages(prompt, system_prompt)
+            response = await self.client.chat.completions.create(
+                model=self.model,
+                messages=messages,
+                max_tokens=max_tokens,
+                **self.kwargs,
+                **kwargs,
+            )
+            return response.choices[0].message.content
+        except Exception as e:
+            return f"Error from OpenAI: {e}"
+
+
+class TogetherAIClient(BaseLLMClient):
+    """
+    Adapter for the Together API client.
+    Uses asyncio.to_thread to run the synchronous client in a separate thread.
+    """
+    def __init__(self, model: str, **kwargs):
+        super().__init__(model, **kwargs)
+        # Note: Together() automatically looks for TOGETHER_API_KEY env var
+        self.client = Together() 
+
+    async def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = 1_000,
+        system_prompt: t.Optional[str] = None,
+        **kwargs
+    ) -> str:
+        # Use asyncio.to_thread to run the synchronous Together client
+        # without blocking the event loop.
+        try:
+            messages = self._create_messages(prompt, system_prompt)
+            response: ChatCompletionResponse = await asyncio.to_thread(
+                self.client.chat.completions.create,
+                model=self.model,
+                messages=messages,
+                max_tokens=max_tokens,
+                **self.kwargs,
+                **kwargs,
+            )
+            return str(response.choices[0].message.content)
+        except Exception as e:
+            return f"Error from TogetherAI: {e}"
+
+
+class GeminiClient(BaseLLMClient):
+    """
+    Adapter for the Gemini REST API, using aiohttp for async HTTP requests.
+    """
+    def __init__(self, model: str, **kwargs):
+        super().__init__(model, **kwargs)
+        self.api_key = os.getenv("GEMINI_API_KEY")
+        self.url = f"https://generativelanguage.googleapis.com/v1beta/models/{self.model}:generateContent?key={self.api_key}"
+
+    async def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = 1_000,
+        system_prompt: t.Optional[str] = None,
+        **kwargs
+    ) -> str:
+        if not self.api_key:
+            return "Error: GEMINI_API_KEY not found."
+
+        contents = self._create_messages(prompt, system_prompt)
+        payload = {
+            "contents": contents,
+            "generationConfig": {"maxOutputTokens": max_tokens},
+            **self.kwargs,
+            **kwargs,
+        }
+        
+        headers = {"Content-Type": "application/json"}
+        
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(self.url, headers=headers, json=payload) as response:
+                    response.raise_for_status()
+                    response_data = await response.json()
+                    return response_data['candidates'][0]['content']['parts'][0]['text']
+        except aiohttp.ClientError as e:
+            return f"Error from Gemini (requests): {e}"
+        except (KeyError, IndexError) as e:
+            return f"Error parsing Gemini response: {e}"
+
+
+class GroqClient(BaseLLMClient):
+    """
+    Adapter for the Groq REST API, using aiohttp for async HTTP requests.
+    """
+    def __init__(self, model: str, **kwargs):
+        super().__init__(model, **kwargs)
+        self.api_key = os.getenv("GROQ_API_KEY")
+        self.url = "https://api.groq.com/openai/v1/chat/completions"
+
+    async def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = 1_000,
+        system_prompt: t.Optional[str] = None,
+        **kwargs
+    ) -> str:
+        if not self.api_key:
+            return "Error: GROQ_API_KEY not found."
+
+        messages = self._create_messages(prompt, system_prompt)
+        payload = {
+            "model": self.model,
+            "messages": messages,
+            "max_tokens": max_tokens,
+            **self.kwargs,
+            **kwargs
+        }
+
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(self.url, headers=headers, json=payload) as response:
+                    response.raise_for_status()
+                    response_data = await response.json()
+                    return response_data['choices'][0]['message']['content']
+        except aiohttp.ClientError as e:
+            return f"Error from Groq (requests): {e}"
+        except (KeyError, IndexError) as e:
+            return f"Error parsing Groq response: {e}"
+
+
+class MistralClient(BaseLLMClient):
+    """
+    Adapter for the Mistral REST API, using aiohttp for async HTTP requests.
+    """
+    def __init__(self, model: str, **kwargs):
+        super().__init__(model, **kwargs)
+        self.api_key = os.getenv("MISTRAL_API_KEY")
+        self.url = "https://api.mistral.ai/v1/chat/completions"
+
+    async def __call__(
+        self,
+        prompt: str,
+        max_tokens: int = 1_000,
+        system_prompt: t.Optional[str] = None,
+        **kwargs
+    ) -> str:
+        if not self.api_key:
+            return "Error: MISTRAL_API_KEY not found."
+
+        messages = self._create_messages(prompt, system_prompt)
+        payload = {
+            "model": self.model,
+            "messages": messages,
+            "max_tokens": max_tokens,
+            **self.kwargs,
+            **kwargs
+        }
+
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(self.url, headers=headers, json=payload) as response:
+                    response.raise_for_status()
+                    response_data = await response.json()
+                    return response_data['choices'][0]['message']['content']
+        except aiohttp.ClientError as e:
+            return f"Error from Mistral (requests): {e}"
+        except (KeyError, IndexError) as e:
+            return f"Error parsing Mistral response: {e}"
+
+        
+# ('openai', OpenAIClient(model="gpt-3.5-turbo")),
+# ('togetherai', TogetherAIClient(model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free")),
+# ('gemini', GeminiClient(model="gemini-1.5-flash-latest")),
+# ('groq', GroqClient(model="llama3-8b-8192")),
+# ('mistral', MistralClient(model="mistral-tiny")),