trying to load the models from HF

app.py

@@ -6,7 +6,6 @@ import os
 import pickle
 from typing import List, Tuple, Optional
 import gradio as gr
-from openai import OpenAI
 from functools import lru_cache
 from tenacity import retry, stop_after_attempt, wait_exponential
 from langchain_community.retrievers import BM25Retriever
@@ -15,25 +14,31 @@ from langchain_core.embeddings import Embeddings
 from langchain_core.documents import Document
 from collections import defaultdict
 import hashlib
-from tqdm import tqdm  
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from sentence_transformers import SentenceTransformer
+from huggingface_hub import login
 
 # --- Configuration ---
 FAISS_INDEX_PATH = "faiss_index"
 BM25_INDEX_PATH = "bm25_index.pkl"
 CACHE_VERSION = "v1"  # Increment when data format changes
-embedding_model = "e5-mistral-7b-instruct"
-generation_model = "meta-llama-3-70b-instruct"
+embedding_model_name = "intfloat/e5-mistral-7b-instruct"
+generation_model_name = "meta-llama/Meta-Llama-3-70B-Instruct"
 data_file_name = "AskNatureNet_data_enhanced.json"
-API_CONFIG = {
-    "api_key": os.getenv("API_KEY"),
-    "base_url": "https://chat-ai.academiccloud.de/v1"
-}
 CHUNK_SIZE = 800
 OVERLAP = 200
 EMBEDDING_BATCH_SIZE = 32  # Batch size for embedding API calls
 
-# Initialize clients
-client = OpenAI(**API_CONFIG)
+# Login to Hugging Face Hub
+login(token="llama3")
+
+# Initialize models
+embedding_model = SentenceTransformer(embedding_model_name)
+tokenizer = AutoTokenizer.from_pretrained(generation_model_name)
+generation_model = AutoModelForCausalLM.from_pretrained(generation_model_name)
+generation_pipeline = pipeline("text-generation", model=generation_model, tokenizer=tokenizer)
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -52,12 +57,7 @@ class MistralEmbeddings(Embeddings):
             # Process in batches with progress tracking
             for i in tqdm(range(0, len(texts), EMBEDDING_BATCH_SIZE), desc="Embedding Progress"):
                 batch = texts[i:i + EMBEDDING_BATCH_SIZE]
-                response = client.embeddings.create(
-                    input=batch,
-                    model=embedding_model,
-                    encoding_format="float"
-                )
-                embeddings.extend([e.embedding for e in response.data])
+                embeddings.extend(embedding_model.encode(batch))
             return embeddings
         except Exception as e:
             logger.error(f"Embedding Error: {str(e)}")
@@ -167,16 +167,12 @@ class EnhancedRetriever:
     @lru_cache(maxsize=500)
     def _hyde_expansion(self, query: str) -> str:
         try:
-            response = client.chat.completions.create(
-                model=generation_model,
-                messages=[{
-                    "role": "user",
-                    "content": f"Generate a technical draft about biomimicry for: {query}\nInclude domain-specific terms."
-                }],
-                temperature=0.5,
-                max_tokens=200
+            response = generation_pipeline(
+                f"Generate a technical draft about biomimicry for: {query}\nInclude domain-specific terms.",
+                max_length=200,
+                temperature=0.5
             )
-            return response.choices[0].message.content
+            return response[0]['generated_text']
         except Exception as e:
             logger.error(f"HyDE Error: {str(e)}")
             return query
@@ -212,23 +208,18 @@ SYSTEM_PROMPT = """**Biomimicry Expert Guidelines**
 2. Cite sources as [Source]
 3. **Bold** technical terms
 4. Include reference links
-
 Context: {context}"""
 
 @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=20))
 def get_ai_response(query: str, context: str) -> str:
     try:
-        response = client.chat.completions.create(
-            model=generation_model,
-            messages=[
-                {"role": "system", "content": SYSTEM_PROMPT.format(context=context)},
-                {"role": "user", "content": f"Question: {query}\nProvide a detailed technical answer:"}
-            ],
-            temperature=0.4,
-            max_tokens=2000  # Increased max_tokens
+        response = generation_pipeline(
+            f"{SYSTEM_PROMPT.format(context=context)}\nQuestion: {query}\nProvide a detailed technical answer:",
+            max_length=2000,
+            temperature=0.4
         )
-        logger.info(f"Raw Response: {response.choices[0].message.content}")  # Log raw response
-        return _postprocess_response(response.choices[0].message.content)
+        logger.info(f"Raw Response: {response[0]['generated_text']}")  # Log raw response
+        return _postprocess_response(response[0]['generated_text'])
     except Exception as e:
         logger.error(f"Generation Error: {str(e)}")
         return "I'm unable to generate a response right now. Please try again later."