Update app.py

app.py

@@ -54,7 +54,35 @@ model_pipeline = pipeline(
 )
 
 # Use the pipeline in HuggingFacePipeline
-llm = HuggingFacePipeline(pipeline=model_pipeline)
+#llm = HuggingFacePipeline(pipeline=model_pipeline)
+
+##### Alternative
+from transformers import pipeline
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
+READER_MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+rmodel = AutoModelForCausalLM.from_pretrained(READER_MODEL_NAME, quantization_config=bnb_config)
+tokenizer = AutoTokenizer.from_pretrained(READER_MODEL_NAME)
+
+llm = pipeline(
+    model=rmodel,
+    tokenizer=tokenizer,
+    task="text-generation",
+    do_sample=True,
+    temperature=0.2,
+    repetition_penalty=1.1,
+    return_full_text=False,
+    max_new_tokens=500,
+)
+
 
 # NLTK Resource Download
 def download_nltk_resources():
@@ -93,7 +121,7 @@ class ModelManager:
         }
 
         
-    def update_model_ranking(self, model_id: str, score: float, feedback: Optional[str] = None):
+    def update_model_ranking(self, model_id: str, score: float, feedback: str = None):
         """Update model ranking based on performance and optional feedback"""
         current_score = self.rankings.get(model_id, 0.0)
         # Weighted average of current score and new score
@@ -361,6 +389,8 @@ def get_text_splitter(split_strategy, chunk_size, overlap_size, custom_separator
         return RecursiveCharacterTextSplitter(
             chunk_size=chunk_size,
             chunk_overlap=overlap_size,
+            add_start_index=True,  # If `True`, includes chunk's start index in metadata
+            strip_whitespace=True,  # If `True`, strips whitespace from the start and end of every document 
             separators=custom_separators or ["\n\n", "\n", " ", ""]
         )
     else:
@@ -369,7 +399,12 @@ def get_text_splitter(split_strategy, chunk_size, overlap_size, custom_separator
 def get_embedding_model(model_type, model_name):
     model_path = model_manager.get_model(model_type, model_name)
     if model_type == 'HuggingFace':
-        return HuggingFaceEmbeddings(model_name=model_path)
+        return = HuggingFaceEmbeddings(
+            model_name=model_path,
+            multi_process=True,
+            model_kwargs={"device": "cuda"},
+            #encode_kwargs={"normalize_embeddings": True},  # Set `True` for cosine similarity
+        ) 
     elif model_type == 'OpenAI':
         return OpenAIEmbeddings(model=model_path)
     elif model_type == 'Cohere':
@@ -605,6 +640,15 @@ def visualize_results(results_df, stats_df):
     
     plt.tight_layout()
     return fig
+
+
+#tokenizer = AutoTokenizer.from_pretrained(EMBEDDING_MODEL_NAME)
+#lengths = [len(tokenizer.encode(doc.page_content)) for doc in tqdm(docs_processed)]
+#fig = pd.Series(lengths).hist()
+#plt.title("Distribution of document lengths in the knowledge base (in count of tokens)")
+#plt.show()
+
+    
 def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
     tokenizer = Tokenizer(models.BPE(unk_token="[UNK]"))