Update app.py

app.py

@@ -16,34 +16,38 @@ from transformers import AutoModelForCausalLM
 from transformers import TextIteratorStreamer
 from threading import Thread
 
+llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+tokenizer = AutoTokenizer.from_pretrained(llm_model)
+# pulling tokeinzer for text generation model
 
 
-#dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
+dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
 #dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
-dataset = load_dataset("epfl-llm/guidelines", split='train')
+#dataset = load_dataset("epfl-llm/guidelines", split='train')
 
 #Returns a list of dictionaries, each representing a row in the dataset.
 print(dataset[1])
-print(dataset)
-# splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=25,separators=["\n"]) # ["\n\n", "\n", " ", ""])
+dataset.features
+splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=25) # ["\n\n", "\n", " ", ""])
 
 
-#docs = splitter.create_documents(str(dataset))
+docs = splitter.create_documents(str(dataset))
 # Returns a list of documents
-#print(docs)
+print(docs)
 embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
 #all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5 working with default dimensions
 #docs_text = [doc.text for doc in docs]
 #embed = embedding_model.embed_documents(docs_text)
+embeddings = embedding_model.encode(docs)
 embedding_dim = embedding_model.get_sentence_embedding_dimension()
 print(embedding_dim)
 #data = FAISS.from_embeddings(embed, embedding_model)
 #data = FAISS.from_texts(docs, embedding_model)
 
 # Returns a FAISS wrapper vector store. Input is a list of strings. from_documents method used documents to Return VectorStore
-
-data = dataset["clean_text"]
-#data = dataset["train"]
+# add_embeddings
+#data = dataset["clean_text"]
+data = dataset["text"]
 
 #print(data)
 d = 384  # vectors dimension
@@ -65,9 +69,6 @@ If you don't know the answer, just say "I do not know." Don't make up an answer.
 print("check2")
 
 
-llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
-tokenizer = AutoTokenizer.from_pretrained(llm_model)
-# pulling tokeinzer for text generation model
 model = AutoModelForCausalLM.from_pretrained(llm_model)
 # Initializing the text generation model