Update app.py

app.py

@@ -32,20 +32,23 @@ dataset.features
 #print(Itemdetails)
 splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=25) # ["\n\n", "\n", " ", ""])
 
+#docs = splitter.create_documents(str(dataset))
 
-docs = splitter.create_documents(str(dataset))
 # Returns a list of documents
 print(docs)
-#embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
-embedding_model = HuggingFaceEmbeddings(model_name = "mixedbread-ai/mxbai-embed-large-v1")
+embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
+#embedding_model = HuggingFaceEmbeddings(model_name = "mixedbread-ai/mxbai-embed-large-v1")
 #all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5 working with default dimensions
 #docs_text = [doc.text for doc in docs]
 #embed = embedding_model.embed_documents(docs_text)
 #embeddings = embedding_model.encode(docs)
+embeddings = torch.from_numpy(dataset["train"].to_pandas().to_numpy()).to(torch.float)
 
-doc_func = lambda x: x.text
-dataset = list(map(doc_func, dataset))
-embeddings = embedding_model.embed_documents(dataset)
+
+
+#doc_func = lambda x: x.text
+#dataset = list(map(doc_func, dataset))
+#embeddings = embedding_model.embed_documents(dataset)
 #embeddings = embedding_model.embed_documents(docs)
 dataset = dataset.add_column('embeddings', embeddings)
 embedding_dim = embedding_model.get_sentence_embedding_dimension()