Update app.py

app.py

@@ -14,6 +14,8 @@ from transformers import AutoTokenizer
 from transformers import AutoModelForCausalLM
 from transformers import TextIteratorStreamer
 from threading import Thread
+from langchain import Dimension
+
 
 
 #dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
@@ -27,8 +29,8 @@ dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
 #docs = splitter.create_documents(str(dataset))
 # Returns a list of documents
 #print(docs)
-embedding_model = SentenceTransformer("BAAI/bge-large-en-v1.5")
-#all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+#all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5 working with default dimensions
 #docs_text = [doc.text for doc in docs]
 #embed = embedding_model.embed_documents(docs_text)
 
@@ -41,6 +43,8 @@ embedding_model = SentenceTransformer("BAAI/bge-large-en-v1.5")
 data = dataset["train"]
 
 print(data)
+d = 384
+faiss = faiss.IndexFlatL2(d)
 data = data.add_faiss_index("embeddings") 
 # adds an index column that for the embeddings