Update app.py
Browse files
app.py
CHANGED
|
@@ -21,8 +21,8 @@ dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
|
|
| 21 |
#Returns a list of dictionaries, each representing a row in the dataset.
|
| 22 |
length = len(dataset)
|
| 23 |
|
| 24 |
-
embedding_model = SentenceTransformer("
|
| 25 |
-
#all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5 working with default dimensions
|
| 26 |
|
| 27 |
df = pd.DataFrame(dataset)
|
| 28 |
#print(df.iloc[[1]])
|
|
@@ -47,9 +47,9 @@ data = dataset
|
|
| 47 |
d = 384 # vectors dimension
|
| 48 |
m = 32 # hnsw parameter. Higher is more accurate but takes more time to index (default is 32, 128 should be ok)
|
| 49 |
#index = faiss.IndexHNSWFlat(d, m)
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
data.add_faiss_index("embeddings")
|
| 53 |
# adds an index column for the embeddings
|
| 54 |
|
| 55 |
print("check1d")
|
|
|
|
| 21 |
#Returns a list of dictionaries, each representing a row in the dataset.
|
| 22 |
length = len(dataset)
|
| 23 |
|
| 24 |
+
embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
|
| 25 |
+
#all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5, mixedbread-ai/mxbai-embed-large-v1 working with default dimensions
|
| 26 |
|
| 27 |
df = pd.DataFrame(dataset)
|
| 28 |
#print(df.iloc[[1]])
|
|
|
|
| 47 |
d = 384 # vectors dimension
|
| 48 |
m = 32 # hnsw parameter. Higher is more accurate but takes more time to index (default is 32, 128 should be ok)
|
| 49 |
#index = faiss.IndexHNSWFlat(d, m)
|
| 50 |
+
index = faiss.IndexFlatL2(embedding_dim)
|
| 51 |
+
data.add_faiss_index(embeddings.shape[1], custom_index=index)
|
| 52 |
+
#data.add_faiss_index("embeddings")
|
| 53 |
# adds an index column for the embeddings
|
| 54 |
|
| 55 |
print("check1d")
|