Spaces:
Sleeping
Sleeping
Update doc_preprocessing.py
Browse files- doc_preprocessing.py +3 -1
doc_preprocessing.py
CHANGED
|
@@ -6,6 +6,8 @@ import streamlit as st
|
|
| 6 |
import numpy as np
|
| 7 |
import os
|
| 8 |
|
|
|
|
|
|
|
| 9 |
def extract_text(file):
|
| 10 |
text = ""
|
| 11 |
# Check if the input is a file path (string) or a file-like object
|
|
@@ -64,7 +66,7 @@ def get_embeddings(texts)-> np.ndarray:
|
|
| 64 |
# embeddings = embedding_model(texts)
|
| 65 |
|
| 66 |
|
| 67 |
-
model = SentenceTransformer(
|
| 68 |
embeddings = model.encode(texts)
|
| 69 |
|
| 70 |
print(f"Generated {len(embeddings)} embeddings.")
|
|
|
|
| 6 |
import numpy as np
|
| 7 |
import os
|
| 8 |
|
| 9 |
+
emb_model = "google/Gemma-Embeddings-v1.0"
|
| 10 |
+
|
| 11 |
def extract_text(file):
|
| 12 |
text = ""
|
| 13 |
# Check if the input is a file path (string) or a file-like object
|
|
|
|
| 66 |
# embeddings = embedding_model(texts)
|
| 67 |
|
| 68 |
|
| 69 |
+
model = SentenceTransformer(emb_model, trust_remote_code=True)
|
| 70 |
embeddings = model.encode(texts)
|
| 71 |
|
| 72 |
print(f"Generated {len(embeddings)} embeddings.")
|