title embed code

src/title_embd/embed.py

@@ -0,0 +1,10 @@
+from sentence_transformers import SentenceTransformer
+from src.configs.config import EMBEDDING_MODEL, TITLE_EMBEDDINGS_FILE
+import numpy as np
+
+def generate_title_embeddings(title_data):
+    model = SentenceTransformer(EMBEDDING_MODEL)
+    titles = [item["title"] for item in title_data]
+    embeddings = model.encode(titles, convert_to_tensor=False)
+    np.save(TITLE_EMBEDDINGS_FILE, embeddings)
+    return embeddings

src/title_embd/index.py

@@ -0,0 +1,11 @@
+import faiss
+import numpy as np
+from src.configs.config import TITLE_EMBEDDINGS_FILE, TITLE_FAISS_INDEX_FILE
+
+def create_title_faiss_index():
+    embeddings = np.load(TITLE_EMBEDDINGS_FILE)
+    dimension = embeddings.shape[1]
+    index = faiss.IndexFlatL2(dimension)
+    index.add(embeddings)
+    faiss.write_index(index, str(TITLE_FAISS_INDEX_FILE))
+    return index

src/title_embd/preprocessing.py

@@ -0,0 +1,8 @@
+import pandas as pd
+from src.configs.config import METADATA_FILE
+
+def preprocess_titles():
+    metadata = pd.read_csv(METADATA_FILE)
+    titles = metadata["Nom du document"].tolist()
+    links = metadata["Lien"].tolist()
+    return [{"title": t, "link": l} for t, l in zip(titles, links)]