arrow -> csv ita data

.gitattributes

@@ -1,3 +1,8 @@
 data/frontend_data/ filter=lfs diff=lfs merge=lfs -text
 data/frontend_data/* filter=lfs diff=lfs merge=lfs -text
 data/frontend_data/**/* filter=lfs diff=lfs merge=lfs -text
+data/frontend_data/us_professor.json filter=lfs diff=lfs merge=lfs -text
+data/frontend_data/all-mpnet-base-v2-embds/weights.pt filter=lfs diff=lfs merge=lfs -text
+data/frontend_data/all-mpnet-base-v2-embds/id_title_author/data-00000-of-00001.arrow filter=lfs diff=lfs merge=lfs -text
+data/frontend_data/all-mpnet-base-v2-embds/id_title_author/dataset_info.json filter=lfs diff=lfs merge=lfs -text
+data/frontend_data/all-mpnet-base-v2-embds/id_title_author/state.json filter=lfs diff=lfs merge=lfs -text

core/recommender.py

@@ -1,7 +1,7 @@
 from collections import Counter, defaultdict
 import json
 
-from datasets import Dataset
+import pandas as pd
 import torch
 import torch.nn.functional as F
 from transformers import AutoTokenizer, AutoModel
@@ -72,7 +72,7 @@ class Recommender:
                  frontend_us_professor_path: str = DataPaths.FRONTEND_PROF_PATH,
     ):
         self.embedding_processor = embedding_processor
-        self.ita = Dataset.load_from_disk(ita_path)
+        self.ita = pd.read_csv(ita_path)
         self.embds = torch.load(weights_path, weights_only=True)
         # dictionary with professor names as keys and their metadata as values
         with open(frontend_us_professor_path, 'r') as f:
@@ -87,8 +87,9 @@ class Recommender:
 
     def get_recommended_data(self, top_indices: torch.Tensor):
         """Returns a list of dictionaries with professors corresponding to their information."""
-        selected = self.ita.select(top_indices)
-        professors = selected["authors"]
+        selected = self.ita.iloc[top_indices]
+
+        professors = [x.split("|-|") for x in selected["authors"]]
         professors = [prof for profs in professors for prof in profs]
 
         # rank professors first by number of times appeared in the list
@@ -98,8 +99,12 @@ class Recommender:
 
         # professor to IDs
         professor2ids = defaultdict(list)
-        for pid_, pt, pauthors in zip(selected['id'], selected['title'], selected['authors']):
-            for prof in pauthors:
+        for pid_, pt, pauthors in zip(
+            selected['id'].tolist(),
+            selected['title'].tolist(),
+            selected['authors'].tolist()
+        ):
+            for prof in pauthors.split("|-|"):
                 professor2ids[prof].append((pid_, pt))
 
         # Build professor metadata

data/frontend_data/all-mpnet-base-v2-embds/id_title_author/state.json

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7961828744086f5b98697879c399db414cbf8e921ae858c62d61ab8f79ecba59
-size 250

data/frontend_data/all-mpnet-base-v2-embds/{id_title_author/dataset_info.json → ita.csv}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a13329a3e3ffb5721d11f0e2af55847ed65dd9cbd5fb0f2d85984312c0b8217a
-size 810
+oid sha256:19672f2088eef5dc28f75010f266bdbccfa5c85b02f1ec0728fad5bae1d8b44d
+size 1424458

data_pipeline/config.py

@@ -28,7 +28,7 @@ class DataPaths:
     FRONTEND_DIR = os.path.join(BASE_DIR, 'frontend_data')
     FRONTEND_PROF_PATH = os.path.join(FRONTEND_DIR, 'us_professor.json')
     FRONTEND_EMBD_PATH = os.path.join(FRONTEND_DIR, EMBD_MODEL)  # contains id, title, author, weights
-    FRONTEND_ITA_PATH = os.path.join(FRONTEND_EMBD_PATH, 'id_title_author')
+    FRONTEND_ITA_PATH = os.path.join(FRONTEND_EMBD_PATH, 'ita.csv')
     FRONTEND_WEIGHTS_PATH = os.path.join(FRONTEND_EMBD_PATH, 'weights.pt')
 
     # create BASE_DIR LOG_DIR FRONTEND_DIR PROF_DIR CONFERENCE_DIR PAPER_DIR

data_pipeline/paper_embeddings_extractor.py

@@ -214,37 +214,45 @@ def main():
     """Downloads arxiv data and extract embeddings for papers."""
 
     ### Download and filter for ML papers written by U.S. professors ###
-    print("Downloading data...")
-    download_arxiv_data()
-    with open(DataPaths.US_PROF_PATH, 'r') as f:
-        authors_of_interest = json.load(f)
-    authors_of_interest = [author['name'] for author in authors_of_interest]
-    print("Filtering data for ML papers...")
-    filter_arxiv_for_ml(authors_of_interest=authors_of_interest)
-
-    ### Create a dataset containing paper info, e.g., title, abstract, authors, etc. ###
-    print("Saving paper data to disk at " + DataPaths.PAPER_DATA_PATH)
-    p2p = get_professors_and_relevant_papers(authors_of_interest)
-    ds = Dataset.from_generator(partial(gen, p2p))
-    ds.save_to_disk(DataPaths.PAPER_DATA_PATH)
-
-    ### Extract paper embeddings ###
-    print("Extracting embeddings (use GPU if possible)...")
-    # Initialize the embedding processor with model names
-    embedding_processor = EmbeddingProcessor(
-        model_name='sentence-transformers/all-mpnet-base-v2',
-        custom_model_name='salsabiilashifa11/sbert-paper'
-    )
-    # Process dataset and save with embeddings
-    embedding_processor.process_dataset(DataPaths.PAPER_DATA_PATH, DataPaths.EMBD_PATH, batch_size=128)
+    # print("Downloading data...")
+    # download_arxiv_data()
+    # with open(DataPaths.US_PROF_PATH, 'r') as f:
+    #     authors_of_interest = json.load(f)
+    # authors_of_interest = [author['name'] for author in authors_of_interest]
+    # print("Filtering data for ML papers...")
+    # filter_arxiv_for_ml(authors_of_interest=authors_of_interest)
+
+    # ### Create a dataset containing paper info, e.g., title, abstract, authors, etc. ###
+    # print("Saving paper data to disk at " + DataPaths.PAPER_DATA_PATH)
+    # p2p = get_professors_and_relevant_papers(authors_of_interest)
+    # ds = Dataset.from_generator(partial(gen, p2p))
+    # ds.save_to_disk(DataPaths.PAPER_DATA_PATH)
+
+    # ### Extract paper embeddings ###
+    # print("Extracting embeddings (use GPU if possible)...")
+    # # Initialize the embedding processor with model names
+    # embedding_processor = EmbeddingProcessor(
+    #     model_name='sentence-transformers/all-mpnet-base-v2',
+    #     custom_model_name='salsabiilashifa11/sbert-paper'
+    # )
+    # # Process dataset and save with embeddings
+    # embedding_processor.process_dataset(DataPaths.PAPER_DATA_PATH, DataPaths.EMBD_PATH, batch_size=128)
 
     ### Create front-end data ###
 
     # Filter ds for paper title, id, authors, and embedding
     embds = Dataset.load_from_disk(DataPaths.EMBD_PATH)
 
+    def join_authors(x):
+        x['authors'] = "|-|".join(x['authors'])
+        return x
+
+    import pdb ; pdb.set_trace()
+
+    embds = embds.map(join_authors)
+
     # save id and title to disk
-    embds.select_columns(['id', 'title', 'authors']).save_to_disk(DataPaths.FRONTEND_ITA_PATH)
+    embds.select_columns(['id', 'title', 'authors']).to_csv(DataPaths.FRONTEND_ITA_PATH)
     # save embeddings as torch tensor
     embds_weights = torch.Tensor(embds['embeddings'])
     torch.save(embds_weights, DataPaths.FRONTEND_WEIGHTS_PATH)

requirements.txt

@@ -1,4 +1,4 @@
-datasets
+pandas
 streamlit
 torch
 transformers