remove duplicates

pages/1_Retrieval_App.py

@@ -130,7 +130,12 @@ def app_main(
 ):
     print("loading data")
 
-    retrieval_df = datasets.load_dataset(data_path)["train"].to_pandas()
+    retrieval_df = (
+        datasets.load_dataset(data_path)["train"]
+        .to_pandas()
+        .drop_duplicates(subset=["repo"])
+        .reset_index(drop=True)
+    )
     print("setting up retrieval_pipe")
     doc_col = "dependencies"
     retrieval_pipeline = setup_retrieval_pipeline(