More improvements

.gitignore

@@ -1 +1,2 @@
-env/
+env/
+*.pyc

app.py

@@ -5,6 +5,7 @@ import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
 
+from datasets import Dataset
 from load_dataframe import get_data
 
 
@@ -48,7 +49,34 @@ def aggregated_data(df, aggregation_level="week"):
     st.pyplot(plt)
 
 
-def display_data(df):
+def show_data_editor(df: pd.DataFrame, key: str):
+    edited_df = st.data_editor(df,
+                hide_index=True,
+                column_order=("reached_out", "reached_out_link", "paper_page", "title", "github", "num_models", "num_datasets", "num_spaces"),
+                column_config={"github": st.column_config.LinkColumn(),
+                                "paper_page": st.column_config.LinkColumn(),
+                                "paper_page_with_title": st.column_config.LinkColumn(display_text=r'\|(.*)')},
+                width=2000,
+                key=key)
+    
+    # Check if the dataframe has been edited
+    # TODO this is wrong
+    # rather we should probably do a merge-join (overwriting the edited rows) and then save the new dataframe
+    # if not edited_df.equals(df):
+    #     save_data(edited_df)
+    #     st.success("Changes saved successfully!")
+
+
+def save_data(df: pd.DataFrame):
+    # load as HF dataset
+    dataset = Dataset.from_pandas(df)
+
+    dataset.push_to_hub("nielsr/daily-papers-enriched")
+
+    return
+
+
+def display_data(df: pd.DataFrame):
     df['has_artifact'] = (df['num_models'] > 0) | (df['num_datasets'] > 0) | (df['num_spaces'] > 0)
     num_artifacts = df['has_artifact'].sum()
     percentage_of_at_least_one_artifact = num_artifacts / df.shape[0] if df.shape[0] > 0 else 0
@@ -67,32 +95,13 @@ def display_data(df):
     """)
 
     st.write("Papers with at least one artifact")
-    st.data_editor(df[df['has_artifact']],
-                hide_index=True,
-                column_order=("reached_out", "reached_out_link", "paper_page", "title", "github", "num_models", "num_datasets", "num_spaces"),
-                column_config={"github": st.column_config.LinkColumn(),
-                                "paper_page": st.column_config.LinkColumn(),
-                                "paper_page_with_title": st.column_config.LinkColumn(display_text=r'\|(.*)')},
-                width=2000,
-                key="papers_with_artifacts")
-    
+    show_data_editor(df[df['has_artifact']], key="papers_with_artifacts")
+
     st.write("Papers without artifacts")
-    st.data_editor(df[~df['has_artifact']],
-                hide_index=True,
-                column_order=("reached_out", "reached_out_link", "paper_page", "title", "github", "num_models", "num_datasets", "num_spaces"),
-                column_config={"github": st.column_config.LinkColumn(),
-                                "paper_page": st.column_config.LinkColumn()},
-                width=2000,
-                key="papers_without_artifacts")
+    show_data_editor(df[~df['has_artifact']], key="papers_without_artifacts")
     
     st.write("Papers with a HF mention in README but no artifacts")
-    st.data_editor(df[(df['hf_mention'] == 1) & (~df['has_artifact'])],
-                hide_index=True,
-                column_order=("reached_out", "reached_out_link", "paper_page", "title", "github", "num_models", "num_datasets", "num_spaces"),
-                column_config={"github": st.column_config.LinkColumn(),
-                                "paper_page": st.column_config.LinkColumn()},
-                width=2000,
-                key="papers_with_hf_mention_no_artifacts")
+    show_data_editor(df[(df['hf_mention'] == 1) & (~df['has_artifact'])], key="papers_with_hf_mention_no_artifacts")
 
 
 def main():
@@ -102,36 +111,29 @@ def main():
     st.sidebar.title("Navigation")
     selection = st.sidebar.selectbox("Go to", ["Daily/weekly/monthly data", "Aggregated data"])
 
-    # TODO use this instead
-    df = get_data()
-
-    print(df.head())
-
-    # df = pd.read_csv('daily_papers_enriched (3).csv')
-    df = df.drop(['Unnamed: 0'], axis=1) if 'Unnamed: 0' in df.columns else df
-    # Use date as index
-    # df = df.set_index('date')
-    # df.index = pd.to_datetime(df.index)
-    df = df.sort_index()
-
     if selection == "Daily/weekly/monthly data":
         # Button to select day, month or week
         # Add streamlit selectbox.
         view_level = st.selectbox(label="View data per day, week or month", options=["day", "week", "month"])
 
         if view_level == "day":
+            # get the latest dataframe
+            df = get_data()
+            
             # make a button to select the day, defaulting to today
             day = st.date_input("Select day", value="today", format="DD/MM/YYYY")
             # convert to the day of a Pandas Timestamp
             day = pd.Timestamp(day)
 
-            df = df[df.index.date == day.date()]
+            filtered_df = df[df.index.date == day.date()]
 
             st.write(f"Showing data for {day.day_name()} {day.strftime('%d/%m/%Y')}")
-
-            display_data(df)
+            display_data(df=filtered_df)
 
         elif view_level == "week":
+            # get the latest dataframe
+            df = get_data()
+            
             # make a button to select the week
             week_number = st.number_input("Select week", value=datetime.today().isocalendar()[1], min_value=1, max_value=52)
             
@@ -139,13 +141,16 @@ def main():
             df['week'] = df.index.isocalendar().week
 
             # Filter the dataframe for the desired week number
-            df = df[df['week'] == week_number]
+            filtered_df = df[df['week'] == week_number]
             
             st.write(f"Showing data for week {week_number}")
             
-            display_data(df)
+            display_data(df=filtered_df)
 
         elif view_level == "month":
+            # get the latest dataframe
+            df = get_data()
+            
             # make a button to select the month, defaulting to current month
             month_str = st.selectbox("Select month", options=["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"])
             year_str = st.selectbox("Select year", options=["2024"])
@@ -160,13 +165,17 @@ def main():
             # Convert month string to number
             month = month_map[month_str]
             year = int(year_str)
-            df = df[(df.index.month == month) & (df.index.year == year)]
+            filtered_df = df[(df.index.month == month) & (df.index.year == year)]
             
             st.write(f"Showing data for {month_str} {year_str}")
             
-            display_data(df)
+            display_data(df=filtered_df)
 
     elif selection == "Aggregated data":
+
+        # get the latest dataframe
+        df = get_data()
+
         aggregated_data(df)
         aggregated_data(df, aggregation_level="month")
 

load_dataframe.py

@@ -20,9 +20,11 @@ class PaperInfo:
     num_comments: int
 
 
-def get_df(start_date: str, end_date: str) -> pd.DataFrame:
+def get_df(start_date: str = None, end_date: str = None) -> pd.DataFrame:
     """
     Load the initial dataset as a Pandas dataframe.
+
+    One can optionally specify a start_date and end_date to only include data between these dates.
     """
 
     df = pd.merge(
@@ -45,8 +47,9 @@ def get_df(start_date: str, end_date: str) -> pd.DataFrame:
     # set date as index
     df = df.set_index('date')
     df.index = pd.to_datetime(df.index)
-    # only include data between start_date and end_date
-    df = df[(df.index >= start_date) & (df.index <= end_date)]
+    if start_date is not None and end_date is not None:
+      # only include data between start_date and end_date
+      df = df[(df.index >= start_date) & (df.index <= end_date)]
 
     return df
 
@@ -150,8 +153,8 @@ def check_hf_mention(batch):
             if response.status_code == 200:
                 # get text
                 text = response.text
-            if "huggingface" in text.lower() or "hugging face" in text.lower():
-                hf_mention = 1
+                if "huggingface" in text.lower() or "hugging face" in text.lower():
+                    hf_mention = 1
             
         hf_mentions.append(hf_mention)
 
@@ -179,18 +182,14 @@ def process_data(start_date: str, end_date: str) -> pd.DataFrame:
     dataset = dataset.map(check_hf_mention, batched=True, batch_size=4, num_proc=cpu_count())
 
     # return as Pandas dataframe
+    # making sure that the date is set as index
     dataframe = dataset.to_pandas()
-
-    # convert date column to datetime
-    dataframe['date'] = pd.to_datetime(dataframe['date'])
-
-    print("First few rows of the dataset:")
-    print(dataframe.head())
+    dataframe = dataframe.set_index('date')
+    dataframe.index = pd.to_datetime(dataframe.index)
 
     return dataframe
 
 
-@st.cache_data
 def get_data() -> pd.DataFrame:
 
     # step 1: load pre-processed data
@@ -200,14 +199,22 @@ def get_data() -> pd.DataFrame:
     df.index = pd.to_datetime(df.index)
 
     # step 2: check how much extra data we need to process
-    latest_day = df.iloc[-1].name.strftime('%d-%m-%Y')
-    today = pd.Timestamp.today().strftime('%d-%m-%Y')
+    latest_day = df.iloc[-1].name.strftime('%Y-%m-%d')
+    today = pd.Timestamp.today().strftime('%Y-%m-%d')
+
+    print("Latest day:", latest_day)
+    print("Today:", today)
 
     # step 3: process the missing data
     if latest_day < today:
         print(f"Processing data from {latest_day} to {today}")
         new_df = process_data(start_date=latest_day, end_date=today)
-        new_df = new_df[new_df.index > latest_day]
+
+        print("Original df:", df.head())
+        print("New df:", new_df.head())
+
         df = pd.concat([df, new_df])
 
+    df = df.sort_index()
+
     return df