update

app.py

@@ -13,23 +13,57 @@ import os
 from functools import lru_cache
 import pandas as pd
 from toolz import frequencies
+from dotenv import load_dotenv
+from typing import List, Any
+from toolz import concat
+import httpx
+from tqdm.auto import tqdm
 
+
+load_dotenv()
 token = os.environ["HUGGINGFACE_TOKEN"]
+user_agent = os.environ["USER_AGENT"]
+user = os.environ["USER_TO_TRACK"]
 assert token
-librarian_bot_avatar = "https://aeiljuispo.cloudimg.io/v7/https://s3.amazonaws.com/moonup/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg?w=200&h=200&f=face"
+assert user_agent
+assert user
+
+headers = {"user-agent": user_agent, "authorization": f"Bearer {token}"}
 
 
 def get_hub_community_activity(user: str) -> List[Any]:
-    all_data = []
-    for i in range(1, 2000, 100):
-        r = httpx.get(
-            f"https://huggingface.co/api/recent-activity?limit=100&type=discussion&skip={i}&user={user}"
-        )
-        activity = r.json()["recentActivity"]
-        all_data.append(activity)
+    with tqdm() as pbar:
+        all_data = []
+        i = 1
+        while True:
+            r = httpx.get(
+                f"https://huggingface.co/api/recent-activity?limit=100&type=discussion&skip={i}&user={user}",
+                headers=headers,
+            )
+            activity = r.json()["recentActivity"]
+            if not activity:
+                break
+            all_data.append(activity)
+            if len(all_data) % 1000 == 0:
+                # print(f"Length of all_data: {len(all_data)}")
+                pbar.write(f"Length of all_data: {len(all_data)}")
+            i += 100
+            pbar.update(100)
+
     return list(concat(all_data))
 
 
+# def get_hub_community_activity(user: str) -> List[Any]:
+#     all_data = []
+#     for i in range(1, 2000, 100):
+#         r = httpx.get(
+#             f"https://huggingface.co/api/recent-activity?limit=100&type=discussion&skip={i}&user={user}"
+#         )
+#         activity = r.json()["recentActivity"]
+#         all_data.append(activity)
+#     return list(concat(all_data))
+
+
 def parse_date_time(date_time: str) -> datetime:
     return datetime.strptime(date_time, "%Y-%m-%dT%H:%M:%S.%fZ")
 
@@ -54,15 +88,18 @@ def parse_pr_data(data):
 
 @cached(cache=TTLCache(maxsize=1000, ttl=timedelta(minutes=30), timer=datetime.now))
 def update_data():
-    previous_df = pl.DataFrame(
-        load_dataset("librarian-bot/stats", split="train").data.table
-    )
-    data = get_hub_community_activity("librarian-bot")
+    try:
+        previous_df = pl.DataFrame(
+            load_dataset(f"librarian-bot/{user}-stats", split="train").data.table
+        )
+    except FileNotFoundError:
+        previous_df = pl.DataFrame()
+    data = get_hub_community_activity(user)
     data = [parse_pr_data(d) for d in data]
     update_df = pl.DataFrame(data)
     df = pl.concat([previous_df, update_df]).unique()
     if len(df) != len(previous_df):
-        Dataset(df.to_arrow()).push_to_hub("librarian-bot/stats", token=token)
+        Dataset(df.to_arrow()).push_to_hub(f"{user}-stats", token=token)
     return df
 
 
@@ -83,20 +120,13 @@ def get_pr_status(user: str):
 
 
 def create_pie():
-    frequencies = get_pr_status("librarian-bot")
+    frequencies = get_pr_status(user)
     df = pd.DataFrame({"status": frequencies.keys(), "number": frequencies.values()})
     return px.pie(df, values="number", names="status", template="seaborn")
 
 
-# def create_pie():
-#     df = update_data()
-#     df = df.filter(pl.col("isPullRequest") is True)
-#     df = df["status"].value_counts().to_pandas()
-#     return px.pie(df, values="counts", names="status", template="seaborn")
-
-
 def group_status_by_pr_number():
-    all_data = get_hub_community_activity("librarian-bot")
+    all_data = get_hub_community_activity(user)
     all_data = [parse_pr_data(d) for d in all_data]
     return (
         pl.DataFrame(all_data).groupby("status").agg(pl.mean("pr_number")).to_pandas()
@@ -104,7 +134,7 @@ def group_status_by_pr_number():
 
 
 def plot_over_time():
-    all_data = get_hub_community_activity("librarian-bot")
+    all_data = get_hub_community_activity(user)
     all_data = [parse_pr_data(d) for d in all_data]
     df = pl.DataFrame(all_data).with_columns(pl.col("createdAt").cast(pl.Date))
     df = df.pivot(
@@ -123,11 +153,11 @@ create_pie()
 
 with gr.Blocks() as demo:
     # frequencies = get_pr_status("librarian-bot")
-    gr.HTML(Path("description.html").read_text())
-    gr.Markdown(f"Total prs and issues opened by librarian-bot: {len(update_data())}")
+    gr.Markdown(f"# {user} PR Stats")
+    gr.Markdown(f"Total prs and issues opened by {user}: {len(update_data()):,}")
     # gr.Markdown(f"Total PRs opened: {sum(frequencies.values())}")
     with gr.Column():
-        gr.Markdown("## Pull requests Status")
+        gr.Markdown("## Pull requests status")
         gr.Markdown(
             "The below pie chart shows the percentage of pull requests made by"
             " librarian bot that are open, closed or merged"