Update API timeout and filter out invalid data

app.py

@@ -26,31 +26,28 @@ assert user
 
 headers = {"user-agent": user_agent, "authorization": f"Bearer {token}"}
 limits = httpx.Limits(max_keepalive_connections=10, max_connections=20)
-client = Client(headers=headers, http2=True, limits=limits, timeout=60.0)
+client = Client(headers=headers, http2=True, limits=limits, timeout=120.0)
 
 
 @lru_cache(maxsize=None)
-def get_hub_community_activity(user: str, max: int = 200_000) -> List[Any]:
+def get_hub_community_activity(user: str) -> List[Any]:
     with tqdm() as pbar:
         all_data = []
         i = 1
-        while i <= max:
-            try:
-                r = client.get(
-                    f"https://huggingface.co/api/recent-activity?limit=100&activityType=discussion&skip={i}1&entity={user}&feedType=user"
-                )
-                activity = r.json()["recentActivity"]
-                if not activity:
-                    break
-                all_data.append(activity)
-                if len(all_data) % 1000 == 0:
-                    # print(f"Length of all_data: {len(all_data)}")
-                    pbar.write(f"Length of all_data: {len(all_data)}")
-                i += 100
-                pbar.update(100)
-            except Exception as e:
-                print(e)
-                continue
+        while True:
+            r = httpx.get(
+                f"https://huggingface.co/api/recent-activity?limit=100&activityType=discussion&skip={i}&entity={user}&feedType=user",
+                headers=headers,
+            )
+            activity = r.json()["recentActivity"]
+            if not activity:
+                break
+            all_data.append(activity)
+            if len(all_data) % 1000 == 0:
+                # print(f"Length of all_data: {len(all_data)}")
+                pbar.write(f"Length of all_data: {len(all_data)}")
+            i += 100
+            pbar.update(100)
 
     return list(concat(all_data))
 
@@ -97,6 +94,7 @@ def update_data():
     except FileNotFoundError:
         previous_df = pl.DataFrame()
     data = get_hub_community_activity(user)
+    data = [d for d in data if d.get("discussionData", None) is not None]
     data = [parse_pr_data(d) for d in data]
     update_df = pl.DataFrame(data)
     df = pl.concat([previous_df, update_df]).unique()
@@ -115,9 +113,21 @@ def update_data():
 @lru_cache(maxsize=512)
 def get_pr_status(user: str):
     all_data = get_hub_community_activity(user)
+    print(all_data)
+    # pr_data = (
+    #     x["discussionData"] for x in all_data if x["discussionData"]["isPullRequest"]
+    # )
+    all_data = [
+        pr_data
+        for pr_data in all_data
+        if pr_data.get("discussionData", None) is not None
+    ]
     pr_data = (
-        x["discussionData"] for x in all_data if x["discussionData"]["isPullRequest"]
+        x.get("discussionData", {})
+        for x in all_data
+        if x.get("discussionData", {}).get("isPullRequest", False)
     )
+
     return frequencies(x["status"] for x in pr_data)
 
 
@@ -129,6 +139,7 @@ def create_pie():
 
 def group_status_by_pr_number():
     all_data = get_hub_community_activity(user)
+    all_data = [d for d in all_data if d.get("discussionData", None) is not None]
     all_data = [parse_pr_data(d) for d in all_data]
     return (
         pl.DataFrame(all_data).groupby("status").agg(pl.mean("pr_number")).to_pandas()
@@ -137,6 +148,7 @@ def group_status_by_pr_number():
 
 def plot_over_time():
     all_data = get_hub_community_activity(user)
+    all_data = [d for d in all_data if d.get("discussionData", None) is not None]
     all_data = [parse_pr_data(d) for d in all_data]
     df = pl.DataFrame(all_data).with_columns(pl.col("createdAt").cast(pl.Date))
     df = df.pivot(
@@ -146,7 +158,7 @@ def plot_over_time():
         aggregate_function="count",
     )
     df = df.fill_null(0)
-    df = df.with_columns(pl.sum(["open", "merged"])).sort("createdAt")
+    df = df.with_columns(pl.sum(["open", "closed", "merged"])).sort("createdAt")
     df = df.to_pandas().set_index("createdAt").cumsum()
     return px.line(df, x=df.index, y=[c for c in df.columns if c != "sum"])