limits

app.py

@@ -1,23 +1,20 @@
+import os
+from datetime import datetime, timedelta
+from functools import lru_cache
 from typing import Any, List
+
 import gradio as gr
-from toolz import concat
+import httpx
+import pandas as pd
 import plotly.express as px
 import polars as pl
-from datasets import load_dataset
 from cachetools import TTLCache, cached
-from datetime import datetime, timedelta
-from datasets import Dataset
-import os
-from functools import lru_cache
-import pandas as pd
-from toolz import frequencies
+from datasets import Dataset, load_dataset
 from dotenv import load_dotenv
-from typing import List, Any
-from toolz import concat
 from httpx import Client
+from toolz import concat, frequencies
 from tqdm.auto import tqdm
 
-
 load_dotenv()
 token = os.environ["HUGGINGFACE_TOKEN"]
 user_agent = os.environ["USER_AGENT"]
@@ -28,27 +25,32 @@ assert user_agent
 assert user
 
 headers = {"user-agent": user_agent, "authorization": f"Bearer {token}"}
-
-client = Client(headers=headers, http2=True)
+limits = httpx.Limits(max_keepalive_connections=10, max_connections=20)
+client = Client(headers=headers, http2=True, limits=limits, timeout=60.0)
 
 
-def get_hub_community_activity(user: str) -> List[Any]:
+@lru_cache(maxsize=None)
+def get_hub_community_activity(user: str, max: int = 35_000) -> List[Any]:
     with tqdm() as pbar:
         all_data = []
         i = 1
-        while True:
-            r = client.get(
-                f"https://huggingface.co/api/recent-activity?limit=100&type=discussion&skip={i}&user={user}",
-            )
-            activity = r.json()["recentActivity"]
-            if not activity:
-                break
-            all_data.append(activity)
-            if len(all_data) % 1000 == 0:
-                # print(f"Length of all_data: {len(all_data)}")
-                pbar.write(f"Length of all_data: {len(all_data)}")
-            i += 100
-            pbar.update(100)
+        while i <= max:
+            try:
+                r = client.get(
+                    f"https://huggingface.co/api/recent-activity?limit=100&type=discussion&skip={i}&user={user}",
+                )
+                activity = r.json()["recentActivity"]
+                if not activity:
+                    break
+                all_data.append(activity)
+                if len(all_data) % 1000 == 0:
+                    # print(f"Length of all_data: {len(all_data)}")
+                    pbar.write(f"Length of all_data: {len(all_data)}")
+                i += 100
+                pbar.update(100)
+            except Exception as e:
+                print(e)
+                continue
 
     return list(concat(all_data))
 

requirements.txt

@@ -25,6 +25,8 @@ attrs==22.2.0
     # via
     #   aiohttp
     #   jsonschema
+backoff==2.2.1
+    # via -r requirements.in
 cachetools==5.3.0
     # via -r requirements.in
 certifi==2022.12.7