draft app

app.py

@@ -0,0 +1,205 @@
+from huggingface_hub import hf_hub_url, list_datasets
+from dotenv import load_dotenv
+import os
+from httpx import Client
+from datetime import datetime
+from datetime import timedelta
+from tqdm.auto import tqdm
+from tqdm.contrib.concurrent import thread_map
+import pandas as pd
+import gradio as gr
+
+from huggingface_hub import hf_hub_url
+import requests
+from diskcache import Cache
+from diskcache import Cache
+from sys import platform
+import gradio as gr
+
+# check if running on macos i.e. local dev
+
+
+load_dotenv()
+
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+USER_AGENT = os.getenv("USER_AGENT")
+
+
+headers = {"authorization": f"Bearer ${HF_TOKEN}", "user-agent": USER_AGENT}
+
+
+client = Client(
+    headers=headers,
+    timeout=60,
+)
+LOCAL = False
+if platform == "darwin":
+    LOCAL = True
+cache_dir = "cache" if LOCAL else "/data/diskcache"
+cache = Cache(cache_dir)
+
+
+def add_created_data(dataset):
+    _id = dataset._id
+    created = datetime.fromtimestamp(int(_id[:8], 16))
+    dataset_dict = dataset.__dict__
+    dataset_dict["created"] = created
+    return dataset_dict
+
+
+def get_three_months_ago():
+    now = datetime.now()
+    return now - timedelta(days=90)
+
+
+def get_readme_len(dataset):
+    try:
+        url = hf_hub_url(dataset["id"], "README.md", repo_type="dataset")
+        resp = client.get(url)
+        if resp.status_code == 200:
+            dataset["len"] = len(resp.text)
+            return dataset
+    except Exception as e:
+        print(e)
+        return None
+
+
+def render_model_hub_link(hub_id):
+    link = f"https://huggingface.co/datasets/{hub_id}"
+    return (
+        f'<a target="_blank" href="{link}" style="color: var(--link-text-color);'
+        f' text-decoration: underline;text-decoration-style: dotted;">{hub_id}</a>'
+    )
+
+
+@cache.memoize(expire=60 * 60 * 12)
+def get_datasets():
+    return list(tqdm(iter(list_datasets(limit=None, full=True))))
+
+
+@cache.memoize(expire=60 * 60 * 12)
+def load_data():
+    datasets = get_datasets()
+    datasets = [add_created_data(dataset) for dataset in tqdm(datasets)]
+    filtered = [ds for ds in datasets if ds.get("cardData")]
+    filtered = [ds for ds in filtered if ds["created"] > get_three_months_ago()]
+
+    ds_with_len = thread_map(get_readme_len, filtered)
+    ds_with_len = [ds for ds in ds_with_len if ds is not None]
+    return ds_with_len
+
+
+remove_orgs = {"HuggingFaceM4", "HuggingFaceBR4"}
+
+
+columns_to_drop = [
+    "cardData",
+    "gated",
+    "sha",
+    "paperswithcode_id",
+    "tags",
+    "description",
+    "siblings",
+    "disabled",
+    "_id",
+    "private",
+    "author",
+    "citation",
+]
+
+
+def prep_dataframe(remove_orgs_and_users=remove_orgs, columns_to_drop=columns_to_drop):
+    ds_with_len = load_data()
+    if remove_orgs_and_users:
+        ds_with_len = [
+            ds for ds in ds_with_len if ds["author"] not in remove_orgs_and_users
+        ]
+    df = pd.DataFrame(ds_with_len)
+    df["id"] = df["id"].apply(render_model_hub_link)
+    if columns_to_drop:
+        df = df.drop(columns=columns_to_drop)
+    return df
+
+
+# def filter_df(
+#     df,
+#     created_after=None,
+#     create_before=None,
+#     min_likes=None,
+#     max_likes=None,
+#     min_len=None,
+#     max_len=None,
+#     min_downloads=None,
+#     max_downloads=None,
+# ):
+#     if min_likes:
+#         df = df[df["likes"] >= min_likes]
+#     if max_likes:
+#         df = df[df["likes"] <= max_likes]
+#     if min_len:
+#         df = df[df["len"] >= min_len]
+#     if max_len:
+#         df = df[df["len"] <= max_len]
+#     if min_downloads:
+#         df = df[df["downloads"] >= min_downloads]
+#     if max_downloads:
+#         df = df[df["downloads"] <= max_downloads]
+#     return df
+
+
+import datetime
+
+import datetime
+
+
+def filter_df_by_max_age(max_age_days=None):
+    df = prep_dataframe()
+    df = df.dropna(subset=["created"])
+
+    now = datetime.datetime.now()
+
+    if max_age_days is not None:
+        max_date = now - datetime.timedelta(days=max_age_days)
+        df = df[df["created"] >= max_date]
+
+    return df
+
+
+# def filter_df(
+#     min_age_days=None,
+#     max_age_days=None,
+#     min_likes=None,
+#     max_likes=None,
+#     min_len=None,
+#     max_len=None,
+#     min_downloads=None,
+#     max_downloads=None,
+# ):
+#     if min_age_days is not None or max_age_days is not None:
+#         df = filter_df_by_date(min_age_days, max_age_days)
+#     else:
+#         df = prep_dataframe()
+#     if min_likes:
+#         df = df[df["likes"] >= min_likes]
+#     if max_likes:
+#         df = df[df["likes"] <= max_likes]
+#     if min_len:
+#         df = df[df["len"] >= min_len]
+#     if max_len:
+#         df = df[df["len"] <= max_len]
+#     if min_downloads:
+#         df = df[df["downloads"] >= min_downloads]
+#     if max_downloads:
+#         df = df[df["downloads"] <= max_downloads]
+#     return df
+
+
+with gr.Blocks() as demo:
+    max_age_days = gr.Slider(
+        label="Max Age (days)", value=7, minimum=0, maximum=90, step=1, interactive=True
+    )
+    output = gr.DataFrame(prep_dataframe(), datatype="markdown", min_width=160 * 2.5)
+    max_age_days.input(filter_df_by_max_age, inputs=[max_age_days], outputs=[output])
+
+demo.launch()