Upload 3 files

data.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32f51e54683c1fa3390bc4e318e1008d686844bb451b82c3c1a91787e2b986d9
+size 3765676

main.py

@@ -0,0 +1,150 @@
+import contextlib
+import gradio as gr
+import polars as pl
+from functools import lru_cache
+from cytoolz import concat, frequencies, topk
+from datasets import load_dataset
+from ast import literal_eval
+from typing import Union, List, Optional
+import numpy as np
+from itertools import combinations
+from toolz import unique
+import pandas as pd
+
+pd.options.plotting.backend = "plotly"
+
+
+def download_dataset():
+    return load_dataset("open-source-metrics/model-repos-stats", split="train")
+
+
+def _clean_tags(tags: Optional[Union[str, List[str]]]):
+    try:
+        tags = literal_eval(tags)
+        if isinstance(tags, str):
+            return [tags]
+        if isinstance(tags, list):
+            return [tag for tag in tags if isinstance(tag, str)]
+        else:
+            return []
+    except (ValueError, SyntaxError):
+        return []
+
+
+def prep_dataset():
+    ds = download_dataset()
+    df = ds.to_pandas()
+    df['languages'] = df['languages'].apply(_clean_tags)
+    df['datasets'] = df['datasets'].apply(_clean_tags)
+    df['tags'] = df['tags'].apply(_clean_tags)
+    df = df.drop(columns=['Unnamed: 0'])
+    df.to_parquet("data.parquet")
+    return df
+
+
+def load_data():
+    return pd.read_parquet("data.parquet")
+
+
+def filter_df_by_library(filter='transformers'):
+    df = load_data()
+    return df[df['library'] == filter] if filter else df
+
+
+@lru_cache()
+def get_library_choices(min_freq: int = 50):
+    df = load_data()
+    library_counts = df.library.value_counts()
+    return library_counts[library_counts > min_freq].index.to_list()
+
+
+@lru_cache()
+def get_all_tags():
+    df = load_data()
+    tags = df['tags'].to_list()
+    return list(concat(tags))
+
+@lru_cache()
+def get_case_sensitive_duplicate_tags():
+    tags = get_all_tags()
+    unique_tags = unique(tags)
+    return [
+        tag_combo
+        for tag_combo in combinations(unique_tags, 2)
+        if tag_combo[0].lower() == tag_combo[1].lower()
+    ]
+
+
+def display_case_sensitive_duplicate_tags():
+    return pd.DataFrame(get_case_sensitive_duplicate_tags())
+
+def tag_frequency(case_sensitive=True):
+    tags = get_all_tags()
+    if not case_sensitive:
+        tags = (tag.lower() for tag in tags)
+    tags_frequencies = dict(frequencies(tags))
+    df = pd.DataFrame.from_dict(tags_frequencies, orient='index', columns=['Count']).sort_values(
+        by='Count', ascending=False)
+    return df
+
+def plot_frequency(filter):
+    df = filter_df_by_library(filter)
+    tags = concat(df['tags'])
+    tags = dict(frequencies(tags))
+    df = pd.DataFrame.from_dict(tags, orient='index', columns=['Count']).sort_values(
+        by='Count', ascending=False)
+    return df
+
+
+def has_model_card_by_library(top_n):
+    df = load_data()
+    if top_n:
+        top_libs = df.library.value_counts().head(int(top_n)).index.to_list()
+        # min_thresh = df.library.value_counts()[:min_number].index.to_list()
+        df = df[df.library.isin(top_libs)]
+    return df.groupby('library')['has_text'].apply(lambda x: np.sum(x) / len(x)).sort_values().plot.barh()
+
+
+def model_card_length_by_library(top_n):
+    df = load_data()
+    if top_n:
+        top_libs = df.library.value_counts().head(int(top_n)).index.to_list()
+        # min_thresh = df.library.value_counts()[:min_number].index.to_list()
+        df = df[df.library.isin(top_libs)]
+    return df.groupby('library')['text_length'].describe().round().reset_index()
+    # df = df.groupby('library')['text_length'].describe().round().reset_index()
+    # df['library'] = df.library.apply(lambda library: f"[{library}](https://huggingface.co/models?library={library})")
+    # return df.to_markdown()
+
+
+df = load_data()
+top_n = df.library.value_counts().shape[0]
+
+with gr.Blocks() as demo:
+    gr.Markdown("# 🤗 Hub Metadata Explorer")
+    gr.Markdown("Some explanation")
+    with gr.Tab("Tags overview"):
+        gr.Markdown("Tags are one of the key...")
+        with gr.Row():
+            gr.Markdown("thsh")
+        with gr.Row():
+            case_sensitive = gr.Checkbox(False,label=)
+            gr.Plot(tag_frequency())
+        with gr.Row():
+            gr.Markdown(f"Number of tags which are case sensitive {len(get_case_sensitive_duplicate_tags())}")
+            with gr.Accordion("View duplicate tags", open=False):
+                gr.Dataframe(display_case_sensitive_duplicate_tags())
+    with gr.Tab("Model Cards"):
+        gr.Markdown("""Model cards are a key component of metadata for a model. Model cards can include both 
+        information created by a human i.e. outlining the goals behind the creation of the model and information 
+        created by a training framework. This automatically generated information can contain information about 
+        number of epochs, learning rate, weight decay etc. """)
+        min_lib_frequency = gr.Slider(minimum=1, maximum=top_n, value=10, label='filter by top n libraries')
+        with gr.Column():
+            plot = gr.Plot()
+            min_lib_frequency.change(has_model_card_by_library, [min_lib_frequency], plot, queue=False)
+        with gr.Column():
+            df = gr.Dataframe()
+            min_lib_frequency.change(model_card_length_by_library, [min_lib_frequency], df, queue=False)
+
+demo.launch(debug=True)

requirements.txt

@@ -0,0 +1,7 @@
+gradio==3.15.0
+pandas
+polars
+datasets
+cytoolz
+plotly
+tabulate