Allow comparing versions

changelog.md

@@ -0,0 +1,7 @@
+Changelog
+
+v0.1
+- Allow pick comparison version
+- Show delta in all metrics
+- Allow picking not transformers in pipeline and library page
+- Show old and new metrics in license, language, and libraries raw tables, with delta columns

models.py

@@ -34,7 +34,7 @@ def main():
             return "unk_modality"
         return None
 
-    supported_revisions = ["27_09_22"]
+    supported_revisions = ["03_10_22", "27_09_22"]
 
     st.cache(allow_output_mutation=True)
     def process_dataset(version):
@@ -52,10 +52,20 @@ def main():
 
         return data
 
-    base = st.selectbox(
-        'What revision do you want to use',
-        supported_revisions)
-    data = process_dataset(base)
+    col1, col2 = st.columns(2)
+    with col1:
+        base = st.selectbox(
+            'Old revision',
+            supported_revisions,
+            index=1)
+    with col2:
+        new = st.selectbox(
+            'Last revision',
+            supported_revisions,
+            index=0)
+    
+    old_data = process_dataset(base)
+    data = process_dataset(new)
 
     def eval_tags(row):
         tags = row["tags"]
@@ -68,10 +78,12 @@ def main():
             return []
         return val
 
+    old_data["tags"] = old_data.apply(eval_tags, axis=1)
     data["tags"] = data.apply(eval_tags, axis=1)
 
+    total_samples_old = old_data.shape[0]
     total_samples = data.shape[0]
-    st.metric(label="Total models", value=total_samples)
+    st.metric(label="Total models", value=total_samples, delta=total_samples-total_samples_old)
 
     # Tabs don't work in Spaces st version
     #tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8 = st.tabs(["Language", "License", "Pipeline", "Discussion Features", "Libraries", "Model Cards", "Super users", "Raw Data"])
@@ -86,9 +98,13 @@ def main():
 
         data.loc[data.languages == "False", 'languages'] = None
         data.loc[data.languages == {}, 'languages'] = None
+        old_data.loc[old_data.languages == "False", 'languages'] = None
+        old_data.loc[old_data.languages == {}, 'languages'] = None
 
         no_lang_count = data["languages"].isna().sum()
+        no_lang_count_old = old_data["languages"].isna().sum()
         data["languages"] = data["languages"].fillna('none')
+        old_data["languages"] = old_data["languages"].fillna('none')
 
         def make_list(row):
             languages = row["languages"]
@@ -103,19 +119,28 @@ def main():
 
         data["languages"] = data.apply(make_list, axis=1)
         data["language_count"] = data.apply(language_count, axis=1)
+        old_data["languages"] = old_data.apply(make_list, axis=1)
+        old_data["language_count"] = old_data.apply(language_count, axis=1)
 
         models_with_langs = data[data["language_count"] > 0]
         langs = models_with_langs["languages"].explode()
         langs = langs[langs != {}]
         total_langs = len(langs.unique())
 
+        models_with_langs_old = old_data[old_data["language_count"] > 0]
+        langs_old = models_with_langs_old["languages"].explode()
+        langs_old = langs_old[langs_old != {}]
+        total_langs_old = len(langs_old.unique())
+
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="Language Specified", value=total_samples-no_lang_count)
+            v = total_samples-no_lang_count
+            v_old = total_samples_old-no_lang_count_old
+            st.metric(label="Language Specified", value=v, delta=int(v-v_old))
         with col2:
-            st.metric(label="No Language Specified", value=no_lang_count)
+            st.metric(label="No Language Specified", value=no_lang_count, delta=int(no_lang_count-no_lang_count_old))
         with col3:
-            st.metric(label="Total Unique Languages", value=total_langs)
+            st.metric(label="Total Unique Languages", value=total_langs, delta=int(total_langs-total_langs_old))
 
         st.subheader("Count of languages per model repo")
         st.text("Some repos are for multiple languages, so the count is greater than 1")
@@ -132,6 +157,8 @@ def main():
 
         models_with_langs = data[data["language_count"] > filter]
         df1 = models_with_langs['language_count'].value_counts()
+        models_with_langs_old = old_data[old_data["language_count"] > filter]
+        df1_old = models_with_langs_old['language_count'].value_counts()
         st.bar_chart(df1)
 
         st.subheader("Most frequent languages")
@@ -150,9 +177,14 @@ def main():
         models_with_langs = data[data["language_count"] > 0]
         langs = models_with_langs["languages"].explode()
         langs = langs[langs != {}]
-
         orig_d = langs.value_counts().rename_axis("language").to_frame('counts').reset_index()
         d = orig_d
+        
+        models_with_langs_old = old_data[old_data["language_count"] > 0]
+        langs = models_with_langs_old["languages"].explode()
+        langs = langs[langs != {}]
+        orig_d_old = langs.value_counts().rename_axis("language").to_frame('counts').reset_index()
+
         if filter == 1:
             d = orig_d.iloc[1:]
         elif filter == 2:
@@ -167,10 +199,22 @@ def main():
         ))
 
         st.subheader("Raw Data")
-        l = df1.rename_axis("lang_count").reset_index().rename(columns={"language_count": "repos_count"})
-        st.dataframe(l)
+        l = df1.rename_axis("lang_count").reset_index().rename(columns={"language_count": "r_c"})
+        l_old = df1_old.rename_axis("lang_count").reset_index().rename(columns={"language_count": "old_r_c"})
+        final_data =  pd.merge(
+            l, l_old, how="outer", on="lang_count"
+        )
+        final_data["diff"] = final_data["r_c"] - final_data["old_r_c"]
+        st.dataframe(final_data)
+        
         d = orig_d.astype(str)
-        st.dataframe(d)
+        orig_d_old = orig_d_old.astype(str).rename(columns={"counts": "old_c"})
+        final_data =  pd.merge(
+            d, orig_d_old, how="outer", on="language"
+        )
+        final_data["diff"] = final_data["counts"].astype(int) - final_data["old_c"].astype(int)
+
+        st.dataframe(final_data)
         
         
 
@@ -179,13 +223,18 @@ def main():
         st.header("License info")
 
         no_license_count = data["license"].isna().sum()
+        no_license_count_old = old_data["license"].isna().sum()
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="License Specified", value=total_samples-no_license_count)
+            v = total_samples-no_license_count
+            v_old = total_samples_old-no_license_count_old
+            st.metric(label="License Specified", value=v, delta=int(v-v_old))
         with col2:
-            st.metric(label="No license Specified", value=no_license_count)
+            st.metric(label="No license Specified", value=no_license_count, delta=int(no_license_count-no_license_count_old))
         with col3:
-            st.metric(label="Total Unique Licenses", value=len(data["license"].unique()))
+            unique_licenses = len(data["license"].unique())
+            unique_licenses_old = len(old_data["license"].unique())
+            st.metric(label="Total Unique Licenses", value=unique_licenses, delta=int(unique_licenses-unique_licenses_old))
 
         st.subheader("Distribution of licenses per model repo")
         license_filter = st.selectbox(
@@ -217,7 +266,12 @@ def main():
 
         st.subheader("Raw Data")
         d = data["license"].value_counts().rename_axis("license").to_frame('counts').reset_index()
-        st.dataframe(d)
+        d_old = old_data["license"].value_counts().rename_axis("license").to_frame('counts').reset_index().rename(columns={"counts": "old_c"})
+        final_data =  pd.merge(
+            d, d_old, how="outer", on="license"
+        )
+        final_data["diff"] = final_data["counts"] - final_data["old_c"]
+        st.dataframe(final_data)
         
     #with tab3:
     if tab == "Pipeline":
@@ -229,14 +283,24 @@ def main():
         s = s[s.apply(type) == str]
         unique_tags = len(s.unique())
 
+        tags_old = old_data["tags"].explode()
+        tags_old = tags_old[tags_old.notna()].value_counts().rename_axis("tag").to_frame('counts').reset_index()
+        s = tags_old["tag"]
+        s = s[s.apply(type) == str]
+        unique_tags_old = len(s.unique())
+
         no_pipeline_count = data["pipeline"].isna().sum()
+        no_pipeline_count_old = old_data["pipeline"].isna().sum()
+
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="# models that have any pipeline", value=total_samples-no_pipeline_count)
+            v = total_samples-no_pipeline_count
+            v_old = total_samples_old-no_pipeline_count_old
+            st.metric(label="# models that have any pipeline", value=v, delta=int(v-v_old))
         with col2:
-            st.metric(label="No pipeline Specified", value=no_pipeline_count)
+            st.metric(label="No pipeline Specified", value=no_pipeline_count, delta=int(no_pipeline_count-no_pipeline_count_old))
         with col3:
-            st.metric(label="Total Unique Pipelines", value=len(data["pipeline"].unique()))
+            st.metric(label="Total Unique Tags", value=unique_tags, delta=int(unique_tags-unique_tags_old))
 
         pipeline_filter = st.selectbox(
             'Modalities',
@@ -260,19 +324,26 @@ def main():
 
         st.subheader("High-level metrics")
         filtered_data = data[data['pipeline'].notna()]
+        filtered_data_old = old_data[old_data['pipeline'].notna()]
 
         if filter == 1:
             filtered_data = data[data["modality"] == "nlp"]
+            filtered_data_old = old_data[old_data["modality"] == "nlp"]
         elif filter == 2:
             filtered_data = data[data["modality"] == "cv"]
+            filtered_data_old = old_data[old_data["modality"] == "cv"]
         elif filter == 3:
             filtered_data = data[data["modality"] == "audio"]
+            filtered_data_old = old_data[old_data["modality"] == "audio"]
         elif filter == 4:
             filtered_data = data[data["modality"] == "rl"]
+            filtered_data_old = old_data[old_data["modality"] == "rl"]
         elif filter == 5:
             filtered_data = data[data["modality"] == "multimodal"]
+            filtered_data_old = old_data[old_data["modality"] == "multimodal"]
         elif filter == 6:
             filtered_data = data[data["modality"] == "tabular"]
+            filtered_data_old = old_data[old_data["modality"] == "tabular"]
 
         col1, col2, col3 = st.columns(3)
         with col1:
@@ -283,7 +354,7 @@ def main():
         with col2:
             l = st.selectbox(
                 'What library do you want to see?',
-                ["all", *filtered_data["library"].unique()]
+                ["all", "not transformers", *filtered_data["library"].unique()]
             )
         with col3:
             f = st.selectbox(
@@ -323,17 +394,26 @@ def main():
         
         if p != "all":
             filtered_data = filtered_data[filtered_data["pipeline"] == p]
-        if l != "all":
+            filtered_data_old = filtered_data_old[filtered_data_old["pipeline"] == p]
+        if l != "all" and l != "not transformers":
             filtered_data = filtered_data[filtered_data["library"] == l]
+            filtered_data_old = filtered_data_old[filtered_data_old["library"] == l]
+        if l == "not transformers":
+            filtered_data = filtered_data[filtered_data["library"] != "transformers"]
+            filtered_data_old = filtered_data_old[filtered_data_old["library"] != "transformers"]
         if f != "all":
             if f == "py":
                 filtered_data = filtered_data[filtered_data["pytorch"] == 1]
+                filtered_data_old = filtered_data_old[filtered_data_old["pytorch"] == 1]
             elif f == "tf":
                 filtered_data = filtered_data[filtered_data["tensorflow"] == 1]
+                filtered_data_old = filtered_data_old[filtered_data_old["tensorflow"] == 1]
             elif f == "jax":
                 filtered_data = filtered_data[filtered_data["jax"] == 1]
+                filtered_data_old = filtered_data_old[filtered_data_old["jax"] == 1]
         if filt != []:
             filtered_data = filtered_data[filtered_data.apply(filter_fn, axis=1)]
+            filtered_data_old = filtered_data_old[filtered_data_old.apply(filter_fn, axis=1)]
 
 
         d = filtered_data["pipeline"].value_counts().rename_axis("pipeline").to_frame('counts').reset_index()
@@ -344,23 +424,31 @@ def main():
         )
         sums = grouped_data.sum()
 
+        d_old = filtered_data_old["pipeline"].value_counts().rename_axis("pipeline").to_frame('counts').reset_index()
+        grouped_data_old = filtered_data_old.groupby("pipeline").sum()[columns_of_interest]
+        final_data_old = pd.merge(
+            d_old, grouped_data_old, how="outer", on="pipeline"
+        )
+        sums = grouped_data.sum()
+        sums_old = grouped_data_old.sum()
+
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="Total models", value=filtered_data.shape[0])
+            st.metric(label="Total models", value=filtered_data.shape[0], delta=int(filtered_data.shape[0] - filtered_data_old.shape[0]))
         with col2:
-            st.metric(label="Cumulative Downloads (30d)", value=sums["downloads_30d"])
+            st.metric(label="Cumulative Downloads (30d)", value=sums["downloads_30d"], delta=int(sums["downloads_30d"] - sums_old["downloads_30d"]))
         with col3:
-            st.metric(label="Cumulative likes", value=sums["likes"])
+            st.metric(label="Cumulative likes", value=sums["likes"], delta=int(sums["likes"] - sums_old["likes"]))
 
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="Total in PT", value=sums["pytorch"])
+            st.metric(label="Total in PT", value=sums["pytorch"], delta=int(sums["pytorch"] - sums_old["pytorch"]))
         with col2:
-            st.metric(label="Total in TF", value=sums["tensorflow"])
+            st.metric(label="Total in TF", value=sums["tensorflow"], delta=int(sums["tensorflow"] - sums_old["tensorflow"]))
         with col3:
-            st.metric(label="Total in JAX", value=sums["jax"])
+            st.metric(label="Total in JAX", value=sums["jax"], delta=int(sums["jax"] - sums_old["jax"]))
         
-        st.metric(label="Unique Tags", value=unique_tags)
+        st.metric(label="Unique Tags", value=unique_tags, delta=int(unique_tags - unique_tags_old))
 
         
 
@@ -414,24 +502,25 @@ def main():
 
         columns_of_interest = ["prs_count", "prs_open", "prs_merged", "prs_closed", "discussions_count", "discussions_open", "discussions_closed"]
         sums = data[columns_of_interest].sum()
+        sums_old = old_data[columns_of_interest].sum()
 
         col1, col2, col3, col4 = st.columns(4)
         with col1:
-            st.metric(label="Total PRs", value=sums["prs_count"])
+            st.metric(label="Total PRs", value=sums["prs_count"],delta=int(sums["prs_count"] - sums_old["prs_count"]))
         with col2:
-            st.metric(label="PRs opened", value=sums["prs_open"])
+            st.metric(label="PRs opened", value=sums["prs_open"], delta=int(sums["prs_open"] - sums_old["prs_open"]))
         with col3:
-            st.metric(label="PRs merged", value=sums["prs_merged"])
+            st.metric(label="PRs merged", value=sums["prs_merged"], delta=int(sums["prs_merged"] - sums_old["prs_merged"]))
         with col4:
-            st.metric(label="PRs closed", value=sums["prs_closed"])
+            st.metric(label="PRs closed", value=sums["prs_closed"], delta=int(sums["prs_closed"] - sums_old["prs_closed"]))
 
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="Total discussions", value=sums["discussions_count"])
+            st.metric(label="Total discussions", value=sums["discussions_count"], delta=int(sums["discussions_count"] - sums_old["discussions_count"]))
         with col2:
-            st.metric(label="Discussions open", value=sums["discussions_open"])
+            st.metric(label="Discussions open", value=sums["discussions_open"], delta=int(sums["discussions_open"] - sums_old["discussions_open"]))
         with col3:
-            st.metric(label="Discussions closed", value=sums["discussions_closed"])
+            st.metric(label="Discussions closed", value=sums["discussions_closed"], delta=int(sums["discussions_closed"] - sums_old["discussions_closed"]))
 
         filtered_data = data[["repo_id", "prs_count", "prs_open", "prs_merged", "prs_closed", "discussions_count", "discussions_open", "discussions_closed"]].sort_values("prs_count", ascending=False).reset_index(drop=True)
         st.dataframe(filtered_data)
@@ -441,23 +530,29 @@ def main():
         st.header("Library info")
 
         no_library_count = data["library"].isna().sum()
+        no_library_count_old = old_data["library"].isna().sum()
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="# models that have any library", value=total_samples-no_library_count)
+            v = total_samples-no_library_count
+            v_old = total_samples_old-no_library_count_old
+            st.metric(label="# models that have any library", value=v, delta=int(v-v_old))
         with col2:
-            st.metric(label="No library Specified", value=no_library_count)
+            st.metric(label="No library Specified", value=no_library_count, delta=int(no_library_count-no_library_count_old))
         with col3:
-            st.metric(label="Total Unique library", value=len(data["library"].unique()))
+            v = len(data["library"].unique())
+            v_old = len(old_data["library"].unique())
+            st.metric(label="Total Unique library", value=v, delta=int(v-v_old))
 
 
         st.subheader("High-level metrics")
         filtered_data = data[data['library'].notna()]
+        filtered_data_old = old_data[old_data['library'].notna()]
 
         col1, col2 = st.columns(2)
         with col1:
             lib = st.selectbox(
                 'What library do you want to see? ',
-                ["all", *filtered_data["library"].unique()]
+                ["all", "not transformers", *filtered_data["library"].unique()]
             )
         with col2:
             pip = st.selectbox(
@@ -465,11 +560,15 @@ def main():
                 ["all", *filtered_data["pipeline"].unique()]
             )
 
-        if pip != "all":
+        if pip != "all" :
             filtered_data = filtered_data[filtered_data["pipeline"] == pip]
-        if lib != "all":
+            filtered_data_old = filtered_data_old[filtered_data_old["pipeline"] == pip]
+        if lib != "all" and lib != "not transformers":
             filtered_data = filtered_data[filtered_data["library"] == lib]
-
+            filtered_data_old = filtered_data_old[filtered_data_old["library"] == lib]
+        if lib == "not transformers":
+            filtered_data = filtered_data[filtered_data["library"] != "transformers"]
+            filtered_data_old = filtered_data_old[filtered_data_old["library"] != "transformers"]
 
         d = filtered_data["library"].value_counts().rename_axis("library").to_frame('counts').reset_index()
         grouped_data = filtered_data.groupby("library").sum()[["downloads_30d", "likes"]]
@@ -478,13 +577,23 @@ def main():
         )
         sums = grouped_data.sum()
 
+        d_old = filtered_data_old["library"].value_counts().rename_axis("library").to_frame('counts').reset_index()
+        grouped_data_old = filtered_data_old.groupby("library").sum()[["downloads_30d", "likes"]]
+        final_data_old = pd.merge(
+            d_old, grouped_data_old, how="outer", on="library"
+        ).add_suffix('_old')
+        final_data_old = final_data_old.rename(index=str, columns={"library_old": "library"})
+        sums_old = grouped_data_old.sum()
+
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="Total models", value=filtered_data.shape[0])
+            v = filtered_data.shape[0]
+            v_old = filtered_data_old.shape[0]
+            st.metric(label="Total models", value=v, delta=int(v-v_old))
         with col2:
-            st.metric(label="Cumulative Downloads (30d)", value=sums["downloads_30d"])
+            st.metric(label="Cumulative Downloads (30d)", value=sums["downloads_30d"], delta=int(sums["downloads_30d"]-sums_old["downloads_30d"]))
         with col3:
-            st.metric(label="Cumulative likes", value=sums["likes"])
+            st.metric(label="Cumulative likes", value=sums["likes"], delta=int(sums["likes"]-sums_old["likes"]))
 
         st.subheader("Most common library types (Learn more in library tab)")
         d = filtered_data["library"].value_counts().rename_axis("library").to_frame('counts').reset_index().head(15)
@@ -496,6 +605,13 @@ def main():
         
 
         st.subheader("Aggregated Data")
+        final_data =  pd.merge(
+            final_data, final_data_old, how="outer", on="library"
+        )
+        final_data["counts_diff"] = final_data["counts"] - final_data["counts_old"]
+        final_data["downloads_diff"] = final_data["downloads_30d"] - final_data["downloads_30d_old"]
+        final_data["likes_diff"] = final_data["likes"] - final_data["likes_old"] 
+
         st.dataframe(final_data)
         
         st.subheader("Raw Data")
@@ -509,28 +625,36 @@ def main():
 
         columns_of_interest = ["has_model_index", "has_metadata", "has_text", "text_length"]
         rows = data.shape[0]
+        rows_old = old_data.shape[0]
 
         cond = data["has_model_index"] | data["has_text"]
         with_model_card = data[cond]
         c_model_card = with_model_card.shape[0]
+
+        cond = old_data["has_model_index"] | old_data["has_text"]
+        with_model_card_old = old_data[cond]
+        c_model_card_old = with_model_card_old.shape[0]
+
         st.subheader("High-level metrics")
         col1, col2, col3 = st.columns(3)
         with col1:
-            st.metric(label="# models with model card file", value=c_model_card)
+            st.metric(label="# models with model card file", value=c_model_card, delta=int(c_model_card-c_model_card_old))
         with col2:
-            st.metric(label="# models without model card file", value=rows-c_model_card)
+            st.metric(label="# models without model card file", value=rows-c_model_card, delta=int((rows-c_model_card)-(rows_old-c_model_card_old)))
         
         with_index = data["has_model_index"].sum()
+        with_index_old = old_data["has_model_index"].sum()
         with col1:
-            st.metric(label="# models with model index", value=with_index)
+            st.metric(label="# models with model index", value=with_index, delta=int(with_index-with_index_old))
         with col2:
-            st.metric(label="# models without model index", value=rows-with_index)
+            st.metric(label="# models without model index", value=rows-with_index, delta=int((rows-with_index)-(rows_old-with_index_old)))
 
         with_text = data["has_text"]
+        with_text_old = old_data["has_text"]
         with col1:
-            st.metric(label="# models with model card text", value=with_text.sum())
+            st.metric(label="# models with model card text", value=with_text.sum(), delta=int(with_text.sum()-with_text_old.sum()))
         with col2:
-            st.metric(label="# models without model card text", value=rows-with_text.sum())
+            st.metric(label="# models without model card text", value=rows-with_text.sum(), delta=int((rows-with_text.sum())-(rows_old-with_text_old.sum())))
 
         
         st.subheader("Length (chars) of model card content")