Create app.py

adding initial functionalities

app.py

@@ -0,0 +1,107 @@
+import streamlit as st
+from evaluate import evaluator
+import evaluate
+import datasets
+import sentencepiece
+from huggingface_hub import HfApi, ModelFilter
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+from transformers import pipeline
+import matplotlib.pyplot as plt
+
+st.title("Metric Compare")
+
+st.markdown("## Choose the dataset you want to use for the comparison:")
+
+api = HfApi()
+datasets = api.list_datasets(filter="task_categories:text-classification", sort = "downloads", direction=-1, limit = 20)
+
+dset = st.selectbox(datasets)
+
+st.markdown("## Now select up to 5 models to compare their performance:")
+
+models = api.list_models(filter="trained_dataset:imdb", sort = "downloads", direction=-1, limit = 20)
+
+options = st.multiselect(
+     'Choose your models', 
+     models)
+
+"""
+tokenizer1 = AutoTokenizer.from_pretrained("lvwerra/distilbert-imdb")
+
+model1 = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb")
+
+tokenizer2 = AutoTokenizer.from_pretrained("sahn/distilbert-base-uncased-finetuned-imdb")
+
+model2 = AutoModelForSequenceClassification.from_pretrained("sahn/distilbert-base-uncased-finetuned-imdb")
+
+tokenizer3 = AutoTokenizer.from_pretrained("aychang/roberta-base-imdb")
+
+model3 = AutoModelForSequenceClassification.from_pretrained("aychang/roberta-base-imdb")
+
+tokenizer4 = AutoTokenizer.from_pretrained("Sreevishnu/funnel-transformer-small-imdb")
+model4 = AutoModelForSequenceClassification.from_pretrained("Sreevishnu/funnel-transformer-small-imdb")
+
+tokenizer5 = AutoTokenizer.from_pretrained("RANG012/SENATOR")
+
+model5 = AutoModelForSequenceClassification.from_pretrained("RANG012/SENATOR")
+
+accuracy = evaluate.load("accuracy")
+f1 = evaluate.load('f1')
+
+data = datasets.load_dataset("imdb", split="test").shuffle().select(range(1000))
+
+eval = evaluator("text-classification")
+
+pipe1 = pipeline("text-classification", model=model1, tokenizer= tokenizer1, device=0)
+res_accuracy1 = eval.compute(model_or_pipeline=pipe1, data=data, metric=accuracy,
+                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)
+res_f11 = eval.compute(model_or_pipeline=pipe1, data=data, metric=f1,
+                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)
+print({**res_accuracy1, **res_f11})
+
+pipe2 = pipeline("text-classification", model=model2, tokenizer= tokenizer2, device=0)
+res_accuracy2 = eval.compute(model_or_pipeline=pipe2, data=data, metric=accuracy,
+                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
+res_f12 = eval.compute(model_or_pipeline=pipe2, data=data, metric=f1,
+                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
+print({**res_accuracy2, **res_f12})
+
+pipe3 = pipeline("text-classification", model=model3, tokenizer= tokenizer3, device=0)
+res_accuracy3 = eval.compute(model_or_pipeline=pipe3, data=data, metric=accuracy,
+                       label_mapping={"neg": 0, "pos": 1},)
+res_f13 = eval.compute(model_or_pipeline=pipe3, data=data, metric=f1,
+                       label_mapping={"neg": 0, "pos": 1},)
+print({**res_accuracy3, **res_f13})
+
+pipe4 = pipeline("text-classification", model=model4, tokenizer= tokenizer4, device=0)
+res_accuracy4 = eval.compute(model_or_pipeline=pipe4, data=data, metric=accuracy,
+                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
+res_f14 = eval.compute(model_or_pipeline=pipe4, data=data, metric=f1,
+                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
+print({**res_accuracy4, **res_f14})
+
+pipe5 = pipeline("text-classification", model=model5, tokenizer= tokenizer5, device=0)
+res_accuracy5 = eval.compute(model_or_pipeline=pipe5, data=data, metric=accuracy,
+                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
+res_f15 = eval.compute(model_or_pipeline=pipe5, data=data, metric=f1,
+                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
+print({**res_accuracy5, **res_f15})
+
+plt.plot(res_accuracy1['accuracy'], res_f11['f1'], marker='o', markersize=6, color="red")
+plt.annotate('distilbert', xy=(res_accuracy1['accuracy']+0.001, res_f11['f1']))
+plt.plot(res_accuracy2['accuracy'], res_f12['f1'], marker='o', markersize=6, color="blue")
+plt.annotate('distilbert-base-uncased-finetuned', xy=(res_accuracy2['accuracy']+0.001, res_f12['f1']))
+plt.plot(res_accuracy3['accuracy'], res_f13['f1'], marker='o', markersize=6, color="green")
+plt.annotate('roberta-base', xy=(res_accuracy3['accuracy']-0.009, res_f13['f1']))
+plt.plot(res_accuracy4['accuracy'], res_f14['f1'], marker='o', markersize=6, color="purple")
+plt.annotate('funnel-transformer-small', xy=(res_accuracy4['accuracy']-0.015, res_f14['f1']))
+plt.plot(res_accuracy5['accuracy'], res_f15['f1'], marker='o', markersize=6, color="black")
+plt.annotate('SENATOR', xy=(res_accuracy5['accuracy']+0.001, res_f15['f1']))
+
+plt.xlabel('Accuracy')
+plt.ylabel('F1 Score') 
+#plt.xlim([0.9, 1.0])
+#plt.ylim([0.9, 1.0])
+plt.title('Comparing the Models')
+"""