app.py

import streamlit as st
from evaluate import evaluator
import evaluate
import datasets
import sentencepiece
from huggingface_hub import HfApi, ModelFilter
from transformers import AutoTokenizer, AutoModelForSequenceClassification
from transformers import AutoTokenizer, AutoModelForMaskedLM
from transformers import pipeline
import matplotlib.pyplot as plt

st.title("Metric Compare")

st.markdown("## Choose the dataset you want to use for the comparison:")

api = HfApi()
datasets = api.list_datasets(filter="task_categories:text-classification", sort = "downloads", direction=-1, limit = 20)

dset = st.selectbox(datasets)

st.markdown("## Now select up to 5 models to compare their performance:")

models = api.list_models(filter="trained_dataset:imdb", sort = "downloads", direction=-1, limit = 20)

options = st.multiselect(
     'Choose your models', 
     models)

"""
tokenizer1 = AutoTokenizer.from_pretrained("lvwerra/distilbert-imdb")

model1 = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb")

tokenizer2 = AutoTokenizer.from_pretrained("sahn/distilbert-base-uncased-finetuned-imdb")

model2 = AutoModelForSequenceClassification.from_pretrained("sahn/distilbert-base-uncased-finetuned-imdb")

tokenizer3 = AutoTokenizer.from_pretrained("aychang/roberta-base-imdb")

model3 = AutoModelForSequenceClassification.from_pretrained("aychang/roberta-base-imdb")

tokenizer4 = AutoTokenizer.from_pretrained("Sreevishnu/funnel-transformer-small-imdb")
model4 = AutoModelForSequenceClassification.from_pretrained("Sreevishnu/funnel-transformer-small-imdb")

tokenizer5 = AutoTokenizer.from_pretrained("RANG012/SENATOR")

model5 = AutoModelForSequenceClassification.from_pretrained("RANG012/SENATOR")

accuracy = evaluate.load("accuracy")
f1 = evaluate.load('f1')

data = datasets.load_dataset("imdb", split="test").shuffle().select(range(1000))

eval = evaluator("text-classification")

pipe1 = pipeline("text-classification", model=model1, tokenizer= tokenizer1, device=0)
res_accuracy1 = eval.compute(model_or_pipeline=pipe1, data=data, metric=accuracy,
                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)
res_f11 = eval.compute(model_or_pipeline=pipe1, data=data, metric=f1,
                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)
print({**res_accuracy1, **res_f11})

pipe2 = pipeline("text-classification", model=model2, tokenizer= tokenizer2, device=0)
res_accuracy2 = eval.compute(model_or_pipeline=pipe2, data=data, metric=accuracy,
                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
res_f12 = eval.compute(model_or_pipeline=pipe2, data=data, metric=f1,
                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
print({**res_accuracy2, **res_f12})

pipe3 = pipeline("text-classification", model=model3, tokenizer= tokenizer3, device=0)
res_accuracy3 = eval.compute(model_or_pipeline=pipe3, data=data, metric=accuracy,
                       label_mapping={"neg": 0, "pos": 1},)
res_f13 = eval.compute(model_or_pipeline=pipe3, data=data, metric=f1,
                       label_mapping={"neg": 0, "pos": 1},)
print({**res_accuracy3, **res_f13})

pipe4 = pipeline("text-classification", model=model4, tokenizer= tokenizer4, device=0)
res_accuracy4 = eval.compute(model_or_pipeline=pipe4, data=data, metric=accuracy,
                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
res_f14 = eval.compute(model_or_pipeline=pipe4, data=data, metric=f1,
                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
print({**res_accuracy4, **res_f14})

pipe5 = pipeline("text-classification", model=model5, tokenizer= tokenizer5, device=0)
res_accuracy5 = eval.compute(model_or_pipeline=pipe5, data=data, metric=accuracy,
                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
res_f15 = eval.compute(model_or_pipeline=pipe5, data=data, metric=f1,
                       label_mapping={"LABEL_0": 0, "LABEL_1": 1},)
print({**res_accuracy5, **res_f15})

plt.plot(res_accuracy1['accuracy'], res_f11['f1'], marker='o', markersize=6, color="red")
plt.annotate('distilbert', xy=(res_accuracy1['accuracy']+0.001, res_f11['f1']))
plt.plot(res_accuracy2['accuracy'], res_f12['f1'], marker='o', markersize=6, color="blue")
plt.annotate('distilbert-base-uncased-finetuned', xy=(res_accuracy2['accuracy']+0.001, res_f12['f1']))
plt.plot(res_accuracy3['accuracy'], res_f13['f1'], marker='o', markersize=6, color="green")
plt.annotate('roberta-base', xy=(res_accuracy3['accuracy']-0.009, res_f13['f1']))
plt.plot(res_accuracy4['accuracy'], res_f14['f1'], marker='o', markersize=6, color="purple")
plt.annotate('funnel-transformer-small', xy=(res_accuracy4['accuracy']-0.015, res_f14['f1']))
plt.plot(res_accuracy5['accuracy'], res_f15['f1'], marker='o', markersize=6, color="black")
plt.annotate('SENATOR', xy=(res_accuracy5['accuracy']+0.001, res_f15['f1']))

plt.xlabel('Accuracy')
plt.ylabel('F1 Score') 
#plt.xlim([0.9, 1.0])
#plt.ylim([0.9, 1.0])
plt.title('Comparing the Models')
"""