app.py

import datetime
import gradio as gr
from huggingface_hub import hf_hub_download
from langdetect import detect, DetectorFactory, detect_langs
import fasttext
from transformers import pipeline

models = {'en': 'Narsil/deberta-large-mnli-zero-cls', # English
          'ru': 'DeepPavlov/xlm-roberta-large-en-ru-mnli', # Russian
          #'uz': 'coppercitylabs/uzbek-news-category-classifier'
          'uz': 'amberoad/bert-multilingual-passage-reranking-msmarco'
          } #Uzbek


hypothesis_templates = {'en': 'This example is {}.', # English
                        'ru': 'Этот пример {}.', # Russian
                        'uz': 'Бу мисол {}.'} # Uzbek
                       
                        
classifiers = {'en': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['en'],
                              model=models['en']),
              
               'ru': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['ru'],
                              model=models['ru']),
                'uz': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['uz'],
                              model=models['uz'])
              }

fasttext_model = fasttext.load_model(hf_hub_download("julien-c/fasttext-language-id", "lid.176.bin"))

def prep_examples():
    example_text1 = "Coronavirus disease (COVID-19) is an infectious disease caused by the SARS-CoV-2 virus. Most \
    people who fall sick with COVID-19 will experience mild to moderate symptoms and recover without special treatment. \
    However, some will become seriously ill and require medical attention."
    example_labels1 = "business,health related,politics,climate change"

    
    example_text2 = "Том был невероятно рад встрече со своим другом, ученным из Китая, который занимается искусственным интелектом."
    example_labels2 = "наука,политика"

    example_text3 = "Алишер Навоий ўзбек классик шоири, буюк ижодкор ва ватанпарвар инсон бўлган."
    example_labels3 = "шеърият,спорт, санъат"

    
    examples = [
        [example_text1, example_labels1],
        [example_text2, example_labels2],
        [example_text3, example_labels3]
        ]

    return examples

def detect_lang(sequence, labels):
    DetectorFactory.seed = 0
    seq_lang = 'en'

    try:
        #seq_lang = detect(sequence)
        #lbl_lang = detect(labels)
        seq_lang = fasttext_model.predict(sequence, k=1)[0][0].split("__label__")[1]
        lbl_lang = fasttext_model.predict(labels, k=1)[0][0].split("__label__")[1]
    except:
        print("Language detection failed!",
              "Date:{}, Sequence:{}, Labels:{}".format(
                  str(datetime.datetime.now()),
                  labels))

    if seq_lang != lbl_lang:
        print("Different languages detected for sequence and labels!",
              "Date:{}, Sequence:{}, Labels:{}, Sequence Language:{}, Label Language:{}".format(
                  str(datetime.datetime.now()),
                  sequence,
                  labels,
                  seq_lang,
                  lbl_lang))

    if seq_lang in models:
        print("Sequence Language detected.",
              "Date:{}, Sequence:{}, Sequence Language:{}".format(
                  str(datetime.datetime.now()),
                  sequence,
                  seq_lang))
    else:
        print("Language not supported. Defaulting to English!",
              "Date:{}, Sequence:{}, Sequence Language:{}".format(
                  str(datetime.datetime.now()),
                  sequence,
                  seq_lang))
        seq_lang = 'en'

    return seq_lang

def sequence_to_classify(sequence, labels):
    classifier = classifiers[detect_lang(sequence, labels)]

    label_clean = str(labels).split(",")
    response = classifier(sequence, label_clean, multi_label=True)

    predicted_labels = response['labels']
    predicted_scores = response['scores']
    clean_output = {idx: float(predicted_scores.pop(0)) for idx in predicted_labels}
    print("Date:{}, Sequence:{}, Labels: {}".format(
        str(datetime.datetime.now()),
        sequence,
        predicted_labels))

    return clean_output

iface = gr.Interface(
    title="En-Ru-Uz Multi-label Zero-shot Classification",
    description="Supported languages are: English, Russian and Uzbek",
    fn=sequence_to_classify,
    inputs=[gr.inputs.Textbox(lines=10,
        label="Please enter the text you would like to classify...",
        placeholder="Text here..."),
        gr.inputs.Textbox(lines=2,
        label="Please enter the candidate labels (separated by comma)...",
        placeholder="Labels here separated by comma...")],
    outputs=gr.outputs.Label(num_top_classes=5),
    #interpretation="default",
    examples=prep_examples())

iface.launch()