Create app.py

app.py

@@ -0,0 +1,122 @@
+import datetime
+import gradio as gr
+from huggingface_hub import hf_hub_download
+from langdetect import detect, DetectorFactory, detect_langs
+import fasttext
+from transformers import pipeline
+
+models = {'en': 'Narsil/deberta-large-mnli-zero-cls', # English
+          'ru': 'DeepPavlov/xlm-roberta-large-en-ru-mnli' # Russian
+          'uz': 'coppercitylabs/uzbek-news-category-classifier' #Uzbek
+
+
+hypothesis_templates = {'en': 'This example is {}.', # English
+                        'ru': 'Этот пример {}.' # Russian
+                        'uz': 'Бу мисол {}.' # Uzbek
+                       
+                        
+classifiers = {'en': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['en'],
+                              model=models['en']),
+              
+               'ru': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['ru'],
+                              model=models['ru']),
+                'uz': pipeline("zero-shot-classification", hypothesis_template=hypothesis_templates['uz'],
+                              model=models['uz']),
+              }
+
+fasttext_model = fasttext.load_model(hf_hub_download("julien-c/fasttext-language-id", "lid.176.bin"))
+
+def prep_examples():
+    example_text1 = "Coronavirus disease (COVID-19) is an infectious disease caused by the SARS-CoV-2 virus. Most \
+    people who fall sick with COVID-19 will experience mild to moderate symptoms and recover without special treatment. \
+    However, some will become seriously ill and require medical attention."
+    example_labels1 = "business,health related,politics,climate change"
+
+    
+    example_text2 = "Россия в среду заявила, что военные учения в аннексированном Москвой Крыму закончились \
+    и что солдаты возвращаются в свои гарнизоны, на следующий день после того, как она объявила о первом выводе \
+    войск от границ Украины."
+    example_labels2 = "новости,комедия"
+
+    example_text3 = "Алишер Навоий ўзбек классик шоири, буюк ижодкор ва ватанпарвар инсон бўлган."
+    example_labels3 = "шеърият,спорт, санъат"
+
+    
+    examples = [
+        [example_text1, example_labels1],
+        [example_text2, example_labels2],
+        ]
+
+    return examples
+
+def detect_lang(sequence, labels):
+    DetectorFactory.seed = 0
+    seq_lang = 'en'
+
+    try:
+        #seq_lang = detect(sequence)
+        #lbl_lang = detect(labels)
+        seq_lang = fasttext_model.predict(sequence, k=1)[0][0].split("__label__")[1]
+        lbl_lang = fasttext_model.predict(labels, k=1)[0][0].split("__label__")[1]
+    except:
+        print("Language detection failed!",
+              "Date:{}, Sequence:{}, Labels:{}".format(
+                  str(datetime.datetime.now()),
+                  labels))
+
+    if seq_lang != lbl_lang:
+        print("Different languages detected for sequence and labels!",
+              "Date:{}, Sequence:{}, Labels:{}, Sequence Language:{}, Label Language:{}".format(
+                  str(datetime.datetime.now()),
+                  sequence,
+                  labels,
+                  seq_lang,
+                  lbl_lang))
+
+    if seq_lang in models:
+        print("Sequence Language detected.",
+              "Date:{}, Sequence:{}, Sequence Language:{}".format(
+                  str(datetime.datetime.now()),
+                  sequence,
+                  seq_lang))
+    else:
+        print("Language not supported. Defaulting to English!",
+              "Date:{}, Sequence:{}, Sequence Language:{}".format(
+                  str(datetime.datetime.now()),
+                  sequence,
+                  seq_lang))
+        seq_lang = 'en'
+
+    return seq_lang
+
+def sequence_to_classify(sequence, labels):
+    classifier = classifiers[detect_lang(sequence, labels)]
+
+    label_clean = str(labels).split(",")
+    response = classifier(sequence, label_clean, multi_label=True)
+
+    predicted_labels = response['labels']
+    predicted_scores = response['scores']
+    clean_output = {idx: float(predicted_scores.pop(0)) for idx in predicted_labels}
+    print("Date:{}, Sequence:{}, Labels: {}".format(
+        str(datetime.datetime.now()),
+        sequence,
+        predicted_labels))
+
+    return clean_output
+
+iface = gr.Interface(
+    title="En-Ru-Uz Multi-label Zero-shot Classification",
+    description="Supported languages are: English, Russian and Uzbek",
+    fn=sequence_to_classify,
+    inputs=[gr.inputs.Textbox(lines=10,
+        label="Please enter the text you would like to classify...",
+        placeholder="Text here..."),
+        gr.inputs.Textbox(lines=2,
+        label="Please enter the candidate labels (separated by comma)...",
+        placeholder="Labels here separated by comma...")],
+    outputs=gr.outputs.Label(num_top_classes=5),
+    #interpretation="default",
+    examples=prep_examples())
+
+iface.launch()