add app file

app.py

@@ -0,0 +1,84 @@
+from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer
+import gradio as gr
+import torch
+import numpy as np
+from mapping_labels import languages_map, id2label
+
+model_checkpoint = "dinalzein/xlm-roberta-base-finetuned-language-identification"
+tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)
+model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)
+trainer = Trainer(model)
+
+class Dataset(torch.utils.data.Dataset):
+    def __init__(self, encodings, labels=None):
+        self.encodings = encodings
+        self.labels = labels
+
+    def __getitem__(self, idx):
+        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+        if self.labels:
+            item["labels"] = torch.tensor(self.labels[idx])
+        return item
+
+    def __len__(self):
+        return len(self.encodings["input_ids"])
+
+def identify_language(txt):
+  txt=[txt]
+  tokenized_txt = tokenizer(txt, truncation=True, max_length=20)
+  txt_dataset = Dataset(tokenized_txt)
+  raw_pred, _, _ = trainer.predict(txt_dataset)
+  # Preprocess raw predictions
+  y_pred = np.argmax(raw_pred, axis=1)
+  return languages_map[id2label[str(y_pred[0])]]
+
+
+
+#with gr.Row():
+examples = [
+    "C'est La Vie",
+    "So ist das Leben",
+    "That is life",
+    "هذه هي الحياة"
+]
+
+
+inputs=gr.inputs.Textbox(placeholder="Enter your text here", label="Text content", lines=5)
+outputs=gr.outputs.Label(label="Language Identified:")
+
+
+article = ('''## Suppoted Langauges \n
+* Arabic (ar)
+* Bulgarian (bg)
+* German (de)
+* Modern greek (el)
+* English (en)
+* Spanish (es)
+* French (fr)
+* Hindi (hi)
+* Italian (it)
+* Japanese (ja)
+* Dutch (nl)
+* Polish (pl)
+* Portuguese (pt)
+* Russian (ru)
+* Swahili (sw)
+* Thai (th)
+* Turkish (tr)
+* Urdu (ur)
+* Vietnamese (vi)
+* Chinese (zh)
+''')
+
+
+gr.Interface(
+    fn=identify_language,
+    inputs=inputs,
+    outputs=outputs,
+    verbose=True,
+    examples = examples,
+    title="Language Identifier",
+    description="It aims at identifing the language a document is written in. It supports 20 different languages.",
+    article=article,
+    theme="huggingface"
+).launch()