Duplicate from saim074/perplexity

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Perplexity
+emoji: ⚡
+colorFrom: blue
+colorTo: gray
+sdk: streamlit
+sdk_version: 1.15.2
+app_file: app.py
+pinned: false
+duplicated_from: saim074/perplexity
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,69 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+import nltk
+nltk.download('punkt')
+from nltk.tokenize import sent_tokenize
+import streamlit as st
+
+def load_model(model_id):
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    model = AutoModelForCausalLM.from_pretrained(model_id)
+    return tokenizer, model
+
+model_id = "asi/gpt-fr-cased-small"
+tokenizer_fr, model_fr = load_model(model_id)
+
+model_id = "gpt2"
+tokenizer_en, model_en = load_model(model_id)
+
+model_id = "dbmdz/german-gpt2"
+tokenizer_de, model_de = load_model(model_id)
+
+with st.form(key='Form'):
+    text = st.text_area("Enter text here.")
+    option = st.selectbox('Select Language',('English', 'German', 'French'))
+    submitted = st.form_submit_button("Submit")
+
+if submitted:
+    text = text.replace('\n', '')
+    
+    with torch.no_grad():
+        if option == 'German':
+            encodings = tokenizer_de(text, return_tensors="pt")
+            input_ids = encodings.input_ids
+            target_ids = input_ids.clone()
+            loss = model_de(input_ids, labels=target_ids).loss
+        elif option == 'English':
+            encodings = tokenizer_en(text, return_tensors="pt")
+            input_ids = encodings.input_ids
+            target_ids = input_ids.clone()
+            loss = model_en(input_ids, labels=target_ids).loss
+        else:
+            encodings = tokenizer_fr(text, return_tensors="pt")
+            input_ids = encodings.input_ids
+            target_ids = input_ids.clone()
+            loss = model_fr(input_ids, labels=target_ids).loss
+        
+        st.write("Entire Text")
+        st.write("Perplexity: ", round(float(torch.exp(loss)), 2))
+    
+    for sentence in sent_tokenize(text):
+        st.write("________________________")
+        st.write(sentence)
+        with torch.no_grad():
+            if option == 'German':
+                encodings = tokenizer_de(sentence, return_tensors="pt")
+                input_ids = encodings.input_ids
+                target_ids = input_ids.clone()
+                loss = model_de(input_ids, labels=target_ids).loss
+            elif option == 'English':
+                encodings = tokenizer_en(sentence, return_tensors="pt")
+                input_ids = encodings.input_ids
+                target_ids = input_ids.clone()
+                loss = model_en(input_ids, labels=target_ids).loss
+            else:
+                encodings = tokenizer_fr(sentence, return_tensors="pt")
+                input_ids = encodings.input_ids
+                target_ids = input_ids.clone()
+                loss = model_fr(input_ids, labels=target_ids).loss
+        st.write("Perplexity: ", round(float(torch.exp(loss)), 2))

requirements.txt

@@ -0,0 +1,3 @@
+torch
+transformers
+nltk