Duplicate from danielcd99/Toxicity-detection

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Predict.py

@@ -0,0 +1,12 @@
+def predict_tweet(tweet, pipeline):
+    label = pipeline(tweet)[0]['label']
+    if label == 'LABEL_0':
+        return 0
+    else:
+        return 1
+
+def get_predictions(tweets, pipeline):
+    predictions = []
+    for tweet in tweets:
+        predictions.append(predict_tweet(tweet, pipeline))
+    return predictions

README.md

@@ -0,0 +1,13 @@
+---
+title: Toxicity Detection
+emoji: 🐠
+colorFrom: indigo
+colorTo: pink
+sdk: streamlit
+sdk_version: 1.17.0
+app_file: app.py
+pinned: false
+duplicated_from: danielcd99/Toxicity-detection
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Scraper.py

@@ -0,0 +1,12 @@
+def check_user_existence(scraper):
+    pass
+
+
+def get_tweets(scraper, number_of_tweets):
+    tweets = []
+
+    for i, tweet in enumerate(scraper.get_items()):
+        if i == number_of_tweets:
+            return tweets
+        tweets.append(tweet.rawContent)
+        

app.py

@@ -0,0 +1,45 @@
+import streamlit as st
+from snscrape.modules.twitter import TwitterUserScraper
+import pandas as pd
+from Predict import *
+from Scraper import *
+from transformers import pipeline
+
+# Model and pipeline
+MODEL_PATH = 'danielcd99/multilanguage-toxicity-classifier'
+
+def load_pipeline():
+    pipe=pipeline(
+    "text-classification",
+    model=MODEL_PATH
+    )
+    return pipe
+
+pipe = load_pipeline()
+
+
+# Title and subtitle
+st.title("Toxicity Detection")
+st.subheader("This is an app for detecting toxicity in tweets written in portuguese. "
+          "Write the name of the user (without @) and select the number of tweets you want to check.")
+
+
+# User information
+with st.form(key='forms'):
+    st.markdown(
+        """#### Tweets are classified in:
+- 0: Harmless
+- 1: Toxic
+        """)
+    username = st.text_input(label='Username:')
+    number_of_tweets = st.selectbox(
+        'How many tweets do you want to check?',
+        (5, 10, 20, 30))
+    submit_button = st.form_submit_button(label='Analyze')
+
+if submit_button:
+    scraper = TwitterUserScraper(username)
+    tweets = get_tweets(scraper, number_of_tweets)
+    predictions = get_predictions(tweets, pipe)
+
+    st.table(pd.DataFrame({'tweet': tweets, 'toxic':predictions}))

requirements.txt

@@ -0,0 +1,4 @@
+snscrape
+numpy
+torch
+transformers