Update app.py

app.py

@@ -1,29 +1,22 @@
 import tweepy as tw
 import streamlit as st
 import pandas as pd
-import torch
-import numpy as np
 import regex as re
+import numpy as np
 import pysentimiento
 import geopy
 import matplotlib.pyplot as plt
+import langdetect
+
 
 from pysentimiento.preprocessing import preprocess_tweet
 from geopy.geocoders import Nominatim
+from transformers import pipeline
+from langdetect import detect
 
-from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
-from transformers import AutoTokenizer, AutoModelForSequenceClassification,AdamW
-tokenizer = AutoTokenizer.from_pretrained('hackathon-pln-es/twitter_sexismo-finetuned-robertuito-exist2021')
-model = AutoModelForSequenceClassification.from_pretrained("hackathon-pln-es/twitter_sexismo-finetuned-robertuito-exist2021")
 
-import torch
-if torch.cuda.is_available():  
-    device = torch.device(	"cuda")
-    print('I will use the GPU:', torch.cuda.get_device_name(0))
-    
-else:
-    print('No GPU available, using the CPU instead.')
-    device = torch.device("cpu")
+model_checkpoint = "hackathon-pln-es/twitter_sexismo-finetuned-robertuito-exist2021" 
+pipeline_nlp = pipeline("text-classification", model=model_checkpoint)
 
     
 consumer_key = "BjipwQslVG4vBdy4qK318KnoA"
@@ -33,34 +26,23 @@ access_token_secret = "pqQ5aFSJxzJ2xnI6yhVtNjQO36FOu8DBOH6DtUrPAU54J"
 auth = tw.OAuthHandler(consumer_key, consumer_secret)
 auth.set_access_token(access_token, access_token_secret)
 api = tw.API(auth, wait_on_rate_limit=True)
-
-def preprocess(text):
-    #text=text.lower()
-    # remove hyperlinks
-    text = re.sub(r'https?:\/\/.*[\r\n]*', '', text)
-    text = re.sub(r'http?:\/\/.*[\r\n]*', '', text)
-    #Replace &amp, &lt, &gt with &,<,> respectively
-    text=text.replace(r'&amp;?',r'and')
-    text=text.replace(r'&lt;',r'<')
-    text=text.replace(r'&gt;',r'>')
-    #remove hashtag sign
-    #text=re.sub(r"#","",text)   
-    #remove mentions
-    text = re.sub(r"(?:\@)\w+", '', text)
-    #text=re.sub(r"@","",text)
-    #remove non ascii chars
-    text=text.encode("ascii",errors="ignore").decode()
-    #remove some puncts (except . ! ?)
-    text=re.sub(r'[:"#$%&\*+,-/:;<=>@\\^_`{|}~]+','',text)
-    text=re.sub(r'[!]+','!',text)
-    text=re.sub(r'[?]+','?',text)
-    text=re.sub(r'[.]+','.',text)
-    text=re.sub(r"'","",text)
-    text=re.sub(r"\(","",text)
-    text=re.sub(r"\)","",text)
-    text=" ".join(text.split())
-    return text
-
+   
+def limpieza_datos(tweet):
+    # Eliminar emojis
+    tweet = re.sub(r'[\U0001F600-\U0001F64F]', '', tweet)
+    tweet = re.sub(r'[\U0001F300-\U0001F5FF]', '', tweet)
+    tweet = re.sub(r'[\U0001F680-\U0001F6FF]', '', tweet)
+    tweet = re.sub(r'[\U0001F1E0-\U0001F1FF]', '', tweet)
+    # Eliminar arrobas
+    tweet = re.sub(r'@\w+', '', tweet)
+    # Eliminar URL
+    tweet = re.sub(r'http\S+', '', tweet)
+    # Eliminar hashtags
+    tweet = re.sub(r'#\w+', '', tweet)
+    # Eliminar caracteres especiales
+    #tweet = re.sub(r'[^a-zA-Z0-9 \n\.]', '', tweet)
+    tweet = re.sub(r'[^a-zA-Z0-9 \n\áéíóúÁÉÍÓÚñÑ.]', '', tweet)
+    return tweet
 
 def highlight_survived(s):
     return ['background-color: red']*len(s) if (s.Sexista == 1) else ['background-color: green']*len(s)
@@ -73,231 +55,168 @@ def color_survived(val):
 st.set_page_config(layout="wide")
 st.markdown('<style>body{background-color: Blue;}</style>',unsafe_allow_html=True)
 
-colT1,colT2 = st.columns([2,8])
-with colT2:
-   # st.title('Analisis de comentarios sexistas en Twitter') 
-    st.markdown(""" <style> .font {
-    font-size:40px ; font-family: 'Cooper Black'; color: #06bf69;} 
-    </style> """, unsafe_allow_html=True)
-    st.markdown('<p class="font">Análisis de comentarios sexistas en Twitter</p>', unsafe_allow_html=True)
-    
-    st.markdown(""" <style> .font1 {
-    font-size:28px ; font-family: 'Times New Roman'; color: #8d33ff;} 
-    </style> """, unsafe_allow_html=True)
-
-    st.markdown(""" <style> .font2 {
-    font-size:16px ; font-family: 'Times New Roman'; color: #3358ff;} 
-    </style> """, unsafe_allow_html=True)
-
-
-
-
-
-def analizar_tweets(search_words, number_of_tweets ):
-  tweets = api.user_timeline(screen_name = search_words, count= number_of_tweets)
-  tweet_list = [i.text for i in tweets]
-  text= pd.DataFrame(tweet_list)
-  text[0] = text[0].apply(preprocess_tweet)
-  text1=text[0].values
-  indices1=tokenizer.batch_encode_plus(text1.tolist(), max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
-  input_ids1=indices1["input_ids"]
-  attention_masks1=indices1["attention_mask"]
-  prediction_inputs1= torch.tensor(input_ids1)
-  prediction_masks1 = torch.tensor(attention_masks1)
-  batch_size = 25
-  # Create the DataLoader.
-  prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
-  prediction_sampler1 = SequentialSampler(prediction_data1)
-  prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
-  #print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
-  # Put model in evaluation mode
-  model.eval()
-  # Tracking variables 
-  predictions = []
-  for batch in prediction_dataloader1:
-    batch = tuple(t.to(device) for t in batch)
-    # Unpack the inputs from our dataloader
-    b_input_ids1, b_input_mask1 = batch
-
-    #Telling the model not to compute or store gradients, saving memory and   # speeding up prediction
-    with torch.no_grad():
-      # Forward pass, calculate logit predictions
-      outputs1 = model(b_input_ids1, token_type_ids=None,attention_mask=b_input_mask1)
-    logits1 = outputs1[0]
-    # Move logits and labels to CPU
-    logits1 = logits1.detach().cpu().numpy()
-    # Store predictions and true labels
-    predictions.append(logits1)
-      
-  #flat_predictions = [item for sublist in predictions for item in sublist]
-  flat_predictions = [item for sublist in predictions for item in sublist]
-  
-  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()
-  
-  probability = np.amax(logits1,axis=1).flatten()
-  Tweets =['Últimos '+ str(number_of_tweets)+' Tweets'+' de '+search_words]
-  df = pd.DataFrame(list(zip(text1, flat_predictions,probability)), columns = ['Tweets' , 'Prediccion','Probabilidad'])
-  
-  df['Prediccion']= np.where(df['Prediccion']== 0, 'No Sexista', 'Sexista')
-  df['Tweets'] = df['Tweets'].str.replace('RT|@', '')   
-  #df['Tweets'] = df['Tweets'].apply(lambda x: re.sub(r'[:;][-o^]?[)\]DpP3]|[(/\\]|[\U0001f600-\U0001f64f]|[\U0001f300-\U0001f5ff]|[\U0001f680-\U0001f6ff]|[\U0001f1e0-\U0001f1ff]','', x))
-
-  tabla = st.table(df.reset_index(drop=True).head(30).style.applymap(color_survived, subset=['Prediccion']))
+#st.markdown('<style>body{background-color: Blue;}</style>',unsafe_allow_html=True)
+#colT1,colT2 = st.columns([2,8])
+st.markdown(""" <style> .fondo {
+background-image: url("https://www.google.com/url?sa=i&url=https%3A%2F%2Flasmujereseneldeportemexicano.wordpress.com%2F2016%2F11%2F17%2Fpor-que-es-importante-hablar-de-genero%2F&psig=AOvVaw0xG7SVXtJoEpwt-fF5Kykt&ust=1676431557056000&source=images&cd=vfe&ved=0CBAQjRxqFwoTCJiu-a6IlP0CFQAAAAAdAAAAABAJ");
+background-size: 180%;} 
+</style> """, unsafe_allow_html=True)
 
-  return tabla
-
-def analizar_frase(frase):
-  #palabra = frase.split()
-  palabra = [frase]
-  
-  indices1=tokenizer.batch_encode_plus(palabra,max_length=128,add_special_tokens=True, 
-                                         return_attention_mask=True,
-                                         pad_to_max_length=True,
-                                         truncation=True)
-  input_ids1=indices1["input_ids"]
-  attention_masks1=indices1["attention_mask"]
-  prediction_inputs1= torch.tensor(input_ids1)
-  prediction_masks1 = torch.tensor(attention_masks1)
-  batch_size = 25
-  prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
-  prediction_sampler1 = SequentialSampler(prediction_data1)
-  prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
-  model.eval()
-  predictions = []
-  # Predict 
-  for batch in prediction_dataloader1:
-    batch = tuple(t.to(device) for t in batch)
-    # Unpack the inputs from our dataloader
-    b_input_ids1, b_input_mask1 = batch
-    # Telling the model not to compute or store gradients, saving memory and   # speeding up prediction
-    with torch.no_grad():
-      # Forward pass, calculate logit predictions
-      outputs1 = model(b_input_ids1, token_type_ids=None,attention_mask=b_input_mask1)
-    logits1 = outputs1[0]
-    # Move logits and labels to CPU
-    logits1 = logits1.detach().cpu().numpy()
-    # Store predictions and true labels
-    predictions.append(logits1)
-  flat_predictions = [item for sublist in predictions for item in sublist]
-  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()
-  tokens = tokenizer.tokenize(frase)
-  # Convertir los tokens a un formato compatible con el modelo
-  input_ids = tokenizer.convert_tokens_to_ids(tokens)
-  attention_masks = [1] * len(input_ids)
- 
-  # Pasar los tokens al modelo
-  outputs = model(torch.tensor([input_ids]), token_type_ids=None, attention_mask=torch.tensor([attention_masks]))
-  scores = outputs[0]
-  #prediccion = scores.argmax(dim=1).item()
-  # Obtener la probabilidad de que la frase sea "sexista"
-  probabilidad_sexista = scores.amax(dim=1).item()
-  #print(probabilidad_sexista)
-  
-  # Crear un Dataframe
-  text= pd.DataFrame({'Frase': [frase], 'Prediccion':[flat_predictions], 'Probabilidad':[probabilidad_sexista]})
-  text['Prediccion'] = np.where(text['Prediccion'] == 0 , 'No Sexista', 'Sexista')
 
+st.markdown(""" <style> .font {
+font-size:40px ; font-family: 'Cooper Black'; color: #301E67;} 
+</style> """, unsafe_allow_html=True)
 
-  tabla = st.table(text.reset_index(drop=True).head(50).style.applymap(color_survived, subset=['Prediccion']))
+st.markdown('<p class="font">Análisis de comentarios sexistas en linea</p>', unsafe_allow_html=True)
     
+st.markdown(""" <style> .font1 {
+font-size:28px ; font-family: 'Times New Roman'; color: #8d33ff;} 
+</style> """, unsafe_allow_html=True)
+
+st.markdown(""" <style> .font2 {
+font-size:16px ; font-family: 'Times New Roman'; color: #5B8FB9;} 
+</style> """, unsafe_allow_html=True)
+
+st.markdown('<p class="font2">Este proyecto consiste en una aplicación web que utiliza la biblioteca Tweepy de Python para descargar tweets de Twitter, permitiendo buscar Tweets por usuario y por localidad. Luego, utiliza modelos de lenguaje basados en Transformers para analizar los tweets y detectar comentarios sexistas. Los resultados se almacenan en un dataframe para su posterior visualización y análisis. El objetivo del proyecto es identificar y proporcionar información sobre el discurso sexista en línea para combatir la discriminación y el acoso hacia las mujeres y otros grupos marginados, y así informar políticas y prácticas que promuevan la igualdad de género y la inclusión.</p>',unsafe_allow_html=True)
+
+
+def tweets_usuario(usuario, cant_de_tweets):
+  tabla = []
+  if(cant_de_tweets > 0 and usuario != "" ):
+      try:
+          # Buscar la información del perfil de usuario
+          user = api.get_user(screen_name=usuario)
+          tweets = api.user_timeline(screen_name = usuario,tweet_mode="extended", count= cant_de_tweets)
+          result = []
+          for tweet in tweets:
+              if (tweet.full_text.startswith('RT')):
+                  continue
+              else:
+                  text = tweet.full_text
+                  try:
+                      language = detect(text)
+                      if language == 'es':
+                          datos=limpieza_datos(text)
+                          if datos == "":
+                              continue
+                          else:
+                              prediction = pipeline_nlp(datos)
+                              for predic in prediction:
+                                etiqueta = {'Tweets': datos, 'Prediccion': predic['label'], 'Probabilidad': predic['score']}
+                                result.append(etiqueta)
+                  except:
+                      pass   
+          df = pd.DataFrame(result)
+          if df.empty:
+              muestra= st.text("No hay tweets Sexistas a analizar")
+              tabla.append(muestra)
+          else:
+              df.sort_values(by=['Prediccion', 'Probabilidad'], ascending=[False, False], inplace=True)
+              df['Prediccion'] = np.where(df['Prediccion'] == 'LABEL_1', 'Sexista', 'No Sexista')
+              df['Probabilidad'] = df['Probabilidad'].apply(lambda x: round(x, 3))
+              muestra = st.table(df.reset_index(drop=True).head(30).style.applymap(color_survived, subset=['Prediccion']))
+              tabla.append(muestra)
+      except Exception as e:
+          muestra = st.text(f"La cuenta {search_words} no existe.") 
+          tabla.append(muestra) 
+  else:
+      muestra= st.text("Ingrese los parametros correspondientes")
+      tabla.append(muestra)      
   return tabla
 
 def tweets_localidad(buscar_localidad):
-  geolocator = Nominatim(user_agent="nombre_del_usuario")
-  location = geolocator.geocode(buscar_localidad)
-  radius = "10km"
-  tweets = api.search_tweets(q="",lang="es",geocode=f"{location.latitude},{location.longitude},{radius}", count = 50)
-  localidad = [i.user.location for i in tweets] 
-  text_localidad = pd.DataFrame(localidad)  
-  username = [i.user.screen_name for i in tweets] 
-  text_user= pd.DataFrame(username)
- 
-  tweet_list = [i.text for i in tweets]
-  text= pd.DataFrame(tweet_list)
-  text[0] = text[0].apply(preprocess_tweet)
-  text1=text[0].values
-  print(text1)
-  indices1=tokenizer.batch_encode_plus(text1.tolist(), max_length=128,add_special_tokens=True, return_attention_mask=True,pad_to_max_length=True,truncation=True)
-  input_ids1=indices1["input_ids"]
-  attention_masks1=indices1["attention_mask"]
-  prediction_inputs1= torch.tensor(input_ids1)
-  prediction_masks1 = torch.tensor(attention_masks1)
-  batch_size = 25
-  # Create the DataLoader.
-  prediction_data1 = TensorDataset(prediction_inputs1, prediction_masks1)
-  prediction_sampler1 = SequentialSampler(prediction_data1)
-  prediction_dataloader1 = DataLoader(prediction_data1, sampler=prediction_sampler1, batch_size=batch_size)
-  #print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs1)))
-  # Put model in evaluation mode
-  model.eval()
-  # Tracking variables 
-  predictions = []
-  for batch in prediction_dataloader1:
-    batch = tuple(t.to(device) for t in batch)
-    # Unpack the inputs from our dataloader
-    b_input_ids1, b_input_mask1 = batch
-
-    #Telling the model not to compute or store gradients, saving memory and   # speeding up prediction
-    with torch.no_grad():
-      # Forward pass, calculate logit predictions
-      outputs1 = model(b_input_ids1, token_type_ids=None,attention_mask=b_input_mask1)
-    logits1 = outputs1[0]
-    # Move logits and labels to CPU
-    logits1 = logits1.detach().cpu().numpy()
-    # Store predictions and true labels
-    predictions.append(logits1)
-      
-  #flat_predictions = [item for sublist in predictions for item in sublist]
-  flat_predictions = [item for sublist in predictions for item in sublist]
-  
-  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()
-  
-  probability = np.amax(logits1,axis=1).flatten()
-  Tweets =['Últimos 50 Tweets'+' de '+ buscar_localidad]
-  df = pd.DataFrame(list(zip(text1, localidad,username, flat_predictions,probability)), columns = ['Tweets' ,'Localidad' , 'Usuario','Prediccion','Probabilidad'])
-  
-  df['Prediccion']= np.where(df['Prediccion']== 0, 'No Sexista', 'Sexista')
-  #df['Tweets'] = df['Tweets'].str.replace('RT|@', '')
-  df_filtrado = df[df["Prediccion"]=="Sexista" ]
-  #df['Tweets'] = df['Tweets'].apply(lambda x: re.sub(r'[:;][-o^]?[)\]DpP3]|[(/\\]|[\U0001f600-\U0001f64f]|[\U0001f300-\U0001f5ff]|[\U0001f680-\U0001f6ff]|[\U0001f1e0-\U0001f1ff]','', x))
-  
-  tabla = st.table(df.reset_index(drop=True).head(50).style.applymap(color_survived, subset=['Prediccion']))
+    tabla = []
+    try:
+        geolocator = Nominatim(user_agent="nombre_del_usuario")
+        location = geolocator.geocode(buscar_localidad)
+        radius = "15km"
+        tweets = api.search_tweets(q="",lang="es",geocode=f"{location.latitude},{location.longitude},{radius}", count = 1000, tweet_mode="extended")
+        result = []
+        for tweet in tweets:
+            if (tweet.full_text.startswith('RT')):
+                continue
+            elif not tweet.full_text.strip():
+                continue
+            else:
+                datos = limpieza_datos(tweet.full_text)
+                prediction = pipeline_nlp(datos)
+                for predic in prediction:
+                    etiqueta = {'Tweets': datos,'Prediccion': predic['label'], 'Probabilidad': predic['score']}
+                    result.append(etiqueta)
+        df = pd.DataFrame(result)
+        if df.empty:
+            muestra=st.text("No se encontraron tweets sexistas dentro de la localidad")
+            tabla.append(muestra)
+        else:
+            #tabla.append(muestra)
+            #df.sort_values(by=['Prediccion', 'Probabilidad'], ascending=[False, False], inplace=True)
+            df.sort_values(by='Prediccion', ascending=False, inplace=True)
+            df['Prediccion'] = np.where(df['Prediccion'] == 'LABEL_1', 'Sexista', 'No Sexista')
+            df['Probabilidad'] = df['Probabilidad'].round(3)
+            muestra = st.table(df.reset_index(drop=True).head(10).style.applymap(color_survived, subset=['Prediccion']))
+            tabla.append(muestra)
+            #resultado=df.groupby('Prediccion')['Probabilidad'].sum()
+            with st.container():
+                resultado = df['Prediccion'].head(10).value_counts()
+                colores=["#EE3555","#aae977"]
+                fig, ax = plt.subplots()
+                fig.set_size_inches(2, 2)
+                plt.pie(resultado,labels=resultado.index,autopct='%1.1f%%',colors=colores,  textprops={'fontsize': 4})
+                ax.set_title("Porcentajes por Categorias", fontsize=5, fontweight="bold")
+                plt.rcParams.update({'font.size':4, 'font.weight':'bold'})
+                ax.legend()
+                # Muestra el gráfico
+                plt.show()
+                st.set_option('deprecation.showPyplotGlobalUse', False)
+                st.pyplot()
+            
+            plt.bar(resultado.index, resultado, color=colores)
+            ax.set_title("Porcentajes por Categorias", fontsize=5, fontweight="bold")
+            plt.rcParams.update({'font.size':4, 'font.weight':'bold'})
+            ax.set_xlabel("Categoría")
+            ax.set_ylabel("Probabilidad")
+            # Muestra el gráfico
+            plt.show()
+            st.set_option('deprecation.showPyplotGlobalUse', False)
+            st.pyplot()
+        
+    except AttributeError as e:
+        muestra=st.text("No existe ninguna localidad con ese nombre") 
+        tabla.append(muestra)
+                 
+    return tabla 
     
-  df_sexista = df[df['Prediccion']== 'Sexista']
-  df_no_sexista = df[df['Probabilidad'] > 0]
-  sexista = len(df_sexista)
-  no_sexista = len(df_no_sexista)
-
-  # Crear un gráfico de barras
-  labels = ['Sexista  ', ' No sexista']
-  counts = [sexista, no_sexista]
-  plt.bar(labels, counts)
-  plt.xlabel('Categoría')
-  plt.ylabel('Cantidad de tweets')
-  plt.title('Cantidad de tweets sexistas y no sexistas')
-  plt.show()
-  st.pyplot()
-  st.set_option('deprecation.showPyplotGlobalUse', False)
+def analizar_frase(frase):
+    language = detect(frase)
+    if frase == "":
+        tabla = st.text("Ingrese una frase")
+        #st.text("Ingrese una frase")
+    elif language == 'es':
+        predictions = pipeline_nlp(frase)
+        # convierte las predicciones en una lista de diccionarios
+        data = [{'Texto': frase, 'Prediccion': prediction['label'], 'Probabilidad': prediction['score']} for prediction in predictions]
+        # crea un DataFrame a partir de la lista de diccionarios
+        df = pd.DataFrame(data)
+        df['Prediccion'] = np.where( df['Prediccion'] == 'LABEL_1', 'Sexista', 'No Sexista')
+        # muestra el DataFrame
+        tabla = st.table(df.reset_index(drop=True).head(1).style.applymap(color_survived, subset=['Prediccion']))
+    else:
+        tabla = st.text("Solo Frase en español")
+        
+    return tabla
     
-  return df
-
- 
 def run():   
  with st.form("my_form"):
    col,buff1, buff2 = st.columns([2,2,1])
    st.write("Escoja una Opción")
-   search_words = col.text_input("Introduzca el termino, usuario o localidad para analizar y pulse el check correspondiente")
-   number_of_tweets = col.number_input('Introduzca número de tweets a analizar. Máximo 50', 0,50,0)
-   termino=st.checkbox('Término')
+   search_words = col.text_input("Introduzca la frase, el usuario o localidad para analizar y pulse el check correspondiente")
+   number_of_tweets = col.number_input('Introduzca número de tweets a analizar del usuario Máximo 50', 0,50,0)
+   termino=st.checkbox('Frase')
    usuario=st.checkbox('Usuario')
    localidad=st.checkbox('Localidad')
    submit_button = col.form_submit_button(label='Analizar')
    error =False
-   submit_button = st.sidebar.button('Submit')
-   clear_button = st.sidebar.button('Clear')
-
-   st.sidebar.row(submit_button, clear_button)
-     
+  
    if submit_button:
             # Condición para el caso de que esten dos check seleccionados
             if ( termino == False and usuario == False and localidad == False):
@@ -312,8 +231,8 @@ def run():
                   analizar_frase(search_words)
                     
                 elif (usuario):
-                    analizar_tweets(search_words,number_of_tweets)
+                    tweets_usuario(search_words,number_of_tweets)
                 elif (localidad):
                     tweets_localidad(search_words)
-                
+     
 run()