29/10/2023

app.py

@@ -1,91 +1,321 @@
+#--------------------------------------------------------------------
+#            DEPENDENCIAS
+#--------------------------------------------------------------------
 import os
+from io import StringIO
+import requests
+import gradio as gr
 import pandas as pd
+import numpy as np
 import openai
-import re
-import json 
+import tiktoken
+#import streamlit as st
+from openai.embeddings_utils import get_embedding, cosine_similarity
+#from langchain.document_loaders import PyPDFLoader
+#from langchain.text_splitter import CharacterTextSplitter
+#from PyPDF2 import PdfReader, PdfFileReader
+from langchain.embeddings import OpenAIEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
 from langchain.chat_models import ChatOpenAI
-import regex
-import gradio as gr
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+from langchain.llms import OpenAI, HuggingFaceHub
+from langchain.chains.question_answering import load_qa_chain
+#from htmlTemplates import css, bot_template, user_template
+
+#import ast 
+#from langchain.schema.vectorstore import Document
 from langchain.schema import Document
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings.openai import OpenAIEmbeddings
-from openai.embeddings_utils import get_embedding
-from openai.embeddings_utils import cosine_similarity
-import gspread # See: https://docs.gspread.org/en/v5.10.0/user-guide.html
-from oauth2client.service_account import ServiceAccountCredentials
-from datetime import datetime
-
-# API de OpenAI
+#import fitz  # PyMuPDF
+#import pytesseract
+#from PIL import Image
+#from io import BytesIO
+#import cv2
+
+
+#--------------------------------------------------------------------
+#            LLAVES
+#--------------------------------------------------------------------
 openai.api_key = os.getenv("OPENAI_API_KEY")
+api_key = os.getenv("OPENAI_API_KEY")
+token = os.getenv("token")
+headers = { 'Authorization': f'token {token}',
+            'Accept': 'application/vnd.github.v3.raw' }
 
-# Establece las credenciales y la API
+#--------------------------------------------------------------------
+#            CARGAR CSV EMBEDDINGS
+#--------------------------------------------------------------------
+#
+url_tomos_conf_DPR = os.getenv("url_tomos_conf_DPR")
+response_tomos_conf_DPR = requests.get( url_tomos_conf_DPR, headers = headers )
+csv_content_tomos_conf_DPR = response_tomos_conf_DPR.text
+tomos_conf_DPR = pd.read_csv(StringIO( csv_content_tomos_conf_DPR ))
 
-#credentials = os.getenv( "credentials" )
+#
+url_tomos_conf_cita = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/04.%20Tomos%20Confidenciales%20WM%20(folios%20citados%20en%20DPR)/tomos_conf_cita.csv'
+response_tomos_conf_cita = requests.get( url_tomos_conf_cita, headers = headers )
+csv_content_tomos_conf_cita = response_tomos_conf_cita.text
+tomos_conf_cita = pd.read_csv(StringIO( csv_content_tomos_conf_cita ))
 
-#credentials =  json.loads( credentials )
+#
+url_df_tomos_1a28_01 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_01.csv'
+response_df_tomos_1a28_01 = requests.get( url_df_tomos_1a28_01, headers = headers )
+csv_content_df_tomos_1a28_01 = response_df_tomos_1a28_01.text
+df_tomos_1a28_01 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_01 ))
 
-#gc = gspread.service_account_from_dict( credentials )
+#
+url_df_tomos_1a28_02 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_02.csv'
+response_df_tomos_1a28_02 = requests.get( url_df_tomos_1a28_02, headers = headers )
+csv_content_df_tomos_1a28_02 = response_df_tomos_1a28_02.text
+df_tomos_1a28_02 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_02 ))
 
-#Google_URL = os.getenv( "Google_URL" )
+#
+url_df_tomos_1a28_03 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_03.csv'
+response_df_tomos_1a28_03 = requests.get( url_df_tomos_1a28_03, headers = headers )
+csv_content_df_tomos_1a28_03 = response_df_tomos_1a28_03.text
+df_tomos_1a28_03 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_03 ))
 
-# Cargar el archivo 
-df_Expediente = pd.read_csv( os.getenv( "Data" ) )
+#
+url_df_tomos_1a28_04 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_04.csv'
+response_df_tomos_1a28_04 = requests.get( url_df_tomos_1a28_04, headers = headers )
+csv_content_df_tomos_1a28_04 = response_df_tomos_1a28_04.text
+df_tomos_1a28_04 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_04 ))
 
-# Main OpenAI Function
 #
-# Function: Get embeddings
+url_df_tomos_1a28_05 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_05.csv'
+response_df_tomos_1a28_05 = requests.get( url_df_tomos_1a28_05, headers = headers )
+csv_content_df_tomos_1a28_05 = response_df_tomos_1a28_05.text
+df_tomos_1a28_05 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_05 ))
 
-def get_embedding( text , model ):
-    text = text.replace("\n", " ")
-    return openai.Embedding.create( input = [text], model = model )['data'][0]['embedding']
+#
+url_df_tomos_1a28_06 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_06.csv'
+response_df_tomos_1a28_06 = requests.get( url_df_tomos_1a28_06, headers = headers )
+csv_content_df_tomos_1a28_06 = response_df_tomos_1a28_06.text
+df_tomos_1a28_06 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_06 ))
 
-# Función de búsqueda
+#
+url_df_tomos_1a28_07 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_07.csv'
+response_df_tomos_1a28_07 = requests.get( url_df_tomos_1a28_07, headers = headers )
+csv_content_df_tomos_1a28_07 = response_df_tomos_1a28_07.text
+df_tomos_1a28_07 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_07 ))
 
-def buscar( busqueda, datos, n_resultados):
-    #
-    busqueda_embed = get_embedding( busqueda, "text-embedding-ada-002" )
-    #
-    datos['Similitud'] = datos['Embedding'].apply( lambda x: cosine_similarity( eval( x ) , busqueda_embed ) )
-    #
-    datos = datos.sort_values('Similitud', ascending = False )
-    #
-    return datos.iloc[:n_resultados][['Documento', 'Pagina', 'Textos', 'NumPalabras', 'Embedding', 'Similitud']]
+#
+url_df_tomos_1a28_08 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_08.csv'
+response_df_tomos_1a28_08 = requests.get( url_df_tomos_1a28_08, headers = headers )
+csv_content_df_tomos_1a28_08 = response_df_tomos_1a28_08.text
+df_tomos_1a28_08 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_08 ))
 
 #
-def Chat( user_message_1 ):
-    #
-    df_final = buscar( user_message_1, df_Expediente, 20).reset_index( drop = True)
+url_df_tomos_1a28_09 = 'https://api.github.com/repos/benjov/Data_Text_WM/contents/df/pages/01.%20Tomos%20Reservados/df_tomos_1a28_09.csv'
+response_df_tomos_1a28_09 = requests.get( url_df_tomos_1a28_09, headers = headers )
+csv_content_df_tomos_1a28_09 = response_df_tomos_1a28_09.text
+df_tomos_1a28_09 = pd.read_csv(StringIO( csv_content_df_tomos_1a28_09 ))
+
+#
+df_tomos_1a28 = pd.concat([df_tomos_1a28_01, df_tomos_1a28_02], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_03], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_04], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_05], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_06], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_07], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_08], ignore_index = True)
+df_tomos_1a28 = pd.concat([df_tomos_1a28, df_tomos_1a28_09], ignore_index = True)
+
+#---------------------------------------------------------------------------------------------------------------
+#     UUUUPS LA COLUMNA EMBEDDINGS NO LA RECONOCE COSINESIMILARITY.. [tomos_conf_DPR, tomos_conf_cita]
+#---------------------------------------------------------------------------------------------------------------
+def clean_and_parse_embedding(embedding_str):
+    embedding_str = embedding_str.split('[')[-1].split(']')[0]
+    embedding_list = ast.literal_eval(embedding_str)
+    return [float(val) for val in embedding_list]
+
+tomos_conf_DPR['Embedding'] = tomos_conf_DPR['Embedding'].apply(clean_and_parse_embedding)
+tomos_conf_cita['Embedding'] = tomos_conf_cita['Embedding'].apply(clean_and_parse_embedding)
+
+#---------------------------------------------------------------------------------------------------------------
+#     UUUUPS LA COLUMNA EMBEDDINGS NO LA RECONOCE COSINESIMILARITY.. [df_tomos_1a28]
+#---------------------------------------------------------------------------------------------------------------
+def parse_embedding(embedding_str):
+    embedding_list = ast.literal_eval(embedding_str)
+    return [float(val) for val in embedding_list]
+
+df_tomos_1a28['Embedding'] = df_tomos_1a28['Embedding'].apply(parse_embedding)
+
+#---------------------------------------------------------------------------------------------------------------
+#     LISTA DE DF
+#---------------------------------------------------------------------------------------------------------------
+list_of_dfs = [tomos_conf_DPR, tomos_conf_cita, df_tomos_1a28] 
+
+#--------------------------------------------------------------------
+#     HACEMOS UNA PREGUNTA Y RANKEA CHUNKS
+#--------------------------------------------------------------------
+def buscar(busqueda, lista_de_datos):
+    resultados = []  # Create an empty list to store individual DataFrame results
+    busqueda_embed = get_embedding(busqueda, engine="text-embedding-ada-002")
+
+    for datos in lista_de_datos:
+        datos["similitud"] = datos['Embedding'].apply(lambda x: cosine_similarity(x, busqueda_embed))
+        datos = datos.sort_values("similitud", ascending=False)
+        resultados.append(datos[['PDFName', 'PageNumber', 'similitud', "PageText", "Folder"]])
+
+    # Concatenate all individual DataFrames into a single DataFrame
+    combined_result = pd.concat(resultados).sort_values("similitud", ascending=False).head(20)
+    return combined_result
+
+#--------------------------------------------------------------------
+#     rank for ai
+#--------------------------------------------------------------------
+def buscar_ai(busqueda, lista_de_datos):
+    resultados = []  # Create an empty list to store individual DataFrame results
+    busqueda_embed = get_embedding(busqueda, engine="text-embedding-ada-002")
+
+    for datos in lista_de_datos:
+        datos["similitud"] = datos['Embedding'].apply(lambda x: cosine_similarity(x, busqueda_embed))
+        datos = datos.sort_values("similitud", ascending=False)
+        resultados.append(datos[['PDFName', 'PageNumber', 'similitud', "PageText", "Folder"]])
+
+    # Concatenate all individual DataFrames into a single DataFrame
+    combined_result = pd.concat(resultados).sort_values("similitud", ascending=False).head(10)
+    return combined_result
+
+#--------------------------------------------------------------------
+#    saque n extraactos de ""
+#--------------------------------------------------------------------
+def count_text_extracted(pregunta):
+    df = buscar(pregunta, list_of_dfs)
+    pdf_counts = df.groupby(['Folder', 'PDFName'])['PageNumber'].count().reset_index()
+    
+    output_string = ""
+    for idx, row in pdf_counts.iterrows():
+        folder_name = row['Folder']
+        pdf_name = row['PDFName']
+        count = row['PageNumber']
+        page_numbers = df[(df['PDFName'] == pdf_name) & (df['Folder'] == folder_name)]['PageNumber'].tolist()
+        page_numbers_str = ', '.join(map(str, page_numbers))
+        output_string += f"Usé el archivo '{pdf_name}' del folder '{folder_name}' {count} (vez/veces) al extraer el texto de las páginas {page_numbers_str}.\n\n"
+    
+    return output_string
+#--------------------------------------------------------------------
+#    file: texto
+#--------------------------------------------------------------------
+
+def print_pdf_info(pregunta):
+    df = buscar(pregunta, list_of_dfs)
+    
+    output_string = ""  # Initialize an empty string to accumulate the output
+    
+    for _, row in df.iterrows():
+        pdf_name = row['PDFName']
+        page_number = row['PageNumber']
+        page_text = row['PageText']
+        
+        # Split page_text into lines and add a tab to each line
+        indented_page_text = '\n'.join(['\t' + line for line in page_text.split('\n')])
+        
+        # Append the formatted output to the output string
+        output_string += f'De "{pdf_name}":\n \tPágina {page_number}:\n\t ------------------------------------------------------------------------------------------------------------------------------------\n{indented_page_text}\n------------------------------------------------------------------------------------------------------------------------------------\n'
+    
+    return output_string
+#--------------------------------------------------------------------
+#    vector -> document
+#-------------------------------------------------------------------
+def vector_document(dataframe):
+    string_vectors = dataframe["PageText"]
+    documents = [Document(page_content=content, metadata={'id': i}) for i, content in enumerate(string_vectors)]
+    return documents
+
+#--------------------------------------------------------------------
+#    AI
+#-------------------------------------------------------------------
+def get_completion_from_messages( messages, model = "gpt-3.5-turbo-16k", 
+                                  temperature = 0, max_tokens = 4500 ): ##Check max_tokens 
+    response = openai.ChatCompletion.create(
+               model = model,
+               messages = messages,
+               temperature = temperature, 
+               max_tokens = max_tokens, 
+    )
+    return response.choices[0].message["content"]
+
+def get_topic( user_message ):
     #
-    Textos = df_final[ ['Documento', 'Pagina', 'Textos'] ]
+    delimiter = "####"
+    system_message = f"""
+    Eres un abogado que trabaja en temas de competencia económica e investiga casos en México.
+    Siempre intenarás responder en el mayor número posible de palabras.
+    Las consultas o preguntas se delimitarán con los caracteres {delimiter}
+    """
     #
-    Textos_Concatenados = '\n\n\n'.join( Textos.apply( lambda row: ' | '.join( row ), axis = 1) )
+    messages =  [  
+    {'role':'system', 
+     'content': system_message},    
+    {'role':'user', 
+     'content': f"{delimiter}{user_message}{delimiter}"},  
+    ] 
+    return get_completion_from_messages( messages )
+
+def get_respuesta( user_message, informacion):
     #
-    # Save Question and date time
-    #update_records( user_message_1 )
-    return Textos_Concatenados
-# 
-# 
-with gr.Blocks() as demo:
+    delimiter = "####"
+    system_message = f"""
+    Eres un abogado que trabaja en temas de competencia económica e investiga casos en México.
+    Siempre intenarás responder en el mayor número posible de palabras.
+    Las consultas o preguntas se delimitarán con los caracteres {delimiter}
+
+    """
     #
-    gr.Markdown("App basada en servicios (Embeddings) de OpenAI (Chat GPT-3.5)")
-    gr.Markdown("Este es un producto de prueba desarrollado por GAMES Economics (https://gamesecon.com/)")
-    gr.Markdown("Contacto: boliva@gamesecon.com")
-    gr.Markdown("Este es un information retrieval system o sistema de recuperación de información.")
-    gr.Markdown("Este tipo de herramientas son un proceso para obtener información relevante \
-                 para una necesidad de información a partir de una colección de textos.")
-    gr.Markdown("Las búsquedas pueden basarse en un texto completo, una frase o una serie de conceptos.")
-    gr.Markdown("No obstante, cuanto más detallada sea la sentencia buscada, la recuperación de \
-                información será más precisa.")
-    gr.Markdown("En alguna medida, los motores de búsqueda web (como Google) son ejemplos de estos sistemas.")
-    busqueda = gr.Textbox(label = "Escribe la pregunta, tema o enunciado.")
-    greet_btn = gr.Button("Preguntar")
-    # Crear dos widgets de salida en lugar de uno
-    output1 = gr.Textbox(label = "El Top 20 de los extractos de documentos que coinciden con la consulta son:")
-
-    # Modificar la función click para asignar cada parte de la tupla a un widget diferente
-    greet_btn.click(fn = Chat, inputs=[busqueda], outputs=[output1])
+    messages =  [  
+    {'role':'system',
+     'content': system_message},   
+    {'role':'user',
+     'content': f"""
+    {delimiter} 
+    Estás intentando recopilar información relevante para tu caso.
+    Usa exclusivamente la información contenida en la siguiente lista: 
+    {informacion} 
+
+    para responder sin límite de palabras lo siguiente: {user_message}
+    Responde de forma detallada.
+    {delimiter}
+    """},   
+    ]
     #
-#demo.launch( share = True )
-demo.launch( )
-#
-#
+    return get_completion_from_messages(messages)
+
+def chat(user_message_1):
+    norma_y_tema_response_1 = get_topic(user_message_1)
+    norma_y_tema_response_1 = norma_y_tema_response_1  # No need to use eval
+    norma_y_tema_response_1 += 'Todos'
+    uno = buscar_ai(user_message_1, list_of_dfs)
+    lista_info = uno[ 'PageText' ].tolist()
+    return get_respuesta(user_message_1, lista_info)
+
+
+#.....................................................................
+#--------------------------------------------------------------------
+#     HACEMOS UNA PREGUNTA Y GENERA UNA RESPUESTA.AI
+#--------------------------------------------------------------------
+#.....................................................................
+llm_mio = OpenAI(openai_api_key=api_key, model_name="gpt-3.5-turbo-16k")
+#RANKING _LO STESSO COME SOPRA_
+def load(query):
+    pregunta = query
+    uno = buscar(pregunta, list_of_dfs)
+    documents = vector_document(uno)
+    chain = load_qa_chain(llm=llm_mio, chain_type="stuff")
+    result1 = chain.run(input_documents=documents, question=pregunta)
+    return result1
+
+with gr.Blocks() as demo:
+    txt = gr.Textbox(label="Pregunta", lines=2)
+    btn = gr.Button(value="Listo")
+    txt_2 = gr.Textbox(value="", label="Donde:")
+    txt_3 = gr.Textbox(value="", label="Cuales:")
+    txt_1 = gr.Textbox(value="", label="Respuesta IA:")
+    btn.click(load, inputs=[txt], outputs=[txt_1])
+    btn.click(count_text_extracted, inputs=[txt], outputs=[txt_2])
+    btn.click(print_pdf_info, inputs=[txt], outputs=[txt_3])
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -1,11 +1,9 @@
+requests==2.28.1
 gradio==3.27.0
 gradio-client==0.1.3
-gspread==5.10.0
-langchain==0.0.163
-oauth2client==4.1.3
-openai==0.27.4
 pandas==2.0.3
-plotly==5.10.0
-regex==2023.3.23
-scikit-learn==1.1.3
-scipy==1.9.3
+numpy==1.25.2
+openai==0.27.4
+tiktoken==0.4.0
+langchain==0.0.163
+