from openai import OpenAI import pandas as pd import psycopg2 import time import gradio as gr import sqlparse import re import os import warnings from persistStorage import saveLog, getAllLogFilesPaths from config import * from constants import * from utils import * from gptManager import ChatgptManager from queryHelperManager import QueryHelper logsDir = os.getenv("HF_HOME", "/data") pd.set_option('display.max_columns', None) pd.set_option('display.max_rows', 10) # Filter out all warning messages warnings.filterwarnings("ignore") dbCreds = DataWrapper(DB_CREDS_DATA) dbEngine = DbEngine(dbCreds) tablesAndCols = getAllTablesInfo(dbEngine, SCHEMA_NAME) metadataLayout = MetaDataLayout(schemaName=SCHEMA_NAME, allTablesAndCols=tablesAndCols) metadataLayout.setSelection(DEFAULT_TABLES_COLS) selectedTablesAndCols = metadataLayout.getSelectedTablesAndCols() openAIClient = OpenAI(api_key=OPENAI_API_KEY) gptInstance = ChatgptManager(openAIClient, model=GPT_MODEL) queryHelper = QueryHelper(gptInstance=gptInstance, schemaName=SCHEMA_NAME,platform=PLATFORM, metadataLayout=metadataLayout, sampleDataRows=SAMPLE_ROW_MAX, gptSampleRows=GPT_SAMPLE_ROWS, dbEngine=dbEngine, getSampleDataForTablesAndCols=getSampleDataForTablesAndCols) def checkAuth(username, password): global ADMIN, PASSWD if username == ADMIN and password == PASSWD: return True return False # Function to save history of chat def respond(message, chatHistory): """gpt response handler for gradio ui""" global queryHelper try: if "modify" in message: botMessage, prospectTablesAndCols = queryHelper.getQueryForUserInput(message, chatHistory) else: botMessage, prospectTablesAndCols = queryHelper.getQueryForUserInput(message, []) except Exception as e: errorMessage = {"function":"queryHelper.getQueryForUserInput","error":str(e), "userInput":message} saveLog(errorMessage, 'error') raise ValueError(str(e)) queryGenerated = extractSqlFromGptResponse(botMessage) logMessage = {"userInput":message, "tablesColsSelectedByGpt":str(prospectTablesAndCols) , "queryGenerated":queryGenerated, "completeGptResponse":botMessage} saveLog(logMessage) chatHistory.append((message, botMessage)) time.sleep(2) return "", chatHistory def preProcessSQL(sql): sql=sql.replace(';', '') disclaimerOutputStripping = "" if ('limit' in sql[-15:].lower())==False: sql = sql + ' ' + 'limit 5' disclaimerOutputStripping = """Results are stripped to show only top 5 rows. Please add your custom limit to get extend result. eg\n select * from schema.table limit 20\n\n""" # sql = str(sql) # sql = sqlparse.format(sql, reindent=True, keyword_case='upper') return sql, disclaimerOutputStripping def testSQL(sql): global dbEngine, queryHelper sql, disclaimerOutputStripping = preProcessSQL(sql=sql) if not isDataQuery(sql): return "Sorry not allowed to run. As the query modifies the data." try: dbEngine.connect() conn = dbEngine.getConnection() df = pd.read_sql_query(sql, con=conn) return disclaimerOutputStripping + str(pd.DataFrame(df)) except Exception as e: errorMessage = {"function":"testSQL","error":str(e), "userInput":sql} saveLog(errorMessage, 'error') print(f"Error occured during running the query {sql}.\n and the error is {str(e)}") # prompt = f"Please correct the following sql query, also it has to be run on {PLATFORM}. sql query is \n {sql}. the error occured is {str(e)}." # modifiedSql = queryHelper.modifySqlQueryEnteredByUser(prompt) # logMessage = {"function":"queryHelper.modifySqlQueryEnteredByUser", "sqlQuery":sql, "modifiedSQLQuery":modifiedSql} # saveLog(logMessage, 'info') return f"The query you entered throws some error. Here is the error. Please try different query\n {str(e)}" def onSelectedTablesChange(tablesSelected): #Updates tables visible and allow selecting columns for them global queryHelper print(f"Selected tables : {tablesSelected}") metadataLayout = queryHelper.getMetadata() allTablesAndCols = metadataLayout.getAllTablesCols() selectedTablesAndCols = metadataLayout.getSelectedTablesAndCols() allTablesList = list(allTablesAndCols.keys()) tableBoxes = [] for i in range(len(allTablesList)): if allTablesList[i] in tablesSelected: dd = gr.Dropdown( allTablesAndCols[allTablesList[i]],visible=True,value=selectedTablesAndCols.get(allTablesList[i],None), multiselect=True, label=allTablesList[i], info="Select columns of a table" ) tableBoxes.append(dd) else: dd = gr.Dropdown( allTablesAndCols[allTablesList[i]],visible=False,value=selectedTablesAndCols.get(allTablesList[i],None), multiselect=True, label=allTablesList[i], info="Select columns of a table" ) tableBoxes.append(dd) return tableBoxes def onSelectedColumnsChange(*tableBoxes): #update selection of columns and tables (include new tables and cols in gpts context) global queryHelper metadataLayout = queryHelper.getMetadata() allTablesAndCols = metadataLayout.getAllTablesCols() allTablesList = list(allTablesAndCols.keys()) tablesAndCols = {} result = '' print("Getting selected tables and columns from gradio") for tableBox, table in zip(tableBoxes, allTablesList): if isinstance(tableBox, list): if len(tableBox)!=0: tablesAndCols[table] = tableBox else: pass metadataLayout.setSelection(tablesAndCols=tablesAndCols) print("metadata updated") print("Updating queryHelper state, and sample data") queryHelper.updateMetadata(metadataLayout) return "Columns udpated" def onResetToDefaultSelection(): global queryHelper metadataLayout = queryHelper.getMetadata() metadataLayout.setSelection(tablesAndCols=tablesAndCols) queryHelper.updateMetadata(metadataLayout) metadataLayout = queryHelper.getMetadata() allTablesAndCols = metadataLayout.getAllTablesCols() selectedTablesAndCols = metadataLayout.getSelectedTablesAndCols() allTablesList = list(allTablesAndCols.keys()) tableBoxes = [] for i in range(len(allTablesList)): if allTablesList[i] in selectedTablesAndCols.keys(): dd = gr.Dropdown( allTablesAndCols[allTablesList[i]],visible=True,value=selectedTablesAndCols.get(allTablesList[i],None), multiselect=True, label=allTablesList[i], info="Select columns of a table" ) tableBoxes.append(dd) else: dd = gr.Dropdown( allTablesAndCols[allTablesList[i]],visible=False,value=selectedTablesAndCols.get(allTablesList[i],None), multiselect=True, label=allTablesList[i], info="Select columns of a table" ) tableBoxes.append(dd) return tableBoxes def onSyncLogsWithDataDir(): downloadableFilesPaths = getAllLogFilesPaths() fileComponent = gr.File(downloadableFilesPaths, file_count='multiple') return fileComponent with gr.Blocks() as demo: # screen 1 : Chatbot for question answering to generate sql query from user input in english with gr.Tab("Query Helper"): gr.Markdown("""