Update app.py

agent inputs looping

app.py

@@ -17,14 +17,16 @@ from urllib.parse import quote
 from multiprocessing import context
 # from agents.tool import WebSearchTool
 from typing_extensions import TypedDict, Any
-from agents import Agent, FunctionTool, ImageGenerationTool, RunContextWrapper,  Runner, function_tool, SQLiteSession
+from agents import Agent, FunctionTool, ImageGenerationTool, RunContextWrapper,  Runner, function_tool, CodeInterpreterTool
+from openai import OpenAI
 from markdown_pdf import MarkdownPdf
 from markdown_pdf import Section
+from docx import Document
 import brave
 import geo_distance
 
 load_dotenv(override=True)
-# key = os.getenv('OPENAI_API_KEY')
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
 users = os.getenv('LOGNAME')
 unames = users.split(',')
 pwds = os.getenv('PASSWORD')
@@ -45,6 +47,9 @@ LOCATIONID_KEY=os.getenv('LOCATIONID_KEY')
 #     dp.mkdir(exist_ok=True)
 #     dataDir = '/data/'
 
+# mimes = {'pdf': 'application/pdf', 'txt': 'text/plain',
+#              'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'}
+
 @function_tool
 async def make_pdf(text: str, title: str)->str:
     '''Creates a pdf document based on input markdown text and title string.
@@ -109,6 +114,12 @@ async def get_distance(addr1: str, addr2: str)->float:
     distance = geo_distance.great_circle_distance_miles(lat1, lon1, lat2, lon2)
     return distance
 
+def extract_text_from_docx(file_path):
+    doc = Document(file_path)
+    text = []
+    for paragraph in doc.paragraphs:
+        text.append(paragraph.text)
+    return "\n".join(text)
 
 def md(txt):
     return str(txt).replace('```', ' ').replace('  ', '&nbsp;&nbsp;').replace('  ', '&nbsp;&nbsp;').replace('  ', '&nbsp;&nbsp;').replace('\n','<br>').replace('~~','~')
@@ -136,11 +147,17 @@ def updatePassword(pwd, user):
         pwd = pwd.lower().strip()
         if pwd == pwd_list[unames.index(user)]:
             password = pwd
-            return [password, "*********"]
+            return [password, "*********",
+                    gr.Button(value='Upload Input File', interactive=True),
+                    gr.Button(value='Upload Image to Analyze', interactive=True)]
         else:
-            return [password, "invalid password"]
+            return [password, "invalid password",
+                gr.Button(value='Upload Input File', interactive=False),
+                gr.Button(value='Upload Image to Analyze', interactive=False)]
     else:
-        return [password, "invalid user"]
+        return [password, "invalid user",
+                gr.Button(value='Upload Input File', interactive=False),
+                gr.Button(value='Upload Image to Analyze', interactive=False)]
 
 def update_user(user_win):
     user_win = user_win.lower().strip()
@@ -155,7 +172,7 @@ def credentials_ok(user, pwd):
     return user in unames and pwd in pwd_list
 
 def clean_up_files():
-    for file in glob('./document.pdf'):
+    for file in glob('./document.*'):
         try:
             os.remove(file)
         except:
@@ -166,31 +183,77 @@ def clean_up_files():
         except:
             pass
 
-def load_image(image, user):
-    status = 'OK, image is ready! Enter prompt and tap submit button'
+def load_image(image, user, output_window):
+    # status = #'OK, image is ready! Enter prompt and tap submit button'
     try:
         with open(image, 'rb') as image_file:
             base64_image = base64.b64encode(image_file.read()).decode('utf-8')
         fpath = user + '_image.b64'
         with open(fpath, 'wt') as fp:
             fp.write(base64_image)
+            output_window += md('\nImage loaded\n')
     except:
-        status = 'Unable to upload image'
-    return [fpath, status]
+        output_window = 'Unable to upload image'
+    return [fpath, output_window]
 
-def upload_image(user, password):
+def upload_image(user, password, output_window):
     if not credentials_ok(user, password):
         return [gr.Image(visible=False, interactive=True), "Incorrect user name and/or password"]
-    return [gr.Image(visible=True, interactive=True), '']
+    return [gr.Image(visible=True, interactive=True), output_window]
+
+def upload_file(user, password, output_window):
+    if not credentials_ok(user, password):
+        return [gr.File(visible=False, label='Upload File'), 'Incorrect user and/or password']
+    return [gr.File(visible=True, label='UploadFile'), output_window]
+
+def load_file(file_uploader, output_window):
+    path = file_uploader
+    fname = os.path.basename(path)
+    return [path, output_window + f'<BR>{fname} loaded<BR>',
+           gr.File(visible=False, label='Upload File', type='filepath', value=None) ]
+
+def create_openai_container(name):
+    url = 'https://api.openai.com/v1/containers'
+    headers= {"Authorization": "Bearer " + OPENAI_API_KEY, "Content-Type": "application/json",}
+    json_data = {"name": name}
+
+    response = requests.post(
+        url,
+        headers=headers,
+        json=json_data
+        )
+
+    return json.loads(response.content)["id"]
+
+def get_openai_file(file_id, container_id):
+    url = f'https://api.openai.com/v1/containers/{container_id}/files/{file_id}/content'
+    headers= {"Authorization": "Bearer " + OPENAI_API_KEY}
+
+    response = requests.get(
+        url,
+        headers=headers
+        )
+    return response
+
+def list_openai_container_files(container_id):
+    url = f'https://api.openai.com/v1/containers/{container_id}/files'
+    headers= {"Authorization": "Bearer " + OPENAI_API_KEY}
+
+    response = requests.get(
+        url,
+        headers=headers
+        )
+    return response
+
 
 async def chat(prompt_window, user_window, password, history, output_window,
-              uploaded_image_file):
+              uploaded_image_file, uploaded_file_path, prior_inputs):
     file_download = gr.DownloadButton(label='Download File', visible=False, value=None)
     image_window = gr.Image(visible=False, value=None)
 
     if not credentials_ok(user_window, password):
         return ['Invalid Credentials', prompt_window, uploaded_image_file,
-                     image_window, file_download, history]
+                     image_window, file_download, history, uploaded_file_path, prior_inputs]
     instructions = '''
     You are a helpful assistant.
     You can call tools to compute straight-line distances and to search the web for
@@ -200,17 +263,58 @@ async def chat(prompt_window, user_window, password, history, output_window,
     straight-line distance by default, and when possible use street addresses for locations.
     '''
 
+
+    code_container = create_openai_container('my_container')
+
     agent = Agent(name="Assistant",
                  instructions=instructions,
-                 tools=[get_distance, search_web, get_news, make_pdf,
+                 tools=[get_distance, search_web, get_news,
+            CodeInterpreterTool(tool_config={"type": "code_interpreter","container": code_container}),     # make_pdf,
               ImageGenerationTool(tool_config={"type": "image_generation", "quality": "low"},)],)
 
     response = output_window
     if not response:
         response = ''
     prompt = prompt_window
-    history.append({"role":"user", "content":prompt})
-    inputs = history
+    # inputs = history.copy()
+    inputs = prior_inputs
+    file_input = ''
+    if uploaded_file_path != '':
+        ext = uploaded_file_path.casefold().split('.')[-1]
+        if ext == 'pdf':
+            client = OpenAI(api_key = OPENAI_API_KEY)
+            file = client.files.create(file=open(f'{uploaded_file_path}','rb'),
+                                      purpose='user_data',
+                                       expires_after={"seconds": 3600, "anchor": "created_at"})
+            file_input=(
+                {"role": "user",
+                "content": [
+                    {
+                        "type": "input_file",
+                        "file_id": file.id,
+                    }
+                ]
+                }
+            )
+            inputs.append(file_input)
+        if ext in ['docx', 'txt', 'py']:
+            if ext == 'docx':
+                extracted_text = extract_text_from_docx(uploaded_file_path)
+            else:
+                with open(uploaded_file_path, 'rt') as fp:
+                    extracted_text = fp.read()
+            file_input=(
+                {"role": "user",
+                "content": [
+                    {
+                        "type": "input_text",
+                        "text": f"{extracted_text}",
+                        }
+                    ]
+                }
+            )
+            inputs.append(file_input)
+        uploaded_file_path == ''
     image_input = ''
     if uploaded_image_file != '':
         with open(uploaded_image_file, 'rt') as fp:
@@ -229,42 +333,110 @@ async def chat(prompt_window, user_window, password, history, output_window,
             }
         )
         inputs.append(image_input)
-    result = await Runner.run(agent, max_turns=20,
-            input=inputs)
-    for item in result.new_items:
-        if (
-            item.type == "tool_call_item"
-            and item.raw_item.type == "image_generation_call"
-            and (img_result := item.raw_item.result)
-        ):
-            # with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
-            image_out_path = f'{user_window}_out.png'
-            with open(image_out_path,'wb') as fp:
-                fp.write(base64.b64decode(img_result))
-            image_window = gr.Image(visible=True, value=image_out_path)
-
-    history.append({"role":"assistant", "content":result.final_output})
-
-    reply = md(result.final_output)
-    response += "\n\n***YOU***: " + prompt + "\n\n***GPT***: " +  reply.replace('```','\n\n```\n\n')
-    usage = result.context_wrapper.usage
-    response += f"\nTotal tokens: = {usage.total_tokens}"
-    pdf_list = glob('./document.pdf')
-    if len(pdf_list):
-        file_download = gr.DownloadButton(label='Download PDF Doc',
-                                         visible=True, value='./document.pdf')
-    return [response, '', uploaded_image_file, image_window, file_download, history]
+    history.append({"role":"user", "content":prompt})
+    inputs.append({"role":"user", "content":prompt})
+    exception_msg = ''
+    result = None
+    try:
+        result = await Runner.run(agent, max_turns=20,
+                input=inputs)
+        for item in result.new_items:
+            if (
+                item.type == "tool_call_item"
+                and item.raw_item.type == "image_generation_call"
+                and (img_result := item.raw_item.result)
+            ):
+                image_out_path = f'{user_window}_out.png'
+                with open(image_out_path,'wb') as fp:
+                    fp.write(base64.b64decode(img_result))
+                image_window = gr.Image(visible=True, value=image_out_path)
+
+        reply = md(result.final_output)
+        response += "\n\n***YOU***: " + prompt + "\n\n***GPT***: " +  reply.replace('```','\n\n```\n\n')
+        history.append({"role":"assistant", "content":result.final_output})
+    except Exception as e:
+        exception_msg = f'Error: {e.message}' 
+        response += "\n\n***YOU***: " + prompt + "\n\n***GPT***: " + exception_msg
+    if result:
+        new_inputs = result.to_input_list()
+        usage = result.context_wrapper.usage
+        response += f"\nTotal tokens: = {usage.total_tokens}"
+        loc = result.final_output.find('/mnt/data/')
+        if loc > -1:
+            ts = result.final_output[loc+10:]
+            loc2 = ts.find(')')
+            if loc2 > -1:
+                fname = ts[:loc2]
+                container_list = list_openai_container_files(code_container)
+                file_list_json = json.loads(container_list.content)
+                latest_file_time = 0
+                download_file_id = None
+                download_ext = ''
+                for item in file_list_json['data']:
+                    if fname in item["path"]:
+                        file_time = item["created_at"]
+                        if file_time > latest_file_time:
+                            latest_file_time = file_time
+                            download_file_id = item["id"]
+                            download_ext = fname.split('.')[-1].casefold()
+                if download_file_id:
+                    fdata = get_openai_file(download_file_id, code_container).content
+                    with open(f'./document.{download_ext}', 'wb') as fp:
+                        fp.write(fdata)
+                        file_download = gr.DownloadButton(label=f'Download {download_ext.upper()} Doc',
+                                        visible=True, value=f'./document.{download_ext}')
+    return [response, '', uploaded_image_file, image_window, file_download, history,
+            uploaded_file_path, new_inputs]
 
 # outputs=[history, output_window, prompt_window, uploaded_image_file,
 #                      image_window, file_download])
 
+def show_help():
+    txt = '''
+    This is an agent using the OpenAI Python Agents SDK.  
+    It has tools to:
+        * Search the Web
+        * Compute straight-line distances between locations
+        * Analyze images you upload.
+        * Create and display images you describe, which you can download
+        * Get news from the web
+        * Make PDF's based on results it generated.
+    Agents perform multiple steps using tools as necessary to satisfy a single request.
+
+    1.  Gemeral:
+        1.1 Login with user name and password (not case-sensitive)
+        1.2 Type prompts (questions, instructions) into "Prompt or Question" window.
+    2.  Chat:
+        2.1 Enter prompt and tap the "Submit Prompt/Question" button.  The responses appear in the Dialog window.
+        2.2 Enter follow-up questions in the Prompt window. Tap "Submit Prompt/Question".
+        2.3 If topic changes or when done chatting, tap the "Start New Session" button.
+    3.  Make Image:
+        3.1 Include description of desired image in prompt window.
+        3.2 Tap the "Submit Prompt/Question" button. This can take a few seconds.
+        3.3 There is a download button on the image display if your system supports file downloads.
+        3.4 When done viewing image, tap the "Start New Session" button
+    4.  Analyze an Image you provide:
+        4.1 Tap the "Upload Image to Analyze" button.
+        4.2 An empty image box will appear lower left. Drag or upload image into it. It offers web cam or camera
+               input also.
+        4.3 The image should appear. This can take some time with a slow internet connection and large image.
+        4.4 Enter what you want done with the image in the "Prompt or Question" box.
+        4.5 Tap the "Submit Prompt/Question" button to start the analysis.  This initiates a chat dialog and
+               you can ask follow-up questions. However, the image is not re-analyzed for follow-up dialog.
+    Hint:
+        Better  results are obtained by including detailed descriptions and instructions
+            of what you want in the prompt.
+    '''
+    return str(txt).replace('```', ' ').replace('  ', '&nbsp;&nbsp;').replace('  ', '&nbsp;&nbsp;').replace('  ', '&nbsp;&nbsp;').replace('\n','<br>')
+
+
 def new_session(user_window, history):
     history = []
     return [prompt_window, history, 'Session cleared',
            gr.Image(visible=False, value=None),
            gr.Image(visible=False, value=None), '',
            gr.DownloadButton(label='Download File', visible=False, value=None),
-           gr.File(visible=False, label='Upload File', type='filepath')]
+           gr.File(visible=False, label='Upload File', type='filepath'), [] ]
 
 
 with gr.Blocks(theme=gr.themes.Soft()) as demo:
@@ -273,6 +445,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
     uploaded_image_file = gr.State('')
     uploaded_file_path = gr.State('')
     history = gr.State([])
+    inputs = gr.State([])
 
     gr.Markdown('# GPT Agent')
     gr.Markdown('Enter user name & password.  Tap "Help & Hints" button for more instructions.')
@@ -283,9 +456,8 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         help_button = gr.Button(value='Help & Hints')
     with gr.Row():
         clear_button = gr.Button(value="Start New Session")
-        button_do_image = gr.Button(value='Make Image')
-        button_upload_file = gr.Button(value='Upload Input File')
-        button_get_image = gr.Button(value='Upload Image to Analyze')
+        button_upload_file = gr.Button(value='Upload Input File', interactive=False)
+        button_get_image = gr.Button(value='Upload Image to Analyze', interactive=False)
         submit_button = gr.Button(value="Submit Prompt/Question")
     with gr.Row():
         prompt_window = gr.Textbox(label = "Prompt or Question", scale=7)
@@ -302,21 +474,25 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         file_uploader = gr.File(visible=False, label='Upload File', type='filepath')
     submit_button.click(chat,
              inputs=[prompt_window, user_window, password, history, output_window,
-                    uploaded_image_file],
+                    uploaded_image_file, uploaded_file_path, inputs],
              outputs=[output_window, prompt_window, uploaded_image_file,
-                     image_window, file_download, history])
+                     image_window, file_download, history, uploaded_file_path, inputs])
     clear_button.click(fn=new_session, inputs=[user_window, history],
-                      outputs=[prompt_window, history, output_window, image_window, image_window2,
-                      uploaded_image_file, file_download, file_uploader])
-    # image_window.change(fn=delete_image, inputs=[user])
-    # help_button.click(fn=show_help, outputs=output_window)
-    button_get_image.click(fn=upload_image,inputs = [user, password],
+                      outputs=[prompt_window, history, output_window,
+                      image_window, image_window2,
+                      uploaded_image_file, file_download, file_uploader, inputs])
+    help_button.click(fn=show_help, outputs=output_window)
+    button_get_image.click(fn=upload_image,inputs = [user, password, output_window],
                           outputs = [image_window2, output_window])
-    image_window2.upload(fn=load_image, inputs=[image_window2, user], outputs=[uploaded_image_file, output_window])
-    pwd_window.blur(updatePassword, inputs = [pwd_window, user], outputs = [password, pwd_window])
-    # button_upload_file.click(fn=upload_file, inputs=[user, password],
-    #                         outputs=[file_uploader, output_window])
-    # file_uploader.upload(fn=load_file, inputs=[file_uploader, user], outputs=[uploaded_file_path, output_window])
+    image_window2.upload(fn=load_image, inputs=[image_window2, user, output_window],
+                        outputs=[uploaded_image_file, output_window])
+    pwd_window.blur(updatePassword,
+                   inputs = [pwd_window, user],
+                   outputs = [password, pwd_window, button_upload_file, button_get_image])
+    button_upload_file.click(fn=upload_file, inputs=[user, password, output_window],
+                            outputs=[file_uploader, output_window])
+    file_uploader.upload(fn=load_file, inputs=[file_uploader, output_window],
+                        outputs=[uploaded_file_path, output_window, file_uploader])
 # demo.launch(share=True, allowed_paths=[dataDir], ssr_mode=False)
     # demo.load(delete_db_files)
     demo.unload(clean_up_files)