connect backend with frontend

src/chatbot/app_ui.py

@@ -1,84 +0,0 @@
-import gradio as gr
-
-def respond(message, history, image):
-    history = history or []
-    if message:
-        history.append({"role": "user", "content": message})
-        bot_reply = f"Echo: {message}"
-    if image:
-        history.append({"role": "user", "content": gr.Image(value=image, interactive=False)})
-        bot_reply = f"Echo: {image.name}"
-        
-    history.append({"role": "assistant", "content": bot_reply})
-    return history, "", None
-
-def upload_image(image, chat_history):
-    return respond("", chat_history, image)
-
-def submit_form(bill_date, amount, account, description):
-    return {
-        "วันที่ของบิล": bill_date,
-        "ค่าใช้จ่ายทั้งหมด (บาท)": amount,
-        "บัญชีที่เกี่ยวข้อง": account,
-        "คำอธิบายเพิ่มเติม": description
-    }
-
-with gr.Blocks() as demo:
-    new_btn = gr.Button("เริ่มใหม่", variant="primary")
-    new_btn.click(lambda: None, [], [])
-
-    with gr.Tabs():
-        # Chat tab
-        with gr.Tab("แชท"):
-            chatbot    = gr.Chatbot(
-                label="Chat History",
-                height=400,
-                type="messages",
-                value=[]
-            )
-            with gr.Row():
-                img_upload = gr.UploadButton(label="📎", file_types=["image"])
-                user_input = gr.Textbox(placeholder="พิมพ์ข้อความ...", lines=1)
-                send_btn   = gr.Button("+")
-
-            # Auto‐send the image and echo its filename, then clear the uploader
-            img_upload.upload(
-                fn=upload_image,
-                inputs=[img_upload, chatbot],
-                outputs=[chatbot, user_input, img_upload]
-            )
-
-            # Manual send for text
-            send_btn.click(
-                fn=respond,
-                inputs=[user_input, chatbot, img_upload],
-                outputs=[chatbot, user_input, img_upload]
-            )
-
-        # Form tab
-        with gr.Tab("ฟอร์ม"):
-            gr.Markdown("**วันที่ของบิล**")
-            dt_input = gr.DateTime(
-                value="now",
-                include_time=True,
-                type="datetime",
-                show_label=False
-            )
-            gr.Markdown("**ค่าใช้จ่ายทั้งหมด**")
-            with gr.Row():
-                amount = gr.Number(label="", container=False)
-                gr.Markdown("บาท")
-            gr.Markdown("**บัญชีที่เกี่ยวข้อง**")
-            account = gr.Textbox(label="", container=False)
-            gr.Markdown("**คำอธิบายเพิ่มเติม**")
-            description = gr.Textbox(label="", container=False, lines=5)
-            submit_btn = gr.Button("Submit", variant="primary")
-            output = gr.JSON(label="ผลลัพธ์")
-            submit_btn.click(
-                fn=submit_form,
-                inputs=[dt_input, amount, account, description],
-                outputs=output
-            )
-
-if __name__ == "__main__":
-    demo.launch()

src/chatbot/graph_manager.py

@@ -69,6 +69,10 @@ class GraphManager:
         snapshot = self.graph.get_state(config)
         return snapshot[0]["messages"][-1].content
     
+def user_input_handler(user_input, session_id):
+    response = graph_manager.process_user_input(user_input, session_id)
+    return response
+
 graph_manager = GraphManager()
 graph_manager.save_graph_image()
 
@@ -78,10 +82,10 @@ graph_manager.save_graph_image()
 #     1
 # )
 
-response = graph_manager.process_user_input(
-    r"C:\Users\Gavin\Desktop\hello-earth\data\jpeg_raw_data\bill\กองทุนพัฒนาบ้านห้วยทราย อ.แม่ออน จ.เชียงใหม่_page-0005.jpg",
-    1
-)
+# response = graph_manager.process_user_input(
+#     r"C:\Users\Gavin\Desktop\hello-earth\data\jpeg_raw_data\bill\กองทุนพัฒนาบ้านห้วยทราย อ.แม่ออน จ.เชียงใหม่_page-0005.jpg",
+#     1
+# )
 
 # response = graph_manager.process_user_input(
 #     r"C:\Users\Gavin\Desktop\hello-earth\data\jpeg_raw_data\bill\กองทุนพัฒนาบ้านห้วยทราย อ.แม่ออน จ.เชียงใหม่_page-0004.jpg",

src/chatbot/kie.py

@@ -8,6 +8,7 @@ from dotenv import load_dotenv
 import os
 # from prompts import kie_prompt
 from prompts.kie_prompts import kie_prompt
+from prompts.extract_form_information_prompts import extract_form_information_prompt
 import ast
 # Load environment variables from .env file
 load_dotenv()
@@ -51,8 +52,8 @@ def receipt_kie(image_path: str) -> str:
         temp_dict = temp_dict.replace("null", "None")
         # print(temp_dict)
         # print(type(temp_dict))
-
-        res_dict = ast.literal_eval(temp_dict)
+        res_dict = extract_form_information(client, temp_dict)
+        # res_dict = ast.literal_eval(format_dict)
     except Exception as e:
         print(f"Error: {e}")
         res_dict = {
@@ -70,6 +71,74 @@ def receipt_kie(image_path: str) -> str:
         }
     return res_dict
 
+def extract_form_information(client, receipt_text, max_retries=100):
+
+    json_schema = {
+        "type": "object",
+        "properties": {
+            "Receipt Title": {"type": "string"},
+            "Project Name": {"type": "string"},
+            "Date": {"type": "string"},
+            "Receipt Number": {"type": "string"},
+            "Payer's Name": {"type": "string"},
+            "Payment Details Table": {
+                "type": "object",
+                "properties": {
+                    "Column Headers": {
+                        "type": "array",
+                        "items": {"type": "string"}
+                    },
+                    "Row Entries": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "Description": {"type": "string"},
+                                "Quantity": {"type": "string"},
+                                "Unit Price": {"type": "number"},
+                                "Total Amount": {"type": "number"}
+                            },
+                            "required": ["Description", "Quantity", "Unit Price", "Total Amount"]
+                        }
+                    }
+                },
+                "required": ["Column Headers", "Row Entries"]
+            },
+            "Total Payment Amount": {"type": "number"},
+            "Incompleteness Description": {"type": "string"},
+            "Expense Justification": {"type": "string"}
+        },
+    }
+
+    for attempt in range(1, max_retries + 1):
+        model = "gpt-4.1-mini"
+
+        response = client.chat.completions.create(
+            model=model,
+            messages=[
+                {
+                    "role": "system",
+                    "content": "You are a financial document analyst. Extract structured receipt data as a Python dictionary."
+                },
+                {
+                    "role": "user",
+                    "content": extract_form_information_prompt.format(receipt_text)
+                }
+            ],
+            response_format={"type": "json_schema", "json_schema": {"name": "form_info", "schema": json_schema}}
+        )
+
+        content = response.choices[0].message.content
+
+        try:
+            print("content: ", content)
+            content = ast.literal_eval(content)
+            return content
+        except:
+            pass
+
+    raise ValueError("Failed to extract valid form info from receipt after multiple retries.")
+
 # # test case
 # res = receipt_kie(r"C:\Users\Gavin\Desktop\hello-earth\data\jpeg_raw_data\bill\กองทุนพัฒนาบ้านห้วยทราย อ.แม่ออน จ.เชียงใหม่_page-0004.jpg")
 # # res = ast.literal_eval(res)

src/chatbot/nodes.py

@@ -32,6 +32,11 @@ form_info = {
         "Expense Justification": 'None',
     }
 
+def get_form_info():
+    return form_info
+
+def set_form_info(key, value):
+    form_info[key] = value
 
 class State(TypedDict):
     messages: Annotated[list, add_messages]