two endpoints

.gitignore

@@ -0,0 +1 @@
+.env

app.py

@@ -7,10 +7,22 @@ from PIL import Image
 import uvicorn
 import os
 import json
+import base64
 import pytesseract  # ✅ OCR
 
+try:
+    from dotenv import load_dotenv  # type: ignore
+    load_dotenv()
+except Exception:
+    # optional in production containers
+    pass
+
 from langchain_openai import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
+try:
+    from langchain_core.messages import HumanMessage
+except Exception:  # Fallback for older langchain
+    from langchain.schema import HumanMessage  # type: ignore
 
 # Init FastAPI
 app = FastAPI()
@@ -22,13 +34,25 @@ os.makedirs("receiptimages", exist_ok=True)
 model = YOLO("receipt-scanner/models/best.pt")
 
 # Init LangChain with OpenAI
-llm = ChatOpenAI(
-    model="gpt-5-mini",  # ✅ text-only model, fine since we feed text now
+openai_api_key = os.getenv("OPENAI_API_KEY")
+if not openai_api_key:
+    print("⚠️ OPENAI_API_KEY is not set; LangChain calls will fail until configured.")
+
+TEXT_LLM_MODEL = os.getenv("TEXT_LLM_MODEL", "gpt-5-mini")
+VISION_LLM_MODEL = os.getenv("VISION_LLM_MODEL", "gpt-4o")
+
+llm_text = ChatOpenAI(
+    model=TEXT_LLM_MODEL,
+    temperature=0,
+)
+
+llm_vision = ChatOpenAI(
+    model=VISION_LLM_MODEL,
     temperature=0,
 )
 
 # Define the prompt for categorization
-prompt = ChatPromptTemplate.from_template("""
+text_prompt = ChatPromptTemplate.from_template("""
 You are an expert at analyzing receipt text extracted with OCR. 
 The text may contain noise or extra characters, but focus only on useful parts.
 
@@ -77,7 +101,7 @@ def run_langchain_with_text(receipt_text: str, image_path: str):
     """Send OCR text to GPT for structured JSON, with logs."""
     print(f"📜 OCR extracted text from {image_path}:\n{receipt_text}")
 
-    chain = prompt | llm
+    chain = text_prompt | llm_text
     result = chain.invoke({"ocr_text": receipt_text})  # ✅ match prompt var
 
     # 🔥 Debug logging
@@ -86,6 +110,38 @@ def run_langchain_with_text(receipt_text: str, image_path: str):
 
     return result.content
 
+
+def detect_and_crop_largest_receipt(img_array: np.ndarray):
+    """Detect the largest bounding box via YOLO and return crop and bbox.
+
+    Returns (cropped_array, bbox_dict)
+    bbox_dict = {"x1": int, "y1": int, "x2": int, "y2": int}
+    """
+    results = model(img_array)
+    # Default fallback to full image
+    cropped = img_array
+    bbox = None
+    for r in results:
+        if len(r.boxes.xyxy) == 0:
+            continue
+        boxes = r.boxes.xyxy
+        i = max(range(len(boxes)), key=lambda j: (boxes[j][2]-boxes[j][0]) * (boxes[j][3]-boxes[j][1]))
+        x1, y1, x2, y2 = map(int, boxes[i].tolist())
+        cropped = img_array[y1:y2, x1:x2]
+        bbox = {"x1": x1, "y1": y1, "x2": x2, "y2": y2}
+        break
+    return cropped, bbox
+
+
+def encode_image_to_data_url(img_array: np.ndarray, format: str = "JPEG") -> str:
+    """Encode an RGB image array to a data URL suitable for GPT-4o image input."""
+    pil_img = Image.fromarray(img_array)
+    buffer = io.BytesIO()
+    pil_img.save(buffer, format=format)
+    b64 = base64.b64encode(buffer.getvalue()).decode("utf-8")
+    mime = "image/jpeg" if format.upper() == "JPEG" else "image/png"
+    return f"data:{mime};base64,{b64}"
+
 @app.post("/process-receipt")
 async def process_receipt(file: UploadFile = File(...)):
     """
@@ -95,47 +151,82 @@ async def process_receipt(file: UploadFile = File(...)):
     image = Image.open(io.BytesIO(contents)).convert("RGB")
     img_array = np.array(image)
 
-    # Run YOLO detection
-    results = model(img_array)
-
     outputs = []
-    for r in results:
-        if len(r.boxes.xyxy) == 0:
-            print("⚠️ No receipts detected, falling back to full image")
-            cropped = img_array
-        else:
-            # ✅ Pick the largest bounding box
-            boxes = r.boxes.xyxy
-            i = max(range(len(boxes)), key=lambda j: (boxes[j][2]-boxes[j][0]) * (boxes[j][3]-boxes[j][1]))
-            x1, y1, x2, y2 = map(int, boxes[i].tolist())
-            cropped = img_array[y1:y2, x1:x2]
-
-        # Save cropped or fallback image
-        save_path = f"receiptimages/receipt_main.jpg"
-        cv2.imwrite(save_path, cv2.cvtColor(cropped, cv2.COLOR_RGB2BGR))
-        print(f"✅ Saved {save_path}")
-
-        # OCR step
-        ocr_text = pytesseract.image_to_string(Image.fromarray(cropped))
-
-        # ✅ Fallback to full image if OCR is empty
-        if not ocr_text.strip():
-            print("⚠️ Empty OCR text, retrying with full image")
-            ocr_text = pytesseract.image_to_string(image)
-
-        # Send OCR text to GPT
-        llm_result = run_langchain_with_text(ocr_text, save_path)
-
-        outputs.append({
-            "cropped_file": save_path,
-            "ocr_text": ocr_text.strip(),
-            "extracted": llm_result
-        })
+    # Detect and crop once per image (YOLO returns one result for input image)
+    cropped, bbox = detect_and_crop_largest_receipt(img_array)
+
+    # Save cropped or fallback image
+    save_path = f"receiptimages/receipt_main.jpg"
+    cv2.imwrite(save_path, cv2.cvtColor(cropped, cv2.COLOR_RGB2BGR))
+    print(f"✅ Saved {save_path}")
+
+    # OCR step
+    ocr_text = pytesseract.image_to_string(Image.fromarray(cropped))
+
+    # ✅ Fallback to full image if OCR is empty
+    if not ocr_text.strip():
+        print("⚠️ Empty OCR text, retrying with full image")
+        ocr_text = pytesseract.image_to_string(image)
+
+    # Send OCR text to GPT
+    llm_result = run_langchain_with_text(ocr_text, save_path)
+
+    outputs.append({
+        "cropped_file": save_path,
+        "bbox": bbox,
+        "ocr_text": ocr_text.strip(),
+        "extracted": llm_result
+    })
 
     # 🔥 Log the full outputs for debugging
     print("📦 Final API response:", json.dumps(outputs, indent=2))
 
     return {"status": "success", "receipts": outputs}
 
+
+@app.post("/process-receipt-vision")
+async def process_receipt_vision(file: UploadFile = File(...)):
+    """
+    Upload receipt image -> detect largest -> crop -> send image to GPT-4o -> JSON result
+    No OCR is performed; the model reads directly from the image.
+    """
+    contents = await file.read()
+    image = Image.open(io.BytesIO(contents)).convert("RGB")
+    img_array = np.array(image)
+
+    cropped, bbox = detect_and_crop_largest_receipt(img_array)
+
+    save_path = f"receiptimages/receipt_vision.jpg"
+    cv2.imwrite(save_path, cv2.cvtColor(cropped, cv2.COLOR_RGB2BGR))
+    print(f"✅ Saved {save_path}")
+
+    data_url = encode_image_to_data_url(cropped, format="JPEG")
+
+    vision_instructions = (
+        "You are an expert at reading receipts from an image. "
+        "Extract these fields as strict JSON: {\\n"
+        "  \"venue\": string or null,\\n"
+        "  \"date\": string or null,\\n"
+        "  \"total\": string or number, include currency if shown,\\n"
+        "  \"category\": string or null\\n"
+        "}. Do not include any extra commentary."
+    )
+
+    message = HumanMessage(
+        content=[
+            {"type": "text", "text": vision_instructions},
+            {"type": "image_url", "image_url": {"url": data_url}},
+        ]
+    )
+
+    result = llm_vision.invoke([message])
+
+    return {
+        "status": "success",
+        "cropped_file": save_path,
+        "bbox": bbox,
+        "extracted": getattr(result, "content", str(result)),
+    }
+
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=7860)