app.py

import gradio as gr
import base64
import requests
import io
from PIL import Image
import numpy as np
import os

URL = os.environ['URL']

def sketch_to_text(image, api_key):
    if image is None or not isinstance(image, dict) or 'composite' not in image:
        return "Please write something first."
    
    # Extract the image data from the dictionary
    image_data = image['composite']
    # Convert the image data to a PIL Image
    pil_image = Image.fromarray(image_data.astype(np.uint8))
    
    # Convert the image to base64
    buffered = io.BytesIO()
    pil_image.save(buffered, format="PNG")
    img_str = base64.b64encode(buffered.getvalue()).decode()

    if api_key:
        API_KEY = api_key
    else:
        API_KEY = os.environ['API_KEY']
        
    # Prepare the API request
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {API_KEY}"
    }
    payload = {
        "model": "Llama-3.2-11B-Vision-Instruct",
        "messages": [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "Please read the forumla in the image and transcribe it into latex code, respond with code only, make sure the code is enclosed within a pair of $$"
                    },
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/png;base64,{img_str}"
                        }
                    }
                ]
            }
        ],
        "max_tokens": 300
    }

    # Make the API request
    response = requests.post(URL, headers=headers, json=payload)
    
    if response.status_code == 200:
        return response.json()["choices"][0]["message"]["content"], response.json()["choices"][0]["message"]["content"]
    else:
        return f"Error: {response.status_code}, {response.text}", f"Error: {response.status_code}, {response.text}"

# Create the Gradio interface
with gr.Blocks() as iface:
    gr.Markdown("# Pix2Latex")
    gr.Markdown("Transcribing handwritten forumla into latex with Llama3.2 instruct. [Powered by SambaNova Cloud, Get Your API Key Here](https://cloud.sambanova.ai/apis)")

    with gr.Row():
        api_key = gr.Textbox(label="API Key", type="password", placeholder="(Optional) Enter your API key here for more availability. ")
        
    with gr.Column(scale=1):
        input_image = gr.ImageEditor()

    with gr.Row():
        with gr.Column(scale=1):
            output1 = gr.Textbox(label="Raw")
        with gr.Column(scale=1):
            output2 = gr.Markdown(label="Rendered")
    
    input_image.change(fn=sketch_to_text, inputs=[input_image, api_key], outputs=[output1, output2])
    
    gr.Markdown("How to use: 1. write your formula in the box above. 2. See it in real time, have fun doing math?")

# Launch the app
iface.launch()