app.py

import argparse
import json
import os

import torch
from PIL import Image
from qwen_vl_utils import process_vision_info
from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration
import gradio as gr

user_prompt = "Analyze the image. Extract and output only the LaTeX formulas present in the image, in LaTeX code format. Ignore inline formulas, all other text, and do not include any explanations."


def read_input_file(input_file):
    with open(input_file, 'r') as file:
        data = json.load(file)

    image_path = data[0]['images'][0]
    gt_latex_code = data[0]['messages'][1]['content']

    return image_path, gt_latex_code


class ImageProcessor:
    def __init__(self, args):
        self.args = args
        self.model, self.vis_processor = self.load_model_and_processor()
        self.generate_kwargs = dict(
            max_new_tokens=2048,
            top_p=0.001,
            top_k=1,
            temperature=0.01,
            repetition_penalty=1.0,
        )

    def load_model_and_processor(self):
        # Load model
        checkpoint = self.args.ckpt
        vis_processor = AutoProcessor.from_pretrained(checkpoint)

        model = Qwen2_5_VLForConditionalGeneration.from_pretrained(checkpoint, torch_dtype="auto", device_map="auto")
        model.eval()

        return model, vis_processor

    def process_single_image(self, image_path):
        question = user_prompt

        try:
            image_local_path = "file://" + image_path

            messages = []
            messages.append(
                {"role": "user", "content": [
                        {"type": "image", "image": image_local_path, "min_pixels": 32 * 32, "max_pixels": 512 * 512},
                        {"type": "text", "text": question},
                    ]
                }
            )

            text = self.vis_processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
            images, videos = process_vision_info([messages])

            inputs = self.vis_processor(text=text, images=images, videos=videos, padding=True, return_tensors='pt')
            inputs = inputs.to(self.model.device)

            with torch.no_grad():
                generated_ids = self.model.generate(
                    **inputs,
                    **self.generate_kwargs,
                )
            generated_ids = [
                output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, generated_ids)
            ]
            out = self.vis_processor.tokenizer.batch_decode(
                generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
            )
            model_answer = out[0]
        except Exception as e:
            print(e, flush=True)
            model_answer = "None"

        return model_answer

def save_image_with_auto_naming(image, save_dir="./tmp"):    
    # 确保目录存在
    os.makedirs(save_dir, exist_ok=True)
    
    # 获取目录中现有的文件名
    existing_files = [f for f in os.listdir(save_dir) if f.endswith('.png') and f.split('.')[0].isdigit()]
    
    # 找到最大的数字
    next_num = 0
    if existing_files:
        next_num = max([int(f.split('.')[0]) for f in existing_files]) + 1
    
    # 生成新文件名
    temp_path = os.path.join(save_dir, f"{next_num}.png")
    
    # 保存图片
    image.save(temp_path)
    
    return temp_path

# {{ edit_1 }}
def process_image_for_gradio(image):
    """处理上传的图片并返回LaTeX结果"""
    if image is None:
        return ""

    # 保存上传的图片到指定目录，并自动命名
    temp_path = save_image_with_auto_naming(image)

    # 处理图片
    pred_latex_code = processor.process_single_image(temp_path)

    # 清理临时文件
    if os.path.exists(temp_path):
        os.remove(temp_path)

    return pred_latex_code

def load_example(example_name):
    """加载示例图片"""
    input_file = os.path.join('./asset/test_jsons', f"{example_name}.json")
    image_path, gt_latex_code = read_input_file(input_file)
    return Image.open(image_path), example_name

# {{ edit_2 }}
def create_gradio_interface(processor):
    """创建Gradio界面"""
    with gr.Blocks(title="DocTron-Formula") as demo:
        gr.Markdown("# DocTron-Formula LaTeX公式识别")
        gr.Markdown("上传图片或选择示例来识别LaTeX公式")

        with gr.Row():
            with gr.Column():
                # 左侧列
                image_input = gr.Image(type="pil", label="上传图片")

                with gr.Row():
                    clear_btn = gr.Button("Clear")
                    submit_btn = gr.Button("Submit", variant="primary")

                gr.Markdown("### 示例图片")
                with gr.Row():
                    line_btn = gr.Button("Line-level")
                    paragraph_btn = gr.Button("Paragraph-level")
                    page_btn = gr.Button("Page-level")

                # 存储示例名称
                example_name = gr.State()

            with gr.Column():
                # 右侧列 - 显示结果
                latex_output = gr.Textbox(label="预测的LaTeX公式", lines=10, interactive=False)

        # 按钮事件绑定
        submit_btn.click(
            fn=process_image_for_gradio,
            inputs=[image_input],
            outputs=[latex_output]
        )

        clear_btn.click(
            fn=lambda: (None, ""),
            inputs=[],
            outputs=[image_input, latex_output]
        )

        # 示例按钮事件
        line_btn.click(
            fn=load_example,
            inputs=gr.Textbox(value="line-level", visible=False),
            outputs=[image_input, example_name]
        ).then(
            fn=lambda img: process_image_for_gradio(img),
            inputs=[image_input],
            outputs=[latex_output]
        )

        paragraph_btn.click(
            fn=load_example,
            inputs=gr.Textbox(value="paragraph-level", visible=False),
            outputs=[image_input, example_name]
        ).then(
            fn=lambda img: process_image_for_gradio(img),
            inputs=[image_input],
            outputs=[latex_output]
        )

        page_btn.click(
            fn=load_example,
            inputs=gr.Textbox(value="page-level", visible=False),
            outputs=[image_input, example_name]
        ).then(
            fn=lambda img: process_image_for_gradio(img),
            inputs=[image_input],
            outputs=[latex_output]
        )

    return demo

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--ckpt", type=str, default="DocTron/DocTron-Formula")
    parser.add_argument("--input_file", type=str, default="line-level")
    args = parser.parse_args()

    # Init model
    processor = ImageProcessor(args)

    # {{ edit_3 }}
    # 创建并启动Gradio界面
    demo = create_gradio_interface(processor)
    # demo.launch(
    #     server_name="10.238.36.208",
    #     server_port=8000,
    #     share=False
    # )
    demo.launch()