app.py

import gradio as gr
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
import torch
from threading import Thread

from marker.convert import convert_single_pdf
from marker.output import markdown_exists, save_markdown, get_markdown_filepath
from marker.pdf.utils import find_filetype
from marker.pdf.extract_text import get_length_of_text
from marker.models import load_all_models
from marker.settings import settings
from marker.logger import configure_logging
from surya.settings import settings as surya_settings
import traceback


# marker
configure_logging()
MAX_PAGES = 20
MIN_LENGTH=200
settings.EXTRACT_IMAGES = False
settings.DEBUG = False
settings.PDFTEXT_CPU_WORKERS = 1
settings.DETECTOR_POSTPROCESSING_CPU_WORKERS = 1
settings.OCR_PARALLEL_WORKERS = 1
surya_settings.IN_STREAMLIT = True

model_refs = load_all_models()
metadata = {}

# prepare LLM
model_name = "maxidl/Llama-OpenReviewer-8B"
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, decode_kwargs=dict(skip_special_tokens=True))

# Define prompts
SYSTEM_PROMPT_TEMPLATE = """You are an expert reviewer for AI conferences. You follow best practices and review papers according to the reviewer guidelines.

Reviewer guidelines:

1. Read the paper: It’s important to carefully read through the entire paper, and to look up any related work and citations that will help you comprehensively evaluate it. Be sure to give yourself sufficient time for this step.

2. While reading, consider the following:

    - Objective of the work: What is the goal of the paper? Is it to better address a known application or problem, draw attention to a new application or problem, or to introduce and/or explain a new theoretical finding? A combination of these? Different objectives will require different considerations as to potential value and impact.

    - Strong points: is the submission clear, technically correct, experimentally rigorous, reproducible, does it present novel findings (e.g. theoretically, algorithmically, etc.)?

    - Weak points: is it weak in any of the aspects listed in b.?

    - Be mindful of potential biases and try to be open-minded about the value and interest a paper can hold for the community, even if it may not be very interesting for you.

3. Answer four key questions for yourself, to make a recommendation to Accept or Reject:

    - What is the specific question and/or problem tackled by the paper?

    - Is the approach well motivated, including being well-placed in the literature?

    - Does the paper support the claims? This includes determining if results, whether theoretical or empirical, are correct and if they are scientifically rigorous.

    - What is the significance of the work? Does it contribute new knowledge and sufficient value to the community? Note, this does not necessarily require state-of-the-art results. Submissions bring value to the community when they convincingly demonstrate new, relevant, impactful knowledge (incl., empirical, theoretical, for practitioners, etc).

4. Write your review including the following information: 

    - Summarize what the paper claims to contribute. Be positive and constructive.

    - List strong and weak points of the paper. Be as comprehensive as possible.

    - Clearly state your initial recommendation (accept or reject) with one or two key reasons for this choice.

    - Provide supporting arguments for your recommendation.

    - Ask questions you would like answered by the authors to help you clarify your understanding of the paper and provide the additional evidence you need to be confident in your assessment.

    - Provide additional feedback with the aim to improve the paper. Make it clear that these points are here to help, and not necessarily part of your decision assessment.

Your write reviews in markdown format. Your reviews contain the following sections:

# Review

{review_fields}

Your response must only contain the review in markdown format with sections as defined above.

"""

USER_PROMPT_TEMPLATE = """Review the following paper:

{paper_text}

"""

# For now, use fixed review fields
REVIEW_FIELDS = """## Summary

Briefly summarize the paper and its contributions. This is not the place to critique the paper; the authors should generally agree with a well-written summary.



## Novelty

Please assign the paper a numerical rating on the following scale to indicate the novelty and originality of the work. Consider whether the paper presents new ideas, methods, or perspectives that have not been explored before. Choose from the following:

4: excellent - Highly original work with groundbreaking ideas or completely novel approaches

3: good - Significant new contributions with clear advances over existing work

2: fair - Some new elements but largely incremental improvements or combinations of existing ideas

1: poor - Little to no novelty, mostly reproducing existing work or trivial variations



## Novelty Explanation

IMPORTANT: Focus ONLY on novelty aspects. DO NOT discuss soundness, presentation, or general contribution here.

Please provide specific justification for your novelty score by addressing:

- What specific new concepts, methods, or approaches does this paper introduce?

- How do these differ from existing work in the field? Cite specific prior work for comparison.

- Are the differences substantial or incremental?

- Is this addressing a problem in a genuinely new way, or applying known methods to a new domain?

DO NOT repeat content from other sections. DO NOT discuss writing quality, experimental rigor, or implementation details here.



## Soundness

Please assign the paper a numerical rating on the following scale to indicate the soundness of the technical claims, experimental and research methodology and on whether the central claims of the paper are adequately supported with evidence. Choose from the following:

4: excellent

3: good

2: fair

1: poor



## Soundness Explanation

IMPORTANT: Focus ONLY on technical correctness and methodological rigor. DO NOT discuss novelty or writing quality here.

Please provide specific reasons for your soundness score by addressing:

- Are the technical claims mathematically/logically correct?

- Is the experimental methodology rigorous and appropriate?

- Are the experiments sufficient to support the claims?

- Are there any methodological flaws or missing controls?

- Is the statistical analysis (if any) appropriate and correctly executed?

DO NOT repeat content from other sections. DO NOT discuss the novelty of the approach or presentation quality here.



## Presentation

Please assign the paper a numerical rating on the following scale to indicate the quality of the presentation. This should take into account the writing style and clarity, as well as contextualization relative to prior work. Choose from the following:

4: excellent

3: good

2: fair

1: poor



## Presentation Explanation

IMPORTANT: Focus ONLY on writing quality, clarity, and organization. DO NOT discuss technical merit or novelty here.

Please explain your presentation score by addressing:

- Is the paper well-organized and easy to follow?

- Are the main ideas clearly explained?

- Are figures, tables, and visualizations effective and well-designed?

- Is the related work section comprehensive and fair?

- Are mathematical notations consistent and clear?

- Is the language precise and grammatically correct?

DO NOT repeat content from other sections. DO NOT discuss the novelty of ideas or soundness of methods here.



## Contribution

Please assign the paper a numerical rating on the following scale to indicate the quality of the overall contribution this paper makes to the research area being studied. Are the questions being asked important? Does the paper bring a significant originality of ideas and/or execution? Are the results valuable to share with the broader ICLR community? Choose from the following:

4: excellent

3: good

2: fair

1: poor



## Contribution Explanation

IMPORTANT: Focus on the OVERALL IMPACT and SIGNIFICANCE to the field. This is different from novelty.

Please justify your contribution score by explaining:

- Why is this work important for the field?

- What practical or theoretical impact could this have?

- Who would benefit from this work and how?

- Does this open new research directions or close important gaps?

- How significant are the improvements over baselines (if applicable)?

Consider both immediate utility and long-term impact. DO NOT simply repeat the novelty assessment here.



## Strengths

List the main strengths of the paper. Be specific and provide evidence. Each strength should be a separate bullet point. Focus on what the paper does well across all dimensions (novelty, soundness, presentation, contribution). Avoid generic statements.



## Weaknesses

List the main weaknesses of the paper. Be specific, constructive, and actionable. Each weakness should be a separate bullet point with suggestions for improvement where possible. Focus on significant issues that affect the paper's validity or impact.



## Questions

List specific questions for the authors that could clarify ambiguities or address concerns. Number each question. These should be questions where the answer could potentially change your assessment of the paper.



## Flag For Ethics Review

If there are ethical issues with this paper, please flag the paper for an ethics review and select area of expertise that would be most useful for the ethics reviewer to have. Please select all that apply. Choose from the following:

No ethics review needed.

Yes, Discrimination / bias / fairness concerns

Yes, Privacy, security and safety

Yes, Legal compliance (e.g., GDPR, copyright, terms of use)

Yes, Potentially harmful insights, methodologies and applications

Yes, Responsible research practice (e.g., human subjects, data release)

Yes, Research integrity issues (e.g., plagiarism, dual submission)

Yes, Unprofessional behaviors (e.g., unprofessional exchange between authors and reviewers)

Yes, Other reasons (please specify below)



## Details Of Ethics Concerns

Please provide details of your concerns. If no ethics review is needed, write "N/A".



## Rating

Please provide an "overall score" for this submission. Choose from the following:

1: strong reject

3: reject, not good enough

5: marginally below the acceptance threshold

6: marginally above the acceptance threshold

8: accept, good paper

10: strong accept, should be highlighted at the conference



## Overall Justification

Provide a comprehensive justification for your overall rating that:

- Synthesizes the assessments from all dimensions (novelty, soundness, presentation, contribution)

- Explains how you weighted different aspects in arriving at your final score

- Clearly states whether the strengths outweigh the weaknesses or vice versa

- Indicates what would need to change for a different rating

This should be a holistic assessment, not a repetition of individual sections.



"""

# functions
def create_messages(review_fields, paper_text):
    messages = [
        {"role": "system", "content": SYSTEM_PROMPT_TEMPLATE.format(review_fields=review_fields)},
        {"role": "user", "content": USER_PROMPT_TEMPLATE.format(paper_text=paper_text)},
    ]
    return messages

@spaces.GPU()
def convert_file(filepath):
    full_text, images, out_metadata = convert_single_pdf(
            filepath, model_refs, metadata=metadata, max_pages=MAX_PAGES
    )
    return full_text

def process_file(file):
    print(file.name)
    filepath = file.name
    try:
        if MIN_LENGTH:
            filetype = find_filetype(filepath)
            if filetype == "other":
                raise ValueError()

            length = get_length_of_text(filepath)
            if length < MIN_LENGTH:
                raise ValueError()
        paper_text = convert_file(filepath)
        paper_text = paper_text.strip()
        if not len(paper_text) > MIN_LENGTH:
            raise ValueError()
    except spaces.zero.gradio.HTMLError as e:
        print(e)
        return "Error. GPU quota exceeded. Please return later."
    # except gradio.exceptions.Error as e:
    #     if 'GPU task aborted' in str(e):
    #         print(e)
    #         return 'GPU task aborted'
    except Exception as e:
        print(traceback.format_exc())
        print(f"Error converting {filepath}: {e}")
        return "Error processing pdf"
    return paper_text


@spaces.GPU(duration=90)
def generate(paper_text, review_template):
    messages = create_messages(review_template, paper_text)
    input_ids = tokenizer.apply_chat_template(
        messages,
        add_generation_prompt=True,
        return_tensors='pt'
    ).to(model.device)
    print(f"input_ids shape: {input_ids.shape}")
    generation_kwargs = dict(input_ids=input_ids, streamer=streamer, max_new_tokens=4096, do_sample=True, temperature=0.6, top_p=0.9)
    thread = Thread(target=model.generate, kwargs=generation_kwargs)
    thread.start()
    generated_text = ""
    for new_text in streamer:
        generated_text += new_text
        yield generated_text.replace("<|eot_id|>", "")



# ui
#8C1B13 red
#4D8093 blue
#767676 med grey
#EFECE3 light grey
#DDDDDD silver below red
#FFFDFA white

title = """<h1 align="center">OpenReviewer</h1>

<div align="center">Using <a href="https://huggingface.co/maxidl/Llama-OpenReviewer-8B" target="_blank"><code>Llama-OpenReviewer-8B</code></a> - Built with Llama</div>

"""

description = """This is an online demo featuring [Llama-OpenReviewer-8B](https://huggingface.co/maxidl/Llama-OpenReviewer-8B), a large language model that generates high-quality reviews for machine learning and AI papers.

## Demo Guidelines

1. Upload you paper as a pdf file. Alternatively you can paste the full text of your paper in markdown format below. We do **not** store your data. User data is kept in ephemeral storage during processing.

2. Once you upload a pdf it will be converted to markdown. This takes some time as it runs multiple transformer models to parse the layout and extract text and tables. Checkout [marker](https://github.com/VikParuchuri/marker/tree/master) for details.

3. Having obtained a markdown version of your paper, you can now click *Generate Review*.

Take a look at the Review Template to properly interpret the generated review. You can also change the review template before generating in case you want to generate a review with a different schema and aspects.

To obtain more than one review, just generate again.

**GPU quota:** If exceeded, either sign in with your HF account or come back later. Your quota has a half-life of 2 hours.

"""

with gr.Blocks() as demo:
    title = gr.HTML(title)
    description = gr.Markdown(description)
    file_input = gr.File(file_types=[".pdf"], file_count="single")
    paper_text_field= gr.Textbox("Upload a pdf or paste the full text of your paper in markdown format here.", label="Paper Text", lines=20, max_lines=20, autoscroll=False)
    with gr.Accordion("Review Template", open=False):
        review_template_description = gr.Markdown("We use the ICLR 2025 review template by default, but you can modify the template below as you like.")
        review_template_field = gr.Textbox(label=" ",lines=20, max_lines=20, autoscroll=False, value=REVIEW_FIELDS)
    generate_button = gr.Button("Generate Review", interactive=not paper_text_field)
    file_input.upload(process_file, file_input, paper_text_field)
    paper_text_field.change(lambda text: gr.update(interactive=True) if len(text) > 200 else gr.update(interactive=False), paper_text_field, generate_button)

    review_field = gr.Markdown("\n\n\n\n\n", label="Review")
    generate_button.click(fn=lambda: gr.update(interactive=False), inputs=None, outputs=generate_button).then(generate, [paper_text_field, review_template_field], review_field).then(fn=lambda: gr.update(interactive=True), inputs=None, outputs=generate_button)

    demo.title = "OpenReviewer"




if __name__ == "__main__":
    demo.launch()