Add initial JSON and Markdown files for multicolumn CNN and switch transformers, including metadata and example images

marker/scripts/server.py

@@ -103,9 +103,75 @@ class CommonParams(BaseModel):
     output_format: Annotated[
         str,
         Field(
-            description="The format to output the text in.  Can be 'markdown', 'json', or 'html'.  Defaults to 'markdown'."
+            description="The format to output the text in.  Can be 'markdown', 'json', 'html', or 'chunks'.  Defaults to 'markdown'."
         ),
     ] = "markdown"
+    output_dir: Annotated[
+        Optional[str],
+        Field(
+            description="Directory where output files will be saved. Defaults to the value specified in settings.OUTPUT_DIR."
+        ),
+    ] = None
+    use_llm: Annotated[
+        bool,
+        Field(
+            description="Uses an LLM to improve accuracy. You will need to configure the LLM backend."
+        ),
+    ] = False
+    block_correction_prompt: Annotated[
+        Optional[str],
+        Field(
+            description="If LLM mode is active, an optional prompt that will be used to correct the output of marker. This is useful for custom formatting or logic that you want to apply to the output."
+        ),
+    ] = None
+    strip_existing_ocr: Annotated[
+        bool,
+        Field(
+            description="Remove all existing OCR text in the document and re-OCR with surya."
+        ),
+    ] = False
+    redo_inline_math: Annotated[
+        bool,
+        Field(
+            description="If you want the absolute highest quality inline math conversion, use this along with --use_llm."
+        ),
+    ] = False
+    disable_image_extraction: Annotated[
+        bool,
+        Field(
+            description="Don't extract images from the PDF. If you also specify --use_llm, then images will be replaced with a description."
+        ),
+    ] = False
+    debug: Annotated[
+        bool,
+        Field(
+            description="Enable debug mode for additional logging and diagnostic information."
+        ),
+    ] = False
+    processors: Annotated[
+        Optional[str],
+        Field(
+            description="Override the default processors by providing their full module paths, separated by commas. Example: 'module1.processor1,module2.processor2'"
+        ),
+    ] = None
+    config_json: Annotated[
+        Optional[str],
+        Field(
+            description="Path to a JSON configuration file containing additional settings."
+        ),
+    ] = None
+    converter_cls: Annotated[
+        Optional[str],
+        Field(
+            description="Converter class to use. One of 'marker.converters.pdf.PdfConverter' (default) or 'marker.converters.table.TableConverter'. The PdfConverter will convert the whole PDF, the TableConverter will only extract and convert tables."
+        ),
+    ] = None
+    llm_service: Annotated[
+        Optional[str],
+        Field(
+            description="Which LLM service to use if --use_llm is passed. This defaults to 'marker.services.gemini.GoogleGeminiService'. Should be full import path, like 'marker.services.gemini.GoogleGeminiService'."
+        ),
+    ] = None
 
 
 async def _convert_pdf(params: CommonParams):
@@ -113,11 +179,13 @@ async def _convert_pdf(params: CommonParams):
         "Invalid output format"
     )
     try:
-        options = params.model_dump()
+        options = params.model_dump(exclude_none=True)
+        # Remove None values to avoid passing them to ConfigParser
+        options = {k: v for k, v in options.items() if v is not None}
         config_parser = ConfigParser(options)
         config_dict = config_parser.generate_config_dict()
         config_dict["pdftext_workers"] = 1
-        converter_cls = PdfConverter
+        converter_cls = config_parser.get_converter_cls()
         converter = converter_cls(
             config=config_dict,
             artifact_dict=app_data["models"],
@@ -163,6 +231,17 @@ async def convert_pdf_upload(
     force_ocr: Optional[bool] = Form(default=False),
     paginate_output: Optional[bool] = Form(default=False),
     output_format: Optional[str] = Form(default="markdown"),
+    output_dir: Optional[str] = Form(default=None),
+    use_llm: Optional[bool] = Form(default=False),
+    block_correction_prompt: Optional[str] = Form(default=None),
+    strip_existing_ocr: Optional[bool] = Form(default=False),
+    redo_inline_math: Optional[bool] = Form(default=False),
+    disable_image_extraction: Optional[bool] = Form(default=False),
+    debug: Optional[bool] = Form(default=False),
+    processors: Optional[str] = Form(default=None),
+    config_json: Optional[str] = Form(default=None),
+    converter_cls: Optional[str] = Form(default=None),
+    llm_service: Optional[str] = Form(default=None),
     file: UploadFile = File(
         ..., description="The PDF file to convert.", media_type="application/pdf"
     ),
@@ -179,6 +258,17 @@ async def convert_pdf_upload(
         force_ocr=force_ocr,
         paginate_output=paginate_output,
         output_format=output_format,
+        output_dir=output_dir,
+        use_llm=use_llm,
+        block_correction_prompt=block_correction_prompt,
+        strip_existing_ocr=strip_existing_ocr,
+        redo_inline_math=redo_inline_math,
+        disable_image_extraction=disable_image_extraction,
+        debug=debug,
+        processors=processors,
+        config_json=config_json,
+        converter_cls=converter_cls,
+        llm_service=llm_service,
     )
     results = await _convert_pdf(params)
     os.remove(upload_path)