Add structured receipt extraction outputs and HF launch entrypoint

.gitignore

@@ -8,3 +8,4 @@ wheels/
 
 # Virtual environments
 .venv
+**/*.log

README.md

@@ -0,0 +1,32 @@
+---
+title: NuMark
+emoji: 🧾
+colorFrom: blue
+colorTo: indigo
+sdk: gradio
+python_version: "3.12"
+app_file: app.py
+pinned: false
+models:
+  - numind/NuExtract3
+short_description: Extract Markdown or structured JSON from document images with NuExtract3.
+---
+
+# NuMark
+
+NuMark is a Gradio app for extracting Markdown or structured JSON from document images with [NuExtract3](https://huggingface.co/numind/NuExtract3).
+
+Upload a receipt, invoice, or other document image, optionally add accompanying text, and choose either content extraction or schema-guided structured extraction.
+
+The inference function uses Hugging Face ZeroGPU dynamic GPU allocation. Select ZeroGPU in the Space hardware settings before launching the app.
+
+## Run locally
+
+Install the project dependencies and launch the Hugging Face Spaces entry point:
+
+```bash
+uv sync
+uv run app.py
+```
+
+The Hugging Face Space starts from `app.py`, which exposes the Gradio app defined in `app/hf_space.py`.

app.py

@@ -0,0 +1,4 @@
+from app.hf_space import demo
+
+if __name__ == "__main__":
+    demo.launch()

app/gradio.py

@@ -1,6 +1,8 @@
+import json
 from pathlib import Path
 
 import gradio as gr
+import spaces
 import torch
 from transformers import AutoModelForImageTextToText, AutoProcessor
 
@@ -17,6 +19,40 @@ structured_json_templates = {
   "paid": "boolean",
   "payment_method": ["cash", "credit-card", "debit-card", "other"],
   "items": ["verbatim-string"]
+}""",
+    "Advanced receipt": """{
+  "merchant": {
+    "name": "verbatim-string",
+    "address": "verbatim-string",
+    "phone": "phone-number"
+  },
+  "purchase_date": "date",
+  "purchase_time": "time",
+  "store_number": "verbatim-string",
+  "operator_number": "verbatim-string",
+  "terminal_number": "verbatim-string",
+  "transaction_number": "verbatim-string",
+  "items": [
+    {
+      "description": "verbatim-string",
+      "sku": "verbatim-string",
+      "quantity": "number",
+      "unit": "unit-code",
+      "unit_price": "number",
+      "discount": "number",
+      "line_total": "number",
+      "tax_code": "verbatim-string"
+    }
+  ],
+  "item_count": "integer",
+  "subtotal": "number",
+  "tax": "number",
+  "total": "number",
+  "currency": "currency",
+  "payment_method": ["cash", "credit-card", "debit-card", "other"],
+  "card_last_four": "verbatim-string",
+  "amount_tendered": "number",
+  "change_due": "number"
 }""",
     "Invoice with line items": """{
   "document_type": "verbatim-string",
@@ -99,19 +135,24 @@ default_structured_json_template = "Invoice with line items"
 def select_structured_json_template(name):
     return structured_json_templates[name]
 
+
 processor = AutoProcessor.from_pretrained(
     model_id,
     trust_remote_code=True,
 )
 
-model = AutoModelForImageTextToText.from_pretrained(
-    model_id,
-    dtype=torch.bfloat16,
-    device_map="auto",
-    trust_remote_code=True,
-).eval()
+model = (
+    AutoModelForImageTextToText.from_pretrained(
+        model_id,
+        dtype=torch.bfloat16,
+        trust_remote_code=True,
+    )
+    .to("cuda")
+    .eval()
+)
 
 
+@spaces.GPU(duration=120)
 def run_nuextract(messages, **chat_template_kwargs):
     inputs = processor.apply_chat_template(
         messages,
@@ -138,7 +179,7 @@ def run_nuextract(messages, **chat_template_kwargs):
 
 
 def extract(image, text, mode, template, enable_thinking):
-    return run_nuextract(
+    result = run_nuextract(
         [
             {
                 "role": "user",
@@ -153,6 +194,11 @@ def extract(image, text, mode, template, enable_thinking):
         enable_thinking=enable_thinking,
     )
 
+    if mode == "content":
+        return result, result, gr.skip()
+
+    return gr.skip(), gr.skip(), json.loads(result)
+
 
 with gr.Blocks(title="NuExtract3") as demo:
     gr.Markdown(
@@ -186,7 +232,21 @@ with gr.Blocks(title="NuExtract3") as demo:
             )
             enable_thinking = gr.Checkbox(label="Enable thinking")
             run = gr.Button("Extract", variant="primary")
-        output = gr.Textbox(label="Output", lines=28, buttons=["copy"])
+        with gr.Column(), gr.Tabs():
+            with gr.Tab("Markdown preview"):
+                markdown_preview = gr.Markdown()
+            with gr.Tab("Markdown text"):
+                markdown_text = gr.Textbox(
+                    label="Markdown source",
+                    lines=28,
+                    buttons=["copy"],
+                )
+            with gr.Tab("Structured output"):
+                structured_output = gr.JSON(
+                    label="Structured JSON",
+                    open=True,
+                    show_indices=True,
+                )
 
     gr.Examples(
         examples=[
@@ -208,9 +268,9 @@ with gr.Blocks(title="NuExtract3") as demo:
                 / "invoice-with-items.png",
                 *sorted(
                     path
-                    for path in (
-                        Path(__file__).parents[1] / "data" / "samples"
-                    ).glob("*.png")
+                    for path in (Path(__file__).parents[1] / "data" / "samples").glob(
+                        "*.png"
+                    )
                     if path.name != "invoice-with-items.png"
                 ),
             ]
@@ -227,7 +287,7 @@ with gr.Blocks(title="NuExtract3") as demo:
     run.click(
         extract,
         inputs=[image, text, mode, template, enable_thinking],
-        outputs=output,
+        outputs=[markdown_preview, markdown_text, structured_output],
         api_name="extract",
     )
 

app/hf_space.py

@@ -0,0 +1,7 @@
+from app.gradio import demo
+
+__all__ = ["demo"]
+
+
+if __name__ == "__main__":
+    demo.launch()

gradio_app.py

@@ -0,0 +1,10 @@
+from runpy import run_path
+
+app = run_path("app/gradio.py")
+demo = app["demo"]
+
+__all__ = ["demo"]
+
+
+if __name__ == "__main__":
+    demo.launch()

mise.toml

@@ -6,3 +6,7 @@ uv = "latest"
 [env]
 UV_LINK_MODE="copy"
 PYTHONPATH=""
+
+[tasks.gradio]
+description = "Launch the Gradio app with hot reload"
+run = '.venv\Scripts\gradio.exe gradio_app.py --watch-dirs app'