import gradio as gr
from openai import OpenAI
import base64
from pathlib import Path
import json
from typing import List, Tuple, Optional
import time
from PIL import Image
import io
import sys
# Global client variable
client = None
def initialize_client(api_key: str) -> Tuple[str, bool]:
"""Initialize OpenAI client with OpenRouter"""
global client
if not api_key or not api_key.strip():
return "⚠️ Please enter a valid API key", False
try:
client = OpenAI(
base_url="https://openrouter.ai/api/v1",
api_key=api_key.strip()
)
return "✅ API Key configured successfully! You can now start chatting.", True
except Exception as e:
return f"❌ Error initializing client: {str(e)}", False
def encode_image(image_path: str) -> str:
"""Encode image to base64"""
with open(image_path, "rb") as image_file:
return base64.b64encode(image_file.read()).decode('utf-8')
def pdf_to_images_pymupdf(pdf_path: str) -> List[Image.Image]:
"""Convert PDF to images using PyMuPDF (primary method)"""
try:
import fitz # PyMuPDF
doc = fitz.open(pdf_path)
images = []
for page_num in range(len(doc)):
page = doc[page_num]
# Render at 2x resolution for better quality
mat = fitz.Matrix(2, 2)
pix = page.get_pixmap(matrix=mat)
# Convert to PIL Image
img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
images.append(img)
doc.close()
return images
except Exception as e:
raise Exception(f"PyMuPDF error: {str(e)}")
def pdf_to_images_pdf2image(pdf_path: str) -> List[Image.Image]:
"""Convert PDF to images using pdf2image (requires poppler)"""
try:
from pdf2image import convert_from_path
images = convert_from_path(pdf_path, dpi=200)
return images
except Exception as e:
raise Exception(f"pdf2image error: {str(e)}")
def pdf_to_images(pdf_path: str) -> Tuple[List[Image.Image], str]:
"""
Convert PDF to images with multiple fallback methods
Returns: (list of images, method used or error message)
"""
# Try PyMuPDF first (doesn't require poppler)
try:
images = pdf_to_images_pymupdf(pdf_path)
return images, "PyMuPDF"
except Exception as e1:
pymupdf_error = str(e1)
# Try pdf2image as fallback
try:
images = pdf_to_images_pdf2image(pdf_path)
return images, "pdf2image"
except Exception as e2:
pdf2image_error = str(e2)
# Both methods failed
error_msg = f"""PDF conversion failed. Tried multiple methods:
1. PyMuPDF: {pymupdf_error}
2. pdf2image: {pdf2image_error}
SOLUTION:
Install PyMuPDF (recommended - no external dependencies):
pip install PyMuPDF
OR install pdf2image + poppler:
pip install pdf2image
Then install poppler:
- Ubuntu/Debian: sudo apt-get install poppler-utils
- macOS: brew install poppler
- Windows: Download from https://github.com/oschwartz10612/poppler-windows/releases/
"""
raise Exception(error_msg)
def image_to_base64(image: Image.Image, format: str = "PNG") -> str:
"""Convert PIL Image to base64"""
buffered = io.BytesIO()
# Convert RGBA to RGB if needed
if image.mode == 'RGBA':
background = Image.new('RGB', image.size, (255, 255, 255))
background.paste(image, mask=image.split()[3])
image = background
elif image.mode != 'RGB':
image = image.convert('RGB')
image.save(buffered, format=format, quality=95)
return base64.b64encode(buffered.getvalue()).decode('utf-8')
def process_file(file_path: str) -> Tuple[List[dict], str]:
"""
Process a file and return content blocks for API
Returns: (content_blocks, status_message)
"""
file_extension = Path(file_path).suffix.lower()
file_name = Path(file_path).name
content_blocks = []
status_message = ""
try:
if file_extension == '.pdf':
# Convert PDF pages to images
images, method = pdf_to_images(file_path)
status_message = f"✅ PDF '{file_name}' converted to {len(images)} page(s) using {method}"
for idx, img in enumerate(images, 1):
base64_image = image_to_base64(img, format="JPEG")
content_blocks.append({
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{base64_image}"
}
})
elif file_extension == '.txt':
# Read text file
try:
with open(file_path, 'r', encoding='utf-8') as f:
text_content = f.read()
except UnicodeDecodeError:
# Try with different encoding
with open(file_path, 'r', encoding='latin-1') as f:
text_content = f.read()
status_message = f"✅ Text file '{file_name}' loaded ({len(text_content)} characters)"
content_blocks.append({
"type": "text",
"text": f"📄 Content from '{file_name}':\n\n{text_content}"
})
else:
# Handle image files
# Determine MIME type
mime_type = "image/jpeg"
if file_extension in ['.png']:
mime_type = "image/png"
elif file_extension in ['.webp']:
mime_type = "image/webp"
elif file_extension in ['.gif']:
mime_type = "image/gif"
elif file_extension in ['.bmp']:
mime_type = "image/bmp"
elif file_extension in ['.tiff', '.tif']:
mime_type = "image/tiff"
# Load and potentially convert the image
try:
img = Image.open(file_path)
# Convert to RGB if necessary
if img.mode in ('RGBA', 'LA', 'P'):
background = Image.new('RGB', img.size, (255, 255, 255))
if img.mode == 'P':
img = img.convert('RGBA')
if img.mode in ('RGBA', 'LA'):
background.paste(img, mask=img.split()[-1] if img.mode in ('RGBA', 'LA') else None)
img = background
elif img.mode != 'RGB':
img = img.convert('RGB')
# Convert to base64
base64_image = image_to_base64(img, format="JPEG")
status_message = f"✅ Image '{file_name}' loaded ({img.width}x{img.height})"
content_blocks.append({
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{base64_image}"
}
})
except Exception as img_error:
# If image processing fails, try direct base64 encoding
base64_image = encode_image(file_path)
status_message = f"✅ Image '{file_name}' loaded (direct encoding)"
content_blocks.append({
"type": "image_url",
"image_url": {
"url": f"data:{mime_type};base64,{base64_image}"
}
})
except Exception as e:
error_msg = f"❌ Error processing '{file_name}': {str(e)}"
content_blocks.append({
"type": "text",
"text": error_msg
})
status_message = error_msg
return content_blocks, status_message
def process_message(
message: str,
history: List[Tuple[str, str]],
files: Optional[List] = None,
enable_reasoning: bool = True,
temperature: float = 0.7,
max_tokens: int = 2000
) -> Tuple[List[Tuple[str, str]], str, str]:
"""
Process user message and generate response
Returns: (updated_history, reasoning_text, status_message)
"""
global client
if client is None:
return history + [(message if message else "No message", "❌ Please configure your API key first in the Settings tab.")], "", "❌ API key not configured"
if not message.strip() and not files:
return history + [("", "⚠️ Please enter a message or upload files.")], "", "⚠️ No input provided"
status_messages = []
try:
# Ensure history is a list
if history is None:
history = []
# Build messages array
messages = []
# Add conversation history
for user_msg, assistant_msg in history:
if user_msg: # Only add if user message exists
messages.append({"role": "user", "content": user_msg if user_msg else "..."})
if assistant_msg: # Only add if assistant message exists
messages.append({"role": "assistant", "content": assistant_msg})
# Build current message content
content = []
# Process files if provided
if files:
file_count = 0
total_pages = 0
for file in files:
if file is not None:
try:
file_blocks, status = process_file(file)
if file_blocks: # Only add if we got valid blocks
content.extend(file_blocks)
status_messages.append(status)
file_count += 1
# Count pages for PDFs
if status and status.startswith("✅") and "page(s)" in status:
try:
pages = int(status.split("converted to ")[1].split(" page(s)")[0])
total_pages += pages
except:
pass
except Exception as file_error:
error_msg = f"❌ Error processing file: {str(file_error)}"
status_messages.append(error_msg)
if file_count > 0:
file_summary = f"📎 {file_count} file(s) uploaded"
if total_pages > 0:
file_summary += f" ({total_pages} PDF pages)"
content.insert(0, {"type": "text", "text": file_summary})
# Add text message
if message and message.strip():
content.append({"type": "text", "text": message})
# If no content at all, return error
if not content:
return history + [(message if message else "", "❌ No valid content to process")], "", "❌ No valid content"
messages.append({"role": "user", "content": content})
# Prepare API call parameters
api_params = {
"model": "nvidia/nemotron-nano-12b-v2-vl:free",
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens
}
# Add reasoning if enabled
if enable_reasoning:
api_params["extra_body"] = {"reasoning": {"enabled": True}}
# Make API call with additional error handling
try:
response = client.chat.completions.create(**api_params)
except Exception as api_error:
error_msg = f"❌ API Error: {str(api_error)}"
return history + [(message if message else "", error_msg)], "", error_msg
# Check if response is valid
if not response:
error_message = "❌ Error: Received None response from API"
return history + [(message if message else "", error_message)], "", error_message
if not hasattr(response, 'choices') or not response.choices or len(response.choices) == 0:
error_message = "❌ Error: Received empty response from API"
return history + [(message if message else "", error_message)], "", error_message
# Get the assistant message safely
try:
assistant_message = response.choices[0].message.content
if not assistant_message:
assistant_message = "⚠️ Model returned an empty response"
except (AttributeError, IndexError) as e:
assistant_message = f"❌ Error extracting response: {str(e)}"
# Extract reasoning if available
reasoning_text = ""
if enable_reasoning:
try:
if hasattr(response.choices[0].message, 'reasoning_details'):
reasoning_details = response.choices[0].message.reasoning_details
if reasoning_details:
reasoning_text = f"**🧠 Reasoning Process:**\n{json.dumps(reasoning_details, indent=2)}"
except Exception as reasoning_error:
# Reasoning extraction failed, but that's okay
pass
# Update history
display_message = message if message and message.strip() else "[Files uploaded]"
new_history = history + [(display_message, assistant_message)]
# Combine status messages
combined_status = "\n".join(status_messages) if status_messages else "✅ Message processed successfully"
return new_history, reasoning_text, combined_status
except Exception as e:
error_message = f"❌ Error: {str(e)}\n\nType: {type(e).__name__}"
import traceback
error_detail = traceback.format_exc()
print(f"Full error: {error_detail}") # For debugging
display_message = message if message and message.strip() else "[Error occurred]"
return history + [(display_message, error_message)], "", error_message
def clear_conversation():
"""Clear conversation history"""
return [], "", ""
def check_dependencies() -> str:
"""Check which PDF processing libraries are available"""
status = "**📦 PDF Processing Dependencies Status:**\n\n"
# Check PyMuPDF
try:
import fitz
status += "✅ **PyMuPDF (fitz)**: Installed and ready!\n"
status += " - No external dependencies needed\n"
status += " - This is the primary PDF processing method\n\n"
except ImportError:
status += "❌ **PyMuPDF (fitz)**: Not installed\n"
status += " - Install: `pip install PyMuPDF`\n\n"
# Check pdf2image
try:
import pdf2image
status += "✅ **pdf2image**: Installed\n"
status += " - Requires poppler-utils (external)\n"
# Try to check if poppler is available
try:
from pdf2image.exceptions import PDFInfoNotInstalledError
from pdf2image import pdfinfo_from_path
# This will throw an error if poppler is not found
status += " - Checking poppler availability...\n"
except:
status += " - ⚠️ poppler-utils may not be installed\n"
status += "\n"
except ImportError:
status += "⚠️ **pdf2image**: Not installed (optional fallback)\n"
status += " - Install: `pip install pdf2image`\n\n"
# Check PIL/Pillow
try:
from PIL import Image
status += "✅ **Pillow (PIL)**: Installed and ready!\n\n"
except ImportError:
status += "❌ **Pillow (PIL)**: Not installed\n"
status += " - Install: `pip install Pillow`\n\n"
status += "**💡 Recommendation:**\n"
status += "Install PyMuPDF for the best PDF support:\n"
status += "`pip install PyMuPDF Pillow`"
return status
# Custom CSS for premium design
custom_css = """
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
* {
font-family: 'Inter', sans-serif;
}
.gradio-container {
background: linear-gradient(135deg, rgb(145 228 242) 0%, rgb(118, 75, 162) 100%) !important;
}
#main-container {
background: rgba(255, 255, 255, 0.98);
border-radius: 24px;
padding: 32px;
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
backdrop-filter: blur(10px);
}
.header-title {
background: linear-gradient(135deg, rgb(108 58 198) 0%, rgb(18 121 44) 100%) text;
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
font-size: 3em;
font-weight: 700;
text-align: center;
margin-bottom: 0.3em;
letter-spacing: -0.02em;
}
.header-subtitle {
text-align: center;
color: #666;
font-size: 1.1em;
margin-bottom: 1.5em;
font-weight: 500;
}
.feature-badge {
display: inline-block;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 6px 16px;
border-radius: 20px;
font-size: 0.85em;
font-weight: 600;
margin: 4px;
box-shadow: 0 4px 12px rgba(102, 126, 234, 0.3);
}
.capability-card {
background: linear-gradient(135deg, #f6f8fb 0%, #ffffff 100%);
border: 2px solid #e0e7ff;
border-radius: 16px;
padding: 20px;
margin: 10px 0;
transition: all 0.3s ease;
}
.capability-card:hover {
transform: translateY(-4px);
box-shadow: 0 12px 24px rgba(102, 126, 234, 0.15);
border-color: #667eea;
}
.tab-nav button {
font-weight: 600 !important;
border-radius: 12px !important;
transition: all 0.3s ease !important;
}
.tab-nav button.selected {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%) !important;
color: white !important;
}
button.primary {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%) !important;
border: none !important;
color: white !important;
font-weight: 600 !important;
border-radius: 12px !important;
padding: 12px 32px !important;
transition: all 0.3s ease !important;
box-shadow: 0 4px 12px rgba(102, 126, 234, 0.3) !important;
}
button.primary:hover {
transform: translateY(-2px) !important;
box-shadow: 0 8px 20px rgba(102, 126, 234, 0.4) !important;
}
button.secondary {
background: white !important;
border: 2px solid #667eea !important;
color: #667eea !important;
font-weight: 600 !important;
border-radius: 12px !important;
transition: all 0.3s ease !important;
}
button.secondary:hover {
background: #f0f4ff !important;
}
.info-box {
background: linear-gradient(135deg, #e0e7ff 0%, #f0f4ff 100%);
border-left: 4px solid #667eea;
border-radius: 12px;
padding: 16px 20px;
margin: 16px 0;
font-size: 0.95em;
line-height: 1.6;
}
.success-box {
background: linear-gradient(135deg, #d4edda 0%, #e8f5e9 100%);
border-left: 4px solid #28a745;
border-radius: 12px;
padding: 16px 20px;
margin: 16px 0;
color: #155724;
font-weight: 500;
}
.chatbot {
border-radius: 16px !important;
border: 2px solid #e0e7ff !important;
box-shadow: 0 8px 24px rgba(102, 126, 234, 0.1) !important;
}
"""
# Build Gradio Interface
with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as app:
gr.HTML("""
📊 Document Intelligence
🎬 Video Understanding
🧠 Reasoning Engine
📈 Chart Analysis
🔤 OCR Excellence
""")
with gr.Row(elem_id="main-container"):
with gr.Column():
with gr.Tabs():
# Chat Tab
with gr.Tab("💬 Chat Interface", elem_classes=["tab-nav"]):
gr.HTML("""
🎯 What can I do?
• Analyze images, documents, and charts
• Perform OCR and text extraction from PDFs
• Reason through complex problems
• Answer questions about visual content
• Process multi-image documents and PDFs
""")
chatbot = gr.Chatbot(
label="Conversation",
height=500,
show_copy_button=True,
avatar_images=(None, "https://www.nvidia.com/favicon.ico"),
elem_classes=["chatbot"]
)
file_status = gr.Textbox(
label="📋 File Processing Status",
lines=2,
interactive=False,
visible=True
)
with gr.Row():
msg = gr.Textbox(
label="Your Message",
placeholder="Ask me anything about images, documents, PDFs, or reasoning tasks...",
lines=3,
scale=4
)
with gr.Row():
files = gr.File(
label="📎 Upload Files (Images, PDFs, Documents - Multi-file support)",
file_count="multiple",
file_types=[".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".tiff", ".pdf", ".txt"],
scale=3
)
with gr.Row():
submit_btn = gr.Button("🚀 Send", variant="primary", scale=2, elem_classes=["primary"])
clear_btn = gr.Button("🗑️ Clear", variant="secondary", scale=1, elem_classes=["secondary"])
reasoning_display = gr.Textbox(
label="🧠 Reasoning Process (when enabled)",
lines=6,
interactive=False
)
# Settings Tab
with gr.Tab("⚙️ Settings", elem_classes=["tab-nav"]):
gr.HTML("""
🔑 API Configuration
Get your free API key from
OpenRouter
""")
api_key_input = gr.Textbox(
label="OpenRouter API Key",
placeholder="sk-or-v1-...",
type="password",
lines=1
)
api_status = gr.Textbox(label="Status", interactive=False)
save_key_btn = gr.Button("💾 Save API Key", variant="primary", elem_classes=["primary"])
gr.HTML("
")
gr.HTML("""
🎛️ Model Parameters
Fine-tune the model's behavior
""")
enable_reasoning = gr.Checkbox(
label="🧠 Enable Reasoning Mode",
value=True,
info="Show the model's step-by-step thinking process"
)
temperature = gr.Slider(
minimum=0.0,
maximum=2.0,
value=0.7,
step=0.1,
label="🌡️ Temperature",
info="Higher = more creative, Lower = more focused"
)
max_tokens = gr.Slider(
minimum=256,
maximum=4096,
value=2000,
step=256,
label="📏 Max Tokens",
info="Maximum length of response"
)
gr.HTML("
")
gr.HTML("""
📦 Check Dependencies
Verify that PDF processing libraries are installed
""")
check_deps_btn = gr.Button("🔍 Check Dependencies", variant="secondary", elem_classes=["secondary"])
deps_status = gr.Markdown(label="Dependency Status")
gr.HTML("""
📦 Installation Guide:
Recommended (PyMuPDF - No external dependencies):
pip install PyMuPDF Pillow openai gradio
Alternative (pdf2image - Requires poppler):
pip install pdf2image Pillow openai gradio
Poppler installation (for pdf2image):
• Ubuntu/Debian:
sudo apt-get install poppler-utils
• macOS:
brew install poppler
• Windows: Download from
GitHub
""")
# File Support Tab
with gr.Tab("📁 Supported Files", elem_classes=["tab-nav"]):
gr.HTML("""
📁 Supported File Types
Nemotron Nano 2 VL accepts a wide variety of file formats
🖼️ Image Files
📷 JPEG/JPG
Standard photo format
🎨 PNG
Lossless with transparency
📐 TIFF
High-quality scans
📄 Document Files
📕 PDF Documents
• Multi-page support
• Automatic conversion to images
• PyMuPDF (recommended)
• Scanned documents
• Forms and tables
📝 Text Files (.txt)
• Plain text documents
• Code snippets
• Notes and logs
• UTF-8 encoding
• Configuration files
🚀 PDF Processing:
This app uses PyMuPDF (fitz) as the primary method for PDF conversion.
• ✅ No external dependencies (no poppler needed)
• ✅ Fast and reliable
• ✅ Automatic fallback to pdf2image if needed
• ✅ Clear error messages with installation instructions
""")
# Examples Tab
with gr.Tab("📚 Use Cases", elem_classes=["tab-nav"]):
gr.HTML("""
📊 Financial Report Analysis
Example: "Extract all the key metrics from this financial report"
What it extracts: Revenue, Net Profit, EBITDA, Cash Flow, Assets, Liabilities, Ratios, YoY Growth
🔤 OCR & Text Extraction
Example: "What text appears in this scanned document?"
State-of-the-art optical character recognition for any text in images or PDFs.
📈 Chart & Data Visualization
Example: "Analyze the trends in these charts"
Understand bar charts, line graphs, pie charts, scatter plots, and complex visualizations.
🧠 Advanced Reasoning
Example: "How many r's are in 'strawberry'? Think step by step."
Transparent reasoning process shows how the model arrives at answers.
📑 Multi-Page Documents
Example: Upload a PDF and ask "Summarize the key points from all pages"
Process entire documents with multiple pages simultaneously.
🏢 Business Document Processing
Example: "Extract information from this invoice/receipt/form"
Handle invoices, receipts, forms, contracts, and structured business documents.
""")
gr.HTML("""
💡 Pro Tips:
• Upload high-quality scans for best OCR results
• Enable reasoning mode for complex financial analysis
• Ask specific questions to get targeted information
• Upload multiple related documents for comparison
• Use clear, descriptive questions for better answers
""")
# About Tab
with gr.Tab("ℹ️ About", elem_classes=["tab-nav"]):
gr.Markdown("""
# 🚀 About Nemotron Nano 12B 2 VL
## 🎯 Model Overview
**NVIDIA Nemotron Nano 2 VL** is a cutting-edge 12-billion-parameter open multimodal reasoning model
designed for video understanding and document intelligence.
## ✨ Key Features
- **🏗️ Hybrid Architecture**: Combines Transformer accuracy with Mamba's efficient sequence modeling
- **⚡ High Performance**: Superior throughput and lower latency
- **📊 Leading Results**: ~74 average score across major benchmarks
- **🎯 Specialized Training**: NVIDIA-curated synthetic datasets
- **🎬 Video Support**: Efficient Video Sampling (EVS) for long-form content
- **📖 Open Source**: Released under permissive NVIDIA open license
## 📈 Benchmark Performance
Achieves leading results on:
- OCRBench v2
- MMMU
- MathVista
- AI2D
- OCR-Reasoning
- ChartQA
- DocVQA
- Video-MME
## 🔧 Deployment
Supported across:
- NVIDIA NeMo
- NVIDIA NIM
- Major inference runtimes
## 🌐 Learn More
- [OpenRouter API](https://openrouter.ai/)
- [NVIDIA NeMo](https://www.nvidia.com/en-us/ai-data-science/products/nemo/)
---
Built with ❤️ using Gradio and powered by NVIDIA Nemotron
""")
# Event Handlers
save_key_btn.click(
fn=initialize_client,
inputs=[api_key_input],
outputs=[api_status]
)
check_deps_btn.click(
fn=check_dependencies,
outputs=[deps_status]
)
submit_btn.click(
fn=process_message,
inputs=[msg, chatbot, files, enable_reasoning, temperature, max_tokens],
outputs=[chatbot, reasoning_display, file_status]
).then(
lambda: ("", None),
outputs=[msg, files]
)
msg.submit(
fn=process_message,
inputs=[msg, chatbot, files, enable_reasoning, temperature, max_tokens],
outputs=[chatbot, reasoning_display, file_status]
).then(
lambda: ("", None),
outputs=[msg, files]
)
clear_btn.click(
fn=clear_conversation,
outputs=[chatbot, reasoning_display, file_status]
)
# Launch the app
if __name__ == "__main__":
app.launch(
share=True
)