Update Project Configs

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Ara Yeroyan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

README.md

@@ -1,18 +1,24 @@
 # Visual RAG Toolkit
 
 [![PyPI version](https://badge.fury.io/py/visual-rag-toolkit.svg)](https://badge.fury.io/py/visual-rag-toolkit)
+[![CI](https://github.com/Ara-Yeroyan/visual-rag-toolkit/actions/workflows/ci.yaml/badge.svg)](https://github.com/Ara-Yeroyan/visual-rag-toolkit/actions/workflows/ci.yaml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
 
-End-to-end visual document retrieval toolkit featuring **two-stage tile-level pooling for scalable search**. Supports multiple vision-language models (ColSmol, ColPali, ColQwen2, etc.) with a unified interface.
+End-to-end visual document retrieval toolkit featuring **fast multi-stage retrieval** (prefetch with pooled vectors + exact MaxSim reranking).
+
+This repo contains:
+- a **Python package** (`visual_rag`)
+- a **Streamlit demo app** (`demo/`)
+- **benchmark & evaluation scripts** for ViDoRe v2 (`benchmarks/`)
 
 ## 🎯 Key Features
 
-- **Modular Architecture** - Use only what you need: PDF processing, embedding, indexing, or retrieval independently
-- **Two-Stage Retrieval** - Our novel contribution: fast pooled prefetch + exact MaxSim reranking
-- **Multi-Model Support** - Works with ColSmol-500M, ColPali, ColQwen2, and more
-- **Special Token Handling** - Proper filtering of special tokens from query embeddings
-- **Production Ready** - Qdrant integration, Cloudinary uploads, batch processing, GPU support
+- **Modular**: PDF → images, embedding, Qdrant indexing, retrieval can be used independently.
+- **Multi-stage retrieval**: two-stage and three-stage retrieval modes built for Qdrant named vectors.
+- **Model-aware embedding**: ColSmol + ColPali support behind a single `VisualEmbedder` interface.
+- **Token hygiene**: query special-token filtering by default for more stable MaxSim behavior.
+- **Practical pipelines**: robust indexing, retries, optional Cloudinary image URLs, evaluation reporting.
 
 ## 📦 Installation
 
@@ -21,65 +27,65 @@ End-to-end visual document retrieval toolkit featuring **two-stage tile-level po
 pip install visual-rag-toolkit
 
 # With specific features
-pip install visual-rag-toolkit[embedding]    # ColPali model support
+pip install visual-rag-toolkit[embedding]    # ColSmol/ColPali embedding support
 pip install visual-rag-toolkit[pdf]          # PDF processing
 pip install visual-rag-toolkit[qdrant]       # Vector database
 pip install visual-rag-toolkit[cloudinary]   # Image CDN
+pip install visual-rag-toolkit[ui]           # Streamlit demo dependencies
 
 # All dependencies
 pip install visual-rag-toolkit[all]
 ```
 
-## 🚀 Quick Start
+### System dependencies (PDF)
 
-### Complete Pipeline
+`pdf2image` requires Poppler.
 
-```python
-from visual_rag import VisualEmbedder, PDFProcessor, TwoStageRetriever
+- macOS: `brew install poppler`
+- Ubuntu/Debian: `sudo apt-get update && sudo apt-get install -y poppler-utils`
 
-# 1. Process PDFs
-processor = PDFProcessor(dpi=140)
-images, texts = processor.process_pdf("report.pdf")
+## 🚀 Quick Start
 
-# 2. Generate embeddings
-embedder = VisualEmbedder(model_name="vidore/colSmol-500M")
-embeddings = embedder.embed_images(images)
+### Minimal: embed a query and run two-stage search (server-side)
 
-# 3. Search with two-stage retrieval
-query_emb = embedder.embed_query("What is the budget allocation?")
-retriever = TwoStageRetriever(qdrant_client, "my_collection")
-results = retriever.search(query_emb, top_k=10, prefetch_k=200)
+```python
+from qdrant_client import QdrantClient
+from visual_rag import VisualEmbedder, TwoStageRetriever
+
+client = QdrantClient(url="https://YOUR_QDRANT", api_key="YOUR_KEY")
+collection_name = "your_collection"
+
+# Embed query tokens
+embedder = VisualEmbedder(model_name="vidore/colpali-v1.3")
+q = embedder.embed_query("What is the budget allocation?")
+
+# Fast path: all stages computed in Qdrant (prefetch + exact rerank)
+retriever = TwoStageRetriever(client, collection_name)
+results = retriever.search_server_side(
+    query_embedding=q,
+    top_k=10,
+    prefetch_k=256,
+    stage1_mode="tokens_vs_experimental",  # or: tokens_vs_tiles / pooled_query_vs_tiles / pooled_query_vs_global
+)
+
+for r in results[:3]:
+    print(r["id"], r["score_final"])
 ```
 
-### Use Components Independently
-
-Each component works on its own - pick what you need:
+### Process a PDF into images (no embedding, no vector DB)
 
 ```python
-# Just PDF processing (no embedding, no vector DB)
-from visual_rag.indexing import PDFProcessor
-processor = PDFProcessor()
-images, texts = processor.process_pdf("doc.pdf")
-
-# Just embedding (bring your own images)
-from visual_rag.embedding import VisualEmbedder
-embedder = VisualEmbedder()
-embeddings = embedder.embed_images(my_images)
-
-# Just Qdrant indexing (bring your own embeddings)
-from visual_rag.indexing import QdrantIndexer
-indexer = QdrantIndexer(url="...", api_key="...", collection_name="my_col")
-indexer.upload_batch(my_points)
-
-# Just retrieval (use existing collection)
-from visual_rag.retrieval import TwoStageRetriever
-retriever = TwoStageRetriever(client, "my_col")
-results = retriever.search(query_embedding)
+from pathlib import Path
+from visual_rag import PDFProcessor
+
+processor = PDFProcessor(dpi=140)
+images, texts = processor.process_pdf(Path("report.pdf"))
+print(len(images), "pages")
 ```
 
-## 🔬 Two-Stage Retrieval (Our Novel Contribution)
+## 🔬 Multi-stage Retrieval (Two-stage / Three-stage)
 
-Traditional ColBERT/ColPali uses exhaustive MaxSim scoring which is O(N × M) where N = query tokens and M = doc tokens. This doesn't scale.
+Traditional ColBERT-style MaxSim scoring compares all query tokens vs all document tokens, which becomes expensive at scale.
 
 **Our approach:**
 
@@ -95,11 +101,7 @@ Stage 2: Exact MaxSim reranking on candidates
          └── Return top-k results (e.g., 10)
 ```
 
-**Benefits:**
-- 🚀 **5-10x faster** than full MaxSim at scale
-- 🎯 **95%+ accuracy** compared to exhaustive search  
-- 💾 **Memory efficient** - don't load all embeddings upfront
-- 📈 **Scalable** - works with millions of documents
+Three-stage extends this with an additional “cheap prefetch” stage before stage 2.
 
 ## 📁 Package Structure
 
@@ -122,9 +124,14 @@ visual-rag-toolkit/
 Configure via environment variables or YAML:
 
 ```bash
-# Environment variables
+# Qdrant credentials (preferred names used by the demo + scripts)
+export SIGIR_QDRANT_URL="https://your-cluster.qdrant.io"
+export SIGIR_QDRANT_KEY="your-api-key"
+
+# Backwards-compatible fallbacks (also supported)
 export QDRANT_URL="https://your-cluster.qdrant.io"
 export QDRANT_API_KEY="your-api-key"
+
 export VISUALRAG_MODEL="vidore/colSmol-500M"
 
 # Special token handling (default: filter them out)
@@ -148,27 +155,36 @@ search:
   top_k: 10
 ```
 
-## 📊 Benchmark Evaluation
-
-Run ViDoRe benchmark evaluation:
+## 🖥️ Demo (Streamlit)
 
 ```bash
-cd benchmarks/
+pip install "visual-rag-toolkit[ui,qdrant,embedding,pdf]"
+streamlit run demo/app.py
+```
 
-# Single dataset
-python run_vidore.py --dataset vidore/docvqa_test_subsampled
+## 📊 Benchmark Evaluation
 
-# With two-stage retrieval
-python run_vidore.py --dataset vidore/docvqa_test_subsampled --two-stage
+Run ViDoRe benchmark evaluation:
 
-# All datasets
-python run_vidore.py --all
+```bash
+# Example: evaluate a collection against ViDoRe BEIR datasets in Qdrant
+python -m benchmarks.vidore_beir_qdrant.run_qdrant_beir \
+  --datasets vidore/esg_reports_v2 vidore/biomedical_lectures_v2 \
+  --collection YOUR_COLLECTION \
+  --mode two_stage \
+  --stage1-mode tokens_vs_experimental \
+  --prefetch-k 256 \
+  --top-k 100 \
+  --evaluation-scope union
 ```
 
+More commands (including multi-stage variants and cropping configs) live in:
+- `benchmarks/vidore_tatdqa_test/COMMANDS.md`
+
 ## 🔧 Development
 
 ```bash
-git clone https://github.com/your-org/visual-rag-toolkit
+git clone https://github.com/Ara-Yeroyan/visual-rag-toolkit
 cd visual-rag-toolkit
 pip install -e ".[dev]"
 pytest tests/ -v
@@ -181,9 +197,9 @@ If you use this toolkit in your research, please cite:
 ```bibtex
 @software{visual_rag_toolkit,
   title = {Visual RAG Toolkit: Scalable Visual Document Retrieval with Two-Stage Pooling},
-  author = {Your Name},
-  year = {2024},
-  url = {https://github.com/your-org/visual-rag-toolkit}
+  author = {Ara Yeroyan},
+  year = {2026},
+  url = {https://github.com/Ara-Yeroyan/visual-rag-toolkit}
 }
 ```
 
@@ -193,7 +209,7 @@ MIT License - see [LICENSE](LICENSE) for details.
 
 ## 🙏 Acknowledgments
 
-- [ColPali](https://github.com/illuin-tech/colpali) - Visual document retrieval models
 - [Qdrant](https://qdrant.tech/) - Vector database with multi-vector support
+- [ColPali](https://github.com/illuin-tech/colpali) - Visual document retrieval models
 - [ViDoRe](https://huggingface.co/spaces/vidore/vidore-leaderboard) - Benchmark dataset
 

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling>=1.21.0"]
 build-backend = "hatchling.build"
 
 [project]
@@ -7,7 +7,7 @@ name = "visual-rag-toolkit"
 version = "0.1.0"
 description = "End-to-end visual document retrieval with ColPali, featuring two-stage pooling for scalable search"
 readme = "README.md"
-license = {text = "MIT"}
+license = {file = "LICENSE"}
 requires-python = ">=3.9"
 authors = [
     {name = "Visual RAG Team"}
@@ -77,7 +77,19 @@ ui = [
 
 # All dependencies
 all = [
-    "visual-rag-toolkit[embedding,pdf,qdrant,cloudinary,ui]",
+    # embedding
+    "colpali-engine>=0.3.0",
+    "transformers>=4.35.0",
+    # pdf
+    "pdf2image>=1.16.0",
+    "pypdf>=3.0.0",
+    # qdrant
+    "qdrant-client>=1.7.0",
+    # cloudinary
+    "cloudinary>=1.30.0",
+    # ui
+    "streamlit>=1.25.0",
+    "httpx>=0.24.0",
 ]
 
 # Development dependencies
@@ -93,13 +105,29 @@ dev = [
 visual-rag = "visual_rag.cli.main:main"
 
 [project.urls]
-Homepage = "https://github.com/your-org/visual-rag-toolkit"
-Documentation = "https://github.com/your-org/visual-rag-toolkit#readme"
-Repository = "https://github.com/your-org/visual-rag-toolkit"
-Issues = "https://github.com/your-org/visual-rag-toolkit/issues"
+Homepage = "https://github.com/Ara-Yeroyan/visual-rag-toolkit"
+Documentation = "https://github.com/Ara-Yeroyan/visual-rag-toolkit#readme"
+Repository = "https://github.com/Ara-Yeroyan/visual-rag-toolkit"
+Issues = "https://github.com/Ara-Yeroyan/visual-rag-toolkit/issues"
 
 [tool.hatch.build.targets.wheel]
-packages = ["visual_rag"]
+packages = ["visual_rag", "benchmarks"]
+
+[tool.hatch.build.targets.sdist]
+# Keep source distributions clean (no local artifacts, eval results, caches).
+exclude = [
+  "/results",
+  "/checkpoints",
+  "/.cache",
+  "/.streamlit",
+  "/.env",
+  "/.env.*",
+  "/**/__pycache__",
+  "/**/*.pyc",
+  "/**/*.pyo",
+  "/**/*.pyd",
+  "/**/.DS_Store",
+]
 
 [tool.ruff]
 line-length = 100

requirements.txt

@@ -0,0 +1,25 @@
+# Core dependencies
+numpy>=1.21.0
+Pillow>=9.0.0
+tqdm>=4.60.0
+pyyaml>=6.0
+python-dotenv>=0.19.0
+
+# Vector database
+qdrant-client>=1.7.0
+
+# Streamlit UI
+streamlit>=1.28.0
+httpx>=0.24.0
+
+# Data processing
+pandas
+altair
+datasets
+
+# PDF processing
+pdf2image>=1.16.0
+pypdf>=3.0.0
+
+# ColPali dependencies (torch/transformers come from base image)
+peft>=0.13.0