Spaces:
Running on Zero
Running on Zero
File size: 4,863 Bytes
a65585a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | """
Multimodal RAG Demo with Nemotron Embed VL and Rerank VL, Pinecone, Redis, and LangGraph
A Gradio demo for multimodal retrieval augmented generation using:
- Dataset: mrdbourke/recipe-synthetic-images-10k
- Embedding model: nvidia/llama-nemotron-embed-vl-1b-v2
- Rerank model: nvidia/llama-nemotron-rerank-vl-1b-v2
- Generation model: Qwen/Qwen3-VL-2B-Instruct
- Vector Store: Pinecone
- Cache: Redis Semantic Cache
- Orchestration: LangGraph
"""
import gradio as gr
import spaces
from PIL import Image
from models import redis_cache
from graph import graph
from utils import create_recipe_cards_html
# ============================================================================
# Main Retrieve Function
# ============================================================================
@spaces.GPU
def retrieve(
query_text: str | None,
query_image: Image.Image | None,
rerank_option: str,
generate_summary_option: str,
cache_threshold: float,
top_k: int
):
if query_text and query_text.strip():
input_query = query_text
elif query_image is not None:
input_query = query_image
else:
raise gr.Error("Please provide either a text query or an image query.")
initial_state = {
"input_query": input_query,
"rerank_option": rerank_option,
"generate_summary_option": generate_summary_option,
"cache_threshold": cache_threshold,
"top_k": top_k,
"timing_dict": {}
}
final_state = graph.invoke(initial_state)
docs = final_state.get("reranked_docs") or final_state.get("retrieved_docs", [])
output_image_gallery = [
(doc["sample"]["image"], doc.get("rerank_string", f"Score: {doc.get('score', 0)}"))
for doc in docs[:3]
]
output_recipe_cards_html = create_recipe_cards_html(
scores_and_samples=docs,
num_results=3
)
summary = final_state.get("summary", "No summary generated.")
timing_dict = final_state.get("timing_dict", {})
return output_image_gallery, output_recipe_cards_html, summary, timing_dict
# ============================================================================
# Gradio Interface
# ============================================================================
def clear_cache_ui():
print("[UI] Clear Cache button clicked.")
if redis_cache:
try:
redis_cache.client.flushdb()
redis_cache.create(overwrite=True)
return "✅ Cache cleared successfully!"
except Exception as e:
return f"❌ Error clearing cache: {e}"
return "❌ Cache not enabled."
with gr.Blocks(title="Multimodal RAG Demo") as demo:
gr.Markdown("""# 👁️📑 Multimodal RAG Demo with Nemotron, Pinecone, Redis & LangGraph""")
with gr.Row():
with gr.Column(scale=1):
query_text = gr.Textbox(label="Text Query", lines=2)
query_image = gr.Image(label="Image Query", type="pil", height=200)
generate_summary_option = gr.Radio(choices=["True", "False"], value="False", label="Generate summary")
rerank_option = gr.Radio(choices=["True", "False"], value="False", label="Rerank results")
cache_threshold = gr.Slider(minimum=0.0, maximum=1.0, value=0.15, step=0.01, label="Cache Vector Distance Threshold")
top_k_slider = gr.Slider(minimum=1, maximum=50, value=20, step=1, label="Retrieval Top K")
search_btn = gr.Button("Search", variant="primary", size="lg")
clear_cache_btn = gr.Button("Clear Cache", variant="secondary")
cache_status_msg = gr.Markdown("")
with gr.Column(scale=2):
gallery_output = gr.Gallery(label="Retrieved Images", columns=3, height="auto", object_fit="cover")
recipes_html = gr.HTML(label="Retrieved Texts")
summary_generation = gr.Markdown(label="Summary")
timing_output = gr.JSON(label="Timings")
gr.Examples(
examples=[
["best omelette recipes", None, "False", "False", 0.15, 20],
["best omelette recipes", None, "False", "True", 0.15, 20],
["best omelette recipes", None, "True", "True", 0.15, 20],
["eggplant dip", None, "True", "True", 0.15, 20]
],
inputs=[query_text, query_image, rerank_option, generate_summary_option, cache_threshold, top_k_slider],
label="Example Queries"
)
search_btn.click(
fn=retrieve,
inputs=[query_text, query_image, rerank_option, generate_summary_option, cache_threshold, top_k_slider],
outputs=[gallery_output, recipes_html, summary_generation, timing_output]
)
clear_cache_btn.click(
fn=clear_cache_ui,
inputs=[],
outputs=[cache_status_msg]
)
if __name__ == "__main__":
demo.launch()
|