"""
Inspired by https://www.loginradius.com/blog/engineering/guest-post/opencv-web-app-with-streamlit/
and https://medium.com/analytics-vidhya/finding-waldo-feature-matching-for-opencv-9bded7f5ab10
"""
import numpy as np
import cv2 as cv
import streamlit as st
from huggingface_hub import hf_hub_download
from streamlit_image_coordinates import streamlit_image_coordinates
@st.cache_data
def compute_correlation(scene: np.array, template: np.array):
"""
COMPUTE_CORRELATION computes the correlation between the pixels in scene and template
when the center of template is placed at location (x, y) on the scene. (x, y) is assumed
to be within bounds of the scene - this function doesn't check for out of bounds.
"""
gray_scene = cv.cvtColor(scene, cv.COLOR_BGR2GRAY)
cv.normalize(gray_scene, gray_scene, 0, 255, cv.NORM_MINMAX)
gray_template = cv.cvtColor(template, cv.COLOR_BGR2GRAY)
cv.normalize(gray_template, gray_template, 0, 255, cv.NORM_MINMAX)
res = cv.matchTemplate(gray_scene, gray_template, cv.TM_CCOEFF_NORMED)
return res
def compute_patch_correlation(scene_patch: np.array, template: np.array):
"""
Compute correlation score between a scene patch and template.
Returns a score between 0 and 1, where 1 is a perfect match.
"""
# Ensure patches are the same size
if scene_patch.shape != template.shape:
return 0.0
gray_patch = cv.cvtColor(scene_patch, cv.COLOR_BGR2GRAY)
cv.normalize(gray_patch, gray_patch, 0, 255, cv.NORM_MINMAX)
gray_template = cv.cvtColor(template, cv.COLOR_BGR2GRAY)
cv.normalize(gray_template, gray_template, 0, 255, cv.NORM_MINMAX)
# Use normalized cross-correlation
result = cv.matchTemplate(gray_patch, gray_template, cv.TM_CCOEFF_NORMED)
return float(result[0, 0]) if result.size > 0 else 0.0
def extract_patch(scene: np.array, x: int, y: int, template_shape):
"""
Extract a patch from the scene centered at (x, y) with the same size as the template.
"""
h, w = template_shape[0], template_shape[1]
# Calculate patch boundaries
y_start = max(0, y - h // 2)
y_end = min(scene.shape[0], y_start + h)
x_start = max(0, x - w // 2)
x_end = min(scene.shape[1], x_start + w)
# Adjust if we hit boundaries
if y_end - y_start < h:
y_start = max(0, y_end - h)
if x_end - x_start < w:
x_start = max(0, x_end - w)
patch = scene[y_start:y_end, x_start:x_end]
# Pad if necessary (edge cases)
if patch.shape[0] < h or patch.shape[1] < w:
patch = cv.copyMakeBorder(
patch,
0,
h - patch.shape[0],
0,
w - patch.shape[1],
cv.BORDER_CONSTANT,
value=[0, 0, 0],
)
return patch, (x_start, y_start, x_end, y_end)
@st.cache_resource
def load_images():
"""Load and cache the template and scene images."""
# Load template
template_path = hf_hub_download(
repo_id="amithjkamath/exampleimages",
filename="waldo-template.jpeg",
repo_type="dataset",
)
template_image = cv.imread(template_path)
template_image = cv.cvtColor(template_image, cv.COLOR_BGR2RGB)
# Load scene
scene_path = hf_hub_download(
repo_id="amithjkamath/exampleimages",
filename="waldo-scene.jpeg",
repo_type="dataset",
)
scene_image = cv.imread(scene_path)
scene_image = cv.cvtColor(scene_image, cv.COLOR_BGR2RGB)
return template_image, scene_image
def main_loop():
"""
MAIN_LOOP is the main loop (duh) for this streamlit App.
"""
st.set_page_config(layout="wide")
st.title("🔍 Interactive Template Matching Demo")
st.markdown(
"""
Welcome! This app teaches you how **template matching** works - a fundamental computer vision technique.
You'll learn by doing: click and drag the template around to see how computers "find" objects in images!
"""
)
# Load images (cached)
template_image, scene_image = load_images()
# Introduction
col1, col2 = st.columns([1, 3])
with col1:
st.markdown("### Meet Waldo 👋")
st.image(template_image, caption="Our template to find")
st.markdown(
"**Template size:** {}×{}".format(
template_image.shape[1], template_image.shape[0]
)
)
with col2:
st.markdown("### The Challenge")
st.markdown(
"""
Can you spot Waldo in this busy scene? Most people take 20+ seconds!
But computers can do it differently. Instead of using intuition, they use **template matching**:
- Slide the template over every possible position
- At each position, compute a **similarity score**
- The highest score reveals where Waldo is!
"""
)
# Show the scene
st.markdown("---")
st.markdown("### 🎯 Try It Yourself: Interactive Template Matching")
st.markdown(
"""
**Instructions:** Click anywhere on the image below to place the template there.
Watch how the correlation score changes! Can you find where Waldo actually is?
"""
)
# Initialize session state for template position
if "template_x" not in st.session_state:
st.session_state.template_x = scene_image.shape[1] // 4
st.session_state.template_y = scene_image.shape[0] // 4
if "last_computed_x" not in st.session_state:
st.session_state.last_computed_x = st.session_state.template_x
st.session_state.last_computed_y = st.session_state.template_y
if "computed_score" not in st.session_state:
st.session_state.computed_score = None
if "computed_patch" not in st.session_state:
st.session_state.computed_patch = None
# Create interactive image
col_left, col_right = st.columns([2, 1])
with col_left:
st.markdown("**Click on the image to move the template:**")
# Create overlay image with template
display_image = scene_image.copy()
t_h, t_w = template_image.shape[0], template_image.shape[1]
# Calculate template position (top-left corner)
x = st.session_state.template_x
y = st.session_state.template_y
x_start = max(0, x - t_w // 2)
y_start = max(0, y - t_h // 2)
x_end = min(scene_image.shape[1], x_start + t_w)
y_end = min(scene_image.shape[0], y_start + t_h)
# Draw rectangle around template position
cv.rectangle(
display_image, (x_start, y_start), (x_end, y_end), (255, 255, 0), 3
)
# Overlay semi-transparent template
overlay = display_image.copy()
if y_end - y_start == t_h and x_end - x_start == t_w:
overlay[y_start:y_end, x_start:x_end] = cv.addWeighted(
overlay[y_start:y_end, x_start:x_end], 0.5, template_image, 0.5, 0
)
display_image = cv.addWeighted(display_image, 0.7, overlay, 0.3, 0)
# Get click coordinates
value = streamlit_image_coordinates(display_image, key="scene_image")
# Update position only if clicked (value changed)
if value is not None:
new_x = value["x"]
new_y = value["y"]
# Only update if position actually changed
if (
new_x != st.session_state.template_x
or new_y != st.session_state.template_y
):
st.session_state.template_x = new_x
st.session_state.template_y = new_y
with col_right:
# Show current position
st.markdown("### 📍 Current Position")
st.markdown(f"**X:** {st.session_state.template_x}px")
st.markdown(f"**Y:** {st.session_state.template_y}px")
# Check if position has changed since last computation
position_changed = (
st.session_state.template_x != st.session_state.last_computed_x
or st.session_state.template_y != st.session_state.last_computed_y
)
# Button to compute match
if position_changed:
st.info("🔄 Position changed! Click below to compute match score.")
compute_button = st.button(
"🔍 Compute Match Score", type="primary", use_container_width=True
) # Will be updated to width='stretch' in future
# Compute correlation if button clicked or initial load
if compute_button or st.session_state.computed_score is None:
with st.spinner("Computing correlation..."):
# Extract patch and compute correlation
patch, _ = extract_patch(
scene_image,
st.session_state.template_x,
st.session_state.template_y,
template_image.shape[:2],
)
score = compute_patch_correlation(patch, template_image)
# Store computed values
st.session_state.computed_score = score
st.session_state.computed_patch = patch
st.session_state.last_computed_x = st.session_state.template_x
st.session_state.last_computed_y = st.session_state.template_y
# Display match score with color coding
st.markdown("### 📊 Match Score")
if st.session_state.computed_score is not None:
score = st.session_state.computed_score
# Determine match quality
if score >= 0.8:
quality = "🎉 Excellent Match!"
color = "green"
explanation = "This is very likely the correct location!"
elif score >= 0.6:
quality = "✅ Good Match"
color = "blue"
explanation = "Strong similarity, but maybe not perfect."
elif score >= 0.4:
quality = "⚠️ Moderate Match"
color = "orange"
explanation = "Some similarity, but probably not the right spot."
else:
quality = "❌ Poor Match"
color = "red"
explanation = "Very low similarity - keep searching!"
# Display score with highlighting
st.markdown(
f"""
""",
unsafe_allow_html=True,
)
st.markdown(f"*{explanation}*")
else:
st.warning("Click 'Compute Match Score' to analyze this position.")
# Show zoomed comparison
st.markdown("### 🔬 Close-up Comparison")
if st.session_state.computed_patch is not None:
st.markdown("**Template vs Current Patch:**")
# Create side-by-side comparison
comparison = np.hstack([template_image, st.session_state.computed_patch])
st.image(
comparison,
caption="Left: Template | Right: Current patch",
width="stretch",
)
else:
st.info("Compute match score to see the comparison.")
# Educational section: Show the full correlation heatmap
st.markdown("---")
st.markdown("### 🧠 How Does the Computer Find Waldo?")
with st.expander("Click here to see the full solution!", expanded=False):
st.markdown(
"""
The computer doesn't guess - it's systematic! It computes the correlation score at **every possible position**.
Here's the resulting **correlation heatmap** where brighter areas indicate better matches:
"""
)
corr = compute_correlation(scene_image, template_image)
norm_corr = (corr - corr.min()) / (corr.max() - corr.min())
col1, col2 = st.columns(2)
with col1:
st.image(
norm_corr,
caption="Correlation Heatmap (bright = high match)",
width="stretch",
)
st.markdown("Notice the bright spot? That's where Waldo is! 🎯")
with col2:
# Show result with bounding boxes
result_image = scene_image.copy()
threshold = 0.6
loc = np.where(corr >= threshold)
template_shape = template_image.shape
for pt in zip(*loc[::-1]):
cv.rectangle(
result_image,
pt,
(pt[0] + template_shape[1], pt[1] + template_shape[0]),
(0, 255, 0),
3,
)
st.image(
result_image,
caption="Detected locations (green boxes)",
width="stretch",
)
st.markdown("Green boxes show all locations with correlation > 0.6")
st.markdown(
"""
**Key Insight:** Template matching is a brute-force approach that checks every possible location.
While simple, it's very effective for finding exact or near-exact matches!
"""
)
# Footer
st.markdown("---")
st.markdown(
"""
🎨 Image copyrights for "Where's Waldo?" are fully attributed to original owners.
Used here purely for educational purposes.
""",
unsafe_allow_html=True,
)
if __name__ == "__main__":
main_loop()