import os
from typing import Tuple
import gradio as gr
import numpy as np
import cv2
import torch
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
from numpy import ndarray
import visualize
CSS = """
#desc, #desc * {
text-align: center !important;
justify-content: center !important;
align-items: center !important;
}
"""
DESCRIPTION = """
MapGlue 🗺️
MapGlue: Multimodal Remote Sensing Image Matching
Advanced feature matching system supporting various image modalities including:
SAR-Visible, Map-Visible, Depth-Visible, Infrared-Visible, Day-Night matching
"""
examples = [
[
"assets/day-night/L1.png",
"assets/day-night/R1.png",
],
[
"assets/day-night/L2.png",
"assets/day-night/R2.png",
],
[
"assets/depth-visible/L1.jpg",
"assets/depth-visible/R1.jpg",
],
[
"assets/depth-visible/L2.png",
"assets/depth-visible/R2.png",
],
[
"assets/infrared-visible/L1.png",
"assets/infrared-visible/R1.png",
],
[
"assets/infrared-visible/L2.png",
"assets/infrared-visible/R2.png",
],
[
"assets/map-visible/L1.jpg",
"assets/map-visible/R1.jpg",
],
[
"assets/map-visible/L2.png",
"assets/map-visible/R2.png",
],
[
"assets/sar-visible/L1.jpg",
"assets/sar-visible/R1.jpg",
],
[
"assets/sar-visible/L2.jpg",
"assets/sar-visible/R2.jpg",
],
[
"assets/sar-visible/L3.png",
"assets/sar-visible/R3.png",
],
]
def fig_to_ndarray(fig: Figure) -> ndarray:
"""Convert matplotlib figure to numpy array."""
fig.canvas.draw()
w, h = fig.canvas.get_width_height()
buffer = fig.canvas.buffer_rgba()
out = np.frombuffer(buffer, dtype=np.uint8).reshape(h, w, 4)
return out
def load_mapglue_model():
"""Load the MapGlue TorchScript model."""
# device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
device = 'cpu'
model_path = './weights/fastmapglue_model.pt'
if not os.path.exists(model_path):
raise FileNotFoundError(
f"Model file not found: {model_path}\n"
f"Please ensure the HF_TOKEN environment variable is set to download the model."
)
model = torch.jit.load(model_path, map_location=device)
model.eval()
model.to(device)
return model, device
def run_mapglue_matching(
path0: str,
path1: str,
model_name: str,
num_keypoints: int,
ransac_threshold: float,
) -> Tuple[ndarray, ndarray, ndarray, ndarray]:
"""
Run MapGlue matching on two input images using Homography RANSAC.
Args:
path0, path1: Paths to input images
model_name: Name of the matching model (currently supports FastMapGlue)
num_keypoints: Number of keypoints to extract
ransac_threshold: RANSAC reprojection threshold
Returns:
Tuple of (raw_keypoint_fig, raw_matching_fig, ransac_keypoint_fig, ransac_matching_fig)
"""
try:
# Load model
model, device = load_mapglue_model()
# Load and preprocess images
image0 = cv2.imread(path0)
image1 = cv2.imread(path1)
if image0 is None or image1 is None:
raise ValueError("Could not load one or both images")
# Convert BGR to RGB
image0 = cv2.cvtColor(image0, cv2.COLOR_BGR2RGB)
image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
# Convert to torch tensors
image0_tensor = torch.from_numpy(image0).to(device)
image1_tensor = torch.from_numpy(image1).to(device)
num_keypoints_tensor = torch.tensor(num_keypoints).to(device)
# Run inference
with torch.no_grad():
points_tensor = model(image0_tensor, image1_tensor, num_keypoints_tensor)
points0 = points_tensor[:, :2]
points1 = points_tensor[:, 2:]
# Create raw matching visualization
plt.figure(figsize=(12, 6))
axes = visualize.show_images([image0, image1])
visualize.draw_matches(points0, points1, line_colors="lime", line_width=0.8)
visualize.add_text(0, f'Raw matches: {len(points0)}', font_size=16)
raw_matching_fig = fig_to_ndarray(plt.gcf())
# Create raw keypoints visualization
plt.figure(figsize=(12, 6))
axes = visualize.show_images([image0, image1])
visualize.draw_keypoints([points0.cpu().numpy(), points1.cpu().numpy()],
kp_color=["lime", "lime"], kp_size=20)
visualize.add_text(0, f'Raw keypoints: {len(points0)}', font_size=16)
raw_keypoint_fig = fig_to_ndarray(plt.gcf())
# Apply RANSAC filtering
points0_np = points0.cpu().numpy()
points1_np = points1.cpu().numpy()
H_pred, inlier_mask = cv2.findHomography(
points0_np, points1_np,
cv2.USAC_MAGSAC,
ransacReprojThreshold=ransac_threshold,
maxIters=10000,
confidence=0.9999
)
if inlier_mask is not None and inlier_mask.sum() > 0:
inlier_mask = inlier_mask.ravel() > 0
mkpts0 = points0_np[inlier_mask]
mkpts1 = points1_np[inlier_mask]
# Create RANSAC matching visualization
plt.figure(figsize=(12, 6))
axes = visualize.show_images([image0, image1])
visualize.draw_matches(mkpts0, mkpts1, line_colors="lime", line_width=1)
visualize.add_text(0, f'RANSAC matches @{ransac_threshold}px: {len(mkpts0)}/{len(points0)}', font_size=16)
ransac_matching_fig = fig_to_ndarray(plt.gcf())
# Create RANSAC keypoints visualization
plt.figure(figsize=(12, 6))
axes = visualize.show_images([image0, image1])
visualize.draw_keypoints([mkpts0, mkpts1],
kp_color=["lime", "lime"], kp_size=20)
visualize.add_text(0, f'RANSAC keypoints @{ransac_threshold}px: {len(mkpts0)}', font_size=16)
ransac_keypoint_fig = fig_to_ndarray(plt.gcf())
else:
# No inliers found
ransac_matching_fig = None
ransac_keypoint_fig = None
plt.close('all') # Clean up matplotlib figures
return (
raw_keypoint_fig,
raw_matching_fig,
ransac_keypoint_fig,
ransac_matching_fig,
)
except Exception as e:
print(f"Error in matching: {str(e)}")
# Return empty arrays in case of error
empty_img = np.zeros((400, 800, 4), dtype=np.uint8)
return (empty_img, empty_img, empty_img, empty_img)
with gr.Blocks(css=CSS) as demo:
with gr.Tab("Image Matching"):
with gr.Row():
with gr.Column(scale=3):
gr.HTML(DESCRIPTION, elem_id="desc")
with gr.Row():
with gr.Column():
gr.Markdown("### Input Panels:")
with gr.Row():
model_name = gr.Dropdown(
choices=["FastMapGlue"],
value="FastMapGlue",
label="Matching Model",
)
with gr.Row():
path0 = gr.Image(
height=300,
image_mode="RGB",
type="filepath",
label="Image 0",
)
path1 = gr.Image(
height=300,
image_mode="RGB",
type="filepath",
label="Image 1",
)
with gr.Row():
stop = gr.Button(value="Stop", variant="stop")
run = gr.Button(value="Run", variant="primary")
with gr.Accordion("Advanced Settings", open=False):
with gr.Accordion("Matching Settings"):
with gr.Row():
num_keypoints = gr.Slider(
minimum=512,
maximum=4096,
value=2048,
step=256,
label="Number of Keypoints",
)
with gr.Accordion("RANSAC Settings"):
with gr.Row():
ransac_threshold = gr.Slider(
minimum=0.5,
maximum=10.0,
value=5.0,
step=0.5,
label="RANSAC Threshold",
)
with gr.Row():
with gr.Accordion("Example Pairs"):
gr.Examples(
examples=examples,
inputs=[path0, path1],
label="Click an example pair below",
)
with gr.Column():
gr.Markdown(
"### Output Panels"
)
with gr.Accordion("Raw Keypoints", open=False):
raw_keypoint_fig = gr.Image(
format="png", type="numpy", label="Raw Keypoints"
)
with gr.Accordion("Raw Matches"):
raw_matching_fig = gr.Image(
format="png", type="numpy", label="Raw Matches"
)
with gr.Accordion("RANSAC Keypoints", open=False):
ransac_keypoint_fig = gr.Image(
format="png", type="numpy", label="RANSAC Keypoints"
)
with gr.Accordion("RANSAC Matches"):
ransac_matching_fig = gr.Image(
format="png", type="numpy", label="RANSAC Matches"
)
inputs = [
path0,
path1,
model_name,
num_keypoints,
ransac_threshold,
]
outputs = [
raw_keypoint_fig,
raw_matching_fig,
ransac_keypoint_fig,
ransac_matching_fig,
]
running_event = run.click(
fn=run_mapglue_matching, inputs=inputs, outputs=outputs
)
stop.click(
fn=None, inputs=None, outputs=None, cancels=[running_event]
)
if __name__ == "__main__":
# Download model weights on startup if HF_TOKEN is available
HF_TOKEN = os.getenv("HF_TOKEN")
if HF_TOKEN:
model_path = './weights/fastmapglue_model.pt'
if not os.path.exists(model_path):
try:
import requests
# 使用 resolve 来直接下载文件
model_url = "https://huggingface.co/wupeihao/mapglue/resolve/main/fastmapglue_model.pt"
headers = {"Authorization": f"Bearer {HF_TOKEN}"}
print("Downloading MapGlue model...")
response = requests.get(model_url, headers=headers)
response.raise_for_status()
os.makedirs('./weights', exist_ok=True)
with open(model_path, 'wb') as f:
f.write(response.content)
print("Model downloaded successfully!")
except Exception as e:
print(f"Failed to download model: {str(e)}")
demo.launch()