Spaces:
Running
Running
wxl
commited on
Commit
·
93bde9f
1
Parent(s):
4150f51
add get bev for front view image
Browse files- app.py +101 -4
- figure/exp1.jpg +0 -0
- models/utils/__pycache__/torch_geometry.cpython-38.pyc +0 -0
- models/utils/torch_geometry.py +496 -0
app.py
CHANGED
|
@@ -1,7 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import gradio as gr
|
|
|
|
|
|
|
| 2 |
|
| 3 |
-
|
| 4 |
-
return "Hello " + name + "!!"
|
| 5 |
|
| 6 |
-
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import torch
|
| 3 |
+
import scipy.io as io
|
| 4 |
+
import numpy as np
|
| 5 |
+
import warnings
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
import gradio as gr
|
| 8 |
+
import torchgeometry as tgm
|
| 9 |
+
from models.utils.torch_geometry import get_perspective_transform, warp_perspective
|
| 10 |
|
| 11 |
+
warnings.filterwarnings("ignore")
|
|
|
|
| 12 |
|
| 13 |
+
def get_BEV_kitti(front_img, fov, pitch, scale, out_size):
|
| 14 |
+
Hp, Wp = front_img.shape[:2]
|
| 15 |
+
|
| 16 |
+
Wo,Ho = int(Wp*scale),int(Wp*scale)
|
| 17 |
+
|
| 18 |
+
fov = fov *torch.pi/180 #
|
| 19 |
+
theta = pitch*torch.pi/180 # Camera pitch angle
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
f = Hp/2/torch.tan(torch.tensor(fov))
|
| 23 |
+
phi = torch.pi/2 - fov
|
| 24 |
+
delta = torch.pi/2+theta - torch.tensor(phi)
|
| 25 |
+
l = torch.sqrt(f**2+(Hp/2)**2)
|
| 26 |
+
h = l*torch.sin(delta)
|
| 27 |
+
f_ = l*torch.cos(delta)
|
| 28 |
+
|
| 29 |
+
######################
|
| 30 |
+
|
| 31 |
+
frame = torch.from_numpy(front_img).to(device)
|
| 32 |
+
|
| 33 |
+
out = torch.zeros((2, 2,2)).to(device)
|
| 34 |
+
|
| 35 |
+
y = (torch.ones((2, 2)).to(device).T *(torch.arange(0,Ho, step=Ho-1)).to(device)).T
|
| 36 |
+
x = torch.ones((2, 2)).to(device) *torch.arange(0, Wo, step=Wo-1).to(device)
|
| 37 |
+
l0 = torch.ones((2, 2)).to(device)*Ho - y
|
| 38 |
+
l1 = torch.ones((2, 2)).to(device) * f_+ l0
|
| 39 |
+
|
| 40 |
+
f1_0 = torch.arctan(h/l1)
|
| 41 |
+
f1_1 = torch.ones((2, 2)).to(device)*(torch.pi/2+theta) - f1_0
|
| 42 |
+
y_ = l0*torch.sin(f1_0)/torch.sin(f1_1)
|
| 43 |
+
j_p = torch.ones((2, 2)).to(device) * Hp - y_
|
| 44 |
+
i_p = torch.ones((2, 2)).to(device) * Wp/2 -(f_+torch.sin(torch.tensor(theta))*(torch.ones((2, 2)).to(device)*Hp-j_p))*(Wo/2*torch.ones((2, 2)).to(device)-x)/l1
|
| 45 |
+
|
| 46 |
+
out[:,:,0] = i_p.reshape((2, 2))
|
| 47 |
+
out[:,:,1] = j_p.reshape((2, 2))
|
| 48 |
+
|
| 49 |
+
four_point_org = out.permute(2,0,1)
|
| 50 |
+
four_point_new = torch.stack((x,y), dim = -1).permute(2,0,1)
|
| 51 |
+
four_point_org = four_point_org.unsqueeze(0).flatten(2).permute(0, 2, 1)
|
| 52 |
+
four_point_new = four_point_new.unsqueeze(0).flatten(2).permute(0, 2, 1)
|
| 53 |
+
H = get_perspective_transform(four_point_org, four_point_new)
|
| 54 |
+
|
| 55 |
+
scale1,scale2 = out_size/Wo,out_size/Ho
|
| 56 |
+
T3 = np.array([[scale1, 0, 0], [0, scale2, 0], [0, 0, 1]])
|
| 57 |
+
Homo = torch.matmul(torch.tensor(T3).unsqueeze(0).to(device).float(), H)
|
| 58 |
+
BEV = warp_perspective(frame.permute(2,0,1).unsqueeze(0).float(), Homo, (out_size,out_size))
|
| 59 |
+
|
| 60 |
+
BEV = BEV[0].cpu().int().permute(1,2,0).numpy().astype(np.uint8)
|
| 61 |
+
|
| 62 |
+
return BEV
|
| 63 |
+
|
| 64 |
+
@torch.no_grad()
|
| 65 |
+
def KittiBEV():
|
| 66 |
+
torch.cuda.empty_cache()
|
| 67 |
+
|
| 68 |
+
with gr.Blocks() as demo:
|
| 69 |
+
gr.Markdown(
|
| 70 |
+
"""
|
| 71 |
+
# HC-Net: Fine-Grained Cross-View Geo-Localization Using a Correlation-Aware Homography Estimator
|
| 72 |
+
## Get BEV from front-view image.
|
| 73 |
+
""")
|
| 74 |
+
|
| 75 |
+
with gr.Row():
|
| 76 |
+
front_img = gr.Image(label="Front-view Image").style(height=450)
|
| 77 |
+
BEV_output = gr.Image(label="BEV Image").style(height=450)
|
| 78 |
+
|
| 79 |
+
fov = gr.Slider(1,90, value=20, label="FOV")
|
| 80 |
+
pitch = gr.Slider(-180, 180, value=0, label="Pitch")
|
| 81 |
+
scale = gr.Slider(1, 10, value=1.0, label="Scale")
|
| 82 |
+
out_size = gr.Slider(500, 1000, value=500, label="Out size")
|
| 83 |
+
btn = gr.Button(value="Get BEV Image")
|
| 84 |
+
btn.click(get_BEV_kitti,inputs= [front_img, fov, pitch, scale, out_size], outputs=BEV_output, queue=False)
|
| 85 |
+
gr.Markdown(
|
| 86 |
+
"""
|
| 87 |
+
### Note:
|
| 88 |
+
- If you wish to acquire **quantitative localization error results** for your uploaded data, kindly supply the real GPS for the ground image as well as the corresponding GPS for the center of the satellite image.
|
| 89 |
+
- When inputting GPS coordinates, please make sure their precision extends to **at least six decimal places**.
|
| 90 |
+
""")
|
| 91 |
+
|
| 92 |
+
gr.Markdown("## Image Examples")
|
| 93 |
+
gr.Examples(
|
| 94 |
+
examples=[['./figure/exp1.jpg', 27, 7, 6, 1000]],
|
| 95 |
+
inputs= [front_img, fov, pitch, scale, out_size],
|
| 96 |
+
outputs=[BEV_output],
|
| 97 |
+
fn=get_BEV_kitti,
|
| 98 |
+
cache_examples=False,
|
| 99 |
+
)
|
| 100 |
+
demo.launch(server_port=7981)
|
| 101 |
+
|
| 102 |
+
if __name__ == '__main__':
|
| 103 |
+
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
| 104 |
+
KittiBEV()
|
figure/exp1.jpg
ADDED
|
models/utils/__pycache__/torch_geometry.cpython-38.pyc
ADDED
|
Binary file (16 kB). View file
|
|
|
models/utils/torch_geometry.py
ADDED
|
@@ -0,0 +1,496 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
import functools
|
| 5 |
+
from typing import Tuple, Optional
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
##########################
|
| 9 |
+
#### from pytorch3d ####
|
| 10 |
+
##########################
|
| 11 |
+
def _axis_angle_rotation(axis: str, angle):
|
| 12 |
+
"""
|
| 13 |
+
Return the rotation matrices for one of the rotations about an axis
|
| 14 |
+
of which Euler angles describe, for each value of the angle given.
|
| 15 |
+
|
| 16 |
+
Args:
|
| 17 |
+
axis: Axis label "X" or "Y or "Z".
|
| 18 |
+
angle: any shape tensor of Euler angles in radians
|
| 19 |
+
|
| 20 |
+
Returns:
|
| 21 |
+
Rotation matrices as tensor of shape (..., 3, 3).
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
cos = torch.cos(angle)
|
| 25 |
+
sin = torch.sin(angle)
|
| 26 |
+
one = torch.ones_like(angle)
|
| 27 |
+
zero = torch.zeros_like(angle)
|
| 28 |
+
|
| 29 |
+
if axis == "X":
|
| 30 |
+
R_flat = (one, zero, zero, zero, cos, -sin, zero, sin, cos)
|
| 31 |
+
if axis == "Y":
|
| 32 |
+
R_flat = (cos, zero, sin, zero, one, zero, -sin, zero, cos)
|
| 33 |
+
if axis == "Z":
|
| 34 |
+
R_flat = (cos, -sin, zero, sin, cos, zero, zero, zero, one)
|
| 35 |
+
|
| 36 |
+
return torch.stack(R_flat, -1).reshape(angle.shape + (3, 3))
|
| 37 |
+
|
| 38 |
+
def euler_angles_to_matrix(euler_angles, convention: str):
|
| 39 |
+
"""
|
| 40 |
+
Convert rotations given as Euler angles in radians to rotation matrices.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
euler_angles: Euler angles in radians as tensor of shape (..., 3).
|
| 44 |
+
convention: Convention string of three uppercase letters from
|
| 45 |
+
{"X", "Y", and "Z"}.
|
| 46 |
+
|
| 47 |
+
Returns:
|
| 48 |
+
Rotation matrices as tensor of shape (..., 3, 3).
|
| 49 |
+
"""
|
| 50 |
+
if euler_angles.dim() == 0 or euler_angles.shape[-1] != 3:
|
| 51 |
+
raise ValueError("Invalid input euler angles.")
|
| 52 |
+
if len(convention) != 3:
|
| 53 |
+
raise ValueError("Convention must have 3 letters.")
|
| 54 |
+
if convention[1] in (convention[0], convention[2]):
|
| 55 |
+
raise ValueError(f"Invalid convention {convention}.")
|
| 56 |
+
for letter in convention:
|
| 57 |
+
if letter not in ("X", "Y", "Z"):
|
| 58 |
+
raise ValueError(f"Invalid letter {letter} in convention string.")
|
| 59 |
+
matrices = map(_axis_angle_rotation, convention, torch.unbind(euler_angles, -1))
|
| 60 |
+
return functools.reduce(torch.matmul, matrices)
|
| 61 |
+
|
| 62 |
+
###########################
|
| 63 |
+
#### from pytorchgemotry ####
|
| 64 |
+
###########################
|
| 65 |
+
def get_perspective_transform(src, dst):
|
| 66 |
+
r"""Calculates a perspective transform from four pairs of the corresponding
|
| 67 |
+
points.
|
| 68 |
+
|
| 69 |
+
The function calculates the matrix of a perspective transform so that:
|
| 70 |
+
|
| 71 |
+
.. math ::
|
| 72 |
+
|
| 73 |
+
\begin{bmatrix}
|
| 74 |
+
t_{i}x_{i}^{'} \\
|
| 75 |
+
t_{i}y_{i}^{'} \\
|
| 76 |
+
t_{i} \\
|
| 77 |
+
\end{bmatrix}
|
| 78 |
+
=
|
| 79 |
+
\textbf{map_matrix} \cdot
|
| 80 |
+
\begin{bmatrix}
|
| 81 |
+
x_{i} \\
|
| 82 |
+
y_{i} \\
|
| 83 |
+
1 \\
|
| 84 |
+
\end{bmatrix}
|
| 85 |
+
|
| 86 |
+
where
|
| 87 |
+
|
| 88 |
+
.. math ::
|
| 89 |
+
dst(i) = (x_{i}^{'},y_{i}^{'}), src(i) = (x_{i}, y_{i}), i = 0,1,2,3
|
| 90 |
+
|
| 91 |
+
Args:
|
| 92 |
+
src (Tensor): coordinates of quadrangle vertices in the source image.
|
| 93 |
+
dst (Tensor): coordinates of the corresponding quadrangle vertices in
|
| 94 |
+
the destination image.
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
Tensor: the perspective transformation.
|
| 98 |
+
|
| 99 |
+
Shape:
|
| 100 |
+
- Input: :math:`(B, 4, 2)` and :math:`(B, 4, 2)`
|
| 101 |
+
- Output: :math:`(B, 3, 3)`
|
| 102 |
+
"""
|
| 103 |
+
if not torch.is_tensor(src):
|
| 104 |
+
raise TypeError("Input type is not a torch.Tensor. Got {}"
|
| 105 |
+
.format(type(src)))
|
| 106 |
+
if not torch.is_tensor(dst):
|
| 107 |
+
raise TypeError("Input type is not a torch.Tensor. Got {}"
|
| 108 |
+
.format(type(dst)))
|
| 109 |
+
if not src.shape[-2:] == (4, 2):
|
| 110 |
+
raise ValueError("Inputs must be a Bx4x2 tensor. Got {}"
|
| 111 |
+
.format(src.shape))
|
| 112 |
+
if not src.shape == dst.shape:
|
| 113 |
+
raise ValueError("Inputs must have the same shape. Got {}"
|
| 114 |
+
.format(dst.shape))
|
| 115 |
+
if not (src.shape[0] == dst.shape[0]):
|
| 116 |
+
raise ValueError("Inputs must have same batch size dimension. Got {}"
|
| 117 |
+
.format(src.shape, dst.shape))
|
| 118 |
+
|
| 119 |
+
def ax(p, q):
|
| 120 |
+
ones = torch.ones_like(p)[..., 0:1]
|
| 121 |
+
zeros = torch.zeros_like(p)[..., 0:1]
|
| 122 |
+
return torch.cat(
|
| 123 |
+
[p[:, 0:1], p[:, 1:2], ones, zeros, zeros, zeros,
|
| 124 |
+
-p[:, 0:1] * q[:, 0:1], -p[:, 1:2] * q[:, 0:1]
|
| 125 |
+
], dim=1)
|
| 126 |
+
|
| 127 |
+
def ay(p, q):
|
| 128 |
+
ones = torch.ones_like(p)[..., 0:1]
|
| 129 |
+
zeros = torch.zeros_like(p)[..., 0:1]
|
| 130 |
+
return torch.cat(
|
| 131 |
+
[zeros, zeros, zeros, p[:, 0:1], p[:, 1:2], ones,
|
| 132 |
+
-p[:, 0:1] * q[:, 1:2], -p[:, 1:2] * q[:, 1:2]], dim=1)
|
| 133 |
+
# we build matrix A by using only 4 point correspondence. The linear
|
| 134 |
+
# system is solved with the least square method, so here
|
| 135 |
+
# we could even pass more correspondence
|
| 136 |
+
p = []
|
| 137 |
+
p.append(ax(src[:, 0], dst[:, 0]))
|
| 138 |
+
p.append(ay(src[:, 0], dst[:, 0]))
|
| 139 |
+
|
| 140 |
+
p.append(ax(src[:, 1], dst[:, 1]))
|
| 141 |
+
p.append(ay(src[:, 1], dst[:, 1]))
|
| 142 |
+
|
| 143 |
+
p.append(ax(src[:, 2], dst[:, 2]))
|
| 144 |
+
p.append(ay(src[:, 2], dst[:, 2]))
|
| 145 |
+
|
| 146 |
+
p.append(ax(src[:, 3], dst[:, 3]))
|
| 147 |
+
p.append(ay(src[:, 3], dst[:, 3]))
|
| 148 |
+
|
| 149 |
+
# A is Bx8x8
|
| 150 |
+
A = torch.stack(p, dim=1)
|
| 151 |
+
|
| 152 |
+
# b is a Bx8x1
|
| 153 |
+
b = torch.stack([
|
| 154 |
+
dst[:, 0:1, 0], dst[:, 0:1, 1],
|
| 155 |
+
dst[:, 1:2, 0], dst[:, 1:2, 1],
|
| 156 |
+
dst[:, 2:3, 0], dst[:, 2:3, 1],
|
| 157 |
+
dst[:, 3:4, 0], dst[:, 3:4, 1],
|
| 158 |
+
], dim=1)
|
| 159 |
+
|
| 160 |
+
# solve the system Ax = b
|
| 161 |
+
# X, LU = torch.gesv(b, A)
|
| 162 |
+
X = torch.linalg.solve(A, b)
|
| 163 |
+
|
| 164 |
+
# create variable to return
|
| 165 |
+
batch_size = src.shape[0]
|
| 166 |
+
M = torch.ones(batch_size, 9, device=src.device, dtype=src.dtype)
|
| 167 |
+
M[..., :8] = torch.squeeze(X, dim=-1)
|
| 168 |
+
return M.view(-1, 3, 3) # Bx3x3
|
| 169 |
+
|
| 170 |
+
def warp_perspective(src, M, dsize, flags='bilinear', border_mode=None,
|
| 171 |
+
border_value=0):
|
| 172 |
+
r"""Applies a perspective transformation to an image.
|
| 173 |
+
|
| 174 |
+
The function warp_perspective transforms the source image using
|
| 175 |
+
the specified matrix:
|
| 176 |
+
|
| 177 |
+
.. math::
|
| 178 |
+
\text{dst} (x, y) = \text{src} \left(
|
| 179 |
+
\frac{M_{11} x + M_{12} y + M_{13}}{M_{31} x + M_{32} y + M_{33}} ,
|
| 180 |
+
\frac{M_{21} x + M_{22} y + M_{23}}{M_{31} x + M_{32} y + M_{33}}
|
| 181 |
+
\right )
|
| 182 |
+
|
| 183 |
+
Args:
|
| 184 |
+
src (torch.Tensor): input image.
|
| 185 |
+
M (Tensor): transformation matrix.
|
| 186 |
+
dsize (tuple): size of the output image (height, width).
|
| 187 |
+
|
| 188 |
+
Returns:
|
| 189 |
+
Tensor: the warped input image.
|
| 190 |
+
|
| 191 |
+
Shape:
|
| 192 |
+
- Input: :math:`(B, C, H, W)` and :math:`(B, 3, 3)`
|
| 193 |
+
- Output: :math:`(B, C, H, W)`
|
| 194 |
+
|
| 195 |
+
.. note::
|
| 196 |
+
See a working example `here <https://github.com/arraiy/torchgeometry/
|
| 197 |
+
blob/master/examples/warp_perspective.ipynb>`_.
|
| 198 |
+
"""
|
| 199 |
+
if not torch.is_tensor(src):
|
| 200 |
+
raise TypeError("Input src type is not a torch.Tensor. Got {}"
|
| 201 |
+
.format(type(src)))
|
| 202 |
+
if not torch.is_tensor(M):
|
| 203 |
+
raise TypeError("Input M type is not a torch.Tensor. Got {}"
|
| 204 |
+
.format(type(M)))
|
| 205 |
+
if not len(src.shape) == 4:
|
| 206 |
+
raise ValueError("Input src must be a BxCxHxW tensor. Got {}"
|
| 207 |
+
.format(src.shape))
|
| 208 |
+
if not (len(M.shape) == 3 or M.shape[-2:] == (3, 3)):
|
| 209 |
+
raise ValueError("Input M must be a Bx3x3 tensor. Got {}"
|
| 210 |
+
.format(src.shape))
|
| 211 |
+
# launches the warper
|
| 212 |
+
return transform_warp_impl(src, M, (src.shape[-2:]), dsize)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def transform_warp_impl(src, dst_pix_trans_src_pix, dsize_src, dsize_dst):
|
| 216 |
+
"""Compute the transform in normalized cooridnates and perform the warping.
|
| 217 |
+
"""
|
| 218 |
+
dst_norm_trans_dst_norm = dst_norm_to_dst_norm(
|
| 219 |
+
dst_pix_trans_src_pix, dsize_src, dsize_dst)
|
| 220 |
+
return homography_warp(src, torch.inverse(
|
| 221 |
+
dst_norm_trans_dst_norm), dsize_dst)
|
| 222 |
+
|
| 223 |
+
def dst_norm_to_dst_norm(dst_pix_trans_src_pix, dsize_src, dsize_dst):
|
| 224 |
+
# source and destination sizes
|
| 225 |
+
src_h, src_w = dsize_src
|
| 226 |
+
dst_h, dst_w = dsize_dst
|
| 227 |
+
# the devices and types
|
| 228 |
+
device = dst_pix_trans_src_pix.device
|
| 229 |
+
dtype = dst_pix_trans_src_pix.dtype
|
| 230 |
+
# compute the transformation pixel/norm for src/dst
|
| 231 |
+
src_norm_trans_src_pix = normal_transform_pixel(
|
| 232 |
+
src_h, src_w).to(device).to(dtype)
|
| 233 |
+
src_pix_trans_src_norm = torch.inverse(src_norm_trans_src_pix)
|
| 234 |
+
dst_norm_trans_dst_pix = normal_transform_pixel(
|
| 235 |
+
dst_h, dst_w).to(device).to(dtype)
|
| 236 |
+
# compute chain transformations
|
| 237 |
+
dst_norm_trans_src_norm = torch.matmul(
|
| 238 |
+
dst_norm_trans_dst_pix, torch.matmul(
|
| 239 |
+
dst_pix_trans_src_pix, src_pix_trans_src_norm))
|
| 240 |
+
return dst_norm_trans_src_norm
|
| 241 |
+
|
| 242 |
+
def normal_transform_pixel(height, width):
|
| 243 |
+
|
| 244 |
+
tr_mat = torch.Tensor([[1.0, 0.0, -1.0],
|
| 245 |
+
[0.0, 1.0, -1.0],
|
| 246 |
+
[0.0, 0.0, 1.0]]) # 1x3x3
|
| 247 |
+
|
| 248 |
+
tr_mat[0, 0] = tr_mat[0, 0] * 2.0 / (width - 1.0)
|
| 249 |
+
tr_mat[1, 1] = tr_mat[1, 1] * 2.0 / (height - 1.0)
|
| 250 |
+
|
| 251 |
+
tr_mat = tr_mat.unsqueeze(0)
|
| 252 |
+
|
| 253 |
+
return tr_mat
|
| 254 |
+
|
| 255 |
+
def homography_warp(patch_src: torch.Tensor,
|
| 256 |
+
dst_homo_src: torch.Tensor,
|
| 257 |
+
dsize: Tuple[int, int],
|
| 258 |
+
mode: Optional[str] = 'bilinear',
|
| 259 |
+
padding_mode: Optional[str] = 'zeros') -> torch.Tensor:
|
| 260 |
+
r"""Function that warps image patchs or tensors by homographies.
|
| 261 |
+
|
| 262 |
+
See :class:`~torchgeometry.HomographyWarper` for details.
|
| 263 |
+
|
| 264 |
+
Args:
|
| 265 |
+
patch_src (torch.Tensor): The image or tensor to warp. Should be from
|
| 266 |
+
source of shape :math:`(N, C, H, W)`.
|
| 267 |
+
dst_homo_src (torch.Tensor): The homography or stack of homographies
|
| 268 |
+
from source to destination of shape
|
| 269 |
+
:math:`(N, 3, 3)`.
|
| 270 |
+
dsize (Tuple[int, int]): The height and width of the image to warp.
|
| 271 |
+
mode (Optional[str]): interpolation mode to calculate output values
|
| 272 |
+
'bilinear' | 'nearest'. Default: 'bilinear'.
|
| 273 |
+
padding_mode (Optional[str]): padding mode for outside grid values
|
| 274 |
+
'zeros' | 'border' | 'reflection'. Default: 'zeros'.
|
| 275 |
+
|
| 276 |
+
Return:
|
| 277 |
+
torch.Tensor: Patch sampled at locations from source to destination.
|
| 278 |
+
|
| 279 |
+
Example:
|
| 280 |
+
>>> input = torch.rand(1, 3, 32, 32)
|
| 281 |
+
>>> homography = torch.eye(3).view(1, 3, 3)
|
| 282 |
+
>>> output = tgm.homography_warp(input, homography, (32, 32)) # NxCxHxW
|
| 283 |
+
"""
|
| 284 |
+
height, width = dsize
|
| 285 |
+
warper = HomographyWarper(height, width, mode, padding_mode)
|
| 286 |
+
return warper(patch_src, dst_homo_src)
|
| 287 |
+
|
| 288 |
+
class HomographyWarper(nn.Module):
|
| 289 |
+
r"""Warps image patches or tensors by homographies.
|
| 290 |
+
|
| 291 |
+
.. math::
|
| 292 |
+
|
| 293 |
+
X_{dst} = H_{src}^{\{dst\}} * X_{src}
|
| 294 |
+
|
| 295 |
+
Args:
|
| 296 |
+
height (int): The height of the image to warp.
|
| 297 |
+
width (int): The width of the image to warp.
|
| 298 |
+
mode (Optional[str]): interpolation mode to calculate output values
|
| 299 |
+
'bilinear' | 'nearest'. Default: 'bilinear'.
|
| 300 |
+
padding_mode (Optional[str]): padding mode for outside grid values
|
| 301 |
+
'zeros' | 'border' | 'reflection'. Default: 'zeros'.
|
| 302 |
+
normalized_coordinates (Optional[bool]): wether to use a grid with
|
| 303 |
+
normalized coordinates.
|
| 304 |
+
"""
|
| 305 |
+
|
| 306 |
+
def __init__(
|
| 307 |
+
self,
|
| 308 |
+
height: int,
|
| 309 |
+
width: int,
|
| 310 |
+
mode: Optional[str] = 'bilinear',
|
| 311 |
+
padding_mode: Optional[str] = 'zeros',
|
| 312 |
+
normalized_coordinates: Optional[bool] = True) -> None:
|
| 313 |
+
super(HomographyWarper, self).__init__()
|
| 314 |
+
self.width: int = width
|
| 315 |
+
self.height: int = height
|
| 316 |
+
self.mode: Optional[str] = mode
|
| 317 |
+
self.padding_mode: Optional[str] = padding_mode
|
| 318 |
+
self.normalized_coordinates: Optional[bool] = normalized_coordinates
|
| 319 |
+
|
| 320 |
+
# create base grid to compute the flow
|
| 321 |
+
self.grid: torch.Tensor = create_meshgrid(
|
| 322 |
+
height, width, normalized_coordinates=normalized_coordinates)
|
| 323 |
+
|
| 324 |
+
def warp_grid(self, dst_homo_src: torch.Tensor) -> torch.Tensor:
|
| 325 |
+
r"""Computes the grid to warp the coordinates grid by an homography.
|
| 326 |
+
|
| 327 |
+
Args:
|
| 328 |
+
dst_homo_src (torch.Tensor): Homography or homographies (stacked) to
|
| 329 |
+
transform all points in the grid. Shape of the
|
| 330 |
+
homography has to be :math:`(N, 3, 3)`.
|
| 331 |
+
|
| 332 |
+
Returns:
|
| 333 |
+
torch.Tensor: the transformed grid of shape :math:`(N, H, W, 2)`.
|
| 334 |
+
"""
|
| 335 |
+
batch_size: int = dst_homo_src.shape[0]
|
| 336 |
+
device: torch.device = dst_homo_src.device
|
| 337 |
+
dtype: torch.dtype = dst_homo_src.dtype
|
| 338 |
+
# expand grid to match the input batch size
|
| 339 |
+
grid: torch.Tensor = self.grid.expand(batch_size, -1, -1, -1) # NxHxWx2
|
| 340 |
+
if len(dst_homo_src.shape) == 3: # local homography case
|
| 341 |
+
dst_homo_src = dst_homo_src.view(batch_size, 1, 3, 3) # NxHxWx3x3
|
| 342 |
+
# perform the actual grid transformation,
|
| 343 |
+
# the grid is copied to input device and casted to the same type
|
| 344 |
+
flow: torch.Tensor = transform_points(
|
| 345 |
+
dst_homo_src, grid.to(device).to(dtype)) # NxHxWx2
|
| 346 |
+
return flow.view(batch_size, self.height, self.width, 2) # NxHxWx2
|
| 347 |
+
|
| 348 |
+
def forward(
|
| 349 |
+
self,
|
| 350 |
+
patch_src: torch.Tensor,
|
| 351 |
+
dst_homo_src: torch.Tensor) -> torch.Tensor:
|
| 352 |
+
r"""Warps an image or tensor from source into reference frame.
|
| 353 |
+
|
| 354 |
+
Args:
|
| 355 |
+
patch_src (torch.Tensor): The image or tensor to warp.
|
| 356 |
+
Should be from source.
|
| 357 |
+
dst_homo_src (torch.Tensor): The homography or stack of homographies
|
| 358 |
+
from source to destination. The homography assumes normalized
|
| 359 |
+
coordinates [-1, 1].
|
| 360 |
+
|
| 361 |
+
Return:
|
| 362 |
+
torch.Tensor: Patch sampled at locations from source to destination.
|
| 363 |
+
|
| 364 |
+
Shape:
|
| 365 |
+
- Input: :math:`(N, C, H, W)` and :math:`(N, 3, 3)`
|
| 366 |
+
- Output: :math:`(N, C, H, W)`
|
| 367 |
+
|
| 368 |
+
Example:
|
| 369 |
+
>>> input = torch.rand(1, 3, 32, 32)
|
| 370 |
+
>>> homography = torch.eye(3).view(1, 3, 3)
|
| 371 |
+
>>> warper = tgm.HomographyWarper(32, 32)
|
| 372 |
+
>>> output = warper(input, homography) # NxCxHxW
|
| 373 |
+
"""
|
| 374 |
+
if not dst_homo_src.device == patch_src.device:
|
| 375 |
+
raise TypeError("Patch and homography must be on the same device. \
|
| 376 |
+
Got patch.device: {} dst_H_src.device: {}."
|
| 377 |
+
.format(patch_src.device, dst_homo_src.device))
|
| 378 |
+
return F.grid_sample(patch_src, self.warp_grid(dst_homo_src),
|
| 379 |
+
mode=self.mode, padding_mode=self.padding_mode)
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def create_meshgrid(
|
| 384 |
+
height: int,
|
| 385 |
+
width: int,
|
| 386 |
+
normalized_coordinates: Optional[bool] = True):
|
| 387 |
+
"""Generates a coordinate grid for an image.
|
| 388 |
+
|
| 389 |
+
When the flag `normalized_coordinates` is set to True, the grid is
|
| 390 |
+
normalized to be in the range [-1,1] to be consistent with the pytorch
|
| 391 |
+
function grid_sample.
|
| 392 |
+
http://pytorch.org/docs/master/nn.html#torch.nn.functional.grid_sample
|
| 393 |
+
|
| 394 |
+
Args:
|
| 395 |
+
height (int): the image height (rows).
|
| 396 |
+
width (int): the image width (cols).
|
| 397 |
+
normalized_coordinates (Optional[bool]): wether to normalize
|
| 398 |
+
coordinates in the range [-1, 1] in order to be consistent with the
|
| 399 |
+
PyTorch function grid_sample.
|
| 400 |
+
|
| 401 |
+
Return:
|
| 402 |
+
torch.Tensor: returns a grid tensor with shape :math:`(1, H, W, 2)`.
|
| 403 |
+
"""
|
| 404 |
+
# generate coordinates
|
| 405 |
+
xs: Optional[torch.Tensor] = None
|
| 406 |
+
ys: Optional[torch.Tensor] = None
|
| 407 |
+
if normalized_coordinates:
|
| 408 |
+
xs = torch.linspace(-1, 1, width)
|
| 409 |
+
ys = torch.linspace(-1, 1, height)
|
| 410 |
+
else:
|
| 411 |
+
xs = torch.linspace(0, width - 1, width)
|
| 412 |
+
ys = torch.linspace(0, height - 1, height)
|
| 413 |
+
# generate grid by stacking coordinates
|
| 414 |
+
base_grid: torch.Tensor = torch.stack(
|
| 415 |
+
torch.meshgrid([xs, ys])).transpose(1, 2) # 2xHxW
|
| 416 |
+
return torch.unsqueeze(base_grid, dim=0).permute(0, 2, 3, 1) # 1xHxWx2
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
def transform_points(trans_01: torch.Tensor,
|
| 420 |
+
points_1: torch.Tensor) -> torch.Tensor:
|
| 421 |
+
r"""Function that applies transformations to a set of points.
|
| 422 |
+
|
| 423 |
+
Args:
|
| 424 |
+
trans_01 (torch.Tensor): tensor for transformations of shape
|
| 425 |
+
:math:`(B, D+1, D+1)`.
|
| 426 |
+
points_1 (torch.Tensor): tensor of points of shape :math:`(B, N, D)`.
|
| 427 |
+
Returns:
|
| 428 |
+
torch.Tensor: tensor of N-dimensional points.
|
| 429 |
+
|
| 430 |
+
Shape:
|
| 431 |
+
- Output: :math:`(B, N, D)`
|
| 432 |
+
|
| 433 |
+
Examples:
|
| 434 |
+
|
| 435 |
+
>>> points_1 = torch.rand(2, 4, 3) # BxNx3
|
| 436 |
+
>>> trans_01 = torch.eye(4).view(1, 4, 4) # Bx4x4
|
| 437 |
+
>>> points_0 = tgm.transform_points(trans_01, points_1) # BxNx3
|
| 438 |
+
"""
|
| 439 |
+
if not torch.is_tensor(trans_01) or not torch.is_tensor(points_1):
|
| 440 |
+
raise TypeError("Input type is not a torch.Tensor")
|
| 441 |
+
if not trans_01.device == points_1.device:
|
| 442 |
+
raise TypeError("Tensor must be in the same device")
|
| 443 |
+
if not trans_01.shape[0] == points_1.shape[0]:
|
| 444 |
+
raise ValueError("Input batch size must be the same for both tensors")
|
| 445 |
+
if not trans_01.shape[-1] == (points_1.shape[-1] + 1):
|
| 446 |
+
raise ValueError("Last input dimensions must differe by one unit")
|
| 447 |
+
# to homogeneous
|
| 448 |
+
points_1_h = convert_points_to_homogeneous(points_1) # BxNxD+1
|
| 449 |
+
# transform coordinates
|
| 450 |
+
points_0_h = torch.matmul(
|
| 451 |
+
trans_01.unsqueeze(1), points_1_h.unsqueeze(-1))
|
| 452 |
+
points_0_h = torch.squeeze(points_0_h, dim=-1)
|
| 453 |
+
# to euclidean
|
| 454 |
+
points_0 = convert_points_from_homogeneous(points_0_h) # BxNxD
|
| 455 |
+
return points_0
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def convert_points_to_homogeneous(points):
|
| 459 |
+
r"""Function that converts points from Euclidean to homogeneous space.
|
| 460 |
+
|
| 461 |
+
See :class:`~torchgeometry.ConvertPointsToHomogeneous` for details.
|
| 462 |
+
|
| 463 |
+
Examples::
|
| 464 |
+
|
| 465 |
+
>>> input = torch.rand(2, 4, 3) # BxNx3
|
| 466 |
+
>>> output = tgm.convert_points_to_homogeneous(input) # BxNx4
|
| 467 |
+
"""
|
| 468 |
+
if not torch.is_tensor(points):
|
| 469 |
+
raise TypeError("Input type is not a torch.Tensor. Got {}".format(
|
| 470 |
+
type(points)))
|
| 471 |
+
if len(points.shape) < 2:
|
| 472 |
+
raise ValueError("Input must be at least a 2D tensor. Got {}".format(
|
| 473 |
+
points.shape))
|
| 474 |
+
|
| 475 |
+
return nn.functional.pad(points, (0, 1), "constant", 1.0)
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def convert_points_from_homogeneous(points):
|
| 480 |
+
r"""Function that converts points from homogeneous to Euclidean space.
|
| 481 |
+
|
| 482 |
+
See :class:`~torchgeometry.ConvertPointsFromHomogeneous` for details.
|
| 483 |
+
|
| 484 |
+
Examples::
|
| 485 |
+
|
| 486 |
+
>>> input = torch.rand(2, 4, 3) # BxNx3
|
| 487 |
+
>>> output = tgm.convert_points_from_homogeneous(input) # BxNx2
|
| 488 |
+
"""
|
| 489 |
+
if not torch.is_tensor(points):
|
| 490 |
+
raise TypeError("Input type is not a torch.Tensor. Got {}".format(
|
| 491 |
+
type(points)))
|
| 492 |
+
if len(points.shape) < 2:
|
| 493 |
+
raise ValueError("Input must be at least a 2D tensor. Got {}".format(
|
| 494 |
+
points.shape))
|
| 495 |
+
|
| 496 |
+
return points[..., :-1] / points[..., -1:]
|