LoomNet / inference.py
Giulio Federico
fix
feb5ad9
Raw History Blame Contribute Delete
4.62 kB
import argparse
import json
import math
import os
from pathlib import Path
from typing import List
import hydra
import lightning as L
import numpy as np
import torch
from einops import rearrange
from lightning import LightningModule
from PIL import Image
from loomnet.utils import load_config
from loomnet.utils.media import get_bg_color, load_image
from loomnet.utils.pose import get_k_near_views
WIDTH, HEIGHT = 256, 256
K_NEAR_VIEWS = 16
def prepare_inputs(input_img: str, input_elevation: float, sample_views_mode: str):
bg_color = get_bg_color("white")
input_img = load_image(
input_img, (WIDTH, HEIGHT), bg_color, return_type="pt"
).permute(2, 0, 1)
meta_fp = f"meta_info/transforms_{sample_views_mode}.json"
with open(meta_fp, "r") as f:
meta = json.load(f)
# Camera intrinsics
fov = meta["camera_angle_x"]
focal_length = 1 / (2 * np.tan(0.5 * fov))
intrinsics_4x4 = torch.tensor(
[
[focal_length, 0, 0.5, 0],
[0, focal_length, 0.5, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
]
)
# Camera extrinsics
num_views = len(meta["frames"])
elevations, azimuths, c2w_matrixs = [], [], []
for frame in meta["frames"]:
elevations.append(frame["elevation"])
azimuths.append(frame["azimuth"])
c2w_matrixs.append(frame["transform_matrix"])
elevations = torch.tensor(elevations) # (N,)
azimuths = torch.tensor(azimuths) # (N,)
c2w_matrixs = torch.tensor(c2w_matrixs) # (N, 4, 4)
c2w_matrixs[:, :, 1:3] *= -1 # blender to opencv
# concat intrinsics and extrinsics
intrinsics_4x4 = (
intrinsics_4x4.unsqueeze(0).repeat(num_views, 1, 1).reshape(num_views, 16)
)
_c2w_matrixs = c2w_matrixs.reshape(num_views, 16)
camera_params = torch.cat([intrinsics_4x4, _c2w_matrixs], dim=1)
# find nearest views
k_near_indices = get_k_near_views(elevations, azimuths, K_NEAR_VIEWS, num_views)
# normalize elevations and azimuths
input_elevation = torch.tensor([input_elevation / 180 * math.pi])
d_elevations = (elevations - input_elevation).reshape(-1, 1)
d_azimuths = azimuths.reshape(-1, 1) % (2 * math.pi)
#distances = torch.zeros_like(d_elevations)
distances = torch.tensor(input_elevation).repeat(num_views).reshape(-1, 1)
return {
"image_0": input_img,
"elevations": d_elevations,
"azimuths": d_azimuths,
"distances": distances,
"c2w_matrixs": c2w_matrixs,
"cameras": camera_params,
"k_near_indices": k_near_indices,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, default="configs/baseline.yaml")
parser.add_argument("--input_img", type=str, default="/raid/homes/giulio.federico/LoomNet/da_eliminare/render_0065.png")
parser.add_argument("--output_dir", type=str, default="/raid/homes/giulio.federico/LoomNet/output_dir")
parser.add_argument("--elevation", type=float, default=0.0)
parser.add_argument(
"--sample_views_mode", type=str, choices=["ele30", "ele40"], default="ele30"
)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--device", type=str, default="cuda")
args, extras = parser.parse_known_args()
output_dir = Path(args.output_dir)
output_dir.mkdir(exist_ok=True, parents=True)
L.seed_everything(args.seed, workers=False)
cfg = load_config(args.config, cli_args=extras)
# prepare model
model: LightningModule = hydra.utils.instantiate(cfg.system)
model.load_weights(cfg.resume)
model = model.to(args.device).eval()
print(f"Loaded model from {cfg.resume}")
# prepare data
data = prepare_inputs(args.input_img, args.elevation, args.sample_views_mode)
for k, v in data.items():
data[k] = v.unsqueeze(0).to(args.device)
if k not in ["k_near_indices"]:
data[k] = data[k].float()
# generate
with torch.no_grad():
images_pred = model._generate_images(data)
# save
image_base_name = os.path.basename(args.input_img).split(".")[0]
image_list = []
for image in images_pred[0]:
image_list.append(Image.fromarray(image))
image_list[0].save(
output_dir / f"{image_base_name}.gif",
save_all=True,
append_images=image_list[1:],
duration=100,
loop=0,
)
full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
Image.fromarray(full_image).save(output_dir / f"{image_base_name}.jpg")
if __name__ == "__main__":
main()