Spaces:
Sleeping
Sleeping
File size: 4,615 Bytes
2039c18 feb5ad9 2039c18 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | import argparse
import json
import math
import os
from pathlib import Path
from typing import List
import hydra
import lightning as L
import numpy as np
import torch
from einops import rearrange
from lightning import LightningModule
from PIL import Image
from loomnet.utils import load_config
from loomnet.utils.media import get_bg_color, load_image
from loomnet.utils.pose import get_k_near_views
WIDTH, HEIGHT = 256, 256
K_NEAR_VIEWS = 16
def prepare_inputs(input_img: str, input_elevation: float, sample_views_mode: str):
bg_color = get_bg_color("white")
input_img = load_image(
input_img, (WIDTH, HEIGHT), bg_color, return_type="pt"
).permute(2, 0, 1)
meta_fp = f"meta_info/transforms_{sample_views_mode}.json"
with open(meta_fp, "r") as f:
meta = json.load(f)
# Camera intrinsics
fov = meta["camera_angle_x"]
focal_length = 1 / (2 * np.tan(0.5 * fov))
intrinsics_4x4 = torch.tensor(
[
[focal_length, 0, 0.5, 0],
[0, focal_length, 0.5, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
]
)
# Camera extrinsics
num_views = len(meta["frames"])
elevations, azimuths, c2w_matrixs = [], [], []
for frame in meta["frames"]:
elevations.append(frame["elevation"])
azimuths.append(frame["azimuth"])
c2w_matrixs.append(frame["transform_matrix"])
elevations = torch.tensor(elevations) # (N,)
azimuths = torch.tensor(azimuths) # (N,)
c2w_matrixs = torch.tensor(c2w_matrixs) # (N, 4, 4)
c2w_matrixs[:, :, 1:3] *= -1 # blender to opencv
# concat intrinsics and extrinsics
intrinsics_4x4 = (
intrinsics_4x4.unsqueeze(0).repeat(num_views, 1, 1).reshape(num_views, 16)
)
_c2w_matrixs = c2w_matrixs.reshape(num_views, 16)
camera_params = torch.cat([intrinsics_4x4, _c2w_matrixs], dim=1)
# find nearest views
k_near_indices = get_k_near_views(elevations, azimuths, K_NEAR_VIEWS, num_views)
# normalize elevations and azimuths
input_elevation = torch.tensor([input_elevation / 180 * math.pi])
d_elevations = (elevations - input_elevation).reshape(-1, 1)
d_azimuths = azimuths.reshape(-1, 1) % (2 * math.pi)
#distances = torch.zeros_like(d_elevations)
distances = torch.tensor(input_elevation).repeat(num_views).reshape(-1, 1)
return {
"image_0": input_img,
"elevations": d_elevations,
"azimuths": d_azimuths,
"distances": distances,
"c2w_matrixs": c2w_matrixs,
"cameras": camera_params,
"k_near_indices": k_near_indices,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, default="configs/baseline.yaml")
parser.add_argument("--input_img", type=str, default="/raid/homes/giulio.federico/LoomNet/da_eliminare/render_0065.png")
parser.add_argument("--output_dir", type=str, default="/raid/homes/giulio.federico/LoomNet/output_dir")
parser.add_argument("--elevation", type=float, default=0.0)
parser.add_argument(
"--sample_views_mode", type=str, choices=["ele30", "ele40"], default="ele30"
)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--device", type=str, default="cuda")
args, extras = parser.parse_known_args()
output_dir = Path(args.output_dir)
output_dir.mkdir(exist_ok=True, parents=True)
L.seed_everything(args.seed, workers=False)
cfg = load_config(args.config, cli_args=extras)
# prepare model
model: LightningModule = hydra.utils.instantiate(cfg.system)
model.load_weights(cfg.resume)
model = model.to(args.device).eval()
print(f"Loaded model from {cfg.resume}")
# prepare data
data = prepare_inputs(args.input_img, args.elevation, args.sample_views_mode)
for k, v in data.items():
data[k] = v.unsqueeze(0).to(args.device)
if k not in ["k_near_indices"]:
data[k] = data[k].float()
# generate
with torch.no_grad():
images_pred = model._generate_images(data)
# save
image_base_name = os.path.basename(args.input_img).split(".")[0]
image_list = []
for image in images_pred[0]:
image_list.append(Image.fromarray(image))
image_list[0].save(
output_dir / f"{image_base_name}.gif",
save_all=True,
append_images=image_list[1:],
duration=100,
loop=0,
)
full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
Image.fromarray(full_image).save(output_dir / f"{image_base_name}.jpg")
if __name__ == "__main__":
main()
|