File size: 4,615 Bytes
2039c18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
feb5ad9
2039c18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import argparse
import json
import math
import os
from pathlib import Path
from typing import List

import hydra
import lightning as L
import numpy as np
import torch
from einops import rearrange
from lightning import LightningModule
from PIL import Image

from loomnet.utils import load_config
from loomnet.utils.media import get_bg_color, load_image
from loomnet.utils.pose import get_k_near_views

WIDTH, HEIGHT = 256, 256
K_NEAR_VIEWS = 16


def prepare_inputs(input_img: str, input_elevation: float, sample_views_mode: str):
    bg_color = get_bg_color("white")
    input_img = load_image(
        input_img, (WIDTH, HEIGHT), bg_color, return_type="pt"
    ).permute(2, 0, 1)

    meta_fp = f"meta_info/transforms_{sample_views_mode}.json"
    with open(meta_fp, "r") as f:
        meta = json.load(f)

    # Camera intrinsics
    fov = meta["camera_angle_x"]
    focal_length = 1 / (2 * np.tan(0.5 * fov))
    intrinsics_4x4 = torch.tensor(
        [
            [focal_length, 0, 0.5, 0],
            [0, focal_length, 0.5, 0],
            [0, 0, 1, 0],
            [0, 0, 0, 1],
        ]
    )

    # Camera extrinsics
    num_views = len(meta["frames"])
    elevations, azimuths, c2w_matrixs = [], [], []
    for frame in meta["frames"]:
        elevations.append(frame["elevation"])
        azimuths.append(frame["azimuth"])
        c2w_matrixs.append(frame["transform_matrix"])
    elevations = torch.tensor(elevations)  # (N,)
    azimuths = torch.tensor(azimuths)  # (N,)
    c2w_matrixs = torch.tensor(c2w_matrixs)  # (N, 4, 4)
    c2w_matrixs[:, :, 1:3] *= -1  # blender to opencv

    # concat intrinsics and extrinsics
    intrinsics_4x4 = (
        intrinsics_4x4.unsqueeze(0).repeat(num_views, 1, 1).reshape(num_views, 16)
    )
    _c2w_matrixs = c2w_matrixs.reshape(num_views, 16)
    camera_params = torch.cat([intrinsics_4x4, _c2w_matrixs], dim=1)

    # find nearest views
    k_near_indices = get_k_near_views(elevations, azimuths, K_NEAR_VIEWS, num_views)

    # normalize elevations and azimuths
    input_elevation = torch.tensor([input_elevation / 180 * math.pi])
    d_elevations = (elevations - input_elevation).reshape(-1, 1)
    d_azimuths = azimuths.reshape(-1, 1) % (2 * math.pi)
    
    #distances = torch.zeros_like(d_elevations)
    distances = torch.tensor(input_elevation).repeat(num_views).reshape(-1, 1)

    return {
        "image_0": input_img,
        "elevations": d_elevations,
        "azimuths": d_azimuths,
        "distances": distances,
        "c2w_matrixs": c2w_matrixs,
        "cameras": camera_params,
        "k_near_indices": k_near_indices,
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", type=str, default="configs/baseline.yaml")
    parser.add_argument("--input_img", type=str, default="/raid/homes/giulio.federico/LoomNet/da_eliminare/render_0065.png")
    parser.add_argument("--output_dir", type=str, default="/raid/homes/giulio.federico/LoomNet/output_dir")
    parser.add_argument("--elevation", type=float, default=0.0)
    parser.add_argument(
        "--sample_views_mode", type=str, choices=["ele30", "ele40"], default="ele30"
    )
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--device", type=str, default="cuda")
    args, extras = parser.parse_known_args()

    output_dir = Path(args.output_dir)
    output_dir.mkdir(exist_ok=True, parents=True)

    L.seed_everything(args.seed, workers=False)
    cfg = load_config(args.config, cli_args=extras)

    # prepare model
    model: LightningModule = hydra.utils.instantiate(cfg.system)
    model.load_weights(cfg.resume)
    model = model.to(args.device).eval()
    print(f"Loaded model from {cfg.resume}")

    # prepare data
    data = prepare_inputs(args.input_img, args.elevation, args.sample_views_mode)
    for k, v in data.items():
        data[k] = v.unsqueeze(0).to(args.device)
        if k not in ["k_near_indices"]:
            data[k] = data[k].float()

    # generate
    with torch.no_grad():
        images_pred = model._generate_images(data)

    # save
    image_base_name = os.path.basename(args.input_img).split(".")[0]
    image_list = []
    for image in images_pred[0]:
        image_list.append(Image.fromarray(image))
    image_list[0].save(
        output_dir / f"{image_base_name}.gif",
        save_all=True,
        append_images=image_list[1:],
        duration=100,
        loop=0,
    )

    full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
    Image.fromarray(full_image).save(output_dir / f"{image_base_name}.jpg")


if __name__ == "__main__":
    main()