diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a2e459abe3a31fc3fc0bf475fabad0dba77f6828 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +build/ +dist/ +data/ +data2/ +output*/ +wandb*/ +checkpoints/ +slurm_scripts* +watch_folder +cross_eval_out +*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..85920ec2dd0f8df35ae5121f50b8d95072b29ead --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 + +RUN apt-get update && apt-get install -y \ + python3.10 python3-pip git wget libgl1-mesa-glx libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -m -u 1000 user +WORKDIR /app +COPY . /app + +RUN pip3 install torch==2.3.1 torchvision==0.18.1 \ + --index-url https://download.pytorch.org/whl/cu118 +RUN pip3 install -r requirements.txt +RUN pip3 install gradio gdown + +RUN cd models/ops && sh make.sh && cd ../.. +RUN cd diff_ras && python3 setup.py build develop && cd .. + +RUN mkdir -p checkpoints && \ + gdown --fuzzy "https://drive.google.com/file/d/1M32HlYwXw-4Q_uajSCvpbF31UFPzQVHP/view?usp=sharing" \ + -O checkpoints/r2g_res256_ep0849.pth + +RUN chown -R user:user /app +USER user + +EXPOSE 7860 +CMD ["python3", "app.py"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d40e4426ec03d3201f1f83dea19d8f7adb98d8c2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Raster2Seq + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index afc932c8ce408b67698b415f839281603886b24f..54abef12d8c16f62d95ad8eeed528b96e65c3828 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,29 @@ --- -title: Raster2seq -emoji: 🐢 -colorFrom: yellow -colorTo: gray +title: Raster2Seq +emoji: 🏠 +colorFrom: blue +colorTo: purple sdk: docker pinned: false +app_port: 7860 --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# Raster2Seq - Floor Plan Vectorization + +Upload a floor plan image to detect room polygons and their semantic labels. + +This Space runs the Raster2Seq model for converting raster floor plan images into vectorized polygon sequences with room type classification. + +## API Usage + +This Space exposes a Gradio API. You can call it programmatically: + +```python +from gradio_client import Client + +client = Client("AGLO-AI/raster2seq") +result = client.predict( + image="path/to/floorplan.png", + api_name="/predict" +) +``` diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3351a274ef9b56b7b3ff1d9a9a919ffc843ef877 --- /dev/null +++ b/app.py @@ -0,0 +1,303 @@ +import argparse +import copy +import json +import math + +import cv2 +import gradio as gr +import numpy as np +import torch +from PIL import Image +from shapely.geometry import Polygon + +from datasets.discrete_tokenizer import DiscreteTokenizer +from datasets.transforms import ResizeAndPad +from detectron2.data import transforms as T +from models import build_model +from util.plot_utils import plot_semantic_rich_floorplan_opencv + +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +MODEL_ARGS = argparse.Namespace( + poly2seq=True, + seq_len=512, + num_bins=32, + image_size=256, + input_channels=3, + backbone="resnet50", + dilation=False, + position_embedding="sine", + position_embedding_scale=2 * np.pi, + num_feature_levels=4, + enc_layers=6, + dec_layers=6, + dim_feedforward=1024, + hidden_dim=256, + dropout=0.1, + nheads=8, + num_queries=800, + num_polys=20, + dec_n_points=4, + enc_n_points=4, + query_pos_type="sine", + with_poly_refine=False, + masked_attn=False, + semantic_classes=13, + aux_loss=False, + dec_attn_concat_src=True, + pre_decoder_pos_embed=False, + learnable_dec_pe=False, + dec_qkv_proj=False, + per_token_sem_loss=True, + add_cls_token=False, + use_anchor=True, + inject_cls_embed=False, + device="cuda" if torch.cuda.is_available() else "cpu", +) + +R2G_LABEL = { + 0: "Living Room", + 1: "Kitchen", + 2: "Bedroom", + 3: "Bathroom", + 4: "Balcony", + 5: "Corridor", + 6: "Dining Room", + 7: "Study", + 8: "Studio", + 9: "Store Room", + 10: "Garden", + 11: "Laundry Room", + 12: "Office", + 13: "Basement", + 14: "Garage", + 15: "Undefined", + 16: "Door", + 17: "Window", +} + + +def _process_predictions( + pred_corners, i, semantic_rich, image_size, pred_room_label, + pred_room_logits, per_token_sem_loss, add_cls_token=False, +): + """Extract polygons from poly2seq model output.""" + np_softmax = lambda x: np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True) + pred_corners_per_scene = pred_corners[i] + room_polys = [] + + if semantic_rich: + room_types = [] + window_doors = [] + window_doors_types = [] + pred_room_label_per_scene = pred_room_label[i].cpu().numpy() + pred_room_logit_per_scene = pred_room_logits[i].cpu().numpy() + + all_room_polys = [] + tmp = [] + all_length_list = [0] + + for j in range(len(pred_corners_per_scene)): + if isinstance(pred_corners_per_scene[j], int): + if pred_corners_per_scene[j] == 2 and tmp: + all_room_polys.append(tmp) + all_length_list.append(len(tmp) + 1 + add_cls_token) + tmp = [] + continue + tmp.append(pred_corners_per_scene[j]) + + if len(tmp): + all_room_polys.append(tmp) + all_length_list.append(len(tmp) + 1 + add_cls_token) + + start_poly_indices = np.cumsum(all_length_list) + final_pred_classes = [] + + for j, poly in enumerate(all_room_polys): + if len(poly) < 2: + continue + corners = np.array(poly, dtype=np.float32) * (image_size - 1) + corners = np.around(corners).astype(np.int32) + + if not semantic_rich: + if len(corners) >= 4 and Polygon(corners).area >= 100: + room_polys.append(corners) + else: + if per_token_sem_loss: + pred_classes, counts = np.unique( + pred_room_label_per_scene[start_poly_indices[j]:start_poly_indices[j + 1]][:-1], + return_counts=True, + ) + pred_class = pred_classes[np.argmax(counts)] + else: + pred_class = pred_room_label_per_scene[start_poly_indices[j + 1] - 1] + final_pred_classes.append(pred_class) + + if len(corners) >= 3 and Polygon(corners).area >= 100: + room_polys.append(corners) + room_types.append(pred_class) + elif len(corners) == 2: + window_doors.append(corners) + window_doors_types.append(pred_class) + + if not semantic_rich: + pred_room_label_per_scene = len(all_room_polys) * [-1] + + return { + "room_polys": room_polys, + "room_types": room_types if semantic_rich else None, + "window_doors": window_doors if semantic_rich else None, + "window_doors_types": window_doors_types if semantic_rich else None, + } + + +@torch.no_grad() +def generate(model, samples, semantic_rich=False, use_cache=True, per_token_sem_loss=False): + """Generate room polygons from model predictions (poly2seq mode only).""" + model.eval() + image_size = samples[0].size(2) + + outputs = model.forward_inference(samples, use_cache) + pred_corners = outputs["gen_out"] + + bs = outputs["pred_logits"].shape[0] + + pred_room_label = None + pred_room_logits = None + if "pred_room_logits" in outputs: + pred_room_logits = outputs["pred_room_logits"] + prob = torch.nn.functional.softmax(pred_room_logits, -1) + _, pred_room_label = prob[..., :-1].max(-1) + + result_rooms = [] + result_classes = [] + + for i in range(bs): + scene_outputs = _process_predictions( + pred_corners, i, semantic_rich, image_size, + pred_room_label, pred_room_logits, per_token_sem_loss, + ) + room_polys = scene_outputs["room_polys"] + room_types = scene_outputs["room_types"] + window_doors = scene_outputs["window_doors"] + window_doors_types = scene_outputs["window_doors_types"] + + if window_doors: + result_rooms.append(room_polys + window_doors) + result_classes.append(room_types + window_doors_types) + else: + result_rooms.append(room_polys) + result_classes.append(room_types) + + return {"room": result_rooms, "labels": result_classes} + + +def load_model(): + tokenizer = DiscreteTokenizer( + MODEL_ARGS.num_bins, MODEL_ARGS.seq_len, add_cls=MODEL_ARGS.add_cls_token + ) + MODEL_ARGS.vocab_size = len(tokenizer) + + model = build_model(MODEL_ARGS, train=False, tokenizer=tokenizer) + model.to(DEVICE) + + ckpt_path = "checkpoints/r2g_res256_ep0849.pth" + checkpoint = torch.load(ckpt_path, map_location="cpu") + ckpt_state_dict = copy.deepcopy(checkpoint["ema"]) + for key in list(ckpt_state_dict.keys()): + if key.startswith("module."): + ckpt_state_dict[key[7:]] = ckpt_state_dict.pop(key) + model.load_state_dict(ckpt_state_dict, strict=False) + + for param in model.parameters(): + param.requires_grad = False + model.eval() + return model + + +print("Loading model...") +MODEL = load_model() +print("Model loaded.") + +DATA_TRANSFORM = T.AugmentationList( + [ResizeAndPad((MODEL_ARGS.image_size, MODEL_ARGS.image_size), pad_value=255)] +) + + +def preprocess_image(pil_image: Image.Image) -> torch.Tensor: + image_np = np.array(pil_image.convert("RGB")) + aug_input = T.AugInput(image_np) + DATA_TRANSFORM(aug_input) + image_np = aug_input.image + + if len(image_np.shape) == 2: + tensor = np.expand_dims(image_np, 0) + else: + tensor = image_np.transpose((2, 0, 1)) + + return (1 / 255) * torch.as_tensor(tensor, dtype=torch.float32) + + +def predict_floorplan(image: Image.Image): + if image is None: + return None, json.dumps({"error": "No image provided"}) + + input_tensor = preprocess_image(image).unsqueeze(0).to(DEVICE) + + outputs = generate( + MODEL, + input_tensor, + semantic_rich=MODEL_ARGS.semantic_classes > 0, + use_cache=True, + per_token_sem_loss=MODEL_ARGS.per_token_sem_loss, + ) + + pred_rooms = outputs["room"][0] + pred_labels = outputs["labels"][0] + image_size = MODEL_ARGS.image_size + + if pred_labels is None: + pred_labels = [-1] * len(pred_rooms) + + result_polygons = [] + for poly, label in zip(pred_rooms, pred_labels): + coords = poly.astype(float).tolist() + result_polygons.append({ + "label": R2G_LABEL.get(int(label), "Unknown"), + "label_id": int(label), + "polygon": coords, + }) + + floorplan_map = plot_semantic_rich_floorplan_opencv( + zip(pred_rooms, pred_labels), + None, + door_window_index=[], + semantics_label_mapping=R2G_LABEL, + plot_text=True, + one_color=False, + is_sem=True, + img_w=image_size * 2, + img_h=image_size * 2, + scale=2, + ) + if floorplan_map is not None and floorplan_map.size > 0: + floorplan_rgb = cv2.cvtColor(floorplan_map, cv2.COLOR_BGR2RGB) + vis_image = Image.fromarray(floorplan_rgb) + else: + vis_image = None + + return vis_image, result_polygons + + +demo = gr.Interface( + fn=predict_floorplan, + inputs=gr.Image(type="pil", label="Floor Plan Image"), + outputs=[ + gr.Image(type="pil", label="Detected Rooms"), + gr.JSON(label="Detected Polygons"), + ], + title="Raster2Seq - Floor Plan Vectorization", + description="Upload a floor plan image to detect room polygons and their semantic labels. Returns both a visualization and structured JSON with polygon coordinates.", +) + +demo.launch(server_name="0.0.0.0", server_port=7860) diff --git a/data_preprocess/README.md b/data_preprocess/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2e76009b7c59e694acfa817bccc56e9d088106bd --- /dev/null +++ b/data_preprocess/README.md @@ -0,0 +1,40 @@ +## Data preprocessing + +### Structured3D + +Simply download preprocessed data by RoomFormer at [here](https://polybox.ethz.ch/index.php/s/wKYWFsQOXHnkwcG). For more details, please refer to [RoomFormer's instructions](https://github.com/ywyue/RoomFormer/tree/main/data_preprocess). + +To render binary floorplan images from GT annotations (as used in our paper), run `bash data_preprocess/tools/run_s3d.sh`. + +### CubiCasa5K +Step 1: Download and extract [CubiCasa5K](https://zenodo.org/record/2613548) dataset. + +Step 2: Run `bash data_preprocess/cubicasa5k/run.sh`. + +### Raster2Graph +The instruction mainly follows Raster2Graph's instruction. + +Step 1: Due to dataset proprietary restrictions, please apply for access to LIFULL HOME'S Data [here](https://www.nii.ac.jp/dsc/idr/en/lifull/). + +Step 2: After obtaining access, download only the "photo-rent-madori-full-00" folder, which contains approximately 300,000 images. + +Step 3: Apply for access to the annotation [here](https://docs.google.com/forms/d/e/1FAIpQLSexqNMjyvPMtPMPN7bSh_1u4Q27LZAT-S9lR_gpipNIMKV5lw/viewform). + +The package has 3 folders: +- annot_npy, annot_json: the annotations saved in npy and json, respectively. +- original_vector_boundary: boundary boxes of "LIFULL HOME'S Data" which is used to create centered 512x512 images. + +These folders should be saved in the same directory as `photo-rent-madori-full-00`. For example: `data/R2G_hr_dataset/`. + +Step 4: Run `bash data_preprocess/tools/run_r2g.sh`. + +### WAFFLE + +It is noted that since WAFFLE only provides segmentation masks for a subset of 100 examples, so we only process this subset for the evaluation, not for training. + +Step 1: Download data at [here](https://tauex-my.sharepoint.com/:f:/g/personal/hadarelor_tauex_tau_ac_il/EqMX9nRbJ9xFiK7dR_m07b8BldS2saoZ4-ockqncJb_Hrg?e=zGIuos) + +Step 2: Run `bash data_preprocess/tools/run_waffle.sh`. + +## Data visualization +Please refer to this script [tools/plot_data.sh](tools/plot_data.sh). \ No newline at end of file diff --git a/data_preprocess/common_utils.py b/data_preprocess/common_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0c6057e75619570b12e3f5a36db3747e4031d399 --- /dev/null +++ b/data_preprocess/common_utils.py @@ -0,0 +1,45 @@ +import os + +import cv2 +import numpy as np +from plyfile import PlyData + + +def read_scene_pc(file_path): + with open(file_path, "rb") as f: + plydata = PlyData.read(f) + dtype = plydata["vertex"].data.dtype + print("dtype of file{}: {}".format(file_path, dtype)) + + points_data = np.array(plydata["vertex"].data.tolist()) + + return points_data + + +def is_clockwise(points): + # points is a list of 2d points. + assert len(points) > 0 + s = 0.0 + for p1, p2 in zip(points, points[1:] + [points[0]]): + s += (p2[0] - p1[0]) * (p2[1] + p1[1]) + return s > 0.0 + + +def resort_corners(corners): + # re-find the starting point and sort corners clockwisely + x_y_square_sum = corners[:, 0] ** 2 + corners[:, 1] ** 2 + start_corner_idx = np.argmin(x_y_square_sum) + + corners_sorted = np.concatenate([corners[start_corner_idx:], corners[:start_corner_idx]]) + + ## sort points clockwise + if not is_clockwise(corners_sorted[:, :2].tolist()): + corners_sorted[1:] = np.flip(corners_sorted[1:], 0) + + return corners + + +def export_density(density_map, out_folder, scene_id): + density_path = os.path.join(out_folder, scene_id + ".png") + density_uint8 = (density_map * 255).astype(np.uint8) + cv2.imwrite(density_path, density_uint8) diff --git a/data_preprocess/cubicasa5k/augmentations.py b/data_preprocess/cubicasa5k/augmentations.py new file mode 100644 index 0000000000000000000000000000000000000000..018a88d38035116a302d7d12007f349fe42cc7e4 --- /dev/null +++ b/data_preprocess/cubicasa5k/augmentations.py @@ -0,0 +1,703 @@ +import random +from math import inf + +import cv2 +import numpy as np +import torch +from floortrans.loaders import svg_utils + + +class Compose(object): + def __init__(self, augmentations): + self.augmentations = augmentations + + def __call__(self, sample): + for a in self.augmentations: + sample = a(sample) + + return sample + + +# 0. I +# 1. I top to right +# 2. I vertical flip +# 3. I top to left +# 4. L horizontal flip +# 5. L +# 6. L vertical flip +# 7. L horizontal and vertical flip +# 8. T +# 9. T top to right +# 10. T top to down +# 11. T top to left +# 12. X or + +# 13. Opening left corner +# 14. Opening right corner +# 15. Opening up corner +# 16. Opening down corer +# 17. Icon upper left +# 18. Icon upper right +# 19. Icon lower left +# 20. Icon lower right + + +class RandomRotations(object): + def __init__(self, format="furu"): + if format == "furu": + self.augment = self.furu + elif format == "cubi": + self.augment = self.cubi + + def __call__(self, sample): + return self.augment(sample) + + def cubi(self, sample): + fplan = sample["image"] + segmentation = sample["label"] + heatmap_points = sample["heatmaps"] + scale = sample["scale"] + num_of_rotations = int(torch.randint(0, 3, (1,))) + hmapp_convert_map = { + 0: 1, + 1: 2, + 2: 3, + 3: 0, + 4: 5, + 5: 6, + 6: 7, + 7: 4, + 8: 9, + 9: 10, + 10: 11, + 11: 8, + 12: 12, + 13: 15, + 14: 16, + 15: 14, + 16: 13, + 17: 18, + 18: 20, + 19: 17, + 20: 19, + } + + for i in range(num_of_rotations): + fplan = fplan.transpose(2, 1).flip(2) + segmentation = segmentation.transpose(2, 1).flip(2) + points_rotated = dict() + for junction_type, points in heatmap_points.items(): + new_junction_type = hmapp_convert_map[junction_type] + new_heatmap_points = [] + for point in points: + x = fplan.shape[1] - 1 - point[1] + y = point[0] + # if y > 256 or x > 256: + # __import__('ipdb').set_trace() + new_heatmap_points.append([x, y]) + + points_rotated[new_junction_type] = new_heatmap_points + + heatmap_points = points_rotated + + sample = {"image": fplan, "label": segmentation, "scale": scale, "heatmaps": heatmap_points} + + return sample + + def furu(self, sample): + fplan = sample["image"] + segmentation = sample["label"] + heatmap_points = sample["heatmap_points"] + num_of_rotations = int(torch.randint(0, 3, (1,))) + for i in range(num_of_rotations): + fplan = fplan.transpose(2, 1).flip(2) + segmentation = segmentation.transpose(2, 1).flip(2) + + hmapp_convert_map = { + 0: 1, + 1: 2, + 2: 3, + 3: 0, + 4: 5, + 5: 6, + 6: 7, + 7: 4, + 8: 9, + 9: 10, + 10: 11, + 11: 8, + 12: 12, + 13: 15, + 14: 16, + 15: 14, + 16: 13, + 17: 18, + 18: 20, + 19: 17, + 20: 19, + } + + points_rotated = dict() + for junction_type, points in heatmap_points.items(): + new_junction_type = hmapp_convert_map[junction_type] + new_heatmap_points = [] + for point in points: + new_heatmap_points.append([fplan.shape[1] - 1 - point[1], point[0]]) + + points_rotated[new_junction_type] = new_heatmap_points + + heatmap_points = points_rotated + + sample = {"image": fplan, "label": segmentation, "heatmap_points": heatmap_points} + + return sample + + +def clip_heatmaps(heatmaps, minx, maxx, miny, maxy): + def clip(p): + return p[0] < maxx and p[0] >= minx and p[1] < maxy and p[1] >= miny + + res = {} + for key, value in heatmaps.items(): + res[key] = list(filter(clip, value)) + for i, e in enumerate(res[key]): + res[key][i] = (e[0] - minx, e[1] - miny) + + return res + + +class DictToTensor(object): + def __init__(self, data_format="cubi"): + if data_format == "cubi": + self.augment = self.cubi + elif data_format == "furukawa": + self.augment = self.furukawa + + def __call__(self, sample): + return self.augment(sample) + + def cubi(self, sample): + image, label = sample["image"], sample["label"] + _, height, width = label.shape + heatmaps = sample["heatmaps"] + scale = sample["scale"] + + heatmap_tensor = np.zeros((21, height, width)) + for channel, coords in heatmaps.items(): + for x, y in coords: + if x >= width: + x -= 1 + if y >= height: + y -= 1 + heatmap_tensor[int(channel), int(y), int(x)] = 1 + + # Gaussian filter + kernel = svg_utils.get_gaussian2D(int(30 * scale)) + for i, h in enumerate(heatmap_tensor): + heatmap_tensor[i] = cv2.filter2D(h, -1, kernel) + + heatmap_tensor = torch.FloatTensor(heatmap_tensor) + + label = torch.cat((heatmap_tensor, label), 0) + + return {"image": image, "label": label} + + def furukawa(self, sample): + image, label = sample["image"], sample["label"] + _, height, width = label.shape + heatmap_points = sample["heatmap_points"] + + heatmap_tensor = np.zeros((21, height, width)) + for channel, coords in heatmap_points.items(): + for x, y in coords: + heatmap_tensor[int(channel), int(y), int(x)] = 1 + + # Gaussian filter + kernel = svg_utils.get_gaussian2D(13) + for i, h in enumerate(heatmap_tensor): + heatmap_tensor[i] = cv2.filter2D(h, -1, kernel, borderType=cv2.BORDER_CONSTANT, delta=0) + + heatmap_tensor = torch.FloatTensor(heatmap_tensor) + + label = torch.cat((heatmap_tensor, label), 0) + + return {"image": image, "label": label} + + +class RotateNTurns(object): + def rot_tensor(self, t, n): + # One turn clock wise + if n == 1: + t = t.flip(2).transpose(3, 2) + # One turn counter clock wise + elif n == -1: + t = t.transpose(3, 2).flip(2) + # Two turns clock wise + elif n == 2: + t = t.flip(2).flip(3) + + return t + + def rot_points(self, t, n): + # Swapping corner ts + t_sorted = t.clone().detach() + # One turn clock wise + if n == 1: + # I junctions + t_sorted[:, 1] = t[:, 0] + t_sorted[:, 2] = t[:, 1] + t_sorted[:, 3] = t[:, 2] + t_sorted[:, 0] = t[:, 3] + # L junctions + t_sorted[:, 5] = t[:, 4] + t_sorted[:, 6] = t[:, 5] + t_sorted[:, 7] = t[:, 6] + t_sorted[:, 4] = t[:, 7] + # T junctions + t_sorted[:, 9] = t[:, 8] + t_sorted[:, 10] = t[:, 9] + t_sorted[:, 11] = t[:, 10] + t_sorted[:, 8] = t[:, 11] + # Opening corners + t_sorted[:, 15] = t[:, 13] + t_sorted[:, 16] = t[:, 14] + t_sorted[:, 14] = t[:, 15] + t_sorted[:, 13] = t[:, 16] + # Icon corners + t_sorted[:, 18] = t[:, 17] + t_sorted[:, 20] = t[:, 18] + t_sorted[:, 17] = t[:, 19] + t_sorted[:, 19] = t[:, 20] + # One turn counter clock wise + elif n == -1: + # I junctions + t_sorted[:, 3] = t[:, 0] + t_sorted[:, 0] = t[:, 1] + t_sorted[:, 1] = t[:, 2] + t_sorted[:, 2] = t[:, 3] + # L junctions + t_sorted[:, 7] = t[:, 4] + t_sorted[:, 4] = t[:, 5] + t_sorted[:, 5] = t[:, 6] + t_sorted[:, 6] = t[:, 7] + # T junctions + t_sorted[:, 11] = t[:, 8] + t_sorted[:, 8] = t[:, 9] + t_sorted[:, 9] = t[:, 10] + t_sorted[:, 10] = t[:, 11] + # Opening corners + t_sorted[:, 16] = t[:, 13] + t_sorted[:, 15] = t[:, 14] + t_sorted[:, 13] = t[:, 15] + t_sorted[:, 14] = t[:, 16] + # Icon corners + t_sorted[:, 19] = t[:, 17] + t_sorted[:, 17] = t[:, 18] + t_sorted[:, 20] = t[:, 19] + t_sorted[:, 18] = t[:, 20] + # Two turns clock wise + elif n == 2: + t_sorted = t.clone().detach() + # I junctions + t_sorted[:, 2] = t[:, 0] + t_sorted[:, 3] = t[:, 1] + t_sorted[:, 0] = t[:, 2] + t_sorted[:, 4] = t[:, 3] + # L junctions + t_sorted[:, 6] = t[:, 4] + t_sorted[:, 7] = t[:, 5] + t_sorted[:, 4] = t[:, 6] + t_sorted[:, 5] = t[:, 7] + # T junctions + t_sorted[:, 10] = t[:, 8] + t_sorted[:, 11] = t[:, 9] + t_sorted[:, 8] = t[:, 10] + t_sorted[:, 9] = t[:, 11] + # Opening corners + t_sorted[:, 14] = t[:, 13] + t_sorted[:, 13] = t[:, 14] + t_sorted[:, 16] = t[:, 15] + t_sorted[:, 15] = t[:, 16] + # Icon corners + t_sorted[:, 20] = t[:, 17] + t_sorted[:, 19] = t[:, 18] + t_sorted[:, 18] = t[:, 19] + t_sorted[:, 17] = t[:, 20] + elif n == 0: + return t_sorted + + return t_sorted + + def __call__(self, sample, data_type, n): + if data_type == "tensor": + return self.rot_tensor(sample, n) + elif data_type == "points": + return self.rot_points(sample, n) + + +class RandomCropToSizeTorch(object): + def __init__( + self, + input_slice=[21, 1, 1], + size=(256, 256), + fill=(0, 0), + data_format="tensor", + dtype=torch.float32, + max_size=None, + ): + self.size = size + self.width = size[0] + self.height = size[1] + self.dtype = dtype + self.fill = fill + self.max_size = max_size + self.input_slice = input_slice + + if data_format == "dict": + self.augment = self.augment_dict + elif data_format == "tensor": + self.augment = self.augment_tesor + elif data_format == "dict furu": + self.augment = self.augment_dict_furu + + def __call__(self, sample): + return self.augment(sample) + + def augment_tesor(self, sample): + image, label = sample["image"], sample["label"] + img_w = image.shape[2] + img_h = image.shape[1] + pad_w = int(self.width / 2) + pad_h = int(self.height / 2) + + new_w = self.width + max(img_w, self.width) + new_h = self.height + max(img_h, self.height) + + new_image = torch.zeros([image.shape[0], new_h, new_w], dtype=self.dtype) + new_image[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = image + + new_heatmaps = torch.zeros([self.input_slice[0], new_h, new_w], dtype=self.dtype) + new_heatmaps[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[: self.input_slice[0]] + + new_rooms = torch.full((self.input_slice[1], new_h, new_w), self.fill[0]) + new_rooms[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[self.input_slice[0]] + new_icons = torch.full((self.input_slice[2], new_h, new_w), self.fill[1]) + new_icons[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[self.input_slice[0] + self.input_slice[1]] + + label = torch.cat((new_heatmaps, new_rooms, new_icons), 0) + image = new_image + + removed_up = random.randint(0, new_h - self.width) + removed_left = random.randint(0, new_w - self.height) + + removed_down = new_h - self.height - removed_up + removed_right = new_w - self.width - removed_left + + if removed_down == 0 and removed_right == 0: + image = image[:, removed_up:, removed_left:] + label = label[:, removed_up:, removed_left:] + elif removed_down == 0: + image = image[:, removed_up:, removed_left:-removed_right] + label = label[:, removed_up:, removed_left:-removed_right] + elif removed_right == 0: + image = image[:, removed_up:-removed_down, removed_left:] + label = label[:, removed_up:-removed_down, removed_left:] + else: + image = image[:, removed_up:-removed_down, removed_left:-removed_right] + label = label[:, removed_up:-removed_down, removed_left:-removed_right] + + return {"image": image, "label": label} + + def augment_dict(self, sample): + image, label = sample["image"], sample["label"] + heatmap_points = sample["heatmaps"] + img_w = image.shape[2] + img_h = image.shape[1] + pad_w = int(self.width / 2) + pad_h = int(self.height / 2) + + new_w = self.width + img_w + new_h = self.height + img_h + + new_image = torch.full([image.shape[0], new_h, new_w], 255) + new_image[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = image + + new_rooms = torch.full((1, new_h, new_w), self.fill[0]) + new_rooms[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[0] + new_icons = torch.full((1, new_h, new_w), self.fill[1]) + new_icons[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[1] + + label = torch.cat((new_rooms, new_icons), 0) + image = new_image + + removed_up = random.randint(0, new_h - self.width) + removed_left = random.randint(0, new_w - self.height) + + removed_down = new_h - self.height - removed_up + removed_right = new_w - self.width - removed_left + + new_heatmap_points = dict() + for junction_type, points in heatmap_points.items(): + new_heatmap_points_per_type = [] + for point in points: + new_heatmap_points_per_type.append([point[0] + pad_w, point[1] + pad_h]) + + new_heatmap_points[junction_type] = new_heatmap_points_per_type + + heatmap_points = new_heatmap_points + + if removed_down == 0 and removed_right == 0: + image = image[:, removed_up:, removed_left:] + label = label[:, removed_up:, removed_left:] + heatmap_points = clip_heatmaps(heatmap_points, removed_left, inf, removed_up, inf) + elif removed_down == 0: + image = image[:, removed_up:, removed_left:-removed_right] + label = label[:, removed_up:, removed_left:-removed_right] + heatmap_points = clip_heatmaps(heatmap_points, removed_left, removed_left + self.width, removed_up, inf) + elif removed_right == 0: + image = image[:, removed_up:-removed_down, removed_left:] + label = label[:, removed_up:-removed_down, removed_left:] + heatmap_points = clip_heatmaps(heatmap_points, removed_left, inf, removed_up, removed_up + self.width) + else: + image = image[:, removed_up:-removed_down, removed_left:-removed_right] + label = label[:, removed_up:-removed_down, removed_left:-removed_right] + heatmap_points = clip_heatmaps( + heatmap_points, removed_left, removed_left + self.width, removed_up, removed_up + self.height + ) + + return {"image": image, "label": label, "heatmaps": heatmap_points, "scale": sample["scale"]} + + def augment_dict_furu(self, sample): + image, label = sample["image"], sample["label"] + heatmap_points = sample["heatmap_points"] + img_w = image.shape[2] + img_h = image.shape[1] + pad_w = int(self.width / 2) + pad_h = int(self.height / 2) + + new_w = self.width + img_w + new_h = self.height + img_h + + new_image = torch.full([image.shape[0], new_h, new_w], 255) + new_image[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = image + + new_rooms = torch.full((1, new_h, new_w), self.fill[0]) + new_rooms[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[0] + new_icons = torch.full((1, new_h, new_w), self.fill[1]) + new_icons[:, pad_h : img_h + pad_h, pad_w : img_w + pad_w] = label[1] + + label = torch.cat((new_rooms, new_icons), 0) + image = new_image + + removed_up = random.randint(0, new_h - self.width) + removed_left = random.randint(0, new_w - self.height) + + removed_down = new_h - self.height - removed_up + removed_right = new_w - self.width - removed_left + + new_heatmap_points = dict() + for junction_type, points in heatmap_points.items(): + new_heatmap_points_per_type = [] + for point in points: + new_heatmap_points_per_type.append([point[0] + pad_w, point[1] + pad_h]) + + new_heatmap_points[junction_type] = new_heatmap_points_per_type + + heatmap_points = new_heatmap_points + + if removed_down == 0 and removed_right == 0: + image = image[:, removed_up:, removed_left:] + label = label[:, removed_up:, removed_left:] + heatmap_points = clip_heatmaps(heatmap_points, removed_left, inf, removed_up, inf) + elif removed_down == 0: + image = image[:, removed_up:, removed_left:-removed_right] + label = label[:, removed_up:, removed_left:-removed_right] + heatmap_points = clip_heatmaps(heatmap_points, removed_left, removed_left + self.width, removed_up, inf) + elif removed_right == 0: + image = image[:, removed_up:-removed_down, removed_left:] + label = label[:, removed_up:-removed_down, removed_left:] + heatmap_points = clip_heatmaps(heatmap_points, removed_left, inf, removed_up, removed_up + self.width) + else: + image = image[:, removed_up:-removed_down, removed_left:-removed_right] + label = label[:, removed_up:-removed_down, removed_left:-removed_right] + heatmap_points = clip_heatmaps( + heatmap_points, removed_left, removed_left + self.width, removed_up, removed_up + self.height + ) + + return {"image": image, "label": label, "heatmap_points": heatmap_points} + + +class ColorJitterTorch(object): + def __init__(self, b_var=0.4, c_var=0.4, s_var=0.4, dtype=torch.float32, version="dict"): + self.b_var = b_var + self.c_var = c_var + self.s_var = s_var + self.dtype = dtype + self.version = version + + def __call__(self, sample): + res = sample + image = sample["image"] + image = self.brightness(image, self.b_var) + image = self.contrast(image, self.c_var) + image = self.saturation(image, self.s_var) + res["image"] = image + + return res + + def blend(self, img_1, img_2, var): + m = torch.tensor([0], dtype=self.dtype).uniform_(-var, var) + alpha = 1 + m + res = img_1 * alpha + (1 - alpha) * img_2 + res = torch.clamp(res, min=0, max=255) + + return res + + def grayscale(self, img): + red = img[0] * 0.299 + green = img[1] * 0.587 + blue = img[2] * 0.114 + gray = red + green + blue + gray = torch.clamp(gray, min=0, max=255) + res = torch.stack((gray, gray, gray), dim=0) + + return res + + def saturation(self, img, var): + res = self.grayscale(img) + res = self.blend(img, res, var) + + return res + + def brightness(self, img, var): + res = torch.zeros(img.shape) + res = self.blend(img, res, var) + + return res + + def contrast(self, img, var): + res = self.grayscale(img) + mean_color = res.mean() + res = torch.full(res.shape, mean_color) + res = self.blend(img, res, var) + + return res + + +class ResizePaddedTorch(object): + def __init__(self, fill, size=(256, 256), both=True, dtype=torch.float32, data_format="tensor"): + self.size = size + self.width = size[0] + self.height = size[1] + self.both = both + self.dtype = dtype + self.fill = fill + self.fill_cval = 255 + if data_format == "tensor": + self.augment = self.augment_tensor + elif data_format == "dict furu": + self.augment = self.augment_dict_furu + elif data_format == "dict": + self.augment = self.augment_dict + self.fill_cval = 1 + + def __call__(self, sample): + # image 1: Bi-cubic interpolation as in original paper + image, _, _, _ = self.resize_padded( + sample["image"], self.size, fill_cval=self.fill_cval, image=True, mode="bilinear", aling_corners=False + ) + sample["image"] = image + + return self.augment(sample) + + def augment_tensor(self, sample): + image, label = sample["image"], sample["label"] + + if self.both: + # labels 0: Nearest-neighbor interpolation + heatmaps, _, _, _ = self.resize_padded(label[:21], self.size, mode="bilinear", aling_corners=False) + rooms_padded, _, _, _ = self.resize_padded(label[[21]], self.size, mode="nearest", fill_cval=self.fill[0]) + icons_padded, _, _, _ = self.resize_padded( + label[[22]], + self.size, + mode="nearest", + fill_cval=self.fill[1], + ) + label = torch.cat((heatmaps, rooms_padded, icons_padded), dim=0) + + return {"image": image, "label": label} + + def augment_dict_furu(self, sample): + image, label = sample["image"], sample["label"] + heatmap_points = sample["heatmap_points"] + + rooms_padded, _, _, _ = self.resize_padded(label[[0]], self.size, mode="nearest", fill_cval=self.fill[0]) + icons_padded, ratio, y_pad, x_pad = self.resize_padded( + label[[1]], self.size, mode="nearest", fill_cval=self.fill[1] + ) + label = torch.cat((rooms_padded, icons_padded), dim=0) + + new_heatmap_points = dict() + for junction_type, points in heatmap_points.items(): + new_heatmap_points_per_type = [] + for point in points: + # Indexing starts from 0 but when we multiply with the ratio we need to start from 0. + new_x = point[0] * ratio + x_pad + new_y = point[1] * ratio + y_pad + new_heatmap_points_per_type.append([new_x, new_y]) + new_heatmap_points[junction_type] = new_heatmap_points_per_type + + heatmap_points = new_heatmap_points + + return {"image": image, "label": label, "heatmap_points": heatmap_points} + + def augment_dict(self, sample): + image, label = sample["image"], sample["label"] + heatmap_points = sample["heatmaps"] + scale = sample["scale"] + + rooms_padded, _, _, _ = self.resize_padded(label[[0]], self.size, mode="nearest", fill_cval=self.fill[0]) + icons_padded, ratio, y_pad, x_pad = self.resize_padded( + label[[1]], self.size, mode="nearest", fill_cval=self.fill[1] + ) + label = torch.cat((rooms_padded, icons_padded), dim=0) + + new_heatmap_points = dict() + for junction_type, points in heatmap_points.items(): + new_heatmap_points_per_type = [] + for point in points: + # Indexing starts from 0 but when we multiply with the ratio we need to start from 0. + new_x = point[0] * ratio + x_pad + new_y = point[1] * ratio + y_pad + if new_y < 256 and new_x < 256 and new_y >= 0 and new_x >= 0: + # __import__('ipdb').set_trace() + new_heatmap_points_per_type.append([new_x, new_y]) + new_heatmap_points[junction_type] = new_heatmap_points_per_type + + heatmap_points = new_heatmap_points + + return {"image": image, "label": label, "heatmaps": heatmap_points, "scale": scale} + + def resize_padded(self, img, new_shape, image=False, fill_cval=0, mode="nearest", aling_corners=None): + new_shape = torch.tensor([img.shape[0], new_shape[0], new_shape[1]], dtype=self.dtype) + old_shape = torch.tensor(img.shape, dtype=self.dtype) + + ratio = (new_shape / old_shape).min() + img_s = torch.tensor(img.shape[1:], dtype=self.dtype) + interm_shape = (ratio * img_s).ceil() + + interm_shape = [interm_shape[0], interm_shape[1]] + + img = img.unsqueeze(0) + interm_img = torch.nn.functional.interpolate(img, size=interm_shape, mode=mode, align_corners=aling_corners) + interm_img = interm_img.squeeze(0) + + a = (interm_img.shape[0], self.size[0], self.size[1]) + + new_img = torch.full(a, fill_cval) + + x_pad = int((self.width - interm_img.shape[1]) / 2) + y_pad = int((self.height - interm_img.shape[2]) / 2) + + new_img[:, x_pad : interm_img.shape[1] + x_pad, y_pad : interm_img.shape[2] + y_pad] = interm_img + + return new_img, ratio, x_pad, y_pad diff --git a/data_preprocess/cubicasa5k/combine_json.py b/data_preprocess/cubicasa5k/combine_json.py new file mode 100644 index 0000000000000000000000000000000000000000..b69750092198e76f367d66d8b187d595fd0c2bb6 --- /dev/null +++ b/data_preprocess/cubicasa5k/combine_json.py @@ -0,0 +1,118 @@ +import glob +import json +import os +import shutil +from pathlib import Path + + +def combine_json_files(input_pattern, data_path, split_type, output_file, start_image_id=0): + """ + Combines multiple COCO-style JSON annotation files into a single file. + + Args: + input_pattern: Glob pattern to match the input JSON files (e.g., "annotations/*.json") + output_file: Path to the output combined JSON file + """ + # Initialize combined data structure + combined_data = {"images": [], "annotations": [], "categories": []} + + # Track image and annotation IDs to avoid duplicates + annotation_ids_seen = set() + + next_image_id = start_image_id + next_annotation_id = 0 + skip_file_list = [] + image_id_mapping = {} + + # Find all matching JSON files + json_files = sorted(glob.glob(input_pattern)) + print(f"Found {len(json_files)} JSON files to combine") + + # Process each file + for i, json_file in enumerate(json_files): + print(f"Processing file {i + 1}/{len(json_files)}: {json_file}") + + with open(json_file, "r") as f: + data = json.load(f) + + # Store categories from the first file + if i == 0 and data.get("categories"): + combined_data["categories"] = data["categories"] + + # empty annos + if len(data["annotations"]) == 0: + skip_file_list.append(data["images"][0]["id"]) + continue + + # Process images + for image in data.get("images", []): + if image["id"] not in image_id_mapping: + image_id_mapping[image["id"]] = next_image_id + else: + skip_file_list.append(image["id"]) + continue + image["id"] = next_image_id + next_image_id += 1 + image["file_name"] = str(image["id"]).zfill(5) + ".png" + org_file_name = os.path.basename(json_file).replace(".json", ".png") + if image["file_name"] != org_file_name and os.path.exists(f"{data_path}/{split_type}/{org_file_name}"): + shutil.move( + f"{data_path}/{split_type}/{org_file_name}", f"{data_path}/{split_type}/{image['file_name']}" + ) + combined_data["images"].append(image) + + # Process annotations + for annotation in data.get("annotations", []): + annotation["id"] = next_annotation_id + next_annotation_id += 1 + annotation["image_id"] = image_id_mapping[annotation["image_id"]] + + annotation_ids_seen.add(annotation["id"]) + combined_data["annotations"].append(annotation) + + # Write combined data to output file + output_path = Path(output_file) + output_path.parent.mkdir(exist_ok=True, parents=True) + + with open(output_file, "w") as f: + json.dump(combined_data, f, indent=2) + + with open(output_path.parent / f"{output_path.name.split('.')[0]}_image_id_mapping.json", "w") as f: + json.dump(image_id_mapping, f, indent=2) + + if len(skip_file_list): + with open(output_path.parent / f"{output_path.name.split('.')[0]}_skipped.txt", "w") as f: + f.write("\n".join([str(x) for x in skip_file_list])) + + print(f"Combined data written to {output_file}") + print(f"Total images: {len(combined_data['images'])}") + print(f"Total annotations: {len(combined_data['annotations'])}") + print(f"Total categories: {len(combined_data['categories'])}") + print(f"Skipped images: {len(skip_file_list)}") + + return combined_data + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Combine multiple COCO-style JSON annotation files") + parser.add_argument("--input", required=True, help="Glob pattern for input JSON files, e.g., 'annotations/*.json'") + parser.add_argument("--output", required=True, help="Output JSON file path") + + args = parser.parse_args() + + splits = ["train", "val", "test"] + for i, split in enumerate(splits): + if split == "train": + start_image_id = 0 + else: + start_image_id += len(list(Path(f"{args.input}/{splits[i - 1]}").glob("*.png"))) + + combine_json_files( + f"{args.input}/annotations_json/{split}/*.json", + args.input, + split, + f"{args.output}/{split}.json", + start_image_id=start_image_id, + ) diff --git a/data_preprocess/cubicasa5k/create_coco_cc5k.py b/data_preprocess/cubicasa5k/create_coco_cc5k.py new file mode 100644 index 0000000000000000000000000000000000000000..695b521b71eacd745729674e67e4b7ef7d39be86 --- /dev/null +++ b/data_preprocess/cubicasa5k/create_coco_cc5k.py @@ -0,0 +1,672 @@ +import argparse +import json +import os +import sys +from multiprocessing import Pool +from pathlib import Path + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +from loaders import FloorplanSVG +from matplotlib.patches import Patch +from PIL import Image +from shapely.geometry import Polygon +from skimage import measure +from tqdm import tqdm + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) + +sys.path.append(str(Path(__file__).resolve().parent.parent)) +from common_utils import resort_corners +from stru3d.stru3d_utils import type2id + +#### ORIGINAL ROOM NAMES & ICON_NAMES #### +ROOM_NAMES = { + 0: "Background", + 1: "Outdoor", + 2: "Wall", + 3: "Kitchen", + 4: "Living Room", + 5: "Bed Room", + 6: "Bath", + 7: "Entry", + 8: "Railing", + 9: "Storage", + 10: "Garage", + 11: "Undefined", +} + +ICON_NAMES = { + 0: "No Icon", + 1: "Window", + 2: "Door", + 3: "Closet", + 4: "Electrical Applience", + 5: "Toilet", + 6: "Sink", + 7: "Sauna Bench", + 8: "Fire Place", + 9: "Bathtub", + 10: "Chimney", +} + + +CC5K_2_S3D_MAPPING = { + 0: None, # "Background" + 1: type2id["balcony"], # "Outdoor" -> balcony (4) + 2: None, # "Wall" has no direct match + 3: type2id["kitchen"], # Kitchen -> kitchen (1) + 4: type2id["living room"], # Living Room -> living room (0) + 5: type2id["bedroom"], # Bed Room -> bedroom (2) + 6: type2id["bathroom"], # Bath -> bathroom (3) + 7: 18, # 'Entry' has no direct match + 8: 19, # "Railing" has no direct match + 9: type2id["store room"], # Storage -> store room (9) + 10: type2id["garage"], # Garage -> garage (14) + 11: type2id["undefined"], # Undefined -> undefined (15) + 12: type2id["window"], # Window -> window (17) + 13: type2id["door"], # Door -> door (16) +} + +CC5K_MAPPING = { + 0: None, + 1: 0, # Outdoor + 2: 1, # Wall + 3: 2, # Kitchen + 4: 3, # Living Room + 5: 4, # Bed Room + 6: 5, # Bath + 7: 6, # Entry + 8: 1, # Railing -> Wall + 9: 7, # Storage + 10: 8, # Garage + 11: 9, # Undefined + 12: 10, # Window + 13: 11, # Door +} + +CC5K_MAPPING_2 = { + 0: None, + 1: 0, # Outdoor + 2: None, # Wall + 3: 1, # Kitchen + 4: 2, # Living Room + 5: 3, # Bed Room + 6: 4, # Bath + 7: 5, # Entry + 8: None, # Railing -> Wall + 9: 6, # Storage + 10: 7, # Garage + 11: 8, # Undefined + 12: 9, # Window + 13: 10, # Door +} + +CC5K_CLASS_MAPPING = { + "Outdoor": 0, + "Wall, Railing": 1, + "Kitchen": 2, + "Living Room": 3, + "Bed Room": 4, + "Bath": 5, + "Entry": 6, + "Storage": 7, + "Garage": 8, + "Undefined": 9, + "Window": 10, + "Door": 11, +} + +CC5K_CLASS_MAPPING_2 = { + "Outdoor": 0, + "Kitchen": 1, + "Living Room": 2, + "Bed Room": 3, + "Bath": 4, + "Entry": 5, + "Storage": 6, + "Garage": 7, + "Undefined": 8, + "Window": 9, + "Door": 10, +} + +CLASS_MAPPING = { + "living room": 0, + "kitchen": 1, + "bedroom": 2, + "bathroom": 3, + "balcony": 4, + "corridor": 5, + "dining room": 6, + "study": 7, + "studio": 8, + "store room": 9, + "garden": 10, + "laundry room": 11, + "office": 12, + "basement": 13, + "garage": 14, + "undefined": 15, + "door": 16, + "window": 17, + "entry": 18, + "railing": 19, +} + + +def fill_holes_in_mask(binary_mask): + """ + Fill 0-pixels in a binary mask that are completely surrounded by 1-pixels. + + Args: + binary_mask (numpy.ndarray): Binary mask with 0 and 1 values. + + Returns: + numpy.ndarray: Binary mask with holes filled. + """ + # Ensure the mask is binary (0 and 1) + binary_mask = (binary_mask > 0).astype(np.uint8) + + # Apply dilation + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 15)) + binary_mask = cv2.dilate(binary_mask, kernel, iterations=1) + + # Find contours in the mask + contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + # Fill the contours + filled_mask = binary_mask.copy() + cv2.fillPoly(filled_mask, contours, 1) + + return filled_mask + + +def close_contour(contour): + if not np.array_equal(contour[0], contour[-1]): + contour = np.vstack((contour, contour[0])) + return contour + + +def binary_mask_to_polygon(binary_mask, tolerance=0): + """Converts a binary mask to COCO polygon representation + Ref: https://github.com/waspinator/pycococreator/blob/master/pycococreatortools/pycococreatortools.py + + Args: + binary_mask: a 2D binary numpy array where '1's represent the object + tolerance: Maximum distance from original points of polygon to approximated + polygonal chain. If tolerance is 0, the original coordinate array is returned. + + """ + polygons = [] + # pad mask to close contours of shapes which start and end at an edge + padded_binary_mask = np.pad(binary_mask, pad_width=1, mode="constant", constant_values=0) + contours = measure.find_contours(padded_binary_mask, 0.5) + contours = np.subtract(contours, 1) + for contour in contours: + contour = close_contour(contour) + contour = measure.approximate_polygon(contour, tolerance) + if len(contour) < 3: + continue + contour = np.flip(contour, axis=1) + segmentation = contour.ravel().tolist() + # after padding and subtracting 1 we may get -0.5 points in our segmentation + segmentation = [0 if i < 0 else i for i in segmentation] + polygons.append(segmentation) + + return polygons + + +def extract_icon_cv2(mask, start_cls_id=11, skip_classes=[]): + room_ids = np.unique(mask) + room_polygons = [] + new_mask = np.zeros(mask.shape) + + # window, door + for room_id in room_ids: + if room_id in skip_classes: + continue + true_room_id = int(room_id) + start_cls_id + # Create binary mask for this room + room_mask = (mask == room_id).astype(np.uint8) + new_mask = np.where(room_mask, true_room_id, 0) + + # Find contours using OpenCV + contours, _ = cv2.findContours(room_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if contours: + # # Get the largest contour + # largest_contour = max(contours, key=cv2.contourArea) + for cnt in contours: + polygon = [tuple(point[0]) for point in cnt] + if len(polygon) < 3: + continue + + poly = Polygon(polygon) + simplified_poly = poly.simplify(tolerance=0.5, preserve_topology=True) + simplified_poly = list(simplified_poly.exterior.coords) + room_polygons.append([simplified_poly, true_room_id]) + + return room_polygons, new_mask + + +def visualize_room_polygons(mask, room_polygons, class_names, save_path="cubicasa_debug.png", bg_polygons=None): + """ + Visualize the extracted room polygons. + + Args: + mask: Original segmentation mask + room_polygons: Dictionary of room polygons as returned by extract_room_polygons + figsize: Figure size for the plot + """ + # Set figure size to exactly 256x256 pixels + dpi = 100 # Standard screen DPI + figsize = (mask.shape[1] / dpi, mask.shape[0] / dpi) # Convert pixels to inches + + # Get unique classes from the mask + unique_classes = np.unique(mask) + + # Create a discrete colormap + cmap = plt.cm.get_cmap("gist_ncar", 256) # nipy_spectral + # cmap = ListedColormap([cmap(x) for x in np.linspace(0, 1, int(20))]) + + fig = plt.figure(figsize=figsize) + ax = fig.add_axes([0, 0, 1, 1]) + plt.imshow(mask, cmap=cmap, interpolation="nearest", alpha=0.6, vmin=0, vmax=20) + + # Plot each room polygon + for polygon, room_cls in room_polygons: + polygon_array = np.array(polygon).copy() + # # flip y + # polygon_array[:, 1] = mask.shape[0] - polygon_array[:, 1] - 1 + ax.plot(polygon_array[:, 0], polygon_array[:, 1], "k-", linewidth=2) + + # Add room ID label at the centroid + centroid_x = np.mean(polygon_array[:, 0]) + centroid_y = np.mean(polygon_array[:, 1]) + ax.text( + centroid_x, + centroid_y, + str(room_cls), + fontsize=12, + ha="center", + va="center", + bbox=dict(facecolor="white", alpha=0.7), + ) + + if bg_polygons is not None: + # Plot each room polygon + for polygon, room_cls in bg_polygons: + polygon_array = np.array(polygon).copy() + # # flip y + # polygon_array[:, 1] = mask.shape[0] - polygon_array[:, 1] - 1 + ax.plot(polygon_array[:, 0], polygon_array[:, 1], "c-", linewidth=2) + + # Create custom legend elements + legend_elements = [] + norm = np.linspace(0, 1, 21) # int(max(unique_classes))+1 + + for i, cls in enumerate(sorted(unique_classes)): + # if int(cls) == 0: + # continue + # Get the exact same color that imshow uses + color = cmap(norm[int(cls)]) + # color = cmap(int(cls)) + + cls_name = f"{int(cls)}_{class_names[int(cls)]}" + # You can replace f"Class {cls}" with your actual class names if available + legend_elements.append(Patch(facecolor=color, edgecolor="black", label=f"{cls_name}", alpha=0.6)) + + # Add the legend to the plot + ax.legend( + handles=legend_elements, + loc="best", + title="Classes", + fontsize=20, + markerscale=4, + title_fontsize=28, + ) + + # plt.title('Room Polygons Extracted from Segmentation Mask') + plt.axis("equal") + plt.axis("off") + fig.savefig(save_path, bbox_inches="tight", pad_inches=0) + plt.close() + + +def config(): + a = argparse.ArgumentParser(description="Generate coco format data for Structured3D") + a.add_argument( + "--data_root", default="Structured3D_panorama", type=str, help="path to raw Structured3D_panorama folder" + ) + a.add_argument("--output", default="coco_cubicasa5k", type=str, help="path to output folder") + a.add_argument("--disable_wd2line", action="store_true") + + args = a.parse_args() + return args + + +def save_image(image_path, output_path, mask=None): + """ + ref: https://github.com/ultralytics/ultralytics/issues/339 + """ + img = Image.open(image_path).convert("RGB") + img.info.pop("icc_profile", None) + + if mask is not None: + img_array = np.array(img) + if len(mask.shape) == 2 and len(img_array.shape) == 3: + mask = mask[:, :, np.newaxis] + masked_img = np.where(mask == 0, 255, img_array) + img = Image.fromarray(masked_img.astype(np.uint8)) + + img.save(output_path) + + +def remove_polygons_by_type(polygons, skip_types=[]): + new_room_polygons = [] + for polygon, poly_type in polygons: + if poly_type in skip_types: + continue + new_room_polygons.append([polygon, poly_type]) + return new_room_polygons + + +def merge_rooms_and_icons(room_polygons, icon_polygons): + new_icon_polygons = [] + for poly, poly_type in icon_polygons: + new_icon_polygons.append([poly, poly_type + 11]) + + return room_polygons + new_icon_polygons + + +def create_coco_bounding_box(bb_x, bb_y, image_width, image_height, bound_pad=2): + bb_x = np.unique(bb_x) + bb_y = np.unique(bb_y) + bb_x_min = np.maximum(np.min(bb_x) - bound_pad, 0) + bb_y_min = np.maximum(np.min(bb_y) - bound_pad, 0) + + bb_x_max = np.minimum(np.max(bb_x) + bound_pad, image_width - 1) + bb_y_max = np.minimum(np.max(bb_y) + bound_pad, image_height - 1) + + bb_width = bb_x_max - bb_x_min + bb_height = bb_y_max - bb_y_min + + coco_bb = [bb_x_min, bb_y_min, bb_width, bb_height] + return coco_bb + + +def process_floorplan( + image_set, + scene_id, + start_scene_id, + args, + save_dir, + annos_folder, + use_org_cc5k_classs=False, + vis_fp=False, + wd2line=False, +): + if use_org_cc5k_classs: + class_mapping_dict = CC5K_MAPPING_2 # old: CC5K_MAPPING + class_to_index_dict = CC5K_CLASS_MAPPING_2 + door_window_index = [10, 9] + else: + class_mapping_dict = CC5K_2_S3D_MAPPING + class_to_index_dict = CLASS_MAPPING + door_window_index = [16, 17] + + mask = image_set["label"].numpy() + room_polygons = [[poly, poly_type] for poly, poly_type in zip(image_set["room_polygon"], image_set["room_type"])] + icon_polygons = [[poly, poly_type] for poly, poly_type in zip(image_set["icon_polygon"], image_set["icon_type"])] + + image_height, image_width = mask.shape[1:] + coco_annotation_dict_list = [] + + # for storing + save_dict = prepare_dict(class_to_index_dict) # old: CC5K_CLASS_MAPPING + + instance_id = 0 + img_id = int(scene_id) + start_scene_id + img_dict = {} + img_dict["file_name"] = str(img_id).zfill(5) + ".png" + img_dict["id"] = img_id + img_dict["width"] = image_width + img_dict["height"] = image_height + + if vis_fp: + os.makedirs(save_dir.rstrip("/") + "_aux", exist_ok=True) + visualize_room_polygons( + mask[0], + room_polygons, + list(ROOM_NAMES.values()), + save_path=f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5)}_room.png", + ) + visualize_room_polygons( + mask[1], + icon_polygons, + list(ICON_NAMES.values()), + bg_polygons=room_polygons, + save_path=f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5)}_icon.png", + ) + + #### FILTER NON-USE TYPES + # DROP BG + room_skip_types = [0] + filtered_room_polygons = remove_polygons_by_type(room_polygons, skip_types=room_skip_types) + # visualize_room_polygons(mask[0], filtered_room_polygons, list(ROOM_NAMES.values()), + # save_path=f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5)}_room_filtered.png") + + # Exclude all furnitures, excepts window, door + icon_skip_types = [0, *list(range(3, 11))] + filtered_icon_polygons = remove_polygons_by_type(icon_polygons, skip_types=icon_skip_types) + # visualize_room_polygons(mask[1], filtered_icon_polygons, list(ICON_NAMES.values()), + # bg_polygons=room_polygons, save_path=f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5)}_icon_filtered.png") + + #### COMBINED + combined_polygons = merge_rooms_and_icons(filtered_room_polygons, filtered_icon_polygons) + + filtered_mask1 = mask[0].copy() + filtered_mask1[np.isin(mask[0], room_skip_types)] = 0 + + filtered_mask2 = mask[1].copy() + filtered_mask2[np.isin(mask[1], icon_skip_types)] = 0 + filtered_mask2[filtered_mask2 != 0] += 11 + + filtered_mask = np.where(filtered_mask2 != 0, filtered_mask2, filtered_mask1) + + new_filtered_mask = filtered_mask.copy() + for src_type, dest_type in class_mapping_dict.items(): + if dest_type is None: + continue + new_filtered_mask[filtered_mask == src_type] = dest_type + 1 + # filtered_mask = new_filtered_mask + + binary_mask = np.zeros_like(filtered_mask) + binary_mask = np.where((mask[0] + mask[1]) != 0, 1, 0).astype(np.uint8) + filled_mask = fill_holes_in_mask(binary_mask) + cv2.imwrite( + f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5) + '_mask.png'}", filled_mask.astype(np.uint8) * 255 + ) + # visualize_room_polygons(combined_mask, combined_polygons, list(ROOM_NAMES.values()) + list(ICON_NAMES.values()), save_path=f"{save_dir}/{str(img_id).zfill(5)}_combined.png") + + save_image( + f"{args.data_root}/{image_set['folder']}/F1_scaled.png", + f"{save_dir}/{str(img_id).zfill(5) + '.png'}", + mask=filled_mask, + ) + if vis_fp: + save_image( + f"{args.data_root}/{image_set['folder']}/F1_scaled.png", + f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5) + '_org.png'}", + mask=None, + ) + + output_polygon_list = [] + combined_polygon_list = [] + for poly_ind, (polygon, poly_type) in enumerate(combined_polygons): + poly_shapely = Polygon(polygon) + area = poly_shapely.area + + org_poly_type = poly_type + poly_type = class_mapping_dict[poly_type] + if poly_type is None: + continue + + if poly_type not in door_window_index and area < 100: + continue + if poly_type in door_window_index and area < 1: + continue + + rectangle_shapely = poly_shapely.envelope + polygon = np.array(polygon) + + ### here we convert door/window annotation into a single line + if poly_type in door_window_index and wd2line: + if polygon.shape[0] > 4: + if len(polygon) == 5 and (polygon[0] == polygon[-1]).all(): + polygon = polygon[:-1] # drop last point since it is same as first + else: + bounding_rect = np.array(poly_shapely.minimum_rotated_rectangle.exterior.coords) + polygon = bounding_rect[:4] + + assert polygon.shape[0] == 4 + midp_1 = (polygon[0] + polygon[1]) / 2 + midp_2 = (polygon[1] + polygon[2]) / 2 + midp_3 = (polygon[2] + polygon[3]) / 2 + midp_4 = (polygon[3] + polygon[0]) / 2 + + dist_1_3 = np.square(midp_1 - midp_3).sum() + dist_2_4 = np.square(midp_2 - midp_4).sum() + if dist_1_3 > dist_2_4: + polygon = np.row_stack([midp_1, midp_3]) + else: + polygon = np.row_stack([midp_2, midp_4]) + + coco_seg_poly = [] + poly_sorted = resort_corners(polygon) + + for p in poly_sorted: + coco_seg_poly += list(p) + + # Slightly wider bounding box + bb_x, bb_y = rectangle_shapely.exterior.xy + coco_bb = create_coco_bounding_box(bb_x, bb_y, image_width, image_height, bound_pad=2) + + coco_annotation_dict = { + "segmentation": [coco_seg_poly], + "area": area, + "iscrowd": 0, + "image_id": img_id, + "bbox": coco_bb, + "category_id": poly_type, + "id": instance_id, + } + coco_annotation_dict_list.append(coco_annotation_dict) + instance_id += 1 + + combined_polygon_list.append([np.array(coco_seg_poly).reshape(-1, 2), org_poly_type]) + output_polygon_list.append([np.array(coco_seg_poly).reshape(-1, 2), poly_type + 1]) + + #### end split_file loop + save_dict["images"].append(img_dict) + save_dict["annotations"] += coco_annotation_dict_list + + json_path = f"{annos_folder}/{str(img_id).zfill(5) + '.json'}" + with open(json_path, "w") as f: + json.dump(save_dict, f) + + if vis_fp: + visualize_room_polygons( + filtered_mask, + combined_polygon_list, + list(ROOM_NAMES.values()) + ["window", "door"], + save_path=f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5)}_combined.png", + ) + visualize_room_polygons( + new_filtered_mask, + output_polygon_list, + ["null"] + list(class_to_index_dict.keys()), + save_path=f"{save_dir.rstrip('/') + '_aux'}/{str(img_id).zfill(5)}_final.png", + ) + + +def prepare_dict(categories_dict): + save_dict = {"images": [], "annotations": [], "categories": []} + for key, value in categories_dict.items(): + type_dict = {"supercategory": "room", "id": value, "name": key} + save_dict["categories"].append(type_dict) + return save_dict + + +if __name__ == "__main__": + args = config() + + ### prepare + outFolder = args.output + if not os.path.exists(outFolder): + os.mkdir(outFolder) + + annotation_outFolder = os.path.join(outFolder, "annotations_json") + if not os.path.exists(annotation_outFolder): + os.mkdir(annotation_outFolder) + + annos_train_folder = os.path.join(annotation_outFolder, "train") + annos_val_folder = os.path.join(annotation_outFolder, "val") + annos_test_folder = os.path.join(annotation_outFolder, "test") + os.makedirs(annos_train_folder, exist_ok=True) + os.makedirs(annos_val_folder, exist_ok=True) + os.makedirs(annos_test_folder, exist_ok=True) + + train_img_folder = os.path.join(outFolder, "train") + val_img_folder = os.path.join(outFolder, "val") + test_img_folder = os.path.join(outFolder, "test") + + for img_folder in [train_img_folder, val_img_folder, test_img_folder]: + if not os.path.exists(img_folder): + os.mkdir(img_folder) + + coco_train_json_path = os.path.join(annotation_outFolder, "train.json") + coco_val_json_path = os.path.join(annotation_outFolder, "val.json") + coco_test_json_path = os.path.join(annotation_outFolder, "test.json") + + ### begin processing + start_scene_id = 3500 # following index of s3d data + split_set = ["train.txt", "val.txt", "test.txt"] + save_folders = [train_img_folder, val_img_folder, test_img_folder] + coco_json_paths = [coco_train_json_path, coco_val_json_path, coco_test_json_path] + annos_folders = [annos_train_folder, annos_val_folder, annos_test_folder] + + def wrapper(scene_id): + image_set = dataset[scene_id] + process_floorplan( + image_set, + scene_id, + start_scene_id, + args, + save_dir, + annos_folder, + use_org_cc5k_classs=True, + vis_fp=scene_id < 100, + wd2line=not args.disable_wd2line, + ) + + def worker_init(dataset_obj): + # Store dataset as global to avoid pickling issues + global dataset + dataset = dataset_obj + + for split_id, split_file in enumerate(split_set): + dataset = FloorplanSVG(args.data_root, split_file, format="txt", original_size=False) + save_dir = save_folders[split_id] + json_path = coco_json_paths[split_id] + print(f"############# {split_file}") + + annos_folder = annos_folders[split_id] + num_processes = 16 + with Pool(num_processes, initializer=worker_init, initargs=(dataset,)) as p: + indices = range(len(dataset)) + list(tqdm(p.imap(wrapper, indices), total=len(dataset))) + + start_scene_id += len(dataset) diff --git a/data_preprocess/cubicasa5k/floorplan_extraction.py b/data_preprocess/cubicasa5k/floorplan_extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..009862ba674b1791b08dc67c951ee86a0ad2a4ec --- /dev/null +++ b/data_preprocess/cubicasa5k/floorplan_extraction.py @@ -0,0 +1,403 @@ +import argparse +import glob +import json +import os +import sys +from multiprocessing import Pool +from pathlib import Path + +import cv2 +import numpy as np +from shapely.geometry import Polygon +from tqdm import tqdm + +sys.path.append(str(Path(__file__).resolve().parent.parent)) +from common_utils import resort_corners +from create_coco_cc5k import create_coco_bounding_box + +from util.plot_utils import plot_semantic_rich_floorplan_opencv + + +def plot_floor(output_coco_polygons, categories_dict, img_w, img_h, save_path, door_window_index=[10, 9]): + gt_sem_rich = [] + for j, (poly, poly_type) in enumerate(output_coco_polygons): + corners = np.array(poly).reshape(-1, 2).astype(np.int32) + # corners_flip_y = corners.copy() + # corners_flip_y[:,1] = 255 - corners_flip_y[:,1] + # corners = corners_flip_y + gt_sem_rich.append([corners, poly_type]) + # plot_semantic_rich_floorplan_nicely(gt_sem_rich, save_path, prec=None, rec=None, + # plot_text=True, is_bw=False, + # door_window_index=door_window_index, + # img_w=img_w, + # img_h=img_h, + # semantics_label_mapping=get_dataset_class_labels(categories_dict), + # ) + plot_semantic_rich_floorplan_opencv( + gt_sem_rich, + save_path, + img_w=img_w, + img_h=img_h, + door_window_index=door_window_index, + semantics_label_mapping=get_dataset_class_labels(categories_dict), + is_bw=False, + ) + + +def prepare_dict(categories_dict): + save_dict = {"images": [], "annotations": [], "categories": categories_dict} + return save_dict + + +def extract_polygons_from_mask(binary_mask, output_mask_path): + """ + Extract polygons from a binary mask where regions with value 1 are polygons + and background regions have value 0. + + Args: + binary_mask (numpy.ndarray): Binary mask with shape (H, W), where 1 represents + the polygon regions and 0 represents the background. + + Returns: + list: A list of polygons, where each polygon is represented as a list of (x, y) coordinates. + """ + # Ensure the mask is binary (0 and 1) + binary_mask = (binary_mask > 0).astype(np.uint8) + + # Find contours in the binary mask + contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + # Extract polygons from contours + polygons = [] + for contour in contours: + # Approximate the contour to reduce the number of points + epsilon = 0.001 * cv2.arcLength(contour, True) # Adjust epsilon for more/less detail + approx_polygon = cv2.approxPolyDP(contour, epsilon, True) + polygons.append(approx_polygon.squeeze().tolist()) # Convert to list of (x, y) points + + # Convert binary_mask to a 3-channel image to draw colored polylines + binary_mask_colored = cv2.cvtColor(binary_mask * 255, cv2.COLOR_GRAY2BGR) + + # Plot polygons on the binary mask with green color + for polygon in polygons: + points = np.array(polygon, dtype=np.int32) + cv2.polylines(binary_mask_colored, [points], isClosed=True, color=(0, 0, 255), thickness=10) + + cv2.imwrite(output_mask_path, binary_mask_colored) + + return polygons + + +def read_polygons_from_json(json_file): + with open(json_file, "r") as f: + data = json.load(f) + category_dict = data["categories"] + polygons = [data["annotations"][i]["segmentation"][0] for i in range(len(data["annotations"]))] + poly_types = [data["annotations"][i]["category_id"] for i in range(len(data["annotations"]))] + source_misc = [data["annotations"][i] for i in range(len(data["annotations"]))] + source_polygons = [(polygons[i], poly_types[i]) for i in range(len(polygons))] + + return source_polygons, source_misc, category_dict + + +def get_dataset_class_labels(category_dict): + return {category_dict[i]["id"]: category_dict[i]["name"] for i in range(len(category_dict))} + + +def check_all_window_door_inside(polygons, door_window_index): + flag = all([poly_type in door_window_index for _, poly_type in polygons]) + return flag + + +def extract_region_and_annotation( + source_image, + source_annot_path, + region_polygons, + start_image_id, + output_image_dir="output", + output_annot_dir="annotations", + output_aux_dir="output_aux", + vis_aux=True, +): + """ + Extract regions of the floorplan from the source image based on polygons + and generate annotations. + + Args: + source_image (numpy.ndarray): The source image (H, W, 3). + polygons (list): List of polygons, where each polygon is a list of (x, y) coordinates. + output_dir (str): Directory to save the extracted regions and annotations. + + Returns: + list: A list of annotations for each extracted region. + """ + door_window_index = [10, 9] + source_polygons, source_misc, categories_dict = read_polygons_from_json(source_annot_path) + source_img_id = os.path.basename(source_annot_path).split(".")[0].zfill(5) + if vis_aux: + gt_sem_rich_path = os.path.join(output_aux_dir, "{}_org_floor.png".format(source_img_id)) + plot_floor( + source_polygons, + categories_dict, + source_image.shape[1], + source_image.shape[0], + gt_sem_rich_path, + door_window_index=door_window_index, + ) + margin = 10 + img_id = start_image_id + + # each region polygon corresponds to an image + for i, polygon in enumerate(region_polygons): + instance_id = 0 + output_coco_polygons = [] + # Create a mask for the current polygon + mask = np.zeros(source_image.shape[:2], dtype=np.uint8) + points = np.array(polygon, dtype=np.int32) + cv2.fillPoly(mask, [points], 255) + + # Crop the ROI to the bounding box of the polygon + x, y, w, h = cv2.boundingRect(points) + # Expand the bounding box by the margin + x_expanded = max(x - margin, 0) + y_expanded = max(y - margin, 0) + w_expanded = min(x + w + margin, source_image.shape[1]) - x_expanded + h_expanded = min(y + h + margin, source_image.shape[0]) - y_expanded + + x, y, w, h = x_expanded, y_expanded, w_expanded, h_expanded + cropped_roi = source_image[y : y + h, x : x + w] + + save_dict = prepare_dict(categories_dict) + + # Create an annotation for the extracted region + img_dict = {} + img_dict["file_name"] = f"{str(img_id).zfill(5)}_{source_img_id}.png" + img_dict["id"] = img_id + img_dict["width"] = w + img_dict["height"] = h + + # Save the cropped ROI + roi_filename = f"{output_image_dir}/{str(img_id).zfill(5)}_{source_img_id}.png" + cv2.imwrite(roi_filename, cropped_roi) + + bounding_box = np.array([x, y, x + w, y + h]) + + # Convert source polygons to NumPy arrays for vectorized operations + source_polygons_np = [np.array(src_poly[0]).reshape(-1, 2) for src_poly in source_polygons] + assert len(source_polygons_np) == len(source_polygons) + + coco_annotation_dict_list = [] + # Iterate through the polygons and filter those inside the bounding box + for j, tmp in enumerate(source_polygons_np): + # Compute the bounding box of the current polygon + poly_bbox = np.hstack([np.min(tmp, axis=0), np.max(tmp, axis=0)]) + + # Check if the polygon is outside the bounding box + if np.any(poly_bbox[:2] < bounding_box[:2]) or np.any(poly_bbox[2:] > bounding_box[2:]): + continue + + # Scale the polygon coordinates relative to the top-left corner of the bounding box + scaled_polygon = tmp - bounding_box[:2] + + coco_seg_poly = [] + poly_sorted = resort_corners(scaled_polygon) + # image = draw_polygon_on_image(image, poly_shapely, "test_poly.jpg") + + for p in poly_sorted: + coco_seg_poly += list(p) + + if len(scaled_polygon) == 2: + area = source_misc[j]["area"] + coco_bb = source_misc[j]["bbox"] + # shift the bounding box + coco_bb[0] -= bounding_box[0] + coco_bb[1] -= bounding_box[1] + else: + poly_shapely = Polygon(scaled_polygon) + area = poly_shapely.area + rectangle_shapely = poly_shapely.envelope + + # Slightly wider bounding box + bb_x, bb_y = rectangle_shapely.exterior.xy + coco_bb = create_coco_bounding_box(bb_x, bb_y, w, h, bound_pad=2) + + coco_annotation_dict = { + "segmentation": [coco_seg_poly], + "area": area, + "iscrowd": 0, + "image_id": img_id, + "bbox": coco_bb, + "category_id": source_polygons[j][1], + "id": instance_id, + } + coco_annotation_dict_list.append(coco_annotation_dict) + output_coco_polygons.append([coco_seg_poly, source_polygons[j][1]]) + + # Remove after obtaining the polygon + # source_polygons.pop(j) + # source_misc.pop(j) + instance_id += 1 + + # skip if just windows and doors are inside + if check_all_window_door_inside(output_coco_polygons, door_window_index): + instance_id -= len(coco_annotation_dict_list) + continue + + save_dict["images"].append(img_dict) + save_dict["annotations"] += coco_annotation_dict_list + + if vis_aux: + gt_sem_rich_path = os.path.join( + output_aux_dir, "{}_{}_floor.png".format(str(img_id).zfill(5), source_img_id) + ) + plot_floor( + output_coco_polygons, categories_dict, w, h, gt_sem_rich_path, door_window_index=door_window_index + ) + + # Save annotations to a JSON file + json_path = f"{output_annot_dir}/{str(img_id).zfill(5)}_{source_img_id}.json" + with open(json_path, "w") as f: + json.dump(save_dict, f) + + img_id += 1 + + start_image_id = img_id + + return start_image_id + + +def config(): + a = argparse.ArgumentParser(description="Generate coco format data for Structured3D") + a.add_argument( + "--data_root", default="Structured3D_panorama", type=str, help="path to raw Structured3D_panorama folder" + ) + a.add_argument("--output", default="coco_cubicasa5k", type=str, help="path to output folder") + + args = a.parse_args() + return args + + +# Example usage +if __name__ == "__main__": + args = config() + + ### prepare + outFolder = args.output + if not os.path.exists(outFolder): + os.mkdir(outFolder) + + annotation_outFolder = os.path.join(outFolder, "annotations_json") + if not os.path.exists(annotation_outFolder): + os.mkdir(annotation_outFolder) + + annos_train_folder = os.path.join(annotation_outFolder, "train") + annos_val_folder = os.path.join(annotation_outFolder, "val") + annos_test_folder = os.path.join(annotation_outFolder, "test") + os.makedirs(annos_train_folder, exist_ok=True) + os.makedirs(annos_val_folder, exist_ok=True) + os.makedirs(annos_test_folder, exist_ok=True) + + train_img_folder = os.path.join(outFolder, "train") + val_img_folder = os.path.join(outFolder, "val") + test_img_folder = os.path.join(outFolder, "test") + + for img_folder in [train_img_folder, val_img_folder, test_img_folder]: + if not os.path.exists(img_folder): + os.mkdir(img_folder) + + ### begin processing + start_image_id = 3500 + save_folders = [train_img_folder, val_img_folder, test_img_folder] + annos_folders = [annos_train_folder, annos_val_folder, annos_test_folder] + splits = ["train", "val", "test"] + + def wrapper(index): + image_path, annot_path, mask_path = packed_input_files[index] + cur_image_id = int(os.path.basename(image_path).split(".")[0]) + binary_mask = cv2.imread(mask_path)[:, :, -1] + source_image = cv2.imread(image_path, cv2.IMREAD_COLOR) + # Extract polygons + region_polygons = extract_polygons_from_mask( + binary_mask, output_mask_path=f"{save_aux_path}/{str(cur_image_id).zfill(5)}_polylines.png" + ) + + return extract_region_and_annotation( + source_image, + annot_path, + region_polygons, + start_image_id + index * 10, + save_path, + save_anno_path, + save_aux_path, + vis_aux=True, + ) + + def worker_init(input_files_object): + # Store dataset as global to avoid pickling issues + global packed_input_files + packed_input_files = input_files_object + + for i, split in enumerate(splits): + image_files = sorted(glob.glob(f"{args.data_root}/{split}/*.png")) + image_id_list = [os.path.basename(image_path).split(".")[0] for image_path in image_files] + anno_files = [f"{args.data_root}/annotations_json/{split}/{id_}.json" for id_ in image_id_list] + mask_files = [f"{args.data_root}/{split}_aux/{id_}_mask.png" for id_ in image_id_list] + save_path = save_folders[i] + save_anno_path = annos_folders[i] + save_aux_path = save_path.rstrip("/") + "_aux" + os.makedirs(save_aux_path, exist_ok=True) + + # for j, (image_path, anno_path, mask_path) in enumerate(zip(image_files, anno_files, mask_files)): + # cur_image_id = int(os.path.basename(image_path).split('.')[0]) + # binary_mask = cv2.imread(mask_path)[:,:,-1] + # source_image = cv2.imread(image_path, cv2.IMREAD_COLOR) + + # # Extract polygons + # polygons = extract_polygons_from_mask(binary_mask, output_mask_path=f'{save_aux_path}/{str(cur_image_id).zfill(5)}_polylines.png') + # # # skip if only one polygon (floorplan) + # # if len(polygons) == 1: + # # print(f"Skipping {image_path} with only one polygon") + # # with open(anno_path, 'r') as f: + # # data = json.load(f) + # # # update image id + # # data['images'][0]['id'] = start_image_id + # # data['images'][0]["file_name"] = f'{str(start_image_id).zfill(5)}_{str(cur_image_id).zfill(5)}.png' + # # for anno in data['annotations']: + # # anno['image_id'] = start_image_id + + # # with open(f"{save_anno_path}/{str(start_image_id).zfill(5)}_{str(cur_image_id).zfill(5)}.json", 'w') as f: + # # json.dump(data, f, indent=2) + # # shutil.copy(image_path, f"{save_path}/{str(start_image_id).zfill(5)}_{str(cur_image_id).zfill(5)}.png") + + # # gt_sem_rich_path = os.path.join(save_aux_path, '{}_{}_floor.png'.format(str(start_image_id).zfill(5), str(cur_image_id).zfill(5))) + # # output_coco_polygons = [(x['segmentation'][0], x['category_id']) for x in data['annotations']] + # # plot_floor(output_coco_polygons, data['categories'], data['images'][0]['width'], data['images'][0]['height'], gt_sem_rich_path, door_window_index=[10, 9]) + + # # start_image_id += 1 + # # continue + + # # # Print the extracted polygons + # # print("Extracted polygons:") + # # for i, polygon in enumerate(polygons): + # # print(f"Polygon {i + 1}: {polygon}") + + # start_image_id = extract_region_and_annotation(source_image, + # anno_path, + # polygons, + # start_image_id, + # output_image_dir=save_path, + # output_annot_dir=save_anno_path, + # output_aux_dir=save_aux_path, + # vis_aux=True) + + packed_input_files = list(zip(image_files, anno_files, mask_files)) + # for j in range(5): + # wrapper(j) + # exit(0) + + num_processes = 16 + with Pool(num_processes, initializer=worker_init, initargs=(packed_input_files,)) as p: + indices = [j for j in range(len(packed_input_files))] + list(tqdm(p.imap(wrapper, indices), total=len(indices))) + + start_image_id += len(packed_input_files) * 10 diff --git a/data_preprocess/cubicasa5k/house.py b/data_preprocess/cubicasa5k/house.py new file mode 100644 index 0000000000000000000000000000000000000000..3d1dc0934b104b2858c5e3a598828513397ace0b --- /dev/null +++ b/data_preprocess/cubicasa5k/house.py @@ -0,0 +1,1131 @@ +import copy +from xml.dom import minidom + +import cv2 +import numpy as np +from skimage.draw import polygon +from svg_utils import ( + PolygonWall, + calc_distance, + get_direction, + get_gaussian2D, + get_icon, + get_icon_number, + get_points, + get_room_number, +) + +all_rooms = { + "Background": 0, # Not in data. The default outside label + "Alcove": 1, + "Attic": 2, + "Ballroom": 3, + "Bar": 4, + "Basement": 5, + "Bath": 6, + "Bedroom": 7, + "Below150cm": 8, + "CarPort": 9, + "Church": 10, + "Closet": 11, + "ConferenceRoom": 12, + "Conservatory": 13, + "Counter": 14, + "Den": 15, + "Dining": 16, + "DraughtLobby": 17, + "DressingRoom": 18, + "EatingArea": 19, + "Elevated": 20, + "Elevator": 21, + "Entry": 22, + "ExerciseRoom": 23, + "Garage": 24, + "Garbage": 25, + "Hall": 26, + "HallWay": 27, + "HotTub": 28, + "Kitchen": 29, + "Library": 30, + "LivingRoom": 31, + "Loft": 32, + "Lounge": 33, + "MediaRoom": 34, + "MeetingRoom": 35, + "Museum": 36, + "Nook": 37, + "Office": 38, + "OpenToBelow": 39, + "Outdoor": 40, + "Pantry": 41, + "Reception": 42, + "RecreationRoom": 43, + "RetailSpace": 44, + "Room": 45, + "Sanctuary": 46, + "Sauna": 47, + "ServiceRoom": 48, + "ServingArea": 49, + "Skylights": 50, + "Stable": 51, + "Stage": 52, + "StairWell": 53, + "Storage": 54, + "SunRoom": 55, + "SwimmingPool": 56, + "TechnicalRoom": 57, + "Theatre": 58, + "Undefined": 59, + "UserDefined": 60, + "Utility": 61, + "Wall": 62, + "Railing": 63, + "Stairs": 64, +} + +rooms_selected = { + "Alcove": 11, + "Attic": 11, + "Ballroom": 11, + "Bar": 11, + "Basement": 11, + "Bath": 6, + "Bedroom": 5, + "CarPort": 10, + "Church": 11, + "Closet": 9, + "ConferenceRoom": 11, + "Conservatory": 11, + "Counter": 11, + "Den": 11, + "Dining": 4, + "DraughtLobby": 7, + "DressingRoom": 9, + "EatingArea": 4, + "Elevated": 11, + "Elevator": 11, + "Entry": 7, + "ExerciseRoom": 11, + "Garage": 10, + "Garbage": 11, + "Hall": 11, + "HallWay": 7, + "HotTub": 11, + "Kitchen": 3, + "Library": 11, + "LivingRoom": 4, + "Loft": 11, + "Lounge": 4, + "MediaRoom": 11, + "MeetingRoom": 11, + "Museum": 11, + "Nook": 11, + "Office": 11, + "OpenToBelow": 11, + "Outdoor": 1, + "Pantry": 11, + "Reception": 11, + "RecreationRoom": 11, + "RetailSpace": 11, + "Room": 11, + "Sanctuary": 11, + "Sauna": 6, + "ServiceRoom": 11, + "ServingArea": 11, + "Skylights": 11, + "Stable": 11, + "Stage": 11, + "StairWell": 11, + "Storage": 9, + "SunRoom": 11, + "SwimmingPool": 11, + "TechnicalRoom": 11, + "Theatre": 11, + "Undefined": 11, + "UserDefined": 11, + "Utility": 11, + "Background": 0, # Not in data. The default outside label + "Wall": 2, + "Railing": 8, +} + +room_name_map = { + "Alcove": "Room", + "Attic": "Room", + "Ballroom": "Room", + "Bar": "Room", + "Basement": "Room", + "Bath": "Bath", + "Bedroom": "Bedroom", + "Below150cm": "Room", + "CarPort": "Garage", + "Church": "Room", + "Closet": "Storage", + "ConferenceRoom": "Room", + "Conservatory": "Room", + "Counter": "Room", + "Den": "Room", + "Dining": "Dining", + "DraughtLobby": "Entry", + "DressingRoom": "Storage", + "EatingArea": "Dining", + "Elevated": "Room", + "Elevator": "Room", + "Entry": "Entry", + "ExerciseRoom": "Room", + "Garage": "Garage", + "Garbage": "Room", + "Hall": "Room", + "HallWay": "Entry", + "HotTub": "Room", + "Kitchen": "Kitchen", + "Library": "Room", + "LivingRoom": "LivingRoom", + "Loft": "Room", + "Lounge": "LivingRoom", + "MediaRoom": "Room", + "MeetingRoom": "Room", + "Museum": "Room", + "Nook": "Room", + "Office": "Room", + "OpenToBelow": "Room", + "Outdoor": "Outdoor", + "Pantry": "Room", + "Reception": "Room", + "RecreationRoom": "Room", + "RetailSpace": "Room", + "Room": "Room", + "Sanctuary": "Room", + "Sauna": "Bath", + "ServiceRoom": "Room", + "ServingArea": "Room", + "Skylights": "Room", + "Stable": "Room", + "Stage": "Room", + "StairWell": "Room", + "Storage": "Storage", + "SunRoom": "Room", + "SwimmingPool": "Room", + "TechnicalRoom": "Room", + "Theatre": "Room", + "Undefined": "Room", + "UserDefined": "Room", + "Utility": "Room", + "Wall": "Wall", + "Railing": "Railing", + "Background": "Background", +} # Not in data. The default outside label + +all_icons = { + "Empty": 0, + "Window": 1, + "Door": 2, + "BaseCabinet": 3, + "BaseCabinetRound": 4, + "BaseCabinetTriangle": 5, + "Bathtub": 6, + "BathtubRound": 7, + "Chimney": 8, + "Closet": 9, + "ClosetRound": 10, + "ClosetTriangle": 11, + "CoatCloset": 12, + "CoatRack": 13, + "CornerSink": 14, + "CounterTop": 15, + "DoubleSink": 16, + "DoubleSinkRight": 17, + "ElectricalAppliance": 18, + "Fireplace": 19, + "FireplaceCorner": 20, + "FireplaceRound": 21, + "GasStove": 22, + "Housing": 23, + "Jacuzzi": 24, + "PlaceForFireplace": 25, + "PlaceForFireplaceCorner": 26, + "PlaceForFireplaceRound": 27, + "RoundSink": 28, + "SaunaBenchHigh": 29, + "SaunaBenchLow": 30, + "SaunaBenchMid": 31, + "Shower": 32, + "ShowerCab": 33, + "ShowerScreen": 34, + "ShowerScreenRoundLeft": 35, + "ShowerScreenRoundRight": 36, + "SideSink": 37, + "Sink": 38, + "Toilet": 39, + "Urinal": 40, + "WallCabinet": 41, + "WaterTap": 42, + "WoodStove": 43, + "Misc": 44, + "SaunaBench": 45, + "SaunaStove": 46, + "WashingMachine": 47, + "IntegratedStove": 48, + "Dishwasher": 49, + "GeneralAppliance": 50, + "ShowerPlatform": 51, +} + +icons_selected = { + "Window": 1, + "Door": 2, + "Closet": 3, + "ClosetRound": 3, + "ClosetTriangle": 3, + "CoatCloset": 3, + "CoatRack": 3, + "CounterTop": 3, + "Housing": 3, + "ElectricalAppliance": 4, + "WoodStove": 4, + "GasStove": 4, + "Toilet": 5, + "Urinal": 5, + "SideSink": 6, + "Sink": 6, + "RoundSink": 6, + "CornerSink": 6, + "DoubleSink": 6, + "DoubleSinkRight": 6, + "WaterTap": 6, + "SaunaBenchHigh": 7, + "SaunaBenchLow": 7, + "SaunaBenchMid": 7, + "SaunaBench": 7, + "Fireplace": 8, + "FireplaceCorner": 8, + "FireplaceRound": 8, + "PlaceForFireplace": 8, + "PlaceForFireplaceCorner": 8, + "PlaceForFireplaceRound": 8, + "Bathtub": 9, + "BathtubRound": 9, + "Chimney": 10, + "Misc": None, + "BaseCabinetRound": None, + "BaseCabinetTriangle": None, + "BaseCabinet": None, + "WallCabinet": None, + "Shower": None, + "ShowerCab": None, + "ShowerPlatform": None, + "ShowerScreen": None, + "ShowerScreenRoundRight": None, + "ShowerScreenRoundLeft": None, + "Jacuzzi": None, +} + +icon_name_map = { + "Window": "Window", + "Door": "Door", + "Closet": "Closet", + "ClosetRound": "Closet", + "ClosetTriangle": "Closet", + "CoatCloset": "Closet", + "CoatRack": "Closet", + "CounterTop": "Closet", + "Housing": "Closet", + "ElectricalAppliance": "ElectricalAppliance", + "WoodStove": "ElectricalAppliance", + "GasStove": "ElectricalAppliance", + "SaunaStove": "ElectricalAppliance", + "Toilet": "Toilet", + "Urinal": "Toilet", + "SideSink": "Sink", + "Sink": "Sink", + "RoundSink": "Sink", + "CornerSink": "Sink", + "DoubleSink": "Sink", + "DoubleSinkRight": "Sink", + "WaterTap": "Sink", + "SaunaBenchHigh": "SaunaBench", + "SaunaBenchLow": "SaunaBench", + "SaunaBenchMid": "SaunaBench", + "SaunaBench": "SaunaBench", + "Fireplace": "Fireplace", + "FireplaceCorner": "Fireplace", + "FireplaceRound": "Fireplace", + "PlaceForFireplace": "Fireplace", + "PlaceForFireplaceCorner": "Fireplace", + "PlaceForFireplaceRound": "Fireplace", + "Bathtub": "Bathtub", + "BathtubRound": "Bathtub", + "Chimney": "Chimney", + "Misc": None, + "BaseCabinetRound": None, + "BaseCabinetTriangle": None, + "BaseCabinet": None, + "WallCabinet": None, + "Shower": "None", + "ShowerCab": "None", + "ShowerPlatform": "None", + "ShowerScreen": None, + "ShowerScreenRoundRight": None, + "ShowerScreenRoundLeft": None, + "Jacuzzi": None, + "WashingMachine": None, + "IntegratedStove": "ElectricalAppliance", + "Dishwasher": "ElectricalAppliance", + "GeneralAppliance": "ElectricalAppliance", +} + + +def complete_polygons(polygons, polygon_types): + new_polygons = [] + new_types = [] + for poly, poly_type in zip(polygons, polygon_types): + if len(poly) < 3: + print(f"Class {poly_type} has less than 3 points. Skipped!") + continue + poly_array = np.array(poly) + t = copy.copy(poly) + # append the beginning point + if len(poly_array) > 2 and (poly_array[0] != poly_array[-1]).any(): + t.append(poly[0]) + new_polygons.append(t) + new_types.append(poly_type) + + return new_polygons, new_types + + +class House: + def __init__(self, path, height, width, icon_list=icons_selected, room_list=rooms_selected): + self.height = height + self.width = width + shape = height, width + svg = minidom.parse(path) + self.walls = np.empty((height, width), dtype=np.uint8) + self.walls.fill(0) + self.wall_ids = np.empty((height, width), dtype=np.uint8) + self.wall_ids.fill(0) + self.icons = np.zeros((height, width), dtype=np.uint8) + # junction_id = 0 + wall_id = 1 + self.wall_ends = [] + self.wall_objs = [] + self.icon_types = [] + self.room_types = [] + self.icon_corners = {"upper_left": [], "upper_right": [], "lower_left": [], "lower_right": []} + self.opening_corners = {"left": [], "right": [], "up": [], "down": []} + self.representation = {"doors": [], "icons": [], "labels": [], "walls": []} + + self.icon_areas = [] + self.wall_coords = [] + self.icon_coords = [] + + for e in svg.getElementsByTagName("g"): + try: + if e.getAttribute("id") == "Wall": + wall = PolygonWall(e, wall_id, shape) + wall.rr, wall.cc = self._clip_outside(wall.rr, wall.cc) + self.wall_objs.append(wall) + self.walls[wall.rr, wall.cc] = room_list["Wall"] + self.wall_ids[wall.rr, wall.cc] = wall_id + self.wall_ends.append(wall.end_points) + + Y, X = self._clip_outside(wall.Y, wall.X) + self.wall_coords.append([(x, y) for x, y in zip(X, Y)]) + self.room_types.append(room_list["Wall"]) + + wall_id += 1 + + if e.getAttribute("id") == "Railing": + wall = PolygonWall(e, wall_id, shape) + wall.rr, wall.cc = self._clip_outside(wall.rr, wall.cc) + self.wall_objs.append(wall) + self.walls[wall.rr, wall.cc] = room_list["Railing"] + self.wall_ids[wall.rr, wall.cc] = wall_id + self.wall_ends.append(wall.end_points) + + Y, X = self._clip_outside(wall.Y, wall.X) + self.wall_coords.append([(x, y) for x, y in zip(X, Y)]) + self.room_types.append(room_list["Railing"]) + + wall_id += 1 + + except ValueError as k: + if str(k) != "small wall": + raise k + continue + + if e.getAttribute("id") == "Window": + X, Y = get_points(e) + rr, cc = polygon(X, Y) + cc, rr = self._clip_outside(cc, rr) + direction = get_direction(X, Y) + locs = np.column_stack((X, Y)) + if direction == "H": + left_index = np.argmin(locs[:, 0]) + left1 = locs[left_index] + locs = np.delete(locs, left_index, axis=0) + left_index = np.argmin(locs[:, 0]) + left2 = locs[left_index] + right = np.delete(locs, left_index, axis=0) + left = np.array([left1, left2]) + + point_left = left.mean(axis=0) + point_right = right.mean(axis=0) + self.opening_corners["left"].append(point_left) + self.opening_corners["right"].append(point_right) + + door_rep = [[list(point_left), list(point_right)], ["door", 1, 1]] + self.representation["doors"].append(door_rep) + else: + up_index = np.argmin(locs[:, 1]) + up1 = locs[up_index] + locs = np.delete(locs, up_index, axis=0) + up_index = np.argmin(locs[:, 1]) + up2 = locs[up_index] + down = np.delete(locs, up_index, axis=0) + up = np.array([up1, up2]) + + point_up = up.mean(axis=0) + point_down = down.mean(axis=0) + self.opening_corners["up"].append(point_up) + self.opening_corners["down"].append(point_down) + + door_rep = [[list(point_up), list(point_down)], ["door", 1, 1]] + self.representation["doors"].append(door_rep) + + self.icons[cc, rr] = 1 + self.icon_types.append(1) + + Y, X = self._clip_outside(Y, X) + self.icon_coords.append([(x, y) for x, y in zip(X, Y)]) + + if e.getAttribute("id") == "Door": + # How to reperesent empty door space + X, Y = get_points(e) + rr, cc = polygon(X, Y) + cc, rr = self._clip_outside(cc, rr) + direction = get_direction(X, Y) + locs = np.column_stack((X, Y)) + if direction == "H": + left_index = np.argmin(locs[:, 0]) + left1 = locs[left_index] + locs = np.delete(locs, left_index, axis=0) + left_index = np.argmin(locs[:, 0]) + left2 = locs[left_index] + right = np.delete(locs, left_index, axis=0) + left = np.array([left1, left2]) + + point_left = left.mean(axis=0) + point_right = right.mean(axis=0) + self.opening_corners["left"].append(left.mean(axis=0)) + self.opening_corners["right"].append(right.mean(axis=0)) + + door_rep = [[list(point_left), list(point_right)], ["door", 1, 1]] + self.representation["doors"].append(door_rep) + else: + up_index = np.argmin(locs[:, 1]) + up1 = locs[up_index] + locs = np.delete(locs, up_index, axis=0) + up_index = np.argmin(locs[:, 1]) + up2 = locs[up_index] + down = np.delete(locs, up_index, axis=0) + up = np.array([up1, up2]) + + point_up = up.mean(axis=0) + point_down = down.mean(axis=0) + self.opening_corners["up"].append(up.mean(axis=0)) + self.opening_corners["down"].append(down.mean(axis=0)) + + door_rep = [[list(point_up), list(point_down)], ["door", 1, 1]] + self.representation["doors"].append(door_rep) + + self.icons[cc, rr] = 2 + self.icon_types.append(2) + + Y, X = self._clip_outside(Y, X) + self.icon_coords.append([(x, y) for x, y in zip(X, Y)]) + + if "FixedFurniture " in e.getAttribute("class"): + num = get_icon_number(e, icon_list) + if num is not None: + rr, cc, X, Y = get_icon(e) + # only four corner icons + if len(X) == 4: + locs = np.column_stack((X, Y)) + up_left_index = locs.sum(axis=1).argmin() + self.icon_corners["upper_left"].append(locs[up_left_index]) + up_left = list(locs[up_left_index]) + locs = np.delete(locs, up_left_index, axis=0) + down_right_index = locs.sum(axis=1).argmax() + self.icon_corners["lower_right"].append(locs[down_right_index]) + down_right = list(locs[down_right_index]) + locs = np.delete(locs, down_right_index, axis=0) + up_right_index = locs[:, 1].argmin() + self.icon_corners["upper_right"].append(locs[up_right_index]) + locs = np.delete(locs, up_right_index, axis=0) + self.icon_corners["lower_left"].append(locs[0]) + + icon_name = e.getAttribute("class").replace("FixedFurniture ", "").split(" ")[0] + icon_name = icon_name_map[icon_name] + + icon_rep = [[up_left, down_right], [icon_name, 1, 1]] + self.representation["icons"].append(icon_rep) + + rr, cc = self._clip_outside(rr, cc) + self.icon_areas.append(len(rr)) + self.icons[rr, cc] = num + self.icon_types.append(num) + + Y, X = self._clip_outside(Y, X) + self.icon_coords.append([(x, y) for x, y in zip(X, Y)]) + + if "Space " in e.getAttribute("class"): + num = get_room_number(e, room_list) + # rr, cc = get_polygon(e) + X, Y = get_points(e) + rr, cc = polygon(Y, X) + if len(rr) != 0: + rr, cc = self._clip_outside(rr, cc) + if len(rr) != 0 and len(cc) != 0: + self.walls[rr, cc] = num + self.room_types.append(num) + + Y, X = self._clip_outside(Y, X) + self.wall_coords.append([(x, y) for x, y in zip(X, Y)]) + + rr_mean = int(round(np.mean(rr))) + cc_mean = int(round(np.mean(cc))) + center_box = [[rr_mean - 10, cc_mean - 10], [rr_mean + 10, cc_mean + 10]] + room_name = e.getAttribute("class").replace("Space ", "").split(" ")[0] + room_name = room_name_map[room_name] + self.representation["labels"].append([center_box, [room_name, 1, 1]]) + + # if "Stairs" in e.getAttribute("class"): + # for c in e.childNodes: + # if c.getAttribute("class") in ["Flight", "Winding"]: + # num = room_list["Stairs"] + # rr, cc = get_polygon(c) + # if len(rr) != 0: + # rr, cc = self._clip_outside(rr, cc) + # if len(rr) != 0 and len(cc) != 0: + # self.walls[rr, cc] = num + # self.room_types.append(num) + + # rr_mean = int(round(np.mean(rr))) + # cc_mean = int(round(np.mean(cc))) + # center_box = [[rr_mean-10, cc_mean-10], [rr_mean+10, cc_mean+10]] + # room_name = "Stairs" + # # room_name = room_name_map[room_name] + # self.representation['labels'].append([center_box, [room_name, 1, 1]]) + + self.avg_wall_width = self.get_avg_wall_width() + + self.new_walls = self.connect_walls(self.wall_objs) + + for w in self.new_walls: + w.change_end_points() + + for w in self.pillar_walls: + self.new_walls.append(w) + + self.points = self.lines_to_points(self.width, self.height, self.new_walls, self.avg_wall_width) + self.points = self.merge_joints(self.points, self.avg_wall_width) + + # walls to representation + for w in self.new_walls: + end_points = w.end_points.round().astype("int").tolist() + if w.name == "Wall": + self.representation["walls"].append([end_points, ["wall", 1, 1]]) + else: + self.representation["walls"].append([end_points, ["wall", 2, 1]]) + + # append begining point at last pos + print("Complete room coords") + self.wall_coords, self.room_types = complete_polygons(self.wall_coords, self.room_types) + print("Complete icon coords") + self.icon_coords, self.icon_types = complete_polygons(self.icon_coords, self.icon_types) + + def get_coords_and_labels(self): + assert len(self.wall_coords) == len(self.room_types) + assert len(self.icon_coords) == len(self.icon_types) + return self.wall_coords, self.room_types, self.icon_coords, self.icon_types + + def get_tensor(self): + heatmaps = self.get_heatmaps() + wall_t = np.expand_dims(self.walls, axis=0) + icon_t = np.expand_dims(self.icons, axis=0) + tensor = np.concatenate((heatmaps, wall_t, icon_t), axis=0) + + return tensor + + def get_segmentation_tensor(self): + wall_t = np.expand_dims(self.walls, axis=0) + icon_t = np.expand_dims(self.icons, axis=0) + tensor = np.concatenate((wall_t, icon_t), axis=0) + + return tensor + + def get_heatmap_dict(self): + # init dict + heatmaps = {} + for i in range(21): + heatmaps[i] = [] + + for p in self.points: + cord, _, p_type = p + x = int(np.round(cord[0])) + y = int(np.round(cord[1])) + channel = self.get_number(p_type) + if y < self.height and x < self.width: + heatmaps[channel - 1] = heatmaps[channel - 1] + [(x, y)] + + channel = 13 + for i in self.opening_corners["left"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + for i in self.opening_corners["right"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + for i in self.opening_corners["up"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + for i in self.opening_corners["down"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + + for i in self.icon_corners["upper_left"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + for i in self.icon_corners["upper_right"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + for i in self.icon_corners["lower_left"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + channel += 1 + for i in self.icon_corners["lower_right"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel] = heatmaps[channel] + [(x, y)] + + return heatmaps + + def get_heatmaps(self): + heatmaps = np.zeros((21, self.height, self.width)) + for p in self.points: + cord, _, p_type = p + x = int(np.round(cord[0])) + y = int(np.round(cord[1])) + channel = self.get_number(p_type) + if y < self.height and x < self.width: + heatmaps[channel - 1, y, x] = 1 + + channel = 13 + for i in self.opening_corners["left"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + for i in self.opening_corners["right"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + for i in self.opening_corners["up"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + for i in self.opening_corners["down"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + + for i in self.icon_corners["upper_left"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + for i in self.icon_corners["upper_right"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + for i in self.icon_corners["lower_left"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + channel += 1 + for i in self.icon_corners["lower_right"]: + y = int(i[1]) + x = int(i[0]) + if y < self.height and x < self.width: + heatmaps[channel, y, x] = 1 + + kernel = get_gaussian2D(13) + for i, h in enumerate(heatmaps): + heatmaps[i] = cv2.filter2D(h, -1, kernel) + + return heatmaps + + def _clip_outside(self, rr, cc): + s = np.column_stack((rr, cc)) + s = s[s[:, 0] < self.height] + s = s[s[:, 1] < self.width] + + return s[:, 0], s[:, 1] + + def lines_to_points(self, width, height, walls, lineWidth): + lines = [h.end_points for h in walls] + + points = [] + usedLinePointMask = [] + + for lineIndex, line in enumerate(lines): + usedLinePointMask.append([False, False]) + + for lineIndex_1, wall_1 in enumerate(walls): + line_1 = wall_1.end_points + + lineDim_1 = self.get_lineDim(line_1, 1) + if lineDim_1 <= -1: + # If wall is diagonal we skip + continue + + fixedValue_1 = (line_1[0][1 - lineDim_1] + line_1[1][1 - lineDim_1]) / 2 + for lineIndex_2, wall_2 in enumerate(walls): + line_2 = wall_2.end_points + + if lineIndex_2 <= lineIndex_1: + continue + + lineDim_2 = self.get_lineDim(line_2, 1) + if lineDim_2 + lineDim_1 != 1: + # if walls have the same direction we skip + continue + + fixedValue_2 = (line_2[0][1 - lineDim_2] + line_2[1][1 - lineDim_2]) / 2 + lineWidth = max(wall_1.max_width, wall_2.max_width) + nearestPair, minDistance = self.findNearestJunctionPair(line_1, line_2, lineWidth) + + if minDistance <= lineWidth: + pointIndex_1 = nearestPair[0] + pointIndex_2 = nearestPair[1] + if pointIndex_1 > -1 and pointIndex_2 > -1: + point = [None, None] + point[lineDim_1] = fixedValue_2 + point[lineDim_2] = fixedValue_1 + side = [None, None] + side[lineDim_1] = line_1[1 - pointIndex_1][lineDim_1] - fixedValue_2 + side[lineDim_2] = line_2[1 - pointIndex_2][lineDim_2] - fixedValue_1 + + if side[0] < 0 and side[1] < 0: + points.append([point, point, ["point", 2, 1]]) + elif side[0] > 0 and side[1] < 0: + points.append([point, point, ["point", 2, 2]]) + elif side[0] > 0 and side[1] > 0: + points.append([point, point, ["point", 2, 3]]) + elif side[0] < 0 and side[1] > 0: + points.append([point, point, ["point", 2, 4]]) + + usedLinePointMask[lineIndex_1][pointIndex_1] = True + usedLinePointMask[lineIndex_2][pointIndex_2] = True + elif (pointIndex_1 > -1 and pointIndex_2 == -1) or (pointIndex_1 == -1 and pointIndex_2 > -1): + if pointIndex_1 > -1: + lineDim = lineDim_1 + pointIndex = pointIndex_1 + fixedValue = fixedValue_2 + pointValue = line_1[pointIndex_1][1 - lineDim_1] + usedLinePointMask[lineIndex_1][pointIndex_1] = True + else: + lineDim = lineDim_2 + pointIndex = pointIndex_2 + fixedValue = fixedValue_1 + pointValue = line_2[pointIndex_2][1 - lineDim_2] + usedLinePointMask[lineIndex_2][pointIndex_2] = True + + point = [None, None] + point[lineDim] = fixedValue + point[1 - lineDim] = pointValue + + if pointIndex == 0: + if lineDim == 0: + points.append([point, point, ["point", 3, 4]]) + else: + points.append([point, point, ["point", 3, 1]]) + else: + if lineDim == 0: + points.append([point, point, ["point", 3, 2]]) + else: + points.append([point, point, ["point", 3, 3]]) + + elif ( + line_1[0][lineDim_1] < fixedValue_2 + and line_1[1][lineDim_1] > fixedValue_2 + and line_2[0][lineDim_2] < fixedValue_1 + and line_2[1][lineDim_2] > fixedValue_1 + ): + point = [None, None] + point[lineDim_1] = fixedValue_2 + point[lineDim_2] = fixedValue_1 + points.append([point, point, ["point", 4, 1]]) + + for lineIndex, pointMask in enumerate(usedLinePointMask): + lineDim = self.get_lineDim(lines[lineIndex], 1) + for pointIndex in range(2): + if pointMask[pointIndex] is True: + continue + point = [lines[lineIndex][pointIndex][0], lines[lineIndex][pointIndex][1]] + if pointIndex == 0: + if lineDim == 0: + points.append([point, point, ["point", 1, 4]]) + elif lineDim == 1: + points.append([point, point, ["point", 1, 1]]) + else: + if lineDim == 0: + points.append([point, point, ["point", 1, 2]]) + elif lineDim == 1: + points.append([point, point, ["point", 1, 3]]) + + return points + + def _pointId2index(self, g, t): + g_ = g - 1 + t_ = t - 1 + k = g_ * 4 + t_ + return k + + def _index2pointId(self, k): + g = k // 4 + 1 + t = k % 4 + 1 + return [g, t] + + def _are_close(self, p1, p2, width): + return calc_distance(p1, p2) < width + + def merge_joints(self, points, wall_width): + lookuptable = {} + lookuptable[0] = {0: 0, 1: 7, 2: None, 3: 6, 4: 9, 5: 11, 6: 6, 7: 7, 8: 8, 9: 9, 10: 12, 11: 11, 12: 12} + lookuptable[1] = {0: 7, 1: 1, 2: 4, 3: None, 4: 4, 5: 10, 6: 8, 7: 7, 8: 8, 9: 9, 10: 10, 11: 12, 12: 12} + lookuptable[2] = {0: None, 1: 4, 2: 2, 3: 5, 4: 4, 5: 5, 6: 11, 7: 9, 8: 12, 9: 9, 10: 10, 11: 11, 12: 12} + lookuptable[3] = {0: 6, 1: None, 2: 5, 3: 3, 4: 10, 5: 5, 6: 6, 7: 8, 8: 8, 9: 12, 10: 10, 11: 11, 12: 12} + lookuptable[4] = {0: 9, 1: 4, 2: 4, 3: 10, 4: 4, 5: 10, 6: 12, 7: 9, 8: 12, 9: 9, 10: 10, 11: 12, 12: 12} + lookuptable[5] = {0: 11, 1: 10, 2: 5, 3: 5, 4: 10, 5: 5, 6: 11, 7: 12, 8: 12, 9: 12, 10: 10, 11: 11, 12: 12} + lookuptable[6] = {0: 6, 1: 8, 2: 11, 3: 6, 4: 12, 5: 11, 6: 6, 7: 8, 8: 8, 9: 12, 10: 12, 11: 11, 12: 12} + lookuptable[7] = {0: 7, 1: 7, 2: 9, 3: 8, 4: 9, 5: 12, 6: 8, 7: 7, 8: 8, 9: 9, 10: 12, 11: 12, 12: 12} + lookuptable[8] = {0: 8, 1: 8, 2: 12, 3: 8, 4: 12, 5: 12, 6: 8, 7: 8, 8: 8, 9: 12, 10: 12, 11: 12, 12: 12} + lookuptable[9] = {0: 9, 1: 9, 2: 9, 3: 12, 4: 9, 5: 12, 6: 12, 7: 9, 8: 12, 9: 9, 10: 12, 11: 12, 12: 12} + lookuptable[10] = { + 0: 12, + 1: 10, + 2: 10, + 3: 10, + 4: 10, + 5: 10, + 6: 12, + 7: 12, + 8: 12, + 9: 12, + 10: 10, + 11: 12, + 12: 12, + } + lookuptable[11] = { + 0: 11, + 1: 12, + 2: 11, + 3: 11, + 4: 12, + 5: 11, + 6: 11, + 7: 12, + 8: 12, + 9: 12, + 10: 12, + 11: 11, + 12: 12, + } + lookuptable[12] = { + 0: 12, + 1: 12, + 2: 12, + 3: 12, + 4: 12, + 5: 12, + 6: 12, + 7: 12, + 8: 12, + 9: 12, + 10: 12, + 11: 12, + 12: 12, + } + + newPoints = [] + merged = [False] * len(points) + for i, point1 in enumerate(points): + if merged[i] is False: + pool = [point1] + for j, point2 in enumerate(points): + if j != i and merged[j] is False and self._are_close(point1[0], point2[0], wall_width): + merged[j] = True + pool.append(point2) + + if len(pool) == 1: + newPoints.append(point1) + merged[i] = True + else: + p_ = pool[0] + for point_id in range(1, len(pool)): + merge_to_p = pool[point_id] + + k_ = self._pointId2index(p_[2][1], p_[2][2]) + k_merge_to_p = self._pointId2index(merge_to_p[2][1], merge_to_p[2][2]) + + knew = lookuptable[k_][k_merge_to_p] + if knew is None: + continue + + typenew = self._index2pointId(knew) + p_ = [p_[0], p_[1], ["point", typenew[0], typenew[1]]] + + newPoints.append(p_) + + return newPoints + + def get_avg_wall_width(self): + res = 0 + for i, w in enumerate(self.wall_objs): + res += w.max_width + res = res / float(i) + + return res + + def connect_walls(self, walls): + new_walls = [] + num_walls = len(walls) + remaining_walls = list(range(1, num_walls + 1)) + + # getting pillars + remaining_pillar_ids = [] + for p_id in range(1, num_walls + 1): + p_wall = self.find_wall_by_id(p_id, walls) + if p_wall.wall_is_pillar(self.avg_wall_width): + for wall_id in range(1, num_walls + 1): + wall = self.find_wall_by_id(wall_id, walls) + if p_wall.merge_possible(wall): + break + else: + remaining_walls.pop(remaining_walls.index(p_wall.id)) + remaining_pillar_ids.append(p_wall.id) + + while len(remaining_walls) > 0: + new_wall_id = remaining_walls.pop(0) + new_wall = self.find_wall_by_id(new_wall_id, walls) + + found = True + while found: + found = False + for merge_wall_id in remaining_walls: + merged = self.find_wall_by_id(merge_wall_id, walls) + temp_wall = new_wall.merge_walls(merged) + + if temp_wall is not None: + remaining_walls.pop(remaining_walls.index(merged.id)) + new_wall = temp_wall + found = True + + new_walls.append(new_wall) + + # connect pillars to walls + new_wall_id = num_walls + 1 + self.pillar_walls = [] + for id in remaining_pillar_ids: + w = self.find_wall_by_id(id, walls) + pws = w.split_pillar_wall(new_wall_id, self.avg_wall_width) + new_wall_id += 4 + for pw in pws: + self.pillar_walls.append(pw) + + return new_walls + + def get_number(self, x): + return (x[1] - 1) * 4 + x[2] + + def get_lineDim(self, line, lineWidth): + lineWidth = lineWidth or 1 + if abs(line[0][0] - line[1][0]) > abs(line[0][1] - line[1][1]) and abs(line[0][1] - line[1][1]) <= lineWidth: + return 0 + elif abs(line[0][1] - line[1][1]) > abs(line[0][0] - line[1][0]) and abs(line[0][0] - line[1][0]) <= lineWidth: + return 1 + else: + return -1 + + def findNearestJunctionPair(self, line_1, line_2, gap): + + minDistance = None + for index_1 in range(0, 2): + for index_2 in range(0, 2): + distance = calc_distance(line_1[index_1], line_2[index_2]) + if minDistance is None or distance < minDistance: + nearestPair = [index_1, index_2] + minDistance = distance + + if minDistance > gap: + lineDim_1 = self.get_lineDim(line_1, 1) + lineDim_2 = self.get_lineDim(line_2, 1) + + if lineDim_1 + lineDim_2 == 1: + fixedValue_1 = (line_1[0][1 - lineDim_1] + line_1[1][1 - lineDim_1]) / 2 + fixedValue_2 = (line_2[0][1 - lineDim_2] + line_2[1][1 - lineDim_2]) / 2 + + if line_2[0][lineDim_2] < fixedValue_1 and line_2[1][lineDim_2] > fixedValue_1: + for index in range(2): + distance = abs(line_1[index][lineDim_1] - fixedValue_2) + if distance < minDistance: + nearestPair = [index, -1] + minDistance = distance + + if line_1[0][lineDim_1] < fixedValue_2 and line_1[1][lineDim_1] > fixedValue_2: + for index in range(2): + distance = abs(line_2[index][lineDim_2] - fixedValue_1) + if distance < minDistance: + nearestPair = [-1, index] + minDistance = distance + + return nearestPair, minDistance + + def find_wall_by_id(self, id, walls): + for wall in walls: + if wall.id == id: + return wall + + return None diff --git a/data_preprocess/cubicasa5k/loaders.py b/data_preprocess/cubicasa5k/loaders.py new file mode 100644 index 0000000000000000000000000000000000000000..dc2d304f51581bba84a81a970aaa5d2dc74eaade --- /dev/null +++ b/data_preprocess/cubicasa5k/loaders.py @@ -0,0 +1,158 @@ +# import lmdb +import pickle + +import cv2 +import numpy as np +import torch +from house import House +from numpy import genfromtxt +from torch.utils.data import Dataset + +ROOM_NAMES = { + 0: "Background", + 1: "Outdoor", + 2: "Wall", + 3: "Kitchen", + 4: "Living Room", + 5: "Bed Room", + 6: "Bath", + 7: "Entry", + 8: "Railing", + 9: "Storage", + 10: "Garage", + 11: "Undefined", +} + +ICON_NAMES = { + 0: "No Icon", + 1: "Window", + 2: "Door", + 3: "Closet", + 4: "Electrical Applience", + 5: "Toilet", + 6: "Sink", + 7: "Sauna Bench", + 8: "Fire Place", + 9: "Bathtub", + 10: "Chimney", +} + + +class FloorplanSVG(Dataset): + def __init__( + self, + data_folder, + data_file, + is_transform=True, + augmentations=None, + img_norm=True, + format="txt", + original_size=False, + lmdb_folder="cubi_lmdb/", + ): + self.img_norm = img_norm + self.is_transform = is_transform + self.augmentations = augmentations + self.get_data = None + self.original_size = original_size + self.image_file_name = "/F1_scaled.png" + self.org_image_file_name = "/F1_original.png" + self.svg_file_name = "/model.svg" + + if format == "txt": + self.get_data = self.get_txt + # if format == 'lmdb': + # self.lmdb = lmdb.open(data_folder+lmdb_folder, readonly=True, + # max_readers=8, lock=False, + # readahead=True, meminit=False) + # self.get_data = self.get_lmdb + # self.is_transform = False + + self.data_folder = data_folder + # Load txt file to list + self.folders = genfromtxt(data_folder + data_file, dtype="str") + + def __len__(self): + """__len__""" + return len(self.folders) + + def __getitem__(self, index): + sample = self.get_data(index) + + if self.augmentations is not None: + sample = self.augmentations(sample) + + if self.is_transform: + sample = self.transform(sample) + + return sample + + def get_txt(self, index): + fplan = cv2.imread(self.data_folder + self.folders[index] + self.image_file_name) + fplan = cv2.cvtColor(fplan, cv2.COLOR_BGR2RGB) # correct color channels + height, width, nchannel = fplan.shape + fplan = np.moveaxis(fplan, -1, 0) + + # Getting labels for segmentation and heatmaps + house = House(self.data_folder + self.folders[index] + self.svg_file_name, height, width) + # Combining them to one numpy tensor + label = torch.tensor(house.get_segmentation_tensor().astype(np.float32)) + heatmaps = house.get_heatmap_dict() + room_polygons, room_types, icon_polygons, icon_types = house.get_coords_and_labels() + coef_width = 1 + if self.original_size: + fplan = cv2.imread(self.data_folder + self.folders[index] + self.org_image_file_name) + fplan = cv2.cvtColor(fplan, cv2.COLOR_BGR2RGB) # correct color channels + height_org, width_org, nchannel = fplan.shape + fplan = np.moveaxis(fplan, -1, 0) + label = label.unsqueeze(0) + label = torch.nn.functional.interpolate(label, size=(height_org, width_org), mode="nearest") + label = label.squeeze(0) + + coef_height = float(height_org) / float(height) + coef_width = float(width_org) / float(width) + for key, value in heatmaps.items(): + heatmaps[key] = [(int(round(x * coef_width)), int(round(y * coef_height))) for x, y in value] + + new_room_polygons = [] + for poly in room_polygons: + new_room_polygons.append([(int(round(x * coef_width)), int(round(y * coef_height))) for x, y in poly]) + room_polygons = new_room_polygons + + new_icon_polygons = [] + for poly in icon_polygons: + new_icon_polygons.append([(int(round(x * coef_width)), int(round(y * coef_height))) for x, y in poly]) + icon_polygons = new_icon_polygons + + img = torch.tensor(fplan.astype(np.float32)) + + sample = { + "image": img, + "label": label, + "folder": self.folders[index], + "heatmaps": heatmaps, + "scale": coef_width, + "room_polygon": room_polygons, + "room_type": room_types, + "icon_polygon": icon_polygons, + "icon_type": icon_types, + } + + return sample + + def get_lmdb(self, index): + key = self.folders[index].encode() + with self.lmdb.begin(write=False) as f: + data = f.get(key) + + sample = pickle.loads(data) + return sample + + def transform(self, sample): + fplan = sample["image"] + # Normalization values to range -1 and 1 + fplan = 2 * (fplan / 255.0) - 1 + + sample["image"] = fplan + + return sample diff --git a/data_preprocess/cubicasa5k/plotting.py b/data_preprocess/cubicasa5k/plotting.py new file mode 100644 index 0000000000000000000000000000000000000000..84bf8b17abae17c944bd02732fa019e9a64e1018 --- /dev/null +++ b/data_preprocess/cubicasa5k/plotting.py @@ -0,0 +1,820 @@ +import matplotlib.path as mplp +import matplotlib.pyplot as plt +import numpy as np +from matplotlib import cm, colors +from shapely.geometry import Point, Polygon +from skimage import draw + + +def discrete_cmap_furukawa(): + """create a colormap with N (N<15) discrete colors and register it""" + # define individual colors as hex values + cpool = [ + "#696969", + "#b3de69", + "#ffffb3", + "#8dd3c7", + "#fdb462", + "#fccde5", + "#80b1d3", + "#d9d9d9", + "#fb8072", + "#577a4d", + "white", + "#000000", + "#e31a1c", + ] + cmap3 = colors.ListedColormap(cpool, "rooms_furukawa") + cm.register_cmap(cmap=cmap3) + + cpool = [ + "#ede676", + "#8dd3c7", + "#b15928", + "#fdb462", + "#ffff99", + "#fccde5", + "#80b1d3", + "#d9d9d9", + "#fb8072", + "#696969", + "#577a4d", + "#e31a1c", + "#42ef59", + "#8c595a", + "#3131e5", + "#48e0e6", + "white", + ] + cmap3 = colors.ListedColormap(cpool, "icons_furukawa") + cm.register_cmap(cmap=cmap3) + + +def drawJunction(h, point, point_type, width, height): + lineLength = 15 + lineWidth = 10 + x, y = point + # plt.text(x,y,str(index),fontsize=25,color='r') + if point_type == -1: + h.scatter(x, y, color="#6488ea") + ########################### + # o + # | #6488ea soft blue + # | drawcode = [1,1] + # + ########################### + if point_type == 0: + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#6488ea") + # plt.scatter(x, y-10, c='k') + ########################### + # + # ---o #6241c7 bluey purple + # drawcode = [1,2] + # + ########################### + elif point_type == 1: + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#6241c7") + # plt.scatter(x+10, y, c='k') + ########################### + # | + # | drawcode = [1,3] + # o #056eee cerulean blue + # + ########################### + elif point_type == 2: + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#056eee") + # plt.scatter(x, y+10, c='k') + ########################### + # + # drawcode = [1,4] + # + # o--- #004577 prussian blue + # + ########################### + elif point_type == 3: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#004577") + # plt.scatter(x-10, y, c='k') + ########################### + # + # |--- drawcode = [2,3] + # | + # + ########################### + elif point_type == 6: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#04d8b2") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#04d8b2") + ########################### + # + # ---| + # | drawcode = [2,4] + # + ########################### + elif point_type == 7: + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#cdfd02") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#cdfd02") + ########################### + # | + # ---| drawcode = [2,1] + # + # + ########################### + elif point_type == 4: + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#ff81c0") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#ff81c0") + ########################### + # + # | + # | drawcode = [2,2] + # -- + # + ########################### + elif point_type == 5: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#f97306") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#f97306") + ########################### + # + # | + # |--- drawcode = [3,4] + # | + # + ########################### + elif point_type == 11: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="b") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="b") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="b") + ########################### + # + # --- + # | drawcode = [3,1] + # | + # + ########################### + elif point_type == 8: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="y") + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="y") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="y") + ########################### + # + # | + # ---| drawcode = [3,2] + # | + # + ########################### + elif point_type == 9: + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="r") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="r") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="r") + ########################### + # + # | + # | drawcode = [3,3] + # --- + # + ########################### + elif point_type == 10: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="m") + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="m") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="m") + ########################### + # + # | + # --- drawcode = [4,1] + # | + # + ########################### + elif point_type == 12: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="k") + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="k") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="k") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="k") + + lineLength = 10 + lineWidth = 5 + + ########################### + # o--- opening left + ########################### + if point_type == 13: + h.plot([x], [y], "o", markersize=30, color="red") + h.plot([x], [y], "o", markersize=25, color="white") + h.text(x, y, "OL", fontsize=30, color="magenta") + ########################### + # ---o opening right + ########################### + elif point_type == 14: + h.plot([x], [y], "o", markersize=30, color="red") + h.plot([x], [y], "o", markersize=25, color="white") + h.text(x, y, "OR", fontsize=30, color="magenta") + ########################### + # o opening up + # | + # | + ########################### + elif point_type == 15: + h.plot([x], [y], "o", markersize=30, color="red") + h.plot([x], [y], "o", markersize=25, color="white") + h.text(x, y, "OU", fontsize=30, color="mediumblue") + ########################### + # | opening down + # | + # o + ########################### + elif point_type == 16: + h.plot([x], [y], "o", markersize=30, color="red") + h.plot([x], [y], "o", markersize=25, color="white") + h.text(x, y, "OD", fontsize=30, color="mediumblue") + + ########################### + # + # |--- drawcode = [2,3] + # | + # + ########################### + elif point_type == 17: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="indianred") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="indianred") + ########################### + # + # ---| + # | drawcode = [2,4] + # + ########################### + elif point_type == 18: + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="darkred") + h.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="darkred") + ########################### + # + # | + # | drawcode = [2,2] + # -- + # + ########################### + elif point_type == 19: + h.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="salmon") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="salmon") + ########################### + # | + # ---| drawcode = [2,1] + # + # + ########################### + elif point_type == 20: + h.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="orangered") + h.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="orangered") + + +def draw_junction_from_dict(point_dict, width, height, size=1, fontsize=30): + index = 0 + markersize_large = 20 * size + markersize_small = 15 * size + for point_type, locations in point_dict.items(): + for loc in locations: + x, y = loc + lineLength = 20 * size + lineWidth = 20 * size + # plt.text(x,y,str(index),fontsize=25,color='r') + ########################### + # o + # | #6488ea soft blue + # | drawcode = [1,1] + # + ########################### + if point_type == 0: + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#6488ea") + # plt.scatter(x, y-10, c='k') + ########################### + # + # ---o #6241c7 bluey purple + # drawcode = [1,2] + # + ########################### + elif point_type == 1: + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#6241c7") + # plt.scatter(x+10, y, c='k') + ########################### + # | + # | drawcode = [1,3] + # o #056eee cerulean blue + # + ########################### + elif point_type == 2: + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#056eee") + # plt.scatter(x, y+10, c='k') + ########################### + # + # drawcode = [1,4] + # + # o--- #004577 prussian blue + # + ########################### + elif point_type == 3: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#004577") + # plt.scatter(x-10, y, c='k') + ########################### + # + # |--- drawcode = [2,3] + # | + # + ########################### + elif point_type == 6: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#04d8b2") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#04d8b2") + ########################### + # + # ---| + # | drawcode = [2,4] + # + ########################### + elif point_type == 7: + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#cdfd02") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#cdfd02") + ########################### + # | + # ---| drawcode = [2,1] + # + # + ########################### + elif point_type == 4: + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#ff81c0") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#ff81c0") + ########################### + # + # | + # | drawcode = [2,2] + # -- + # + ########################### + elif point_type == 5: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#f97306") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#f97306") + ########################### + # + # | + # |--- drawcode = [3,4] + # | + # + ########################### + elif point_type == 11: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="b") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="b") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="b") + ########################### + # + # --- + # | drawcode = [3,1] + # | + # + ########################### + elif point_type == 8: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="y") + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="y") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="y") + ########################### + # + # | + # ---| drawcode = [3,2] + # | + # + ########################### + elif point_type == 9: + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="r") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="r") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="r") + ########################### + # + # | + # | drawcode = [3,3] + # --- + # + ########################### + elif point_type == 10: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="m") + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="m") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="m") + ########################### + # + # | + # --- drawcode = [4,1] + # | + # + ########################### + elif point_type == 12: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="k") + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="k") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="k") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="k") + + lineLength = 15 * size + lineWidth = 15 * size + + ########################### + # o--- opening left + ########################### + if point_type == 13: + plt.plot([x], [y], "o", markersize=markersize_large, color="red") + plt.plot([x], [y], "o", markersize=markersize_small, color="white") + plt.text(x, y, "OL", fontsize=fontsize, color="magenta") + ########################### + # ---o opening right + ########################### + elif point_type == 14: + plt.plot([x], [y], "o", markersize=markersize_large, color="red") + plt.plot([x], [y], "o", markersize=markersize_small, color="white") + plt.text(x, y, "OR", fontsize=fontsize, color="magenta") + ########################### + # o opening up + # | + # | + ########################### + elif point_type == 15: + plt.plot([x], [y], "o", markersize=markersize_large, color="red") + plt.plot([x], [y], "o", markersize=markersize_small, color="white") + plt.text(x, y, "OU", fontsize=fontsize, color="mediumblue") + ########################### + # | opening down + # | + # o + ########################### + elif point_type == 16: + plt.plot([x], [y], "o", markersize=markersize_large, color="red") + plt.plot([x], [y], "o", markersize=markersize_small, color="white") + plt.text(x, y, "OD", fontsize=fontsize, color="mediumblue") + + ########################### + # + # |--- drawcode = [2,3] + # | + # + ########################### + elif point_type == 17: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="indianred") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="indianred") + ########################### + # + # ---| + # | drawcode = [2,4] + # + ########################### + elif point_type == 18: + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="darkred") + plt.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="darkred") + ########################### + # + # | + # | drawcode = [2,2] + # -- + # + ########################### + elif point_type == 19: + plt.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="salmon") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="salmon") + ########################### + # | + # ---| drawcode = [2,1] + # + # + ########################### + elif point_type == 20: + plt.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="orangered") + plt.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="orangered") + + index += 1 + + +def plot_pre_rec_4(instances, classes): + walls = ["Wall", "Railing"] + openings = ["Window", "Door"] + rooms = [ + "Outdoor", + "Kitchen", + "Living Room", + "Bed Room", + "Entry", + "Dining", + "Storage", + "Garage", + "Undefined Room", + "Sauna", + "Fire Place", + "Bathtub", + "Chimney", + ] + icons = [ + "Bath", + "Closet", + "Electrical Appliance", + "Toilet", + "Shower", + "Sink", + "Sauna", + "Fire Place", + "Bathtub", + "Chimney", + ] + + def make_sub_plot(classes_to_plot): + plt.ylim([0.0, 1.0]) + plt.xlim([0.0, 1.0]) + plt.xlabel("Recall") + plt.ylabel("Precision") + indx = [classes.index(i) for i in classes_to_plot] + ins = instances[:, indx].sum(axis=1) + + correct = ins[:, 0] + false_positive = ins[:, 2] + false_negatives = ins[:, 1] + precision = correct / (correct + false_positive) + recall = correct / (correct + false_negatives) + + plt.step(recall[::-1], precision, color="b", alpha=0.2, where="post") + plt.fill_between(recall[::-1], precision, step="post", alpha=0.2, color="b") + + plt.subplot(2, 2, 1) + plt.title("Walls") + make_sub_plot(walls) + plt.subplot(2, 2, 2) + plt.title("Openings") + make_sub_plot(openings) + plt.subplot(2, 2, 3) + plt.title("Rooms") + make_sub_plot(rooms) + plt.subplot(2, 2, 4) + plt.title("Icons") + make_sub_plot(icons) + + +def discrete_cmap(): + """create a colormap with N (N<15) discrete colors and register it""" + # define individual colors as hex values + cpool = [ + "#DCDCDC", + "#b3de69", + "#000000", + "#8dd3c7", + "#fdb462", + "#fccde5", + "#80b1d3", + "#808080", + "#fb8072", + "#696969", + "#577a4d", + "#ffffb3", + ] + cmap3 = colors.ListedColormap(cpool, "rooms") + cm.register_cmap(cmap=cmap3) + + cpool = [ + "#DCDCDC", + "#8dd3c7", + "#b15928", + "#fdb462", + "#ffff99", + "#fccde5", + "#80b1d3", + "#808080", + "#fb8072", + "#696969", + "#577a4d", + ] + cmap3 = colors.ListedColormap(cpool, "icons") + cm.register_cmap(cmap=cmap3) + + """create a colormap with N (N<15) discrete colors and register it""" + # define individual colors as hex values + cpool = [ + "#DCDCDC", + "#b3de69", + "#000000", + "#8dd3c7", + "#fdb462", + "#fccde5", + "#80b1d3", + "#808080", + "#fb8072", + "#696969", + "#577a4d", + "#ffffb3", + "d3d5d7", + ] + cmap3 = colors.ListedColormap(cpool, "rooms_furu") + cm.register_cmap(cmap=cmap3) + + cpool = [ + "#DCDCDC", + "#8dd3c7", + "#b15928", + "#fdb462", + "#ffff99", + "#fccde5", + "#80b1d3", + "#808080", + "#fb8072", + "#696969", + "#577a4d", + ] + cmap3 = colors.ListedColormap(cpool, "rooms_furu") + cm.register_cmap(cmap=cmap3) + + +def segmentation_plot(rooms_pred, icons_pred, rooms_label, icons_label): + room_classes = [ + "Background", + "Outdoor", + "Wall", + "Kitchen", + "Living Room", + "Bed Room", + "Bath", + "Entry", + "Railing", + "Storage", + "Garage", + "Undefined", + ] + icon_classes = [ + "No Icon", + "Window", + "Door", + "Closet", + "Electrical Applience", + "Toilet", + "Sink", + "Sauna Bench", + "Fire Place", + "Bathtub", + "Chimney", + ] + discrete_cmap() # custom colormap + + fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(30, 15)) + axes[0].set_title("Room Ground Truth") + axes[0].imshow(rooms_label, cmap="rooms", vmin=0, vmax=len(room_classes) - 1) + + axes[1].set_title("Room Prediction") + im = axes[1].imshow(rooms_pred, cmap="rooms", vmin=0, vmax=len(room_classes) - 1) + + cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) + cbar = fig.colorbar(im, cax=cbar_ax, ticks=np.arange(12) + 0.5) + + fig.subplots_adjust(right=0.8) + cbar.ax.set_yticklabels(room_classes) + plt.show() + + fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(30, 15)) + axes[0].set_title("Icon Ground Truth") + axes[0].imshow(icons_label, cmap="icons", vmin=0, vmax=len(icon_classes) - 1) + + axes[1].set_title("Icon Prediction") + im = axes[1].imshow(icons_pred, cmap="icons", vmin=0, vmax=len(icon_classes) - 1) + + cbar_ax = fig.add_axes([0.85, 0.15, 0.05, 0.7]) + cbar = fig.colorbar(im, cax=cbar_ax, ticks=np.arange(11) + 0.5) + + fig.subplots_adjust(right=0.8) + cbar.ax.set_yticklabels(icon_classes) + plt.show() + + +def polygons_to_image(polygons, types, room_polygons, room_types, height, width): + pol_room_seg = np.zeros((height, width)) + pol_icon_seg = np.zeros((height, width)) + + for i, pol in enumerate(room_polygons): + mask = shp_mask(pol, np.arange(width), np.arange(height)) + + # jj, ii = draw.polygon(pol[:, 1], pol[:, 0]) + pol_room_seg[mask] = room_types[i]["class"] + + for i, pol in enumerate(polygons): + jj, ii = draw.polygon(pol[:, 1], pol[:, 0]) + if types[i]["type"] == "wall": + pol_room_seg[jj, ii] = types[i]["class"] + else: + pol_icon_seg[jj, ii] = types[i]["class"] + + return pol_room_seg, pol_icon_seg + + +def plot_room(r, name, n_classes=12): + discrete_cmap() # custom colormap + plt.figure(figsize=(40, 30)) + plt.axis("off") + plt.tight_layout() + plt.imshow(r, cmap="rooms", vmin=0, vmax=n_classes - 1) + plt.savefig(name + ".png", format="png") + plt.show() + + +def plot_icon(i, name, n_classes=11): + discrete_cmap() # custom colormap + plt.figure(figsize=(40, 30)) + plt.axis("off") + plt.tight_layout() + plt.imshow(i, cmap="icons", vmin=0, vmax=n_classes - 1) + plt.savefig(name + ".png", format="png") + plt.show() + + +def plot_heatmaps(h, name): + for index, i in enumerate(h): + plt.figure(figsize=(40, 30)) + plt.axis("off") + plt.tight_layout() + plt.imshow(i, cmap="Reds", vmin=0, vmax=1) + plt.savefig(name + str(index) + ".png", format="png") + plt.show() + + +def outline_to_mask(line, x, y): + """Create mask from outline contour + + Parameters + ---------- + line: array-like (N, 2) + x, y: 1-D grid coordinates (input for meshgrid) + + Returns + ------- + mask : 2-D boolean array (True inside) + + Examples + -------- + >>> from shapely.geometry import Point + >>> poly = Point(0,0).buffer(1) + >>> x = np.linspace(-5,5,100) + >>> y = np.linspace(-5,5,100) + >>> mask = outline_to_mask(poly.boundary, x, y) + """ + mpath = mplp.Path(line) + X, Y = np.meshgrid(x, y) + points = np.array((X.flatten(), Y.flatten())).T + mask = mpath.contains_points(points).reshape(X.shape) + return mask + + +def _grid_bbox(x, y): + dx = dy = 0 + return x[0] - dx / 2, x[-1] + dx / 2, y[0] - dy / 2, y[-1] + dy / 2 + + +def _bbox_to_rect(bbox): + l, r, b, t = bbox + return Polygon([(l, b), (r, b), (r, t), (l, t)]) + + +def shp_mask(shp, x, y, m=None): + """ + Adapted from code written by perrette + form: https://gist.github.com/perrette/a78f99b76aed54b6babf3597e0b331f8 + Use recursive sub-division of space and shapely contains method to create a raster mask on a regular grid. + + Parameters + ---------- + shp : shapely's Polygon (or whatever with a "contains" method and intersects method) + x, y : 1-D numpy arrays defining a regular grid + m : mask to fill, optional (will be created otherwise) + + Returns + ------- + m : boolean 2-D array, True inside shape. + + Examples + -------- + >>> from shapely.geometry import Point + >>> poly = Point(0,0).buffer(1) + >>> x = np.linspace(-5,5,100) + >>> y = np.linspace(-5,5,100) + >>> mask = shp_mask(poly, x, y) + """ + rect = _bbox_to_rect(_grid_bbox(x, y)) + + if m is None: + m = np.zeros((y.size, x.size), dtype=bool) + + if not shp.intersects(rect): + m[:] = False + + elif shp.contains(rect): + m[:] = True + + else: + k, l = m.shape + + if k == 1 and l == 1: + m[:] = shp.contains(Point(x[0], y[0])) + + elif k == 1: + m[:, : l // 2] = shp_mask(shp, x[: l // 2], y, m[:, : l // 2]) + m[:, l // 2 :] = shp_mask(shp, x[l // 2 :], y, m[:, l // 2 :]) + + elif l == 1: + m[: k // 2] = shp_mask(shp, x, y[: k // 2], m[: k // 2]) + m[k // 2 :] = shp_mask(shp, x, y[k // 2 :], m[k // 2 :]) + + else: + m[: k // 2, : l // 2] = shp_mask(shp, x[: l // 2], y[: k // 2], m[: k // 2, : l // 2]) + m[: k // 2, l // 2 :] = shp_mask(shp, x[l // 2 :], y[: k // 2], m[: k // 2, l // 2 :]) + m[k // 2 :, : l // 2] = shp_mask(shp, x[: l // 2], y[k // 2 :], m[k // 2 :, : l // 2]) + m[k // 2 :, l // 2 :] = shp_mask(shp, x[l // 2 :], y[k // 2 :], m[k // 2 :, l // 2 :]) + + return m diff --git a/data_preprocess/cubicasa5k/run.sh b/data_preprocess/cubicasa5k/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..a72ee35d30ea02b7fa9c61f8912b6b74738d2293 --- /dev/null +++ b/data_preprocess/cubicasa5k/run.sh @@ -0,0 +1,15 @@ +# create COCO-style dataset for CubiCasa5k +python create_coco_cc5k.py --data_root=data/cubicasa5k/ \ + --output=data/coco_cubicasa5k_nowalls_v4/ \ + --disable_wd2line + +# Split example has more than 1 floorplan into separate samples +python floorplan_extraction.py \ + --data_root data/coco_cubicasa5k_nowalls_v4/ \ + --output data/coco_cubicasa5k_nowalls_v4-1_refined/ + +# Merge individual JSONs into single JSON file per split (train/val/test) +# This must be done after floorplan_extraction.py +python combine_json.py \ + --input data/coco_cubicasa5k_nowalls_v4-1_refined/ \ + --output data/coco_cubicasa5k_nowalls_v4-1_refined/annotations/ \ \ No newline at end of file diff --git a/data_preprocess/cubicasa5k/svg_utils.py b/data_preprocess/cubicasa5k/svg_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5575301478a6bac07d7e7e9f8a9b3a071e77e362 --- /dev/null +++ b/data_preprocess/cubicasa5k/svg_utils.py @@ -0,0 +1,746 @@ +import math +from logging import warning + +import numpy as np +from skimage.draw import polygon +from svgpathtools import parse_path + + +def get_room_number(e, rooms): + name_list = e.getAttribute("class").split(" ") + room_type = name_list[1] + try: + return rooms[room_type] + except KeyError: + warning("Room type " + e.getAttribute("class") + " not defined.") + return rooms["Undefined"] + + +def get_icon_number(e, icons): + name_list = e.getAttribute("class").split(" ") + icon_type = name_list[1] + + try: + return icons[icon_type] + except KeyError: + warning("Icon type " + e.getAttribute("class") + " not defined.") + return icons["Misc"] + + +def get_icon(ee): + parent_transform = None + if ee.parentNode.getAttribute("class") == "FixedFurnitureSet": + parent_transform = ee.parentNode.getAttribute("transform") + strings = parent_transform.split(",") + a_p = float(strings[0][7:]) + b_p = float(strings[1]) + c_p = float(strings[2]) + d_p = float(strings[3]) + e_p = float(strings[-2]) + f_p = float(strings[-1][:-1]) + M_p = np.array([[a_p, c_p, e_p], [b_p, d_p, f_p], [0, 0, 1]]) + + transform = ee.getAttribute("transform") + strings = transform.split(",") + a = float(strings[0][7:]) + b = float(strings[1]) + c = float(strings[2]) + d = float(strings[3]) + e = float(strings[-2]) + f = float(strings[-1][:-1]) + + M = np.array([[a, c, e], [b, d, f], [0, 0, 1]]) + + X = np.array([]) + Y = np.array([]) + + try: + toilet = next(p for p in ee.childNodes if p.nodeName == "g" and p.getAttribute("class") == "BoundaryPolygon") + + for p in toilet.childNodes: + if p.nodeName == "polygon": + X, Y = get_icon_polygon(p) + break + else: + x_all, y_all = get_corners(toilet) + points = np.column_stack((x_all, y_all)) + + X, Y = get_max_corners(points) + # if p.nodeName == "path": + # X, Y = get_icon_path(p) + + except StopIteration: + X, Y = make_boudary_polygon(ee) + + if len(X) < 4: + return None, None, X, Y + + if parent_transform is not None: + for i in range(len(X)): + v = np.matrix([[X[i]], [Y[i]], [1]]) + vv = np.matmul(M, v) + new_x, new_y, _ = np.round(np.matmul(M_p, vv)) + X[i] = new_x + Y[i] = new_y + else: + for i in range(len(X)): + v = np.matrix([[X[i]], [Y[i]], [1]]) + vv = np.matmul(M, v) + new_x, new_y, _ = np.round(vv) + X[i] = new_x + Y[i] = new_y + + rr, cc = polygon(Y, X) + + return rr, cc, X, Y + + +def get_corners(g): + x_all, y_all = [], [] + for pol in g.childNodes: + if pol.nodeName == "polygon": + x, y = get_icon_polygon(pol) + x_all = np.append(x_all, x) + y_all = np.append(y_all, y) + elif pol.nodeName == "path": + x, y = get_icon_path(pol) + x_all = np.append(x_all, x) + y_all = np.append(y_all, y) + elif pol.nodeName == "rect": + x = pol.getAttribute("x") + if x == "": + x = 1.0 + else: + x = float(x) + y = pol.getAttribute("y") + if y == "": + y = 1.0 + else: + y = float(y) + x_all = np.append(x_all, x) + y_all = np.append(y_all, y) + w = float(pol.getAttribute("width")) + h = float(pol.getAttribute("height")) + x_all = np.append(x_all, x + w) + y_all = np.append(y_all, y + h) + + return x_all, y_all + + +def get_max_corners(points): + if len(points) == 0: + return [], [] + + minx, miny = float("inf"), float("inf") + maxx, maxy = float("-inf"), float("-inf") + for x, y in points: + # Set min coords + if x < minx: + minx = x + if y < miny: + miny = y + # Set max coords + if x > maxx: + maxx = x + elif y > maxy: + maxy = y + + X = np.array([minx, maxx, maxx, minx]) + Y = np.array([miny, miny, maxy, maxy]) + + return X, Y + + +def make_boudary_polygon(pol): + g_gen = (c for c in pol.childNodes if c.nodeName == "g") + + x_all, y_all = [], [] + for g in g_gen: + x, y = get_corners(g) + x_all = np.append(x_all, x) + y_all = np.append(y_all, y) + + points = np.column_stack((x_all, y_all)) + X, Y = get_max_corners(points) + + return X, Y + + +def get_icon_path(pol): + path = pol.getAttribute("d") + try: + path_alt = parse_path(path) + minx, maxx, miny, maxy = path_alt.bbox() + except ValueError as e: + print("Error handled") + print(e) + return np.array([]), np.array([]) + + X = np.array([minx, maxx, maxx, minx]) + Y = np.array([miny, miny, maxy, maxy]) + + if np.unique(X).size == 1 or np.unique(Y).size == 1: + return np.array([]), np.array([]) + + return X, Y + + +def get_icon_polygon(pol): + points = pol.getAttribute("points").split(" ") + + return get_XY(points) + + +def get_XY(points): + if points[-1] == "": + points = points[:-1] + + if points[0] == "": + points = points[1:] + + X, Y = np.array([]), np.array([]) + i = 0 + for a in points: + if "," in a: + if len(a) == 2: + x, y = a.split(",") + else: + num_list = a.split(",") + x, y = num_list[0], num_list[1] + X = np.append(X, np.round(float(x))) + Y = np.append(Y, np.round(float(y))) + else: + # if no comma every other is x and every other is y + if i % 2: + Y = np.append(Y, float(a)) + else: + X = np.append(X, float(a)) + + i += 1 + + return X, Y + + +def get_points(e): + pol = next(p for p in e.childNodes if p.nodeName == "polygon") + points = pol.getAttribute("points").split(" ") + points = points[:-1] + + X, Y = np.array([]), np.array([]) + for a in points: + x, y = a.split(",") + X = np.append(X, np.round(float(x))) + Y = np.append(Y, np.round(float(y))) + + return X, Y + + +def get_direction(X, Y): + max_diff_X = abs(max(X) - min(X)) + max_diff_Y = abs(max(Y) - min(Y)) + + if max_diff_X > max_diff_Y: + return "H" # horizontal + else: + return "V" # vertical + + +def get_polygon(e): + pol = next(p for p in e.childNodes if p.nodeName == "polygon") + points = pol.getAttribute("points").split(" ") + points = points[:-1] + + X, Y = np.array([]), np.array([]) + for a in points: + y, x = a.split(",") + X = np.append(X, np.round(float(x))) + Y = np.append(Y, np.round(float(y))) + + rr, cc = polygon(X, Y) + + return rr, cc + + +def calc_distance(point_1, point_2): + return math.sqrt(math.pow(point_1[0] - point_2[0], 2) + math.pow(point_1[1] - point_2[1], 2)) + + +def calc_center(points): + return list(np.mean(np.array(points), axis=0)) + + +def get_gaussian2D(ndim, sigma=0.25): + over_sigmau = 1.0 / (sigma * ndim) + over_sigmav = 1.0 / (sigma * ndim) + dst_data = np.zeros((ndim, ndim)) + + mean_u = 0.5 * ndim + 0.5 + mean_v = 0.5 * ndim + 0.5 + + for v in range(ndim): + for u in range(ndim): + du = (u + 1 - mean_u) * over_sigmau + dv = (v + 1 - mean_v) * over_sigmav + value = np.exp(-0.5 * (du * du + dv * dv)) + dst_data[v][u] = value + + return dst_data + + +def draw_junction(index, point, width, height, axes): + lineLength = 15 + lineWidth = 7 + x, y = point[0] + axes.text(x, y, str(index), fontsize=15, color="k") + ########################### + # o + # | #6488ea soft blue + # | drawcode = [1,1] + # + ########################### + if point[2][1] == 1 and point[2][2] == 1: + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#6488ea") + ########################### + # + # ---o #6241c7 bluey purple + # drawcode = [1,2] + # + ########################### + elif point[2][1] == 1 and point[2][2] == 2: + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#6241c7") + ########################### + # | + # | drawcode = [1,3] + # o #056eee cerulean blue + # + ########################### + elif point[2][1] == 1 and point[2][2] == 3: + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#056eee") + ########################### + # + # drawcode = [1,4] + # + # o--- #004577 prussian blue + # + ########################### + elif point[2][1] == 1 and point[2][2] == 4: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#004577") + ########################### + # + # |--- drawcode = [2,3] + # | + # + ########################### + elif point[2][1] == 2 and point[2][2] == 3: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#04d8b2") + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#04d8b2") + ########################### + # + # ---| + # | drawcode = [2,4] + # + ########################### + elif point[2][1] == 2 and point[2][2] == 4: + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#cdfd02") + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="#cdfd02") + ########################### + # | + # ---| drawcode = [2,1] + # + # + ########################### + elif point[2][1] == 2 and point[2][2] == 1: + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="#ff81c0") + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#ff81c0") + ########################### + # + # | + # | drawcode = [2,2] + # -- + # + ########################### + elif point[2][1] == 2 and point[2][2] == 2: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="#f97306") + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="#f97306") + ########################### + # + # | + # |--- drawcode = [3,4] + # | + # + ########################### + elif point[2][1] == 3 and point[2][2] == 4: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="b") + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="b") + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="b") + ########################### + # + # --- + # | drawcode = [3,1] + # | + # + ########################### + elif point[2][1] == 3 and point[2][2] == 1: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="y") + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="y") + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="y") + ########################### + # + # | + # ---| drawcode = [3,2] + # | + # + ########################### + elif point[2][1] == 3 and point[2][2] == 2: + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="r") + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="r") + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="r") + ########################### + # + # | + # | drawcode = [3,3] + # --- + # + ########################### + elif point[2][1] == 3 and point[2][2] == 3: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="m") + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="m") + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="m") + ########################### + # + # | + # --- drawcode = [4,1] + # | + # + ########################### + elif point[2][1] == 4 and point[2][2] == 1: + axes.plot([x, min(x + lineLength, width - 1)], [y, y], linewidth=lineWidth, color="k") + axes.plot([x, max(x - lineLength, 0)], [y, y], linewidth=lineWidth, color="k") + axes.plot([x, x], [y, max(y - lineLength, 0)], linewidth=lineWidth, color="k") + axes.plot([x, x], [y, min(y + lineLength, height - 1)], linewidth=lineWidth, color="k") + + +class Wall: + def __init__(self, id, end_points, direction, width, name): + self.id = id + self.name = name + self.end_points = end_points + self.direction = direction + self.max_width = width + self.min_width = width + + def change_end_points(self): + if self.direction == "V": + self.end_points[0][0] = np.mean(np.array(self.min_coord)) + self.end_points[1][0] = self.end_points[0][0] + elif self.direction == "H": + self.end_points[0][1] = np.mean(np.array(self.min_coord)) + self.end_points[1][1] = self.end_points[0][1] + + def get_length(self, end_points): + return calc_distance(end_points[0], end_points[1]) + + +class LineWall(Wall): + def __init__(self, id, end_points, direction, width, name): + Wall.__init__(self, id, end_points, direction, width, name) + + +class PolygonWall(Wall): + def __init__(self, e, id, shape=None): + self.id = id + self.name = e.getAttribute("id") + self.X, self.Y = self.get_points(e) + if abs(max(self.X) - min(self.X)) < 4 or abs(max(self.Y) - min(self.Y)) < 4: + # wall is too small and we ignore it. + raise ValueError("small wall") + if shape: + self.X = np.clip(self.X, 0, shape[1]) + self.Y = np.clip(self.Y, 0, shape[0]) + # self.X, self.Y = self.sort_X_Y(self.X, self.Y) + self.rr, self.cc = polygon(self.Y, self.X) + direction = self.get_direction(self.X, self.Y) + end_points = self.get_end_points(self.X, self.Y, direction) + self.min_width = self.get_width(self.X, self.Y, direction) + self.max_width = self.min_width + + Wall.__init__(self, id, end_points, direction, self.max_width, self.name) + self.length = self.get_length(self.end_points) + self.center = self.get_center(self.X, self.Y) + self.min_coord, self.max_coord = self.get_width_coods(self.X, self.Y) + + def get_points(self, e): + pol = next(p for p in e.childNodes if p.nodeName == "polygon") + points = pol.getAttribute("points").split(" ") + points = points[:-1] + + X, Y = np.array([]), np.array([]) + for a in points: + x, y = a.split(",") + X = np.append(X, np.round(float(x))) + Y = np.append(Y, np.round(float(y))) + + return X, Y + + def get_direction(self, X, Y): + max_diff_X = abs(max(X) - min(X)) + max_diff_Y = abs(max(Y) - min(Y)) + + if max_diff_X > max_diff_Y: + return "H" # horizontal + else: + return "V" # vertical + + def get_center(self, X, Y): + return np.mean(X), np.mean(Y) + + def get_width(self, X, Y, direction): + _, _, p1, p2 = self._get_min_points(X, Y) + + if direction == "H": + return (abs(p1[0][1] - p1[1][1]) + abs(p2[0][1] - p2[1][1])) / 2 + elif "V": + return (abs(p1[0][0] - p1[1][0]) + abs(p2[0][0] - p2[1][0])) / 2 + + def _width(self, values): + temp = values.tolist() if type(values) is not list else values + + mean_1 = min(temp) + mean_2 = max(temp) + + return abs(mean_1 - mean_2) + + def merge_possible(self, merged): + max_dist = max([self.max_width, merged.max_width]) + + if self.id == merged.id: + return False + + # walls have to be in the same direction + if self.direction != merged.direction: + return False + + # walls have too big width difference + if abs(self.max_width - merged.max_width) > merged.max_width: + return False + + # If endpoints are near + # self up and left endpoint to merged down and right end point + dist1 = calc_distance(self.end_points[0], merged.end_points[1]) + # self down and right endpoint to merged up and left end point + dist2 = calc_distance(self.end_points[1], merged.end_points[0]) + + if dist1 <= max_dist * 1.5 or dist2 <= max_dist * 1.5: + return True + else: + return False + + def _get_overlap(self, a, b): + return max(0, min(a[1], b[1]) - max(a[0], b[0])) + + def merge_walls(self, merged): + max_dist = max([self.max_width, merged.max_width]) + + if self.id == merged.id: + return None + + # walls have to be in the same direction + if self.direction != merged.direction: + return None + + # If endpoints are near + # self up and left endpoint to merged down and right end point + dist1 = calc_distance(self.end_points[0], merged.end_points[1]) + # self down and right endpoint to merged up and left end point + dist2 = calc_distance(self.end_points[1], merged.end_points[0]) + + if dist1 <= max_dist * 1.5: + if self._get_overlap(self.min_coord, merged.min_coord) <= 0: + return None + # merged is on top or on left + return self.do_merge(merged, 0) + elif dist2 <= max_dist * 1.5: + if self._get_overlap(self.min_coord, merged.min_coord) <= 0: + return None + # merged is on down or on right + return self.do_merge(merged, 1) + else: + return None + + def _get_min_points(self, X, Y): + assert len(X) == 4 and len(Y) == 4 + length = len(X) + min_dist1 = np.inf + min_dist2 = np.inf + point1 = None + point2 = None + corners1 = None + corners2 = None + + for i in range(length): + x1, y1 = X[i], Y[i] + x2, y2 = X[(i + 1) % 4], Y[(i + 1) % 4] + + dist = np.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2) + if dist < min_dist1: + point2 = point1 + point1 = np.array([(x1 + x2) / 2, (y1 + y2) / 2]) + min_dist2 = min_dist1 + min_dist1 = dist + corners2 = corners1 + corners1 = np.array([[x1, y1], [x2, y2]]) + elif dist <= min_dist2: + point2 = np.array([(x1 + x2) / 2, (y1 + y2) / 2]) + min_dist2 = dist + corners2 = np.array([[x1, y1], [x2, y2]]) + + return point1, point2, corners1, corners2 + + def get_end_points(self, X, Y, direction): + point1, point2, _, _ = self._get_min_points(X, Y) + + if point1[0] != point2[0] or point1[1] != point2[1]: + if abs(point1[0] - point2[0]) > abs(point1[1] - point2[1]): + # horizontal + point1[1] = point1[1] + point2[1] / 2.0 + point2[1] = point1[1] + # point1[1] = int(np.round(point1[1])) + # point2[1] = int(np.round(point2[1])) + else: + # vertical + point1[0] = point1[0] + point2[0] / 2.0 + point2[0] = point1[0] + # point1[0] = int(np.round(point1[0])) + # point2[0] = int(np.round(point2[0])) + + return self.sort_end_points(direction, point1, point2) + + def sort_end_points(self, direction, point1, point2): + if direction == "V": + if point1[1] < point2[1]: + return np.array([point1, point2]) + else: + return np.array([point2, point1]) + else: + if point1[0] < point2[0]: + return np.array([point1, point2]) + else: + return np.array([point2, point1]) + + def do_merge(self, merged, direction): + # update width + self.max_width = max([self.max_width, merged.max_width]) + self.min_width = min([self.min_width, merged.min_width]) + + # update polygon + self.X = np.concatenate((self.X, merged.X)) + self.Y = np.concatenate((self.Y, merged.Y)) + + # update width coordinates + self.max_coord = self.get_max_width_coord(merged) + self.min_coord = self.get_min_width_coord(merged) + + if direction == 0: + # merged wall is up or left to the original wall + self.end_points = np.array([merged.end_points[0], self.end_points[1]]) + else: + # merged wall is down or right to the original wall + self.end_points = np.array([self.end_points[0], merged.end_points[1]]) + + self.length = self.get_length(self.end_points) + + return self + + def get_max_width_coord(self, merged): + width_1 = abs(self.max_coord[0] - self.max_coord[1]) + width_2 = abs(merged.max_coord[0] - merged.max_coord[1]) + return self.max_coord if width_1 > width_2 else merged.max_coord + + def get_min_width_coord(self, merged): + width_1 = max(merged.min_coord[0], self.min_coord[0]) + # width_1 = abs(self.min_coord[0] - self.min_coord[1]) + width_2 = min(merged.min_coord[1], self.min_coord[1]) + # width_2 = abs(merged.min_coord[0] - merged.min_coord[1]) + # return self.min_coord if width_1 < width_2 else merged.min_coord + return [width_1, width_2] + + def get_width_coods(self, X, Y): + if self.direction == "H": + dist_1 = abs(Y[0] - Y[2]) + dist_2 = abs(Y[1] - Y[3]) + if dist_1 < dist_2: + return [Y[0], Y[2]], [Y[1], Y[3]] + else: + return [Y[1], Y[3]], [Y[0], Y[2]] + + elif self.direction == "V": + dist_1 = abs(X[0] - X[3]) + dist_2 = abs(X[1] - X[2]) + if dist_1 < dist_2: + return [X[0], X[3]], [X[1], X[2]] + else: + return [X[1], X[2]], [X[0], X[3]] + + def sort_X_Y(self, X, Y): + max_x = max(X) + min_x = min(X) + max_y = max(Y) + min_y = min(Y) + res_X, res_Y = [0] * 4, [0] * 4 + # top left 0, top right 1, bottom left 2, bottom right 3 + directions = [[min_x, min_y], [max_x, min_y], [min_x, max_y], [max_x, max_y]] + length = len(X) + for i in range(length): + min_dist = 1000000 + direction_candidate = None + for j, direc in enumerate(directions): + dist = calc_distance([X[i], Y[i]], direc) + if dist < min_dist: + min_dist = dist + direction_candidate = j + + res_X[direction_candidate] = X[i] + res_Y[direction_candidate] = Y[i] + + return res_X, res_Y + + def wall_is_pillar(self, avg_wall_width): + if self.max_width > avg_wall_width: + if self.length < 3 * self.max_width: + return True + + return False + + def split_pillar_wall(self, ids, avg_wall_width): + half = avg_wall_width / 3.0 + end_points = [[[0, 0], [0, 0]], [[0, 0], [0, 0]], [[0, 0], [0, 0]], [[0, 0], [0, 0]]] + self.X[np.argmax(self.X)] = max(self.X) - half + self.X[np.argmax(self.X)] = max(self.X) - half + self.X[np.argmin(self.X)] = min(self.X) + half + self.X[np.argmin(self.X)] = min(self.X) + half + self.Y[np.argmax(self.Y)] = max(self.Y) - half + self.Y[np.argmax(self.Y)] = max(self.Y) - half + self.Y[np.argmin(self.Y)] = min(self.Y) + half + self.Y[np.argmin(self.Y)] = min(self.Y) + half + for i in range(4): + x = self.X[i] + y = self.Y[i] + end = [x, y] + j = i % 2 + end_points[i][j] = end + end_points[(i + 3) % 4][j] = end + + walls = [] + for i, e in enumerate(end_points): + if abs(e[0][1] - e[1][1]) > abs(e[0][0] - e[1][0]): + # vertical wall + direction = "V" + else: + # horizontal wall + direction = "H" + + e = self.sort_end_points(direction, e[0], e[1]) + wall = LineWall(ids + i, e, direction, avg_wall_width / 2.0, self.name) + walls.append(wall) + + return walls diff --git a/data_preprocess/raster2graph/combine_json.py b/data_preprocess/raster2graph/combine_json.py new file mode 100644 index 0000000000000000000000000000000000000000..a99d5b0299853e5476800da502b64f9a262d6892 --- /dev/null +++ b/data_preprocess/raster2graph/combine_json.py @@ -0,0 +1,122 @@ +import glob +import json +import os +import shutil +from pathlib import Path + + +def combine_json_files(input_pattern, data_path, split_type, output_file, output_image_dir, start_image_id=0): + """ + Combines multiple COCO-style JSON annotation files into a single file. + + Args: + input_pattern: Glob pattern to match the input JSON files (e.g., "annotations/*.json") + output_file: Path to the output combined JSON file + """ + os.makedirs(output_image_dir, exist_ok=True) + + # Initialize combined data structure + combined_data = {"images": [], "annotations": [], "categories": []} + + # Track image and annotation IDs to avoid duplicates + annotation_ids_seen = set() + + next_image_id = start_image_id + next_annotation_id = 0 + skip_file_list = [] + image_id_mapping = {} + + # Find all matching JSON files + json_files = sorted(glob.glob(input_pattern)) + print(f"Found {len(json_files)} JSON files to combine") + + # Process each file + for i, json_file in enumerate(json_files): + print(f"Processing file {i + 1}/{len(json_files)}: {json_file}") + + with open(json_file, "r") as f: + data = json.load(f) + + # Store categories from the first file + if i == 0 and data.get("categories"): + combined_data["categories"] = data["categories"] + + # empty annos + if len(data["annotations"]) == 0: + skip_file_list.append(data["images"][0]["id"]) + continue + + # Process images + for image in data.get("images", []): + if image["id"] not in image_id_mapping: + image_id_mapping[image["id"]] = next_image_id + else: + skip_file_list.append(image["id"]) + continue + image["id"] = next_image_id + next_image_id += 1 + # org_file_name = copy(image['file_name']) + image["file_name"] = str(image["id"]).zfill(6) + ".png" + org_file_name = os.path.basename(json_file).replace(".json", ".png") + if image["file_name"] != org_file_name and os.path.exists(f"{data_path}/{split_type}/{org_file_name}"): + shutil.copy(f"{data_path}/{split_type}/{org_file_name}", f"{output_image_dir}/{image['file_name']}") + combined_data["images"].append(image) + + # Process annotations + for annotation in data.get("annotations", []): + annotation["id"] = next_annotation_id + next_annotation_id += 1 + annotation["image_id"] = image_id_mapping[annotation["image_id"]] + + annotation_ids_seen.add(annotation["id"]) + combined_data["annotations"].append(annotation) + + # Write combined data to output file + output_path = Path(output_file) + output_path.parent.mkdir(exist_ok=True, parents=True) + + with open(output_file, "w") as f: + json.dump(combined_data, f, indent=2) + + with open(output_path.parent / f"{output_path.name.split('.')[0]}_image_id_mapping.json", "w") as f: + json.dump(image_id_mapping, f, indent=2) + + if len(skip_file_list): + with open(output_path.parent / f"{output_path.name.split('.')[0]}_skipped.txt", "w") as f: + f.write("\n".join([str(x) for x in skip_file_list])) + + print(f"Combined data written to {output_file}") + print(f"Total images: {len(combined_data['images'])}") + print(f"Total annotations: {len(combined_data['annotations'])}") + print(f"Total categories: {len(combined_data['categories'])}") + print(f"Skipped images: {len(skip_file_list)}") + + image_id_mapping_list = [[f"{k} {v}"] for k, v in image_id_mapping.items()] # Reverse mapping for easier lookup + + return combined_data, image_id_mapping_list + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Combine multiple COCO-style JSON annotation files") + parser.add_argument("--input", required=True, help="Glob pattern for input JSON files, e.g., 'annotations/*.json'") + parser.add_argument("--output", required=True, help="Output JSON file path") + + args = parser.parse_args() + + splits = ["train", "val", "test"] + for i, split in enumerate(splits): + if split == "train": + start_image_id = 0 + else: + start_image_id += len(list(Path(f"{args.input}/{splits[i - 1]}").glob("*.png"))) + + _, image_id_mapping_list = combine_json_files( + f"{args.input}/{split}_jsons/*.json", + args.input, + split, + f"{args.output}/annotations/{split}.json", + output_image_dir=f"{args.output}/{split}", + start_image_id=start_image_id, + ) diff --git a/data_preprocess/raster2graph/combine_mapping_ids.py b/data_preprocess/raster2graph/combine_mapping_ids.py new file mode 100644 index 0000000000000000000000000000000000000000..bc778f7d06c131e86b4a406ef9f6c6c320c0c4b8 --- /dev/null +++ b/data_preprocess/raster2graph/combine_mapping_ids.py @@ -0,0 +1,95 @@ +import json + + +def generate_combined_mapping(file_mapping_path, image_id_mapping_path, output_path): + """ + Generates a combined mapping file from an original filename mapping + and an image ID mapping. + + Args: + file_mapping_path (str): Path to the text file mapping original filenames + to intermediate 6-digit IDs. + image_id_mapping_path (str): Path to the JSON file mapping intermediate + IDs to destination IDs. + output_path (str): Path where the new combined mapping file will be saved. + """ + # 1. Read test_file_mapping.txt + org_fn_to_intermediate_id = {} + try: + with open(file_mapping_path, "r") as f: + for line in f: + parts = line.strip().split() + if len(parts) == 2: + org_fn = parts[0] + # Convert the 6-digit string ID to an integer for lookup + intermediate_id_str = parts[1] + # Remove leading zeros and convert to int + intermediate_id = int(intermediate_id_str) + org_fn_to_intermediate_id[org_fn] = intermediate_id + except FileNotFoundError: + print(f"Error: The file '{file_mapping_path}' was not found.") + return + except Exception as e: + print(f"Error reading '{file_mapping_path}': {e}") + return + + # 2. Read test_image_id_mapping.json + intermediate_id_to_dst_fn = {} + try: + with open(image_id_mapping_path, "r") as f: + image_id_data = json.load(f) + for key, value in image_id_data.items(): + # Keys in JSON are strings, convert to int for consistency + intermediate_id_to_dst_fn[int(key)] = value + except FileNotFoundError: + print(f"Error: The file '{image_id_mapping_path}' was not found.") + return + except json.JSONDecodeError: + print(f"Error: Could not decode JSON from '{image_id_mapping_path}'. Please ensure it's valid JSON.") + return + except Exception as e: + print(f"Error reading '{image_id_mapping_path}': {e}") + return + + # 3. Create the combined mapping and write to output file + combined_mappings = [] + found_mappings_count = 0 + for org_fn, intermediate_id in org_fn_to_intermediate_id.items(): + if intermediate_id in intermediate_id_to_dst_fn: + dst_fn = intermediate_id_to_dst_fn[intermediate_id] + combined_mappings.append(f"{org_fn} {dst_fn}") + found_mappings_count += 1 + else: + # Optionally, you can print a warning for IDs not found + print(f"Warning: Intermediate ID '{intermediate_id}' for '{org_fn}' not found in image ID mapping.") + + try: + with open(output_path, "w") as f: + for mapping_line in combined_mappings: + f.write(mapping_line + "\n") + print(f"\nSuccessfully generated combined mapping to '{output_path}'.") + print(f"Total original filenames processed: {len(org_fn_to_intermediate_id)}") + print(f"Total combined mappings written: {found_mappings_count}") + except Exception as e: + print(f"Error writing to output file '{output_path}': {e}") + + +# Define file paths +file_mapping_path = "data/R2G_hr_dataset_processed/test_file_mapping.txt" +image_id_mapping_path = "data/R2G_hr_dataset_processed_v1/annotations/test_image_id_mapping.json" +output_mapping_path = "data/R2G_hr_dataset_processed_v1/annotations/test_combined_mapping.txt" + +# Run the mapping function +generate_combined_mapping(file_mapping_path, image_id_mapping_path, output_mapping_path) + +# You can optionally print the content of the generated file to verify +print("\n--- Content of combined_mapping.txt ---") +try: + with open(output_mapping_path, "r") as f: + print(f.read()) +except FileNotFoundError: + print("Output file was not created.") + +# Clean up dummy files (optional) +# os.remove(file_mapping_path) +# os.remove(image_id_mapping_path) diff --git a/data_preprocess/raster2graph/convert_to_coco.py b/data_preprocess/raster2graph/convert_to_coco.py new file mode 100644 index 0000000000000000000000000000000000000000..686924f916e2878714294356833a63178654f74a --- /dev/null +++ b/data_preprocess/raster2graph/convert_to_coco.py @@ -0,0 +1,472 @@ +import gc +import os +import sys + +print(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import argparse +import json +import shutil +from multiprocessing import Pool + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch +from datasets.dataset import MyDataset +from matplotlib.patches import Patch +from shapely.geometry import Polygon +from tqdm import tqdm +from util.data_utils import edge_inside +from util.graph_utils import get_cycle_basis_and_semantic, tensors_to_graphs_batch + +mean = [0.920, 0.913, 0.891] +std = [0.214, 0.216, 0.228] + +ID2CLASS = { + 0: "unknown", + 1: "living_room", + 2: "kitchen", + 3: "bedroom", + 4: "bathroom", + 5: "restroom", + 6: "balcony", + 7: "closet", + 8: "corridor", + 9: "washing_room", + 10: "PS", + 11: "outside", + # 12: 'wall' +} + + +def plot_room_map(preds, room_map, room_id=0, im_size=256, plot_text=True): + """Draw room polygons overlaid on the density map""" + centroid_x = int(np.mean(preds[:, 0])) + centroid_y = int(np.mean(preds[:, 1])) + + # Get text size to create a background box + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 0.3 + thickness = 1 + text = str(room_id) + (text_width, text_height), baseline = cv2.getTextSize(text, font, font_scale, thickness) + border_color = (252, 252, 0) + + for i, corner in enumerate(preds): + if i == len(preds) - 1: + cv2.line( + room_map, + (round(corner[0]), round(corner[1])), + (round(preds[0][0]), round(preds[0][1])), + border_color, + 2, + ) + else: + cv2.line( + room_map, + (round(corner[0]), round(corner[1])), + (round(preds[i + 1][0]), round(preds[i + 1][1])), + border_color, + 2, + ) + cv2.circle(room_map, (round(corner[0]), round(corner[1])), 2, (0, 0, 255), 2) + # cv2.putText(room_map, str(i), (round(corner[0]), round(corner[1])), cv2.FONT_HERSHEY_SIMPLEX, + # 0.4, (0, 255, 0), 1, cv2.LINE_AA) + + # Draw white background box with transparency + # overlay = room_map.copy() + # cv2.addWeighted(overlay, 0.7, room_map, 0.3, 0, room_map) # 70% opacity + + # Draw text + if plot_text: + cv2.rectangle( + room_map, + (centroid_x - text_width // 2 - 2, centroid_y - text_height // 2 - 2), + (centroid_x + text_width // 2 + 2, centroid_y + text_height // 2 + 2), + (255, 255, 255), # (0, 0, 0), + -1, + ) # Filled rectangle + cv2.putText( + room_map, + text, + (centroid_x - text_width // 2, centroid_y + text_height // 2), + font, + font_scale, + (0, 100, 0), + thickness, + ) + + return room_map + + +def plot_density_map(sample, image_size, room_polys, pred_room_label_per_scene, plot_text=True): + if not isinstance(sample, np.ndarray): + density_map = np.transpose(sample.cpu().numpy(), [1, 2, 0]) + # # Convert to grayscale if not already + # if density_map.shape[2] > 1: + # density_map = cv2.cvtColor(density_map, cv2.COLOR_RGB2GRAY)[:, :, np.newaxis] + else: + density_map = sample + if density_map.shape[2] == 3: + density_map = density_map * (image_size - 1) + else: + density_map = np.repeat(density_map, 3, axis=2) * (image_size - 1) + pred_room_map = np.zeros([image_size, image_size, 3]) + + for room_poly, room_id in zip(room_polys, pred_room_label_per_scene): + pred_room_map = plot_room_map( + np.array(room_poly), pred_room_map, room_id, im_size=image_size, plot_text=plot_text + ) + + alpha = 0.4 # Adjust for desired transparency + pred_room_map = cv2.addWeighted(density_map.astype(np.uint8), alpha, pred_room_map.astype(np.uint8), 1 - alpha, 0) + return pred_room_map + + +def is_clockwise(points): + # points is a list of 2d points. + assert len(points) > 0 + s = 0.0 + for p1, p2 in zip(points, points[1:] + [points[0]]): + s += (p2[0] - p1[0]) * (p2[1] + p1[1]) + return s > 0.0 + + +def resort_corners(corners): + # re-find the starting point and sort corners clockwisely + x_y_square_sum = corners[:, 0] ** 2 + corners[:, 1] ** 2 + start_corner_idx = np.argmin(x_y_square_sum) + + corners_sorted = np.concatenate([corners[start_corner_idx:], corners[:start_corner_idx]]) + + ## sort points clockwise + if not is_clockwise(corners_sorted[:, :2].tolist()): + corners_sorted[1:] = np.flip(corners_sorted[1:], 0) + + return corners + + +def create_coco_bounding_box(bb_x, bb_y, image_width, image_height, bound_pad=2): + bb_x = np.unique(bb_x) + bb_y = np.unique(bb_y) + bb_x_min = np.maximum(np.min(bb_x) - bound_pad, 0) + bb_y_min = np.maximum(np.min(bb_y) - bound_pad, 0) + + bb_x_max = np.minimum(np.max(bb_x) + bound_pad, image_width - 1) + bb_y_max = np.minimum(np.max(bb_y) + bound_pad, image_height - 1) + + bb_width = bb_x_max - bb_x_min + bb_height = bb_y_max - bb_y_min + + coco_bb = [bb_x_min, bb_y_min, bb_width, bb_height] + return coco_bb + + +def prepare_dict(): + save_dict = {"images": [], "annotations": [], "categories": []} + for key, value in ID2CLASS.items(): + if key == 0: + continue + type_dict = {"supercategory": "room", "id": key, "name": value} + save_dict["categories"].append(type_dict) + return save_dict + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset_path", + type=str, + required=True, + help="Path to the dataset directory", + ) + parser.add_argument( + "--output_dir", + type=str, + required=True, + help="Path to the dataset directory", + ) + # Add more arguments as needed + return parser + + +def visualize_room_polygons(room_polygons, room_classes, image_size=512, save_path="cubicasa_debug.png"): + """ + Visualize the extracted room polygons. + + Args: + room_polygons: Dictionary of room polygons as returned by extract_room_polygons + figsize: Figure size for the plot + """ + # Set figure size to exactly 256x256 pixels + dpi = 100 # Standard screen DPI + figsize = (image_size / dpi, image_size / dpi) # Convert pixels to inches + class_names = [v for k, v in ID2CLASS.items()] + + # Get unique classes from the mask + unique_classes = list(ID2CLASS.keys()) + + # Create a discrete colormap + cmap = plt.cm.get_cmap("gist_ncar", 256) # nipy_spectral + norm = np.linspace(0, 1, 13) # int(max(unique_classes))+1 + + fig = plt.figure(figsize=figsize, dpi=dpi) + ax = fig.add_axes([0, 0, 1, 1]) + ax.set_xlim(0, image_size) + ax.set_ylim(0, image_size) + ax.set_aspect("equal") + ax.axis("off") + + # Plot each room polygon and fill with color + for polygon, room_cls in zip(room_polygons, room_classes): + polygon_array = np.array(polygon).copy() + polygon_array[:, 1] = image_size - 1 - polygon_array[:, 1] # flip + # Fill the polygon with its class color + color = cmap(norm[int(room_cls)]) + ax.fill(polygon_array[:, 0], polygon_array[:, 1], color=color, alpha=0.4, zorder=1) + # Draw the polygon border + ax.plot(polygon_array[:, 0], polygon_array[:, 1], "k-", linewidth=2, zorder=2) + + # Add room ID label at the centroid + centroid_x = np.mean(polygon_array[:, 0]) + centroid_y = np.mean(polygon_array[:, 1]) + ax.text( + centroid_x, + centroid_y, + str(room_cls), + fontsize=12, + ha="center", + va="center", + bbox=dict(facecolor="white", alpha=0.7), + zorder=3, + ) + + # Create custom legend elements + legend_elements = [] + for i, cls in enumerate(sorted(unique_classes)): + color = cmap(norm[int(cls)]) + cls_name = f"{int(cls)}_{class_names[int(cls)]}" + legend_elements.append(Patch(facecolor=color, edgecolor="black", label=f"{cls_name}", alpha=0.6)) + ax.legend( + handles=legend_elements, + loc="best", + title="Classes", + fontsize=10, + markerscale=1, + title_fontsize=12, + framealpha=0.5, + ) + + plt.tight_layout(pad=0) + fig.savefig(save_path, bbox_inches="tight", pad_inches=0) + plt.close() + + +def process_floorplan(image_set, split, source_data_path, save_dir, save_aux_dir, vis_fp=False): + img, target = image_set + img = img * torch.tensor(std)[:, None, None] + torch.tensor(mean)[:, None, None] # unnormalize + graph = tensors_to_graphs_batch([target["graph"]]) + del target["graph"] + + tgt_this_preds = [] + tgt_this_edges = [] + for _ in range(len(target["points"])): + tgt_p_d = {} + tgt_p_d["scores"] = torch.tensor(1.0000, device="cpu") + tgt_p_d["points"] = target["unnormalized_points"][_] + tgt_p_d["edges"] = target["edges"][_] + tgt_p_d["size"] = target["size"] + if "semantic_left_up" in target: + tgt_p_d["semantic_left_up"] = target["semantic_left_up"][_] + tgt_p_d["semantic_right_up"] = target["semantic_right_up"][_] + tgt_p_d["semantic_right_down"] = target["semantic_right_down"][_] + tgt_p_d["semantic_left_down"] = target["semantic_left_down"][_] + tgt_this_preds.append(tgt_p_d) + for __ in range(4): + adj = graph[0][tuple(tgt_p_d["points"].tolist())][__] + if adj != (-1, -1): + tgt_p_d1 = tgt_p_d + tgt_p_d2 = {} + indx = 99999 + for ___, up in enumerate(target["unnormalized_points"].tolist()): + if abs(up[0] - adj[0]) + abs(up[1] - adj[1]) <= 2: + indx = ___ + break + # assert indx != 99999 + if indx == 99999: # No match found + # Log a warning or skip this iteration + print(f"Warning: No match found for adj {adj}") + continue # Skip to the next iteration + # tgt_p_d2['scores'] = torch.tensor(1.0000, device='cuda:0') + tgt_p_d2["points"] = target["unnormalized_points"][indx] + tgt_p_d2["edges"] = target["edges"][indx] + tgt_p_d2["size"] = target["size"] + if "semantic_left_up" in target: + tgt_p_d2["semantic_left_up"] = target["semantic_left_up"][indx] + tgt_p_d2["semantic_right_up"] = target["semantic_right_up"][indx] + tgt_p_d2["semantic_right_down"] = target["semantic_right_down"][indx] + tgt_p_d2["semantic_left_down"] = target["semantic_left_down"][indx] + tgt_e_l = (tgt_p_d1, tgt_p_d2) + if not edge_inside((tgt_p_d2, tgt_p_d1), tgt_this_edges): + tgt_this_edges.append(tgt_e_l) + tgt = [(tgt_this_preds, [], tgt_this_edges)] + target_d_rev, target_simple_cycles, target_results = get_cycle_basis_and_semantic((2, 999999, tgt)) + + # convert to coco format + polys_list = [] + polys_semantic_list = [] + output_json = [] + + image_width, image_height = target["size"][0].item(), target["size"][1].item() + filename = target["file_name"].split(".")[0] + img_id = int(target["image_id"]) + + img_dict = {} + img_dict["file_name"] = str(img_id).zfill(6) + ".png" + img_dict["id"] = img_id + img_dict["width"] = image_width + img_dict["height"] = image_height + save_dict = prepare_dict() + + os.makedirs(os.path.join(save_dir, split), exist_ok=True) + os.makedirs(f"{save_dir}/{split}_jsons/", exist_ok=True) + json_path = f"{save_dir}/{split}_jsons/{str(img_id).zfill(6)}.json" + + for instance_id, (poly, poly_cls) in enumerate(zip(target_simple_cycles, target_results)): + t = [(int(pt[0]), int(pt[1])) for pt in poly] + class_id = int(poly_cls) + + polys_list.append(t) + polys_semantic_list.append(class_id) + + poly_shapely = Polygon(t) + area = poly_shapely.area + coco_seg_poly = [] + polygon = np.array(t) + poly_sorted = resort_corners(polygon) + + for p in poly_sorted: + coco_seg_poly += list(p) + + if area < 100: + continue + + if class_id not in ID2CLASS: + print(f"Warning: Class ID {class_id} not found in ID2CLASS mapping. Skipping instance.") + continue + + # Slightly wider bounding box + rectangle_shapely = poly_shapely.envelope + bb_x, bb_y = rectangle_shapely.exterior.xy + coco_bb = create_coco_bounding_box(bb_x, bb_y, image_width, image_height, bound_pad=2) + + output_json.append( + { + "image_id": img_id, + "segmentation": [coco_seg_poly], + "category_id": class_id, + "id": instance_id, + "area": area, + "bbox": coco_bb, + "iscrowd": 0, + } + ) + + if vis_fp: + visualize_room_polygons( + polys_list, + polys_semantic_list, + image_size=image_width, + save_path=os.path.join(save_aux_dir, str(img_id).zfill(6) + ".png"), + ) + room_map = plot_density_map( + img, + image_width, + polys_list, + polys_semantic_list, + plot_text=False, + ) + cv2.imwrite(os.path.join(save_aux_dir, str(img_id).zfill(6) + "_density_map.png"), room_map) + + print(f"Processed image {img_id} with {len(output_json)} instances.") + # print(f"Class: {target_results}") + # min_class_id = min(target_results) + # max_class_id = max(target_results) + # if max_class_id == 12: + # breakpoint() + # print(f"Min class ID: {min_class_id}, Max class ID: {max_class_id}") + save_dict["images"].append(img_dict) + save_dict["annotations"] += output_json + with open(json_path, "w") as json_file: + # Convert all numpy and torch types to native Python types for JSON serialization + def convert(o): + if isinstance(o, (np.integer, np.int32, np.int64)): + return int(o) + if isinstance(o, (np.floating, np.float32, np.float64)): + return float(o) + if isinstance(o, (np.ndarray,)): + return o.tolist() + if isinstance(o, torch.Tensor): + return o.item() if o.numel() == 1 else o.tolist() + return str(o) + + json.dump(save_dict, json_file, default=convert) + + # rename image file + shutil.copy( + os.path.join(source_data_path, split, filename + ".png"), + os.path.join(save_dir, split, str(img_id).zfill(6) + ".png"), + ) + + # Write mapping from source file name to target file name (safe for parallel) + mapping_line = f"{filename} {str(img_id).zfill(6)}\n" + # Each process writes to its own temp file + pid = os.getpid() + os.makedirs(os.path.join(save_dir, f"{split}_logs"), exist_ok=True) + mapping_file = os.path.join(save_dir, f"{split}_logs", f"{split}_file_mapping_{pid}.txt") + with open(mapping_file, "a") as f: + f.write(mapping_line) + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + torch.set_printoptions(threshold=np.inf, linewidth=999999) + np.set_printoptions(threshold=np.inf, linewidth=999999) + gc.collect() + torch.cuda.empty_cache() + + def wrapper(scene_id): + try: + image_set = dataset[scene_id] + except Exception as e: + print(f"Error processing scene {scene_id}: {e}. Skipping...") + return + process_floorplan(image_set, split, args.dataset_path, args.output_dir, save_aux_dir, vis_fp=scene_id < 100) + + def worker_init(dataset_obj): + # Store dataset as global to avoid pickling issues + global dataset + dataset = dataset_obj + + splits = ["train", "val", "test"] + for split in splits: + dataset = MyDataset( + args.dataset_path + f"/{split}", + args.dataset_path + "/annot_json" + f"/instances_{split}.json", + extract_roi=False, + ) + + save_aux_dir = os.path.join(args.output_dir, f"{split}_aux") + os.makedirs(save_aux_dir, exist_ok=True) + + # for i, image_set in enumerate(tqdm(dataset)): + # save_aux_dir = os.path.join(args.output_dir, f"{split}_aux") + # os.makedirs(save_aux_dir, exist_ok=True) + # process_floorplan(image_set, split, args.dataset_path, args.output_dir, save_aux_dir, vis_fp=i < 100) + + num_processes = 16 + with Pool(num_processes, initializer=worker_init, initargs=(dataset,)) as p: + indices = range(len(dataset)) + list(tqdm(p.imap(wrapper, indices), total=len(dataset))) diff --git a/data_preprocess/raster2graph/dataset.py b/data_preprocess/raster2graph/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..9379b090e2438f169d6e0119acc53c671298e26e --- /dev/null +++ b/data_preprocess/raster2graph/dataset.py @@ -0,0 +1,296 @@ +import copy +import json +import os +from collections import defaultdict + +import numpy as np +import torch +import torch.multiprocessing +import torch.utils.data +import torchvision.transforms.functional as F +from PIL import Image +from torch.utils.data import Dataset +from util.data_utils import l1_dist +from util.graph_utils import graph_to_tensor +from util.image_id_dict import d +from util.mean_std import mean, std +from util.semantics_dict import semantics_dict + +torch.multiprocessing.set_sharing_strategy("file_system") + + +class MyDataset(Dataset): + def __init__(self, img_path, annot_path, extract_roi, image_size=512): + self.img_path = img_path + self.quadtree_path = "/".join(img_path.split("/")[:-1]) + "/annot_npy" + self.mode = img_path.split("/")[-1] + self.image_size = image_size + + # load annotation + with open(annot_path, "r") as f: + dataset = json.load(f) + # images + self.imgs = {} + for img in dataset["images"]: + self.imgs[img["id"]] = img + self.imgToAnns = defaultdict(list) + for ann in dataset["annotations"]: + self.imgToAnns[ann["image_id"]].append(ann) + self.ids = list(sorted(self.imgs.keys())) + if "0c-10-c468a57377ff8ef63d3b26a6d1fa-0002" in self.ids: + self.ids.remove("0c-10-c468a57377ff8ef63d3b26a6d1fa-0002") + if "0c-10-8486f08035ba152d5244ac54099c-0001" in self.ids: + self.ids.remove("0c-10-8486f08035ba152d5244ac54099c-0001") + + def __getitem__(self, index): + img_id = self.ids[index] + img_file_name = self.imgs[img_id]["file_name"].replace(".jpg", ".png") + img = Image.open(os.path.join(self.img_path, img_file_name)).convert("RGB") + image_scale = self.image_size / img.size[0] + if img.size[0] != self.image_size or img.size[1] != self.image_size: + img = img.resize((self.image_size, self.image_size), Image.BILINEAR) + + if 1: + # get structure annotations + anns = self.imgToAnns[img_id] + new_anns = [] + for ann in anns: + new_ann = copy.deepcopy(ann) + new_ann["point"] = [int(ann["point"][0] * image_scale), int(ann["point"][1] * image_scale)] + new_anns.append(new_ann) + target = {"image_id": img_id, "annotations": new_anns} + orig_quadtree = np.load( + os.path.join(self.quadtree_path, img_file_name[:-4] + ".npy"), allow_pickle=True + ).item()["quatree"][0] + quadtree = {} + for k, v in orig_quadtree.items(): + new_k = k + new_v = [] + for pos in v: + new_pos = (int(pos[0] * image_scale), int(pos[1] * image_scale)) + new_v.append(new_pos) + quadtree[new_k] = new_v + + orig_graph = np.load( + os.path.join(self.quadtree_path, img_file_name[:-4] + ".npy"), allow_pickle=True + ).item() + del orig_graph["quatree"] + new_graph = {} + for k, v in orig_graph.items(): + new_k = (int(k[0] * image_scale), int(k[1] * image_scale)) + new_v = [] + for adj in v: + if adj == (-1, -1): + new_v.append((-1, -1)) + else: + new_v.append((int(adj[0] * image_scale), int(adj[1] * image_scale))) + new_graph[new_k] = new_v + + target_layers = [] + for layer, layer_points in quadtree.items(): + target_layer = [] + for layer_point in layer_points: + for target_i in target["annotations"]: + if l1_dist(target_i["point"], list(layer_point)) <= 2: + target_layer.append(target_i) + break + target_layers.extend(target_layer) + layer_indices = [] + count = 0 + for k, v in quadtree.items(): + if k == 0: + layer_indices.append(0) + else: + layer_indices.append(count) + count += len(v) + + image_id = torch.tensor([d[img_id]]) + + points = [obj["point"] for obj in target_layers] + points = torch.as_tensor(points, dtype=torch.int64).reshape(-1, 2) + edges = [obj["edge_code"] for obj in target_layers] + edges = torch.tensor(edges, dtype=torch.int64) + + # get semantic annotations + semantic_left_up = [semantics_dict[obj["semantic"][0]] for obj in target_layers] + semantic_right_up = [semantics_dict[obj["semantic"][1]] for obj in target_layers] + semantic_right_down = [semantics_dict[obj["semantic"][2]] for obj in target_layers] + semantic_left_down = [semantics_dict[obj["semantic"][3]] for obj in target_layers] + semantic_left_up = torch.tensor(semantic_left_up, dtype=torch.int64) + semantic_right_up = torch.tensor(semantic_right_up, dtype=torch.int64) + semantic_right_down = torch.tensor(semantic_right_down, dtype=torch.int64) + semantic_left_down = torch.tensor(semantic_left_down, dtype=torch.int64) + + # annotations + target = {} + target["edges"] = edges + target["file_name"] = img_file_name + target["image_id"] = image_id + target["size"] = torch.as_tensor([img.size[1], img.size[0]]) + + target["semantic_left_up"] = semantic_left_up + target["semantic_right_up"] = semantic_right_up + target["semantic_right_down"] = semantic_right_down + target["semantic_left_down"] = semantic_left_down + + # get image + img = F.to_tensor(img) + img = F.normalize(img, mean=mean, std=std) + target["unnormalized_points"] = points + # normalize + points = points / torch.tensor([img.shape[2], img.shape[1]], dtype=torch.float32) + target["points"] = points + target["layer_indices"] = torch.tensor(layer_indices) + + target["graph"] = graph_to_tensor(new_graph) + + return img, target + + def __len__(self): + return len(self.ids) + + +class MyDataset2(Dataset): + def __init__(self, img_path, annot_path, extract_roi, disable_sem_info=False): + self.disable_sem_info = disable_sem_info + self.img_path = img_path + self.quadtree_path = "/".join(img_path.split("/")[:-1]) + "/annotations_npy/" + img_path.split("/")[-1] + self.edgecode_path = "/".join(img_path.split("/")[:-1]) + "/annotations_edge/" + img_path.split("/")[-1] + self.mode = img_path.split("/")[-1] + + available_ids = {int(x.replace(".npy", "")) for x in os.listdir(self.quadtree_path)} + + # load annotation + with open(annot_path, "r") as f: + dataset = json.load(f) + # images + self.imgs = {} + for img in dataset["images"]: + if img["id"] not in available_ids: + continue + self.imgs[img["id"]] = img + self.imgToAnns = defaultdict(list) + for ann in dataset["annotations"]: + if ann["image_id"] not in available_ids: + continue + self.imgToAnns[ann["image_id"]].append(ann) + self.ids = list(sorted(self.imgs.keys())) + + def __getitem__(self, index): + img_id = self.ids[index] + img_file_name = self.imgs[int(img_id)]["file_name"] + img = Image.open(os.path.join(self.img_path, img_file_name)).convert("RGB") + + if 1: + # get structure annotations + # anns = self.imgToAnns[int(img_id)] + + data = np.load(os.path.join(self.quadtree_path, img_file_name[:-4] + ".npy"), allow_pickle=True).item() + orig_quadtree = data["quadtree"] + orig_graph = data["graph"] + image_points = data["points"] + + new_anns = [] + for pt in image_points: + new_ann = { + "point": [int(pt[0]), int(pt[1])], + } + new_anns.append(new_ann) + target = {"image_id": img_id, "annotations": new_anns} + + quadtree = {} + for k, v in orig_quadtree.items(): + new_k = k + new_v = [] + for pos in v: + new_pos = (int(pos[0]), int(pos[1])) + new_v.append(new_pos) + quadtree[new_k] = new_v + + new_graph = {} + for k, v in orig_graph.items(): + new_k = (int(k[0]), int(k[1])) + new_v = [] + for adj in v: + if adj == (-1, -1): + new_v.append((-1, -1)) + else: + new_v.append((int(adj[0]), int(adj[1]))) + new_graph[new_k] = new_v + + target_layers = [] + for layer, layer_points in quadtree.items(): + target_layer = [] + for layer_point in layer_points: + for target_i in target["annotations"]: + if l1_dist(target_i["point"], list(layer_point)) <= 2: + target_layer.append(target_i) + break + target_layers.extend(target_layer) + layer_indices = [] + count = 0 + for k, v in quadtree.items(): + if k == 0: + layer_indices.append(0) + else: + layer_indices.append(count) + count += len(v) + + image_id = torch.tensor([int(img_id)]) + + points = [obj["point"] for obj in target_layers] + with open(os.path.join(self.edgecode_path, img_file_name[:-4] + ".json"), "r") as f: + edge2code = json.load(f) + edge2code = { + tuple(map(lambda x: int(float(x)), key.strip("()").split(", "))): value + for key, value in edge2code.items() + } + + edges = [edge2code[(int(pt[0]), int(pt[1]))] for pt in points] + points = torch.as_tensor(points, dtype=torch.int64).reshape(-1, 2) + edges = torch.tensor(edges, dtype=torch.int64) + + # annotations + target = {} + target["edges"] = edges + target["image_id"] = image_id + target["file_name"] = img_file_name + target["size"] = torch.as_tensor([img.size[1], img.size[0]]) + + # get semantic annotations + if not self.disable_sem_info: + semantic_left_up = [semantics_dict[obj["semantic"][0]] for obj in target_layers] + semantic_right_up = [semantics_dict[obj["semantic"][1]] for obj in target_layers] + semantic_right_down = [semantics_dict[obj["semantic"][2]] for obj in target_layers] + semantic_left_down = [semantics_dict[obj["semantic"][3]] for obj in target_layers] + semantic_left_up = torch.tensor(semantic_left_up, dtype=torch.int64) + semantic_right_up = torch.tensor(semantic_right_up, dtype=torch.int64) + semantic_right_down = torch.tensor(semantic_right_down, dtype=torch.int64) + semantic_left_down = torch.tensor(semantic_left_down, dtype=torch.int64) + + target["semantic_left_up"] = semantic_left_up + target["semantic_right_up"] = semantic_right_up + target["semantic_right_down"] = semantic_right_down + target["semantic_left_down"] = semantic_left_down + + # get image + img = F.to_tensor(img) + img = F.normalize(img, mean=mean, std=std) + target["unnormalized_points"] = points + # normalize + points = points / torch.tensor([img.shape[2], img.shape[1]], dtype=torch.float32) + target["points"] = points + target["layer_indices"] = torch.tensor(layer_indices) + + # padding (-1,-1) if not enough 4 neighbors + for pt, neighbors in new_graph.items(): + if len(neighbors) < 4: + new_graph[pt].extend([(-1, -1)] * (4 - len(neighbors))) + elif len(neighbors) > 4: + new_graph[pt] = neighbors[:4] + target["graph"] = graph_to_tensor(new_graph) + + return img, target + + def __len__(self): + return len(self.ids) diff --git a/data_preprocess/raster2graph/image_process.py b/data_preprocess/raster2graph/image_process.py new file mode 100644 index 0000000000000000000000000000000000000000..726554b2d6199249085010f7d1bfff11a8d94e5a --- /dev/null +++ b/data_preprocess/raster2graph/image_process.py @@ -0,0 +1,67 @@ +import argparse +import json +import os + +import numpy as np +from PIL import Image +from tqdm import tqdm + +parser = argparse.ArgumentParser("Preprocess LIFULL HOMES DATA (HIGH RESOLUTION) Dataset") +parser.add_argument("--data_root", type=str, default=r"R2G_hr_dataset/", help="path to the root folder of the dataset") +args = parser.parse_args() + +SIZE = 512 +MARGIN = 64 +np.set_printoptions(threshold=np.inf, linewidth=999999) + +# original_images_path = r'E:/LIFULL HOMES DATA (HIGH RESOLUTION)/photo-rent-madori-full-00' +original_images_path = args.data_root + +with open(f"{args.data_root}/annot_json/instances_train.json", mode="r") as f_train: + train_jpgs = [_["file_name"] for _ in json.load(f_train)["images"]] +with open(f"{args.data_root}/annot_json/instances_val.json", mode="r") as f_val: + val_jpgs = [_["file_name"] for _ in json.load(f_val)["images"]] +with open(f"{args.data_root}/instances_test.json", mode="r") as f_test: + test_jpgs = [_["file_name"] for _ in json.load(f_test)["images"]] +jpgs = {"train": train_jpgs, "val": val_jpgs, "test": test_jpgs} + +start_idx = 0 +for mode in ["train", "val", "test"]: + output_dir = "./" + mode + os.makedirs(output_dir, exist_ok=True) + for fnames in [jpgs[mode]]: + for i in tqdm(range(len(fnames))): + fn = fnames[i].replace(".jpg", "") + if os.path.exists(os.path.join(f"{args.data_root}/annot_npy", fn + ".npy")) and os.path.exists( + os.path.join(f"{args.data_root}/original_vector_boundary", fn + ".npy") + ): + img_original = Image.open(os.path.join(original_images_path, fn.replace("-", "/") + ".jpg")) + boundary_path = os.path.join(f"{args.data_root}/original_vector_boundary", fn + ".npy") + boundary = np.load(boundary_path, allow_pickle=True).item() + x_min = boundary["x_min"] + x_max = boundary["x_max"] + y_min = boundary["y_min"] + y_max = boundary["y_max"] + width = x_max - x_min + mid_width = (x_max + x_min) / 2 + height = y_max - y_min + mid_height = (y_max + y_min) / 2 + if width > height: + scale = (SIZE - 2 * MARGIN) / width + else: + scale = (SIZE - 2 * MARGIN) / height + # print(x_min, y_min, x_max, y_max, width, height, scale) + + original_width, original_height = img_original.size + new_width = int(original_width * scale) + new_height = int(original_height * scale) + scaled_image = img_original.resize((new_width, new_height), Image.Resampling.LANCZOS) + canvas = Image.new("RGB", (512, 512), (255, 255, 255)) + # print(new_width, new_height) + x_topleft_offset = int(512 / 2 - mid_width * scale) + y_topleft_offset = int(512 / 2 - mid_height * scale) + canvas.paste(scaled_image, (x_topleft_offset, y_topleft_offset)) + + canvas.save(os.path.join(output_dir, fn + ".png")) + + start_idx += 1 diff --git a/data_preprocess/raster2graph/util/data_utils.py b/data_preprocess/raster2graph/util/data_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fcfc27a2095952bbd754f810abda24cacb40cde2 --- /dev/null +++ b/data_preprocess/raster2graph/util/data_utils.py @@ -0,0 +1,966 @@ +import copy +import random + +import torch +from util.edges_utils import get_edges_alldirections_rev +from util.math_utils import clip +from util.mean_std import mean, std + + +def data_to_cuda(samples, targets): + return samples.to(torch.device("cuda")), [ + {k: v if isinstance(v, str) else v.to(torch.device("cuda")) for k, v in t.items()} for t in targets + ] + + +def get_random_layer_targets(targets, gt_layer): + random_targets = [] + for batch_i, target_i in enumerate(targets): + random_layer_targets_i = copy.deepcopy(target_i) + if gt_layer[batch_i] != len(random_layer_targets_i["layer_indices"]) - 1: + start = random_layer_targets_i["layer_indices"][gt_layer[batch_i]].item() + end = random_layer_targets_i["layer_indices"][gt_layer[batch_i] + 1].item() + else: + start = random_layer_targets_i["layer_indices"][gt_layer[batch_i]].item() + end = len(random_layer_targets_i["points"]) + random_points_i = random_layer_targets_i["points"][start:end, :] + random_edges_i = random_layer_targets_i["edges"][start:end] + random_unnormalized_points_i = random_layer_targets_i["unnormalized_points"][start:end, :] + random_layer_targets_i["points"] = random_points_i + random_layer_targets_i["edges"] = random_edges_i + random_layer_targets_i["unnormalized_points"] = random_unnormalized_points_i + del random_layer_targets_i["layer_indices"] + random_targets.append(random_layer_targets_i) + return random_targets + + +def random_layers(targets): + return [random.randint(0, len(targets[i]["layer_indices"]) - 1) for i in range(len(targets))] + + +def get_given_layers_random_region(targets, graphs): + random_regions = [] + for bs_i in range(len(targets)): + # target + targets_i = targets[bs_i] + graphs_i = graphs[bs_i] + # level 0: start + start_i = tuple(targets_i["unnormalized_points"][0].tolist()) + + # sampled prob: for a neighborhood, each node is sampled by this probability + # sampled_prob = 0.0001 + # sampled_prob = random.random() + sampled_prob = 0.5 + # sampled_prob = 1 + + # sampled nodes + sampled_points = {} + for point_tensor in targets_i["unnormalized_points"]: + pos = tuple(point_tensor.tolist()) + sampled_points[pos] = 0 + # edges of sampled nodes + sampled_edges = [] + + # nodes number of subgraph + # sampled_amount = random.randint(0, len(sampled_points) + 2) + # if sampled_amount in [len(sampled_points) + 1]: + # sampled_amount = 0 + # if sampled_amount in [len(sampled_points) + 2]: + # sampled_amount = len(sampled_points) + sampled_amount = random.randint(0, len(sampled_points)) # TODO: 1~len(sampled_points) + + # Note that when sampled_prob = 1, the number of sampled nodes must be in 'layer_indices' or be the total number of points to ensure that the entire layers is sampled. + # equal to BFS + if sampled_prob == 1: + l = targets_i["layer_indices"].tolist() + l.append(len(sampled_points)) + l.append(0) + l.append(len(sampled_points)) + sampled_amount = l[random.randint(0, len(l) - 1)] + + # start sampling + if sampled_amount == 0: + random_regions.append((sampled_points, sampled_edges)) + continue + sampled_points[start_i] = 1 + if sampled_amount == 1: + random_regions.append((sampled_points, sampled_edges)) + continue + + max_iterations = max(1000, 10 * sampled_amount) # Ensure at least 1000 iterations + iteration_count = 0 + while sum(sampled_points.values()) < sampled_amount: + iteration_count += 1 + if iteration_count > max_iterations: + print("Reached maximum iterations, breaking to avoid infinite loop.") + break + all_sampled_points = set([k for k, v in sampled_points.items() if v == 1]) + all_sampled_points_adjs = set() + for sampled_point in all_sampled_points: + adj = set([(int(x[0]), int(x[1])) for x in graphs_i[sampled_point]]) + all_sampled_points_adjs = all_sampled_points_adjs.union(adj) + + if (-1, -1) in all_sampled_points_adjs: + all_sampled_points_adjs.remove((-1, -1)) + all_sampled_points_adjs = list(all_sampled_points_adjs.difference(all_sampled_points)) + + if not all_sampled_points_adjs: # If no more adjacent points to sample, break + print("No more adjacent points to sample, breaking the loop.") + break + + # shuffle the last layer to let it uniform (no bias of sample order) + random.shuffle(all_sampled_points_adjs) + # determine whether to sample nodes in each neighborhood based on probability + for all_sampled_points_adj_index, all_sampled_points_adj in enumerate(all_sampled_points_adjs): + all_sampled_points = set([k for k, v in sampled_points.items() if v == 1]) + if sum(sampled_points.values()) == sampled_amount: + break + else: + if 1: + if random.random() < sampled_prob: + sampled_points[all_sampled_points_adj] = 1 + # sample edges + all_pos1s = graphs_i[all_sampled_points_adj] + pos2 = all_sampled_points_adj + for pos1 in all_pos1s: + if pos1 in all_sampled_points: + sampled_edges.append((pos1, pos2)) + else: + sampled_points[all_sampled_points_adj] = 0 + random_regions.append((sampled_points, sampled_edges)) + return random_regions + + +def get_random_region_targets(given_layers, graphs, targets): + random_region_targets = [] + for bs_i in range(len(targets)): + random_region_target = {} + targets_i = targets[bs_i] + graphs_i = graphs[bs_i] + given_layers_i = given_layers[bs_i] + sampled_points_i, sampled_edges_i = given_layers_i + + if sum(sampled_points_i.values()) == 0: + random_region_target["edges"] = targets_i["edges"][:1] + + if "semantic_left_up" in targets_i: + random_region_target["semantic_left_up"] = targets_i["semantic_left_up"][:1] + random_region_target["semantic_right_up"] = targets_i["semantic_right_up"][:1] + random_region_target["semantic_right_down"] = targets_i["semantic_right_down"][:1] + random_region_target["semantic_left_down"] = targets_i["semantic_left_down"][:1] + + random_region_target["image_id"] = targets_i["image_id"] + random_region_target["size"] = targets_i["size"] + random_region_target["unnormalized_points"] = targets_i["unnormalized_points"][:1] + random_region_target["points"] = targets_i["points"][:1] + random_region_target["last_edges"] = torch.zeros( + (1,), dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_target["this_edges"] = torch.zeros( + (1,), dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_targets.append(random_region_target) + elif 1 <= sum(sampled_points_i.values()) <= len(sampled_points_i) - 1: + sampled_points_i_given = set([k for k, v in sampled_points_i.items() if v == 1]) + unnormalized_points = [] + for point, sampled_or_not in sampled_points_i.items(): + if sampled_or_not == 0: + adjs = graphs_i[point] + for adj in adjs: + if adj in sampled_points_i_given: + unnormalized_points.append(point) + break + + if len(unnormalized_points) == 0: + random_region_target["edges"] = targets_i["edges"][:1] + + if "semantic_left_up" in targets_i: + random_region_target["semantic_left_up"] = targets_i["semantic_left_up"][:1] + random_region_target["semantic_right_up"] = targets_i["semantic_right_up"][:1] + random_region_target["semantic_right_down"] = targets_i["semantic_right_down"][:1] + random_region_target["semantic_left_down"] = targets_i["semantic_left_down"][:1] + + random_region_target["image_id"] = targets_i["image_id"] + random_region_target["size"] = targets_i["size"] + random_region_target["unnormalized_points"] = targets_i["unnormalized_points"][:1] + random_region_target["points"] = targets_i["points"][:1] + random_region_target["last_edges"] = torch.zeros( + (1,), dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_target["this_edges"] = torch.zeros( + (1,), dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_targets.append(random_region_target) + continue + + indices_for_semantic = [] + for unnormalized_point in unnormalized_points: + for ind, every_point in enumerate(targets_i["unnormalized_points"]): + every_point = tuple(every_point.tolist()) + if ( + abs(every_point[0] - unnormalized_point[0]) <= 2 + and abs(every_point[1] - unnormalized_point[1]) <= 2 + ): + indices_for_semantic.append(ind) + # assert len(unnormalized_points) == len(indices_for_semantic) + semantic_left_up = [] + semantic_right_up = [] + semantic_right_down = [] + semantic_left_down = [] + + if "semantic_left_up" in targets_i: + for ind in indices_for_semantic: + semantic_left_up.append(targets_i["semantic_left_up"][ind].item()) + semantic_right_up.append(targets_i["semantic_right_up"][ind].item()) + semantic_right_down.append(targets_i["semantic_right_down"][ind].item()) + semantic_left_down.append(targets_i["semantic_left_down"][ind].item()) + + edges = [] + for unnormalized_point in unnormalized_points: + edge = "" + adjs = graphs_i[unnormalized_point] + for adj in adjs: + if adj != (-1, -1): + edge += "1" + else: + edge += "0" + edge = get_edges_alldirections_rev(edge) + edges.append(edge) + last_edges = [] + for unnormalized_point in unnormalized_points: + last_edge = "" + adjs = graphs_i[unnormalized_point] + for adj in adjs: + if adj in sampled_points_i_given: + last_edge += "1" + else: + last_edge += "0" + last_edge = get_edges_alldirections_rev(last_edge) + last_edges.append(last_edge) + this_edges = [] + for unnormalized_point in unnormalized_points: + this_edge = "" + adjs = graphs_i[unnormalized_point] + for adj in adjs: + if adj in unnormalized_points: + this_edge += "1" + else: + this_edge += "0" + this_edge = get_edges_alldirections_rev(this_edge) + this_edges.append(this_edge) + + random_region_target["edges"] = torch.tensor( + edges, dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + + if "semantic_left_up" in targets_i: + random_region_target["semantic_left_up"] = torch.tensor( + semantic_left_up, + dtype=targets_i["semantic_left_up"].dtype, + device=targets_i["semantic_left_up"].device, + ) + random_region_target["semantic_right_up"] = torch.tensor( + semantic_right_up, + dtype=targets_i["semantic_right_up"].dtype, + device=targets_i["semantic_right_up"].device, + ) + random_region_target["semantic_right_down"] = torch.tensor( + semantic_right_down, + dtype=targets_i["semantic_right_down"].dtype, + device=targets_i["semantic_right_down"].device, + ) + random_region_target["semantic_left_down"] = torch.tensor( + semantic_left_down, + dtype=targets_i["semantic_left_down"].dtype, + device=targets_i["semantic_left_down"].device, + ) + + random_region_target["image_id"] = targets_i["image_id"] + random_region_target["size"] = targets_i["size"] + + # NEW + # if len(unnormalized_points) == 0: + # print("Warning: unnormalized_points is empty. Initializing to default value.") + # unnormalized_points = torch.zeros((1, 2), dtype=targets_i['unnormalized_points'].dtype, + # device=targets_i['unnormalized_points'].device) + # else: + # random_region_target['unnormalized_points'] = torch.tensor(unnormalized_points, + # dtype=targets_i['unnormalized_points'].dtype, + # device=targets_i['unnormalized_points'].device) + + random_region_target["unnormalized_points"] = torch.tensor( + unnormalized_points, + dtype=targets_i["unnormalized_points"].dtype, + device=targets_i["unnormalized_points"].device, + ) + random_region_target["points"] = ( + torch.tensor(unnormalized_points, dtype=targets_i["points"].dtype, device=targets_i["points"].device) + / targets_i["size"] + ) + random_region_target["last_edges"] = torch.tensor( + last_edges, dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_target["this_edges"] = torch.tensor( + this_edges, dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_targets.append(random_region_target) + else: + random_region_target["edges"] = 16 * torch.ones( + targets_i["edges"][:1].shape, dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + + if "semantic_left_up" in targets_i: + random_region_target["semantic_left_up"] = 11 * torch.ones( + targets_i["semantic_left_up"][:1].shape, + dtype=targets_i["semantic_left_up"].dtype, + device=targets_i["semantic_left_up"].device, + ) + random_region_target["semantic_right_up"] = 11 * torch.ones( + targets_i["semantic_right_up"][:1].shape, + dtype=targets_i["semantic_right_up"].dtype, + device=targets_i["semantic_right_up"].device, + ) + random_region_target["semantic_right_down"] = 11 * torch.ones( + targets_i["semantic_right_down"][:1].shape, + dtype=targets_i["semantic_right_down"].dtype, + device=targets_i["semantic_right_down"].device, + ) + random_region_target["semantic_left_down"] = 11 * torch.ones( + targets_i["semantic_left_down"][:1].shape, + dtype=targets_i["semantic_left_down"].dtype, + device=targets_i["semantic_left_down"].device, + ) + + random_region_target["image_id"] = targets_i["image_id"] + random_region_target["size"] = targets_i["size"] + random_region_target["unnormalized_points"] = 505 * torch.ones( + targets_i["unnormalized_points"][:1].shape, + dtype=targets_i["unnormalized_points"][:1].dtype, + device=targets_i["unnormalized_points"][:1].device, + ) + random_region_target["points"] = ( + 505 + * torch.ones( + targets_i["unnormalized_points"][:1].shape, + dtype=targets_i["points"][:1].dtype, + device=targets_i["points"][:1].device, + ) + ) / targets_i["size"] + random_region_target["last_edges"] = 16 * torch.ones( + (1,), dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_target["this_edges"] = 16 * torch.ones( + (1,), dtype=targets_i["edges"].dtype, device=targets_i["edges"].device + ) + random_region_targets.append(random_region_target) + + return random_region_targets + + +def random_pertubation(sampled_points_i, sampled_edges_i): + random_pertube_map = {} + sigma = 2 + pertube_threshold = 5 + for sampled_point in sampled_points_i: + random_pertube_map[sampled_point] = ( + sampled_point[0] + clip(int(random.gauss(0, sigma)), -1 * pertube_threshold, pertube_threshold), + sampled_point[1] + clip(int(random.gauss(0, sigma)), -1 * pertube_threshold, pertube_threshold), + ) + new_sampled_points_i = {} + new_sampled_edges_i = [] + for sampled_point in sampled_points_i: + new_sampled_points_i[random_pertube_map[sampled_point]] = sampled_points_i[sampled_point] + for pos1, pos2 in sampled_edges_i: + new_sampled_edges_i.append((random_pertube_map[pos1], random_pertube_map[pos2])) + return new_sampled_points_i, new_sampled_edges_i + + +def draw_given_layers_on_tensors_random_region(given_layers, tensors, graphs): + """draw 9*9 yellow squares and width 2 blue lines""" + tensors_list = [] + unnormalized_list = [] + for i in range(len(given_layers)): + temp_tensor = tensors[i] + + temp_tensor_0 = (temp_tensor[0] * std[0] + mean[0]) * 255 + temp_tensor_1 = (temp_tensor[1] * std[1] + mean[1]) * 255 + temp_tensor_2 = (temp_tensor[2] * std[2] + mean[2]) * 255 + + rectangle_radius = 5 + + # end sign + endsign = (505, 505) + valid_violet_endsign_up = endsign[1] - rectangle_radius + valid_violet_endsign_down = endsign[1] + rectangle_radius + valid_violet_endsign_left = endsign[0] - rectangle_radius + valid_violet_endsign_right = endsign[0] + rectangle_radius + temp_tensor_0[ + valid_violet_endsign_up : valid_violet_endsign_down + 1, + valid_violet_endsign_left : valid_violet_endsign_right + 1, + ] = 255 + temp_tensor_1[ + valid_violet_endsign_up : valid_violet_endsign_down + 1, + valid_violet_endsign_left : valid_violet_endsign_right + 1, + ] = 0 + temp_tensor_2[ + valid_violet_endsign_up : valid_violet_endsign_down + 1, + valid_violet_endsign_left : valid_violet_endsign_right + 1, + ] = 255 + + sampled_points_i, sampled_edges_i = given_layers[i] + sampled_points_i, sampled_edges_i = random_pertubation(sampled_points_i, sampled_edges_i) + + given_points = [k for k, v in sampled_points_i.items() if v == 1] + + for j, pos in enumerate(given_points): + valid_yellow_pos_up = int(pos[1] - rectangle_radius) if (pos[1] - rectangle_radius) >= 0 else 0 + valid_yellow_pos_down = ( + int(pos[1] + rectangle_radius) + if (pos[1] + rectangle_radius) < temp_tensor.shape[2] + else temp_tensor.shape[2] - 1 + ) + valid_yellow_pos_left = int(pos[0] - rectangle_radius) if (pos[0] - rectangle_radius) >= 0 else 0 + valid_yellow_pos_right = ( + int(pos[0] + rectangle_radius) + if (pos[0] + rectangle_radius) < temp_tensor.shape[1] + else temp_tensor.shape[1] - 1 + ) + + temp_tensor_0[ + valid_yellow_pos_up : valid_yellow_pos_down + 1, valid_yellow_pos_left : valid_yellow_pos_right + 1 + ] = 255 + temp_tensor_1[ + valid_yellow_pos_up : valid_yellow_pos_down + 1, valid_yellow_pos_left : valid_yellow_pos_right + 1 + ] = 255 + temp_tensor_2[ + valid_yellow_pos_up : valid_yellow_pos_down + 1, valid_yellow_pos_left : valid_yellow_pos_right + 1 + ] = 0 + + # draw blue lines + line_width = 2 + for edge in sampled_edges_i: + pos1 = (int(edge[0][0]), int(edge[0][1])) + pos2 = (int(edge[1][0]), int(edge[1][1])) + if abs(pos1[0] - pos2[0]) < abs(pos1[1] - pos2[1]): + if pos1[1] > pos2[1]: + temp_tensor_0[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_1[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_2[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 255 + else: + temp_tensor_0[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_1[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_2[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 255 + else: + if pos1[0] > pos2[0]: + temp_tensor_0[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 0 + temp_tensor_1[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 0 + temp_tensor_2[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 255 + else: + temp_tensor_0[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 0 + temp_tensor_1[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 0 + temp_tensor_2[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 255 + + unnormalized = torch.stack((temp_tensor_0, temp_tensor_1, temp_tensor_2), dim=0) + unnormalized_list.append(unnormalized) + + temp_tensor_0_renorm = ((temp_tensor_0 / 255) - mean[0]) / std[0] + temp_tensor_1_renorm = ((temp_tensor_1 / 255) - mean[1]) / std[1] + temp_tensor_2_renorm = ((temp_tensor_2 / 255) - mean[2]) / std[2] + + temp_tensor = torch.stack([temp_tensor_0_renorm, temp_tensor_1_renorm, temp_tensor_2_renorm], dim=0) + + tensors_list.append(temp_tensor) + + return torch.stack(tensors_list, dim=0), torch.stack(unnormalized_list, dim=0) + + +def initialize_tensors(tensors): + tensors_list = [] + unnormalized_list = [] + for i in range(len(tensors)): + temp_tensor = tensors[i] + + temp_tensor_0 = (temp_tensor[0] * std[0] + mean[0]) * 255 + temp_tensor_1 = (temp_tensor[1] * std[1] + mean[1]) * 255 + temp_tensor_2 = (temp_tensor[2] * std[2] + mean[2]) * 255 + + rectangle_radius = 5 # 4+1+4=9 + + # end sign (when predict this, AR iteration terminates) + endsign = (505, 505) + valid_violet_endsign_up = endsign[1] - rectangle_radius + valid_violet_endsign_down = endsign[1] + rectangle_radius + valid_violet_endsign_left = endsign[0] - rectangle_radius + valid_violet_endsign_right = endsign[0] + rectangle_radius + temp_tensor_0[ + valid_violet_endsign_up : valid_violet_endsign_down + 1, + valid_violet_endsign_left : valid_violet_endsign_right + 1, + ] = 255 + temp_tensor_1[ + valid_violet_endsign_up : valid_violet_endsign_down + 1, + valid_violet_endsign_left : valid_violet_endsign_right + 1, + ] = 0 + temp_tensor_2[ + valid_violet_endsign_up : valid_violet_endsign_down + 1, + valid_violet_endsign_left : valid_violet_endsign_right + 1, + ] = 255 + + unnormalized = torch.stack((temp_tensor_0, temp_tensor_1, temp_tensor_2), dim=0) + unnormalized_list.append(unnormalized) + + temp_tensor_0_renorm = ((temp_tensor_0 / 255) - mean[0]) / std[0] + temp_tensor_1_renorm = ((temp_tensor_1 / 255) - mean[1]) / std[1] + temp_tensor_2_renorm = ((temp_tensor_2 / 255) - mean[2]) / std[2] + + temp_tensor = torch.stack([temp_tensor_0_renorm, temp_tensor_1_renorm, temp_tensor_2_renorm], dim=0) + + tensors_list.append(temp_tensor) + + return torch.stack(tensors_list, dim=0), torch.stack(unnormalized_list, dim=0) + + +def l1_dist(pos1, pos2): + return abs(pos1[0] - pos2[0]) + abs(pos1[1] - pos2[1]) + + +def delete_graphs(targets): + no_graph_targets = [] + for target in targets: + target_ = copy.deepcopy(target) + del target_["graph"] + no_graph_targets.append(target_) + return no_graph_targets + + +def delete_graphs_and_unnormpoints(targets): + no_graph_targets = [] + for target in targets: + target_ = copy.deepcopy(target) + del target_["graph"] + del target_["unnormalized_points"] + no_graph_targets.append(target_) + return no_graph_targets + + +def get_remove_point(this_preds, dist_threshold): + for point1 in this_preds: + for point2 in this_preds: + # if point1 != point2: + if not ( + (point1["points"].tolist()[0] == point2["points"].tolist()[0]) + and (point1["points"].tolist()[1] == point2["points"].tolist()[1]) + ): + dist_chebyshev = max( + abs(point1["points"].tolist()[0] - point2["points"].tolist()[0]), + abs(point1["points"].tolist()[1] - point2["points"].tolist()[1]), + ) + if dist_chebyshev <= dist_threshold: + point1_confidence = point1["scores"].item() + point2_confidence = point2["scores"].item() + if point1_confidence < point2_confidence: + return point1 + elif point2_confidence < point1_confidence: + return point2 + else: + return [point1, point2][random.randint(0, 1)] + return None + + +def point_inside(point, points_list): + point1 = tuple(point["points"].tolist()) + for point_i in points_list: + point1_i = tuple(point_i["points"].tolist()) + if point1 == point1_i: + return True + return False + + +def remove_points(need_to_remove_in_last_edges, this_preds): + result = [] + for this_pred in this_preds: + if not point_inside(this_pred, need_to_remove_in_last_edges): + result.append(this_pred) + return result + + +def nms(this_preds): + if len(this_preds) <= 1: + return this_preds + else: + dist_threshold = 5 + while True: + remove_point = get_remove_point(this_preds, dist_threshold) + if remove_point is None: + break + else: + # this_preds.remove(remove_point) + this_preds = remove_points([remove_point], this_preds) + + return this_preds + + +def nms_givenpoints(this_preds, preds): + if len(this_preds) == 0: + return this_preds + else: + all_given_points = [] + for given_points, given_last_edges, given_this_edges in preds: + all_given_points.extend(given_points) + if len(all_given_points) == 0: + return this_preds + this_preds_copy = copy.deepcopy(this_preds) + dist_threshold = 5 + for this_pred in this_preds_copy: + for given_point in all_given_points: + this_pred_pos = tuple(this_pred["points"].tolist()) + given_point_pos = tuple(given_point["points"].tolist()) + dist_chebyshev = max( + abs(this_pred_pos[0] - given_point_pos[0]), abs(this_pred_pos[1] - given_point_pos[1]) + ) + if dist_chebyshev <= dist_threshold: + this_preds = remove_points([this_pred], this_preds) + break + return this_preds + + +def random_keep(this_preds): + if len(this_preds) <= 1: + return this_preds + else: + while True: + random_keep_this_preds = [] + for point in this_preds: + # is_keep = random.random() < point['scores'].item() + is_keep = random.random() < 1.01 + # is_keep = random.random() < 0.5 + if is_keep: + random_keep_this_preds.append(point) + if len(random_keep_this_preds) > 0: + return random_keep_this_preds + + +def is_stop(this_preds): + if len(this_preds) == 0: + return 1 # stop + elif (len(this_preds) >= 1) and (16 in [p["edges"].item() for p in this_preds]): + return 2 # normally terminate + else: + return 0 # not stop + + +def draw_preds_on_tensors(preds, tensors): + tensors_list = [] + unnormalized_list = [] + + for i in range(len(tensors)): + temp_tensor = tensors[i] + + temp_tensor_0 = (temp_tensor[0] * std[0] + mean[0]) * 255 + temp_tensor_1 = (temp_tensor[1] * std[1] + mean[1]) * 255 + temp_tensor_2 = (temp_tensor[2] * std[2] + mean[2]) * 255 + + rectangle_radius = 5 + + this_preds, last_edges, this_edges = preds[-1] + for this_pred in this_preds: + point = tuple([int(_) for _ in this_pred["points"].tolist()]) + up = point[1] - rectangle_radius + down = point[1] + rectangle_radius + left = point[0] - rectangle_radius + right = point[0] + rectangle_radius + temp_tensor_0[up : down + 1, left : right + 1] = 255 + temp_tensor_1[up : down + 1, left : right + 1] = 255 + temp_tensor_2[up : down + 1, left : right + 1] = 0 + line_width = 2 + for last_edge in last_edges: + pos1 = tuple([int(_) for _ in last_edge[0]["points"].tolist()]) + pos2 = tuple([int(_) for _ in last_edge[1]["points"].tolist()]) + if abs(pos1[0] - pos2[0]) < abs(pos1[1] - pos2[1]): + if pos1[1] > pos2[1]: + temp_tensor_0[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_1[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_2[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 255 + else: + temp_tensor_0[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_1[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_2[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 255 + else: + if pos1[0] > pos2[0]: + temp_tensor_0[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 0 + temp_tensor_1[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 0 + temp_tensor_2[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 255 + else: + temp_tensor_0[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 0 + temp_tensor_1[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 0 + temp_tensor_2[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 255 + for this_edge in this_edges: + pos1 = tuple([int(_) for _ in this_edge[0]["points"].tolist()]) + pos2 = tuple([int(_) for _ in this_edge[1]["points"].tolist()]) + if abs(pos1[0] - pos2[0]) < abs(pos1[1] - pos2[1]): + if pos1[1] > pos2[1]: + temp_tensor_0[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_1[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_2[ + pos2[1] : pos1[1] + 1, + int((pos1[0] + pos2[0]) / 2) - int(line_width / 2) : int((pos1[0] + pos2[0]) / 2) + + int(line_width / 2) + + 1, + ] = 255 + else: + temp_tensor_0[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_1[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 0 + temp_tensor_2[ + pos1[1] : pos2[1] + 1, + int((pos2[0] + pos1[0]) / 2) - int(line_width / 2) : int((pos2[0] + pos1[0]) / 2) + + int(line_width / 2) + + 1, + ] = 255 + else: + if pos1[0] > pos2[0]: + temp_tensor_0[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 0 + temp_tensor_1[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 0 + temp_tensor_2[ + int((pos1[1] + pos2[1]) / 2) - int(line_width / 2) : int((pos1[1] + pos2[1]) / 2) + + int(line_width / 2) + + 1, + pos2[0] : pos1[0] + 1, + ] = 255 + else: + temp_tensor_0[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 0 + temp_tensor_1[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 0 + temp_tensor_2[ + int((pos2[1] + pos1[1]) / 2) - int(line_width / 2) : int((pos2[1] + pos1[1]) / 2) + + int(line_width / 2) + + 1, + pos1[0] : pos2[0] + 1, + ] = 255 + + unnormalized = torch.stack((temp_tensor_0, temp_tensor_1, temp_tensor_2), dim=0) + unnormalized_list.append(unnormalized) + + temp_tensor_0_renorm = ((temp_tensor_0 / 255) - mean[0]) / std[0] + temp_tensor_1_renorm = ((temp_tensor_1 / 255) - mean[1]) / std[1] + temp_tensor_2_renorm = ((temp_tensor_2 / 255) - mean[2]) / std[2] + + temp_tensor = torch.stack([temp_tensor_0_renorm, temp_tensor_1_renorm, temp_tensor_2_renorm], dim=0) + + tensors_list.append(temp_tensor) + + return torch.stack(tensors_list, dim=0), torch.stack(unnormalized_list, dim=0) + + +def edge_inside(edge, edges_list): + edge_point1 = tuple(edge[0]["points"].tolist()) + edge_point2 = tuple(edge[1]["points"].tolist()) + for edge_i in edges_list: + edge_i_point1 = tuple(edge_i[0]["points"].tolist()) + edge_i_point2 = tuple(edge_i[1]["points"].tolist()) + if ((edge_point1 == edge_i_point1) and (edge_point2 == edge_i_point2)) or ( + (edge_point1 == edge_i_point2) and (edge_point2 == edge_i_point1) + ): + return True + return False + + +def remove_edge(edge, edges_list): + result = [] + edge_point1 = tuple(edge[0]["points"].tolist()) + edge_point2 = tuple(edge[1]["points"].tolist()) + for edge_i in edges_list: + edge_i_point1 = tuple(edge_i[0]["points"].tolist()) + edge_i_point2 = tuple(edge_i[1]["points"].tolist()) + if (edge_point1 == edge_i_point1) and (edge_point2 == edge_i_point2): + pass + else: + result.append(edge_i) + return result + + +def get_edges_amount(preds): + count = 0 + for this_preds, last_edges, this_edges in preds: + count += len(last_edges) + count += len(this_edges) + return count + + +def get_reserve_preds(results, keep_confidence_threshold, targets): + reserve_preds = [] + + valid_label_indices_edges = torch.where(results["edges"] != 0)[0] + valid_label_indices_scores = torch.where(results["scores"] <= keep_confidence_threshold)[0] + valid_label_indices = torch.tensor( + list(set(valid_label_indices_edges.tolist()).intersection(set(valid_label_indices_scores.tolist()))), + dtype=valid_label_indices_edges.dtype, + device=valid_label_indices_edges.device, + ) + for valid_label_indice in valid_label_indices: + valid_results_i = {} + valid_results_i["scores"] = results["scores"][valid_label_indice] + valid_results_i["points"] = results["points"][valid_label_indice] + valid_results_i["last_edges"] = results["last_edges"][valid_label_indice] + valid_results_i["this_edges"] = results["this_edges"][valid_label_indice] + valid_results_i["edges"] = results["edges"][valid_label_indice] + valid_results_i["size"] = targets[0]["size"] + reserve_preds.append(valid_results_i) + return reserve_preds diff --git a/data_preprocess/raster2graph/util/edges_utils.py b/data_preprocess/raster2graph/util/edges_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e13d9ad7de56477d25f10bca2df73ed116db1c5f --- /dev/null +++ b/data_preprocess/raster2graph/util/edges_utils.py @@ -0,0 +1,46 @@ +edges = { + 0: "0000", + 1: "0001", + 2: "0010", + 3: "0011", + 4: "0100", + 5: "0110", + 6: "0111", + 7: "1000", + 8: "1001", + 9: "1011", + 10: "1100", + 11: "1101", + 12: "1110", + 13: "1111", + 14: "0101", + 15: "1010", +} + + +def get_edges_alldirections(edges_class): + return edges[edges_class] + + +edges_rev = { + "0000": 0, + "0001": 1, + "0010": 2, + "0011": 3, + "0100": 4, + "0110": 5, + "0111": 6, + "1000": 7, + "1001": 8, + "1011": 9, + "1100": 10, + "1101": 11, + "1110": 12, + "1111": 13, + "0101": 14, + "1010": 15, +} + + +def get_edges_alldirections_rev(edges_class_rev): + return edges_rev[edges_class_rev] diff --git a/data_preprocess/raster2graph/util/geom_utils.py b/data_preprocess/raster2graph/util/geom_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c660fa518c85c9951667f4805fd241be0eb9f309 --- /dev/null +++ b/data_preprocess/raster2graph/util/geom_utils.py @@ -0,0 +1,124 @@ +import math + +from shapely.geometry import Polygon + + +def poly_iou(poly1: Polygon, poly2: Polygon): + try: + intersection_area = poly1.intersection(poly2).area + union_area = poly1.union(poly2).area + return intersection_area / union_area + except Exception: + poly1 = poly1.buffer(1) + poly2 = poly2.buffer(1) + intersection_area = poly1.intersection(poly2).area + union_area = poly1.union(poly2).area + return intersection_area / union_area + + +def is_clockwise_or_not(points): + s = 0 + for i in range(0, len(points) - 1): + s += points[i][0] * points[i + 1][1] - points[i][1] * points[i + 1][0] + return s > 0 + + +def x_axis_angle(y): + # 以图像坐标系为准，(1,0)方向记为0度，逆时针绕一圈到360度 + # print('-------------') + # print(y) + y_right_hand = (y[0], -y[1]) + # print(y_right_hand) + + x = (1, 0) + inner = x[0] * y_right_hand[0] + x[1] * y_right_hand[1] + # print(inner) + y_norm2 = (y_right_hand[0] ** 2 + y_right_hand[1] ** 2) ** 0.5 + # print(y_norm2) + cosxy = inner / (y_norm2 + 1e-8) + # print(cosxy) + angle = math.acos(cosxy) + # print(angle, math.degrees(angle)) + # print('-------------') + return math.degrees(angle) if y_right_hand[1] >= 0 else 360 - math.degrees(angle) + + +def get_quadrant(angle): + if angle[0] < angle[1]: + if 0 <= angle[0] < 90 and 0 <= angle[1] < 90: + quadrant = (angle[1] - angle[0], 0, 0, 0) + elif 0 <= angle[0] < 90 and 90 <= angle[1] < 180: + quadrant = (90 - angle[0], angle[1] - 90, 0, 0) + elif 0 <= angle[0] < 90 and 180 <= angle[1] < 270: + quadrant = (90 - angle[0], 90, angle[1] - 180, 0) + elif 0 <= angle[0] < 90 and 270 <= angle[1] < 360: + quadrant = (90 - angle[0], 90, 90, angle[1] - 270) + elif 90 <= angle[0] < 180 and 90 <= angle[1] < 180: + quadrant = (0, angle[1] - angle[0], 0, 0) + elif 90 <= angle[0] < 180 and 180 <= angle[1] < 270: + quadrant = (0, 180 - angle[0], angle[1] - 180, 0) + elif 90 <= angle[0] < 180 and 270 <= angle[1] < 360: + quadrant = (0, 180 - angle[0], 90, angle[1] - 270) + elif 180 <= angle[0] < 270 and 180 <= angle[1] < 270: + quadrant = (0, 0, angle[1] - angle[0], 0) + elif 180 <= angle[0] < 270 and 270 <= angle[1] < 360: + quadrant = (0, 0, 270 - angle[0], angle[1] - 270) + elif 270 <= angle[0] < 360 and 270 <= angle[1] < 360: + quadrant = (0, 0, 0, angle[1] - angle[0]) + else: + if 0 <= angle[1] < 90 and 0 <= angle[0] < 90: + quadrant_ = (angle[0] - angle[1], 0, 0, 0) + elif 0 <= angle[1] < 90 and 90 <= angle[0] < 180: + quadrant_ = (90 - angle[1], angle[0] - 90, 0, 0) + elif 0 <= angle[1] < 90 and 180 <= angle[0] < 270: + quadrant_ = (90 - angle[1], 90, angle[0] - 180, 0) + elif 0 <= angle[1] < 90 and 270 <= angle[0] < 360: + quadrant_ = (90 - angle[1], 90, 90, angle[0] - 270) + elif 90 <= angle[1] < 180 and 90 <= angle[0] < 180: + quadrant_ = (0, angle[0] - angle[1], 0, 0) + elif 90 <= angle[1] < 180 and 180 <= angle[0] < 270: + quadrant_ = (0, 180 - angle[1], angle[0] - 180, 0) + elif 90 <= angle[1] < 180 and 270 <= angle[0] < 360: + quadrant_ = (0, 180 - angle[1], 90, angle[0] - 270) + elif 180 <= angle[1] < 270 and 180 <= angle[0] < 270: + quadrant_ = (0, 0, angle[0] - angle[1], 0) + elif 180 <= angle[1] < 270 and 270 <= angle[0] < 360: + quadrant_ = (0, 0, 270 - angle[1], angle[0] - 270) + elif 270 <= angle[1] < 360 and 270 <= angle[0] < 360: + quadrant_ = (0, 0, 0, angle[0] - angle[1]) + quadrant = (90 - quadrant_[0], 90 - quadrant_[1], 90 - quadrant_[2], 90 - quadrant_[3]) + return quadrant + + +def find_which_angle_to_counterclockwise_rotate_from(t): + if t > 270: + return 630 - t + else: + return 270 - t + + +def counter_degree(d): + if d >= 180: + return d - 180 + else: + return d + 180 + + +def rotate_degree_clockwise_from_counter_degree(src_degree, dest_degree): + delta = src_degree - dest_degree + return delta if delta >= 0 else 360 + delta + + +def rotate_degree_counterclockwise_from_counter_degree(src_degree, dest_degree): + delta = dest_degree - src_degree + return delta if delta >= 0 else 360 + delta + + +def poly_area(points): + s = 0 + points_count = len(points) + for i in range(points_count): + point = points[i] + point2 = points[(i + 1) % points_count] + s += (point[0] - point2[0]) * (point[1] + point2[1]) + return s / 2 diff --git a/data_preprocess/raster2graph/util/graph_utils.py b/data_preprocess/raster2graph/util/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0ad1a59f3959dc7e0761c29cc19bda784f6c1e04 --- /dev/null +++ b/data_preprocess/raster2graph/util/graph_utils.py @@ -0,0 +1,879 @@ +import copy +import random + +import networkx as nx +import numpy as np +import torch +from util.geom_utils import ( + get_quadrant, + is_clockwise_or_not, + poly_area, + rotate_degree_counterclockwise_from_counter_degree, + x_axis_angle, +) +from util.metric_utils import get_results, get_results_float_with_semantic + + +def graph_to_tensor(graph): + t_l = [] + for k, v in graph.items(): + a = [] + a.append(k) + a.extend(v) + b = [list(i) for i in a] + c = torch.tensor(b) + t_l.append(c) + return torch.stack(t_l, dim=0) + + +def tensor_to_graph(tensor): + gr = {} + for kv in tensor: + k = tuple([i.item() for i in kv[0]]) + v = kv[1:5] + v = v.tolist() + v = [tuple(i) for i in v] + gr[k] = v + return gr + + +def tensors_to_graphs_batch(tensors): + return [tensor_to_graph(ts) for ts in tensors] + + +def get_cycle_basis_and_semantic_deprecated(best_result): + output_points, output_edges = get_results_float_with_semantic(best_result) + d = {} + for output_point_index, output_point in enumerate(output_points): + d[output_point] = output_point_index + d_rev = {} + for output_point_index, output_point in enumerate(output_points): + d_rev[output_point_index] = output_point + es = [] + for output_edge in output_edges: + es.append((d[output_edge[0]], d[output_edge[1]])) + + G = nx.Graph() + for e in es: + G.add_edge(e[0], e[1]) + + nx.draw(G) + # plt.show() + simple_cycles = nx.cycle_basis(G) + + results = [] + + for cycle_ind, cycle in enumerate(simple_cycles): + points = [d_rev[ind] for ind in cycle] + points.append(points[0]) + + is_clockwise = is_clockwise_or_not([(p[0], p[1]) for p in points]) + if is_clockwise: + points.reverse() + + cross_products = [] + poses = [(p[0], p[1]) for p in points] + for ind in range(len(poses) - 1): + ei = [ + poses[(ind + 1) % (len(poses) - 1)][0] - poses[ind][0], + poses[(ind + 1) % (len(poses) - 1)][1] - poses[ind][1], + ] + eiplus1 = [ + poses[(ind + 2) % (len(poses) - 1)][0] - poses[(ind + 1) % (len(poses) - 1)][0], + poses[(ind + 2) % (len(poses) - 1)][1] - poses[(ind + 1) % (len(poses) - 1)][1], + ] + cross_products.append(np.cross(ei, eiplus1).tolist()) + cross_products.insert(0, cross_products[-1]) + cross_products.pop(-1) + + while 0 in cross_products: + for point_ind, cross_product in enumerate(cross_products): + if cross_product == 0: + if point_ind == 0: + p0 = copy.deepcopy(points[0]) + points[0] = ( + p0[0] + 0.000001 * random.random() * [-1, 1][random.randint(0, 1)], + p0[1] + 0.000001 * random.random() * [-1, 1][random.randint(0, 1)], + p0[2], + p0[3], + p0[4], + p0[5], + ) + points[-1] = copy.deepcopy(points[0]) + else: + pi = copy.deepcopy(points[point_ind]) + points[point_ind] = ( + pi[0] + 0.000001 * random.random() * [-1, 1][random.randint(0, 1)], + pi[1] + 0.000001 * random.random() * [-1, 1][random.randint(0, 1)], + pi[2], + pi[3], + pi[4], + pi[5], + ) + # print(points) + cross_products = [] + poses = [(p[0], p[1]) for p in points] + for ind in range(len(poses) - 1): + ei = [ + poses[(ind + 1) % (len(poses) - 1)][0] - poses[ind][0], + poses[(ind + 1) % (len(poses) - 1)][1] - poses[ind][1], + ] + eiplus1 = [ + poses[(ind + 2) % (len(poses) - 1)][0] - poses[(ind + 1) % (len(poses) - 1)][0], + poses[(ind + 2) % (len(poses) - 1)][1] - poses[(ind + 1) % (len(poses) - 1)][1], + ] + cross_products.append(np.cross(ei, eiplus1).tolist()) + cross_products.insert(0, cross_products[-1]) + cross_products.pop(-1) + + semantics = [[p[2], p[3], p[4], p[5]] for p in points] + + degrees = [] + for ind in range(len(poses) - 1): + ei_minus = [ + -(poses[(ind + 1) % (len(poses) - 1)][0] - poses[ind][0]), + -(poses[(ind + 1) % (len(poses) - 1)][1] - poses[ind][1]), + ] + + eiplus1 = [ + poses[(ind + 2) % (len(poses) - 1)][0] - poses[(ind + 1) % (len(poses) - 1)][0], + poses[(ind + 2) % (len(poses) - 1)][1] - poses[(ind + 1) % (len(poses) - 1)][1], + ] + + degrees.append((x_axis_angle(ei_minus), x_axis_angle(eiplus1))) + degrees.insert(0, degrees[-1]) + degrees.pop(-1) + + angles = [] + for degree in degrees: + angles.append(((min(degree), max(degree)), (max(degree), min(degree)))) + + angles_to_semantics = [] + for angle_ind, angle in enumerate(angles): + angle1 = angle[0] + angle2 = angle[1] + quadrant1 = get_quadrant(angle1) + quadrant2 = get_quadrant(angle2) + + semantic1 = ( + semantics[angle_ind][1] if quadrant1[0] >= 45 else -1, + semantics[angle_ind][0] if quadrant1[1] >= 45 else -1, + semantics[angle_ind][3] if quadrant1[2] >= 45 else -1, + semantics[angle_ind][2] if quadrant1[3] >= 45 else -1, + ) + semantic2 = ( + semantics[angle_ind][1] if quadrant2[0] >= 45 else -1, + semantics[angle_ind][0] if quadrant2[1] >= 45 else -1, + semantics[angle_ind][3] if quadrant2[2] >= 45 else -1, + semantics[angle_ind][2] if quadrant2[3] >= 45 else -1, + ) + + angle1_degree = sum(quadrant1) + angle2_degree = sum(quadrant2) + + xproduct = cross_products[angle_ind] + + if xproduct < 0: + if angle1_degree < angle2_degree: + angles_to_semantics.append(semantic1) + else: + angles_to_semantics.append(semantic2) + elif xproduct > 0: + if angle1_degree < angle2_degree: + angles_to_semantics.append(semantic2) + else: + angles_to_semantics.append(semantic1) + else: + assert 0 + + semantic_result = {} + for semantic_label in range(0, 13): + semantic_result[semantic_label] = 0 + for everypoint_semantic in angles_to_semantics: + everypoint_semantic = [s for s in everypoint_semantic if s != -1] + for label in everypoint_semantic: + semantic_result[label] += 1 / len(everypoint_semantic) + + this_cycle_semantic1 = sorted(semantic_result.items(), key=lambda d: d[1], reverse=True) + this_cycle_result = None + if this_cycle_semantic1[0][1] > this_cycle_semantic1[1][1]: + this_cycle_result = this_cycle_semantic1[0][0] + else: + this_cycle_results = [i[0] for i in this_cycle_semantic1 if i[1] == this_cycle_semantic1[0][1]] + this_cycle_result = this_cycle_results[random.randint(0, len(this_cycle_results) - 1)] + results.append(this_cycle_result) + + return d_rev, simple_cycles, results + + +def get_cycle_basis_and_semantic(best_result): + output_points, output_edges = get_results_float_with_semantic(best_result) + output_points = copy.deepcopy(output_points) + output_edges = copy.deepcopy(output_edges) + + d = {} + for output_point_index, output_point in enumerate(output_points): + d[output_point] = output_point_index + d_rev = {} + for output_point_index, output_point in enumerate(output_points): + d_rev[output_point_index] = output_point + es = [] + for output_edge in output_edges: + es.append((d[output_edge[0]], d[output_edge[1]])) + # print(d) + + G = nx.Graph() + for e in es: + G.add_edge(e[0], e[1]) + + simple_cycles = [] + simple_cycles_number = [] + simple_cycles_semantics = [] + bridges = list(nx.bridges(G)) + for b in bridges: + if (d_rev[b[0]], d_rev[b[1]]) in output_edges: + output_edges.remove((d_rev[b[0]], d_rev[b[1]])) + es.remove((b[0], b[1])) + G.remove_edge(b[0], b[1]) + if (d_rev[b[1]], d_rev[b[0]]) in output_edges: + output_edges.remove((d_rev[b[1]], d_rev[b[0]])) + es.remove((b[1], b[0])) + G.remove_edge(b[1], b[0]) + connected_components = list(nx.connected_components(G)) + for c in connected_components: + if len(c) == 1: + pass + else: + simple_cycles_c = [] + simple_cycles_number_c = [] + simple_cycle_semantics_c = [] + # output_points_c = [p for p in output_points if d[p] in c] + output_edges_c = [e for e in output_edges if d[e[0]] in c or d[e[1]] in c] + output_edges_c_copy_for_traversing = copy.deepcopy(output_edges_c) + + for edge_c in output_edges_c: + if edge_c not in output_edges_c_copy_for_traversing: + pass + else: + simple_cycle_semantics = [] + simple_cycle = [] + simple_cycle_number = [] + point1 = edge_c[0] + point2 = edge_c[1] + point1_number = d[point1] + point2_number = d[point2] + + initial_point = None + initial_point_number = None + if point1_number < point2_number: + initial_point = point1 + initial_point_number = point1_number + else: + initial_point = point2 + initial_point_number = point2_number + simple_cycle.append(initial_point) + simple_cycle_number.append(initial_point_number) + + last_point = initial_point + + current_point = None + current_point_number = None + if point1_number < point2_number: + current_point = point2 + current_point_number = point2_number + else: + current_point = point1 + current_point_number = point1_number + simple_cycle.append(current_point) + simple_cycle_number.append(current_point_number) + + next_initial_point = copy.deepcopy(current_point) + + next_point = None + next_point_number = None + + while next_point != next_initial_point: + relevant_edges = [] + for edge in output_edges_c: + if edge[0] == current_point or edge[1] == current_point: + relevant_edges.append(edge) + + relevant_edges_degree = [] + for relevant_edge in relevant_edges: + vec = None + if relevant_edge[0] == current_point: + vec = ( + relevant_edge[1][0] - relevant_edge[0][0], + relevant_edge[1][1] - relevant_edge[0][1], + ) + elif relevant_edge[1] == current_point: + vec = ( + relevant_edge[0][0] - relevant_edge[1][0], + relevant_edge[0][1] - relevant_edge[1][1], + ) + else: + assert 0 + + vec_degree = x_axis_angle(vec) + relevant_edges_degree.append(vec_degree) + + vec_from_current_point_to_last_point_degree = None + for relevant_edge_ind, relevant_edge in enumerate(relevant_edges): + if relevant_edge == (current_point, last_point): + vec_from_current_point_to_last_point_degree = relevant_edges_degree[relevant_edge_ind] + relevant_edges.remove(relevant_edge) + relevant_edges_degree.remove(vec_from_current_point_to_last_point_degree) + elif relevant_edge == (last_point, current_point): + vec_from_current_point_to_last_point_degree = relevant_edges_degree[relevant_edge_ind] + relevant_edges.remove(relevant_edge) + relevant_edges_degree.remove(vec_from_current_point_to_last_point_degree) + else: + continue + + rotate_deltas_counterclockwise = [] + + interior_angles = [] + for relevant_edge_degree in relevant_edges_degree: + rotate_delta = rotate_degree_counterclockwise_from_counter_degree( + vec_from_current_point_to_last_point_degree, relevant_edge_degree + ) + rotate_deltas_counterclockwise.append(rotate_delta) + interior_angles.append((relevant_edge_degree, vec_from_current_point_to_last_point_degree)) + # print(rotate_deltas_counterclockwise) + + max_rotate_index = rotate_deltas_counterclockwise.index(max(rotate_deltas_counterclockwise)) + + interior_angle_counterclockwise = interior_angles[max_rotate_index] + + current_point_semantic = [ + current_point[3], + current_point[2], + current_point[5], + current_point[4], + ] + + interior_angle_counterclockwise_degree_smaller = min(interior_angle_counterclockwise) + interior_angle_counterclockwise_degree_bigger = max(interior_angle_counterclockwise) + quadrant_smaller_to_bigger_counterclockwise = get_quadrant( + ( + interior_angle_counterclockwise_degree_smaller, + interior_angle_counterclockwise_degree_bigger, + ) + ) + # print(quadrant_smaller_to_bigger_counterclockwise) + if interior_angle_counterclockwise.index(interior_angle_counterclockwise_degree_smaller) == 0: + pass + elif ( + interior_angle_counterclockwise.index(interior_angle_counterclockwise_degree_smaller) == 1 + ): + quadrant_smaller_to_bigger_counterclockwise = ( + 90 - quadrant_smaller_to_bigger_counterclockwise[0], + 90 - quadrant_smaller_to_bigger_counterclockwise[1], + 90 - quadrant_smaller_to_bigger_counterclockwise[2], + 90 - quadrant_smaller_to_bigger_counterclockwise[3], + ) + else: + assert 0 + + current_point_semantic_valid = [] + for qd, seman in enumerate(current_point_semantic): + if quadrant_smaller_to_bigger_counterclockwise[qd] >= 45: + current_point_semantic_valid.append(seman) + else: + current_point_semantic_valid.append(-1) + + simple_cycle_semantics.append(current_point_semantic_valid) + + max_rotate_edge = relevant_edges[max_rotate_index] + + if max_rotate_edge[0] == current_point: + next_point = max_rotate_edge[1] + next_point_number = d[next_point] + elif max_rotate_edge[1] == current_point: + next_point = max_rotate_edge[0] + next_point_number = d[next_point] + else: + assert 0 + + last_point = current_point + current_point = next_point + current_point_number = next_point_number + simple_cycle.append(current_point) + simple_cycle_number.append(current_point_number) + + for point_number_ind, point_number in enumerate(simple_cycle_number): + if point_number_ind < len(simple_cycle_number) - 1: + edge_number = (point_number, simple_cycle_number[point_number_ind + 1]) + # print(simple_cycle_number) + if edge_number[0] < edge_number[1]: + if ( + d_rev[edge_number[0]], + d_rev[edge_number[1]], + ) in output_edges_c_copy_for_traversing: + output_edges_c_copy_for_traversing.remove( + (d_rev[edge_number[0]], d_rev[edge_number[1]]) + ) + elif ( + d_rev[edge_number[1]], + d_rev[edge_number[0]], + ) in output_edges_c_copy_for_traversing: + output_edges_c_copy_for_traversing.remove( + (d_rev[edge_number[1]], d_rev[edge_number[0]]) + ) + + simple_cycle.pop(-1) + simple_cycle_number.pop(-1) + + polygon_counterclockwise = [(int(p[0]), -int(p[1])) for p in simple_cycle] + polygon_counterclockwise.pop(-1) + # print('poly_area(polygon_counterclockwise)', poly_area(polygon_counterclockwise)) + if poly_area(polygon_counterclockwise) > 0: + simple_cycles_c.append(simple_cycle) + simple_cycles_number_c.append(simple_cycle_number) + + semantic_result = {} + for semantic_label in range(0, 13): + semantic_result[semantic_label] = 0 + for everypoint_semantic in simple_cycle_semantics: + everypoint_semantic = [s for s in everypoint_semantic if s != -1] + for label in everypoint_semantic: + semantic_result[label] += 1 / len(everypoint_semantic) + # print(semantic_result) + del semantic_result[11] + del semantic_result[12] + + this_cycle_semantic = sorted(semantic_result.items(), key=lambda d: d[1], reverse=True) + # print(this_cycle_semantic) + this_cycle_result = None + if this_cycle_semantic[0][1] > this_cycle_semantic[1][1]: + this_cycle_result = this_cycle_semantic[0][0] + else: + this_cycle_results = [ + i[0] for i in this_cycle_semantic if i[1] == this_cycle_semantic[0][1] + ] + this_cycle_result = this_cycle_results[random.randint(0, len(this_cycle_results) - 1)] + # print(this_cycle_result) + simple_cycle_semantics_c.append(this_cycle_result) + + simple_cycles.extend(simple_cycles_c) + simple_cycles_number.extend(simple_cycles_number_c) + simple_cycles_semantics.extend(simple_cycle_semantics_c) + + # print([[(int(j[0]), int(j[1])) for j in i] for i in simple_cycles]) + + # print(len(simple_cycles_number)) + # print(simple_cycles_semantics) + + return d_rev, simple_cycles, simple_cycles_semantics + + +def get_cycle_basis_and_semantic_2(best_result): + output_points, output_edges = get_results_float_with_semantic(best_result) + output_points = copy.deepcopy(output_points) + output_edges = copy.deepcopy(output_edges) + # print(output_points) + # print(output_edges) + # assert 0 + d = {} + for output_point_index, output_point in enumerate(output_points): + d[output_point] = output_point_index + d_rev = {} + for output_point_index, output_point in enumerate(output_points): + d_rev[output_point_index] = output_point + es = [] + for output_edge in output_edges: + es.append((d[output_edge[0]], d[output_edge[1]])) + # print(d) + + G = nx.Graph() + for e in es: + G.add_edge(e[0], e[1]) + + simple_cycles = [] + simple_cycles_number = [] + simple_cycles_semantics = [] + + bridges = list(nx.bridges(G)) + + for b in bridges: + if (d_rev[b[0]], d_rev[b[1]]) in output_edges: + output_edges.remove((d_rev[b[0]], d_rev[b[1]])) + es.remove((b[0], b[1])) + G.remove_edge(b[0], b[1]) + if (d_rev[b[1]], d_rev[b[0]]) in output_edges: + output_edges.remove((d_rev[b[1]], d_rev[b[0]])) + es.remove((b[1], b[0])) + G.remove_edge(b[1], b[0]) + + connected_components = list(nx.connected_components(G)) + for c in connected_components: + if len(c) == 1: + pass + else: + simple_cycles_c = [] + simple_cycles_number_c = [] + simple_cycle_semantics_c = [] + output_edges_c = [e for e in output_edges if d[e[0]] in c or d[e[1]] in c] + output_edges_c_copy_for_traversing = copy.deepcopy(output_edges_c) + + for edge_c in output_edges_c: + if edge_c not in output_edges_c_copy_for_traversing: + pass + else: + simple_cycle_semantics = [] + simple_cycle = [] + simple_cycle_number = [] + point1 = edge_c[0] + point2 = edge_c[1] + point1_number = d[point1] + point2_number = d[point2] + + initial_point = None + initial_point_number = None + if point1_number < point2_number: + initial_point = point1 + initial_point_number = point1_number + else: + initial_point = point2 + initial_point_number = point2_number + simple_cycle.append(initial_point) + simple_cycle_number.append(initial_point_number) + + last_point = initial_point + + current_point = None + current_point_number = None + if point1_number < point2_number: + current_point = point2 + current_point_number = point2_number + else: + current_point = point1 + current_point_number = point1_number + simple_cycle.append(current_point) + simple_cycle_number.append(current_point_number) + + next_initial_point = copy.deepcopy(current_point) + + next_point = None + next_point_number = None + + while next_point != next_initial_point: + relevant_edges = [] + for edge in output_edges_c: + if edge[0] == current_point or edge[1] == current_point: + relevant_edges.append(edge) + + relevant_edges_degree = [] + for relevant_edge in relevant_edges: + vec = None + if relevant_edge[0] == current_point: + vec = ( + relevant_edge[1][0] - relevant_edge[0][0], + relevant_edge[1][1] - relevant_edge[0][1], + ) + elif relevant_edge[1] == current_point: + vec = ( + relevant_edge[0][0] - relevant_edge[1][0], + relevant_edge[0][1] - relevant_edge[1][1], + ) + else: + assert 0 + + vec_degree = x_axis_angle(vec) + relevant_edges_degree.append(vec_degree) + + vec_from_current_point_to_last_point_degree = None + for relevant_edge_ind, relevant_edge in enumerate(relevant_edges): + if relevant_edge == (current_point, last_point): + vec_from_current_point_to_last_point_degree = relevant_edges_degree[relevant_edge_ind] + relevant_edges.remove(relevant_edge) + relevant_edges_degree.remove(vec_from_current_point_to_last_point_degree) + elif relevant_edge == (last_point, current_point): + vec_from_current_point_to_last_point_degree = relevant_edges_degree[relevant_edge_ind] + relevant_edges.remove(relevant_edge) + relevant_edges_degree.remove(vec_from_current_point_to_last_point_degree) + else: + continue + + rotate_deltas_counterclockwise = [] + interior_angles = [] + for relevant_edge_degree in relevant_edges_degree: + rotate_delta = rotate_degree_counterclockwise_from_counter_degree( + vec_from_current_point_to_last_point_degree, relevant_edge_degree + ) + rotate_deltas_counterclockwise.append(rotate_delta) + interior_angles.append((relevant_edge_degree, vec_from_current_point_to_last_point_degree)) + # print(rotate_deltas_counterclockwise) + max_rotate_index = rotate_deltas_counterclockwise.index(max(rotate_deltas_counterclockwise)) + interior_angle_counterclockwise = interior_angles[max_rotate_index] + current_point_semantic = [ + current_point[3], + current_point[2], + current_point[5], + current_point[4], + ] + interior_angle_counterclockwise_degree_smaller = min(interior_angle_counterclockwise) + interior_angle_counterclockwise_degree_bigger = max(interior_angle_counterclockwise) + quadrant_smaller_to_bigger_counterclockwise = get_quadrant( + ( + interior_angle_counterclockwise_degree_smaller, + interior_angle_counterclockwise_degree_bigger, + ) + ) + if interior_angle_counterclockwise.index(interior_angle_counterclockwise_degree_smaller) == 0: + pass + elif ( + interior_angle_counterclockwise.index(interior_angle_counterclockwise_degree_smaller) == 1 + ): + quadrant_smaller_to_bigger_counterclockwise = ( + 90 - quadrant_smaller_to_bigger_counterclockwise[0], + 90 - quadrant_smaller_to_bigger_counterclockwise[1], + 90 - quadrant_smaller_to_bigger_counterclockwise[2], + 90 - quadrant_smaller_to_bigger_counterclockwise[3], + ) + else: + assert 0 + current_point_semantic_valid = [] + for qd, seman in enumerate(current_point_semantic): + if 1: + current_point_semantic_valid.append(seman) + else: + current_point_semantic_valid.append(-1) + simple_cycle_semantics.append(current_point_semantic_valid) + + max_rotate_edge = relevant_edges[max_rotate_index] + if max_rotate_edge[0] == current_point: + next_point = max_rotate_edge[1] + next_point_number = d[next_point] + elif max_rotate_edge[1] == current_point: + next_point = max_rotate_edge[0] + next_point_number = d[next_point] + else: + assert 0 + + last_point = current_point + current_point = next_point + current_point_number = next_point_number + simple_cycle.append(current_point) + simple_cycle_number.append(current_point_number) + + for point_number_ind, point_number in enumerate(simple_cycle_number): + if point_number_ind < len(simple_cycle_number) - 1: + edge_number = (point_number, simple_cycle_number[point_number_ind + 1]) + if edge_number[0] < edge_number[1]: + if ( + d_rev[edge_number[0]], + d_rev[edge_number[1]], + ) in output_edges_c_copy_for_traversing: + output_edges_c_copy_for_traversing.remove( + (d_rev[edge_number[0]], d_rev[edge_number[1]]) + ) + elif ( + d_rev[edge_number[1]], + d_rev[edge_number[0]], + ) in output_edges_c_copy_for_traversing: + output_edges_c_copy_for_traversing.remove( + (d_rev[edge_number[1]], d_rev[edge_number[0]]) + ) + + simple_cycle.pop(-1) + simple_cycle_number.pop(-1) + polygon_counterclockwise = [(int(p[0]), -int(p[1])) for p in simple_cycle] + polygon_counterclockwise.pop(-1) + if poly_area(polygon_counterclockwise) > 0: + simple_cycles_c.append(simple_cycle) + simple_cycles_number_c.append(simple_cycle_number) + semantic_result = {} + for semantic_label in range(0, 13): + semantic_result[semantic_label] = 0 + for everypoint_semantic in simple_cycle_semantics: + for _ in range(0, 13): + if _ in everypoint_semantic: + semantic_result[_] += 1 + del semantic_result[11] + del semantic_result[12] + + this_cycle_semantic = sorted(semantic_result.items(), key=lambda d: d[1], reverse=True) + this_cycle_result = None + if this_cycle_semantic[0][1] > this_cycle_semantic[1][1]: + this_cycle_result = this_cycle_semantic[0][0] + else: + this_cycle_results = [ + i[0] for i in this_cycle_semantic if i[1] == this_cycle_semantic[0][1] + ] + this_cycle_result = this_cycle_results[random.randint(0, len(this_cycle_results) - 1)] + simple_cycle_semantics_c.append(this_cycle_result) + + simple_cycles.extend(simple_cycles_c) + simple_cycles_number.extend(simple_cycles_number_c) + simple_cycles_semantics.extend(simple_cycle_semantics_c) + + return d_rev, simple_cycles, simple_cycles_semantics + + +def get_cycle_basis(best_result): + output_points, output_edges = get_results(best_result) + output_points = copy.deepcopy(output_points) + output_edges = copy.deepcopy(output_edges) + + d = {} + for output_point_index, output_point in enumerate(output_points): + d[output_point] = output_point_index + d_rev = {} + for output_point_index, output_point in enumerate(output_points): + d_rev[output_point_index] = output_point + es = [] + for output_edge in output_edges: + es.append((d[output_edge[0]], d[output_edge[1]])) + + G = nx.Graph() + for e in es: + G.add_edge(e[0], e[1]) + + simple_cycles = [] + simple_cycles_number = [] + bridges = list(nx.bridges(G)) + for b in bridges: + if (d_rev[b[0]], d_rev[b[1]]) in output_edges: + output_edges.remove((d_rev[b[0]], d_rev[b[1]])) + es.remove((b[0], b[1])) + G.remove_edge(b[0], b[1]) + if (d_rev[b[1]], d_rev[b[0]]) in output_edges: + output_edges.remove((d_rev[b[1]], d_rev[b[0]])) + es.remove((b[1], b[0])) + G.remove_edge(b[1], b[0]) + connected_components = list(nx.connected_components(G)) + for c in connected_components: + if len(c) == 1: + pass + else: + simple_cycles_c = [] + simple_cycles_number_c = [] + output_edges_c = [e for e in output_edges if d[e[0]] in c or d[e[1]] in c] + output_edges_c_copy_for_traversing = copy.deepcopy(output_edges_c) + + for edge_c in output_edges_c: + if edge_c not in output_edges_c_copy_for_traversing: + pass + else: + simple_cycle = [] + simple_cycle_number = [] + point1 = edge_c[0] + point2 = edge_c[1] + point1_number = d[point1] + point2_number = d[point2] + + if point1_number < point2_number: + initial_point = point1 + initial_point_number = point1_number + current_point = point2 + current_point_number = point2_number + else: + initial_point = point2 + initial_point_number = point2_number + current_point = point1 + current_point_number = point1_number + + simple_cycle.append(initial_point) + simple_cycle_number.append(initial_point_number) + simple_cycle.append(current_point) + simple_cycle_number.append(current_point_number) + + last_point = initial_point + next_initial_point = copy.deepcopy(current_point) + next_point = None + + while next_point != next_initial_point: + relevant_edges = [] + for edge in output_edges_c: + if edge[0] == current_point or edge[1] == current_point: + relevant_edges.append(edge) + + relevant_edges_degree = [] + for relevant_edge in relevant_edges: + vec = None + if relevant_edge[0] == current_point: + vec = ( + relevant_edge[1][0] - relevant_edge[0][0], + relevant_edge[1][1] - relevant_edge[0][1], + ) + elif relevant_edge[1] == current_point: + vec = ( + relevant_edge[0][0] - relevant_edge[1][0], + relevant_edge[0][1] - relevant_edge[1][1], + ) + else: + assert 0 + vec_degree = x_axis_angle(vec) + relevant_edges_degree.append(vec_degree) + + vec_from_current_point_to_last_point_degree = None + for relevant_edge_ind, relevant_edge in enumerate(relevant_edges): + if relevant_edge == (current_point, last_point): + vec_from_current_point_to_last_point_degree = relevant_edges_degree[relevant_edge_ind] + relevant_edges.remove(relevant_edge) + relevant_edges_degree.remove(vec_from_current_point_to_last_point_degree) + elif relevant_edge == (last_point, current_point): + vec_from_current_point_to_last_point_degree = relevant_edges_degree[relevant_edge_ind] + relevant_edges.remove(relevant_edge) + relevant_edges_degree.remove(vec_from_current_point_to_last_point_degree) + else: + continue + + rotate_deltas_counterclockwise = [] + for relevant_edge_degree in relevant_edges_degree: + rotate_delta = rotate_degree_counterclockwise_from_counter_degree( + vec_from_current_point_to_last_point_degree, relevant_edge_degree + ) + rotate_deltas_counterclockwise.append(rotate_delta) + + max_rotate_index = rotate_deltas_counterclockwise.index(max(rotate_deltas_counterclockwise)) + max_rotate_edge = relevant_edges[max_rotate_index] + + if max_rotate_edge[0] == current_point: + next_point = max_rotate_edge[1] + next_point_number = d[next_point] + elif max_rotate_edge[1] == current_point: + next_point = max_rotate_edge[0] + next_point_number = d[next_point] + else: + assert 0 + + last_point = current_point + current_point = next_point + current_point_number = next_point_number + simple_cycle.append(current_point) + simple_cycle_number.append(current_point_number) + + for point_number_ind, point_number in enumerate(simple_cycle_number): + if point_number_ind < len(simple_cycle_number) - 1: + edge_number = (point_number, simple_cycle_number[point_number_ind + 1]) + if edge_number[0] < edge_number[1]: + if ( + d_rev[edge_number[0]], + d_rev[edge_number[1]], + ) in output_edges_c_copy_for_traversing: + output_edges_c_copy_for_traversing.remove( + (d_rev[edge_number[0]], d_rev[edge_number[1]]) + ) + elif ( + d_rev[edge_number[1]], + d_rev[edge_number[0]], + ) in output_edges_c_copy_for_traversing: + output_edges_c_copy_for_traversing.remove( + (d_rev[edge_number[1]], d_rev[edge_number[0]]) + ) + + simple_cycle.pop(-1) + simple_cycle_number.pop(-1) + + polygon_counterclockwise = [(int(p[0]), -int(p[1])) for p in simple_cycle] + polygon_counterclockwise.pop(-1) + if poly_area(polygon_counterclockwise) > 0: + simple_cycles_c.append(simple_cycle) + simple_cycles_number_c.append(simple_cycle_number) + + simple_cycles.extend(simple_cycles_c) + simple_cycles_number.extend(simple_cycles_number_c) + + return d_rev, simple_cycles, simple_cycles_number diff --git a/data_preprocess/raster2graph/util/image_id_dict.py b/data_preprocess/raster2graph/util/image_id_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..a4911668af6c5242cd433b06adc3a4c99d067e01 --- /dev/null +++ b/data_preprocess/raster2graph/util/image_id_dict.py @@ -0,0 +1,10826 @@ +d = { + "00-00-081549d99cf1e3f5be593e560799-0002": 1, + "00-00-3e6fd70bea51c029e5289f4c7627-0004": 2, + "00-00-c41333aafe250ee08ffef30361aa-0001": 3, + "00-00-d85242cf8f1ae5620f874bac9fba-0001": 4, + "00-01-79b6db19059440f73e39abee88af-0001": 5, + "00-02-5b4d2fa305df28eea9551d615207-0001": 6, + "00-02-7488ac85219b41907d04d3a85650-0001": 7, + "00-02-77af30c0c4799f23d915c3377dfb-0002": 8, + "00-02-af8514017426072e1bf1b0e9c145-0001": 9, + "00-02-c05441a2b7ce87779453ef0c13fe-0001": 10, + "00-03-0bd5e984efd6d0df4fc32cf7acd3-0001": 11, + "00-03-13b6c8175fa6b94902d9c06c5a22-0001": 12, + "00-03-19c050a40be4e87136469a1edfce-0003": 13, + "00-03-61365475b603d9d47f80db62d30c-0002": 14, + "00-03-762df73c741c3af941cdd9c0bee9-0001": 15, + "00-03-cf405ceefda83a4cd61bf0ca4b47-0002": 16, + "00-03-f4806b671b184f9d7a4ac9eb96da-0001": 17, + "00-04-095dcd3bf6a9b0a7a70cb4984428-0001": 18, + "00-04-1184ba1a42e623e3ef0d1c454d9e-0001": 19, + "00-05-101cb28e2288e2b8b46d93dbdd9d-0001": 20, + "00-05-30212c0b27e69a2d7b8583395a2f-0001": 21, + "00-05-521b0815db82a98a363d75e9000c-0001": 22, + "00-05-7b56cd5420b8878750dbe3088f2d-0002": 23, + "00-05-e94c72b66ea46482833aea7ec710-0001": 24, + "00-06-94db28655d836be5a79b9d4afcbe-0001": 25, + "00-06-aa3cbfeaee05fc4e68a4be544320-0003": 26, + "00-06-d49c7427145f9b3456a97fe6cdc6-0001": 27, + "00-07-1e1794ae5618b008a2f804c919ef-0001": 28, + "00-07-b5ea736b34a0ecbbdd2478604116-0002": 29, + "00-08-218a5d9fe2a78495e2f0acd84fc5-0001": 30, + "00-08-360f045f4195a7cdc35e08970a5e-0001": 31, + "00-09-1509cc128a0ca1b62c636502307d-0003": 32, + "00-09-474a97b49d838fde83fc464d8eb2-0001": 33, + "00-09-bdc8129165f578363819fd8da2e0-0001": 34, + "00-09-d0bdb504649b90da028b77e6998b-0001": 35, + "00-0a-1618347270d03f6d03ab4d4a2cee-0001": 36, + "00-0a-432c07758d0741233b6167fe23bf-0001": 37, + "00-0a-bec1e7dabc0656eccae4d5caac7e-0001": 38, + "00-0a-d8097c16eb99fb7d1b3cc393f710-0001": 39, + "00-0a-e12777a4acfca0a935651292115c-0002": 40, + "00-0a-e18e11ed80ab8a20776827e2d65b-0001": 41, + "00-0b-223dcea56bde883a4c9d1ee5d1c0-0001": 42, + "00-0b-411ff7a6e9d34b79e4a576e849e7-0002": 43, + "00-0c-86faa0b8263dd5911a38ad521e62-0002": 44, + "00-0d-5e09e8cca35277b1941096c0f57d-0001": 45, + "00-0e-265cb57a2c91f3fc6e19108cb77f-0001": 46, + "00-0e-e13ea66c1fded65cce70246f0e80-0001": 47, + "00-0f-2b2f9544a4475d32a25afb0406b2-0001": 48, + "00-0f-6181efed7560781c9d1aa07506c6-0001": 49, + "00-0f-61989f9d030c882dec7e77e2a0ae-0002": 50, + "00-10-00d49541c1a4ba4d286b2012e0ed-0001": 51, + "00-10-635001b67fd181d23d7173532859-0001": 52, + "00-10-cb950828a2f3e160babc5d682eee-0001": 53, + "00-11-0205f3cf08f5c5f15ec90babff81-0002": 54, + "00-11-4a1072c49b11c2923caa8d59d912-0002": 55, + "00-11-57d1118d7076037a4793fce345bd-0001": 56, + "00-11-b33327b5d2ab39dc7f0a64ce4081-0001": 57, + "00-11-e585f6165422132477c3bde363e9-0002": 58, + "00-12-231ae9b254df2089c353e901e30c-0001": 59, + "00-12-2db5fc9ddbe913308cc92e4b584b-0001": 60, + "00-12-504da839e909658d33fcced32424-0002": 61, + "00-12-5e36bcd89c9ebe0b854518f914a3-0001": 62, + "00-12-7a4d46c59c516b1b488a1be9714e-0001": 63, + "00-12-81a7246326edd440620b8208fd38-0002": 64, + "00-13-04174a28a57ed9cbf803eaea78e1-0001": 65, + "00-13-4d4ced9265d59072f00eb7de6e3e-0001": 66, + "00-13-ec70b663143245fbc90f65cbb685-0001": 67, + "00-13-f621e5e43e75d80aab472c268a68-0001": 68, + "00-14-081e4fb4f14de3f44bebc4341a1f-0001": 69, + "00-14-19bed632d0e4952915b5ca28ccea-0002": 70, + "00-14-815ce2b049683d6ecaf622567f33-0002": 71, + "00-14-955aa91a52f95ac0f68066493d39-0001": 72, + "00-14-9d99be7a60782ae63a872a266388-0002": 73, + "00-15-43afb50eedcb5e695e03dddef74a-0001": 74, + "00-16-1f97180f432629740cd310584811-0001": 75, + "00-16-2bbc171d6d854f695d9b7e4921ee-0003": 76, + "00-16-37e05663bc83faa36f007e61b0c0-0001": 77, + "00-16-387ad4137aaa30145aadc0c48695-0001": 78, + "00-16-7c0bc72a9fa15a7943d749fd9f74-0002": 79, + "00-16-89a8200b48d389bc9f6330b06ea7-0001": 80, + "00-16-963f2baa2a73a88803a13421cdfe-0002": 81, + "00-16-a57bb9d72f47de256064cc11cdb7-0001": 82, + "00-17-1565154767a429651df2b297b276-0002": 83, + "00-17-6dd8df07fe6d9b90ab4c7c486683-0001": 84, + "00-17-81d44a2979fd06ff1e820e91b031-0002": 85, + "00-17-8eb4e517ff6fd188f5103b00b3ce-0002": 86, + "00-17-8f93a6bdc1dff2b3c28bc81e147a-0001": 87, + "00-17-adb790f4c770a7d608fd44f53ec4-0001": 88, + "00-18-05bd15702680ce566a5655d0786e-0001": 89, + "00-18-0f7b8ef5459527eccde272812eb5-0001": 90, + "00-18-1127918c000a24c7447a33948500-0001": 91, + "00-18-5b74e3d2d1f4cb03d0175f3b3f2a-0001": 92, + "00-18-8e23249f5b466ae2da42d39f0b40-0002": 93, + "00-18-c71891068613535d18a7a78a31ee-0001": 94, + "00-18-c854c73926dbf3683539b42a7c43-0001": 95, + "00-18-f833a11292f8f168282186cbdaa0-0002": 96, + "00-19-0ad477cdc12c8ce9587a457bba39-0001": 97, + "00-19-3130d8728b4b48679da5244d7b9a-0001": 98, + "00-19-42b3a9fdbdea34ddcdc9c57dc37e-0002": 99, + "00-19-4c72d028974f5c099e0449582a05-0001": 100, + "00-19-568f9cc642f0368dc8ec1cfd354e-0002": 101, + "00-19-a505e4a0fb8d1251baa32ca665ee-0002": 102, + "00-19-ca789c065dc237b580105eca5976-0001": 103, + "00-19-d45156bb28e574cbbad11a43c512-0001": 104, + "00-19-dbd41a71762b2c0f050eecc97994-0001": 105, + "00-19-ee225028d87cf149dfb0608a716c-0001": 106, + "00-1a-7799838133e0bade8b97e18802a2-0001": 107, + "00-1a-9f5c4b0f7ceefa79c0665a234d99-0001": 108, + "00-1b-7ffc044859493aeba957d1c94a3b-0001": 109, + "00-1b-e1c0b59fbf91bc8da0e1406d2fef-0001": 110, + "00-1b-ee62777b4aa75a3e11aa7a8c3780-0001": 111, + "00-1c-2d294561a389a298e61a6e3e3032-0002": 112, + "00-1c-3f58c9f8c6eba3e415722ce2620b-0001": 113, + "00-1c-49dd9c413c988701ac7a1697292c-0001": 114, + "00-1c-df941c495b3f55bd4e93314aa77c-0001": 115, + "00-1d-896b0975d5d224f915aed4e824ab-0004": 116, + "00-1d-b4e5fae1c2cf2691ab0e152ddcbc-0006": 117, + "00-1d-d6fd540fc996cd9be551dc8b4281-0001": 118, + "00-1e-46dfef9b2c7931aea63da6d436d5-0004": 119, + "00-1e-548bffab8ada7c554def67aa4b55-0001": 120, + "00-1e-57aa2bc110c2ed0db4df5bdeb0d9-0002": 121, + "00-1e-90ea8b0db811ced27583059a34c5-0003": 122, + "00-1e-9d04ae41a5e9f13cf80bf7387f67-0001": 123, + "00-1e-abc722cb608a3a5339ac6b517d07-0002": 124, + "00-1e-bc8cb86129ba4708c9045d32239c-0001": 125, + "00-1f-02f97b31ba619badcfd067f42cff-0002": 126, + "00-1f-440d44d6deba40115d91442666d1-0001": 127, + "00-1f-6a856cd4a04cf417eb72779836a9-0001": 128, + "00-1f-72acc07b5dc85ad4da0200f19695-0002": 129, + "00-1f-7c22026b91d223a00613d4578deb-0001": 130, + "00-20-024dfd02682affdfff7eeca39cf3-0001": 131, + "00-20-117c4451b538120183b96c72de5f-0001": 132, + "00-20-1424fef55708fe2a1554332b583b-0002": 133, + "00-20-3457e0b943033fa0656e061ac167-0001": 134, + "00-20-677d94b01f3314a4cd13a8ddb469-0002": 135, + "00-20-86205fedaad1350c91ff7dedce9b-0002": 136, + "00-20-b1d2324783c12eb0bd42513ea485-0002": 137, + "00-20-d79c6c2cec725ac23cd9a92dd1d0-0002": 138, + "00-20-d968b314f5e70da4babac943d8af-0002": 139, + "00-20-ed9990830d3c0d65a529e5584086-0001": 140, + "00-21-57b5aee6f29611250e7dc2b8f49e-0002": 141, + "00-21-71a7e1e4ea5a0989425498b5df37-0001": 142, + "00-21-bf1cbcb6bd596639d0b29fdc0bf3-0001": 143, + "00-22-770b6295d1fd2b79d8d62bf345fc-0001": 144, + "00-22-7cfbba2a545af8819024f004c2ce-0002": 145, + "00-22-82f413becd783d5ed37b7dca1e00-0002": 146, + "00-22-8bea5c265573208f0903b757152f-0001": 147, + "00-22-bb8a2094791827fbaf4d59658412-0001": 148, + "00-22-ea2a3c0d9149596f57fce1268eb3-0001": 149, + "00-22-f29ff3b483dac444b2cfc8c9394d-0001": 150, + "00-23-016b16e1fadb0aaf7b21b18f37ff-0002": 151, + "00-23-0c53c9e3a3daad1a79ffc1aaa895-0001": 152, + "00-23-3e97a4ae43092c4483834facd939-0002": 153, + "00-23-3ebbec493a73275636fad84d850b-0001": 154, + "00-23-500a167ab926df601817631d6733-0002": 155, + "00-23-f0aff0461ec8ef9c7540591e6973-0001": 156, + "00-24-70a4bc607f4699fcc4557ba625f6-0003": 157, + "00-24-fa15529018416944fd02d74f9c97-0001": 158, + "00-25-61505f847f863d68c8faeda2a4da-0002": 159, + "00-25-75385eff683bb4979122162b7b9c-0001": 160, + "00-25-7bb67ea772f6b89215e9f52e4756-0001": 161, + "00-25-a4618d75b7b5e934b4bccdb22f7e-0001": 162, + "00-25-a96be150d4b1adb5700d722e8b1b-0001": 163, + "00-26-2c5577ffea3105c2bf35b8eff3f7-0001": 164, + "00-26-84daadad4cf584eee5063ade99ba-0004": 165, + "00-26-963f311f889df5cc867e66570c75-0001": 166, + "00-26-db891b5eb2d3bf37a927d5aba2fd-0001": 167, + "00-27-36973c9ed66e8e868e84a848d03a-0002": 168, + "00-27-52b36ee12e6529d4a1c1ef2c848b-0001": 169, + "00-27-67ecad017170742537f37da504e6-0001": 170, + "00-27-78a252e38cd16b4394f676d97ac8-0001": 171, + "00-27-7bede1c8b1704c87b4434bbaab95-0001": 172, + "00-27-8ebce2fd13e5b9bbc228ca6f8ce6-0001": 173, + "00-27-90517feb0ba31131e22f991c5553-0001": 174, + "00-27-9b68c43ba5eda05dcb9273a82449-0002": 175, + "00-27-b9aca13025bd92b82f9c9532f1d6-0002": 176, + "00-28-160740ae3151ba0fa50ca7926f85-0001": 177, + "00-28-1cad73c7fb09240396096979adc6-0001": 178, + "00-28-1e8b3e808e47ed4701e159d39564-0001": 179, + "00-28-4ee2e59e50ae2bd8c70995cbd643-0002": 180, + "00-28-6b89fde550375c9832d432a41550-0002": 181, + "00-28-71c8e6e60d00d8bc1a68ac53c816-0002": 182, + "00-28-a16986ca5370b1b306dc684d8f16-0001": 183, + "00-28-e9ba2ad9c52d67bfe28e57dcda0b-0002": 184, + "00-29-073d51a4185a9e2333f73def52fb-0001": 185, + "00-29-2d5a5c433d8e84386a83924f9191-0002": 186, + "00-29-4ba85177e5c273049ea67fddb9de-0002": 187, + "00-29-5402c2b02fbd5bcaa4cba96382bc-0001": 188, + "00-29-72eba356ecb0c1fa8d27d8adebf2-0002": 189, + "00-29-91896a5d229181f6c98f8ad23b01-0003": 190, + "00-29-97857f572476a2ff724897a33f98-0002": 191, + "00-29-d2bde3ed5d526d432fbeeb109ccf-0002": 192, + "00-2a-5302cf6d798e6d266329a29b03d9-0001": 193, + "00-2a-9b5155a18f887c7d7fa09177211c-0001": 194, + "00-2a-cf3d22ab6d1f61477513db717c4d-0003": 195, + "00-2b-0aa0bd7f20fadd76091183f492ee-0001": 196, + "00-2b-1f0ff9f71e8f1ae906cb026b477d-0001": 197, + "00-2b-7c01e323ee5b4d961c62a6c5bf7c-0002": 198, + "00-2b-b5a0f2d458882ad5ea6cfa708476-0001": 199, + "00-2b-cd6e4dd23c4a38f24dd5f5ffccca-0001": 200, + "00-2c-0a88ac04b4f434ca2c2f49d5e186-0001": 201, + "00-2c-289132b72f88ba82fcae8b4327f9-0001": 202, + "00-2c-30019528c0d0a92320de7d1c1ded-0001": 203, + "00-2c-6ab2038e81727ebbe3d3052de232-0002": 204, + "00-2c-88909c2f98c87ee3ba39777fe3f8-0001": 205, + "00-2c-89e3aefc81605339bd655e03e1d8-0001": 206, + "00-2c-8d9cc003b28d26ad9acaf0090962-0002": 207, + "00-2c-a4a7856c4b4f887ef3af4909ec20-0001": 208, + "00-2c-b6c3aff45f00e2f504bdec878d4b-0003": 209, + "00-2c-bb4e49d67e7a5eec668ed39d6e6e-0001": 210, + "00-2c-de0e43e25c372c92ea5b179b3c16-0001": 211, + "00-2c-fec3460cf4d3823a4476eedf2541-0001": 212, + "00-2d-8cd6b0209ec1dac85c070dab0bfe-0001": 213, + "00-2e-e277dc3adaa1551d1b96aa83c4f8-0002": 214, + "00-2e-fbca25af5ca62817b9035d8a0167-0001": 215, + "00-2f-26db378e624d504d9312f93662ea-0001": 216, + "00-2f-691054d42b7410b3a67a810998f9-0001": 217, + "00-2f-7ed94a302d66886eddba699ff4ce-0001": 218, + "00-2f-b6315c0b1b3e9ced33755df67fc8-0001": 219, + "00-2f-e80e0f22c3f6520e21bbd26c8f85-0001": 220, + "00-2f-f480ec0fec8a74be7e97722765c7-0002": 221, + "00-30-3d163ad9c021e78b63fcad22ba3f-0004": 222, + "00-30-6adf793510aacc4cfbf2dfe389a3-0001": 223, + "00-31-066ac22edf958b7c3a7eb0d48c7c-0001": 224, + "00-31-3bd2acf59ed1fb72444e8ca3abb6-0001": 225, + "00-31-5a898da9e7a8cf2d969239ab257c-0001": 226, + "00-31-9a452cf50250c89d83cca24a945f-0002": 227, + "00-31-ad4aa63e0580fe59eba57935bf16-0003": 228, + "00-31-be653f952616dee0694e2f547878-0001": 229, + "00-31-d7738beeb8bf91c96c3dc9ee7d65-0001": 230, + "00-32-40031854330762144a5dbc2f114c-0002": 231, + "00-32-749255007409e44d9d50f4720f40-0001": 232, + "00-32-8edb577b4f3cfc89362d8084b83b-0001": 233, + "00-32-94ddfe829fc7730c36b0e876c53d-0001": 234, + "00-32-dc8e6587406fca5e95d502a32411-0002": 235, + "00-33-36ea583b2ccb84e29bdda35f90f3-0001": 236, + "00-33-7f643fec9aa23c5048b1ba121a7b-0003": 237, + "00-33-fcaeca6beb0dd6db5ffb0bf54ff6-0001": 238, + "00-34-0cb3c0ffbf84da36ff9fa17bb6bb-0001": 239, + "00-34-5443f2c860361a03c43854fd8646-0001": 240, + "00-34-b07564bb3413cf19b0dfa52b048f-0002": 241, + "00-34-d63fd68d173bf883fc5553b596fc-0002": 242, + "00-35-0a95f55623b0ffd22d653aea14e2-0002": 243, + "00-35-5ecf9be4a8e4fd35118db1d66e89-0002": 244, + "00-35-6ab2098aab867ac8d0abfa440960-0002": 245, + "00-35-a90708538b1070fababfe79578d0-0003": 246, + "00-35-c1ec95ce25c968a8802e40b117d7-0001": 247, + "00-35-f3359d52214e2fe3dbac60ff5dd3-0001": 248, + "00-35-f637ccecea1799f1407c6dd69144-0001": 249, + "00-36-70aad120aed012f30b91262df5b1-0002": 250, + "00-36-aa982a46e1864348ab301c50c131-0002": 251, + "00-36-bc40787e7e5ad0b6e46c37aa167e-0001": 252, + "00-36-d478ad84264b8ec8a97331ca81d0-0002": 253, + "00-36-e716f9ba42ea5cc8bcb405952bdc-0001": 254, + "00-36-f0a1800f4f759d1a2e94ed6b1514-0002": 255, + "00-37-0008be7b9662575f40d7854e90f0-0003": 256, + "00-37-2bc4bd8597b317ad14afd257498d-0002": 257, + "00-37-754e613be865c41e4b488f6e81f7-0001": 258, + "00-37-e8b3b980e4e35e857e5288d64aab-0001": 259, + "00-38-36b52a02505abdb0d779b508f7ce-0004": 260, + "00-38-9e62f25478e8a2273accc0745fc3-0002": 261, + "00-38-e628c020fd1acada7edd9ad22acd-0001": 262, + "00-39-0787b6c6e43202117a8babd8d961-0001": 263, + "00-39-359e1780bdbe60d876c6aacd24cf-0002": 264, + "00-39-478df52af2ef81bfc1b23d40af76-0004": 265, + "00-39-7e630555c1f1b759bcefbffb0b87-0001": 266, + "00-39-c432573c3149f12b89dd819946c5-0001": 267, + "00-3a-02c0fca13d8a51474a26f5cd7351-0001": 268, + "00-3a-27363a47255877b37792a79f600d-0001": 269, + "00-3a-5f0643c34e2f75ab3e109e69a4cd-0002": 270, + "00-3a-74929a1c18201a490d0dcaf028cd-0004": 271, + "00-3a-b20ab32cf1ec8262e1e447181ec4-0004": 272, + "00-3a-c9cb83749325d3483c3d6c4bdd89-0002": 273, + "00-3b-2ebfb29e2c393b288197c3b83f29-0001": 274, + "00-3b-9aebded5889387d312eb6e5cf96a-0002": 275, + "00-3b-e38f55ed8955fa18c60875ef0013-0001": 276, + "00-3b-ee546ec0094d4ad52e9bf632ed44-0002": 277, + "00-3c-4cbea171518d4b921f40a8e7e464-0001": 278, + "00-3c-6b5b5a009618990d68fa532a5f52-0002": 279, + "00-3c-98d77e100c5ac22eede96115b2cc-0002": 280, + "00-3c-a05a20ce8f64b262575ea29ada56-0001": 281, + "00-3c-bcd3788e6af427ff044412a9160e-0001": 282, + "00-3c-be4b2016e0ebc0f447ac24c1f5fe-0002": 283, + "00-3c-c062e9506248059317a034af7039-0001": 284, + "00-3c-e65c1c52b3111a8288e3d08ee2a0-0002": 285, + "00-3d-8093325a4067e7d57de41e8571a9-0001": 286, + "00-3d-d6675a44d7f7df7d752b4b24e77a-0002": 287, + "00-3d-e947828f0faa90bca34806bb6646-0002": 288, + "00-3d-efde255985603c50c1f3b572906f-0002": 289, + "00-3e-9c8d3c2fbd419c5da85150d22ca4-0001": 290, + "00-3e-9cce20748de4c6921460cbaebe5e-0002": 291, + "00-3e-c5cd44c6da484479553e7da20696-0001": 292, + "00-3f-23196863539647f95fdbb112ca51-0001": 293, + "00-3f-2d397ee85ccffb737d5a86280fe4-0002": 294, + "00-3f-655323c0754e97f942bbf6199f94-0002": 295, + "00-40-208032acd71999157bd0624fb27f-0001": 296, + "00-40-2892f39473a1193c7ec39d51a105-0001": 297, + "00-40-742df65911601862b60067f1f9dc-0002": 298, + "00-40-b542d5e5c369a23aa73719a20c89-0001": 299, + "00-40-c11c0909480a53c1037c5d458c62-0002": 300, + "00-41-106b363fe9b3d7a63c7bda30a388-0002": 301, + "00-41-52b0ef5703d846aa5c3cc0093db8-0003": 302, + "00-41-6d96f4d9c83dd6e57edd90852a8a-0001": 303, + "00-41-8fa416732c6cd070f3444e25fe63-0003": 304, + "00-41-9e3470495305fd2b55f3c4cd9fd5-0001": 305, + "00-41-d12adc7e285cf59b61db01768f3d-0001": 306, + "00-41-e0c60a724f1a1b71b76fbac1f671-0001": 307, + "00-41-f38b760b2f99846a0d787d5e4ad7-0001": 308, + "00-42-5815706b2dcda9ddf28071b435e9-0001": 309, + "00-42-6a14bbe5b404694d484f31723f62-0001": 310, + "00-42-7b586db8c5d13da8a25846555d25-0002": 311, + "00-42-93d6e485865f869f7347e6c7a5c7-0001": 312, + "00-42-e44903de08884d5b6a8d73cfb431-0001": 313, + "00-43-023d69a18887436a053165dded7b-0001": 314, + "00-43-0cd62cc76e8a2bdf06e3c604e712-0001": 315, + "00-43-2c351fea5a42edfada06f5b2ae76-0002": 316, + "00-43-5d141fbf904b882bab09d62a33aa-0001": 317, + "00-43-7fc069f1edabbea8df0b4863d4b3-0001": 318, + "00-44-7f4f99bcfc280abc8729c79935b4-0001": 319, + "00-45-b5a76bf012e02b24bd89b757c9e2-0001": 320, + "00-46-4f2cc5dbd526e1cbfd2a5ec112e1-0001": 321, + "00-46-e6176c755501ec5579220f051baf-0001": 322, + "00-46-fc5f58837b2948fe9de01653566f-0001": 323, + "00-47-51d95a2ebb1e929ecc17f9da9057-0001": 324, + "00-47-6a23b85aebb616e9d04de6adb747-0001": 325, + "00-47-f2e8d7f3b9dd49b242ec6f876f4e-0001": 326, + "00-48-165944d6908f77a593209b13318b-0007": 327, + "00-48-20c8343e225c600a30a381fcae2c-0002": 328, + "00-48-52f0725ab2445c4de8735c5203ab-0001": 329, + "00-48-f41ed72d3cdcc21ed1e018c9a70d-0001": 330, + "00-48-f613bf6aaf246256359b01cd36bf-0002": 331, + "00-48-fabaa2bf5993a8d42e4a24db40ac-0002": 332, + "00-49-22f589e6dd1174565bcdeafa6afb-0001": 333, + "00-49-5a39fd0f0a5aaec3f239c773945c-0001": 334, + "00-49-67735ebc3c52c11f556ad7515e8c-0001": 335, + "00-49-8cb470570d81c15bdb5cb5263ac1-0001": 336, + "00-49-945c206ac9065d1c77d30693155a-0003": 337, + "00-49-b46a8b463a6f5c334375cf03cc06-0002": 338, + "00-49-b857ba9e2d017f37bc0363406705-0001": 339, + "00-49-d51b32fe5db8e360942d18ac2417-0001": 340, + "00-49-f74be09d2f1d43998590be652a91-0001": 341, + "00-4a-6ec364c755cab7f526e97a0b5ea3-0002": 342, + "00-4a-7582781377e14c15244496cf8fe7-0001": 343, + "00-4a-9953d69b9771976735586f43534f-0001": 344, + "00-4a-a20743bcd52c1c88b8f7d6f9ca32-0002": 345, + "00-4b-128d93444dd6842244c5c8e17356-0001": 346, + "00-4b-bc467191d69f128783035faae23e-0003": 347, + "00-4c-9b3a772f682d6840a58582c3d7f8-0001": 348, + "00-4c-eb7be223e71f96ad6ce83666cb8d-0001": 349, + "00-4d-81c6cb107b7f225d3000113a5891-0001": 350, + "00-4d-aba1eeb8ee374eb90e27447cdb73-0001": 351, + "00-4e-2ea480d95ca16e9709e07720670e-0002": 352, + "00-4e-476bd4f1842fe132eb83077d4c8c-0002": 353, + "00-4e-7c65b15995e7b11e1bfea1147414-0002": 354, + "00-4e-be50d2709768d5320485e8ecc44e-0001": 355, + "00-4e-d164e489ee920fe75eeed6de0e83-0002": 356, + "00-4e-dbe3e455fc2d249857b28ba63458-0003": 357, + "00-4e-eb78db9fc16c88ace66a1a0ed784-0001": 358, + "00-4f-2444e3668f9f7f9cd00336e5af9a-0001": 359, + "00-4f-b56d901519638626d67900451638-0003": 360, + "00-50-1c4938a90deb3a78b17866efc03f-0002": 361, + "00-50-528faa6223291bfddde8464a515a-0001": 362, + "00-50-89cdcc13d5e456bb26a76dceadd5-0002": 363, + "00-50-b9cde83799a1302b42be464f1510-0001": 364, + "00-50-c02254e598895e333e698c93fe14-0001": 365, + "00-50-da93ac85a15a54294d596a9682f6-0001": 366, + "00-50-df926be40060e638a69db7438db5-0002": 367, + "00-50-fd1380e54d9d0a4c46974a45f7f2-0002": 368, + "00-51-1986150ed02ce43aa367ea6f3c62-0003": 369, + "00-51-35e8ec52500843c6f1f7da0845a8-0001": 370, + "00-51-720ee538542f645a893b34fd6f82-0001": 371, + "00-51-d6f5012e8e6d97dd739e3c5fcb91-0003": 372, + "00-51-de75e6dd0fda9bdbeab8852687a0-0001": 373, + "00-52-02ffcff20204c67ca4595243033d-0001": 374, + "00-52-0e98be4346829f90cc4ba0cb789f-0002": 375, + "00-52-1d417b593c5d9a61c7555be99f9e-0001": 376, + "00-52-caebe662919bca834c8cf3662668-0001": 377, + "00-53-022eb58ca8c93c4997883b4f7e5f-0002": 378, + "00-53-3ce07d59aaf78286c45fb23e9249-0001": 379, + "00-55-1b0f6504a79601025965952929c1-0003": 380, + "00-55-3fb00de5cd6084d8444edf8aa038-0001": 381, + "00-55-7829b097c56544592c016528b797-0002": 382, + "00-55-83368e39678983ba570930675a35-0001": 383, + "00-55-a24961a9d20b303b656d5ec82ae9-0002": 384, + "00-55-cae3a3fa8826df53beec04f3f62f-0001": 385, + "00-55-da3bb751cbff10589c4331badcc8-0001": 386, + "00-55-e95f4243679f770ec40fbe32e99d-0001": 387, + "00-56-68420c4768e04f600e4a4520ee1f-0002": 388, + "00-56-c7712ad36acf84c37daf8f61ae30-0001": 389, + "00-57-0602e4bb1f5c440acb4d82b7faca-0004": 390, + "00-57-12fde4819ea9c54e367d3dc97c4b-0001": 391, + "00-57-45c2e78a25b598863e0d503bcc15-0002": 392, + "00-57-7b816d0f53330f6d58486b510ab5-0001": 393, + "00-57-92699c92c6bf4bf1595200b63ab7-0001": 394, + "00-57-c589a713740de91ffd3330a8864a-0001": 395, + "00-57-d198c7f9e63c197b44f42546b1e5-0001": 396, + "00-58-20cf4814bf4703b1a5cb054744bd-0001": 397, + "00-58-785b7ffbfa05389bfda287abf4ed-0001": 398, + "00-58-8a122bcca06fadb1ea5b15e70b24-0001": 399, + "00-58-96568a264cd47e4e7f3b64b75045-0001": 400, + "00-58-9d7769d41739210d64bfb70f1e50-0001": 401, + "00-58-9e5824da3b4b363ef4bb68a0be8a-0001": 402, + "00-59-245a3d3f8f97f2d369addd307ab1-0001": 403, + "00-59-383fa11961ab24a54dda2bf6ace3-0001": 404, + "00-59-3842d0f55e2ae59c88c1afc00e83-0003": 405, + "00-59-75a3d9fd79bce92596759c81bb01-0001": 406, + "00-59-8a7a036cbf53f28bbba17e01bbf9-0001": 407, + "00-59-8c1e21c7e0accdb29f352a0f0f18-0001": 408, + "00-59-95ee69991afc8b8eb370097e56b5-0002": 409, + "00-59-a80bbb660fdb067cbc9957b2366f-0001": 410, + "00-59-c81c2e333a9827370c73bc055b7d-0001": 411, + "00-59-cc80cf7f57012aa5a3a44b9ae75b-0001": 412, + "00-59-e8bf62a771d26e8a0aa6dd1d2742-0001": 413, + "00-5a-1d102ccb68fcf401a4fd138a78df-0001": 414, + "00-5b-7fed42523c6c450041577d7c144a-0001": 415, + "00-5b-c553a7a6848e3b7a167ec8f3a982-0001": 416, + "00-5b-cade75d832e6f920c9ec74ec31d1-0001": 417, + "00-5b-e3d2ff6fb7c0df09768b3ab63130-0001": 418, + "00-5c-247bde814fbe8ab64926f177dc94-0002": 419, + "00-5c-2762d979d80a4bf73e22003d375a-0001": 420, + "00-5c-3606d897ecc8d08f8e0a97cdbe30-0002": 421, + "00-5c-3e12408c93124c684fbf23a1339f-0001": 422, + "00-5c-4012dadd5f2301b5b0a6108f1b6e-0001": 423, + "00-5c-5254d0b240adabbe95af4af9f848-0002": 424, + "00-5c-6bff2a91bfa23d728341ac07721f-0001": 425, + "00-5c-807f72432bf34d48cd39accc85c8-0001": 426, + "00-5c-a1e66d43e86151ce05fd21ca50fb-0002": 427, + "00-5c-b5a1d5371c364e995110b10ae0ab-0001": 428, + "00-5c-d0ad8343f2d386dccd9ce04c2344-0002": 429, + "00-5d-13dd17bfbf5448156e2e42e45bce-0002": 430, + "00-5d-3203dbcf7700e0154489ed63fe7b-0001": 431, + "00-5d-32f3821dd389e1742dacb8af4b99-0002": 432, + "00-5d-8142ad4e68e7217dbaa79d058f56-0002": 433, + "00-5d-d09832560c40507f47a7d4032081-0002": 434, + "00-5d-e1341f4f3518da1cd0986589968e-0001": 435, + "00-5e-ab1b27faaa480f5c0d5933fb7570-0002": 436, + "00-5e-bd12b7ba277d5a8084749b9918c0-0001": 437, + "00-5f-1116844134a91f3ace681209792a-0001": 438, + "00-5f-2c3daa577392718f05cd75c05e76-0003": 439, + "00-5f-6668d215e1950d61fa4632d2697a-0002": 440, + "00-5f-a4055527c38f1042982b7de69994-0001": 441, + "00-5f-dc5eff42d93c24bf3d344ed82e3b-0001": 442, + "00-60-0504bb965a509cd15210744bfd92-0001": 443, + "00-60-135dc15eb1e60350dc4bd560e63d-0001": 444, + "00-60-7725711798aeb1cd9e982453132c-0003": 445, + "00-60-c42478294a1277a79fa9b6198f78-0001": 446, + "00-60-fc2d25fac2d27fb757b4ded36802-0002": 447, + "00-61-a008072afccbfd7093ce0817c4a9-0002": 448, + "00-62-094cbc4ad31b2b7299feac1247a3-0001": 449, + "00-62-194b1e6f612d71d63dd0384ae3ab-0002": 450, + "00-62-1ecac75c1697cf03762703d1c64f-0002": 451, + "00-62-7dfcfbea4486b79c597654699900-0001": 452, + "00-62-9602131f5a204832a0c767b42a28-0002": 453, + "00-62-c215db944f349f206ca496482171-0001": 454, + "00-62-e02cd87f14691609f7ff1b366598-0001": 455, + "00-62-ec438da71dcd9d152473e7abfe6c-0001": 456, + "00-63-1638027496d573a2c82e1e1c2651-0001": 457, + "00-63-3cfd532cc1cb8e16f2a1b68c626b-0002": 458, + "00-63-51ecf50d6ee97f37ee43f958d496-0002": 459, + "00-63-bfabfac4f4e16419d54daa6aaab0-0001": 460, + "00-63-e09993c8b51556655ad2d0c389e3-0001": 461, + "00-64-2fa9737de65da696f5f27a8575ff-0002": 462, + "00-64-5e78cc91abb0832ae482e49d38dc-0002": 463, + "00-64-ba7162fb344d4081f17816ca423a-0001": 464, + "00-64-f7ad26097750467d86bf33d08da3-0001": 465, + "00-65-01ed616f1f129c005a67c3694950-0003": 466, + "00-65-8ccb884f866debb0488ec4d92038-0002": 467, + "00-65-b246cabaac4aba49fa9b6d6a3ae1-0001": 468, + "00-66-1c225d54b345c1f434010ec8a44e-0001": 469, + "00-66-763892d42da1f9d505aca1a84950-0003": 470, + "00-67-4b7f0289c29f21014fefe784b989-0001": 471, + "00-67-7e7d7b4c3c1f7a4959f243343ac4-0001": 472, + "00-67-ceb105d2888fbd2a0d88ac1ba439-0002": 473, + "00-67-f24903de5e82732539976bdeaaf1-0001": 474, + "00-68-257ac2ed2ff220ac34630f378dee-0002": 475, + "00-68-46ca80ba6e8f30bd4ab1a0a28991-0001": 476, + "00-68-5d92ebf3ef85c0d6f811c9124c23-0002": 477, + "00-68-a18a90a32f5fd7219d0f0566b2ed-0002": 478, + "00-69-0315dadae60995ffe9750d396c80-0002": 479, + "00-69-5c076823fb3c5a27439a90c8fbde-0004": 480, + "00-6a-74a38ae7fce7573b4ab96c6a5f5f-0002": 481, + "00-6a-8a81e1337455d81002beb1b8dbde-0003": 482, + "00-6a-fe90c85f11dccef40d026da06550-0001": 483, + "00-6b-058b3d271dd68c24e665f4a19baf-0003": 484, + "00-6b-0ef7c294fc8c9dd29974f871f755-0001": 485, + "00-6b-69de4984f4c090dacdb38582a10c-0001": 486, + "00-6b-74a9863a328568e6ff6862877417-0002": 487, + "00-6b-b6a64619819dd1f703e8cc901746-0001": 488, + "00-6c-0f45adbc0f8aa5b30b14365e8f48-0001": 489, + "00-6c-1a7c47fa908b7f44af11d7a21ffb-0001": 490, + "00-6c-251a939f022f363a9be25ee8bf68-0001": 491, + "00-6c-5bf6a774497ad56f646e6aeaa3f1-0001": 492, + "00-6c-5d287ae3eef73eddae42723beca9-0002": 493, + "00-6c-9f6fab7da688e91f168e2c8ee1c5-0002": 494, + "00-6c-bb4ff33b7ef4affcd7b1e20ddc1d-0002": 495, + "00-6c-ed6094c6ac617e48b5ce8bde28a7-0002": 496, + "00-6d-a0d690792b42a3b906a522c6e133-0004": 497, + "00-6d-bf244466f3928b37a34ea3b8d744-0002": 498, + "00-6d-e1a279541120fb8f41717db07275-0001": 499, + "00-6e-03c6c535e2b6e2bd2f063c769e2f-0002": 500, + "00-6e-078a22c4c750bd8afa1cdbc331e8-0001": 501, + "00-6e-0ff482d2f4a243876ebe2dc6ceb9-0001": 502, + "00-6e-14b1363101ca4c3195f16562bf45-0001": 503, + "00-6e-9ebf86041fdaaa1cfb4c7106dafe-0002": 504, + "00-6e-bb4f5ff835f2575270a928ec513e-0001": 505, + "00-6e-bd92ab0fbccfcbe37bea1794076c-0001": 506, + "00-6e-de5e7bd373deeb9aadd9dbd32c10-0001": 507, + "00-6f-0a6ff8f53d2eb03512a4c28025ec-0002": 508, + "00-6f-57c8599f07d94e084a32946e6a89-0001": 509, + "00-6f-637da40cb10a4453711812d9b57c-0001": 510, + "00-6f-7330737ccbe47119103bd06786aa-0001": 511, + "00-70-0dfe7ca32ab29f0a643c99df1c25-0001": 512, + "00-70-1e34d916261cf77dbd13e7b33eaa-0001": 513, + "00-70-1e490dd4ffa4e7b136e9fab62dc3-0001": 514, + "00-71-0cd6b18e3497f8787dc433cb5c1b-0001": 515, + "00-71-21efc8bb1aebf61edc26075d49a8-0001": 516, + "00-71-4d2a83b09e6f92d69f5f9bb06f36-0001": 517, + "00-71-6704d57312d1142846148fad710c-0001": 518, + "00-71-8f09c9a113d3f9f8b29a58812a9f-0001": 519, + "00-71-c8e0e396f82c1042d7e96638903b-0001": 520, + "00-72-3d02845da39387a3c481bde7fbe2-0002": 521, + "00-72-6d97e1f92c5ee60610e21caee3c6-0001": 522, + "00-72-6f840d72c863746a4755ab1b648d-0001": 523, + "00-72-7fd1e77eb8e830433daa4603de90-0002": 524, + "00-72-d476426c11e772445d8048e0b2dc-0001": 525, + "00-73-224f3ad9f4d1bbdc77d5e7f2c47a-0002": 526, + "00-73-bb7ecbccb7c81385870ebcf2a87f-0003": 527, + "00-73-c9151b2df9ece578b59dc9c5122d-0002": 528, + "00-74-01862418adeb5785582e94ea0889-0001": 529, + "00-74-196a9299b59406d5d70c0e98cef5-0001": 530, + "00-74-5a3305a2e242599bf8ab7015a243-0002": 531, + "00-74-5cce3d747d9ce4f8931923dcc4a2-0001": 532, + "00-74-6edc6313649679afa4630153c0a0-0001": 533, + "00-74-85b6d9e9c61400e15f9cc5330042-0001": 534, + "00-74-8dda89b081a85b0d59a37229dad7-0001": 535, + "00-74-fa6b47e4716d71af6899b5c27a58-0001": 536, + "00-75-0e18ca2b5f7b1377b05a1caab820-0001": 537, + "00-75-993f8c5998c18e5dff4476ff2b8e-0002": 538, + "00-75-bb10f274b78904fccc6fa668731b-0001": 539, + "00-75-c9c191aaaa1d7237e3cd44926816-0002": 540, + "00-75-d0e02508a036c0b11956314b80e0-0001": 541, + "00-75-f84671297c3b4e18227430c357f6-0002": 542, + "00-75-fef41bed09ed0214fb2b246ecd3f-0001": 543, + "00-76-27cc31a06b0d22fea3eb21b61673-0002": 544, + "00-76-556d13e3c56d87072efb2cb181fe-0001": 545, + "00-76-ce84e4fa949b98cf648c69c451fe-0001": 546, + "00-76-d85fbdc7e31ce8a7f2726c7634d4-0002": 547, + "00-77-20d99bbb560f19bf22f5087addc0-0001": 548, + "00-77-633fdc2aab97dfd9f4dbe2dd77ad-0001": 549, + "00-77-70898bf54e91d4799d115ae81f06-0001": 550, + "00-77-a5bac8d65b3fc46e5d68e0ec8f4c-0002": 551, + "00-77-ae67cfb460497507bdd950d5f147-0001": 552, + "00-77-b9ebf22b33e6540ee561035e2ded-0001": 553, + "00-77-de34b437e504cb431a9dafda05dd-0002": 554, + "00-77-e7942eb128f742f4758fe470a162-0003": 555, + "00-78-2a67f3d88e6f9995afc3060d3cee-0001": 556, + "00-78-376fa8277a928099705fe4d79654-0002": 557, + "00-78-563d482a885e292b6ff08fbec6c1-0002": 558, + "00-78-586ba548febada3e957465ff2915-0002": 559, + "00-78-68a10957e282e3f178d7775ff1fc-0002": 560, + "00-78-7bb51994dd796c7170e1bd1006a1-0003": 561, + "00-78-7c34610df3c30e593bcd78d6fd96-0001": 562, + "00-78-9a3a24a5c245c0e1f09274f55e71-0001": 563, + "00-78-cf9fdb6bd6ec0798a1e3f6e36c74-0001": 564, + "00-79-499687a374717a95735c34fe250c-0001": 565, + "00-79-8c33788ac673c691226d297ef414-0001": 566, + "00-79-9cc877ca554fa4d38b5672c460fe-0001": 567, + "00-79-af58ff21645b75b7f680b6f664e8-0001": 568, + "00-79-fb6e7e060acf17d7a8b64cadbe5a-0002": 569, + "00-7a-1b80960d1891c471caf86a268ac8-0001": 570, + "00-7a-4cf7fd8bc83167bf4410426ebc26-0001": 571, + "00-7a-a790bc07b98e0b0af0485f0a1335-0002": 572, + "00-7a-c466035bd609f6b0d0515d69d7ff-0001": 573, + "00-7a-fffc0bc8a17e3d2b6c1a6273486a-0001": 574, + "00-7b-85d02f3bf150a8f0158e24c32a01-0002": 575, + "00-7b-9bb587be22b5d4a1f2669c0f29f7-0001": 576, + "00-7b-d6f5a308b8c766335ea926050dd7-0001": 577, + "00-7b-d96ff2266240fdc970b2f0c0c2c3-0002": 578, + "00-7c-1a73f65d30631010fbbd3be35f20-0002": 579, + "00-7c-3b2a63dcecc96a73b7a91efda962-0001": 580, + "00-7c-9ab9b6e7b481d1c7afc6333ee539-0002": 581, + "00-7c-da2560542c7e8a9a25f47fc2849b-0001": 582, + "00-7c-ec4936bfcdbb0d5140e7393a56ca-0002": 583, + "00-7d-23352e3f6a3611c34c3647638840-0003": 584, + "00-7d-5bdca044c616e2a7b02c8f45c91a-0001": 585, + "00-7d-94aa75456340ea6c8d8f2c0c5a66-0001": 586, + "00-7d-ca085337d35e50f33d7efd952cb1-0001": 587, + "00-7d-f8ac3caafc026f027f54b09bf750-0001": 588, + "00-7e-0b430974e37160cad3fcd59961ea-0002": 589, + "00-7e-2187554eae7b2d48cb779a25aee4-0001": 590, + "00-7e-464fa7eac32c08a5c5ae05862397-0001": 591, + "00-7e-5e57b3f276dd60c63c7351bbb6ff-0004": 592, + "00-7e-6af7f5f66627fc384a11beba9271-0002": 593, + "00-7e-6ce6eded2bdafdb4c896a293b998-0002": 594, + "00-7f-2d30539392f1c9229d9dd8b4023f-0003": 595, + "00-7f-652dcc750a4f7097bd676dcff6bd-0001": 596, + "00-7f-65c312d6cb47e0dedd869655187b-0001": 597, + "00-7f-9a6cc8b5f0c1bcb8fc909c4f6a67-0003": 598, + "00-7f-9f68d4a07af2ae4955de30d4394a-0004": 599, + "00-7f-af819c337b3fc6f22b1f1266bbe3-0002": 600, + "00-80-1d44b3ad253a99688ddca0b4b10e-0001": 601, + "00-80-4bf79ea543d79d94032f61d0a050-0002": 602, + "00-80-ae053e7613f7fd805e140698f221-0002": 603, + "00-80-af9f6b90fe17b47fed72f0d65de9-0001": 604, + "00-81-44b0e9639243aea754a942ad2048-0002": 605, + "00-81-9faadb3dd3b23ab326ec324234c0-0002": 606, + "00-81-d3f7477f7c03ddaeec94cc1158b5-0001": 607, + "00-82-6e3f2be01b29f872116b01f5e58a-0001": 608, + "00-82-7442208beb65e5075b94a565e62d-0002": 609, + "00-82-d356d088da870c3dc6d05533b88e-0001": 610, + "00-82-ff40b42fd59da76c9e975395c88f-0002": 611, + "00-83-9c9f8581163ff8c3389fcecd391d-0001": 612, + "00-84-342f6f55dc976f5cf6fa64543c1e-0002": 613, + "00-84-502d94892caf2c3298e7230dc964-0002": 614, + "00-84-79c2bb5bfc2802d2496338ae2f84-0001": 615, + "00-84-c9f19175ba51ba562a9509d351b3-0002": 616, + "00-84-ea93478d6968d240ed14c19b409a-0001": 617, + "00-85-1345c32971927b8a7bcc389eefba-0001": 618, + "00-85-1764f6c82efb940f9deecb982947-0001": 619, + "00-85-24f47dcd05ac8c580a86bbb78e2c-0001": 620, + "00-85-4cbca825460c4c19ff9129817fc7-0001": 621, + "00-85-5688793020ec2ca8b79fbb25a7f8-0001": 622, + "00-85-7da5395b612f6e3925dd3471dc5c-0003": 623, + "00-85-b1d33fbd85d68d0087b7727a25b8-0002": 624, + "00-85-fdb275ce619d692554bd69f9ab2d-0002": 625, + "00-86-04903887f137201ceb5e4cf1f3b1-0001": 626, + "00-86-512abb498e7a652bc13833510740-0001": 627, + "00-86-69925394a352b7fdb2977bc3bb88-0001": 628, + "00-86-c5a04330f0b34e34819a6043d2ae-0001": 629, + "00-86-cb2fa18ede7142be45778a84b0be-0003": 630, + "00-86-dfb40ada26f5583501a38bbe2828-0001": 631, + "00-87-2b3ab9b850de1c9c406be029f6f8-0001": 632, + "00-88-34e628db2d39a47ba3fe28e2ae7a-0001": 633, + "00-88-61fb06133836e0d8b2425672cace-0002": 634, + "00-88-875f02127d344b76088099745dc2-0002": 635, + "00-88-a16cdf56ef6b349396402cfc3364-0001": 636, + "00-88-a1c36e347c5a8ca18af82c88d94f-0004": 637, + "00-88-b780faea56440450dd3a376e845e-0001": 638, + "00-88-db76c1bbbae0e455af2d9d7e0104-0002": 639, + "00-88-fc0b6e6e2c613fcbada45c328e0b-0001": 640, + "00-89-1b51c0f8d0f33bc53639d667ffb3-0002": 641, + "00-89-5f33d30a76f52189698e89ef3d11-0002": 642, + "00-89-70387273b4d297e4b74ec0f1e8c9-0001": 643, + "00-89-9833ecf318911420d65f0910a15d-0002": 644, + "00-89-ab45991198e89de26563e01c778d-0001": 645, + "00-89-d117581eb365cb3cb831c450bd97-0002": 646, + "00-89-e6e0112334cf5b8c3b991dfe0feb-0003": 647, + "00-8a-6b6240babe193b75ea1b01692292-0001": 648, + "00-8a-700424938c0719218480682efe29-0001": 649, + "00-8a-7b817cb40c2749250487341f00ec-0002": 650, + "00-8a-a61965e4f90bd6973d80492dc1e0-0001": 651, + "00-8a-e218b9aec5c97d6b7adb51cf7d94-0002": 652, + "00-8b-1cc386878f2a8251d33e80a02e84-0001": 653, + "00-8b-a17bb32b81107b68e27c3c6ae068-0002": 654, + "00-8b-b76e3b3bbcc09a281c8cc290e0f8-0004": 655, + "00-8b-c3a4cf489378753d8af119bbc342-0001": 656, + "00-8b-ca21be495dedf56851acf1fca1e2-0001": 657, + "00-8c-2717b6e7d39bc32d6cc454a7a684-0003": 658, + "00-8c-29c0897772b4a4b0dad98f7b8134-0001": 659, + "00-8c-4b71e143ca3e9f0aefdd7fd1e82b-0003": 660, + "00-8c-53082fa710b0025d7a6fce36c8e5-0001": 661, + "00-8c-55dd1a961d1677ce887f064e5de7-0001": 662, + "00-8d-0b039079387c6c2add2881410aca-0002": 663, + "00-8d-7bdef238a3ff77b82c0131bc3dfd-0001": 664, + "00-8d-8dd2d17765f51e7101552854c72a-0001": 665, + "00-8d-a7cdc98f43c4f599353456ac6bf6-0001": 666, + "00-8d-b8a4d7853f0f8095f2cf15d3bfdf-0001": 667, + "00-8d-db0f4b5b04a8ff87f024c24cf4be-0001": 668, + "00-8d-f86d5a7d4004feafd987a7c066d3-0001": 669, + "00-8e-1ea22a29582cc7ef9c7584527243-0002": 670, + "00-8e-7527e82d1d446690620c41bc5bbd-0002": 671, + "00-8f-495cd67f399f1bb896b8544849b7-0001": 672, + "00-8f-7799db1fe69b0139876db7ea1819-0001": 673, + "00-8f-9cfb04be184e326c27525a9f222c-0001": 674, + "00-90-1728e96e5ace6217aa61f62f5954-0001": 675, + "00-90-6844b2ccee1693c6bfa10ec0b6f0-0001": 676, + "00-90-716398abd2a73a2fc45594715e28-0004": 677, + "00-90-996f2fbb1767d0a6d41cbe7e2979-0001": 678, + "00-90-b99da5e8eb129ec78c985631d125-0002": 679, + "00-90-ec32ec9c2741c3fa9fbdb0328da6-0001": 680, + "00-90-f31c46862a141f5c64f3e559ff4f-0001": 681, + "00-91-3af8a2d9bdb788538e9f3fd27dd2-0001": 682, + "00-91-b9b92159ae3ee76140b068348794-0002": 683, + "00-91-bc140fbfed44a66b7103ee2de4fa-0001": 684, + "00-91-cd35934577a05cb42010cfb2ea25-0001": 685, + "00-91-f2eb516fdd4e9f8f23c25ff39d17-0001": 686, + "00-92-03f9a47843be525fa2675397a1b5-0002": 687, + "00-92-11768aa3f46e249a23310d36ce38-0001": 688, + "00-92-32d42da775d0d0d1d2e8dc186c55-0002": 689, + "00-92-3bc690bfa9b5983e4787ab369f7e-0001": 690, + "00-92-401675c45ebd014f5baf58d4cd82-0001": 691, + "00-92-56e199a2f7643e80cfc20e3b7a2a-0002": 692, + "00-92-8666938dcee48dfb97323581562d-0001": 693, + "00-92-90e378b80f1275282c8c37780271-0001": 694, + "00-92-a6b102c7344d80ff7a56048c442f-0001": 695, + "00-92-b6c5d81709309848f45ca54522ea-0001": 696, + "00-92-d76284da45abcb81bc3cd61e5087-0001": 697, + "00-92-fcc10efdabdf0f7eea600ffc213d-0001": 698, + "00-93-2cc0763989dbed446811b3cb7ecb-0002": 699, + "00-93-2dff1776b0a03d6981f876be683f-0001": 700, + "00-93-a7fa51207fbed2d7a8e8cae94496-0003": 701, + "00-93-bac3e559055af0fed63ccc29f98c-0001": 702, + "00-93-c6f4bb7e2a30379889e70ac102da-0001": 703, + "00-94-862e861cdc31d96d1b1841dfb399-0002": 704, + "00-94-dd8050ec5e003a133bf4b01f9886-0002": 705, + "00-94-eb2e1cf96151a34a98687d294ac0-0002": 706, + "00-94-f61eaea7425673eac59c48256b69-0002": 707, + "00-94-f67e0eeabe977b9bebbf14d2ff44-0002": 708, + "00-95-0007d9966e092f31b29077131098-0001": 709, + "00-95-16a311cf567ba0cf32b6dc99e75c-0001": 710, + "00-95-26fd6440f367039533529e90cbad-0001": 711, + "00-95-77c93bdf6093b8f51ff08263b35b-0001": 712, + "00-95-b17e8dbfa5db5dd306bdd89df56e-0001": 713, + "00-95-c2037adf4e0920f93e911c775ff4-0002": 714, + "00-96-47e569edaa710f7a07d8ca72a3e4-0001": 715, + "00-96-5a81bc0e6f57f2fbffebf249fa62-0001": 716, + "00-96-97602f107dd825452c464609a8ae-0001": 717, + "00-96-bec5fa78ee02f5d27c03b9bebbf9-0001": 718, + "00-96-c0b08259482f81c43c5d6e012a0b-0001": 719, + "00-96-cb890250c18cd63495c11db65539-0001": 720, + "00-96-fc4b72bba341ead5bce1ed002cee-0004": 721, + "00-97-65ae35171a7448167a48dfe1d429-0002": 722, + "00-97-7dc714c2deaba6484de19a9eedfd-0001": 723, + "00-97-8b48d4d6a6f062e3d5120848d9f2-0002": 724, + "00-97-d8620022a377af77f52d26d3ea90-0002": 725, + "00-98-06b68b49a3df1f87bd15a99262f8-0002": 726, + "00-98-382e4b132e41b8334252e8f9514f-0001": 727, + "00-98-49272d0897760637d74f9669c838-0006": 728, + "00-98-895a42f8e0f64cc9c8cd183b9eb8-0002": 729, + "00-99-061449e0905d7220b875ef9d371c-0002": 730, + "00-99-1f0e619861f1787ec14d8fb4c460-0001": 731, + "00-9a-5833a01dc0b886e966155d5ff1c3-0001": 732, + "00-9a-fe939842d50a298c08ec32acac33-0002": 733, + "00-9b-183ee2a9daab0ac4f5cd27f04a79-0003": 734, + "00-9b-30de74869c60edb2292cb69b4e0a-0001": 735, + "00-9b-38b2fb53e0219c82eda510f53171-0001": 736, + "00-9b-5a678ec977ae1e17408c5811331f-0002": 737, + "00-9b-ad1590609b5524537a9aaf7bd004-0002": 738, + "00-9c-1b79abd5301c0234c477934705dd-0001": 739, + "00-9c-2b2ed1ad737b889e1853509e5a85-0001": 740, + "00-9c-592a21f423738380005920b59159-0002": 741, + "00-9c-61b801b2220ba1ee22d549a67911-0001": 742, + "00-9c-6b9514e3ccb39b737d85e05c9b72-0001": 743, + "00-9c-91e4e54580db6e46821df47f5b6f-0001": 744, + "00-9c-f8ad784636f8fc6351d344094899-0002": 745, + "00-9d-3e7d0c6eaed64feae837ae9d6ef2-0001": 746, + "00-9d-47797136baf9a78ae0ccfbfb0d36-0002": 747, + "00-9d-54dedfa3175cfb8303e59875d71e-0001": 748, + "00-9d-9d9dfaf711a2fdb28590b93d9f6e-0001": 749, + "00-9d-d61d9d27c82a3a674bfd5ef2e611-0001": 750, + "00-9e-0743e78eff36d383aecce7e41517-0001": 751, + "00-9e-402141464f6a6b25184a1b65d3d0-0001": 752, + "00-9e-5c756efd48a39291d8894938fdbc-0001": 753, + "00-9e-a0623036697f18e642c802edd8fc-0001": 754, + "00-9e-cb915c590878adab9c4caa9a0887-0001": 755, + "00-9f-19072343a601d07c7bcb21de9fdb-0001": 756, + "00-9f-44e1f2fe94998aec24b77e0cdc95-0002": 757, + "00-9f-57ecf91ebb75d96c717538681016-0002": 758, + "00-9f-7d3d9c56c5da8a26c755f9b5a039-0004": 759, + "00-9f-8d4c975e6a723077d2abb2c27ff0-0001": 760, + "00-9f-91b7810a3a2b1ce52ed47c34a9ba-0001": 761, + "00-9f-a8801101d8d69ae7186d75bfb794-0001": 762, + "00-9f-bf73b24543619ef6ac721cd10a1a-0002": 763, + "00-9f-c49f81777e4cbb22728f95c3b86a-0001": 764, + "00-9f-e904d1bd157e7b4ec5309b89301d-0001": 765, + "00-a0-3c288ddb69f1ccab5a2441fd3793-0001": 766, + "00-a0-587ba4afe2235c2181f945a6e978-0001": 767, + "00-a0-99ac6c1a44f7408f6a044fbcc27d-0002": 768, + "00-a0-a8f29b8799fe6e736b6a59c91197-0002": 769, + "00-a1-134f4ec811db34a383727d0b11b9-0003": 770, + "00-a1-261bc55d66255e9655c87a417573-0001": 771, + "00-a1-4cf1f55dcc7e56e90f3410f2431b-0001": 772, + "00-a1-63e9f3dfd1a1e5582d5570a54f50-0001": 773, + "00-a1-88ba24ff6cbb9a6fb9d1c26409d5-0003": 774, + "00-a1-9a641cb24517a95c66e8a8aab1a0-0001": 775, + "00-a1-d58f7960ebc0aca0fe5a89c597c6-0003": 776, + "00-a2-12e948b6bc2e67ea265edb1c935b-0001": 777, + "00-a2-16b5a4184148f492a29d9ecd8efb-0001": 778, + "00-a2-1c6c4258c80f56cbfe9febe8401f-0001": 779, + "00-a2-268eb9b5473a80591e5a3d6a2715-0002": 780, + "00-a2-86e95496266df6467bdaf1ae69d2-0001": 781, + "00-a2-bb507d29902a4fc4b442efd617ab-0003": 782, + "00-a3-398ec8ee7e2c92ac665812835988-0001": 783, + "00-a3-abfaaa9d066ce952e7db10d4c270-0002": 784, + "00-a4-6eebd3f0c3187579b412ef2e7d01-0002": 785, + "00-a4-81d00f284b6bdc0b2fbc1ddf8087-0001": 786, + "00-a4-eebb9c8dbce9b07c7269e63e91ec-0002": 787, + "00-a5-ab3c169f83b7e227ef9f7736e9f7-0001": 788, + "00-a5-f60a0a622ff7acc63e248a1cb19a-0002": 789, + "00-a6-14dc1367f7314964df5c115a5fcc-0003": 790, + "00-a6-242117ed53c0e4464bb2e5686b7d-0001": 791, + "00-a6-b92a9563328bcafd7e80ad229743-0002": 792, + "00-a6-f7732dfc6aef851f611c790164cc-0002": 793, + "00-a7-559376a5010651901a5b2dcc05f8-0002": 794, + "00-a7-c7ba40e362516339626b54fe3bfe-0002": 795, + "00-a7-d7d1cab66c09fadaf4df5366672b-0001": 796, + "00-a7-e513317f0f2e800d60600437d909-0001": 797, + "00-a8-2d2254851c7ac97ca1b52edacd20-0002": 798, + "00-a8-2fbbbcbb3986bb356a75ff328987-0001": 799, + "00-a8-40ca496a4b0966c5dd3edba541ec-0002": 800, + "00-a8-6dcef6e71ae937cd13e08f72b68d-0001": 801, + "00-a8-f89b26de5ed77d39526b610aed3b-0001": 802, + "00-a9-4244363c437d48516cde3ddcd936-0002": 803, + "00-a9-6b5811dfa286ebdb29513b0eb5ce-0001": 804, + "00-a9-864c47042f62dc1f183da89fa6ff-0001": 805, + "00-a9-90e2e6e5b449fadc2379cd37e61d-0002": 806, + "00-a9-ba57dff89bcea9cf9af5acce489d-0001": 807, + "00-aa-36abe9f3ea2a613921926121be8b-0001": 808, + "00-aa-567fe7b749b183d3d47c67f3cfb1-0001": 809, + "00-aa-5ed3c0386b7fd8972cd071d99d8b-0001": 810, + "00-aa-9f4885caf71d74c851ef13930e6f-0002": 811, + "00-aa-d0827e47a352e93893f2c0d3b01a-0004": 812, + "00-aa-d4c3dd78c3322138cbf79d6f80d9-0002": 813, + "00-aa-dc8beca3f064010ccc07a11d2554-0002": 814, + "00-aa-e86556147e9c22ca552676301086-0001": 815, + "00-aa-e8ac1e8d1d6e9a470a308d7f274d-0001": 816, + "00-aa-fe2a0db5e41a3da6523557c662ae-0002": 817, + "00-ab-25b6777e74d7cf27a4e6084df0f7-0002": 818, + "00-ab-72b8f839f4579b4200f71878d46e-0001": 819, + "00-ab-b2448cfd73342056b6f9a79990ce-0001": 820, + "00-ac-11b78a5971747f99a94336dab120-0002": 821, + "00-ac-3d2e1b18244cd93c22570c1fb82e-0002": 822, + "00-ac-653a926b570a1cf69d8f45bda938-0001": 823, + "00-ac-68a256c26da9b88e7b49f8612f16-0003": 824, + "00-ac-e4ed581f5f57720add3924dd47ab-0001": 825, + "00-ac-fbd4a0e4036987d36e53998e30d6-0001": 826, + "00-ad-18dd4b4c4120564463c29472b73a-0003": 827, + "00-ad-3660f716d1c17e0b7d90ef8e77c8-0002": 828, + "00-ad-a6ddd79d1c44a96f56dd44c2341d-0002": 829, + "00-ad-b5f184fd43a65b5754d99a83b3f4-0002": 830, + "00-ad-be3929c4aef118a4e7fefcbd26ca-0001": 831, + "00-ad-d070b0289e56c2b6983d143be5f8-0001": 832, + "00-ad-f57817ddbeff950b038c73c8e54c-0001": 833, + "00-ae-647cbfe20113e1a7dc382d0883ad-0002": 834, + "00-ae-975ec333f4ef3130215ff786fee3-0001": 835, + "00-af-0c923999024c8ef014e92c3a6e2d-0001": 836, + "00-af-1331bd3d3726315361c01e194071-0001": 837, + "00-af-518e7a7a2fa31b942ee751474647-0001": 838, + "00-af-566d17ab1a1b5550d00a3cb5b523-0002": 839, + "00-af-a23ee0aac0640ae0855cda72085a-0001": 840, + "00-af-c3e200ec0194cab3a1a16752b03f-0001": 841, + "00-af-cb438d6663a80fca767e2610cf67-0001": 842, + "00-b0-6793097f3063965cde54ec3302e7-0002": 843, + "00-b0-813eea1f888729bd957d37dfca75-0001": 844, + "00-b0-b378cc550cee82cb09d0035d211e-0001": 845, + "00-b0-bcc79d818bff6d99afa655b0ee14-0003": 846, + "00-b0-d4660b0a292e7d8c4c85e91116e9-0001": 847, + "00-b0-eca89029e6351af5b00fca24bdb2-0001": 848, + "00-b0-faa6541e199e42c382ab15ba4147-0001": 849, + "00-b1-12d5fdaad3e39967996e1d184b49-0001": 850, + "00-b1-1e488deba5158f52cf16da42c7fd-0002": 851, + "00-b1-3a6ac4761269c1a077b5f7ae8429-0002": 852, + "00-b1-7ff70b266ca6cdefb78fe3f8f008-0001": 853, + "00-b2-075c268ec0476dd84057385d9f3e-0003": 854, + "00-b2-13addc151248aa58f104a8ff0212-0001": 855, + "00-b2-8279e18ea7b109c4d3a6040bf2be-0004": 856, + "00-b2-a2a82254890ebbb6796a60546cf3-0001": 857, + "00-b2-a8fbd116099a7143500d5fca9863-0001": 858, + "00-b2-c32ec2a6ae0b65445e9e3069e587-0001": 859, + "00-b2-dd262a98a4eade28e2544c4f0bf3-0001": 860, + "00-b3-1dd4b79a5b67de2c754c3c108b6f-0002": 861, + "00-b3-2c92909f7a203c417d5ae206e470-0003": 862, + "00-b3-5051ee8ba2150d403a76db7098aa-0002": 863, + "00-b3-968382052ab0356666cabb903921-0001": 864, + "00-b3-de8a09a1c4409996b9a3bbce66e6-0001": 865, + "00-b3-e372427fe595d42ee5a56b9a2b2d-0001": 866, + "00-b3-ff867eb1ed8da2d062aec29a2c8e-0001": 867, + "00-b4-1b713fdf4ae414c2b670a622bca7-0003": 868, + "00-b4-eeafbd2cd8e7842e172e63850c24-0001": 869, + "00-b5-0d29f9f1125eb04efca49bfbe1e2-0002": 870, + "00-b5-66bb299549e3f081a49136c13a21-0001": 871, + "00-b5-74da8f362322cb1a921a9868d022-0001": 872, + "00-b5-d6362d0a03f7fa2837897c131015-0001": 873, + "00-b6-120b827f05f0436c2ba0aa01a516-0003": 874, + "00-b6-5463b1191e3b778b8fb528361f24-0001": 875, + "00-b7-15283e4b4399795ef67640dc8479-0001": 876, + "00-b7-184e77d501eb8b4046dbb78050fa-0001": 877, + "00-b7-2430396896f50c862733cd004418-0001": 878, + "00-b7-2b86768811156ff29b574a0b87fd-0001": 879, + "00-b7-3c65b74fc9b00b9890e55297b5c6-0001": 880, + "00-b7-5ec31f19426ec4a7ad933292c40b-0001": 881, + "00-b7-72199fabc0b916d57407d72e7378-0002": 882, + "00-b7-8427fca5fbedb93f3a3747f9b2e6-0001": 883, + "00-b7-b876745f47ccf4a061434134113a-0001": 884, + "00-b8-1739439c72fd7194de374476ecef-0001": 885, + "00-b8-57a202294759d3f7ff9bac0d78c1-0002": 886, + "00-b8-7fcb771ed7d01c11f8949444bffa-0001": 887, + "00-b8-95f2a6d2be3e4e77521570fa4d89-0001": 888, + "00-b8-a6aadc5e20b59e0b10a7df524bcc-0001": 889, + "00-b8-e30da423670ad18d30f8da07af2b-0002": 890, + "00-b8-ea697ad4fcdaae86b5fa0646ba32-0002": 891, + "00-b9-473a14f7f8aef79b226249fd15fd-0001": 892, + "00-b9-863dec1920f42941a0e37522bac8-0001": 893, + "00-ba-0c724609be240ccbfe309f58c3a2-0002": 894, + "00-ba-40bee5d1460ab6a96b50f705f46c-0001": 895, + "00-ba-89d6a09cdbe1f1f58a0e0ab992b6-0001": 896, + "00-ba-920e19acf3102922b7da57177f4c-0002": 897, + "00-ba-ff8ddeff8234590164539b8eb56d-0001": 898, + "00-bb-289f078929212d5e9ffe6576d393-0001": 899, + "00-bb-76fddc7b2adfd0c15c13c79d9571-0001": 900, + "00-bb-b29945b53565313569c9429eda22-0002": 901, + "00-bc-1231deb2fcdff857bfe836a028f6-0001": 902, + "00-bc-15ac2222aeca57349ec83342826a-0001": 903, + "00-bc-5b31c8e950ce06062679383d3c37-0001": 904, + "00-bc-5df65458bf904d88c68e025c2ac8-0002": 905, + "00-bc-db89650b7dd49a78239b486e79f8-0001": 906, + "00-bd-1b2e883b3ff1d9eb98325b137485-0002": 907, + "00-bd-465729bde0415937b9f497dc9823-0002": 908, + "00-bd-7ddb2ce8b744167f9fad8763918b-0001": 909, + "00-bd-d95d7948c7d1204d4463bd827268-0001": 910, + "00-be-03efcb7f7ea02d39e28abfe0a344-0001": 911, + "00-be-0c8273a321409aa8a7271aec3664-0003": 912, + "00-be-10aba454d7e07437d426d5aa805b-0001": 913, + "00-bf-572ea720032d4e3647cd9ed6060f-0002": 914, + "00-bf-d27242956cd059e5b0a3c7705faf-0001": 915, + "00-c0-0cf6c33cdb43a7102f26e86fd328-0001": 916, + "00-c0-6409f6403d40e849e5c02c1196de-0001": 917, + "00-c0-ba5a26208cce331348256e7c7e48-0001": 918, + "00-c0-c0ab8ae3955ff9baa26d47308a21-0002": 919, + "00-c1-0d2d1846822f446b6f0487f23b07-0001": 920, + "00-c1-34ebdff423a0e7a3bf60e14a25e8-0001": 921, + "00-c1-503d01f99fc45c5db14c098b40b9-0001": 922, + "00-c1-585bac477524ec8742ffd0073e9d-0002": 923, + "00-c1-68b676a4edf305e194a822752449-0003": 924, + "00-c1-840f1941ca9fd0a371e3002323f0-0002": 925, + "00-c1-cdf820e9f562bacb1ee09a6d559c-0002": 926, + "00-c1-d2cb88b2a2a9cfd9205f6616491d-0002": 927, + "00-c1-d838db34e6580d6b0a1f808fef21-0001": 928, + "00-c2-0a2b2ec50ca5d6f50bd0e2f950c9-0001": 929, + "00-c2-5d6f7e194694d2655d3507f05b74-0001": 930, + "00-c2-693fc58f76319bf19d59cfbd8b72-0001": 931, + "00-c2-81c133746ede012447d477bb3239-0001": 932, + "00-c2-a68e891d2dd000b23fca2750a154-0002": 933, + "00-c2-afaa47ac9cf88f80e6248e49e36c-0002": 934, + "00-c2-bfc92bfc0a2c2669ee0684d253f8-0003": 935, + "00-c2-dea09dc7101697d6171419fe2ac2-0001": 936, + "00-c2-eb2d0b592e8c05d4d7a68915d9ea-0002": 937, + "00-c3-1d20c8a08bc46e97e44a16dfd532-0001": 938, + "00-c3-57a95d84d41c63143fd37ef4daf7-0005": 939, + "00-c4-12b4a5bf7f26d610037d05107dea-0003": 940, + "00-c4-3cd2ac88713100b7cd5547d822a4-0001": 941, + "00-c4-cdb6e29eb8bd9240785f14c066e2-0001": 942, + "00-c4-e0e20f12601f414db8a11d66586f-0001": 943, + "00-c5-0876b4daa09f2a6ec4d7ec6d286c-0001": 944, + "00-c5-08b3b816ff821a428baa475d157f-0002": 945, + "00-c5-12af0da2e0d4778d8de6beb3e708-0002": 946, + "00-c5-14c43a08810b287132a2694736ec-0001": 947, + "00-c5-4273e722950681f4bad0fcd67ece-0001": 948, + "00-c5-77e358fd2c840055bb3e5ebbcb16-0003": 949, + "00-c5-7d4828395caeafb9636dd1aa1a3b-0001": 950, + "00-c5-bb2c685262fa57b822ef303c486f-0001": 951, + "00-c5-d0113c0b45371d851d505354dda3-0001": 952, + "00-c5-ddb8d38a91756be8598ed117378a-0002": 953, + "00-c6-06518746be98d797815bd907b17a-0003": 954, + "00-c6-62080c91bcacecff4f14b7b150c4-0001": 955, + "00-c6-9f51de37eabb3c82e601322f602d-0001": 956, + "00-c6-a821c510474e26dedc88448b7e96-0002": 957, + "00-c7-7bf474827a2276c92c423f7221c8-0001": 958, + "00-c7-8bf4f1af02519e55a616eb6bed08-0001": 959, + "00-c7-92deffe30b3d56680ec90c0256b9-0004": 960, + "00-c7-cf0f1f349aae246c74a9df091b2f-0002": 961, + "00-c8-10e00c2c1691697d5a4331a004d6-0002": 962, + "00-c8-47537351bf7e9518a156df6e7a5d-0001": 963, + "00-c8-5a575c36cfcf525129a1837a390b-0001": 964, + "00-c8-6f7c8ff83f42f2b88e0fdce5fec4-0001": 965, + "00-c8-82a7c14ad773c99893b9e4109686-0001": 966, + "00-c8-d2586b96e43ccfc551906b66e0d3-0002": 967, + "00-c8-e99cafc29ab200195bf078202a1d-0001": 968, + "00-c9-0c251dd50a12989876aee9a98cec-0003": 969, + "00-c9-35714cb537f5d1317c1373183446-0001": 970, + "00-c9-42939c696d3f278f358d491c2826-0001": 971, + "00-c9-4e3c499f5152c0b205c52e022e3a-0002": 972, + "00-c9-56d636962184c1e19423d7ffa2f9-0001": 973, + "00-c9-8c9860236119c9dcbc96e8413bb0-0001": 974, + "00-c9-997c9ccd2a21165268203c241b40-0002": 975, + "00-ca-020996d4069d28cc77a694b9d9b5-0001": 976, + "00-ca-16bcb45c40fbe932d95c05d1abdc-0002": 977, + "00-ca-21c54ed8d41b01aca766859be31f-0001": 978, + "00-ca-613235676eb3a904dd9e3e6cf98e-0001": 979, + "00-ca-6ef386790354b367ba7b4c8d4fae-0001": 980, + "00-ca-fb516076ef8a484b7e4dc2bafa9b-0001": 981, + "00-cb-23a5b950c7ea434bbaa4f62c845f-0002": 982, + "00-cb-26b532652801bb77dd6cc5b9f435-0002": 983, + "00-cb-353d2ed51e6b1467778e9369ccc2-0001": 984, + "00-cb-49d2cf80568ad80ffab0571e3101-0001": 985, + "00-cb-a1548a1c331af96c386787c66d70-0002": 986, + "00-cc-65453e8584959152496a78b9446a-0002": 987, + "00-cc-cc53e6b845610c32b28996ef0f35-0001": 988, + "00-cd-12b27c679653803abe57456e8e02-0001": 989, + "00-cd-911e5244b10cc606350f70882845-0001": 990, + "00-ce-551266aa303931efafd495080c0e-0001": 991, + "00-ce-6b31df7ae5a25a301a66534e3b0d-0001": 992, + "00-ce-aad6842fd53e28422c67ca476918-0003": 993, + "00-ce-b634b62b0d83a6fd2e3efdd1a8a7-0003": 994, + "00-ce-e8fc86220fb25e8237658787d9b6-0003": 995, + "00-ce-ec088d123296ff30c8ce60da904f-0001": 996, + "00-ce-f1aedb3c78bb53b836543039a8f8-0004": 997, + "00-ce-fc188fc815f067b5e3585c9d5c87-0002": 998, + "00-cf-039320ba3229554ed14ce25666dd-0001": 999, + "00-cf-0c31d96e72ce00c10ebd57b9d6b7-0003": 1000, + "00-cf-be68f96754959b6d8d7e451d9fde-0003": 1001, + "00-cf-dfea20b504f368fa2f24a583c32b-0001": 1002, + "00-d0-417d933da35f8a604a91baf7925e-0002": 1003, + "00-d1-3d8563ca9128c601f2ad9a21be81-0002": 1004, + "00-d1-6124b097ac8fa649f65de8849243-0002": 1005, + "00-d1-9091e14f55823f03f411ed109329-0001": 1006, + "00-d1-a4f89c1738c717321110a3f72ddf-0002": 1007, + "00-d1-b3e40cafd236b1ab4c954c915c49-0001": 1008, + "00-d2-28d4c163b2cfdd80a51368b5918c-0002": 1009, + "00-d2-926c82ca06f403832c150bbcb17b-0001": 1010, + "00-d2-ae6254dd3aeab6823e2fd53a5a65-0003": 1011, + "00-d3-390a35fd41886da06fda17720f99-0001": 1012, + "00-d3-8e44ea32b5b187f3c5da40a0fa6f-0002": 1013, + "00-d4-821a0fcbdd0b270a39307387f05e-0002": 1014, + "00-d4-956d4803769d26f975286fa44839-0002": 1015, + "00-d4-e58b988ca37382b2f4f1938d7a40-0001": 1016, + "00-d5-6503a38e71b5631b55eeb58dea1a-0001": 1017, + "00-d6-0b4d52164914b24d4471021e505a-0001": 1018, + "00-d6-a47568f9a130fe74b32dc69cb209-0001": 1019, + "00-d6-bac0210197c2280e78c850c8df18-0002": 1020, + "00-d7-115907a2bc9ea877369993be9417-0001": 1021, + "00-d7-17266af1d8358ac1587b8df8b63d-0001": 1022, + "00-d7-22c42be86ef50a840b451d923202-0002": 1023, + "00-d7-22f59138a1ba506e57b0c46879fa-0002": 1024, + "00-d7-244140826fd38226c0835869d4cd-0001": 1025, + "00-d7-368b73bc23554c9ae4bb8e5625de-0001": 1026, + "00-d7-505e1404c31aaa0238f772770a65-0001": 1027, + "00-d7-86c3c59f7bbd5d236c31019e150d-0003": 1028, + "00-d7-8f92b62ee6f19f76ebaef5208aae-0001": 1029, + "00-d7-ff94532edb1a382dc1b3c934d929-0001": 1030, + "00-d8-02e36acebed34cce11a1782f8ea6-0001": 1031, + "00-d8-57c77dbb57a061622d99a7d0c712-0001": 1032, + "00-d8-5f9f47702f3057bbba24d7577460-0002": 1033, + "00-d8-8737475851f5a2ceaa826d46da69-0001": 1034, + "00-d8-8badb3c7882af249741bc1405e16-0003": 1035, + "00-d8-92d7c54ed16b07c995583be651c5-0001": 1036, + "00-d8-a0050e5262c54d9f89a88bcec135-0002": 1037, + "00-d8-ca2da16d7410fa6935dfec3294df-0001": 1038, + "00-d8-e61ac8251f7c7da49eb8058a96ab-0002": 1039, + "00-d8-f29f79262f0132893284e039700e-0001": 1040, + "00-d9-096cf2353c7d7adff4611e3503e7-0003": 1041, + "00-d9-0c5ed9db9ae62d9a61cdd2aa2a66-0001": 1042, + "00-d9-19893dd46c27316b6db7097192ce-0002": 1043, + "00-d9-2878570522b21676e0d479b5190c-0001": 1044, + "00-d9-3998ee391e2c96fd2ef7fdf188e1-0002": 1045, + "00-d9-443a947128840fd35d66d6ad719e-0002": 1046, + "00-d9-5234ebc6d20b346683f85f2c1ef5-0004": 1047, + "00-d9-d420a3a790e1d53457b60cbab01f-0001": 1048, + "00-d9-d9fae62e4b887545088d49bd3713-0002": 1049, + "00-d9-e0c895c56a51e04f03671b0dbee1-0003": 1050, + "00-da-22acc47952aa2f90b693eda6d8f7-0001": 1051, + "00-da-5de9a24dffce5f43178bc0266d9a-0001": 1052, + "00-da-6034cda95372280734b0921747ac-0002": 1053, + "00-da-a0ab0b59584c1f439a2cc03d2b72-0001": 1054, + "00-da-c5625ee56452bdf7c658f2f778f4-0001": 1055, + "00-db-5e45044c8418dc720482c62289ee-0001": 1056, + "00-db-7f2a9a055b4f9db5387364c4ddf1-0002": 1057, + "00-db-854539f2c7c38dfffab02bd95917-0002": 1058, + "00-db-8f6aa1092bdd800851e2abcd3912-0001": 1059, + "00-dc-06767dff29c704406b41fefcd3aa-0002": 1060, + "00-dc-2d32811f823d5bcf9ad96b9b59de-0002": 1061, + "00-dc-33b0ed9c75991175dee3a4d822c0-0001": 1062, + "00-dc-755e5d152846d214458df32d3dd0-0001": 1063, + "00-dc-7ae55de964dff3d3daa2df6c8940-0001": 1064, + "00-dc-801e60925056ed75e078652c240c-0003": 1065, + "00-dc-8fc047ceabb2a39332b5a14bcd7a-0001": 1066, + "00-dc-b43e322a5c9f9cd14d021a94983b-0001": 1067, + "00-dc-b7f400ccbe887dc4296c0981de6e-0001": 1068, + "00-dc-d19b99be0a94fade333d11154a9d-0001": 1069, + "00-dc-e5696d1690110a611e2e6c1fd394-0001": 1070, + "00-dc-fac3a6d1fd2120f667905583d956-0002": 1071, + "00-dd-0426cfb549ea07d79a4e70637f75-0001": 1072, + "00-dd-0f7c63a96f96ed08fc5bde3fa60f-0002": 1073, + "00-dd-3516fb793262704806e0cffa60c5-0002": 1074, + "00-dd-49ebf2d404adb35887d188e513fa-0002": 1075, + "00-dd-7661daf24a0db3cad58ae3ff7b03-0001": 1076, + "00-dd-7c16aad08d0c37ea177c1f4e3898-0001": 1077, + "00-dd-cdfd65f1b6e1db9ac74b2dde127e-0002": 1078, + "00-dd-e8b1949cbc79d81c96b2352097d2-0001": 1079, + "00-de-03545316f7c0fd277a8f3fb1a296-0002": 1080, + "00-de-0a0e52e3b9522d16d139c6770eee-0001": 1081, + "00-de-50ad6befbf19f569e7c54e7ca5dc-0001": 1082, + "00-de-b22ccc9c3633c520c66602f4537b-0002": 1083, + "00-de-dd75aad43beb563fd4f66b72fd6f-0002": 1084, + "00-df-541cf1b4184d4b7528bc6181cb7e-0001": 1085, + "00-df-834856bc7d4ef834ba72f4cfaeb9-0001": 1086, + "00-df-9cb31dad7f999eb0efc66361d628-0001": 1087, + "00-df-a08d0f9e1cf97ee1e18d3f209fc4-0001": 1088, + "00-df-ab9340a02711a3f67574cfd15dfa-0002": 1089, + "00-df-db139ce1036eac867a28437588ba-0003": 1090, + "00-e0-0a4a909fe83c26577f923d0b2a8e-0002": 1091, + "00-e0-4618d0c7f97a98f28b18dd27e5bc-0001": 1092, + "00-e0-69e7e1ec845482a9f370ba64e31e-0001": 1093, + "00-e0-80cc90131f5b4fa2dfd4ea143862-0002": 1094, + "00-e0-b450df3961844d20aecef10f6f5c-0002": 1095, + "00-e1-3c50444387fe923a1fffc849cfe0-0001": 1096, + "00-e1-6285e27976a4cc8b9b0a20a43c84-0001": 1097, + "00-e1-782484dd61daf03bebcce2398e60-0001": 1098, + "00-e1-7d5bb7f060c7b3beae6cf33d58d4-0001": 1099, + "00-e2-1e788871561e94e3237ce369f889-0002": 1100, + "00-e2-1f1561c00a2aa48e6d783a43a338-0001": 1101, + "00-e2-25d3de717094ac237fb5f77aca7e-0002": 1102, + "00-e2-2ffc6bf4c04343f74df0ed9ef979-0001": 1103, + "00-e2-3ac2a17dcef1cb1c7236bb62e3c3-0003": 1104, + "00-e2-3dd96b4185c5f274a8ba7a68cd30-0001": 1105, + "00-e2-46bd579980bcd7773413b2c26bbc-0001": 1106, + "00-e2-8ce8b049f52931ddc456f4efd4a4-0001": 1107, + "00-e2-b48054931c165ff453f4ac8c66e2-0001": 1108, + "00-e2-f5c9295e24290c103c5523e5a890-0003": 1109, + "00-e2-ff76e4f3f3254ee6a3225b779758-0001": 1110, + "00-e3-7e0a9f1545560e489b3b8e86c477-0001": 1111, + "00-e3-852ec7f8d1a38f23c8d7eb36987e-0002": 1112, + "00-e3-8e893786eb91a348e09d2365d275-0003": 1113, + "00-e3-a29b00609a043143caf726346535-0002": 1114, + "00-e3-a9fc5a8c4a36b98458d5b7486917-0001": 1115, + "00-e3-b9e3bf18a14870f17483f12ccbec-0002": 1116, + "00-e3-fc8744032ea4172aeeb47f4a8962-0003": 1117, + "00-e4-5271b7c684ebc9b6939c7343ce4d-0001": 1118, + "00-e4-57dac5aa99936ecd44ba4cfcffde-0001": 1119, + "00-e4-6bf09e85bf1da138b85037f0491a-0001": 1120, + "00-e4-79cf868060523925e25b79d54414-0001": 1121, + "00-e4-ef7746e2234ea98abe836c8914ac-0001": 1122, + "00-e5-193e6a50493975fa515ea541a630-0003": 1123, + "00-e5-6456690805e52bdc7642556658c1-0001": 1124, + "00-e5-7a3751790fbb5732233fe7270d66-0001": 1125, + "00-e5-a00ca92dc0034778187d03ac4203-0001": 1126, + "00-e5-cbf5629e5dc1da8bf98fb4ea722e-0002": 1127, + "00-e5-d3f7af295ff2a87636bc966c25ad-0001": 1128, + "00-e5-d4eeaf0f2f417425ffe0d66a459a-0002": 1129, + "00-e5-dd22282eddc04be197d652fe0151-0001": 1130, + "00-e5-e6ab7da74713ad97ea5340d6eba4-0001": 1131, + "00-e6-21e614fe1146a55c5ace9fa5713d-0002": 1132, + "00-e6-338dd01ea03471d770e4317164cb-0002": 1133, + "00-e6-f10ed0901d55f9335c8e1df23fbe-0003": 1134, + "00-e7-053142760c20f0a36f7212a787b5-0002": 1135, + "00-e7-1a5d7836315fb36b374cefa84b0d-0001": 1136, + "00-e7-4fc96d1306193fde7ad5862c7807-0001": 1137, + "00-e7-b5042f39dfc3bdc8d0a9db6557e2-0001": 1138, + "00-e7-bd4c8f0e406d1b9dd60bbcc52993-0001": 1139, + "00-e7-d629ac391e821858000559952200-0003": 1140, + "00-e7-f45908e036c70d6c0102d162c8aa-0004": 1141, + "00-e8-21bb0ab46cecda11c9c053a0a113-0002": 1142, + "00-e8-22d87d8d99ee0f48c8bddff0bbaa-0001": 1143, + "00-e8-65cd4059395fff2264b039bf4f97-0002": 1144, + "00-e8-b45d6c1e0db30825045930587c9b-0001": 1145, + "00-e8-be6522f2a716a01c8bab72a94bca-0001": 1146, + "00-e8-c20d3ae2474656ce7c861c54dc1c-0001": 1147, + "00-e8-e70079a391f8edefd716462f161f-0001": 1148, + "00-e9-0803d60840267f4c2f0d7c3b92ae-0002": 1149, + "00-e9-111d68f886533a1f9dddce5b3a10-0002": 1150, + "00-e9-27e32e2bb3ed3ef921ed25e1dd41-0001": 1151, + "00-e9-48ab09f3d3d73b27725a6385069b-0001": 1152, + "00-e9-ae873546f0fcaee164c542933f7a-0001": 1153, + "00-e9-ce74539dfe011607dcc4df90263e-0001": 1154, + "00-ea-162ff08065248274d352376d9daa-0001": 1155, + "00-ea-33a2e25c6a0aabb8c81d4b8f65c7-0002": 1156, + "00-ea-58c3c9dbdf734dee1b7b7417321a-0001": 1157, + "00-ea-768d5cd524d23ca06b866ea9e09c-0002": 1158, + "00-ea-df025d50949c1e82e255b98565f8-0001": 1159, + "00-ea-f85406a1963a17bd62126a3548f4-0002": 1160, + "00-eb-3b2406181f67cee60787e50697cf-0001": 1161, + "00-eb-8142cb919a5457991ffa19008fae-0001": 1162, + "00-eb-be2b8df5477b55347d9ad5e5625d-0001": 1163, + "00-eb-d0c3463046c08f2cd496ef17ba6a-0001": 1164, + "00-eb-d3e11e4300618f87482f75152a5f-0001": 1165, + "00-ec-36627bd0eb1c878c7da7575a970a-0001": 1166, + "00-ec-643d2bd281640c7a66d6ae5901e9-0001": 1167, + "00-ec-7363cee7a58a115494e3fa914028-0001": 1168, + "00-ec-890773358b7e150d6e40821a6386-0001": 1169, + "00-ec-8f3003633dc14bf32f87e92f2681-0002": 1170, + "00-ec-a32c7f52bf428b7e259c8d6548c5-0001": 1171, + "00-ec-b91df095ead28420026135e3f4fa-0001": 1172, + "00-ec-c313fa523d94f71ed15c21d18b72-0001": 1173, + "00-ed-02b951009042a341fc39de917316-0002": 1174, + "00-ed-446ff6f71b50fbdf2fc19e197083-0001": 1175, + "00-ed-45aef4be2ff77255f39fe9549951-0001": 1176, + "00-ed-6499d83cf71792444e1390cba942-0002": 1177, + "00-ed-8134c80c34bcc7dac74c2ad75e5c-0002": 1178, + "00-ed-9aeefe50cc6c15942aeeb44b5a0e-0001": 1179, + "00-ed-ade2a47d0b72017d065b4eb7e248-0001": 1180, + "00-ed-b3b850fa678b875c20c3fa34800b-0001": 1181, + "00-ee-061e2c78f96c0f774799c8d8ac91-0001": 1182, + "00-ee-0e048d3f7b6155408a2b883e2d19-0001": 1183, + "00-ee-18fc5cf31bd57bf7d409cc41e5a1-0002": 1184, + "00-ee-53fddaba611dd414b1f0948b2a5a-0002": 1185, + "00-ee-6fa2f2c9e50e9fb25c5c53e67c8c-0001": 1186, + "00-ef-043a7076b02ae8f806bf33cb916f-0001": 1187, + "00-ef-44e2a86aee808e6ed397640208be-0002": 1188, + "00-ef-71013f31d541ffcbc6e7ef3e8637-0001": 1189, + "00-f0-0931f493d6275d1bdd54d0c7759f-0001": 1190, + "00-f0-30641d4bd82fec6ab7f8e7753de5-0001": 1191, + "00-f0-33984c8381ca8d2dc3e03823b7f5-0002": 1192, + "00-f0-34d02d111c2cd67f6d15684b011b-0001": 1193, + "00-f0-3e9a26c7f10a0c08f21656de0e32-0002": 1194, + "00-f0-6cdddaa8ea203f1f9ff28ff9d98b-0001": 1195, + "00-f0-7f71707fcf0e7bf76126cd6ac9fd-0001": 1196, + "00-f0-957211b6752ec6fcce3991343ebe-0001": 1197, + "00-f0-d98d54736a7585dcbf71d14f68df-0001": 1198, + "00-f0-f6158f02935bf82d084d3d1d9348-0002": 1199, + "00-f1-162b971d5e1f915472d7608f084f-0001": 1200, + "00-f1-193901694afa10fd9c7223702c22-0001": 1201, + "00-f1-59e92a9f55173c88fb865f0a9789-0002": 1202, + "00-f1-5be0b2d75ee750bfe4390d10c2fb-0002": 1203, + "00-f1-61563df41b3bf8a9d5e478f9f81b-0001": 1204, + "00-f1-8913a41bc3230fefb95907d1894d-0002": 1205, + "00-f1-8f5d470c68836ada2c4d7e4c0f9d-0002": 1206, + "00-f1-d17533c625616df4910e3c21f050-0003": 1207, + "00-f2-8b3151c70c9babdf4c2c5a1b5647-0002": 1208, + "00-f2-d3b989789bb071937efb476e8869-0001": 1209, + "00-f2-d519f195dd49dc5b1fc52ddbf304-0003": 1210, + "00-f2-eedbedc19fc8799b1914d2c0a5f0-0002": 1211, + "00-f3-323cffa2c183f86cbdb123cea0ab-0004": 1212, + "00-f3-93ac56d3c0e16ff847f8d5b43887-0002": 1213, + "00-f3-bc58864686bcaacb0e2ed17e03ef-0002": 1214, + "00-f3-cdd1a71c7121907781d5695e9291-0001": 1215, + "00-f3-f7d5b9ceeef40a5a870766d94f64-0002": 1216, + "00-f3-fa99bf71d6a555af74c921434be0-0002": 1217, + "00-f4-39f65bb79ebb6b27329825662166-0001": 1218, + "00-f4-7db58b8b2d47ad8ba13b6fc7e399-0001": 1219, + "00-f4-7e2496b2a5b391172c5168be3fda-0002": 1220, + "00-f4-805f72e2ca19c218dc51c5918a81-0001": 1221, + "00-f4-93fffcc11e896d54b180b83c8ad2-0001": 1222, + "00-f4-b4bc1f972889de6e2687b20f358e-0001": 1223, + "00-f4-e3c3e9ae39e93a3d7a33ad2417a8-0001": 1224, + "00-f5-1c4cb396de2390c24394d5209d25-0002": 1225, + "00-f5-21893396a95c96959a224bcc11cc-0001": 1226, + "00-f5-495738ae3d3bfbd38c49aded6861-0002": 1227, + "00-f5-b639e796cb43195a0c224e6c28a7-0002": 1228, + "00-f5-f24407e7a61d57fb715cc16a64c5-0001": 1229, + "00-f6-18248abd89259cc751bd9c18dec5-0007": 1230, + "00-f6-3cf19b30f937dbff7c0d040ef8b0-0001": 1231, + "00-f6-88086072020f0d4bb53af5de0d71-0003": 1232, + "00-f6-b64cc7ea264ac428f58fb27e4893-0001": 1233, + "00-f7-27f4c1cdbac9c3898c8a17f86167-0002": 1234, + "00-f7-43475706834ade0a723575abf62b-0001": 1235, + "00-f7-ef2144edae1f9b8ab2e20bd03fac-0001": 1236, + "00-f8-085353f3aab0d9a5b5c387bc518f-0002": 1237, + "00-f8-abec32e383776f44da1f3d700b51-0002": 1238, + "00-f8-d4920e7fe571eb502bd78a022868-0001": 1239, + "00-f8-ed4bfab884a62c504754eafd3cca-0001": 1240, + "00-f9-18cd98f7270de01294ef6568906d-0001": 1241, + "00-f9-7546cf09581061411af29d2496ae-0001": 1242, + "00-f9-a25166192d984c0a6da41700fa1d-0001": 1243, + "00-f9-e85c0db186da2f093f7d04ac3688-0001": 1244, + "00-fa-2860d91864aa42377a2734ae1c4f-0003": 1245, + "00-fa-82c7ed95b6b5b9ef17c40fd242a5-0001": 1246, + "00-fa-93811d9fde84227becfefd2ebb89-0001": 1247, + "00-fa-9d8915970040684fb799f9ddcc18-0001": 1248, + "00-fa-9f6e88089128a82d553a2ce07192-0001": 1249, + "00-fa-cc7b67a5233c44f6733bb1ee6961-0002": 1250, + "00-fa-e529d38593922fea72e3e72e39c2-0002": 1251, + "00-fa-e5ecf91ce28e4c7cc914757a0fa7-0001": 1252, + "00-fb-5864064af2b2d81613aa40de1c09-0001": 1253, + "00-fb-85fcfc2386827cb01165e4f21ac2-0001": 1254, + "00-fc-4c60c89f68cb6b3852fcb4b68f10-0001": 1255, + "00-fc-6341475b4c4efeaceafaecad0936-0002": 1256, + "00-fc-73b1ae176659bf7f182ce982dd24-0001": 1257, + "00-fc-c2204f90ba5dadfe6cbf19f758c6-0001": 1258, + "00-fc-ebecf5004b43b3c3825c809d541f-0001": 1259, + "00-fd-0c59d9e480186a58694ed8f4d44a-0002": 1260, + "00-fd-2c753bd832df1a6bbcf9ad6d2c09-0001": 1261, + "00-fd-d0aa66e63bf3a9de79462e6add9d-0001": 1262, + "00-fd-ef48859675fe75db72bd946fed3b-0001": 1263, + "00-fe-71f9348e6eea369bc7c7f0ba2cf0-0001": 1264, + "00-fe-794cd97cc9b8be7ab922c458a017-0001": 1265, + "00-fe-82303e974205b857f9931b2fe664-0001": 1266, + "00-fe-d86ea7a301cdbec34a1952a2fca0-0003": 1267, + "00-fe-e36380bf5f2d9bc9bbda55b71356-0001": 1268, + "00-ff-13fddb3e048ff040341a222c28c8-0001": 1269, + "00-ff-3688b0a602bf33caa3e0360e38e4-0001": 1270, + "00-ff-52221ae623eaad44f2ede461b4a2-0001": 1271, + "00-ff-59851b49229d7f26b5f0555fa9e2-0001": 1272, + "00-ff-6ac1b3001ba913ad42da4d3d7909-0001": 1273, + "00-ff-7bf3776210359320c496320d44fc-0001": 1274, + "00-ff-883c3fd1c1771b7324c1e03347ff-0002": 1275, + "00-ff-b55313090ca9af48467d1d4c1786-0001": 1276, + "00-ff-f2533c1dbe8b3cbaa36908596824-0001": 1277, + "01-00-22b0fe94d8814696f80a48f587e4-0002": 1278, + "01-00-777f0cd6bea0ce338ddf9924a293-0001": 1279, + "01-00-93fd564988087e449a04d0207999-0003": 1280, + "01-00-b40c38a4777e5fcc01e22fb60c64-0002": 1281, + "01-00-f88a5b66aac528a096913478f08c-0001": 1282, + "01-00-fcb1f98fd4b1b1b7b3c8f3be7cca-0001": 1283, + "01-01-15ca773e6991e9f4336e3db9eaa5-0001": 1284, + "01-01-26c6bc76f236662b4b920c0577d7-0001": 1285, + "01-01-4b5694791427f03089c51c04c9e7-0002": 1286, + "01-01-56720ad2e04d292e89fe958d68c4-0002": 1287, + "01-01-589fa71b21d87621aea079f5ea5d-0001": 1288, + "01-01-9c0f32382803d5477e1a49b1e15f-0002": 1289, + "01-01-9e796a6085b11717fadc8d6749ec-0002": 1290, + "01-01-a165bd52f3640044f04acc9734cf-0001": 1291, + "01-01-ceb018a9777f368eb0acac6500b4-0001": 1292, + "01-01-e4b133be80caf00e702d0f5bfb34-0001": 1293, + "01-01-eef18bebce113dbb27d77c83f9be-0001": 1294, + "01-01-f8d72d7cad54a83da23c2754262e-0001": 1295, + "01-02-11c962e9bcbaeccaec3084c94082-0001": 1296, + "01-02-203e6d4fefc4abbf3b49a9a1167e-0001": 1297, + "01-02-2ae548ee78e6b8643514492de809-0001": 1298, + "01-02-61c2268b90d0f4092c9b28c8ae9b-0003": 1299, + "01-02-b472d827c8f3730c4dac8d59aa59-0002": 1300, + "01-02-be56d98b9221b5160b2904296c46-0002": 1301, + "01-02-cb33b593bf85bbde23ffaac43d8e-0002": 1302, + "01-02-da0e5311c7d3cc8641a84964a4bd-0003": 1303, + "01-02-e1165fc2092bc0fd69e78d10aba1-0001": 1304, + "01-02-e2b85ccbdc164e19d8817e090b51-0001": 1305, + "01-03-14e43ed00e8ec815aa4d5a40b8a2-0001": 1306, + "01-03-1a3a7aa2a75d0b2ad00651fc8dd4-0001": 1307, + "01-03-485f705bc27f4ecb51135a2b710f-0002": 1308, + "01-03-b67d8db6e267cccbd79cda762dbd-0001": 1309, + "01-03-f2fb841ad5cd5ef2e862727611df-0001": 1310, + "01-04-152eaa556bc6ffb71cbb1d1e4001-0001": 1311, + "01-04-5e350002e9d8fffde8375b8f327f-0001": 1312, + "01-04-7f01de9033ce6da9cc5f66b1df44-0002": 1313, + "01-04-995edca8c51b967dd3a12dede677-0002": 1314, + "01-05-08429a9b494ce4bbc05a85060c74-0002": 1315, + "01-05-3625381a0797173e709ba358fd00-0001": 1316, + "01-05-5a3dbd2dc471bcabd1cab5025705-0001": 1317, + "01-05-6b3745a8369b1964b47643479ddd-0001": 1318, + "01-05-a4045003a41ea74ab9a97a03dafe-0001": 1319, + "01-06-2c086e630d476f6e874cd73469f0-0001": 1320, + "01-06-881aad4209389adf77d15253c640-0001": 1321, + "01-06-8cf035ac577da412e058928d4628-0003": 1322, + "01-06-a27c94916f706b70019fb0dd87b3-0001": 1323, + "01-06-a7b225851e2bcc944f975357fe6b-0002": 1324, + "01-06-c384e30ab264b4b6468e8a81225f-0001": 1325, + "01-06-e9ab3f15dc3c68ba3ce8ce92c3e5-0001": 1326, + "01-07-0f80822f959309df1c7c72e8d7d5-0003": 1327, + "01-07-1e69c81d387d41a25129bc300892-0001": 1328, + "01-07-28029e9b1e0f4e4834ae04e6528d-0001": 1329, + "01-07-42f6b718b3ef0e2e0a0241814102-0001": 1330, + "01-07-480643c51a1eb65e6ef607101f8d-0002": 1331, + "01-07-b76072b83bdc39dabb17e3da1e78-0002": 1332, + "01-07-c5fa2f4e80871de19f6cf7f37130-0001": 1333, + "01-07-cbfe4f3ad5dde04b091e829d12c0-0001": 1334, + "01-07-d0ddc5b0b731967d5e3243f3d0c3-0001": 1335, + "01-07-e04514523e3775783c9d090dd06b-0001": 1336, + "01-08-0b3ecb7f89dc94baac30e4eee2d6-0001": 1337, + "01-08-5871fd52958fdaf78da5ba115dd4-0003": 1338, + "01-08-5f088fd1171d7b6fe71764c160c7-0001": 1339, + "01-08-76f5775f79b8face9fada894fca1-0001": 1340, + "01-08-904e3a0ec81aff0ee48f9e9755f9-0001": 1341, + "01-08-a728b4e1998151b94419b851eff6-0001": 1342, + "01-08-b4b4ee19936c8bf71adfca163193-0002": 1343, + "01-08-c8e249bb074a6fe397e39753627e-0002": 1344, + "01-08-d295e90827b7e476ad73220d81e3-0001": 1345, + "01-08-f18ecceddc78f03fc603d59960c4-0003": 1346, + "01-09-0f1280c407952b06eb5605eef9d2-0002": 1347, + "01-09-41e9b1631d78a6f8cbdcb0aa4f09-0001": 1348, + "01-09-56a4209c62458428da5c17d9a808-0002": 1349, + "01-09-61651b77e48231263a9e4f44352b-0001": 1350, + "01-09-eb6e383cb968a851afc91ed57c1a-0003": 1351, + "01-09-f08a2b8b9c14a2adc4a9fa996e45-0001": 1352, + "01-0a-10f98143a2f3b82f708e113917d1-0004": 1353, + "01-0a-1aaa7f3f782424cc7e630b580451-0001": 1354, + "01-0a-39b7380ec88080c868c3e582aa7d-0001": 1355, + "01-0a-411d775369f788d10cfa799c411c-0001": 1356, + "01-0a-604d6b53f08aa07b4a47a8a99111-0001": 1357, + "01-0a-87b5a76ea45bae1ca25ed78c837a-0002": 1358, + "01-0a-9cce2f51d70dc52664504efef35f-0003": 1359, + "01-0a-bdd9d4eea0f3cc98fba8060014db-0001": 1360, + "01-0a-c145f72531c8c320513fcea1e2dd-0001": 1361, + "01-0a-c299bacbaafb871a2e56a5f11448-0001": 1362, + "01-0b-0918afa057deb8d6a3b51197d9cf-0001": 1363, + "01-0b-0c8499e486a90837e2a19859335a-0001": 1364, + "01-0b-1e24bba9a933d75071b5fc5e5c2e-0001": 1365, + "01-0b-2c12ad6b84d7f9b28459a35c431b-0001": 1366, + "01-0b-4b19ae0b30df0429baf92317457d-0001": 1367, + "01-0b-4bb0137a46f387b9363f1aa9ad24-0001": 1368, + "01-0b-67765c754563a25b44e88809e3d1-0002": 1369, + "01-0b-7aaf6f318533103e8f704f6d1aa1-0001": 1370, + "01-0b-8595e79d64c5421df11a2db9a252-0001": 1371, + "01-0b-ad0948db505dcc714a5f58a2c8f0-0001": 1372, + "01-0b-b138c8207136e599abc5b615a821-0001": 1373, + "01-0b-b2ef05c7e276cd46c9cf56708c6a-0004": 1374, + "01-0b-bd4323356ddbab04963cc79da619-0001": 1375, + "01-0b-dae3f381ca1d438a4ed72af0d95b-0001": 1376, + "01-0b-e0dc58b85833635a4b94dc0a7943-0002": 1377, + "01-0c-2dd898d4d57cc6c4b228dcfe137f-0001": 1378, + "01-0c-7883bfe176e93cfc29ee81eae779-0002": 1379, + "01-0c-97cc3b29e5f6fa3f5a46d22ee3e6-0001": 1380, + "01-0c-a509d7210002933516a9d24e324e-0003": 1381, + "01-0c-b2b2f58a06dce0c5bba383026ca8-0002": 1382, + "01-0c-c2c6e8eaaaf3a1bab0041dbb9933-0001": 1383, + "01-0c-f1d5123c3b49205a4c5dd53e28ae-0001": 1384, + "01-0d-066c6a04f1db5594d4b5a4a4e5a2-0002": 1385, + "01-0d-49ed551193a5513422c6f16ac3bc-0001": 1386, + "01-0d-7bec4f2abaed8187264588ccc891-0001": 1387, + "01-0e-5ecd49b395dca8ac509e54635f00-0001": 1388, + "01-0e-a0c5a00cafdfd701b97887edd18f-0002": 1389, + "01-0e-af13a58611f7dbd61a2c051a47d7-0001": 1390, + "01-0e-c29c9a2d6bb0183cd9b6fc9c96af-0001": 1391, + "01-0f-08a96ed0a8a04a4b9bbf067ac4c1-0002": 1392, + "01-0f-2699e159200f4b22043dfdf6df12-0001": 1393, + "01-0f-28ad563c918c01714e7e002a4c7e-0001": 1394, + "01-0f-3c8aac1bc3d50ccfec9beae8001d-0004": 1395, + "01-0f-3d32ee108d077ee46d7f275ecb62-0002": 1396, + "01-0f-80e03ebfac8247f62f667b2f8682-0002": 1397, + "01-0f-8619915d09f0e8e2e26e73f3855e-0001": 1398, + "01-10-04790f12e73968c950e0abd80863-0004": 1399, + "01-10-0d8291dab20039d43cb4b0b24c4c-0001": 1400, + "01-10-57cb4b1d27a7992bab0046126093-0002": 1401, + "01-10-6635950235e4678e829d17c97053-0001": 1402, + "01-10-6c4dd98ffc87b4d5972b61ab9dbb-0002": 1403, + "01-10-7756c4d02e931789d9ba27be7938-0001": 1404, + "01-10-791bf1567187dee4a32e307ab2c9-0002": 1405, + "01-10-8dfec130c223e82c917e5fe3d112-0001": 1406, + "01-10-b12d52fc6c143e24a01989aa1d77-0002": 1407, + "01-10-bf59b9f47d6de1a9a38e21720df0-0001": 1408, + "01-10-d33528b067e1d39d6b4d481d7f57-0001": 1409, + "01-10-d91a32c123f0d9fda938545846db-0002": 1410, + "01-10-df3f8ce4a9f7536424ae4dc7406f-0002": 1411, + "01-11-3763855ebe6fbeedcbfbb866ba7c-0001": 1412, + "01-11-4c51b2ce156967c0c1db204c0d01-0001": 1413, + "01-11-56174466a6c6762b1458a5b9bfb9-0002": 1414, + "01-11-5ad456da648970bd6597fc89c139-0001": 1415, + "01-11-69f29a5ef513a6d39072dd090eb3-0001": 1416, + "01-11-6a5a328e84205c631fe926b1b13f-0002": 1417, + "01-11-8221b45e637b28b1ac0dc36ea153-0001": 1418, + "01-11-91b75d8df6e39c0288f2ed286fff-0002": 1419, + "01-11-a7f784be76115bf20ee2770318b5-0002": 1420, + "01-11-b1c92db5e3fa221ca94aaba0fb5f-0001": 1421, + "01-12-0622a9525e350b146f24897cec0e-0001": 1422, + "01-12-2c41826489b235ef675b88ddd427-0001": 1423, + "01-12-7baaa8169b3276c6792717553f15-0001": 1424, + "01-12-7f70679c94943480bcaccaa64908-0001": 1425, + "01-12-91d3000c105d593b9d10f826b608-0002": 1426, + "01-12-9c23b56088afa72634e4d82dd9a9-0002": 1427, + "01-12-c4d22e7c08c3b67c7922ab50a5d5-0001": 1428, + "01-12-d83113d4152306df6eaab47717aa-0002": 1429, + "01-12-d8c8641bd0516f6ce4322d7c2876-0001": 1430, + "01-12-de4c27aa1ae3017d67448a7e065d-0001": 1431, + "01-13-191c88cc6146cce32f4b9d4f0a9e-0002": 1432, + "01-13-1fe5feaeee97311f7598f89976ff-0001": 1433, + "01-13-363e5b9e8ff67e808ca4139af967-0002": 1434, + "01-13-3e4ea619eda7e783964866c272fd-0001": 1435, + "01-13-59a7e44bdf7b12083ea0b46b1f84-0001": 1436, + "01-13-844b823d04277ab9bf90f984f4a7-0002": 1437, + "01-13-ab4f5e4e4feecb6852f3d0fe7b46-0001": 1438, + "01-13-bdd75b785facbe479d5135840e55-0001": 1439, + "01-13-e6d9000e8c972125fa77c95fb5f5-0001": 1440, + "01-13-e85222fd69bc07cfe7e98772f207-0001": 1441, + "01-14-3711e316cbc5bc4cb6bc01d42f94-0002": 1442, + "01-14-47859200db660977009934b460d2-0002": 1443, + "01-14-49a492b84ee426a0985e3ff51707-0004": 1444, + "01-14-4dacfdf051e1f986e7a7c64c2adb-0001": 1445, + "01-14-5d7d0dc8f2cb62905682bdabb2e0-0001": 1446, + "01-14-7b1d24eec61808ddd65a9be1e151-0002": 1447, + "01-14-82aa5a37b3c38694bc8ba64a691e-0001": 1448, + "01-14-c1e81bb218d182a9a6053bb3f133-0001": 1449, + "01-14-ccb7639537d7d804875e9e74955a-0001": 1450, + "01-14-f8b3423d953fafd3bc930f9bdeb9-0002": 1451, + "01-15-0dc8745ca3de6d23798797f9c54a-0001": 1452, + "01-15-29dc3b723aabe54c5218e01d0399-0001": 1453, + "01-15-359de901ec34a4a54f741a7ecac3-0001": 1454, + "01-15-7ce56c63d6c52a25dd9171f88db8-0002": 1455, + "01-15-8dfe2fc7946617f80ae57d8bd312-0001": 1456, + "01-15-a7c0e150ad0645b647381f888f01-0003": 1457, + "01-15-a8ae41081b3e41aee0466569ada0-0004": 1458, + "01-15-b4e83ef4316a5828b9b434e18b5c-0002": 1459, + "01-15-d49c63275579c5741c82fac0897e-0001": 1460, + "01-15-ffb3bfb743fd2c2515309346e0b3-0001": 1461, + "01-16-149c09cff63c1eac9f56d5c52543-0002": 1462, + "01-16-3b5968af7f0ff0db9caa9ec0bb03-0001": 1463, + "01-16-3d0973135bd53d7c5e2954263779-0001": 1464, + "01-16-4e09895c8b4f41eb806b316f6394-0002": 1465, + "01-16-818b213948adc9db0eedfe4b5898-0001": 1466, + "01-16-938851d97874f9f4bf9c1157b328-0001": 1467, + "01-17-159888edbaf635611b4d4eae0786-0002": 1468, + "01-17-1e1fd92419101c0557cd33e639e2-0001": 1469, + "01-17-4b6d0fd04575e215922fa5917c58-0001": 1470, + "01-17-9c3b5eae8d65c81efd5159b8fa65-0002": 1471, + "01-17-a212b6ac953c025aa6968ce3d813-0001": 1472, + "01-17-ab3e3b290bef6d6e9bc9fcab446e-0002": 1473, + "01-17-b252e28254b34a57d88746f93ee1-0002": 1474, + "01-18-091815dd94d4aac06aee74676bac-0001": 1475, + "01-18-12765d687eb783c095962c5f57be-0001": 1476, + "01-18-2438f7cfad3bc97a9a18f9f2cbf9-0002": 1477, + "01-18-35073f7ea70d5ef3c4cd9e5cb95f-0001": 1478, + "01-18-786c70ca893b4283bee4ca3ef4a3-0003": 1479, + "01-18-8a90cfa6e612a94cfc15b50d20f0-0002": 1480, + "01-18-954265466ca6a48f87544011238b-0002": 1481, + "01-18-9df0d06fdc89d0ac5c8794258d73-0001": 1482, + "01-18-d9fbe74e354896021315c6e6002f-0001": 1483, + "01-19-2c3ec720c05573b935bf349eec2a-0002": 1484, + "01-19-36b698d168ff932763596fcf9556-0001": 1485, + "01-19-78edf6286a3ffc73f8e314c150af-0001": 1486, + "01-19-7dd0af5ae2f84919a69452fa9b49-0002": 1487, + "01-19-89ae8163dc5d66da8c39878558b8-0001": 1488, + "01-19-944586711998503711cba9921efb-0002": 1489, + "01-19-c09ffdeb2e06fda6e10b4c2625bf-0001": 1490, + "01-1a-04757b544fbef960e3bd8219d432-0001": 1491, + "01-1a-2b8a763b07de932bf90184e62656-0001": 1492, + "01-1a-54e963fae8a0d6edfea3e6519b55-0002": 1493, + "01-1a-98e780c0e369d03792889be92468-0002": 1494, + "01-1a-a30f040ded85ae130e435b37a303-0001": 1495, + "01-1a-d2dec1db72eabf7f5fd6f1bac727-0001": 1496, + "01-1a-f9871ee26d1cdc4671906b0b7a81-0001": 1497, + "01-1a-fa88436d8775ac30cfd6deceb317-0003": 1498, + "01-1a-fd23cdd4d287e06be02ca3a8e077-0001": 1499, + "01-1b-151ed2dd85b39619fb2c21fc9434-0001": 1500, + "01-1b-1e1657318616f6f213ab163f8ef7-0001": 1501, + "01-1b-5070c597c93412c81f992a56f617-0001": 1502, + "01-1b-704b793bfeaa7c97643d97f76a30-0004": 1503, + "01-1b-7b26ffc8f5623f9c805239bb7d51-0001": 1504, + "01-1b-7b3f33be8bb0d5c8617697bd2203-0001": 1505, + "01-1b-8036b0a5be0f536ed31b861c038e-0001": 1506, + "01-1b-963f77b4846c0d3e1bf4538bfded-0001": 1507, + "01-1b-bbf696e93266c44fffcf1d8f95ec-0002": 1508, + "01-1c-65dfd498f1725992784d5f0f82d0-0001": 1509, + "01-1c-9f03661ef73c53e7d2a2944196a4-0001": 1510, + "01-1c-a5e3e3e3759dd76b4a7f0332a138-0001": 1511, + "01-1c-cce84db683c531c37c1eb50e6517-0001": 1512, + "01-1c-e2f6cc1ebfe6304df879f164000a-0001": 1513, + "01-1c-e60593ccb3e5766aa5e2769a9d27-0001": 1514, + "01-1d-0ce515fcbe6137354547d5e8c1f7-0002": 1515, + "01-1d-42117a38e55bc9f13964962cb007-0002": 1516, + "01-1d-769e05ff9234967db0de3cb0681e-0002": 1517, + "01-1d-ea30058219b98a5659301338ec81-0003": 1518, + "01-1d-fdacf4877d57e53d5c5bd3f380b9-0001": 1519, + "01-1e-6cc7e9ee9d5de0938b9353c4cb81-0002": 1520, + "01-1e-7c90e2a81b009a33350d4ec0dc04-0003": 1521, + "01-1e-7ed1924623aa6f58e6b6e956c99f-0001": 1522, + "01-1e-bd4586f9051762601c5075393e23-0001": 1523, + "01-1e-c98434a5c37c1f8534aaf7e47751-0001": 1524, + "01-1e-d885231b4da520249ae025a38755-0001": 1525, + "01-1e-e37fba1c7007c5e274f7e3412c29-0002": 1526, + "01-1e-ed7e3cd2c494373dd03c5855ce61-0001": 1527, + "01-1f-5343edbd313b8724efdfc0be0238-0004": 1528, + "01-1f-69252d20689bbfd93b103cf862d6-0003": 1529, + "01-1f-862f9b805026abd51a04d8a43f46-0002": 1530, + "01-1f-a61f60efc17da594494e19c6f7dc-0001": 1531, + "01-1f-b7adcc057bcf6455ff513b7f9faf-0002": 1532, + "01-20-47c3f2f5131bc2df473ccb4cddb3-0001": 1533, + "01-20-612475b5d0c43437a0d4e319cb94-0001": 1534, + "01-20-79fb32e61ef09e71886c55177741-0001": 1535, + "01-21-039e7f57121de010332933778c26-0001": 1536, + "01-21-526bde23e699bf7dcbdf05545cc1-0001": 1537, + "01-21-75e5e3f02eaab238f4672f048858-0003": 1538, + "01-21-7bdd9107b248b8da2cd7444765f2-0002": 1539, + "01-21-889ef21b29edd5adbc937abecc7d-0001": 1540, + "01-21-93c67d40109747286c670f193b9a-0002": 1541, + "01-21-9d3417378af11a57243aa6817846-0002": 1542, + "01-21-bd40e164a9c8087532cb1d201c9a-0001": 1543, + "01-21-c33ee09baa5af544409792ac08ca-0004": 1544, + "01-21-d127e29656f9d08f10a4a01bba8b-0001": 1545, + "01-21-f970c9e4e9362dfe4dd838760379-0002": 1546, + "01-22-05b6d45cbd512526dbff121d3281-0002": 1547, + "01-22-343ed0a7304b8b66152cfba0b6e4-0001": 1548, + "01-22-547f0019e8458a5eb16833913777-0003": 1549, + "01-22-560adad14e24375fbd0e73192284-0001": 1550, + "01-22-91c80194408d0742ab71825c9050-0001": 1551, + "01-22-a6e200e7cce6396e592941483a07-0002": 1552, + "01-22-b4eed34615ee89addbecebbd2498-0001": 1553, + "01-22-c6e4b29245d24bbc7bcd1249a6ce-0001": 1554, + "01-22-deee15fb5105063609a23cfd3a9c-0001": 1555, + "01-22-e391055dc5d8a522c5a82142df38-0002": 1556, + "01-22-e5571a3b70f1324e3477c1efc4e1-0001": 1557, + "01-22-f704119ca63be38783d742244ec3-0001": 1558, + "01-23-1f31115863fd69c80be644b82169-0001": 1559, + "01-23-248fa509c4fe4814fa46dfeca3f4-0002": 1560, + "01-23-287026b80c65ce6cf89b9b457d56-0002": 1561, + "01-23-4eb152e2f75f06876606e15c5de6-0001": 1562, + "01-23-58dc5f922a300ab8815e6506319e-0001": 1563, + "01-23-62c366ff46c7f60c65d8c9bef0cd-0002": 1564, + "01-23-6aa81a31ce350b46f72da3f3f81c-0003": 1565, + "01-23-790d763d95f9318e4379d8e3765c-0001": 1566, + "01-24-1ce57163debd8e44558b96fcef49-0001": 1567, + "01-24-8eabe5e0c113ba646aed1030d644-0002": 1568, + "01-24-9a04ffb9057718e33de7efd60aa6-0002": 1569, + "01-24-bd87ebfa872e195fbf111707cbdd-0001": 1570, + "01-24-c70b0e5cf9d6afee7df7b51c2e2e-0001": 1571, + "01-24-eab99193e163a9ff8029707111ce-0001": 1572, + "01-25-10b684a6942a4315165c9eabf278-0001": 1573, + "01-25-502ab325c810cc302564f8b267ce-0001": 1574, + "01-25-59425e1c6985c936f261f6eabe4b-0002": 1575, + "01-25-69677a07f666dc79f4d88dfa6ed6-0001": 1576, + "01-25-84d76dd6a72d6e54180402d615cc-0001": 1577, + "01-25-b618661d41d900c4337353070f13-0004": 1578, + "01-25-b9d1f0f6625042f9a18e68d97571-0002": 1579, + "01-25-d9c5ac591b1ce4d5b20a028d705f-0001": 1580, + "01-25-f134c049dacad161279b59092ba5-0001": 1581, + "01-25-ff0596def14965a4b43fbeea71e4-0001": 1582, + "01-26-047c152cc9d37495a4d2ecdaa053-0002": 1583, + "01-26-10ef337f18c94a6deaf45ba45817-0003": 1584, + "01-26-36a7608415ad725d0f6c04bbded8-0001": 1585, + "01-26-5bde0cf3eb6c6d4721e30d894e47-0001": 1586, + "01-26-5e26bd142b830dfc70811fbd0ebb-0001": 1587, + "01-26-68431e921efae431d54958228052-0001": 1588, + "01-26-9dcfbc76237315d8166d4e17aee0-0001": 1589, + "01-26-a0ade1ce627628baf0bf85a16793-0001": 1590, + "01-26-b1b5131b662f48bbc1c9dbc2c375-0002": 1591, + "01-26-c50669325bbb0b136eb903ef403f-0001": 1592, + "01-26-ca677b27bd1769ded51163ff365a-0001": 1593, + "01-26-d566e602ccb059ea3a83bd45cd8f-0003": 1594, + "01-26-eea43688a612e80cade519ad443f-0001": 1595, + "01-26-f69170227253f50b8e5e881058e3-0003": 1596, + "01-27-20d158721f70e755834b7452069d-0003": 1597, + "01-27-36c1cff93233420eb5cb124aaad8-0001": 1598, + "01-27-5c88b073541c9ff26058813aaf33-0002": 1599, + "01-27-719019951448a481a126c4f2f906-0001": 1600, + "01-27-a74a1a692b80b3e82cd7cbedabea-0002": 1601, + "01-27-aa404117e3b3f0411e2ea2288e49-0002": 1602, + "01-27-e38e33c35ed158e7edcae030a552-0003": 1603, + "01-27-f36e795a91ba1cda10f93d836d73-0001": 1604, + "01-28-79a9c6bcb65974a31e254ada6923-0001": 1605, + "01-28-9439a8630ec38709ef6cdd838a7b-0001": 1606, + "01-28-9594f1b30a6027258d3639c47433-0001": 1607, + "01-28-a5d0b8cbf644a513f23fac90118d-0002": 1608, + "01-28-b52da760aa1bd199cb8105e85f38-0002": 1609, + "01-28-ee2b996013e876e71933df18fa1d-0002": 1610, + "01-29-0cfb6136214fdfeb17dcade56553-0001": 1611, + "01-29-424b91662eaf1f28d5759a00a87d-0002": 1612, + "01-29-45c170f4d5582e5800e8d21fcb97-0001": 1613, + "01-29-7089e397c2e5aebb331413bd0203-0001": 1614, + "01-29-83e2b3928684d5600da57b0bdd19-0001": 1615, + "01-29-a49fd25b9470366b670a3689cf26-0001": 1616, + "01-29-a7daaeeeee655ed766812460a9b4-0002": 1617, + "01-29-c2d466a24e57dea268f8890d9ca6-0002": 1618, + "01-29-d833209477b5d716f6a6a615d025-0001": 1619, + "01-29-d9d66e1b933b5542c1d045dbf110-0003": 1620, + "01-29-dfebb2057dfce266874a06c13816-0002": 1621, + "01-2a-18e5a8fb1848f36b1f18a670e74a-0001": 1622, + "01-2a-75cb3a65550c6464d632bf35299d-0001": 1623, + "01-2a-8bac28fa97728aec7c0be4972d0b-0001": 1624, + "01-2a-b7ed85c72b2b64699bcca0c814c6-0002": 1625, + "01-2a-c80c80e902891886e121e45826f7-0001": 1626, + "01-2a-e5ed375bb6acff31a0cf20a40162-0001": 1627, + "01-2b-1a2601fd58f2889983a555d77bb6-0003": 1628, + "01-2b-47b6b78ecfec58ff760d47121e34-0001": 1629, + "01-2b-503fcb4a4c108c31389a7a7ea5a8-0001": 1630, + "01-2b-873f296666e7a6c7cabca7b642b5-0003": 1631, + "01-2b-a8fc20f12bfea26f54fecf03fbd6-0001": 1632, + "01-2c-3fd03bd7ae9e223026b7074c323b-0002": 1633, + "01-2c-550e9408bb97a6b67185ecf9406b-0001": 1634, + "01-2c-7c4d6e0ee41990f4e9dd6ee286a4-0001": 1635, + "01-2c-806c93ed5277c89706727dd25bbd-0003": 1636, + "01-2c-850fd7abc4fb63c0652809f9b30d-0001": 1637, + "01-2c-8cdb0f996a36ca56a3934a86bd55-0001": 1638, + "01-2c-8e1b93abebce54666f51962eba0f-0001": 1639, + "01-2c-9727f877d4cc2684be95b19596f1-0001": 1640, + "01-2c-b38a7c46ce3a13213c527c21040c-0002": 1641, + "01-2c-fd3b0d6e2b9b4ab2b28a63a59e2a-0002": 1642, + "01-2d-0bea95ecf618e4de2e4cc8baf812-0002": 1643, + "01-2d-28146770f16f7363cfc2c912f22c-0001": 1644, + "01-2d-2941d4449259dbae6f65b904213f-0001": 1645, + "01-2d-2e55c91859b34528b8ba4105af35-0001": 1646, + "01-2d-3a477e94f4f293e2874e480a0f85-0002": 1647, + "01-2d-3a6943feb93f4e9ff6a558bb21d9-0001": 1648, + "01-2d-a52471ac7570c476ee50801abae0-0001": 1649, + "01-2d-f600bba563e383c204012faa18de-0002": 1650, + "01-2e-0271610440f3a0dbf76bb031a71d-0001": 1651, + "01-2e-1b0395e32f8ee50bb38d56e4a27c-0002": 1652, + "01-2e-2cd23a65bd3d09932286b5a3226e-0001": 1653, + "01-2e-3fcded2d1d978e0cfce7e4874232-0001": 1654, + "01-2e-705614c15d49c31d2d0872559d31-0002": 1655, + "01-2e-7602ec7d3ff3ff671fe0138dcad8-0001": 1656, + "01-2e-a7871279b5a37f5cc1bec387a14f-0001": 1657, + "01-2e-ae6dd9b619421ca6d014fefa2e4e-0003": 1658, + "01-2e-b865d564a6e8abe7a1fe182d4501-0001": 1659, + "01-2e-e6b31e4d9a84df3bba8f37186bef-0002": 1660, + "01-2e-fb9cd2a64bafaff8185daf9b9c16-0002": 1661, + "01-2f-16bcebf62162d11db7e6924160bb-0001": 1662, + "01-2f-27acd9fce016fc15781d5b4971d5-0002": 1663, + "01-2f-2aea952fe1efc3d7d7f1b48631f6-0001": 1664, + "01-2f-c40797a2af7351f00a7640f0ba1d-0001": 1665, + "01-30-742215cad158fd5ed6f2eec1140d-0001": 1666, + "01-30-e0472105742ff02aa6648d19b789-0001": 1667, + "01-31-10f14b4d118f39a5248deb4a2ad2-0001": 1668, + "01-31-3cf83c1f9570c8c84159cdd740b7-0001": 1669, + "01-31-9706cb320f30aebbdde7f0211c82-0004": 1670, + "01-31-9cf19c314b5fd380cb896eddc350-0002": 1671, + "01-31-b0678301566d73d8153558f0cfc1-0002": 1672, + "01-31-c258675cf9c1bc2355683df353aa-0001": 1673, + "01-32-9186cd7e4ad2b1c32aa7a1829ee6-0003": 1674, + "01-32-99b162976cb144cf647ea8667360-0001": 1675, + "01-32-b4cb59a758a1b7b6328ffcaaadd3-0001": 1676, + "01-32-cf32a70cdff4e75406c88c4a2c38-0003": 1677, + "01-33-20f60b64642c1bbea4ca3e60b6a2-0003": 1678, + "01-33-3788a551b7a7c4a201a835be9f86-0001": 1679, + "01-33-7a8c728c4735d91f9bb2f2209e70-0002": 1680, + "01-33-dded68b6d58d4283e438ab2c2c56-0003": 1681, + "01-33-e40555fb8abd1527991fdf69d2cb-0001": 1682, + "01-34-396022f64603c4820a7d4e1a62c1-0001": 1683, + "01-34-54ba5aabe8f19a190d4c4e2d469f-0001": 1684, + "01-34-626716525499f1e52d1c1661d72f-0003": 1685, + "01-34-a85f41f9fadaeb7e426864198fe7-0001": 1686, + "01-34-a8a46970c8deb9a6680fe617a3b1-0001": 1687, + "01-34-aad97fe7c9449cedddc0e41d674b-0001": 1688, + "01-34-ebd3133497512d2b5ccb4f81e9f2-0001": 1689, + "01-35-06c0c41fc54882aeb132e3b1aaa4-0001": 1690, + "01-35-07729229af479132d16ac870ae28-0001": 1691, + "01-35-0c2a6c5f2d62f3ca820f6b2c2b8d-0001": 1692, + "01-35-0c625a6c163ecc8ffcc8eddfae72-0002": 1693, + "01-35-2f8111934852aacf1d1c98925155-0001": 1694, + "01-35-321b014555afc5f9de95d3b7460b-0001": 1695, + "01-35-6a22ae8ea02c1d89f35e65d0dce7-0002": 1696, + "01-35-8b949d0ffc3f302c81fbede8a98f-0001": 1697, + "01-35-8e7d4c50b8d611362f5b1f254d70-0003": 1698, + "01-35-8f6263505df039c3764688f5687e-0001": 1699, + "01-35-98309e6e5841a77f13ce44895ab1-0002": 1700, + "01-35-9b0259dab0d5a32164d59585b2e2-0001": 1701, + "01-35-f880db415d72d4bfc3ff5fd82ff9-0002": 1702, + "01-36-19085f8ea27a4acef7fc813e97da-0001": 1703, + "01-36-4968d52c9ffc92aced0b152be53b-0001": 1704, + "01-36-7f5ad8faa5d5d291918b7bf61917-0001": 1705, + "01-36-9e4a7a09a25a82cbf1bf373ea7d0-0001": 1706, + "01-37-3cf52c762ba41baa4a821ef7b602-0001": 1707, + "01-37-85f26edcbbfcc9bee26dfba14845-0001": 1708, + "01-37-984d076839274ddea5deeb068e87-0002": 1709, + "01-37-996b05ec1c08f2f428b38b3ddbd3-0001": 1710, + "01-37-b04597810f29facfd7152670a1e1-0003": 1711, + "01-37-bba7d1939edfd900a6c02438ff4b-0001": 1712, + "01-37-ee7ad7f8bcf0fab0997e3ba77505-0001": 1713, + "01-37-f722363727cb9edee353a094958d-0001": 1714, + "01-37-fd7a11543d811930d570eeb37b88-0001": 1715, + "01-38-1ea0278d4245f89eeb0d21bbd132-0001": 1716, + "01-38-35418cf967ccde7dad29fa3588e8-0001": 1717, + "01-38-37412f9c792e5146c1d7f0b3b8cb-0001": 1718, + "01-38-481ecefa78ed2e7b7569f73c3538-0002": 1719, + "01-38-6b051d859c92a3e4334b21ad79e6-0002": 1720, + "01-38-6d2b4af27dbd0b6e4d9d5814eabe-0002": 1721, + "01-38-827cca1fa09216b572d42f01aee3-0001": 1722, + "01-38-b57ba494043af62fe6a042968043-0002": 1723, + "01-38-bf736d5988fcf9ffb1a809f94a7d-0002": 1724, + "01-38-c710dc3749d03a7c4b0f7289047a-0001": 1725, + "01-38-d0fe9cbf58bcf4eaa076dc4ba2ff-0002": 1726, + "01-38-e475798741c0a7e3c47140272cca-0002": 1727, + "01-38-fa690484d9eb6f76572e345ad135-0001": 1728, + "01-39-06042c102dde0cc991338a8ce87a-0002": 1729, + "01-39-1373e9d4aacd2359fc1b3bc0496c-0001": 1730, + "01-39-202919962c76e4b7b3178cecd081-0001": 1731, + "01-39-3862e3ee16b8a4db2777f1d7e1ed-0001": 1732, + "01-39-5ac31cb13c4b209f460cbf22b261-0001": 1733, + "01-39-6e32049124d88399cd58a837842c-0002": 1734, + "01-39-9edcc498353f54e0524644d37af4-0001": 1735, + "01-39-cf99032be8e13d877f7dfc2eab9b-0002": 1736, + "01-39-e982a0a2e044d82328bfe9881f0f-0001": 1737, + "01-39-f1b973a25cc0e31077fd4579f6a5-0001": 1738, + "01-3a-04ea2a3b9b4a55dcc2fabe59caf3-0001": 1739, + "01-3a-169577de086dc046e6402800a9fb-0001": 1740, + "01-3a-1d19b01e1a89b7ea90da32c47c1d-0001": 1741, + "01-3a-42364f80d4259c34925c9aa7d9ca-0001": 1742, + "01-3a-5e55346cf309eec5de4aab52e3d6-0002": 1743, + "01-3a-7670d0c588a83f4b4868317ef831-0001": 1744, + "01-3a-7ad9524f6e5b641709ff58ac5df0-0002": 1745, + "01-3a-7f4e2490e91a55e55cd42cfcef0b-0002": 1746, + "01-3a-8f9b884bba8686ad4c5003cc820d-0002": 1747, + "01-3a-e2166c6154d768ab43d7a9d1d007-0001": 1748, + "01-3a-e3e837b38dbbb191514e372e947d-0002": 1749, + "01-3a-f06428baecc4f23447539e041fff-0001": 1750, + "01-3a-f2ec9dbb98271b35b9b26944e461-0001": 1751, + "01-3a-f539f34843c333ea539646c5b5a1-0001": 1752, + "01-3a-f69241390a20af446ef95c017a71-0001": 1753, + "01-3b-2172fa031c1718bcb017383ee221-0001": 1754, + "01-3b-2f7bca4d777fbf025f73f4cac743-0001": 1755, + "01-3b-3ed241c61d45761a7bc8d633df7f-0002": 1756, + "01-3b-5764a9fee790d620f4fcfece21a0-0001": 1757, + "01-3b-83caebdf89ab82d6ccb420922dff-0001": 1758, + "01-3b-9d5af155f34a48777489ab7d4e06-0001": 1759, + "01-3b-b886fb03d00b4ccf19accf1ebf50-0001": 1760, + "01-3b-b8a75ed97817f7b89865097714e2-0001": 1761, + "01-3b-fccf3eda6c3764fdc0ebe0cd6cd5-0001": 1762, + "01-3c-30dee4062f90d645851d0e234a34-0003": 1763, + "01-3c-525d09fb3e2fedc792327eac530a-0002": 1764, + "01-3c-61fe4cc5ca8a1aa26f1c180c5b98-0001": 1765, + "01-3c-644e9327192328fb44d10c9bde3e-0001": 1766, + "01-3c-7a6ff4738ce9b26c84294f8c28f7-0001": 1767, + "01-3c-8dd2e03326358e32c2a8fb44e0ba-0001": 1768, + "01-3c-fc6d411478d3ce98e1b104b5fa5e-0002": 1769, + "01-3c-ff168673bd78f67deea94017b38d-0002": 1770, + "01-3d-0a81ba9c139cf38dd22e5f4dbefb-0002": 1771, + "01-3d-2f6ff7172351e0d88f7f2704d72f-0001": 1772, + "01-3d-3939a50ab4dca3b42b8275d37e50-0001": 1773, + "01-3d-ab978773313d4d0293525ccceed6-0001": 1774, + "01-3d-d1a3d1b3d391a64b1eaf6f001264-0001": 1775, + "01-3e-05032feff7485bf60b64b8389aee-0001": 1776, + "01-3e-0c5c25873236510b25b35f8492a8-0002": 1777, + "01-3e-3bc775faa47621ca35793678b086-0001": 1778, + "01-3e-4b200815b00d6d5eda393df92299-0001": 1779, + "01-3e-5c56954b2a445a85ad4f32fd8c34-0003": 1780, + "01-3e-5d653e10044e7999250abe33ceb2-0002": 1781, + "01-3e-af3dc2a1156e35a3f194222bcf58-0002": 1782, + "01-3e-b4d77ba98caf82db53dd60eb5068-0001": 1783, + "01-3e-b911890694bb67e77201923c6d82-0001": 1784, + "01-3e-ed135af2c2197a62a5cd30a303ea-0002": 1785, + "01-3f-04d86209daebfd19711a87fefcb8-0003": 1786, + "01-3f-1aec7ff44dd2516d851afbe6063f-0002": 1787, + "01-3f-205776cdbd6df78d9ef3444b2c1d-0002": 1788, + "01-3f-248c2f867382284bc1e60d255c88-0001": 1789, + "01-3f-253e5d86eb64f808dcc8846e2a80-0001": 1790, + "01-3f-4b47e0b0dc1b89631b54336a3004-0001": 1791, + "01-3f-8a57424c5583ada139b344a7d26a-0001": 1792, + "01-3f-91704ad1c18723d4575ec9a6ed49-0001": 1793, + "01-3f-a069792ff31262aebb69c2e26c75-0001": 1794, + "01-40-262160c1dc7671e0ae76a7ffb14e-0001": 1795, + "01-40-5db3c3a5a723f24bcb8e1d1cb136-0002": 1796, + "01-40-87a61ed35181e6ad4c5178a9cd51-0001": 1797, + "01-40-c58eacb2f509568d96cc8b92e51f-0001": 1798, + "01-40-e88c06cff39ab4faf926d1c612e5-0001": 1799, + "01-40-f22ae31e3a568afb6041a3b0c0ab-0002": 1800, + "01-41-2e5673974891a41794adadd12d71-0002": 1801, + "01-41-590931f0687aeb6cfd07c727516b-0002": 1802, + "01-41-6de042fe6d455ebb41c980cca4f7-0001": 1803, + "01-41-82b526d42f37739d9e4d58701445-0001": 1804, + "01-41-8b421e02e7d89e7267e90335827a-0001": 1805, + "01-41-9ca9e1d6adef0f09b607641c09f3-0003": 1806, + "01-41-9cbad5ab850fe7f948d8173a0646-0001": 1807, + "01-41-adc4e6d3158d4f1694116c44a1c4-0001": 1808, + "01-41-e919a0b110841b21cb980e14d2c9-0001": 1809, + "01-42-0b84d2e3bbf7816728d442c7ae77-0001": 1810, + "01-42-2354679e72eb6dab6e919cd5f879-0002": 1811, + "01-42-ad89788a467dbcc2cc7073f85ca7-0001": 1812, + "01-42-cd4041163fb16f8f8b907b0c8cd2-0001": 1813, + "01-43-1208b9c817341b007ae6017cd56b-0001": 1814, + "01-43-1db032aed33745ee9d8c54894408-0002": 1815, + "01-43-4ee0432d67160a8f4844ec3c9c58-0001": 1816, + "01-43-6c2f80d054974c004aba3127fb04-0001": 1817, + "01-43-70bd5a991b4d97c5565b87da2244-0001": 1818, + "01-43-7d0863e8f10e09ec1fc7eea8d156-0001": 1819, + "01-43-89e0dff943045c170e32feb26741-0001": 1820, + "01-43-8f59ff50b580b7866ba1c4af6672-0001": 1821, + "01-43-a238aa9154600aa7c5d9d64702e1-0003": 1822, + "01-43-a64d081caf1d64ad0f49c3e79186-0001": 1823, + "01-43-ea9882769aa33c1903e8984bc814-0001": 1824, + "01-43-fae2b7b3804c393b3c6de5454267-0001": 1825, + "01-44-09f3149ce966b36a40c204dc95dd-0001": 1826, + "01-44-15260cc2ec13513350245ae581f8-0001": 1827, + "01-44-947f6a440551bc051b61a333b948-0001": 1828, + "01-44-da8bf6799d4a756d6fedccbbb2b6-0002": 1829, + "01-44-e262d709f3dfd7183e47f55f0f83-0001": 1830, + "01-44-f10a8f809bd699e9e9f2378d7419-0002": 1831, + "01-44-fb334152bcad3990622c0f59241d-0003": 1832, + "01-45-153bd26899c8ff8188369bb6610b-0002": 1833, + "01-45-4289c0d28fd6821e3c7185dac6ac-0002": 1834, + "01-45-95e7af9a766d7f43bdaa1d95e255-0001": 1835, + "01-45-ad4846258e0627e6e8b894149ea5-0001": 1836, + "01-45-b27da87a763efca53f0ec3819f5c-0001": 1837, + "01-45-d12458454b9701c4a853a26f187f-0003": 1838, + "01-46-6e49af805018a74555dbc4d75974-0001": 1839, + "01-46-7c4484bf02fe6baf5aeed8050c1c-0003": 1840, + "01-46-7c80180cdab448c2c54ce2855919-0004": 1841, + "01-46-b5a6a35d01e661961a8edfbb8fe6-0001": 1842, + "01-46-b919aa45b05d6fa42ae65490334f-0001": 1843, + "01-46-eab3882d9a5a0d1083e3dfb57fef-0001": 1844, + "01-46-f439486254654777e2a0b19ce46a-0001": 1845, + "01-46-fc8a7866380342f308a5db4fd442-0003": 1846, + "01-47-2367d1d57174ba5d8a6e5f46a8a0-0001": 1847, + "01-47-2a919f0eb3635964ff4602730b75-0001": 1848, + "01-47-4ee0a39c4e81a6e14bdf0953693a-0001": 1849, + "01-47-5d32a26ab57e6b6295c31546cc19-0002": 1850, + "01-47-a228684f46b76751aa5fc9d41f67-0001": 1851, + "01-47-aa5268a13cdea478b2218a514467-0001": 1852, + "01-47-b992143403dc260eed68870c9b8c-0002": 1853, + "01-47-cbe698033e307069dd4e64a96482-0001": 1854, + "01-47-e21f1d6fdd89d48505adb49254d6-0001": 1855, + "01-48-023e59454ac63fe1616f23cb39ca-0001": 1856, + "01-48-0d1d1b6b20e021cee4e6d16e8320-0001": 1857, + "01-48-542a18bfe885d6c6b4bb65193fc0-0001": 1858, + "01-48-7040dea917849834032b82e874f2-0001": 1859, + "01-48-882bd00c916d161582ccf9ff35c2-0001": 1860, + "01-48-991837bb50ded479d27f410cb5a9-0002": 1861, + "01-48-b27e62aa1db7e812e4b4750edd3f-0002": 1862, + "01-48-b35e60157dd9e085981de21658da-0002": 1863, + "01-48-d8d350abb60393860cea818c140e-0001": 1864, + "01-49-1075636fcb63f08a4db6714ead69-0001": 1865, + "01-49-133bd4faf0133e244844ed5a6bae-0001": 1866, + "01-49-33974fe5070dc26f315033c5b1a0-0002": 1867, + "01-49-670c755ab30574c52c02a3d6cffe-0001": 1868, + "01-49-739d2c852d4d95b42e40d7b8a0fd-0001": 1869, + "01-49-75bb5f36f17d4863d98c9ad673d6-0001": 1870, + "01-49-7ae20a3e15d4938aa1572f8ba7fe-0003": 1871, + "01-49-864fa72a8b9d768d8b22be8c6882-0003": 1872, + "01-49-93cc4e38284dbb66d24e4433a75b-0001": 1873, + "01-49-97108481da75c10fe43f5acffb86-0001": 1874, + "01-49-b3da3f4f002f9a65d0b2ea8dc7fe-0003": 1875, + "01-49-d28abbae43988c26722c4d09de38-0001": 1876, + "01-49-e92a3f2f163a3e1e8b3df544066d-0001": 1877, + "01-4a-32bd5323df54f4bc31b70632865f-0001": 1878, + "01-4a-3636ae3b31d7112cb6c9df708ebe-0002": 1879, + "01-4a-390d3061b29014153e0c3f4befd6-0001": 1880, + "01-4a-6d6dc0ddf716b6b4755458a39016-0001": 1881, + "01-4a-e5507d86b3846ab9e42342decaf2-0001": 1882, + "01-4b-04ce944cb92584f3f11886bd2b5b-0001": 1883, + "01-4b-353694fdea9f2593f353048694c0-0004": 1884, + "01-4b-3d0a7b5e65bfb3ad8eea90f7615a-0001": 1885, + "01-4b-8377f2e4e8a91f613323188addb5-0002": 1886, + "01-4b-a70df57671f28d9312348a34b32f-0001": 1887, + "01-4b-ad5eb54bc63674d0bd824ee6af0b-0001": 1888, + "01-4b-ca1447a5c7577d258c8577eefe99-0001": 1889, + "01-4b-e48b140b7dcdc8f40ce154bafc75-0002": 1890, + "01-4b-fb56abf92e75740fcb61ac0345af-0001": 1891, + "01-4c-51bd27c3344b49e68e5b30d49c4c-0001": 1892, + "01-4c-768e981698f825169e57378e9d79-0001": 1893, + "01-4c-8fcd2fd745f1d5e6ecb1f6a64e4a-0002": 1894, + "01-4c-e1ccdead9da273939461dd299abf-0001": 1895, + "01-4d-11c4505a9f50ac1717ffda29b477-0001": 1896, + "01-4d-5a06deddec5957ec4bf7f807abe4-0002": 1897, + "01-4d-640ac3191183a92fddb0e51407e9-0002": 1898, + "01-4d-7e409c5a2283e94e29ad2db44a57-0002": 1899, + "01-4d-879a48655964396d46b48d527246-0001": 1900, + "01-4d-902a8e2956a44f3589e2ae8160b3-0002": 1901, + "01-4e-0812a18b0cb3a24c8591826d6c03-0002": 1902, + "01-4e-08c05a1febec0ac7097275274e18-0001": 1903, + "01-4e-70ba3f462e3d4d899643811855fb-0003": 1904, + "01-4e-f5d463281938e2b654110e59ac11-0001": 1905, + "01-4f-25fdb30e6832b642bb3adb623c75-0003": 1906, + "01-4f-4dc9a6d4d9f32d09650ca1be5501-0001": 1907, + "01-4f-4dff6d02bbf1959049cf6e8c6b49-0001": 1908, + "01-4f-60fb946bc6d910e4955f0d1fd2bf-0001": 1909, + "01-4f-7069031688013bed44c1325f2d58-0002": 1910, + "01-4f-729bb53c3f8663409f752493fd68-0001": 1911, + "01-4f-8f5f2d2f9b77582203d134496b43-0002": 1912, + "01-4f-b5353289ac72af314d9b8eaefd87-0002": 1913, + "01-4f-bc61b38a99ccbf30a701b9261639-0001": 1914, + "01-4f-c0c2d0aab29fa09173cf499d2431-0003": 1915, + "01-4f-ca21b5b2b0356f153932636fe982-0001": 1916, + "01-4f-eb1ff30920bfd86188f2842cfed5-0002": 1917, + "01-50-07c6287311bda7f5cf338a32087a-0003": 1918, + "01-50-14df157911fd67af1fc703c4a4ea-0001": 1919, + "01-50-dadbf204a1334c5dcff9df3ad8bb-0001": 1920, + "01-50-f21bc2e4e27e9e1c7d370068ff50-0001": 1921, + "01-51-088f54737f03577b5e4983f0e4b9-0001": 1922, + "01-51-48cdcd2f0895539ba9033f623725-0002": 1923, + "01-51-5a0de778851af57784a2ca29c133-0002": 1924, + "01-51-70e3cf26264a62b7d777f025dc18-0001": 1925, + "01-51-8cf03fc5041c05baed282819025a-0001": 1926, + "01-51-928c61ba05ee3aa8a24c53f3e8c0-0001": 1927, + "01-51-9316337678f1947806b6b279d96c-0001": 1928, + "01-51-97e2e7deca3f41b8ff4ca13cc7aa-0002": 1929, + "01-51-9d83a10450677194cb861944b9c5-0002": 1930, + "01-51-b90658d8fb34cf72dcbc89c18282-0003": 1931, + "01-51-f8106afdbe4906d6ed950251d2f2-0001": 1932, + "01-52-05ed3eec1c1bce92f5e9c30c672c-0001": 1933, + "01-52-54bb3e9c843116ec4467a5df87fb-0001": 1934, + "01-52-67abbe87db5099f73a0d15dfe4bb-0002": 1935, + "01-52-73044edeac8b6558c1ee0ed89962-0001": 1936, + "01-52-7d61397e4af10cca1731610799d1-0002": 1937, + "01-52-82bd28a736a7bb281dd30ac35365-0001": 1938, + "01-52-d090de4f3fd1d67a837369bfb84c-0001": 1939, + "01-53-21475e2b839f12d769eb2079be76-0002": 1940, + "01-53-2397b84da105d88fdc13cd772815-0001": 1941, + "01-53-23c44fc75895105f3b3de0b1cbfa-0002": 1942, + "01-53-272bbebb5d814f65eb13988c8887-0001": 1943, + "01-53-280030944d9b6c26cb142fdb1906-0002": 1944, + "01-53-2ae51d2f457a5f83c4fbb7d6dc6e-0002": 1945, + "01-53-a33318c67011369c3b928deca4f9-0002": 1946, + "01-54-4cbab630373fe33b6bfe56f079dc-0001": 1947, + "01-54-538c761b7bc95d8f18659381d920-0003": 1948, + "01-54-78d7d1c91518e3607ecc7b18e35d-0002": 1949, + "01-54-8144adedd5686081c4065bef1684-0001": 1950, + "01-54-845c99c4ef5940e3b12f4705853e-0001": 1951, + "01-54-a33202d889698d18bfc02aefeb88-0002": 1952, + "01-54-c5b4d5ef5d87de785340e2646fa3-0001": 1953, + "01-54-ccf4d5d154bb579e833ad768f32f-0003": 1954, + "01-55-09a55a2034d9be0b55fa06e82422-0002": 1955, + "01-55-0d298b07c34d38f957527c068dfc-0001": 1956, + "01-55-40bd24c7aebe93ef2835ad992f5d-0001": 1957, + "01-55-86bd75b1c8dc1d58bed7feffc088-0001": 1958, + "01-55-8e8d24f3c4c091234ceceb45b519-0001": 1959, + "01-55-9a0001c9d51fbcf5ea1e09f5ea56-0004": 1960, + "01-55-b17da912cd5be2be4f5182a39ce2-0002": 1961, + "01-55-bce4f44542592fe8eb62a33a4019-0001": 1962, + "01-55-fbe94fdf3b4fd82cf3ed6bb2e576-0001": 1963, + "01-56-136f1eed5ee7307c6ef9c5318b08-0001": 1964, + "01-56-2f05c6a00f354b2fddbe2c02f2d8-0001": 1965, + "01-56-32b29751640f3f82ebc61f322af8-0002": 1966, + "01-56-78b918b0927119360e392882de5c-0001": 1967, + "01-56-a55070d5b952b611a18ca539e041-0001": 1968, + "01-56-da0dacb0c7ca32af45f6bbab5d99-0001": 1969, + "01-57-0bb49f1b2eaa398b89a826661c59-0001": 1970, + "01-57-1ee1b58a820e5cd6f466835efe3d-0002": 1971, + "01-57-4510738daef2505920100aec2354-0002": 1972, + "01-57-4bae0d06aea44e04a33d8f86ee4f-0001": 1973, + "01-57-81169f3ffe799d21d12786c78cd3-0001": 1974, + "01-57-c5d49f0fcac47c5705f320554b66-0001": 1975, + "01-58-04e8d0d3819fe509845bced7eb02-0001": 1976, + "01-58-75d20c1156694a22013fa32c9df1-0001": 1977, + "01-58-77c2496e1ac463c9432658c8022c-0001": 1978, + "01-58-e73483f26a2140066f20cb64e2c2-0002": 1979, + "01-59-51f74eb1a8184f916504e3a2c9e6-0001": 1980, + "01-59-6131d112358117e6eac8d3601bcd-0001": 1981, + "01-59-7d28a3a955d33cd8d5fa5ab00d7d-0002": 1982, + "01-59-86817e27cb4ac73911b79dd0d558-0003": 1983, + "01-59-a08b44285af471cce0f876530215-0001": 1984, + "01-59-e19ccbe2e7ae157dfd7b6e881407-0001": 1985, + "01-59-e7fec23763881a037b46367245bb-0001": 1986, + "01-5a-00b6c4578604f4821904d4f37eaf-0002": 1987, + "01-5a-1984dac700c9b8fa14898a831eca-0001": 1988, + "01-5a-213e527d0cd4ab23d68103262336-0003": 1989, + "01-5a-2c32d79aa7a830c0ddd927dd3a0c-0002": 1990, + "01-5a-300ab87755db8b17379408d1d3cb-0002": 1991, + "01-5a-4784bc2436a89711a7df97d6d634-0001": 1992, + "01-5a-6525c5fcbff55465e9d9783d888c-0002": 1993, + "01-5a-9b51f7003714e96961a9690dbbf6-0002": 1994, + "01-5a-e4c52bd6c3c5eac9f27c16e42fce-0002": 1995, + "01-5a-efdbf66ef998b6f8009bc8714d2d-0002": 1996, + "01-5b-0afa3040281147d5eda811387345-0001": 1997, + "01-5b-2097f74e252254e50af9d3db7887-0001": 1998, + "01-5b-415d5ce814e940c18ff4efb75435-0001": 1999, + "01-5b-451a7cca9de63c492d11e1051925-0003": 2000, + "01-5b-7edb9c6df60d7b61b6b007dbd211-0002": 2001, + "01-5b-821ea329422ac296239ad78f9d8f-0003": 2002, + "01-5b-d973f8822163ad6b88383b427e38-0001": 2003, + "01-5b-dc290e2f0ac09db08ad4dddc7b58-0003": 2004, + "01-5b-e9911ebb65779d9052b1779a8286-0001": 2005, + "01-5c-186609bddda194d96f82861c567c-0001": 2006, + "01-5c-6d91a369ffc0467666d43d2e3fd5-0001": 2007, + "01-5c-a24b04c39107f25d72775cfacd24-0002": 2008, + "01-5c-af4d2e3c7c1277e6f52e7b491a53-0001": 2009, + "01-5c-b5df81e5be872027a81ae216cfd7-0001": 2010, + "01-5c-d66435e3d0af65d78682a57b02af-0001": 2011, + "01-5d-16695dd934db92fc429b692185e3-0002": 2012, + "01-5d-1ce2095a90bc9b2da261bb24eb98-0001": 2013, + "01-5d-27b1fffd69cdbf7d6cc2349c7435-0002": 2014, + "01-5d-3c53861ee768fa3f4788fb331cf4-0001": 2015, + "01-5d-4638b6393057cb113436926aaa1e-0002": 2016, + "01-5d-66c0d8f2747ccb877cd792d9ef4f-0002": 2017, + "01-5d-9c3921ba3c82021fc5937ebc13ae-0002": 2018, + "01-5d-a61812264057d8418ea852c57ee7-0002": 2019, + "01-5e-108f3775b4d64a9d101a6d24793d-0001": 2020, + "01-5e-472afcd136aa24d2fc53768b39a2-0001": 2021, + "01-5e-57ccdd8436cbfc8f498ab12260e7-0002": 2022, + "01-5e-dc8daac372d596f8f164e10a5a62-0001": 2023, + "01-5e-e224abe0c0471ccc56b241cff5fa-0001": 2024, + "01-5e-ea720b613700c858104ae5c4f866-0001": 2025, + "01-5f-16bf159f0596482e05483b3e33b9-0001": 2026, + "01-5f-3c5b7c0fb564fab555b0305c16d6-0002": 2027, + "01-5f-7a921fe23f83f10fda180b598988-0001": 2028, + "01-5f-8a62cf26a425a60d7473e8323891-0003": 2029, + "01-5f-8ed8fe3f1381e2fce7f6a6c978c6-0001": 2030, + "01-5f-a1806bdf8e34b49738f98e2c6771-0001": 2031, + "01-5f-ec60674a0e3a43e0dda19f476a79-0002": 2032, + "01-60-21ec773b081b2ef1d83631e804b4-0001": 2033, + "01-60-23cd32ea6a86b974eb7b5a4b83ac-0001": 2034, + "01-60-6879d61727248079405ecfb8fecd-0002": 2035, + "01-60-7c22e236971a22fd5e2d251bb930-0001": 2036, + "01-60-b84ea146f59fdb04d14767474714-0001": 2037, + "01-60-b87a7314b77f8b9165ed281c5f4d-0001": 2038, + "01-60-c00b293a8f823c78129795919d04-0002": 2039, + "01-60-d78fb83aabc9655630ac51ad5a91-0002": 2040, + "01-60-e1ddad3b5e77a02bad4790f09aa8-0002": 2041, + "01-61-063296aa9d7ac0a44ea22479ed51-0001": 2042, + "01-61-23e695d02aa1ba8e9a6031248e9d-0001": 2043, + "01-61-53dd187994ced4691fd5913a34de-0001": 2044, + "01-61-7e36a08875ae887b15abcc6e43ec-0003": 2045, + "01-61-9e18da04d7065535fc233dd80c7e-0001": 2046, + "01-61-a23c27460a5aaae1ef519947815a-0001": 2047, + "01-61-c75e2c3b0c8be611ef7f7ce90a73-0001": 2048, + "01-61-d8e29cedaff98b9031e46d32ae4a-0001": 2049, + "01-62-104f51e0e2ccc52d6759ac767c2f-0001": 2050, + "01-62-21b931bc18fd7fa5b59d8db08947-0001": 2051, + "01-62-c34a4c2537a7bd5744bd7d73fd02-0001": 2052, + "01-62-c82fa61318aa0664332448bf9a83-0003": 2053, + "01-62-cc241e84ec7831cbbf5c4bdce690-0001": 2054, + "01-63-2ad00debeae8e2254735b3ddd991-0001": 2055, + "01-63-31ccdeb9dd19ed433d23bd381579-0002": 2056, + "01-63-69ea572ae968c89bd6482db76d05-0001": 2057, + "01-63-b57f4320385a93c2b263eeb9efbb-0001": 2058, + "01-63-c80227a5d26413b71b001445f4f6-0002": 2059, + "01-63-edad254bf24ea41973d2bc3c4f34-0001": 2060, + "01-64-2b8ee75c2ec6fa66bae081317e06-0001": 2061, + "01-64-49c8fe01309612dc4cfadcd76367-0001": 2062, + "01-64-854ca4b1c7195ec78e7c18e4cde9-0002": 2063, + "01-64-868b759f2f66cab4f26c03243469-0001": 2064, + "01-64-9acf8e5e057dcd40a265e2726da8-0001": 2065, + "01-64-dc60f30df1bf748a7fdb26d46c97-0002": 2066, + "01-65-183f42c2d995ded85fc740b7efeb-0002": 2067, + "01-65-231208c9e924699862cdf128ccf2-0001": 2068, + "01-65-245e64dbf859ee829d354caedf3c-0001": 2069, + "01-65-8d7b227cdfd194ef60b21afe6fa1-0001": 2070, + "01-65-9c98f03f27556cee5b7020444357-0001": 2071, + "01-65-9e2bf6d94d1bc558314663013875-0002": 2072, + "01-65-a7318d75f4dc6a0d869f77f40d1b-0001": 2073, + "01-65-b2f947ade520a38849790c534f2b-0002": 2074, + "01-65-f28ef3588072a31eae098a16bea0-0001": 2075, + "01-65-f7cbea01835b48a7a54291ea631a-0001": 2076, + "01-66-174e4d1f84ddf25598e6d6e2752f-0001": 2077, + "01-66-2a218269e847f36eac3e84253ccd-0001": 2078, + "01-66-2a88730ebff30776199469a39b96-0001": 2079, + "01-66-2c2f09679f676dad2c838b3b24fd-0001": 2080, + "01-66-2c90242f6e82f0db32b070f6622d-0001": 2081, + "01-66-3a9f9fa99cfe809e01d3a1ce611f-0001": 2082, + "01-66-3cace34d0558bcd61c360368d5f3-0002": 2083, + "01-66-7d2bb11b09ca4bb0131aa9bdcca3-0002": 2084, + "01-66-dea94009619bb462d51cc3167cd8-0001": 2085, + "01-67-36e718394d1e06cea7a185c2a730-0002": 2086, + "01-67-c47bf9e6546edf9e8004a9922435-0001": 2087, + "01-67-e85d856851a156ea6a3169d47528-0001": 2088, + "01-67-e90bed4dd8ea7089fa7d0a0110db-0001": 2089, + "01-67-fd73d9d3b3c8fc3a5eb80a56bf9b-0001": 2090, + "01-68-3d6a1f139687386430e514b22fab-0001": 2091, + "01-68-7a23be453764717b88d7248b2641-0001": 2092, + "01-68-9c108639d043466e059a1c37a2da-0002": 2093, + "01-68-f95a12085ce85687e33197b2aad9-0002": 2094, + "01-69-2ae3757cb5b9ed2648f6cc5b9f41-0001": 2095, + "01-69-2b58c369396a076da8be8c175ebb-0001": 2096, + "01-69-2c0ca75476ea05c8396e5051c823-0002": 2097, + "01-69-4ad80deaaf3d4fb0b26f0d22e42c-0004": 2098, + "01-69-7bcf146346ee610359db0630cd06-0001": 2099, + "01-69-aeb51c708870d5c6fac7d26702ce-0001": 2100, + "01-69-c8aa8b70775eb709f59b0d84913f-0001": 2101, + "01-69-cd4a88165a7f2a5e15ca6191635d-0001": 2102, + "01-69-d5113cbebd64d21324cc3cb02b9b-0001": 2103, + "01-6a-2b36c88c7e9be11c726b1c275aef-0003": 2104, + "01-6a-62acac2c11d5d7ce67d2d72c8e18-0001": 2105, + "01-6a-64c3ee54d0033a1111bce5943acb-0001": 2106, + "01-6a-73be2ded637a2566e90ecb9339fa-0002": 2107, + "01-6a-90ba8f7d1e09ff7003161cfb2143-0001": 2108, + "01-6a-910eb09130679097a45ea8ea59ae-0002": 2109, + "01-6a-c126e75abcd518d7bb00f0530356-0001": 2110, + "01-6a-c5dba15f02cda2aa255875f004c4-0001": 2111, + "01-6a-c6615f8498e597b00d9089e35ca5-0003": 2112, + "01-6b-2d0d47a980201741482362b40639-0002": 2113, + "01-6b-5ed690de6ad657bc38a5ed63c401-0002": 2114, + "01-6b-6c1d3f9ccfde8094dbce3724fdf6-0001": 2115, + "01-6b-90c92e689475401c92af006963d3-0002": 2116, + "01-6b-a9c4293aba85d2063a46b89df9ca-0003": 2117, + "01-6b-d30dcd891dcf4d7caaa61d5096b0-0001": 2118, + "01-6b-f8a6abe128165546a8503bac0041-0001": 2119, + "01-6c-11cc581c7ff30c9baac84377c3e5-0001": 2120, + "01-6c-1765094a275a65f80d9a9f61da50-0001": 2121, + "01-6c-1cdd25ccfcc1334f94dc61e16ce7-0001": 2122, + "01-6c-2a4eec93bae65eebebd8f92391db-0001": 2123, + "01-6c-2e376339ddf68b813fb0873070a5-0001": 2124, + "01-6c-3a8cfc57d7cc2ae570d6698e1249-0001": 2125, + "01-6c-98295ecbb3417301bfd9f8ccbd08-0002": 2126, + "01-6c-ba72c39c461ed2a611ebf0e168e4-0001": 2127, + "01-6c-ba954fbc65886de60b023c872578-0001": 2128, + "01-6c-c2e0bcb0d14521994f2a0f86a5d1-0002": 2129, + "01-6c-cb273149efe87b5c58f9a7497597-0001": 2130, + "01-6c-dc20dc01002a4d1977ec203214a1-0003": 2131, + "01-6c-e3f3171255e55605a9c9c39d4a66-0001": 2132, + "01-6c-e8929b28b1a9e912af0397202770-0001": 2133, + "01-6d-1416c5aaf93ce0ed19751fe86844-0001": 2134, + "01-6d-15c73e8eb741b7fffb6fa8f2f7c7-0002": 2135, + "01-6d-4e668c90001351e92207dcdd2b13-0001": 2136, + "01-6d-99cb0e7286d57d2c61f680cb9ca8-0002": 2137, + "01-6d-a44f4dabf7005fbb1724fecfe6a3-0002": 2138, + "01-6d-b34b1935877e02742179ae672491-0002": 2139, + "01-6d-e812428a57349c7895c547bbc05a-0001": 2140, + "01-6d-f3dd9c79aaf294e5583f8a4fb801-0002": 2141, + "01-6e-03907940efd61053ddfb544071a1-0002": 2142, + "01-6e-2195fdf868944bed0b409ad65449-0001": 2143, + "01-6e-2604014c833c04ed27d457334bbb-0001": 2144, + "01-6e-317ed948ee3c1a3517144432e641-0002": 2145, + "01-6e-ae55674ec091b57fa69efaae757f-0002": 2146, + "01-6e-f2f0f4988bc72766535372a97418-0001": 2147, + "01-6e-f551cae910bfa4b77992a82a63a1-0001": 2148, + "01-6e-f927687a65b54c80575637aac392-0002": 2149, + "01-6f-0e195ad19a12e5a327c7fb2f3d69-0002": 2150, + "01-6f-0ff448a3266a85f2afb70c951aae-0001": 2151, + "01-6f-1f41f421f079bfce658c17930a12-0001": 2152, + "01-6f-3eb3b7a09a9f0a017f99d479e5e5-0001": 2153, + "01-6f-5270bc2e74b6fab1652a5b63287a-0001": 2154, + "01-6f-a1e0ca2c60e4b96d9c8a45e0c54b-0001": 2155, + "01-6f-c9f04283c5d2c7116e24fd654523-0002": 2156, + "01-6f-d720ab0718840a6d264b0ae2e1f0-0001": 2157, + "01-70-4574f39c7768348b68de5e868507-0001": 2158, + "01-70-4f25a3dd213444b475e5e570a86b-0002": 2159, + "01-70-597772927f4faf84e5232d761bf0-0001": 2160, + "01-70-988c2cbed57b0a831bf869167af1-0001": 2161, + "01-70-d68538567d168400641fbfa07785-0001": 2162, + "01-70-f3c70b23bdb30799e46943837cde-0001": 2163, + "01-71-1c0c9908a5fe0003be6fe4b6a155-0001": 2164, + "01-71-1fda0ee594a308069b55f4adfe7c-0002": 2165, + "01-71-21808816edb73e3fed682e509f69-0002": 2166, + "01-71-34a4ab42e2faf09e68261af00058-0002": 2167, + "01-71-8a43a3aeadce8f199668ee67409a-0001": 2168, + "01-71-8b9f724667ff5289edd3c4102d1c-0001": 2169, + "01-71-928b6e90e52d09d9766ef69cd202-0001": 2170, + "01-71-9f597381bce689eb1f052241686a-0001": 2171, + "01-71-a37c736158bba9bdd8ac60b03378-0002": 2172, + "01-72-1416f0783778f5dcfeabe322f039-0001": 2173, + "01-72-2225732f4df66be91ec5ca4260f5-0004": 2174, + "01-72-27a83dee7021dc8086baac3dd8a4-0002": 2175, + "01-72-2accb3be8d04a68e230f1c366844-0001": 2176, + "01-72-2e1d13c6be7ccfd1a87b74ce9c7b-0001": 2177, + "01-72-3edab740ce6dbdfe23f946ef8cd5-0001": 2178, + "01-72-468a4d91d7e7d6290acaf556e9c4-0002": 2179, + "01-72-b27355aad5c673819dbe03bb73bf-0002": 2180, + "01-72-b66d10e12c35b6e23d2678473add-0001": 2181, + "01-72-d50396138ada8b59e8921e9c54e0-0001": 2182, + "01-72-dec235cb2962daae2460f9164b13-0002": 2183, + "01-73-188c4731a24a186156b95ca9bc42-0001": 2184, + "01-73-3d92b7581a88c79298823eadbc6a-0001": 2185, + "01-73-5c549f8b83cb8f9b1351c69fc0e3-0001": 2186, + "01-73-6ad7620c8c17bfc3ea175db07981-0002": 2187, + "01-73-c3352ebcfeb5c30a9231f4bd32a1-0001": 2188, + "01-74-2a03aba0a17b614f7ad476b5fc5b-0001": 2189, + "01-74-3d07e7dd2a98390a43d2b0940c4c-0002": 2190, + "01-74-5947607a14aebc57b2907d8f0169-0001": 2191, + "01-74-9443d8a74e72ef11e988f959ec4a-0004": 2192, + "01-74-a055f6ab0d9e385459c70e22ab75-0001": 2193, + "01-74-b0ee9ca0c8bef6393053f14a741b-0002": 2194, + "01-74-fe57297abb1a2f052c857c2946c9-0002": 2195, + "01-75-38b5d63bcd3a72c39ba04fc5f6b3-0003": 2196, + "01-75-a164cc75d1a8cc23cd9a54dec0b6-0001": 2197, + "01-76-1665c836804646b18b70a52e7677-0001": 2198, + "01-76-6eeb15bd9e22315cc42d235a1bd7-0001": 2199, + "01-76-7e5b6b5f92b049fd35f95deb5bab-0001": 2200, + "01-76-81a6c104d329b07e677947bd6e74-0002": 2201, + "01-76-db7e687e5e8d6f920ddf84db338f-0002": 2202, + "01-76-dff1819195efc505d7f26d6d2c9c-0001": 2203, + "01-76-e3ed8f7e5b8b5ba491104322a2c8-0001": 2204, + "01-76-f37293411fe421a012836783a1c4-0002": 2205, + "01-77-0770f7df5d5c95de74b6defa4872-0001": 2206, + "01-77-1527d1b678edf2c83bf51c6594cb-0003": 2207, + "01-77-b656d06593fb2ab1bf7b19d01a50-0001": 2208, + "01-77-cd027b19b45ffc710b5b8a5229b0-0001": 2209, + "01-77-d17418561ebc42dd5756217772b8-0002": 2210, + "01-77-ecb94ee6df40220b88635a1da074-0002": 2211, + "01-77-f52fa506bcc49b327707110f6f5a-0001": 2212, + "01-78-026c5778d648cd379932e59f9726-0001": 2213, + "01-78-03964d65681c41ef6833cd967823-0001": 2214, + "01-78-0ed52252ad0442882574b05be3fa-0003": 2215, + "01-78-3e674146e67e05415b3862781377-0001": 2216, + "01-78-9d091855c4aaf2f19f1f65d06803-0001": 2217, + "01-78-9fcbb534ba03826849db42139610-0001": 2218, + "01-78-cf483575114c47a3a7aff66330cd-0002": 2219, + "01-78-eb19ff0227b933293a5d3a06c7bd-0001": 2220, + "01-78-ef8ecf736b1453159dfc636c0101-0002": 2221, + "01-79-19803fc9bedf198f25546c396861-0001": 2222, + "01-79-2256ea2b708a283075acc1ec980b-0001": 2223, + "01-79-230aff6adb09f99ecfe71814a4f9-0001": 2224, + "01-79-2b2d0c3607e58980c924b81c5c6d-0001": 2225, + "01-79-5ee574c347d427ce88fe36eaefe8-0001": 2226, + "01-79-619b108730916ad668d9f4233c99-0001": 2227, + "01-79-6e353344b87529f7ecc8ca02e297-0001": 2228, + "01-79-76eabe3e667851d0ff765961d674-0002": 2229, + "01-79-7bc83bc318587a9f145623f25423-0002": 2230, + "01-79-803b51c1aa14992b6584fca145ee-0001": 2231, + "01-79-8b93aaf66ee93b6aae1c4ca63b61-0002": 2232, + "01-79-b6cd5c209e614444499f6caf318b-0003": 2233, + "01-79-bb0a45e0738798365bea80947158-0001": 2234, + "01-79-db06be68c38e841ce26138a504eb-0002": 2235, + "01-7a-08595c44ac8f102873dcc308a685-0002": 2236, + "01-7a-0c4aeac9a65341f27ae8ad42ce07-0002": 2237, + "01-7a-25426f0b98ae26337fca0e8f717b-0002": 2238, + "01-7a-2b4e48d873982ef1a4db8dc2ce84-0002": 2239, + "01-7a-4e54372378a8cad0b3b03f911501-0001": 2240, + "01-7a-542c8a30046042bf5e25e5a8c8ab-0001": 2241, + "01-7a-70bf9a566ec66ebdc43344c34c46-0001": 2242, + "01-7a-834cfbab96e48966cd94d13e0592-0003": 2243, + "01-7a-861a7d45020ddc5ec3d6ea199def-0002": 2244, + "01-7b-5af2deb58b9a4b03920e765e1f4a-0001": 2245, + "01-7b-7352ce3bc9ab6b5ec4442019c98e-0002": 2246, + "01-7b-7853bf146a0b1b648955f5f27ed8-0001": 2247, + "01-7c-34523853e2902e3be599dd4878e5-0001": 2248, + "01-7c-61eaf9ce4f0dcacfea567337b92c-0002": 2249, + "01-7c-672af638e150cc4e6f34ba2e303e-0001": 2250, + "01-7c-74376d43283f371e98630bfe679c-0001": 2251, + "01-7c-9c5c797428ad56d1211cb71fc0eb-0001": 2252, + "01-7c-aa386bb5a00b8ee56c1ba4a7e542-0001": 2253, + "01-7c-df38354431ac4c64ab5d3930bd1f-0001": 2254, + "01-7c-fda583d99ff898e6fd5d05cbf986-0002": 2255, + "01-7d-2511984713b0883da8d43c42a4c1-0002": 2256, + "01-7d-401a061e5c3d1d58b2a11f5bc13b-0001": 2257, + "01-7d-a7ac72fdb762aec754c60c9f39f9-0001": 2258, + "01-7d-c584b06faebb75c2cbdeb2287e9e-0002": 2259, + "01-7d-f97cc024f19712e2c2bdeb871a94-0002": 2260, + "01-7e-393f62d5c8d640c210bdcbc30db1-0001": 2261, + "01-7e-46c7f4f952bde0e143048e305f93-0001": 2262, + "01-7e-8ebc23fc5ee406d70bfda97c1f38-0002": 2263, + "01-7e-e55cf87305e66d9698677472215c-0001": 2264, + "01-7e-f11980ca7ecc0704bb3beb154f35-0001": 2265, + "01-7f-04a6f8e215c47a2bb8e3dd1e5afb-0001": 2266, + "01-7f-06b00e809400cb4a0172063ef7a3-0002": 2267, + "01-7f-3b5deaf737ed6553b80b70dbf419-0002": 2268, + "01-7f-3fd03c0478bfaccb117776f35711-0001": 2269, + "01-7f-4d0065974d352a461f8cccfc750b-0001": 2270, + "01-7f-4e50a81264626f51379a6fb74e2e-0002": 2271, + "01-7f-65de32e33369f02bd12dacf80560-0003": 2272, + "01-7f-92b06a4249797e4c7302af5e6ca3-0001": 2273, + "01-7f-a456cf32a5c31a8f08ebde02ffd1-0001": 2274, + "01-7f-b438fb7dc23ce10fe87cc6da451e-0001": 2275, + "01-7f-b7d183f5d0033b7dcb43a3b00e03-0001": 2276, + "01-80-0380c73236e77ace4ec17aa3319c-0001": 2277, + "01-80-03b44ff85fcb43c4cd27a85f52ce-0002": 2278, + "01-80-13124a5cae32fee45589b3733c32-0001": 2279, + "01-80-d84860b147624e51af6cf13efae8-0003": 2280, + "01-80-e7c87fed6898192d22bf794e0fe1-0003": 2281, + "01-81-36152907d4bfe08878f2b9a64c66-0001": 2282, + "01-81-5f0f9c02da1c6fa69d6fa7f8a53c-0002": 2283, + "01-81-6499c161c09590d6e1a3019b3dc4-0001": 2284, + "01-81-66e2187243a27e458b24cad490c1-0002": 2285, + "01-81-781f50c4065919968c71ec3a9b29-0001": 2286, + "01-81-a6b558c7428f38f788e90cbd52f5-0001": 2287, + "01-81-b5151136b5abcf42393fa1a4f790-0002": 2288, + "01-81-d2bf93a092af34cc15748c7b95f7-0001": 2289, + "01-81-d799929456aeae09552a3518e8b1-0002": 2290, + "01-81-ffbfdd1c29b738474117525b1e22-0001": 2291, + "01-81-ffc2544f6d2506b3bf946ebb63a2-0002": 2292, + "01-82-16a5f5aea4032c945f9c01d46e51-0003": 2293, + "01-82-3ccfd1f252790a4620b9c52abfd2-0001": 2294, + "01-82-90ec591b9468ce3709b84ed07d4f-0001": 2295, + "01-82-9f32b0798b115a22d63918707088-0001": 2296, + "01-83-09ef04ec9d384e26f25e116a1e86-0002": 2297, + "01-83-447b1970516f0ef3f8891e2081e2-0001": 2298, + "01-83-4aa515a4073dfdc7b6c2e4664b8a-0002": 2299, + "01-83-6cf8d5fdf8e5b72567b14f99a02d-0002": 2300, + "01-83-7cb57946207d2181cfebdcbc666c-0001": 2301, + "01-83-93a435ae8c332cea9652b8a1e047-0001": 2302, + "01-83-e05b9db4706dd3478c50221b5d45-0003": 2303, + "01-84-2d0e13fb5c792f2d25b7932efff1-0002": 2304, + "01-84-3b18daba54c066abddef35e59f28-0001": 2305, + "01-84-425926597bcd5ff5dc062efcb106-0001": 2306, + "01-84-675cd85cb13f1a31e05ab440b963-0003": 2307, + "01-84-a0a988320dc176f0cd9bb7b99a41-0002": 2308, + "01-84-bc2aeb1fdf09e58246ffc28c2368-0001": 2309, + "01-84-ec141417c4f73da22d2fc31a363a-0001": 2310, + "01-85-166ee36b18d96b8540795ff8c0a5-0002": 2311, + "01-85-191be5da9777001ff0ebce0f1ab3-0001": 2312, + "01-85-23d1bf828501f6b122407c441dcd-0001": 2313, + "01-85-32939710bab1e6f3c52e494b9c65-0001": 2314, + "01-85-3b875e3f491e151cebf8f692df3a-0002": 2315, + "01-85-97b430ab7db790fb100d86c51609-0002": 2316, + "01-85-99771c5f2fe2bfe8d4cb5e8e68c2-0002": 2317, + "01-85-ae5f5a86bc21b5f52629bc7357b4-0002": 2318, + "01-85-d887f4710fbae26caeab51c41b63-0004": 2319, + "01-86-0a3dcca13ea62cd02ddc197dd136-0001": 2320, + "01-86-0f5a6e6abe0e1945c2d3f575602d-0001": 2321, + "01-86-1c7d85074fb28f045e4dd6276850-0001": 2322, + "01-86-45a8e9c28cd9d803a234b70f45b7-0001": 2323, + "01-86-460bb6b21a3366a1928cf20e4b4a-0001": 2324, + "01-86-7b562ae07cd3bb67474087795685-0001": 2325, + "01-86-99b2c33033ba87e101cc156bf68b-0001": 2326, + "01-86-d277bfcc5e243b0b596302696337-0001": 2327, + "01-86-e69d5305abba98a9713fe83e66d5-0003": 2328, + "01-86-eba4686600c13381838d09445309-0003": 2329, + "01-87-12989d01ef7b16743ff57e9e8707-0003": 2330, + "01-87-1940929fb5478e10493e294edff0-0001": 2331, + "01-87-22b5fe821fc53abb5c616f5fd161-0001": 2332, + "01-87-7fe8da248699f534012e2347fc19-0001": 2333, + "01-87-92ab26bd9aca34d6c22abd7088bc-0002": 2334, + "01-87-993511e527605288057de2000369-0002": 2335, + "01-87-e74312f211df2c9ef1a80e226966-0004": 2336, + "01-87-f0bec8d54a75c0c26098e6a0aae7-0001": 2337, + "01-88-01f3a72d05b55d7874043e3e558e-0002": 2338, + "01-88-10fd26d6ff3dbf2c939dd4ed6d12-0002": 2339, + "01-88-33f667d0026bfe865a0812e07990-0001": 2340, + "01-88-3bad45071b2306fc8196510db052-0001": 2341, + "01-88-4b148d354eee6c655d8606cf1d61-0001": 2342, + "01-88-6ecad26a1a56f24837d1d30229b9-0001": 2343, + "01-88-731d89921065128ec0983ea62f6b-0002": 2344, + "01-88-9bbde72d57d3dcd611ed56fe51e4-0001": 2345, + "01-88-c60834581f86319c2dc9cb0b2aad-0001": 2346, + "01-88-e3f19b7ae309968e2a89d1b71c82-0001": 2347, + "01-88-f321d6e7dc55ec7fe6a664ebde98-0001": 2348, + "01-88-fdbd896855c72324956edb70de3d-0002": 2349, + "01-89-24df7b626fb8ae58b42fa1b16aca-0001": 2350, + "01-89-5cbbab36796a4a3b41a70cf338e4-0001": 2351, + "01-89-5fc75c70dbe5fe2fd50d11378543-0001": 2352, + "01-89-672e77d63b406f5360ec9f8f3708-0002": 2353, + "01-89-8946c00e2a986ea56626413f642d-0001": 2354, + "01-89-a385b7684fa44a65836ccbd08ab6-0002": 2355, + "01-89-aee39d2e7627230aba9746e38db5-0002": 2356, + "01-89-bfd59f89a339b3a3b6815727dd36-0002": 2357, + "01-89-e209c50ecff3c1c0bbd6bac1978d-0001": 2358, + "01-89-e8aae6bd4e41818add9cb8bfc128-0001": 2359, + "01-8a-5c4d2c8e1120d4d3cdf6806b1782-0002": 2360, + "01-8a-5f5f8a90601f01f67045df700f72-0001": 2361, + "01-8a-9e74b84be597487da8eaf1a01708-0001": 2362, + "01-8b-287b72e4f308b47c51d311502e89-0001": 2363, + "01-8b-2cb648a640d672c1f5c86af50388-0002": 2364, + "01-8b-31a44e70211207a3943092a5ea00-0002": 2365, + "01-8b-3334ed0db3de723a32a0dc38dc15-0001": 2366, + "01-8b-444d2e4f4509aabc91dc9bd3d469-0002": 2367, + "01-8b-f585f063abb71885473f69630cef-0001": 2368, + "01-8c-26df8628786594bab6fbd423e260-0002": 2369, + "01-8c-33e21bf492962dae3b51004f7a5a-0002": 2370, + "01-8c-34533901e7d4c8fdf1009914a8d4-0003": 2371, + "01-8c-644c419ab012bcceac6233ee77e4-0002": 2372, + "01-8c-749300e94447ff2e2ea1b59149a1-0001": 2373, + "01-8c-92153c10e170ced94c6d27ad4d68-0001": 2374, + "01-8c-d45a854c9b48bdd8896c346b34ad-0002": 2375, + "01-8c-e0a8d0b9eddd1edebfe2d3010206-0001": 2376, + "01-8c-e3610349d3c0c08e8e86e5ec3424-0001": 2377, + "01-8d-40e202bb52b78355fde5cfed8a36-0001": 2378, + "01-8d-58d36919d4bd87ee920b3d9fd3b5-0002": 2379, + "01-8d-82e6a7cdd0a4400526cb70a5c596-0001": 2380, + "01-8d-a3719c091760e503b2a76b50e676-0002": 2381, + "01-8e-00cc4cdbe5c474c0b76fc4235928-0001": 2382, + "01-8e-0c90e21749f83fdbd8b046da6331-0001": 2383, + "01-8e-3470f34ff08eee20f105f2c0179d-0002": 2384, + "01-8e-3a51b7ee17bc66fe1b3d7d4cd0ba-0001": 2385, + "01-8e-3f24cab7948f872f2135ef512945-0001": 2386, + "01-8e-41b930000bd0a8283829ef10c7c4-0002": 2387, + "01-8e-55398dac49ade5234c5492edf75c-0004": 2388, + "01-8e-696b523581d040fe70b2db5a14bf-0002": 2389, + "01-8e-7a575df99f3de2efad98d0ac927e-0002": 2390, + "01-8e-c561f3d0b78a35fa88854f0dadf8-0002": 2391, + "01-8e-c63ce00a49e611fb0c054055ce96-0002": 2392, + "01-8f-1b6841bbec5cf32bc37fff66fcdf-0001": 2393, + "01-8f-3f9fa2239333d7236e4353f0816b-0001": 2394, + "01-8f-4136c8da8dbc225d77e2a1f18106-0003": 2395, + "01-8f-52e2ad13f6317a59c7615523bfc3-0002": 2396, + "01-8f-5bbcf78815e5fbb2c4d9f5bb025d-0001": 2397, + "01-8f-9ba9f6c6564cf11bd782ed0fd88e-0002": 2398, + "01-8f-a50c22c7c431b2a57e1f54073d97-0001": 2399, + "01-8f-faa208f5a29277dac2abeb3d76d0-0002": 2400, + "01-90-0476e696f6b97bd71650c8663e3e-0001": 2401, + "01-90-52904c043b1394c7a952d2140081-0001": 2402, + "01-90-5cd46df16918913006cca9c4a471-0001": 2403, + "01-90-5fe8f47d7aa44c68104f307b217b-0001": 2404, + "01-90-84eb5da0c212ba99cba37574abe6-0003": 2405, + "01-90-d4f506c2fe6f601e7ae1b75de993-0001": 2406, + "01-91-4c5d725dc66cdb273a3e88e04a7b-0001": 2407, + "01-91-57fdd607e5d17e6e58ea315723e0-0001": 2408, + "01-91-5fbd3471a73c5d2b606af358c97c-0001": 2409, + "01-91-7ead2d3d761c3acfecd5b3fd6d90-0001": 2410, + "01-91-828eca660bf3a8aaff80fe73c5ed-0001": 2411, + "01-91-ccab1b09c061f3e980959c396d9b-0001": 2412, + "01-92-6c192347f37b99caa48777667587-0001": 2413, + "01-92-8391e9e0576964082ae8ebdc0508-0002": 2414, + "01-92-995498ccc18a5cb7ff25dd782597-0002": 2415, + "01-92-abe3a55b209a25d90616e252bbdc-0002": 2416, + "01-92-f44322c2e93e012e575651b86d14-0001": 2417, + "01-93-323318f0369342a07030e59c784b-0002": 2418, + "01-93-5748d024e8809bc494759e5e70ce-0002": 2419, + "01-93-6dff812974c3f096c8a3aec0f640-0002": 2420, + "01-93-a1f7eedaadec5e41843792c1ff86-0002": 2421, + "01-93-b7d55fd8c3d9e10b944ac2c877b8-0002": 2422, + "01-93-c63868687a361a8a2770e387c722-0001": 2423, + "01-93-c980ef499c370dade394c2503661-0003": 2424, + "01-93-cd835c4f7085e398bc1c8463e5bd-0001": 2425, + "01-93-d7149ab580cc4608cd381a87cec3-0002": 2426, + "01-94-4e44b9271cd3ad298ea607a51a58-0002": 2427, + "01-94-521b570555098005fb89ae551951-0001": 2428, + "01-94-8f46e919518c0f7f18f45677356a-0001": 2429, + "01-94-9e9c7f38dab33ffedd2ee91823ac-0001": 2430, + "01-94-b088ec66ac57efaede9726ba5f56-0002": 2431, + "01-94-b2b2cf1ec1858fc9b8368a432c1c-0001": 2432, + "01-94-c905534adc1ecec54c348fc54883-0002": 2433, + "01-94-d627a3bde6295ebcd2c2b7183e87-0001": 2434, + "01-94-dbbc2515eace42ce93bae4b39cb2-0001": 2435, + "01-95-081ba255a13aed04cb7dae649075-0001": 2436, + "01-95-19837b3331bc5e3bfa747cef1499-0001": 2437, + "01-95-573bba5c09bc1fe6c5260c9d1828-0002": 2438, + "01-95-a107de05199926ee74ffe546d0cb-0001": 2439, + "01-95-ae2cf0a054df52a2c72fd309f1e8-0002": 2440, + "01-95-bffcea8fd140a494de3cea315d10-0002": 2441, + "01-95-e27326c3e5702f64f618afed9749-0001": 2442, + "01-95-e2f73eb1546c6e8b9dcb08a44b85-0001": 2443, + "01-96-21795299f5b798e79ae90820f824-0003": 2444, + "01-96-5561a037f3f8bfb8ca1872725f6c-0001": 2445, + "01-96-7f8d685c30a9aa63d0625580387a-0001": 2446, + "01-96-86ff04223d19ae8f670fac85b0bd-0002": 2447, + "01-96-8d052f905ef0d817aae0c5109666-0001": 2448, + "01-96-c5e343a0716aa5b92564046e80ff-0002": 2449, + "01-96-e0095b7b984b946556ea3f49c798-0001": 2450, + "01-96-fdf987470f7c99c1638905e724be-0002": 2451, + "01-97-06def37087920bd4e5a6b934b81e-0002": 2452, + "01-97-14fc04ec0f1ded1daf6b767fff0b-0001": 2453, + "01-97-3288951135668fccec7a61a7d7a7-0002": 2454, + "01-97-6b33e564bffa5ad5f97dd1fb20ed-0001": 2455, + "01-97-b69ca3f75f7d1027e763aa63b959-0002": 2456, + "01-98-03a79a7faa78e9fa8f71203379ad-0001": 2457, + "01-98-367c0636921d195ec25e05410c3c-0001": 2458, + "01-98-7c17abc2b7f7b151f08da6b27971-0001": 2459, + "01-98-8928713796bbcad4ecca4637465b-0002": 2460, + "01-98-9696b18f55f24d3985c47b3f617e-0001": 2461, + "01-98-9f26a2881d8f7b2848845469e849-0002": 2462, + "01-98-a5731ccb537fb822f783ba8ff7a0-0001": 2463, + "01-98-c1c60ed192a9f0a1e93678501267-0002": 2464, + "01-98-cb0d69d6fc36316d8b72150c7626-0002": 2465, + "01-98-efe83d815e9b9a9a0df26333ef5a-0001": 2466, + "01-98-fbf31181e77a0cdefbd4781067c7-0002": 2467, + "01-98-fc23428deeb4e4beaf24f57728ae-0001": 2468, + "01-98-ffef84e6529d6a89b38681085bdb-0001": 2469, + "01-99-54ef168544624b82fd67f72228c8-0001": 2470, + "01-99-6bd5aa7b8a85dc333314aa155214-0001": 2471, + "01-99-8c2331da85f72be6795d85ab5d0b-0001": 2472, + "01-99-fa245d73b11c272f8a444b796e48-0001": 2473, + "01-9a-52c69e50a39b364df2c065fb4af3-0001": 2474, + "01-9a-567f38a05d6712b317f168a0cd50-0004": 2475, + "01-9a-56f0e8d2e45c15d9bac8612c18af-0003": 2476, + "01-9a-5885fc0c734c160efe4ea5a7f573-0002": 2477, + "01-9a-74a048920895d8a377e53b23478d-0002": 2478, + "01-9a-8e59ef5c88be1e70ec5989a3f55a-0001": 2479, + "01-9a-f88981eee2cf9179c50cb63b1fb1-0002": 2480, + "01-9b-3f0f20280d9914e37ea923112800-0002": 2481, + "01-9b-455426cc62075ab7c855cf587d6b-0003": 2482, + "01-9b-4766da4bdc51f0d18873cd65f668-0001": 2483, + "01-9b-692c9de8c47ba64d870690700dc7-0002": 2484, + "01-9b-d6f042ce88f59d4f8194428589db-0001": 2485, + "01-9c-0c1daa029dd482b9d23700f2d36f-0001": 2486, + "01-9c-2bf63dac000a9a3ce42a3043bacb-0001": 2487, + "01-9c-639b6baf4ae8f248745e39cde79c-0001": 2488, + "01-9c-86475427f432f867e0a053602264-0001": 2489, + "01-9c-920284e638eda3c2eecd72830667-0001": 2490, + "01-9c-d60c8d647daa89926f4135572ede-0001": 2491, + "01-9c-df1fce62771ba37a946885bb0bac-0001": 2492, + "01-9c-eda968502d212df40e7f8ad3bea5-0003": 2493, + "01-9c-fff44ad228cb8b03f48df5aaee06-0001": 2494, + "01-9d-46ae5e36e53dfcbd715b78eb3225-0002": 2495, + "01-9d-57481abb8440b28f0fdbe59e0e43-0001": 2496, + "01-9d-70daa6e5a61808fa919feb7dbc4e-0002": 2497, + "01-9d-a1e8dd2a8ae5fd5ef6a4b53f99ac-0001": 2498, + "01-9d-a2e667ee34667e5e962d08255f4b-0002": 2499, + "01-9d-bea4ca23e2a0b081554dc8ee9781-0001": 2500, + "01-9d-dd2d9f2d86b92d5598b4cc7c2fe9-0001": 2501, + "01-9e-2bdf89c3d8dbcc4cecdb9689706d-0001": 2502, + "01-9e-421dfcc6a7405dc23bc4815cbe45-0001": 2503, + "01-9e-a190bc957f89f9d5e580f5b1a261-0002": 2504, + "01-9e-a70e0619e4dc6d22c1e2bfad9a72-0002": 2505, + "01-9e-ed468b3f1ffdf42d2756bd78c168-0001": 2506, + "01-9e-f30471b44d38402ef3dc68425443-0001": 2507, + "01-9e-fe5b11a126a9fa706985ec961661-0002": 2508, + "01-9f-072fdd984d8bb51189d2120a4581-0002": 2509, + "01-9f-1558b1468668b3ba4e58a9395d37-0002": 2510, + "01-9f-687c86feb38c907f24bc35e1783c-0001": 2511, + "01-9f-6e74e07e53eec266109d9f559bd0-0002": 2512, + "01-9f-8ebcf2b3156157de7554be4d95f4-0002": 2513, + "01-9f-9180d3a13c6f832e3ab8a51f661c-0002": 2514, + "01-9f-9ae77eb482779c5adeda57df6a2d-0001": 2515, + "01-9f-bb5229f4d084d34c3f852809854e-0001": 2516, + "01-9f-f2a92d4b9091e80a5815b2338920-0002": 2517, + "01-a0-1ce3c4b302119d86d244c552e71c-0001": 2518, + "01-a0-24e222f181b69e96c62326a96fcd-0001": 2519, + "01-a0-3d57b50653476fabf379c9288ea0-0001": 2520, + "01-a0-87b85be60bba894a2a0e7d3361ac-0003": 2521, + "01-a0-99fb781df5e6a5379059274abb08-0001": 2522, + "01-a0-9b2e84bdb37e46d0c42435c89f86-0002": 2523, + "01-a0-d3f428be5325cf63ee54cfdf1d6e-0001": 2524, + "01-a1-11254a1c6518ca0237129bd2684c-0001": 2525, + "01-a1-19fde0658b49b51332d608ae21f4-0001": 2526, + "01-a1-71bbca5bd2dbd0989e7e725a31ed-0001": 2527, + "01-a1-7e604b18d7540bfc0f04ef79c66a-0002": 2528, + "01-a1-80282ecf8efe0b6be1e4384329e5-0010": 2529, + "01-a1-85d95d5faff18f2e5a281f96efe8-0001": 2530, + "01-a1-949c1fc141b0b0cb57fd913fd983-0002": 2531, + "01-a1-b9e7adbcd06545d31898d607182b-0001": 2532, + "01-a1-bae8663e9573a6db1e93bf727e93-0001": 2533, + "01-a1-bd285bc58e6a2c97e3a89bbd7872-0001": 2534, + "01-a1-c6cbd03000e0786d4a7bc792e1fc-0001": 2535, + "01-a1-dc7f12ef4e6cb412c285166ef7e7-0001": 2536, + "01-a1-e486c3ff96b9c08c1f6d5df5decf-0002": 2537, + "01-a2-70d9fc33b297f22e265bf8510168-0002": 2538, + "01-a2-a9cf47cca57ab8dde07c8073d64d-0001": 2539, + "01-a2-b33ecad0732b1a9bbaa7ef40fa97-0001": 2540, + "01-a2-cc255b14ef8311e2fd657ab52611-0002": 2541, + "01-a3-1078f5e29c5dab96e455a98d05ba-0001": 2542, + "01-a3-1ef0277f03676e2e24722f8bef7e-0001": 2543, + "01-a3-2c69d713424b8e600232da4b563d-0001": 2544, + "01-a3-2cc88b85413a07c994b3c3a31472-0001": 2545, + "01-a3-83e319cb6c27074fc5f590639688-0002": 2546, + "01-a3-8740a8064900e406d0ac59894a30-0002": 2547, + "01-a3-8e450b699d6d7bc98b1fc7b6948f-0001": 2548, + "01-a3-98c2dd273fb16bc5a849cf2afce1-0001": 2549, + "01-a3-9c4bbced1a709fd8a09412ec4ac3-0001": 2550, + "01-a3-a5dfcaf2d1e56495c89e38833b46-0002": 2551, + "01-a3-a7cb8ac809313b174cd9768e694a-0001": 2552, + "01-a3-b12fc354b40ac66b52eeaabb7ffc-0003": 2553, + "01-a3-bc3a71842eabfab7ec9646d18633-0002": 2554, + "01-a4-0b3e9c124e8b65976902656b70ef-0001": 2555, + "01-a4-3788c4066a728018559d72dc70d1-0003": 2556, + "01-a4-9d61132231195acc09e88fd94029-0001": 2557, + "01-a4-a3883a32423f9cd9ea444ae6b028-0001": 2558, + "01-a4-dcce5848e853c783a5635e251aef-0003": 2559, + "01-a5-0422091becfd2ec59ca5bca0ea42-0001": 2560, + "01-a5-3c34b9e9be30fce24bcbec2dea5a-0001": 2561, + "01-a5-4a34fb9619178ac3a7c83391341e-0001": 2562, + "01-a5-680f808d88e2043ac38762034c32-0002": 2563, + "01-a5-a8b6a109132466453d2465488494-0002": 2564, + "01-a5-b50de963d2e62ed83a2ded99ffae-0001": 2565, + "01-a5-b710177062018f1e4504474df4a3-0001": 2566, + "01-a5-c98b1c01fbfb36ec41c2fce4b4fb-0001": 2567, + "01-a5-cc8eabb0642793a7fc7e4a4ce917-0001": 2568, + "01-a5-d8a305e4c927f1a60f9cec87244d-0001": 2569, + "01-a6-00e0e0aa364cac70e89ae8d47425-0001": 2570, + "01-a6-0ebd3855371396fe2d5f7cf5adb7-0002": 2571, + "01-a6-6cada15eafe14d7521ccc1380117-0002": 2572, + "01-a6-ba7ca2e0abd5f40ccde903f24ba1-0001": 2573, + "01-a6-be0e2e286c3f44235537c033a0e0-0002": 2574, + "01-a6-cd9159a3c3932ca4272880dc326e-0003": 2575, + "01-a6-e2cadc6ce6f9cac7e65269de49af-0001": 2576, + "01-a6-faee0f4d3293979222bd9f7c5498-0001": 2577, + "01-a7-8fafa21c4b9e1fa2c53c146d5e9d-0003": 2578, + "01-a7-a8c56ceb0dbf75cc7e37c8c25833-0001": 2579, + "01-a7-b4a04d008bfad7205442c7262fd9-0002": 2580, + "01-a7-b9e4afd2cec86f60a3e8907e63b1-0002": 2581, + "01-a7-c29c2e36ca125d301a9d8889e111-0001": 2582, + "01-a7-f8223d45412ebc9eb04eb3540447-0001": 2583, + "01-a8-11c407d9b4c4f83071f66a5aa297-0002": 2584, + "01-a8-1ab344da9103b32293be80461265-0002": 2585, + "01-a8-1df63bd97f5ea1f61a84c98d4f4a-0002": 2586, + "01-a8-3e583e471669a660e4690d6f4b28-0001": 2587, + "01-a8-42f9d8086fd03b46e865647ab25b-0001": 2588, + "01-a8-4426ed10c3f06e6ec001ff1a8be0-0001": 2589, + "01-a8-46a7d502b428a0a79e315ebc5707-0002": 2590, + "01-a8-5c6d16df53d129a6ab84df7c3fdd-0001": 2591, + "01-a8-6617ab86d357ca6fff7b58236b6f-0001": 2592, + "01-a8-920158a99605abed721c0c37a87b-0001": 2593, + "01-a8-b10bef625401067defc8e1e74f54-0002": 2594, + "01-a8-db0ce5b217afae3eeb26f87476a0-0001": 2595, + "01-a9-151b390d0265a2d76c540f3eb411-0001": 2596, + "01-a9-1c3f0fd172c08da50b7b5897bf4c-0002": 2597, + "01-a9-1cac4722c3e88d839489fcecc3d6-0002": 2598, + "01-a9-216e05c2bf385ffe33fce71e6e77-0001": 2599, + "01-a9-24bcf533ceeb4c03776589050eda-0001": 2600, + "01-a9-3c49d37fcbe96ad513b3be26e960-0001": 2601, + "01-a9-40e3486b7d7d9ad1296009db2422-0002": 2602, + "01-a9-57deb4948da730f59091fd661d54-0004": 2603, + "01-a9-6592157cbfc1aa1c78b6c4973802-0001": 2604, + "01-a9-7386a89c2ecea5f81f5e6f414359-0001": 2605, + "01-a9-b0cb59067e385382698a5ca96e9c-0001": 2606, + "01-a9-d021de23aed9ab809f8adc139c93-0002": 2607, + "01-aa-11c8e61fd2be03a97390ae374902-0001": 2608, + "01-aa-183f9e9c79aa85dd55eb5fff04de-0001": 2609, + "01-aa-207fedee7c27ea89c031d03f6cdb-0003": 2610, + "01-aa-2623e79aa58f6fbbd4e6c837e8d2-0001": 2611, + "01-aa-4b904a70419c2866217324f69b7b-0001": 2612, + "01-aa-57eedf0a88890608aec29ba26fdb-0003": 2613, + "01-aa-678296199af37fdd1b8ca66ea2a2-0001": 2614, + "01-aa-85f9bccb15813ca6eaf12baeb307-0002": 2615, + "01-aa-e2fe58fc8621da9e4aeb5816735f-0001": 2616, + "01-aa-f7f3550debf3fbffc02deecb6f8e-0001": 2617, + "01-ab-18ffb176c5ec058b982de1d5eff8-0001": 2618, + "01-ab-200c6810cb262a4f0fe5c66f989c-0001": 2619, + "01-ab-229ec90d10be957079e32d9049d3-0001": 2620, + "01-ab-807150151332d9d05c2294c35f78-0001": 2621, + "01-ab-acc332eb15372276455d9bdce475-0001": 2622, + "01-ab-c61233810f1784dab9582e936d2d-0001": 2623, + "01-ab-c6ec25128e8aeda7c350c6f939b8-0001": 2624, + "01-ab-eca2cdeb994fee24f5cbbc0207f9-0001": 2625, + "01-ab-fc942c4d4f3e21396764b5bc5e43-0001": 2626, + "01-ab-fdf373528adeee2e49f4338fa2b9-0001": 2627, + "01-ac-041a5b114031e2c530f64d66ca4e-0001": 2628, + "01-ac-41003edf76c30f5c9d0d25f31ef4-0001": 2629, + "01-ac-49340b272eb78c2b69be2254cc09-0003": 2630, + "01-ac-61964be79bd4fc47922f71c3c0de-0002": 2631, + "01-ac-76985125019787177bec4b8d3b86-0001": 2632, + "01-ac-c673794c38a53bd58cfab78634c4-0001": 2633, + "01-ac-d1d401a55d42af8cddbd742d2551-0001": 2634, + "01-ad-1c4d1660e91eee9a784c0bec269b-0002": 2635, + "01-ad-49374ac46c2db734a8a024a73c1a-0002": 2636, + "01-ad-55137f10987e7c510cc417b62df6-0001": 2637, + "01-ad-739efbedcbb372f5be999fce6f14-0001": 2638, + "01-ad-84b8db17c121c8263ddf9247abdf-0001": 2639, + "01-ad-a52e74d91fdb6761b33eb5e2720d-0001": 2640, + "01-ae-5029f42f6c3029ce52e9716e57e6-0001": 2641, + "01-ae-5aa76ef972f74eefa97996987d10-0004": 2642, + "01-ae-7ba058f810ef28f6aaa1ae8eeb9d-0001": 2643, + "01-ae-9ab32df040d1084069fca9737c2d-0001": 2644, + "01-ae-9e301b51e2ccfa8c51a30e574d2f-0001": 2645, + "01-ae-b2de58dda3087b07f6174fd013ae-0001": 2646, + "01-af-01592bfea93c333825bdd61e3a72-0001": 2647, + "01-af-499743029b8eb494fd4938a3fec7-0001": 2648, + "01-af-52760406ca905f7d825f03585a69-0002": 2649, + "01-af-73522962a3668e1d12eb51777383-0003": 2650, + "01-af-87ad12ea17e85fe2e2edf84ed75b-0001": 2651, + "01-af-ba7c27c8d9f9f37120a57200638d-0001": 2652, + "01-af-d725c4fc7f3a592f185eae6734e2-0001": 2653, + "01-af-e47635f52c205d0a0986a1602b96-0001": 2654, + "01-af-fa40a6e5faf3680507babb97298e-0002": 2655, + "01-b0-0c60deaab05f437623b661a52aae-0001": 2656, + "01-b0-2d1ab8f361845cb32a1c90e1d0e9-0001": 2657, + "01-b0-3285a3da234659a894600b32863a-0002": 2658, + "01-b0-3289789adb174959b505f4eb78d0-0002": 2659, + "01-b0-3d06be5bdd265b0433b7ddfb2ee8-0001": 2660, + "01-b0-af9852e6f16d4d4434e5fbaadb53-0001": 2661, + "01-b0-b3883b583d1a6e9617a8afc9c7aa-0001": 2662, + "01-b0-c5f4900bf645d84a87202e13f548-0001": 2663, + "01-b0-fb76028ad96348871c30111cc09d-0001": 2664, + "01-b1-268fe7e538fa2584f10950119f98-0001": 2665, + "01-b1-280754035bc90bd86af68f01ed66-0002": 2666, + "01-b1-2c9fbe3be5d3ce2d4ce6ac4a81ad-0002": 2667, + "01-b1-47aebd82d7d3b2993bbd8bb5ed4f-0001": 2668, + "01-b1-58cd1325e6e4edb2b039073d1d56-0001": 2669, + "01-b1-685f31fb3b55bfb43a9e81746115-0001": 2670, + "01-b1-76fe6152160fcf1eb69a00d36cf1-0001": 2671, + "01-b1-97b4366b166bcd0b81ea849171f2-0001": 2672, + "01-b1-ce4ac1c355e337553e9a9438f3fc-0001": 2673, + "01-b1-cef4ee02e018a913736fabfe4969-0001": 2674, + "01-b1-d1e7668c32f9437f7089d793e041-0001": 2675, + "01-b1-d70b34697685d74d202910ad88ed-0001": 2676, + "01-b1-ddeaccdcbfc8758376bd703f2809-0002": 2677, + "01-b2-05304fe82e69351ecc76c587fc52-0002": 2678, + "01-b2-0d495d35780f8c174a309fb898d3-0001": 2679, + "01-b2-1f242ec550dcb6793f93b4759235-0001": 2680, + "01-b2-41e1f0e700d89be9617566fc41af-0001": 2681, + "01-b2-69e8249183f76c963768fff04cae-0001": 2682, + "01-b2-bae81ddd3fe70c316f7aae8d056e-0001": 2683, + "01-b2-be8a1615e40d47f8d9fb4d9f4445-0001": 2684, + "01-b2-eb99726a3be7a466ee759057f423-0001": 2685, + "01-b2-edc201e74c443b11a57f1705f278-0001": 2686, + "01-b3-08383939bb6c77a3eb78a42651a6-0001": 2687, + "01-b3-181628dd52e3907f10e036468d8c-0001": 2688, + "01-b3-350e9a45612dd2ae52cf0e2bcda6-0001": 2689, + "01-b3-6b15350b37cd4728ca9aa208e465-0002": 2690, + "01-b3-6f2c7fd820893c398e6530d2699d-0001": 2691, + "01-b3-7029cf38e38c31c0efec79389a35-0002": 2692, + "01-b3-b22aef4109460ca8d83e4529867e-0003": 2693, + "01-b3-b252c5c1f4529d7d9e1ace0fb998-0002": 2694, + "01-b3-d38598986a31072d8550981f938a-0002": 2695, + "01-b3-f8b4efe40eb2e73e86894068abce-0002": 2696, + "01-b4-0f5945dae933a6ce178ff704e35b-0002": 2697, + "01-b4-5b315ef0e777f41d58a315760249-0001": 2698, + "01-b4-69e8fdf44b8745abd2398f17ae26-0002": 2699, + "01-b4-8044dd6b875a111ef88787d6e379-0003": 2700, + "01-b4-94e8f1e5f196014959d22acb1d30-0001": 2701, + "01-b4-cd5579073c52b416c8f8ea7506d6-0001": 2702, + "01-b4-cdc2ce20c6062d6a4d94f844906e-0001": 2703, + "01-b4-d73cf6ad4e2c45cc14b380f307a7-0002": 2704, + "01-b4-ef096e0552124086de014a2c4827-0001": 2705, + "01-b5-13d04b9cefd63898db7d716fadae-0002": 2706, + "01-b5-306c0f948841c75ccbce5f9727aa-0002": 2707, + "01-b5-34dc0842760676dd9f798b81f2b3-0002": 2708, + "01-b5-3ce28241142e7216fe82db24c9d0-0001": 2709, + "01-b5-4828dfed96f4522d786b1b66faa7-0002": 2710, + "01-b5-4cfb90cd069022c795bdaf3892d9-0001": 2711, + "01-b5-62775c7a3cf00e6e8d48dfd12dfc-0002": 2712, + "01-b5-720bec1528abe39da9f91f641331-0001": 2713, + "01-b5-745c38af57029f70dc15dd6cdb0e-0002": 2714, + "01-b5-ac9952b39ebfe099ace445d53d44-0001": 2715, + "01-b5-b281033d65199f6bc8eadb13cb09-0002": 2716, + "01-b5-c977d8bf14aa5a49388890833804-0001": 2717, + "01-b5-d388381dfa054024bdc4e57e7d66-0001": 2718, + "01-b5-e4388795cc1fa7177b10068860e9-0001": 2719, + "01-b5-fd58698187bb22e65a8e4bf26e2f-0001": 2720, + "01-b6-1e70646dec93e0b4acde808cf257-0001": 2721, + "01-b6-2b5ad31da16af478f659fea7b979-0002": 2722, + "01-b6-448d6ad09f1c4d3660342d38930e-0001": 2723, + "01-b6-6daa12722e5318bb583ae411defe-0001": 2724, + "01-b6-810e22d7727924d5e37bace65a69-0002": 2725, + "01-b6-886e790a5bf6bcfb1068cd462b62-0002": 2726, + "01-b6-b2102de1dab7d01f89e3893a2808-0001": 2727, + "01-b6-edc7dc5cbcab6f16948f6e2d6230-0002": 2728, + "01-b6-f4cc400b650e259202eeabb2ee84-0003": 2729, + "01-b7-87877dc282bd75066a823b716885-0002": 2730, + "01-b7-8f25822949915569d0f55def6c6d-0002": 2731, + "01-b8-2c698a46da35d51a84d5af8351b9-0002": 2732, + "01-b8-4ddece5aa8dcbd970b11281a5089-0001": 2733, + "01-b8-556601e48b680644ea4f17d63bc2-0003": 2734, + "01-b8-558fb0bb7eb4fe6bd36ff3844b50-0002": 2735, + "01-b8-6d030f3beb885ed47e96041f1855-0001": 2736, + "01-b8-823e33cf503dca6d92529b2aa0d2-0001": 2737, + "01-b8-9731f0fec6b7f8ee9de04dc47d09-0001": 2738, + "01-b8-9f7bb099a68ec150607c50f32890-0001": 2739, + "01-b8-aecf41cb7aa5d2e6d4274bc0819e-0001": 2740, + "01-b8-b857532acf918bef13ce2149e3e2-0001": 2741, + "01-b8-d5215e36397c9c26aa0fbd863020-0001": 2742, + "01-b8-fddf2b87b3d100c8ecea2b6188e3-0002": 2743, + "01-b9-125d423951f6fbd5eac05070b678-0001": 2744, + "01-b9-59657eb637fe15cb4ebdd77fe205-0001": 2745, + "01-b9-6b56c4e85344d749d78852d04d30-0001": 2746, + "01-b9-7537dd0b0b4d8c7ee849e56970e9-0001": 2747, + "01-b9-f138560efb1a2e5ebe25bdb6b277-0001": 2748, + "01-b9-f5dfdb0c70f43d47d6534dee5ac1-0001": 2749, + "01-ba-00316f374727e65cde2ed4498f61-0001": 2750, + "01-ba-30e1d37b9eee17e149a9f385b4fc-0001": 2751, + "01-ba-4eaa63b301973aa8efd7e1e57c65-0002": 2752, + "01-ba-582d3ea652b50b81abfeb4869c79-0001": 2753, + "01-ba-671d2ba08d0811abac693ac9e3e5-0002": 2754, + "01-ba-682804485c406465ad6ac49eb03d-0002": 2755, + "01-ba-6e509903ae37b9bfc9cc100aa63d-0002": 2756, + "01-ba-b62801ee0dc1292dfe98f89b52fa-0004": 2757, + "01-ba-e5c43419d91f9dcd1780767017cc-0001": 2758, + "01-ba-ec20759e7b2a8b9b67864f8b3923-0001": 2759, + "01-ba-f24afcbb35d57b668f2c2b24b81f-0001": 2760, + "01-bb-0677af645bfd21d3b35b21fa2a5a-0001": 2761, + "01-bb-4a5b96bbeeeb67992f05c3d6663a-0001": 2762, + "01-bb-767c17c6de441f4e04ea03532891-0002": 2763, + "01-bb-7bf3ed246c7c219d4a73fb0bd38a-0002": 2764, + "01-bb-7e9650f66b4f5a305cfb4d794937-0001": 2765, + "01-bb-99e62ae43d0208eb62acf025b61c-0002": 2766, + "01-bb-9f231a12b7b7162657582d0e1855-0001": 2767, + "01-bb-a23e597da883f6d088e0e56698ac-0001": 2768, + "01-bb-e958bc1e4590ca873ee48d265ead-0002": 2769, + "01-bb-f0796ada37750c7954373c685f6e-0001": 2770, + "01-bb-ffb976b11fcc66041a4d46e2ff35-0002": 2771, + "01-bc-0fab71118f3cc8e874e5e241d0b7-0002": 2772, + "01-bc-332053d8745b4a3da181f1087bad-0001": 2773, + "01-bc-42846c304e5bbc8a5c433fbca347-0002": 2774, + "01-bc-55c10e444851b79a4be3e9d3482b-0002": 2775, + "01-bc-7593df7c665ca341bb69d95fb2fc-0001": 2776, + "01-bc-82760aa8eb3f08e487eeaf540adc-0002": 2777, + "01-bc-d2a4aedca39771760785b3a0bb61-0001": 2778, + "01-bc-daed6a705dc8bfa062c63beaff0c-0001": 2779, + "01-bd-11b78a3745c4093708bc016269d2-0001": 2780, + "01-bd-15946c83ab843b176c463fa497f1-0001": 2781, + "01-bd-3b99186f3be4c2d9e6e5914017b2-0001": 2782, + "01-bd-3d6ec2a6bc9eb9d7fba10b7b105b-0003": 2783, + "01-bd-6728bbf72ac0a86ed20d1e258974-0001": 2784, + "01-bd-6c546b7ac78561343a3dcf15a4d1-0001": 2785, + "01-bd-8a43fdc60d41c1c987554a4a18bb-0001": 2786, + "01-bd-c00d058a81997b58f53c2882c719-0001": 2787, + "01-bd-ca0657319ef6eca83774d61c71f8-0002": 2788, + "01-be-0bc8936af7b089a2b92844d1536b-0002": 2789, + "01-be-0cc5895c36ff3f9d033bec721665-0001": 2790, + "01-be-25064adca8c313331d489ea08f66-0001": 2791, + "01-be-34906e909a532db35774c2c87a07-0001": 2792, + "01-be-3b89e8afcef063e3a66edef4962e-0001": 2793, + "01-be-5aebd46db75252c4e140de124926-0001": 2794, + "01-be-cb6a8ddd28531d1953d2002beece-0001": 2795, + "01-be-dc0dbd4bde36a4855b90eb524c34-0001": 2796, + "01-be-e0d4f471490fb8b8afe1de651049-0002": 2797, + "01-bf-46b32d8aeb41d0483b76c908f494-0001": 2798, + "01-bf-61cf05b02b3240adec7d6f91a8b4-0002": 2799, + "01-bf-6cbc3d1da1a0e4741873ee081687-0001": 2800, + "01-bf-858820ade5bbba39012e5b1d2a66-0002": 2801, + "01-bf-b66fb4ffcf3c4ed8af4252a62ed5-0002": 2802, + "01-bf-e0ff9f2829df7874fb2994667ab0-0001": 2803, + "01-c0-36b6c8e0311b7550a9195001925e-0001": 2804, + "01-c0-4545e9566d7e32272fbf0819e4df-0001": 2805, + "01-c0-6166b25fe833e1a86043c5813fe7-0001": 2806, + "01-c0-65e7a4510a8394c6be61fe34f77a-0001": 2807, + "01-c0-9288041d0905019cb9ebd5b6e47a-0001": 2808, + "01-c0-9e90ebfe588354544afa231c53c8-0001": 2809, + "01-c0-acc134e02d8a1609b7721ea8731d-0002": 2810, + "01-c1-0e1ab8047eef60173d2fd31292cd-0002": 2811, + "01-c1-13742a9f296f84b94f336b5ed2f7-0001": 2812, + "01-c1-1c15f96f5d9989e8e0684be9927d-0001": 2813, + "01-c1-364a3aeede752ea0f1799827ccf7-0001": 2814, + "01-c1-3f45eb68f84c55bac6d3b61d4df1-0001": 2815, + "01-c1-419d2644bd311b1a3728f59911f2-0001": 2816, + "01-c1-422132f838871e23ff887d8d35e3-0004": 2817, + "01-c1-51a6d723a319f223df08ed9a3ac7-0002": 2818, + "01-c1-5cb80c8e7de6fc505305da8fee67-0001": 2819, + "01-c1-8f55b03afcf02ece71a29da42b0f-0002": 2820, + "01-c1-9d8a4dab032c03bc75edcb10823e-0001": 2821, + "01-c1-abcce23b55d1260e930b5a1604a8-0001": 2822, + "01-c1-c2aa760ca101bc43890e35371915-0001": 2823, + "01-c2-3413d24aa774324a6054e56d2fc7-0002": 2824, + "01-c2-5f1c405483cecfeb4246cb6eaa13-0001": 2825, + "01-c2-6fbe3c72305e8be7f8c0c50be354-0001": 2826, + "01-c2-8f5e43f7d86749b5c5e25dc96018-0001": 2827, + "01-c2-8ff6517f354456966231327fc3dc-0002": 2828, + "01-c2-98b7dc9e35a0d31b25760999e6b2-0001": 2829, + "01-c2-c08271372f5d2cf05e67853c65cb-0001": 2830, + "01-c2-dba4f1bcd5923bb8ef066679c12f-0001": 2831, + "01-c3-1fc1eb9f6719577ff748c1309f01-0001": 2832, + "01-c3-26fcae197e14df19a8fa1b9ab841-0001": 2833, + "01-c3-440a799c31d3d5e6f47da2bbf731-0001": 2834, + "01-c3-7591682a0ac42f56cb11f4f2c5aa-0001": 2835, + "01-c3-fdbd046a73186b2169b444d13db4-0002": 2836, + "01-c4-29f42cde0ac8581aaa9f8f0510c6-0001": 2837, + "01-c4-69de59eea6121096cf6d8016bc8b-0001": 2838, + "01-c4-7bbada2135d5c9dbe0fa4f9f68f7-0001": 2839, + "01-c4-acfe07510e6e490ca96f66789e26-0001": 2840, + "01-c4-b078e6c4039a03691901c6052114-0001": 2841, + "01-c4-be93d0a2a41a861a41d931955d40-0001": 2842, + "01-c4-d90e580c08d16de76fa522c2263b-0001": 2843, + "01-c4-da7a0303754acab82d5c2d4a8552-0003": 2844, + "01-c4-f82f374ff9538e6594348d42ad65-0001": 2845, + "01-c5-1690536f258dab5c6cb72609c000-0001": 2846, + "01-c5-50d02dfdd6eb53e82e12f2375a9f-0002": 2847, + "01-c5-61c03ae095753cab2515d1acb2a6-0001": 2848, + "01-c5-6fc726e4d5fe79441376edbc42e8-0001": 2849, + "01-c5-7d76327ff1ab16b8ec7a366e0e6c-0001": 2850, + "01-c5-8a327e0964808d1044727b87fd33-0003": 2851, + "01-c5-a1e0b920b75010b3eb36d939a24d-0001": 2852, + "01-c5-bd5cbdb1c38e9a94c51b0b69f894-0002": 2853, + "01-c5-d2b323f529768e396b982c94704e-0002": 2854, + "01-c5-ee62073cf012399e8211c75c2669-0003": 2855, + "01-c5-efe1a5e70b73b7e22942fe6bd78d-0002": 2856, + "01-c5-f848dd320e732d03d8783d0b27ce-0001": 2857, + "01-c6-1a88723db6a612426015ea7b9202-0002": 2858, + "01-c6-710635501a8b4181a70f45077892-0001": 2859, + "01-c6-7bc0c5b7babb07f60887a27bcd10-0001": 2860, + "01-c6-7d4ed40d9c9ae4b822e11da7cec7-0002": 2861, + "01-c6-85a44d36cb3d4945082cabd7074d-0001": 2862, + "01-c6-eed92295c541005c09a23919d584-0002": 2863, + "01-c6-fa8b0766482962e86441f7bb599c-0001": 2864, + "01-c7-0a12cd5492a25ee38ba27ac1b5ef-0001": 2865, + "01-c7-18f863b3526e0d477a1321ca3990-0001": 2866, + "01-c8-047645bf9f5d7df9e9ad86102f9f-0003": 2867, + "01-c8-237530430222b9705b9474b8773a-0001": 2868, + "01-c8-3566c55b51d56811ddff40aa4a2d-0001": 2869, + "01-c8-47b8381b88d34794d8ad90ad6c82-0001": 2870, + "01-c8-486916bd93b4d30a6f9277d68f16-0001": 2871, + "01-c8-5e5ac2688dcfec557084a40f0698-0001": 2872, + "01-c8-6cdc640008188d35e59784d1425d-0001": 2873, + "01-c8-a9357015aacea95329a16e4bcdc4-0002": 2874, + "01-c8-d6841d7f00338a4b86fb86a3d06b-0002": 2875, + "01-c8-e0c06334672e2ac48f55863b6ce6-0001": 2876, + "01-c8-ee15e7824583defb1bdca7ce7cb3-0001": 2877, + "01-c9-1e475a6fcb486021e0b4e92cc03a-0001": 2878, + "01-c9-33842447b1e40d838e077015b8d0-0002": 2879, + "01-c9-3caa17535f19de4e4dcdc7d0d69f-0001": 2880, + "01-c9-3eaae1502894d12c7a943bbc97fb-0003": 2881, + "01-c9-5418409df63c2d87e6485664b5bb-0003": 2882, + "01-c9-a1ce4895a9696732afc895bc5f5f-0001": 2883, + "01-c9-a2d3e6c79faa8034af6e99b41334-0002": 2884, + "01-c9-e067c2f533b3d15c7d8cb4aaf28f-0001": 2885, + "01-ca-0f202105b16046cfc7744db48adb-0001": 2886, + "01-ca-8f469800c4b57a7c8d479a70244a-0002": 2887, + "01-ca-954ad1ee0ed7553f6bb3a3272a08-0002": 2888, + "01-ca-a6349d46402018a43ce1daf0648b-0001": 2889, + "01-ca-a99e7048ae92a04fd3fae27e1600-0002": 2890, + "01-ca-f122f4017da04247586eb3846202-0001": 2891, + "01-cb-78e1c80980a01a243fb8154f1ecb-0001": 2892, + "01-cb-eb464ae3486734039fd8d4c42f7c-0001": 2893, + "01-cb-fb1fa1806fa8a51c678cd24d2234-0002": 2894, + "01-cc-7a50b4791548b5ebcc08bd61abe2-0001": 2895, + "01-cc-7c9ee0e2484efd10f9c6da19075a-0001": 2896, + "01-cc-8a68fe35fe1161e67c430fa5a467-0002": 2897, + "01-cc-8e2df2409f75b5c0f7b69eb6c250-0001": 2898, + "01-cc-9357187ccbed5dc8cc2b6bd4b164-0001": 2899, + "01-cc-bef334989cde5c72d4b741d716b4-0001": 2900, + "01-cc-d24a7505644715e3a5f97f9e24c6-0001": 2901, + "01-cd-0e842ab9cb54a2b936dc34b48748-0001": 2902, + "01-cd-179d820695fab8769b30588ceb2b-0002": 2903, + "01-cd-25fdcbfc89024331b4712bab08d1-0001": 2904, + "01-cd-4e3590d4d2217395414abc2069ae-0001": 2905, + "01-cd-4f02c613c8665fd19c82b759a5a0-0001": 2906, + "01-cd-50fdd09f620d6269a2f5fdb301eb-0002": 2907, + "01-cd-7315ab8c263927ff4b749f711e0b-0001": 2908, + "01-cd-899c0f890561d5ecaf7b1dc83407-0001": 2909, + "01-cd-8b8b91e0e2d400cc77c2d7dbb55b-0001": 2910, + "01-cd-a9e202e46d72169264dccd9c915f-0002": 2911, + "01-ce-0179eeb60a5a5d0f6877507fdf40-0002": 2912, + "01-ce-06e83b617024b1a65f4a290b8093-0001": 2913, + "01-ce-1f002201d06df265761cc231c74f-0001": 2914, + "01-ce-2edcb38db273b4a2039dd115c868-0001": 2915, + "01-ce-4243b4464553cdd15a7a07b21c23-0001": 2916, + "01-ce-6a41cdb14e9cb807966735d7de6d-0001": 2917, + "01-ce-badba5c07e638ceccfb1d156776c-0002": 2918, + "01-ce-d9104a51ae708225e3c0851585f7-0003": 2919, + "01-ce-e65fe6b546ae92c16e5c3c9a59d2-0001": 2920, + "01-cf-c1a0625bc702fe232ae510fa32d8-0001": 2921, + "01-cf-c225673422555398b42ca5ca0775-0001": 2922, + "01-cf-e77c8c612879b63b475cb6b7842e-0002": 2923, + "01-cf-fb99d30d089109e55f3fd4316f50-0001": 2924, + "01-d0-5dc72acfe56149340ce68c2a1142-0001": 2925, + "01-d0-7afefd3be2464bcea31c3f08ee60-0001": 2926, + "01-d0-8482812c4deafd7569361dbbbc52-0001": 2927, + "01-d0-8899c08db4cbe45dfe92731ce10c-0001": 2928, + "01-d0-91e4de013aff62e8810a1cdec026-0001": 2929, + "01-d0-a5c69081071f910634ee14a52a36-0002": 2930, + "01-d0-b085bda5487983c50aca6b1776aa-0003": 2931, + "01-d0-c8b30decc494ac79c9e8907a9918-0003": 2932, + "01-d1-0130991fa7979f38e83ed589615c-0001": 2933, + "01-d1-06bc7f9d684ab58a84c943c73e65-0001": 2934, + "01-d1-458267ba8992dea9fd8759b444b3-0002": 2935, + "01-d1-4eda59513ac061f028151af3ff50-0002": 2936, + "01-d2-014cb2fb928413daaac5497b9dd0-0001": 2937, + "01-d2-05f43d059cc9af29bf8a0b71c271-0002": 2938, + "01-d2-11cc072cbaad50ba372fab6d6c5f-0002": 2939, + "01-d2-385f86f1cbce4999c8c0584a30d6-0001": 2940, + "01-d2-417dc82667dc5b97b5702e53e422-0002": 2941, + "01-d2-5ab7d35f1d0e92662208ceeb609d-0001": 2942, + "01-d2-66e20dca1f8f716dab65fa9227dc-0002": 2943, + "01-d2-77ff18e09002bb6ffe7d38a26589-0002": 2944, + "01-d2-7d384698f7db44cf38aef22f3412-0002": 2945, + "01-d2-9cd5c18cd7d9fc4421e3e76e34bf-0001": 2946, + "01-d2-a2333037cb97476b8708f74e867c-0001": 2947, + "01-d2-a3b42291d1a671a5b8f739bf4644-0001": 2948, + "01-d2-a98533aa9eabe3c78081fccfaba0-0002": 2949, + "01-d2-b3243ca1d8e2a7a07e8412d193d7-0002": 2950, + "01-d2-b7f38875d3784b6b00c127ee9640-0001": 2951, + "01-d2-b80bf5ae75075269815e986a3ac7-0001": 2952, + "01-d2-b96d3ff3c29f58e831ac12fee032-0002": 2953, + "01-d2-d730af7945d9a6cafbdda3fcb593-0002": 2954, + "01-d3-1d999c627c248e60d621b1251ed2-0001": 2955, + "01-d3-3d141d91dfa2751b2d29c8731bf1-0001": 2956, + "01-d3-3d550d89d2a3988c8808e8391b69-0002": 2957, + "01-d3-4f647a71fed83280743bb9cd88b8-0001": 2958, + "01-d3-5d603393b18124d7851a5ebba965-0002": 2959, + "01-d3-6e6aea587e5124d49c2e09ccd63e-0002": 2960, + "01-d3-dde5b1ef1191e1ec4e3623fd314f-0001": 2961, + "01-d3-e1b2e69e558dcdf1a410662a8bc3-0001": 2962, + "01-d3-f069665b19ce6ea89509f140ac6a-0002": 2963, + "01-d4-2f06d1bcf656b1fcc7e531dae743-0002": 2964, + "01-d4-697b6e934a7682542fd12610a670-0001": 2965, + "01-d4-6e82e5d3bea889317d515b6f7d95-0002": 2966, + "01-d4-78fa5bdb5a7eb7a5a090498931f4-0001": 2967, + "01-d4-f1f7846f400acbccdc1636963bce-0001": 2968, + "01-d5-03472e1c5cf815bf6e87b1302b9d-0001": 2969, + "01-d5-035e3c01c4b072c3d475d7ad802c-0001": 2970, + "01-d5-15426d72868bbf086455e3fb9280-0001": 2971, + "01-d5-1e595c6b17ba8199acefd86992ab-0002": 2972, + "01-d5-39d3e5c1d53f96f8b95d4e24f71e-0001": 2973, + "01-d5-5321440c323f71cd1755fa659a21-0001": 2974, + "01-d5-62267567940ba07f3c50ad7efaf8-0002": 2975, + "01-d5-87942b03232b1ddae3a90b78baa7-0001": 2976, + "01-d5-ad9d4ddb65e5b771dad4482c9d30-0001": 2977, + "01-d5-f7eb64fdf3f6f5b5d9be32275521-0001": 2978, + "01-d6-0714f7ddc741728f0971ca728501-0001": 2979, + "01-d6-1143742e8a0ec20640d1d7c5993e-0001": 2980, + "01-d6-22fb6ba36d4920b2c4448cb1c2c2-0002": 2981, + "01-d6-6af9716180482e745cd3bea7a044-0001": 2982, + "01-d6-73b1d864987aa2f405a760c0f620-0003": 2983, + "01-d6-9968420a94e4d6f5a8ca7bc2423c-0001": 2984, + "01-d6-be77afef164e0d493d43135b40f6-0002": 2985, + "01-d6-f97f6b8e84c1277f890c1cdf179c-0001": 2986, + "01-d7-232a6694b545ff4b7389f07a9553-0001": 2987, + "01-d7-6c8b68b1803968db55e4a330704a-0001": 2988, + "01-d7-781214a2430e6179f446b25bbe56-0001": 2989, + "01-d7-aba61ea5376dcfcd806035aa3086-0001": 2990, + "01-d7-b862cd4db54bf76ec600a4d86061-0001": 2991, + "01-d7-c721944ceb255d81fa4f6cbbecb5-0001": 2992, + "01-d7-d33332e6f05372ea195149265be9-0001": 2993, + "01-d7-d89c0be549e909f5eb3f71041f6a-0002": 2994, + "01-d8-02d38ae6d3029ba705d9d5198657-0002": 2995, + "01-d8-180dc231acde4204987c27c09712-0003": 2996, + "01-d8-51b24ad473d9ea91883f97215742-0002": 2997, + "01-d8-7ce467b3f82eb268c23fea874b64-0001": 2998, + "01-d8-cbf4a6c1395d934bc29f324b02d1-0001": 2999, + "01-d8-ee27dfc8420f7c00b9fee0864667-0001": 3000, + "01-d9-01d4e264b017386e6822d72cb852-0001": 3001, + "01-d9-22a740f1f7f61e5554fa89746251-0002": 3002, + "01-d9-4fdc7cd9e8cb515ee7c2bac39929-0001": 3003, + "01-d9-624fd046c958d452762d6e12988a-0001": 3004, + "01-d9-9fca4bd852d1a6db5ae70a6a132f-0002": 3005, + "01-d9-e64411146d3ba677d3b8191dc21a-0002": 3006, + "01-d9-f32942b091f033f5329e9bc0af5e-0001": 3007, + "01-da-0f28a292e7852c9a32a465dbb343-0004": 3008, + "01-da-337b80f381d5dcfd5a296d66cd3b-0002": 3009, + "01-da-3af0231786b1a8e771ded65113c4-0002": 3010, + "01-da-451ace2d73a3cdfdb6420585125a-0001": 3011, + "01-da-46ebc784e8a8eda98a315e1dab26-0002": 3012, + "01-da-5764702c631b4b952acede7dfad7-0001": 3013, + "01-da-7d629276e16b00575aaaac48e34b-0001": 3014, + "01-da-869c86843622737f37b73c303f11-0002": 3015, + "01-da-af8c77adc33adaf3360b43ce7849-0003": 3016, + "01-da-cbd7dddd3534f20412b73c85d89e-0002": 3017, + "01-db-15dd367fad3e700c65f4af9de1d5-0001": 3018, + "01-db-561de812e397fd5e0b4d9092c9c3-0001": 3019, + "01-db-8a453dd9bec3125f5e194aad4fdb-0001": 3020, + "01-db-8ada32ee108d197238a8d216db4c-0002": 3021, + "01-db-acd695271d439464df8f8d782157-0001": 3022, + "01-db-ae2af4afdab18ad023cbc9fac1bd-0001": 3023, + "01-db-b5c161490686f63312fc1baa9930-0002": 3024, + "01-db-c08e61db5141e1be89e1b1c81b67-0002": 3025, + "01-db-d28ebcab8d58aeda6d94abce633e-0001": 3026, + "01-db-ee6365c418bc00361e110bee7d77-0001": 3027, + "01-db-fcd465c4e8e4dfa7fd49f2dafbd9-0001": 3028, + "01-dc-3c421c56667630f763d1c897adcb-0001": 3029, + "01-dc-3fe655c95e807d5ac85f83c2f896-0001": 3030, + "01-dc-43d109965fd99073ca660857f1f7-0003": 3031, + "01-dc-45d19cf3d9bc9089a07ec4463887-0001": 3032, + "01-dc-c349cc0d8e274b0ad0bf7a7bcd2e-0001": 3033, + "01-dc-c65c41b0f9408cce548865327149-0001": 3034, + "01-dc-de73061171303985184baa42e290-0001": 3035, + "01-dc-ebe742050f05676fd6515cc66724-0001": 3036, + "01-dd-2c81565a2fc3529327fd105bb4d7-0001": 3037, + "01-dd-65a59e70c763b6ea6ac9c2ab1072-0001": 3038, + "01-dd-6a19c64a491ea1cc8ab55f87aaad-0002": 3039, + "01-dd-763ecfb27eb337233b51c62a0713-0002": 3040, + "01-dd-8a89956165b59696664c9b8624f3-0001": 3041, + "01-dd-9f49bc5806eed0021b4e76bfeafc-0002": 3042, + "01-dd-b5373c85cf4d8a07f819546a0cd9-0002": 3043, + "01-dd-c269a583e689f0dc738eac91bc9e-0002": 3044, + "01-dd-ce17b82ca4ceab4e5d29bffbd403-0001": 3045, + "01-dd-d4e62ff23598bbe111219b28a4e2-0001": 3046, + "01-dd-e098512b436d524016430dfb80de-0001": 3047, + "01-dd-ec895e3bbc337a878ea6f50ac434-0004": 3048, + "01-dd-f58fe1a90471b31ee891e01f8966-0002": 3049, + "01-dd-fac144d3610b7f57095442e3abba-0001": 3050, + "01-de-0882339617dd332356c62f64523e-0002": 3051, + "01-de-112b04cc3b0ae20759b89a64f201-0001": 3052, + "01-de-156cb9cefeacf5f71c8c2666ddc7-0001": 3053, + "01-de-50cb20eab8936e23741bdf8d3e67-0001": 3054, + "01-de-67cd0c0dcd50fbee91267bc64f07-0003": 3055, + "01-de-70c91f21f1d70adcd2a0cc9019d8-0002": 3056, + "01-de-7e5558803e712977539ae6f9b722-0001": 3057, + "01-de-a1cb0c8b4b88ad89f833225b65e0-0001": 3058, + "01-de-ae15ef297981e29e7c097caa5f6f-0001": 3059, + "01-df-43286c02a4db8ce385a3c2d2987c-0003": 3060, + "01-df-63418a5b9154d7b6eb5c49046836-0001": 3061, + "01-df-6ddd332f60392342ce630de17dbb-0001": 3062, + "01-df-a99666ae7ca20f8f38f294bd5ee0-0001": 3063, + "01-df-a9d38d92d4d0ea8c8ec7764805f7-0001": 3064, + "01-df-d6415c1c4c62a01208db89490988-0002": 3065, + "01-df-fb8a3992956b96b94ad5c0c893d5-0004": 3066, + "01-e0-225b46be118d267c25abf452e780-0001": 3067, + "01-e0-243ce3dff367b103c244d282645e-0001": 3068, + "01-e0-25104f73eed6ddba4ccd2ee3b9b4-0001": 3069, + "01-e0-4176bc6c68123478ddf166b7e023-0001": 3070, + "01-e0-532a3ac8274fe82fae2d09a73d7b-0001": 3071, + "01-e0-a7847fb937333dac649c3ae59ef2-0001": 3072, + "01-e0-b819f13dc446684d5d05a1ec7da0-0002": 3073, + "01-e0-cacafcf53564dc3111d4208c8b2e-0001": 3074, + "01-e0-d52e43ab1cd290609b952a4fdb00-0001": 3075, + "01-e0-e33a6f80534d737ab000d800077e-0001": 3076, + "01-e0-f6522934425b0138e641abef9d0e-0002": 3077, + "01-e1-05dc7444d75cd53310d0a2f751d4-0001": 3078, + "01-e1-3ae26c96c6a8ea24ea357ff25d6c-0001": 3079, + "01-e1-4d49c5ec4a85f21380107d798006-0001": 3080, + "01-e1-574c44f1a1e67477bb2766685e5b-0001": 3081, + "01-e1-5ff101df7c3dc9c4bc81cb37d994-0001": 3082, + "01-e1-784bcd4e9da71ffaccf7d5b94d03-0001": 3083, + "01-e1-8d3b151ad674fcdccda9641f7587-0002": 3084, + "01-e1-b525743787231ba17dfa0a2daa4c-0002": 3085, + "01-e1-c6102d05464471d59739e4c256c4-0001": 3086, + "01-e2-086d9d7303e49948b80a2ab59560-0001": 3087, + "01-e2-087dc5e36bcd94a63d6cbb4e20fb-0002": 3088, + "01-e2-0ef874fc50941bc727cffad5dbb7-0001": 3089, + "01-e2-6bcbc64ba84d97366c2c77e4c12b-0001": 3090, + "01-e2-789d45487f77fee253bea9c92486-0001": 3091, + "01-e2-812db1d1962a725affca34d5123f-0001": 3092, + "01-e2-a9123578dacca4cdc9015906308b-0001": 3093, + "01-e2-aa53468bedaf0122ed71b62cc26b-0001": 3094, + "01-e2-b843a8af6018f8a0d2de9286f0a5-0001": 3095, + "01-e2-d13f64e04874a2e13fbedf0bc056-0001": 3096, + "01-e2-d629b3014256ee7577a0f823b6c5-0001": 3097, + "01-e3-0e9b1aeb9ded7dc8f0f8c179f1e4-0001": 3098, + "01-e3-0ec248ad572e6176dd3eb7235770-0001": 3099, + "01-e3-4531e60c576a65a9eb9d9748a33d-0001": 3100, + "01-e3-552aae7f706f87f34c5d56d8a320-0003": 3101, + "01-e3-576e8ff7d1e0a440d99f051319a3-0004": 3102, + "01-e3-7ab23839a1fb0641b7a729fdfc1b-0001": 3103, + "01-e3-aeb053db1aa404e47266e82ed985-0001": 3104, + "01-e3-cc7f3af43d8cbb77443c53248632-0001": 3105, + "01-e3-daea50ecf8f70985d0cd5279e1bc-0001": 3106, + "01-e3-dc7932b46e017589096364d528b4-0001": 3107, + "01-e3-ed75cdb79c0c007d7bd31e388a53-0002": 3108, + "01-e3-f95d762dfab45cca13d042926683-0003": 3109, + "01-e4-013ad28536226c78b8f329fc6f74-0002": 3110, + "01-e4-191f4b132f270e484e7e6fe82bbf-0001": 3111, + "01-e4-5009c5aa6a0acf76c7a197e689d8-0001": 3112, + "01-e4-57019030f1051f349678c945234e-0001": 3113, + "01-e4-6564ee938a3ab4811769fc7d3e27-0002": 3114, + "01-e4-b9e02b44604e97c3aaf45f1684b3-0001": 3115, + "01-e4-c3deb6383a51deb323cd48d663c8-0002": 3116, + "01-e5-5f4f430d2534cab98006b59837aa-0001": 3117, + "01-e5-c9793716cc96580a682a48e88a58-0002": 3118, + "01-e5-df6969c71c4cb3b786cde09b4efc-0002": 3119, + "01-e6-070102ae7dcef4608b1131422d23-0002": 3120, + "01-e6-2371ecb3e7cf9a94a8b66ddea2c2-0002": 3121, + "01-e6-4237965d0ec33ffc4028b3aecb9f-0013": 3122, + "01-e6-b64d91a227c05212257bf894735f-0001": 3123, + "01-e6-ba45c61bff99528769057482509d-0002": 3124, + "01-e6-d861dc94d9a2bf824c49069cfa0c-0001": 3125, + "01-e7-0d7f8928f7291da10601c23aa94d-0002": 3126, + "01-e7-0e1ccbbf3b8cbe26aa4b7c3a6145-0004": 3127, + "01-e7-71ef106a9535e4eb11b76929500f-0002": 3128, + "01-e7-887559cddadc49f1ed445322dc0b-0002": 3129, + "01-e7-c3ad448e659a5a5fba3b082a1efe-0001": 3130, + "01-e7-cd5f0c97937bae553d6c6c999367-0002": 3131, + "01-e7-ddd6b8936b925e29f7b3a9528d7d-0001": 3132, + "01-e8-05b736f50b46f9f28bf12ce7dcf6-0002": 3133, + "01-e8-3e5b6f5e073ec9600e238bcb1017-0001": 3134, + "01-e8-5743fa425ac39452ba0103b0ceb1-0003": 3135, + "01-e8-5b09d39804afef7eab5a026b0b75-0001": 3136, + "01-e8-76d965901d0ddef2b2ad7cae650b-0001": 3137, + "01-e8-aa4a0a1650d71291a9fc96c6bb76-0002": 3138, + "01-e9-01e146652459a2693fe6d4f420dd-0003": 3139, + "01-e9-0a5fae6f205ea23675e4ed15a69c-0001": 3140, + "01-e9-1632b86062aa006ad0c05e158b79-0001": 3141, + "01-e9-205cb7430f5e7f882e8027b2afda-0001": 3142, + "01-e9-485247ae4df3ef2ac7c0cb7ea9e6-0001": 3143, + "01-e9-4ede7d78fd43fa15a2f8c64e157d-0001": 3144, + "01-e9-63b9214251785b307e240836e8c7-0001": 3145, + "01-e9-7d1ddb01370528c02a02c8f006ea-0001": 3146, + "01-e9-83a06bf8b0bbaf910dd8fc2e8420-0003": 3147, + "01-e9-859830b5a240d4f65e7fe893db40-0001": 3148, + "01-e9-891782063fca2453c2809004f4ae-0001": 3149, + "01-e9-fa00cd4ec982ee704239f34064a7-0002": 3150, + "01-ea-0610c31aac3020218b0bfd0ef25f-0001": 3151, + "01-ea-1188c466c05b782829a1f63685b9-0002": 3152, + "01-ea-1e53158bd9c8cc8c2c4a1cbd488b-0003": 3153, + "01-ea-27a46cc0555629bffe0ecaaeb9cb-0001": 3154, + "01-ea-486f705817408d6733c0e31853cc-0001": 3155, + "01-ea-4c2cd7a10e4a76b693ac09b64343-0001": 3156, + "01-ea-944e769d35fc402b9bd9cacd90ce-0001": 3157, + "01-ea-addaf1942072ae172df2d51b4f05-0002": 3158, + "01-ea-c41fdd0cdf04e3f542837b31aeef-0001": 3159, + "01-ea-f4b190070a02aec711fbe3dfae50-0001": 3160, + "01-ea-f9b9d4aa70c441217bbdf08afc43-0002": 3161, + "01-ea-ff2f7a5c592e9079627a950b2b93-0001": 3162, + "01-eb-60d53bbb9bf7324dec298981d467-0003": 3163, + "01-eb-e9539991aa57657090060498d568-0001": 3164, + "01-ec-0dd262754c24b8685600393612c7-0001": 3165, + "01-ec-106e9c73b52e26a8c629badf8df0-0001": 3166, + "01-ec-1d2ea9fa451062d8587d4b3dd444-0001": 3167, + "01-ec-3de4c987fe0a6d8e196271e904a8-0003": 3168, + "01-ec-4609fbb2ab5c1b5d0585ed0b286b-0001": 3169, + "01-ec-5eb946174655294afbf8a3b06559-0001": 3170, + "01-ec-67c8f444d5c30ab4736b761b19e1-0003": 3171, + "01-ec-7d9bcd5abbb462d7dc55609029a8-0002": 3172, + "01-ec-98761e3a6569c460d2ca8824e68a-0001": 3173, + "01-ec-f5d59765c9346ce2351cb9e294b9-0002": 3174, + "01-ed-0e4cec280d7dc88b47403d480d1b-0002": 3175, + "01-ed-25fc544b4c28da0c53d1e57844d3-0001": 3176, + "01-ed-43c281d7e61da871a4d9f7fc0209-0001": 3177, + "01-ed-86f5d55e880b625c98d7d1858ab5-0001": 3178, + "01-ed-a18000f0abc41c539c55a124ee3c-0001": 3179, + "01-ed-a1af932bfb5cb8aedf02e733f0ae-0001": 3180, + "01-ed-d998364901fba818d6e043468a1c-0001": 3181, + "01-ed-fd277ad0f3f20f3ed5bee0bd91c8-0001": 3182, + "01-ee-2e43e67a065f77db000690977b6e-0001": 3183, + "01-ee-44c7d51f3a9339c53c178a56bde0-0004": 3184, + "01-ee-922867f8bb324c234e7ea6137ced-0002": 3185, + "01-ee-d17783510c6c742d1a973a8d28d6-0002": 3186, + "01-ee-e17be7cf7f65a905ad753af83321-0003": 3187, + "01-ee-e29e12ae08d10469cefdcbd4073f-0002": 3188, + "01-ee-fa4828f0f49338258bd0ade7b79c-0001": 3189, + "01-ee-fea4eb2386d616cd1ecf40b0d840-0001": 3190, + "01-ef-2341d92f21b5c12a31ccf5c5b0a7-0001": 3191, + "01-ef-3aa88e01e804f1cfc482349e432f-0002": 3192, + "01-ef-4986556af6fc0706903030cf6caa-0001": 3193, + "01-ef-627c6d4eda8751abb79b346a10b9-0001": 3194, + "01-ef-69601f6444ee8bf3b60e3d6f4339-0001": 3195, + "01-ef-8c6fa5a29a3b964e7e4752825c41-0001": 3196, + "01-ef-c990b2a3c1eff1c1d188c29b9401-0002": 3197, + "01-f0-065dc2f7bf26ab5f9c203f778695-0002": 3198, + "01-f0-76b0d7c9d01225b59cb00c578a06-0002": 3199, + "01-f0-7d8ea9f688e1999642e93fe6ccac-0001": 3200, + "01-f0-a20ad0dd97264f279fdfadf91306-0001": 3201, + "01-f0-c13ae58adfba59446b48be8170bc-0001": 3202, + "01-f0-c55ec3a99bb3bbc04005650990e1-0001": 3203, + "01-f0-e0164e6ba93e40136ab9047618c6-0001": 3204, + "01-f1-3528656356d82ff53ccd975d99cb-0004": 3205, + "01-f1-49fe3d2003dbaf3071e9b7892ccf-0002": 3206, + "01-f1-686e7ab08d35966e0127af27a5fb-0002": 3207, + "01-f1-a966130898747740b5fd1c9d3f04-0002": 3208, + "01-f1-aeb4df0167930075a1921751da0e-0001": 3209, + "01-f1-be9f8a8983855b7164cd84e21e75-0001": 3210, + "01-f2-2845e0cfea80a1998fa4fd559656-0003": 3211, + "01-f2-2f40679221f5e2a565cd0e671768-0002": 3212, + "01-f2-48107250efcdd296b90cfb5cacd0-0002": 3213, + "01-f2-87a3e1ef90bcc54f33b2858a983b-0002": 3214, + "01-f2-8eb552cfdc8ce764ef00c52289bc-0002": 3215, + "01-f2-cb0f8fbf5db74c4247255a579ef3-0001": 3216, + "01-f2-d48a5eefd30077f105ccea7e5fa0-0001": 3217, + "01-f3-4caf8847e34e9288c1276362abbd-0002": 3218, + "01-f3-5526a9877911349ae5cb9e114d17-0001": 3219, + "01-f3-88224298e84eabd24eb360f94a7e-0002": 3220, + "01-f3-95aac743995577f2821a3d2037a5-0003": 3221, + "01-f3-9aaa5015197753f28b81ebea8086-0001": 3222, + "01-f3-a6a2a72af10f71620bd7815e8677-0001": 3223, + "01-f3-f2ffff8c935f8add62f2f16d5e76-0001": 3224, + "01-f4-05176d332965b7cf855b5c068deb-0001": 3225, + "01-f4-337ab5a477a42f90d4d7ed5f99b9-0002": 3226, + "01-f4-87d2c83eee72720aa4a1a246bf36-0001": 3227, + "01-f4-c19dd8d5e40eae894b94e5fa1271-0001": 3228, + "01-f5-0e2a1aa8cded4db1fc6be5b36f9c-0003": 3229, + "01-f5-28e41113f008cf6307b0e9a05f94-0001": 3230, + "01-f5-2f071ad30daa2034b4bcb4967d7c-0001": 3231, + "01-f5-30bed8721586bde42bf06fa15af9-0001": 3232, + "01-f5-53b3888c1a8e0ccca5345c678b7f-0001": 3233, + "01-f5-54432fe66a82c99973223e5eab54-0001": 3234, + "01-f5-91d1bfc01f914119402141869c56-0001": 3235, + "01-f5-9e2d73936e2b791ad57133f19df5-0002": 3236, + "01-f5-c4ff13b62a6405f7dfef4cd2e585-0002": 3237, + "01-f5-e6da501537d743ccf532af5b00f9-0001": 3238, + "01-f5-eadf9a4f9b992f6781bffc63b51f-0004": 3239, + "01-f6-15cdb2dc0d19ffee47bca5189741-0001": 3240, + "01-f6-2c730e536bbcc0545a1cd7dc02bc-0002": 3241, + "01-f6-49ad95f461a9e6396769dffa0a22-0001": 3242, + "01-f6-61c9673ced74737aeefbe8dbcce9-0001": 3243, + "01-f6-66457314576bcf66d7cd6b5262bd-0002": 3244, + "01-f6-6d8280c92db215e99010b7a96594-0002": 3245, + "01-f6-91902c88df32bcc08d1c1fb68870-0001": 3246, + "01-f6-e3f1687767c19fdfa9ff25650010-0001": 3247, + "01-f7-09f21522b6ac10fde2d8eeb356c6-0001": 3248, + "01-f7-75ae8cd94e37f8ba6a522814127f-0002": 3249, + "01-f7-a1c42a3e2e91d84eeaeb8ec75866-0001": 3250, + "01-f7-d016b2d759d59bd897d6f17e7b64-0001": 3251, + "01-f7-ea36732cff8811e2802d45ab3b48-0001": 3252, + "01-f8-2fd7a8c016913cc49045ed683de4-0002": 3253, + "01-f8-34a472d26e6820758ad092880da2-0001": 3254, + "01-f8-3ef58005acf56f46fdbe6f312c87-0001": 3255, + "01-f8-66dbfa498c387a07b110d080280f-0001": 3256, + "01-f8-6db878ac126a44c9d2763c13180e-0001": 3257, + "01-f8-7bdef8ae751e07ba8dc5e0b15abf-0001": 3258, + "01-f8-c0be9e4c2dfd4d4337c7c93e2ff2-0002": 3259, + "01-f8-c4936d9b0cf1c6dfe8eaf8712577-0001": 3260, + "01-f8-fe58b22a92a48b11184e0dbff4f0-0001": 3261, + "01-f9-405a0d35b5ce0e76052a0fad9013-0001": 3262, + "01-f9-4fe07fe49938359f91c161aefd8f-0003": 3263, + "01-f9-640582790c561b036d063a5f1e97-0001": 3264, + "01-f9-6525d894e701ff4d1f3525cbda68-0002": 3265, + "01-f9-6653dc86e868373e67a272e3c0f1-0001": 3266, + "01-f9-bb64ac17ec495104dac76d27ee65-0001": 3267, + "01-f9-c65c6699fb7583228a4553180822-0001": 3268, + "01-f9-eac568da22857ec3bd1369466bc4-0001": 3269, + "01-f9-fde72c4b2d513c484f8446a624bb-0001": 3270, + "01-fa-2139f44d11475e629859892a112d-0002": 3271, + "01-fa-27b3d57b89dc28c7ea4c2a985e88-0001": 3272, + "01-fa-89e85614832c445bf94e16482e4f-0002": 3273, + "01-fa-8f299bad6b2d4c81585b67d255f2-0004": 3274, + "01-fa-97ad38a5042ebb9215fc8581fb93-0001": 3275, + "01-fa-a588e78415f54c9eb52e77b6f329-0001": 3276, + "01-fa-bc04835e84ff80c2bc59b70281e1-0001": 3277, + "01-fa-ca84291f52b1448ba9fc937fb55e-0002": 3278, + "01-fa-ccc986388179d23559d78fd25cf0-0002": 3279, + "01-fb-0d813a36936e96841a1d61e0404d-0001": 3280, + "01-fb-200b3234d839a17721e8e7fc736e-0002": 3281, + "01-fb-28dff9321d8e599774cf1c8ed0f2-0001": 3282, + "01-fb-3932212def796b4111a6a75b4a89-0001": 3283, + "01-fb-5bafc886353bc1884590176a087b-0001": 3284, + "01-fb-90d25ce4367fc31ed899c4bf3a23-0006": 3285, + "01-fb-a5784288ec24e950f8a04152a826-0002": 3286, + "01-fb-a909ca0d402b2db5dcf5cd400414-0001": 3287, + "01-fb-c607d77eddfa508ee78195520063-0001": 3288, + "01-fb-ecbb008abbe09f2fbbd6a2876ae3-0003": 3289, + "01-fb-ef2ea265b040521ab0d750ec9aaf-0001": 3290, + "01-fc-0a38b10d57f4b4c62d551680bb2d-0001": 3291, + "01-fc-3b3ba6d03eda6a86e525fa783a81-0002": 3292, + "01-fc-3e2dd1e3a34e7e4ef6fa90a88fce-0001": 3293, + "01-fc-5566a5f261bf28f704f02bb62d3d-0002": 3294, + "01-fc-5dd766c5f7bf694d883e12bbb183-0001": 3295, + "01-fc-891bd6d270d2e7e4a919268c4170-0001": 3296, + "01-fc-905fac34492f56dd57024134fedb-0001": 3297, + "01-fc-b85b9f151c85c3af6e06dc650f30-0001": 3298, + "01-fc-c11e118d46677ef61d76c0da4546-0001": 3299, + "01-fc-e396ef759da7bc20af1835cae666-0001": 3300, + "01-fd-52866226c8476bbffae5848fa725-0001": 3301, + "01-fd-5e7c90a71affbd1825e1642e2ed4-0001": 3302, + "01-fd-b87ed90080b57f35ae6bd8959dcc-0001": 3303, + "01-fd-bc7604f33683ae8416e08881d24a-0002": 3304, + "01-fd-c9e0cce0ac12c9829702f2d54f4a-0002": 3305, + "01-fd-d9df53759b8e66139b827f5cc643-0003": 3306, + "01-fd-da8a8dfed4dcd5b7a8e29b04e44a-0003": 3307, + "01-fd-db1d15d7d379b0fb76fd614454c9-0001": 3308, + "01-fd-fed8e46efbca26316b6419942e3f-0001": 3309, + "01-fe-1bd2fb04a52f62b28e983c9f1397-0002": 3310, + "01-fe-4e39283a2488a70139f7730a93ef-0001": 3311, + "01-fe-c4a342d369f500694fabcaf969d8-0002": 3312, + "01-fe-d228713f2a710e8b20ad839f4810-0001": 3313, + "01-ff-14fc1552d4d412a148ca96f9f835-0001": 3314, + "01-ff-5e578a3b27a3186a306563472949-0003": 3315, + "01-ff-8bee7cef1f3461b3555473bb6098-0001": 3316, + "01-ff-b20a5cc219b03418a3e1078835b8-0001": 3317, + "01-ff-bf5e59196a2beb2d4f926c0dc498-0001": 3318, + "01-ff-c4066e8a3ef30b99a5cb92fae51a-0001": 3319, + "01-ff-e7a55c03ad33a15e62bc51d9a138-0002": 3320, + "01-ff-f90dcfe01819bd6f9f561957acee-0001": 3321, + "02-00-0bb130523cb5205e0fac2ab2d9e9-0001": 3322, + "02-00-ad290df2829518083b455b66fab6-0001": 3323, + "02-00-e04489344819040a6c02e6e4eca3-0002": 3324, + "02-00-fa3ee5966eab1b9ce08b852edb07-0001": 3325, + "02-01-2ecaf17839401042cd669f200169-0001": 3326, + "02-01-4e96a603c0de2df3d564c87e8bb0-0002": 3327, + "02-01-51cbc79a9204953a6f6789eea463-0002": 3328, + "02-01-6c084aac3c1eb5b7d0bc1ca65feb-0002": 3329, + "02-01-7527f7a596e0721a03f1b98a0ed0-0001": 3330, + "02-01-978a6949453f8b4747a71580cb6b-0001": 3331, + "02-01-9accc11a9aeea94124c50a3bbc5b-0001": 3332, + "02-01-c54af971575cac0b32c3fa5bbd9d-0001": 3333, + "02-01-d9c2db8fc43de058cdb2b25d2053-0001": 3334, + "02-01-da32b19ef8b6caed178e725692e7-0002": 3335, + "02-02-190c6a8d24e9cb57f5711fe0c104-0002": 3336, + "02-02-1f4e2285d3136a4b686ad2be445c-0002": 3337, + "02-02-24df604ed5ca3480bd0b62b4e81e-0004": 3338, + "02-02-559f0ac17108c1c38bf7b0660719-0002": 3339, + "02-02-5c0b4bc0107a55da6ad690a475a0-0001": 3340, + "02-02-6d6288868598c2262f628e2d1367-0001": 3341, + "02-02-7c60fe19bc4ba3c770c7888eedf6-0001": 3342, + "02-02-9631acef3be58edf77a1717277e1-0002": 3343, + "02-02-bfb43bdc289d1cb16169dfc6c4ed-0002": 3344, + "02-02-cdb1fa9fca5742584d25b7547f2c-0001": 3345, + "02-02-cf35dcaaf6ba542ca7a03ec0411a-0001": 3346, + "02-03-0a54b4a27f08c37b7e76a241070d-0002": 3347, + "02-03-3a7db74f0c102f53af256c7501ff-0002": 3348, + "02-03-40e5c54642040357947693793def-0001": 3349, + "02-03-5633bd546ddfdb3a0300a0d24ccd-0003": 3350, + "02-03-6efa6cb84effbeebcc1b76224e9a-0002": 3351, + "02-03-76135ae8e469e6304d28f84cc05f-0001": 3352, + "02-03-8df7388f14fcb58ac353a0ab4366-0001": 3353, + "02-03-970d5c2f1d26a2af47f05e4a838f-0002": 3354, + "02-03-a7b1f770c31d11d24153c067149f-0003": 3355, + "02-03-df60cbd54e53d59fb7e3d0df9d42-0001": 3356, + "02-04-20fb9e63f5fc4fd64e2a86308129-0003": 3357, + "02-04-277ac494db10f47c5f3195fe989c-0001": 3358, + "02-04-2fe78632faf35be9cbdff606920e-0002": 3359, + "02-04-3480306ff87502a7721923ab9a24-0001": 3360, + "02-04-569e02968466bbdb6029e9d02bec-0001": 3361, + "02-05-99810f962868cf7d9689f2c7528c-0001": 3362, + "02-05-b134c193a1b06e80e8ea559d8628-0001": 3363, + "02-05-bb8052ec77272d5b360e16ced8d9-0001": 3364, + "02-05-cc41a30b12c4accbc24f3643fbe2-0001": 3365, + "02-05-dfe763f8c6ddf921a94a696f8892-0001": 3366, + "02-05-e52ca65d87c34fd18d4a0511e76f-0002": 3367, + "02-05-ff1954a513c0484ac262405fa0e9-0001": 3368, + "02-06-18cd03a4a98e1832b48792d89820-0001": 3369, + "02-06-27948fa0a79f00fa48292ff9da1e-0001": 3370, + "02-06-2ef837602befff1715da4b252601-0002": 3371, + "02-06-3234c50fe86d4bc4ec5b90e9843d-0001": 3372, + "02-06-4eef1e4fc4f4b4f86a4f234d7663-0003": 3373, + "02-06-682d2acb321eaf9ffe30abbf36d2-0001": 3374, + "02-06-6f050e3c749d396134a3dc7f9284-0001": 3375, + "02-06-8910b1cb48315131c59c63a4c62e-0003": 3376, + "02-06-8db15be91805dc168ef628e08b18-0001": 3377, + "02-06-9208a89e30cc45d95f955f6b1e71-0001": 3378, + "02-06-aa8b98d2c79c5547db730a5967d7-0002": 3379, + "02-06-cd3d09606d5669bb1756f996b887-0001": 3380, + "02-07-0d0b50a2cdb19c593a12ff7c7e2a-0001": 3381, + "02-07-30fbfa582c6c320b54c1ddf97e9d-0001": 3382, + "02-07-7ef55953e778ab738a7bc0bb7447-0001": 3383, + "02-07-8a9ec51a8d7594de17c60f7b94c7-0001": 3384, + "02-07-955ae2bcf407d00bd17f88c2bb31-0001": 3385, + "02-07-a9b6c715140c40aa2d54e0f36f49-0003": 3386, + "02-07-dab00a8721e18131e5ae5a0da350-0002": 3387, + "02-07-e6c356021ff67bdd364aa2d222f7-0002": 3388, + "02-08-18265ca379938d6a07a939ebdab2-0001": 3389, + "02-08-1cf3913a98be1a162ac60ed115b3-0001": 3390, + "02-08-63dea025fde2136a9c241b678e53-0002": 3391, + "02-08-a11e63be0aaab2cc6137524e8459-0002": 3392, + "02-08-a9bd0280b5264972723afe2f5e0a-0001": 3393, + "02-08-ac00fb4890bd7770bceb3ed7624b-0001": 3394, + "02-08-b5f7da9740284dcf2eb162290997-0001": 3395, + "02-08-f52d9f72baa2505bd5a8bfbf862e-0003": 3396, + "02-08-f7c512ff161fb821139a050e2847-0002": 3397, + "02-09-0e9d8f1ac304c4d21e3c59c42386-0002": 3398, + "02-09-80dbdf448ff8fcd54ad094c308d8-0001": 3399, + "02-09-816d330d0b77fb4a886b6735594a-0001": 3400, + "02-09-8786495f661ea388718d93e3e1ff-0001": 3401, + "02-09-8b519853fc9496bf3e0eb2deaf9e-0002": 3402, + "02-09-b594c852fbc5913df6d44151a02e-0001": 3403, + "02-09-d2adf72552782b3ba26d623842e4-0001": 3404, + "02-09-db204f6e0222c352d40dc51d0cba-0002": 3405, + "02-0a-0f6c8f6c7d050e19304caffd17c3-0002": 3406, + "02-0a-1984f3ced1eaf49032af313dfd73-0001": 3407, + "02-0a-20feeb66312907dfc6ebb2b3bcd3-0002": 3408, + "02-0a-258a9fa3991e80e925938f963c73-0001": 3409, + "02-0a-34619471a1ab3b8d4d3ab532f53d-0001": 3410, + "02-0a-64e7c2a5eae63578b2094c12c146-0002": 3411, + "02-0a-77b78c1fa0e9f3e4f834427eef86-0002": 3412, + "02-0a-93d443accc42d94b15b844e47111-0001": 3413, + "02-0a-e51bd093dd5329785bc928566fa1-0001": 3414, + "02-0a-fb332c193bf7d5548627ec1690c6-0001": 3415, + "02-0b-003371c5e1cb3d7a4c22000656ba-0001": 3416, + "02-0b-1472dd6bf37a305bd42db0024f27-0002": 3417, + "02-0b-38066f15bae9ced7999a0840708d-0002": 3418, + "02-0b-5d3650ce6b8900356796aebf712c-0001": 3419, + "02-0b-9ac96d9d9ae4333d1ab69a48bc36-0001": 3420, + "02-0b-a343bcc5d045f39a31da005a7691-0001": 3421, + "02-0b-a5e15c2fcf5de9415409e2b95a89-0001": 3422, + "02-0b-ba589ab10533e21fb5c4648b8b19-0001": 3423, + "02-0c-01316c71bc53500c90ba49ef546b-0002": 3424, + "02-0c-66c07c8584bc7ff6ab634955eb3b-0002": 3425, + "02-0c-70bce22840679bcfe9bcdac2416c-0001": 3426, + "02-0c-b3505ba2a30fe20624e6ebe93d70-0002": 3427, + "02-0c-c69f38245de48c8b9e55eca9fc6a-0001": 3428, + "02-0c-dd2c02b370041251d3af95d04248-0001": 3429, + "02-0c-e3c708fd631697d3c6b60a7f7352-0001": 3430, + "02-0c-e6377b019e03bd39d97a4bde6581-0001": 3431, + "02-0c-eae251e0497d9e7568d38617ec99-0001": 3432, + "02-0c-f4accccdaa7e25e634aa0f1cc089-0001": 3433, + "02-0d-191c78da98abe142a9d062ed8829-0002": 3434, + "02-0d-1b1f042d652f8b4f7756b3009586-0001": 3435, + "02-0d-1f927306e29db990fa5c51fc3640-0001": 3436, + "02-0d-4c48b43865bcd50ebe6b0b9484ba-0001": 3437, + "02-0d-c8ecd39a20432c29a911fd45808d-0001": 3438, + "02-0d-e9fc7ad3acaca7b3f3df519327da-0002": 3439, + "02-0d-ef7899c5e40d306d9442645f7e9a-0001": 3440, + "02-0d-fcfb2678af807916bd0acf75f19c-0001": 3441, + "02-0e-151d8d332a7b9fb6d920bbe24202-0002": 3442, + "02-0e-3854ef859246a467c8682a808542-0002": 3443, + "02-0e-769e96b68b959f9c3d78d1c6dfa8-0001": 3444, + "02-0e-7d86dfd03c99608f2b1a0bc5ede9-0002": 3445, + "02-0e-a58b04ce2ef7b50ed5244c9dc0cb-0001": 3446, + "02-0e-b43b3222abcce4d210916eecb219-0003": 3447, + "02-0e-bcd01281b62c64ac66c3fb3e9df2-0002": 3448, + "02-0e-d1de5f9cc6c7f967a0b0ab570dbf-0002": 3449, + "02-0f-044dc7537cb6d22fba4c8f69961a-0001": 3450, + "02-0f-07e0f01509fd2b57153498dd00f6-0001": 3451, + "02-0f-0d931abd9934fffa364b721eba60-0001": 3452, + "02-0f-3500599b85aeea85f7e96458fb65-0001": 3453, + "02-0f-6ae7d78e19f1b9b82dd648b5f47b-0001": 3454, + "02-0f-6d8fd800a16694085a7cccd7d2fe-0001": 3455, + "02-0f-6f92f143731ab4d279251d74b972-0001": 3456, + "02-0f-750962474ca6fbcb0560138770c9-0001": 3457, + "02-0f-7b6a318138f529955fb66dc69caf-0001": 3458, + "02-0f-b570115886de4e137cb99cd2b57b-0001": 3459, + "02-0f-d769dc716610442d644145a9c15b-0001": 3460, + "02-0f-fe6e57f034693fc14927ebad42eb-0001": 3461, + "02-0f-fe7475c31113f0e0f8073acc7888-0001": 3462, + "02-0f-ffc32a9f6b386473c207798735c2-0001": 3463, + "02-10-1b2f98b7b99f2da0daac0b0430f8-0001": 3464, + "02-10-1c6c4db0308b392b177425ff886b-0001": 3465, + "02-10-321b7d0715a5397751afc3528666-0003": 3466, + "02-10-3d38e01ad87fb0c9f3006943cb92-0001": 3467, + "02-10-3d57a50147ff74ff761f4327f8b3-0001": 3468, + "02-10-4705c75f424581df8c9bfd898b97-0001": 3469, + "02-10-6be23e5cc97fe493e33377d91987-0001": 3470, + "02-10-6bffbd7a796090c5bb46e476f336-0001": 3471, + "02-10-6ecf0399a97b66a08c38713c61f1-0001": 3472, + "02-10-a9bc3adc4a6a2c53d35e47b19606-0001": 3473, + "02-10-e430516b2bc312c87d8967c6abbe-0001": 3474, + "02-10-ed451cef0b35f826a4956e423b87-0001": 3475, + "02-11-1bd9ba181a7782b4cdd28a9b2cb6-0002": 3476, + "02-11-2098b6807d3930cc59d96b3ea444-0003": 3477, + "02-11-2e65e4261836810e95f1c122323d-0001": 3478, + "02-11-362b8d891e7b298cee529426723b-0002": 3479, + "02-11-3d7945442e9cd67885db3e211747-0003": 3480, + "02-11-3ebcf5d38a4b2e6b08829215fcf3-0001": 3481, + "02-11-61d49343cec990f0044a6003ed84-0002": 3482, + "02-11-6bc4d17b76d83a21eabbafb24869-0002": 3483, + "02-11-7c557632528248943c9a7ef1d3fe-0001": 3484, + "02-11-a3fd99164de3b007f3b8fae67561-0001": 3485, + "02-12-19511f9d4cfb290204a58550aace-0003": 3486, + "02-12-40514b737d0ad4dfd329001442d8-0001": 3487, + "02-12-505de100d970b1d61f88f4e7d925-0001": 3488, + "02-12-a3951d084b2465ec3253e2a565e0-0002": 3489, + "02-12-a4d64d9515d1b5297274cf026d51-0002": 3490, + "02-12-b0ecb9605320b55731f546288ebe-0002": 3491, + "02-12-ba1d5e11559140b58cb66ab9e4bd-0004": 3492, + "02-12-c8b66462253847fff956b2a2c0f3-0001": 3493, + "02-13-1a00b9f3ef532cdc1131fc9d6472-0001": 3494, + "02-13-1de7f62ece09664890c2a74be2dc-0001": 3495, + "02-13-296bd285314b2c0fb22c759f94f1-0002": 3496, + "02-13-43b169250339b46d4f3729fe4dd3-0001": 3497, + "02-13-d223424af794d47565ba895000fb-0003": 3498, + "02-14-5f98c021fb74a9fbd7fcfcd7ffed-0001": 3499, + "02-14-853b766af9f3349dc257ad28d5a1-0001": 3500, + "02-14-9b79531dc263affebd5675c2febc-0001": 3501, + "02-14-a44acda175419bfa6c74984ddd61-0003": 3502, + "02-14-dbe17c58ffe7f249f345673daf71-0002": 3503, + "02-15-028f019b9b2e203444f7956dd3a4-0002": 3504, + "02-15-2389015b765ff47ca7eb02e78797-0003": 3505, + "02-15-354da34319b3c1206d95e734c3f7-0001": 3506, + "02-15-43be67ad7dd6900b44d669fd1626-0001": 3507, + "02-15-43ead1261dceb14a443057cc9643-0001": 3508, + "02-15-5ae5bde3d1e993e9b93a0110638b-0002": 3509, + "02-15-9323edc8c8723c7c5b0bcc895eea-0001": 3510, + "02-15-ac307353ec5142eb92d5240ef108-0002": 3511, + "02-15-d4a357bbb5dd16642bd2b1ae4029-0001": 3512, + "02-16-2a3d9e7eacf94f4005b4ef4d566e-0001": 3513, + "02-16-2af5f19c5ca0161c97c1ee8df3b4-0002": 3514, + "02-16-4ba033c42f0ef819320981e51cf6-0001": 3515, + "02-16-5b945437accf71be7277a1e22ba8-0001": 3516, + "02-16-6e42b7693620f1c85a7c63c1280c-0001": 3517, + "02-16-b2ca678434f58225ebaeb9d26973-0001": 3518, + "02-16-b5b40b81631e3a1caa5b0fff05f1-0001": 3519, + "02-16-b5c5f2e1ab0f2d94e8869d969e76-0002": 3520, + "02-16-ca6d0e88d79c6dc1f1e4d2d49a90-0001": 3521, + "02-16-edb96354ce0d02cea1db72acca9e-0001": 3522, + "02-16-eff2d09c7f65ed564e78e1d1b875-0003": 3523, + "02-17-0318d07fa22abd3aa7abff4d69a0-0002": 3524, + "02-17-41c4982c7e396bed03bb82952429-0001": 3525, + "02-17-51cc3314a0ed55e2ba9b06ef57a2-0001": 3526, + "02-17-6d77b22c9e51e3836e7e557b07cf-0001": 3527, + "02-17-c674f6a0aeb79c79c1635280b849-0001": 3528, + "02-18-1236930022e34b465449f4fec0aa-0001": 3529, + "02-18-1993881eb33b41c6f2ac19e4a80c-0001": 3530, + "02-18-5c2f4db4d8714d73f14fc10b7991-0002": 3531, + "02-18-60ca19a44a24dfec5d174fc927b9-0001": 3532, + "02-18-b22f3eb014c43100a17f15349f1c-0001": 3533, + "02-18-f3253aa80aeda8da4a66f5960088-0002": 3534, + "02-18-fff30934b9630cbb9f70f76b7893-0002": 3535, + "02-19-02c2c7baa5c1c183d2436ec252c4-0002": 3536, + "02-19-298cb51ab3b48e12c9cebd88a8d7-0001": 3537, + "02-19-2bd60e08ea5e47178346a19e0296-0001": 3538, + "02-19-64b71a2639c87bf8353a21474391-0002": 3539, + "02-19-94ef6bba86e2040c339d2cfcd071-0001": 3540, + "02-1a-16241c3ed5bbdd6daa8c58422d9f-0001": 3541, + "02-1a-1d61da56c695764dbae79a7e7289-0001": 3542, + "02-1a-49aab51e07336cfb9f4ebadc7ee9-0001": 3543, + "02-1a-723d62224d18bc8f52d2bd8e8946-0001": 3544, + "02-1a-784fe82efc23aa16ded1c8d2b3ff-0001": 3545, + "02-1a-85d6b63d98d86afa528bdc7c121e-0002": 3546, + "02-1a-8e082b306a73829c83426438b64b-0001": 3547, + "02-1a-9d45eeb1c108a9488d6d28a0a351-0003": 3548, + "02-1a-b42379718dd08bf0d3ad398cf01d-0001": 3549, + "02-1a-c97bc453d505ed2879ff49cadbaa-0002": 3550, + "02-1b-17dd6b75b8826ef9a7e035ef70ab-0001": 3551, + "02-1b-30c300ef7d61abaa4dde0583b02d-0002": 3552, + "02-1b-38bdd04d8417bc7400d567b34412-0002": 3553, + "02-1b-3a6e91a04912a5c4336781bfca1a-0001": 3554, + "02-1b-426a17db2ffa8b0e1183cd65cac5-0002": 3555, + "02-1b-539ebc0b4e103d78a2a2d0897514-0001": 3556, + "02-1b-8929e658becb15b21043b6559583-0002": 3557, + "02-1b-9a667e70620c3c174e41d91a0a60-0001": 3558, + "02-1b-dbce41302dfac4d9949275108cb2-0001": 3559, + "02-1b-f452149d3f23542058ff1eb5457a-0001": 3560, + "02-1c-a0b395f96676baf86699022b4dbc-0002": 3561, + "02-1c-a46cdd30d3772b510e2d06eadf27-0002": 3562, + "02-1c-b91d300a37e8b954d4f8a5826d92-0001": 3563, + "02-1c-e6d37cc4ea1aef58189eadb34e0a-0003": 3564, + "02-1c-e72a1e7c1ffada4d17cd5638d1f0-0001": 3565, + "02-1c-f41fb409284c06004706ed1c29ac-0001": 3566, + "02-1d-3fc5ee0ade56487ff77f132df680-0003": 3567, + "02-1d-4a260ce56449d0740da9df054a34-0001": 3568, + "02-1d-54b8f71becdbc07964a13d315aaa-0002": 3569, + "02-1d-6c8e7ca3daf51d1223a486ca2a0e-0001": 3570, + "02-1d-6e0fdc0bb916277479f78fb0dd8f-0001": 3571, + "02-1d-8552ed0441f113bb3088c272c7b6-0001": 3572, + "02-1d-cabe9f58bda979789bc8f200cea4-0001": 3573, + "02-1d-cba43dc08ef747cf9533a29cb91c-0001": 3574, + "02-1e-01f29a6a3aa841221e783a0a94a1-0002": 3575, + "02-1e-114b01e18db558033124f9e1d760-0001": 3576, + "02-1e-143b979c1f7c894b062e718da933-0002": 3577, + "02-1e-218963539ffa8c42a1849b70affb-0002": 3578, + "02-1e-370701af9b12ce7aa8c55175f141-0001": 3579, + "02-1e-40452f16b1afefa5ff9243130b85-0002": 3580, + "02-1e-423ebf106ff29e20e8dba4c6320c-0001": 3581, + "02-1e-4560db9b98e93713bee3684a9afb-0002": 3582, + "02-1e-5ad51e19a233570b7f95b060ab45-0001": 3583, + "02-1e-66c6becd525e8acec4fad11e597c-0001": 3584, + "02-1e-78ea6a110a1b9c9ba5cb45ffc415-0002": 3585, + "02-1e-9a4eae7db377fbab4d43672a1f33-0002": 3586, + "02-1e-cf2d7b535bbcd02087dc9e92a843-0001": 3587, + "02-1f-962d47cb0ab87cd0ba03b8231f2b-0002": 3588, + "02-1f-9735963f55fa0e009f350f1e7eab-0001": 3589, + "02-1f-ffef0f788b7e9922c4bd8083b8d4-0001": 3590, + "02-20-92ed94a53f9c0afe18ae489e7320-0001": 3591, + "02-20-c5feab2220247083d400b2195603-0002": 3592, + "02-20-c8b3f1614673b2e3d7a157107865-0002": 3593, + "02-20-e781bc2ce541d4a4699b7d6329f1-0001": 3594, + "02-20-e87d3fbe87d626470b07c95cd725-0001": 3595, + "02-20-e9412b0f552cdfb1c841f86d7699-0001": 3596, + "02-21-1d9efaaa98b608b409e8d19ae235-0001": 3597, + "02-21-369b903597db27288b9cfcda3c7a-0002": 3598, + "02-21-4eac37b8373ea10aad6d0b8eb502-0001": 3599, + "02-21-919f2c5198acac9fea2010508cad-0003": 3600, + "02-21-9774eb6bdda0916dc87d5c7c3f93-0001": 3601, + "02-21-b16f8c5c03ad83cc27cbe8e3d60f-0003": 3602, + "02-21-bd851b7d52b30f3ef59728a6a2c2-0002": 3603, + "02-21-c7d7f18df505337ed695a9c12702-0002": 3604, + "02-21-cdf50a18df5741888bc6661d4997-0001": 3605, + "02-21-eacb6c4772ce22516dc758605bd8-0001": 3606, + "02-21-ed5c03844f7377eef4541bd96d19-0001": 3607, + "02-22-1526f22c4426e2db8f015b100696-0001": 3608, + "02-22-7194fd51a66e7a3a8e6462348161-0001": 3609, + "02-22-7257117fed2b558af94e6017d9f1-0001": 3610, + "02-22-9323dd368343f26249b9b14add8b-0001": 3611, + "02-22-a51ff7c72435485e7a6ee048df76-0001": 3612, + "02-22-fe0c6cd6eacfa58b56793751bfc4-0001": 3613, + "02-23-02eb14330f7c16a4045c55565bf0-0001": 3614, + "02-23-1ca899e507ff00d875cd03d3d3a2-0001": 3615, + "02-23-287d2cdccf07119acb1610385b4d-0002": 3616, + "02-23-64b680a2f48a616864de32fffbce-0001": 3617, + "02-23-bc628690f4b24f49de4e73ad8edc-0001": 3618, + "02-23-d442a7262554cf1667d0f2d036e2-0003": 3619, + "02-23-f1a9454aae27cf43c2f0553afaba-0002": 3620, + "02-24-2a2feff6218a971699513320a6e7-0002": 3621, + "02-24-302c50c90253a2e206a4362ed5d4-0001": 3622, + "02-24-428da5b9f678800e3e48f9cb1542-0001": 3623, + "02-24-54b85f12f79c089afd0b7123683e-0001": 3624, + "02-24-633fa3cb249479d57c46cced2177-0001": 3625, + "02-24-898952ccdd8e11f461dcde3f3402-0006": 3626, + "02-24-c7acbc863540de9a25c69e77cee7-0002": 3627, + "02-24-d99255e054feb15d8d4ed43c834b-0002": 3628, + "02-24-e19b57020c1e9a75677e8243cc47-0002": 3629, + "02-25-0f3fa35278cf8bbb3144f56ec333-0001": 3630, + "02-25-1d8c5c1b80813a81ae872eacfc15-0001": 3631, + "02-25-1f21e7d83b6f769c6fba8dd0b45e-0001": 3632, + "02-25-339192c665144440f2dc4a7e8b10-0001": 3633, + "02-25-693bd78da0f9579ef326fd4c92da-0001": 3634, + "02-25-75295ff163fea34ee812d20429b5-0002": 3635, + "02-25-ad960931cb42a1a0bc3224832057-0001": 3636, + "02-25-af99cf9ec08cd8ef7ab67bfd3fd0-0001": 3637, + "02-25-f88933fa142b9542e7ada411a092-0001": 3638, + "02-26-088080d912be6f5c01a2ae1b9d4f-0001": 3639, + "02-26-9949e5f8ce1bf6295ecfd772b749-0002": 3640, + "02-26-fa21d4dc8eee425b554ee43f4d2e-0003": 3641, + "02-27-1fcf2e82fb981eb8d3427b68995a-0001": 3642, + "02-27-35d973a0e335b458a0b3c9017d03-0001": 3643, + "02-27-4e3eeb0001504a3694e34ebbfeac-0002": 3644, + "02-27-651ad01478bac5cbdaa638ca5f8f-0004": 3645, + "02-27-73dda72968762a8221eac4947f80-0001": 3646, + "02-27-836c2391dc2bd1268b34f3897c8a-0003": 3647, + "02-27-a1f6ad916a940bf05758aec75057-0001": 3648, + "02-27-ea9a29f7dab9251ee7fc5d4479a4-0001": 3649, + "02-27-f6a87f201719a11ca8eabf25a0fd-0002": 3650, + "02-28-327812e785dd42cf40be3d369cf6-0001": 3651, + "02-28-6bace124052d1a9d36de95154b3f-0002": 3652, + "02-28-b34cd03d641585bbeb7dca4840e4-0002": 3653, + "02-28-c299d4c0831f3c35859829595a15-0001": 3654, + "02-29-10b0414b878409a6e4a18269154e-0002": 3655, + "02-29-2602f8c8ebb9e9a7c716de0704a7-0002": 3656, + "02-29-3b5001ebc61b1351a127520baf9b-0003": 3657, + "02-29-3d5b384f8161cd301ef6d2d6dee5-0001": 3658, + "02-29-86b9cf959450b02276c72347b76a-0003": 3659, + "02-29-913539344ba5b45f5b5fa3784fc6-0001": 3660, + "02-29-fa18485226e0c7f8b9906d176e74-0002": 3661, + "02-2a-38d517b5bdf87330a41380c0884a-0001": 3662, + "02-2a-5046204a741b1c4e9c6a6d979dd9-0002": 3663, + "02-2a-511a0b87b7dcd0ac4ae6160e2638-0001": 3664, + "02-2a-798d8c5f67a7fd64aff5b7b97434-0001": 3665, + "02-2a-8e85c44caa0a7ebd20048b2dcc7f-0001": 3666, + "02-2a-8faa891c8999f11c2c223bd5fd3c-0002": 3667, + "02-2a-922e88c7edff65fc324f510634cc-0001": 3668, + "02-2a-a38cda7e6587ec51e30febcf0191-0001": 3669, + "02-2a-b6d0aa092b31cae9908635a1440d-0001": 3670, + "02-2a-ecdbb935ba308b1b77e14d3ddf42-0002": 3671, + "02-2a-f688f261b3eb8d52270e7ffaf6dc-0001": 3672, + "02-2b-31e6dabd9cc488d656e7aa6d181b-0003": 3673, + "02-2b-4f7e94c89b184bc51e9d5445774b-0001": 3674, + "02-2b-5caeb1027215393d14f961ac5304-0001": 3675, + "02-2b-820b6cfb810d95f57eea3775b424-0003": 3676, + "02-2b-c82bf8a881f9ddb74a4975875cb9-0001": 3677, + "02-2b-f78edf168fb659535da6d9ee3306-0001": 3678, + "02-2c-060debb1cd3d3d478698996b3788-0001": 3679, + "02-2c-1625a9262cadf4b0a6b7927fbc8e-0001": 3680, + "02-2c-2126f78476a93135f46e3bb4bf6d-0002": 3681, + "02-2c-852693525632ffdd131e34f95988-0001": 3682, + "02-2c-ab212bd1c631981e621bedb26eec-0001": 3683, + "02-2c-c738f597cc83eee290fdfebd36ef-0002": 3684, + "02-2c-c9dd5cde3513580b018902e6149c-0001": 3685, + "02-2c-ce7a7c5856547f0bef461aa1ca55-0001": 3686, + "02-2c-cea8b1e48eb3ab988ddd00d64e53-0001": 3687, + "02-2c-dddd60c352df66a0479f1cd5ae78-0004": 3688, + "02-2c-f57eeba0a78ab8201e4ac6b99f55-0002": 3689, + "02-2d-0a3a56d9fcc77a29af245382d785-0001": 3690, + "02-2d-295d45af68219eafd50ae94da9ba-0001": 3691, + "02-2d-427c17aa8289a6b02aba44e3701e-0001": 3692, + "02-2d-4ebe381cbcd7d1a70f3695a2987d-0001": 3693, + "02-2d-6f12f564cb96d3f6d20b845b6828-0002": 3694, + "02-2d-c80c35f09476d03fc395b18e852e-0001": 3695, + "02-2d-c8816101db7069cd74c245f74d5b-0002": 3696, + "02-2d-f8a3c1d7c2413abdb6dbe335152e-0002": 3697, + "02-2e-08106fa8f56ceeb2928846b232f2-0001": 3698, + "02-2e-2921d26b6e336a51bd21b82c0ec7-0001": 3699, + "02-2e-b6508e09e709b858ec5c33992cf3-0001": 3700, + "02-2e-b67e46edb8057f292e6ed0dab07d-0002": 3701, + "02-2e-b724d2bb33f75141a69cc28c8b00-0001": 3702, + "02-2e-db48ba2f1b458478318bbbd6c9de-0001": 3703, + "02-2e-e91b7db64295fa6e9511c85bbe85-0001": 3704, + "02-2f-069ba19955390315ca4f96f4f7fb-0002": 3705, + "02-2f-2cef841d5ffe62267a5aa22564b1-0001": 3706, + "02-2f-3c8e232098f79995cf05bababb75-0002": 3707, + "02-2f-580d0004ef3b73687b312166acc7-0002": 3708, + "02-2f-5d380a78a9b1ac3aeee7d336917b-0001": 3709, + "02-2f-6fe87262dc077546902f9972ee43-0001": 3710, + "02-2f-81c18a34180fe372d62be8be98de-0003": 3711, + "02-2f-86801e83544e209e6d093fc186c6-0001": 3712, + "02-2f-912aa171955c89c0d8be8f037d45-0001": 3713, + "02-2f-a9afb1f463f1e0a638868a580ff6-0001": 3714, + "02-2f-dd3a01c7d32a4194151fb3e24caa-0001": 3715, + "02-30-0f0d0ec674f5106d2db818924e68-0002": 3716, + "02-30-1110d7f09b9a22c5cfc9b1e40141-0002": 3717, + "02-30-1f656f2405befb30786d5260d8a8-0001": 3718, + "02-30-269c8efc651d3635a3cac77b3b22-0002": 3719, + "02-30-3026c9304323103aae2e759e86f0-0002": 3720, + "02-30-3d9f5a9db58fcc006e7fc5c04a9c-0001": 3721, + "02-30-7f31b43724dfb18ee9743d99f310-0001": 3722, + "02-30-8d64df3126e78cfb5ac808f8709f-0001": 3723, + "02-30-de9810d3d76f6caba6d0174ff80d-0001": 3724, + "02-31-0354461017d5f4c6c7a43e4296f4-0001": 3725, + "02-31-74372acf817675d516dd535eb2cc-0001": 3726, + "02-31-8792a4c546a37c79d24a70358596-0001": 3727, + "02-31-9fa647b4b911beb769b8910953fd-0001": 3728, + "02-31-b761f5e7161f77d4346c82b09ffe-0001": 3729, + "02-31-c59db78ecbfea52a55a0c8988f1e-0001": 3730, + "02-32-2a1f98d4ace7b87e015ec7a148da-0002": 3731, + "02-32-7251faa772d716a6c98b2c58f63f-0001": 3732, + "02-32-9d54938d44deedaadafb52f614ae-0001": 3733, + "02-33-55788b822393445e4b01bca37c07-0001": 3734, + "02-33-6208ad3edcdea8c89f840b2cfa36-0001": 3735, + "02-33-7c83119982b2d2439f46513b8cf3-0002": 3736, + "02-33-92d5886e578c4341e2754b450ea4-0001": 3737, + "02-33-9a1eeb6c95770e5eea4f56b5188a-0002": 3738, + "02-33-acdda4c64e39a675c1844efd9c38-0002": 3739, + "02-33-adc563780e89be6c304e793b02be-0001": 3740, + "02-33-db9c3e5c16934ee029e60a42d1d9-0001": 3741, + "02-33-eff20879c1526b8c135069562b6b-0002": 3742, + "02-33-fcf77d1b7dfa020ea0132ab12c20-0001": 3743, + "02-34-18e1bf6a6fd0ed65f29e92011e61-0001": 3744, + "02-34-27307e1f4a2dd40a6e252ec96b08-0003": 3745, + "02-34-36ddc06e99066d193ab052cbe9ea-0001": 3746, + "02-34-3ff7219d079a52181ad6212a2d92-0003": 3747, + "02-34-417d3a05cab61bf02773eef0a98b-0001": 3748, + "02-34-d82ae64e129e99119e3083ab6140-0001": 3749, + "02-35-196c2d9bf8600573486b463d7d86-0002": 3750, + "02-35-7689d19bfe0d936d829eaa464be4-0001": 3751, + "02-35-87ba8d1be368973468e35d6de572-0002": 3752, + "02-35-8ebe14361279a09d083c10562873-0001": 3753, + "02-35-a131e49935e86bf8a77f517d211e-0002": 3754, + "02-35-a3cdae5a7c4bf36208b861d0668b-0002": 3755, + "02-35-dd7e5f8da73c74d66cd0b0aca9dc-0002": 3756, + "02-36-0ed6672e6613f3b7e7edbcc1b236-0001": 3757, + "02-36-37083bfb8bdee3c2b79141448e9c-0002": 3758, + "02-36-50a6b57732ca04f12dd57a15ac32-0001": 3759, + "02-36-7d47d168ce1bdda7928949b21b61-0002": 3760, + "02-36-840ae69e51153f373518e6b0ff97-0002": 3761, + "02-36-87d4a58b4394999dce49d0f84925-0002": 3762, + "02-36-90bb5a63edf177512506d7398548-0001": 3763, + "02-36-f7f81d4fa8f24de2a0cbc53f86b3-0001": 3764, + "02-37-7682c2d6cec9b21f13c3b4729de1-0001": 3765, + "02-37-8779690f5df8672320d6f43a9a48-0001": 3766, + "02-37-88ba0da2d609d8310ab3bb8126e4-0002": 3767, + "02-38-168eb8f2ac751b5817555bea315c-0002": 3768, + "02-38-651512e90f11ad1dcd8bb0659788-0001": 3769, + "02-38-aea03161b487eaec1a5e883c86a8-0002": 3770, + "02-38-bab46787fc8a333c9b92456989cb-0001": 3771, + "02-38-d3b8ba1346c3dcec1ad9b4c11844-0002": 3772, + "02-38-df5d40b39e598f8da0f36172868e-0001": 3773, + "02-38-e138f162233d20cfa963f831202f-0001": 3774, + "02-39-0406abbd4b8ea1d69b1304668577-0001": 3775, + "02-39-32c3c92e3e46f3ecc2dac294d89d-0001": 3776, + "02-39-399c001eea7b3080e5707dd42381-0001": 3777, + "02-39-3d1425f249bef847e9903ae83631-0001": 3778, + "02-39-45fde818058ddf37579eecf37340-0002": 3779, + "02-39-65f13adf696cc03da59cfef22521-0001": 3780, + "02-39-aad144753c85f57e473841075b19-0001": 3781, + "02-39-ad7a6324992e0b3355e9e81648da-0002": 3782, + "02-39-cb94d205627342add23acdbd90cc-0001": 3783, + "02-39-ce095096f4dcc914f17ccca7bbaa-0001": 3784, + "02-39-d42c6d93e78eb72a2d25f8c760bc-0001": 3785, + "02-3a-033678c50bbf687dd63613c01e94-0001": 3786, + "02-3a-069f5a1654dc83d582cd36697839-0001": 3787, + "02-3a-0c0a824ca8536ce83edf20bc7034-0004": 3788, + "02-3a-24a6f60e727871ef3654b109470e-0001": 3789, + "02-3a-6392d0e302ff7b90d93a47ef77de-0001": 3790, + "02-3a-9ad1e018a7dfb1e690207ea47db7-0001": 3791, + "02-3a-9c580c5eae947f7d1245277bc3ee-0001": 3792, + "02-3a-a4009fc5afa17af8fa9a17730028-0002": 3793, + "02-3a-f359e3d38563a09821bfde0b2879-0001": 3794, + "02-3b-0ffe7f31feb9307a361f0b7ee10f-0001": 3795, + "02-3b-3feaa218745bb9bb44af6a47e0d7-0001": 3796, + "02-3b-455d3bbfd7adfb3ade500b1d9ee6-0004": 3797, + "02-3b-b3026b8c873f64adcec3cfa7f199-0001": 3798, + "02-3c-15e84d0f7ca16a788f4c89246075-0002": 3799, + "02-3c-6d58455c5afb9d2819e87658f9db-0003": 3800, + "02-3c-8739dd642a249f803875ffe6c659-0001": 3801, + "02-3c-b0ff8a0ec2ae6d6251c37822758e-0002": 3802, + "02-3c-b5913e7f3e6dc3f346cc36bb01a7-0001": 3803, + "02-3c-c8cea7fd1fec1a6a0e70f5cfca9e-0003": 3804, + "02-3c-d19b18e933aab9914a27aef869af-0001": 3805, + "02-3c-f050355b9ffe97aaafbd5932946f-0001": 3806, + "02-3c-fa4c37c8d48663241477b56b8260-0001": 3807, + "02-3d-11e2c7a431e8d4d9eae75b5fe772-0001": 3808, + "02-3d-3953d1a8a494cccd4056b0d6a59d-0002": 3809, + "02-3d-4f2a4106acfb142cbab5d86f9146-0002": 3810, + "02-3d-5774a31cf7f902ceae8a7906a2d6-0001": 3811, + "02-3d-8e80ceb40fc6ccbf6ee4cccd95d3-0001": 3812, + "02-3d-ce908c33086e85a2133b2f0832fe-0001": 3813, + "02-3d-d44b98416df272bf867ff06359d6-0006": 3814, + "02-3d-d6865fee98d7ca6dbeb0e2e7ad65-0001": 3815, + "02-3d-ef6b29cf1c00695c6ff46f859b19-0001": 3816, + "02-3e-37030d7372c3a460a2bb8e845c7c-0001": 3817, + "02-3e-6b458db295839186a657d052bb54-0001": 3818, + "02-3e-87ce85b836112f77b5a2dc40bee6-0001": 3819, + "02-3e-cee11ef4da4f382cbd54eb8891e1-0001": 3820, + "02-3e-e52f503340fee22d84fee67da47c-0002": 3821, + "02-3e-fcab5b3da6fe982cfb832b6c558d-0002": 3822, + "02-3f-0f04acd5ec89ff0fe6d067618e91-0001": 3823, + "02-3f-1ae4a0aaa8141c83ad4b3c2afaf8-0002": 3824, + "02-3f-29b2d1a0a529b015ff9e9d4290c9-0001": 3825, + "02-3f-537411ac320d4b3f2575f066477e-0001": 3826, + "02-3f-800c21b07c2df67aa9fc86441cdf-0001": 3827, + "02-3f-9649fa533b0052ae56903ecaf402-0001": 3828, + "02-3f-9ebb61ab1f770ba270d2f9cb57f8-0001": 3829, + "02-3f-a3d0636b39ca80921d43ffde37c2-0001": 3830, + "02-3f-b2b4d72e885c2834cfc594c98ba1-0001": 3831, + "02-3f-bbcc5817dd167bb6bbcc89a6a23e-0001": 3832, + "02-3f-dcf5f725dd25e49d4f39bb342d94-0001": 3833, + "02-40-2a8f4d584cf11248b9d300d76ed5-0001": 3834, + "02-40-3efd97e180bee41b9e31001d05fb-0002": 3835, + "02-40-47eee64399260e9b7ca74ba69ef0-0003": 3836, + "02-40-58ce956922ab4e516581a1f8517a-0001": 3837, + "02-40-99b992d550908b870246cfeeac5c-0002": 3838, + "02-40-d336a680c7998cfbb03d6714c3cd-0003": 3839, + "02-41-11225a636ab97a406b225e69d85a-0002": 3840, + "02-41-22e4a2ae517efe21f52dbbdafb2e-0002": 3841, + "02-41-2c9a8c61849cebc3ab0e2eb2c77b-0001": 3842, + "02-41-6877b892b084c37f1eb98e3d4dff-0001": 3843, + "02-41-6ead10a528d590b37486e065995d-0001": 3844, + "02-41-7e5149dd36048b1d9bff41c835af-0002": 3845, + "02-41-8a88b7db6e56affd5aedbaecc1aa-0001": 3846, + "02-41-8d550bd611fb963cab38e61801b3-0001": 3847, + "02-41-a6afe15d00ac8a42764af72def4d-0001": 3848, + "02-41-ae22571a568e3af1290889d89847-0002": 3849, + "02-41-bc5588222e9cc7f99c7697cd19ea-0001": 3850, + "02-42-0dafbe2dbbce45f2f4316c67d46c-0002": 3851, + "02-42-2c8f6a75592dd5292a1f56317209-0001": 3852, + "02-42-30e5f41670e4d9111c88b2922589-0001": 3853, + "02-42-374939b947c07b12cbaf9becb111-0003": 3854, + "02-42-42d95080de314831796f6541bc46-0001": 3855, + "02-42-4b7790efc281061e6e4d360b1fd9-0003": 3856, + "02-42-757d0a92b9b0d05698bee524c543-0001": 3857, + "02-42-daf737c31e1377982127eab681aa-0003": 3858, + "02-42-dd0c0dbce23161f5929c765bfed6-0001": 3859, + "02-43-0f5eed01f145d54c3f253db73052-0003": 3860, + "02-43-0fdbcf6092402a7c651fac9f9a3b-0003": 3861, + "02-43-2937d6afbd7ba0ffd761869dc9dd-0003": 3862, + "02-43-4086af103beef73c60bce25d1383-0001": 3863, + "02-43-579982d15d3f1239a17c30926ab9-0003": 3864, + "02-43-6b20258696f8953bd6ec56177154-0004": 3865, + "02-43-808365303238fbea292ae4222590-0001": 3866, + "02-43-af3095e020f55af24e5ef427986a-0003": 3867, + "02-43-e66c8ea467deca3a3c54e42e46da-0002": 3868, + "02-43-ed38e35274f4e8746b07aba9b473-0001": 3869, + "02-43-f29a08c26512d493c42176c0c1ed-0002": 3870, + "02-44-09f77dcc48ba20c4c7816462db2d-0002": 3871, + "02-44-1a80386af4d9017d890c678a8673-0002": 3872, + "02-44-4a67a62f8c231b3cfd28917e9343-0002": 3873, + "02-44-5bbf23aedfbb3aa051372d601ced-0002": 3874, + "02-44-5ec7cb4ee728612bd6da15cf2d0d-0001": 3875, + "02-44-648c3aeaed53e02b59f7f0c89d28-0002": 3876, + "02-44-755d0336b584e9a24ee7be4966b4-0001": 3877, + "02-44-7f0d0627dcc9108cbbbbf507b4ed-0001": 3878, + "02-44-8205714a82856cf41658813a1ff4-0001": 3879, + "02-44-884e313d8b5a67727a1ecd3580b8-0002": 3880, + "02-44-a40687350c7bcd0877a3d5ec59bb-0002": 3881, + "02-44-beddcfe7eb9879562490e22b366e-0002": 3882, + "02-44-cf01e36dea08396e7e243539eee5-0001": 3883, + "02-44-dc311ab2c02119a0eba975928fa3-0003": 3884, + "02-44-fb98f12ed176122ea16e3028aca2-0001": 3885, + "02-45-0a05a72a15db83211e983ac19cbd-0001": 3886, + "02-45-25492edfb4103295218f64eb93ca-0001": 3887, + "02-45-6122b04ef538ccedca99caab0d82-0001": 3888, + "02-45-b462a46c51834c820a42533021fe-0001": 3889, + "02-45-c5cc9c877d4083a4c170ba886d4b-0002": 3890, + "02-45-f66475519585738f373e221276d0-0001": 3891, + "02-46-087d1af0a965a339a4c2c04d3835-0001": 3892, + "02-46-0b059d46e4a50fbedfad55981190-0001": 3893, + "02-46-518c84c164dbca506e91a0bd1953-0002": 3894, + "02-46-78d4fb7f68a7362e64cf44a87c73-0002": 3895, + "02-46-e35e550d036885478b81641850be-0001": 3896, + "02-46-eda7d8f4fdd902dd66be31f09e9a-0001": 3897, + "02-47-0cd4969f91958c949efc3252207b-0003": 3898, + "02-47-431034bc88eecdd3ed5a68d3213d-0001": 3899, + "02-47-51fc3948747d96d4cf8e3eb5e1d6-0001": 3900, + "02-47-7098e17af98928060c9e7ddc1b45-0001": 3901, + "02-47-74e366e7d8998e9d9913240601f8-0002": 3902, + "02-47-d51c053323f50e9332c8e295343c-0002": 3903, + "02-48-0abdfc563ab1122712b91f2905ee-0001": 3904, + "02-48-2b5a67785ff3b2f4f8369d2ec061-0001": 3905, + "02-48-2cf189c01e7c4377e5c27e53bf19-0001": 3906, + "02-48-360ad1523e6f58453ac3f748515d-0002": 3907, + "02-48-75378e7d1375be1cf509cf50f468-0002": 3908, + "02-48-9cd424476e4db8907828910f7cb7-0001": 3909, + "02-48-ef724cfb10cec35d98fc9a120193-0001": 3910, + "02-48-f84767e9ece031539ed72c452e20-0001": 3911, + "02-49-0d709af868341bb19005ed0b75b6-0002": 3912, + "02-49-1007b8abb5f1cf3404f7143f3690-0002": 3913, + "02-49-1da5ae89acad7875af8f6a1c57f6-0001": 3914, + "02-49-3c8100cc4535b123a2f3b5ea25d1-0002": 3915, + "02-49-534b519ef2e855cb9957cd71431d-0001": 3916, + "02-49-55ae25c1401a7de4cf8c918cee21-0001": 3917, + "02-49-62ad853f55ba391f6898b2946145-0002": 3918, + "02-49-77d5daed2a18e60210f01806677f-0001": 3919, + "02-49-9b99200d6588323ace6d47aa289b-0001": 3920, + "02-49-b410293771c543fcbe11c0c5ffa0-0001": 3921, + "02-49-bd2a8a775503bf19fdb38242158c-0001": 3922, + "02-4a-076ac9eac8cc39301ca82edf7991-0002": 3923, + "02-4a-097f11696d3076180acb8103f80a-0002": 3924, + "02-4a-27fd0d41c1a1f907f8cdf0c315fe-0002": 3925, + "02-4a-640cf46a96875fcdcc060157ee7f-0003": 3926, + "02-4a-716fa30b0fec0619f57e7d283bed-0003": 3927, + "02-4a-8465618b88c46601251d815bcf67-0001": 3928, + "02-4a-b383678088bd08694e941111febf-0002": 3929, + "02-4a-cfc81cea613fe2badb338fcf4ca3-0001": 3930, + "02-4a-e44a7fd5adb1fa45f3ba1623d4aa-0001": 3931, + "02-4b-002bbc91790a88c494cbf2168607-0002": 3932, + "02-4b-1cad9aed06b5ce8374dcb88d2723-0001": 3933, + "02-4b-2e043622b908318db9021f50218d-0002": 3934, + "02-4b-3947d00f5f5de078f6b622e4a2fa-0001": 3935, + "02-4b-440d73dea7b8f692192c0e89f4c4-0001": 3936, + "02-4b-6afa4fe4aa5213ea04086869e387-0001": 3937, + "02-4b-a58c81f7ac1543dfcdd7edd2b719-0001": 3938, + "02-4b-c0b0d9fc8feab8e36b26e8cc66aa-0001": 3939, + "02-4b-d613b0e42c3ab2e10a8584f24cd3-0001": 3940, + "02-4b-e02bd7bc475ac59da6cf4bbbfd3c-0001": 3941, + "02-4b-e7b7769c4a9370dbb1550510ec0b-0003": 3942, + "02-4c-5a92d0c12c90ea218251aa6d9c43-0001": 3943, + "02-4c-9c5deb22da75914f93365207c501-0001": 3944, + "02-4c-c1988361e2ac914ead9b48ca7e9a-0001": 3945, + "02-4c-f39059dc88c407253c4340d19c23-0002": 3946, + "02-4d-1a40e70090551bfe50d473ef1a14-0002": 3947, + "02-4d-44fa896ce5aa64ca67b61a16feb7-0001": 3948, + "02-4d-a9f57642353ed4ba3f6b0cbc2d5c-0003": 3949, + "02-4d-e5fc326d52f879c603697031a41e-0001": 3950, + "02-4d-ef5cf728a5be8565341eb0a08b50-0001": 3951, + "02-4d-f60ad93f3a389f03fb512947bd64-0002": 3952, + "02-4e-740c10053a2b5c7bd1d7b073548f-0001": 3953, + "02-4e-76577b10ec14706384210c1742da-0003": 3954, + "02-4e-82b8c8cc3bfec26692517b3b7062-0002": 3955, + "02-4e-a249c3b9aa91e9637a7adfb4cbfc-0001": 3956, + "02-4e-cbf0a156b91112e6bc42ecf44944-0001": 3957, + "02-4e-d6a2fc91f5933a26b6a410da8756-0001": 3958, + "02-4f-4bd423ffb3fddef1f720c4dbf5c4-0001": 3959, + "02-4f-7c26572177e1882e553ad1c2cdc7-0001": 3960, + "02-4f-815c73883c674225e57435f501f7-0002": 3961, + "02-4f-8bc5799ec0a8b35c133f4d7aa594-0003": 3962, + "02-4f-9c01f19921e79fa97010d60c8ebb-0001": 3963, + "02-4f-e9d1987668e0158aa31688df3032-0001": 3964, + "02-50-0b370541261591f4ff1a6ffb728d-0001": 3965, + "02-50-18747e6f1f772853b8570ffd3c74-0002": 3966, + "02-50-1ad189987bd3fd54ac0ed5327534-0001": 3967, + "02-50-23cbbecbec25dafe4ce7809d066a-0001": 3968, + "02-50-89614cdad1f984a616387a26a436-0001": 3969, + "02-50-9418fa3a7ee0696f97bc0e33344c-0001": 3970, + "02-50-bf67f56fb43c013170f3cc4f086a-0001": 3971, + "02-50-dd0e5f26b13bf785d878fe9cd0af-0002": 3972, + "02-51-10f8d83aad320bd24a7d3a969689-0001": 3973, + "02-51-4e3f5dee718d79808e46cb96f2f7-0002": 3974, + "02-51-50e66aa0bb16eea817f8bbc1d71a-0001": 3975, + "02-51-56de6f3b6dd14264e18cf113effa-0001": 3976, + "02-51-78a4ad2d7922c1a9d087270faa02-0001": 3977, + "02-51-8a82687a9f695633f027bee165f3-0001": 3978, + "02-51-8c04f007f4f3a8f3d2fcabe278f3-0001": 3979, + "02-51-cede568a8ecc1dd9f9ddd1d34269-0002": 3980, + "02-51-d397e634486d7bae7bcc9d1c0d1a-0002": 3981, + "02-51-db151fd9792fd6538ef251345290-0002": 3982, + "02-51-ef352dd0c35f294c49d7e297fd37-0001": 3983, + "02-51-fdb4e5d6f7397a6acc6f458cd421-0001": 3984, + "02-51-fe3243627e8e3cd8bc081c588606-0001": 3985, + "02-52-0007563d84b0a81085889b40f3a6-0001": 3986, + "02-52-00171b0545340bc15c5e209f7863-0001": 3987, + "02-52-4d76305dc04c3c29b95afaf1853f-0001": 3988, + "02-52-90e43d908338c8f6234fcdaab19f-0001": 3989, + "02-52-9fb0e61aa3cec7950742b9df3f3b-0001": 3990, + "02-52-b22e3ffb46f3de792212503740b2-0001": 3991, + "02-52-bc7e66ccf89fb6b92ee33fa8f056-0001": 3992, + "02-52-cf818721500fd0ed62ac2a214ce6-0001": 3993, + "02-53-15ee2e2b06fdb8643a0e42335e4c-0001": 3994, + "02-53-18284fcae24363dfb090ffb0c6f2-0001": 3995, + "02-53-41d8eee7077e5c23ea98c1dc67bb-0001": 3996, + "02-53-429fa57948b6b1c11d785bf0e3e9-0004": 3997, + "02-53-9d00a2799517482a66e2e74083d8-0001": 3998, + "02-53-b31300706cc889d671cdec752403-0001": 3999, + "02-53-cd3b31491ce1735bb3d7f46ec38d-0002": 4000, + "02-54-10407809fc03bb48b87c2992510e-0003": 4001, + "02-54-194f9c8e86de8737eb2062112384-0001": 4002, + "02-54-26071e64bb5a76e8815d6a532989-0002": 4003, + "02-54-366d859a08d0350bae2f27b1de18-0003": 4004, + "02-54-3bf0af3b5343d4955812d9ec1662-0001": 4005, + "02-54-4bf0815fff459c23cc9c9036e717-0001": 4006, + "02-54-7ab3002190f0abf65109dc77daa3-0001": 4007, + "02-54-a7da5171b6350960d2280d345918-0001": 4008, + "02-54-aada6d4ac4c926fd04aca19d9c0d-0003": 4009, + "02-54-bed11546655830b17d43abb9bdd3-0001": 4010, + "02-54-e1e6bd2b3ff2f6f45939aca21ba3-0002": 4011, + "02-54-eeb6de01e5a4c5af126015ea57de-0001": 4012, + "02-54-f80f5ccf762f0b5b9690927162b8-0001": 4013, + "02-55-089a06837895fa446bbf93f5f5ce-0002": 4014, + "02-55-0ff686362e973dbf3b608fb722d3-0002": 4015, + "02-55-3da884b9170b2c1793cb45660d25-0001": 4016, + "02-55-4206c3368c54178c9ec67be9998e-0001": 4017, + "02-55-4ff190f910c78aadf59ba906eec1-0001": 4018, + "02-55-68888bc6d0555064a9bf05e778dd-0001": 4019, + "02-55-7127e6cdb62fedc1bbf19db051b8-0002": 4020, + "02-55-7f760b0a116f558c801cfa60e86a-0002": 4021, + "02-55-942d89df2ffce7f24f9347a494a9-0001": 4022, + "02-55-a7374e140c22b875001a39648032-0001": 4023, + "02-55-bb3ffcc019aae00121d119b66e32-0001": 4024, + "02-55-c4de9751dbc036378080009476d6-0001": 4025, + "02-55-de022b4a3ea371d19dc537849f44-0001": 4026, + "02-56-09330e189e02a3aae76ae70d9413-0001": 4027, + "02-56-266070e6b6a4bf9dfccb35fcc86b-0002": 4028, + "02-56-506ce184c21201fc1fa7b74eb527-0001": 4029, + "02-56-8864723d3e0cefe82a9f0de51e39-0001": 4030, + "02-56-a0cf39ba8b91078e42f992459cde-0001": 4031, + "02-56-c25e5f2b0c71144f0cd902e2e47c-0001": 4032, + "02-57-053c7d239c01bacce2feaff06ac7-0001": 4033, + "02-57-0be8798acad3a3c9c8ea7153a1d4-0002": 4034, + "02-57-1376092ce931928e7d5433f08b41-0001": 4035, + "02-57-14546154a1e77bde710a6fdc51c2-0002": 4036, + "02-57-18dd918e8acfeb6fc5c35e82d957-0002": 4037, + "02-57-1dfd1b3616f0726b57f4c9a32249-0001": 4038, + "02-57-3616ecf63de324aca0157ec96b2b-0002": 4039, + "02-57-4784298be3d1249fd967915af048-0001": 4040, + "02-57-7081a8d11e5250ce41ab45bc2e7c-0002": 4041, + "02-57-909cbbfaa6763a443c500e3f950f-0003": 4042, + "02-57-a4cf7a514e2e27ade76784505c96-0001": 4043, + "02-57-c941d9707091422389d31f6b7def-0001": 4044, + "02-57-cbf21dbb485443317af720654c28-0002": 4045, + "02-57-df18a1905748b1ff1a6df4eb962d-0001": 4046, + "02-58-0cadaa3ec9c5e0241ae2839330c6-0002": 4047, + "02-58-3aa421247d7324a9017cc9b43bec-0001": 4048, + "02-58-4c20c5fad48399eddf7f498ee2d7-0001": 4049, + "02-58-5c5018d367c1ad1d04dd326c8dff-0003": 4050, + "02-58-7d8f8ba923492637fe08d314d848-0001": 4051, + "02-58-a94bc6796681b1ff73f9cf3a59b4-0001": 4052, + "02-58-b70e90487dec072c26e7b7402943-0002": 4053, + "02-58-cd697c1f4909c5ff04eda99eb94e-0001": 4054, + "02-58-f2edbab9888b6905d23c6dace41d-0001": 4055, + "02-59-07f3dcb157788f768c4358d27345-0003": 4056, + "02-59-1bd6fa3ee53ca0d5a767fd5cde85-0001": 4057, + "02-59-4b44bd6237bd962ea1b54e48d9c9-0003": 4058, + "02-59-6a60497e7f52592640caf5d274ce-0002": 4059, + "02-59-b5755c2856bce1d13a8a74f8ca67-0001": 4060, + "02-59-d75b698fbb1d99adc3555b3bce85-0001": 4061, + "02-59-e4cf4ae0803068596c1dcc64a6f5-0002": 4062, + "02-59-ed032a7debb9d507e790ace3b073-0002": 4063, + "02-5a-05f3b8b13a8c7d7e758b59d7d919-0001": 4064, + "02-5a-273bd5f968caae4bc43c7e1d372f-0002": 4065, + "02-5a-830314c418ec9692df60a5caae27-0002": 4066, + "02-5a-c5a0a94a9601021758be8c118a33-0001": 4067, + "02-5a-c6f95b9e789af6028742060ba3a3-0001": 4068, + "02-5a-d0476a513d4a4923c66244625b5a-0001": 4069, + "02-5a-e57b350572fc8d2401ec6b9843ac-0001": 4070, + "02-5a-f7670dea3e5f3d4b3f7818fc9513-0001": 4071, + "02-5b-1f440c6252ac53c24a580430ecff-0002": 4072, + "02-5b-81a3425328db9635d147e511f928-0001": 4073, + "02-5b-a8a9c698909e3722deee46e8865f-0001": 4074, + "02-5b-a9f366c3164f817e3a5d95830d64-0001": 4075, + "02-5b-c4c2e03a6fd0840922dfee66ed6a-0002": 4076, + "02-5b-c983379f6dadde82f721487fcaf4-0001": 4077, + "02-5b-ccadd759d6ab47b0fdb362cd8cbf-0002": 4078, + "02-5c-13412b36f7e7d0ddba60cb0b7e3f-0001": 4079, + "02-5c-4f5d684ca891007c2399b5bc02ad-0002": 4080, + "02-5c-5113a024f5c834b8ad6736f9bebd-0003": 4081, + "02-5c-74707344f538e4808a1131e297c5-0001": 4082, + "02-5c-c5fcf247f55ba2b5daaf4e95e86d-0002": 4083, + "02-5c-e5fe4ac4198854e200019c6e2920-0001": 4084, + "02-5c-fa485fd96ac40fe14ef94e9b5630-0001": 4085, + "02-5d-c7715425335287a0b125e9c2a90f-0001": 4086, + "02-5d-e02c98aea9afa009e53c10078e9e-0001": 4087, + "02-5e-008c6d803040de7c53d3e8b883ab-0001": 4088, + "02-5e-2edeb2cf06e44f861c4c256d85cc-0002": 4089, + "02-5e-481602e058807c55a0b6526a0080-0001": 4090, + "02-5e-6ec8e2b89f86afbb0b3e8be02be9-0002": 4091, + "02-5e-83280f93ad5bc920ba0b46336f70-0001": 4092, + "02-5e-8b1ec27baeb175b0fc575772b4ee-0001": 4093, + "02-5e-c358b9c2369a87f637624c13778e-0002": 4094, + "02-5e-e9554710d8ae5e3b009ae75def76-0001": 4095, + "02-5f-0db2b4967173b310a498ae2f8148-0001": 4096, + "02-5f-1519e961e918036496bc09205247-0001": 4097, + "02-5f-1d9f4c6da75520639b28743e6012-0002": 4098, + "02-5f-704a1c5140050497deb9d79c4a07-0001": 4099, + "02-5f-a3ba3ef76fe6b16da97991c44181-0002": 4100, + "02-5f-de82a9567358bb785fbcdd04bda5-0002": 4101, + "02-60-172a0736531f4d12a59392841187-0002": 4102, + "02-60-2dc7525d9bc01c5e6a5f7cd8c532-0001": 4103, + "02-60-6f4b35d74933e30f4b0a0512ada6-0001": 4104, + "02-60-79672977d8e335c8562616cff27a-0001": 4105, + "02-60-90e77e1b550c3766b71b8665b9d7-0002": 4106, + "02-60-b94eec7b3c562212f6f4cfc6dd3e-0003": 4107, + "02-60-d6e4dda547b135060026cf034320-0003": 4108, + "02-60-ef35e32c89b73d4daba8a6fc146a-0002": 4109, + "02-60-f5b60a0e4aff95be315fe57513d0-0001": 4110, + "02-61-0d27bfce9a4fab4530c2a6848067-0001": 4111, + "02-61-19a5aa345cdc8c727abcfcd54e32-0001": 4112, + "02-61-4adea36427f35164722d93d9c6a4-0001": 4113, + "02-61-73d6fae554e4091e074bec4722c3-0002": 4114, + "02-61-7c9ae9761c682b7b097ccd688dcf-0002": 4115, + "02-61-a856fe2a31e5e2e97311d69f601b-0001": 4116, + "02-61-b586075b78acba9ce44f74ceedfc-0001": 4117, + "02-61-c561c666fd6a00b4fd651e98d96c-0001": 4118, + "02-61-ceb7e2ee0aa49e79361a0be619ac-0001": 4119, + "02-61-dffd6d347972db8eea02399b01cc-0002": 4120, + "02-61-f1e6406dc304392e7629ffd7ee86-0001": 4121, + "02-61-fb5b37997d1a998b005c8e7f427f-0003": 4122, + "02-62-034d7e2295b651e70bcf70d53653-0001": 4123, + "02-62-1c77fc747d8097fd09dd556a300f-0001": 4124, + "02-62-3f7c62efccbbc632e0dd6e3effd5-0001": 4125, + "02-62-486d8aceacb3df1b8dfeab37e376-0001": 4126, + "02-62-739936eaa542eebbb3ba171ed9e9-0003": 4127, + "02-62-797019b72e6bef41c0ab2ba8ea85-0001": 4128, + "02-62-79bd2ee3987955e0831e3ff12855-0002": 4129, + "02-62-bbe4b5334cd4156c0040e80cf9b9-0001": 4130, + "02-62-e6aee8b76519a911fa15422dfe87-0001": 4131, + "02-63-3b0f33a86ec718abf1b43bf6f014-0001": 4132, + "02-63-414a0df10cbb1de83e27bc15bc5a-0001": 4133, + "02-63-54542b58749355e4c424a92c6473-0002": 4134, + "02-63-81080606523f94054137e71a1a4f-0001": 4135, + "02-63-8842366343315c186218eb135b5f-0002": 4136, + "02-63-909efa7ff142db0b0c758857328a-0001": 4137, + "02-63-9d58ecc9b55edc070a83c86d2214-0001": 4138, + "02-63-b0d6a5028d4af2edbf3671399d5b-0001": 4139, + "02-63-bcc706dee3e3ce5d5fc3f5b8e397-0003": 4140, + "02-63-c94d73b6fd64b9b13af9748a43f9-0002": 4141, + "02-63-d9b093051ea5940a88d627677142-0002": 4142, + "02-63-eaf3d83c7d607d85db567ddb41c4-0001": 4143, + "02-64-0f142cab6531c962a58f0d88561d-0001": 4144, + "02-64-15b8d6b74452e17a814ca343d0d5-0002": 4145, + "02-64-1786249bd2a382f490785375c67b-0001": 4146, + "02-64-1a3034eeef15079a3db723990dda-0001": 4147, + "02-64-460853e0410b96f776643e36fe10-0001": 4148, + "02-64-821548301ecaa0488b3070750ad7-0001": 4149, + "02-64-87391d948b58ec78e759e9d97d8c-0001": 4150, + "02-64-908c1f5c3df2557b2efefd72442e-0002": 4151, + "02-64-90d054fb5eb0692275577639e015-0002": 4152, + "02-64-a3064d5f3b89a4ee0bbbd74d7e38-0001": 4153, + "02-64-a72180cbea57d8b51e3da37cc061-0001": 4154, + "02-64-d367529bb49493ba28d3f20d1fb6-0003": 4155, + "02-64-ea94666084ad3777d9d0b2727cb4-0002": 4156, + "02-65-b67ac4c69f2011360171c0badaba-0001": 4157, + "02-65-dbfc254038bf2fa8dc562003e645-0002": 4158, + "02-65-e68159e2d15a5409c23f1c2e6558-0001": 4159, + "02-66-14764aa96eb84cd3d99c579aaac3-0001": 4160, + "02-66-17a584e7ae1ae0b09675016074f2-0001": 4161, + "02-66-6299fb4dd55356e8c6649b9568ad-0001": 4162, + "02-66-6dec37c7d1f0c1ceca6ccfafce50-0001": 4163, + "02-66-87906cef7cabdf567177f00716ab-0001": 4164, + "02-66-8eda5686b463219b304f65b09922-0002": 4165, + "02-66-931912b535a5fa59762e75e9df78-0001": 4166, + "02-66-ae6ae0ace9a4a847201c4378660f-0001": 4167, + "02-66-bdffcd74b116bbc1661a38f770ef-0001": 4168, + "02-67-34136f8affc12e3c25430f0948b7-0002": 4169, + "02-67-53044d08c8be5df38aafdcebc1d3-0001": 4170, + "02-67-53ee229dec6461fd6e36e978b0ca-0001": 4171, + "02-67-599b552e2eff57c97d8645a1ceae-0002": 4172, + "02-67-803393572f28d60f74c7f9bfcee5-0002": 4173, + "02-67-9492b0efc2fe9f487db3dde3f44b-0002": 4174, + "02-67-ae6d17a95c8969b4b9617052e197-0001": 4175, + "02-67-afe607c4b2cebb1e3ffa18daa070-0002": 4176, + "02-67-c3d1cb6d92d8d95e948c3fc54a11-0004": 4177, + "02-67-cad71077bf5a20dec9094eb9c9eb-0001": 4178, + "02-67-de023463de4691a468f52e0f6c87-0002": 4179, + "02-68-0f358932f2347f1d2b46e072618c-0001": 4180, + "02-68-10029dc31eccbc2388865320e682-0001": 4181, + "02-68-326e49997b579d794b83651fbf05-0001": 4182, + "02-68-71bcc04dfefb679ac7e88b3357a9-0004": 4183, + "02-68-afa88074246fcbdd54855c0c4799-0002": 4184, + "02-68-b6922cb4804de3fb214b3ed25345-0001": 4185, + "02-68-df4c3af49fc3842ad2b7f536674b-0003": 4186, + "02-68-f151ff4a2de95acff8f18126545b-0001": 4187, + "02-69-27e1c1607f5e7129fc11c2b43b42-0001": 4188, + "02-69-796d4b0d9eaedd5e5b0dfdf3d5ae-0002": 4189, + "02-69-79f84ef0e64838d3e0ea2d0717dc-0002": 4190, + "02-69-a0b28fad3fad042b6e5749eb9259-0001": 4191, + "02-69-b72207cde752c6b0513e866471da-0001": 4192, + "02-69-b8f2da6febaac217afc28c62e281-0001": 4193, + "02-69-e2fd58d78ea5871ab1bb2a4fcadb-0002": 4194, + "02-69-fbde404895d168e04638be02b200-0001": 4195, + "02-6a-5030fdd653a62c89d9445f07b303-0001": 4196, + "02-6a-844dcca001b5d4a1d42c877a75f8-0002": 4197, + "02-6a-8d04825eafa482741e35acd1252c-0001": 4198, + "02-6a-9c6309aba33edfff60115414516c-0001": 4199, + "02-6a-bae1cf172a31e3b8e69124a1dbfc-0002": 4200, + "02-6a-de1a594bcd03ab06105b0575538a-0001": 4201, + "02-6b-325f74314cc04dcf8073a7334d8a-0001": 4202, + "02-6b-ce91387c661e72440f103360fd54-0001": 4203, + "02-6b-d19c9d6a53b85fea5dba189cdbfe-0001": 4204, + "02-6b-e763d10642d95fbc37ded2b3a311-0001": 4205, + "02-6c-200ce86607d187f8581981b1bacc-0001": 4206, + "02-6c-38b00a6e91571f8ec896ea25c8d3-0003": 4207, + "02-6c-4a6d28c1d3ab3ba7a57c34964bde-0001": 4208, + "02-6c-77d5db94ffe9995094fc79c61833-0001": 4209, + "02-6d-115107a2dab1ed9ed1985622abd8-0001": 4210, + "02-6d-1950caa81b2e0c141e3fc5917357-0001": 4211, + "02-6d-4d3d4530510364b9fdd6c59f0159-0001": 4212, + "02-6d-88adb41046b588e3b8f64bb4eeee-0004": 4213, + "02-6d-94d42f683385d7bd8c2f2a6fe331-0001": 4214, + "02-6d-e1c4d8e958771f4bf5e981953db3-0002": 4215, + "02-6d-ff3fbeb912fa0763ab66a16b0279-0002": 4216, + "02-6e-36c538665c8b947de3489755706d-0002": 4217, + "02-6e-4a66794e75d0b0bdb67c0953c532-0001": 4218, + "02-6e-4b7397abf3d58921e4ec2b60e91f-0001": 4219, + "02-6e-6a2361abf9184e36a9dc04eb1244-0002": 4220, + "02-6e-99f3ad9d90f95ff69d683b75f4c5-0001": 4221, + "02-6e-ce13f7dfdb2c1ce6c5b82fa451d4-0001": 4222, + "02-6f-1c4d80c3b7c6a4f04bcb607510b5-0001": 4223, + "02-6f-57c77bfaef35a3563081954903b3-0002": 4224, + "02-6f-79b0102285ed4f3fb9d0f6281059-0001": 4225, + "02-6f-a2cf3cde1f26c6fae5050ad90a52-0001": 4226, + "02-6f-a9628b4fb4b1cbfc8efb82ef3050-0002": 4227, + "02-6f-d116a157e83a715407fca67ec7b2-0001": 4228, + "02-70-13bad9970c3bf1abcdb095f51585-0001": 4229, + "02-70-1f0711ff8593c3294b86d1ee776d-0002": 4230, + "02-70-3a130bd375048a870f0c93e97eb5-0001": 4231, + "02-70-427230030a0ff31e0dd5783ff5e8-0003": 4232, + "02-70-597319c7326cea9bc945a2dd0190-0001": 4233, + "02-70-7f301e2486532685a3c3b39dda46-0001": 4234, + "02-70-921fd9a7ce426da6d0759e7d3599-0001": 4235, + "02-70-a5445df19ec93eb6093aaffa2ce9-0002": 4236, + "02-70-afdcc3adf3c95a1f75b642f62257-0003": 4237, + "02-70-b14ad13859b162aba3ab54d2c4cf-0001": 4238, + "02-70-b1e09ee869525e393c0eb15aa55e-0001": 4239, + "02-70-c828d097321ccb84064576472fca-0001": 4240, + "02-70-d78686ca5f4f83368432546fed66-0001": 4241, + "02-70-e1e781d8476c7406afb58b0ee340-0001": 4242, + "02-71-21934bbe2b9cca00c5c74ab8fa91-0001": 4243, + "02-71-5b28c6d73207475434c9a8f05a1f-0001": 4244, + "02-71-6bfcae50df773c166e7cdefb001d-0001": 4245, + "02-71-f432960dd29226684ed2a6083521-0001": 4246, + "02-72-0746de0dfc0d1ce132f38218c46f-0001": 4247, + "02-72-6af21096ffedb419be01533d8f35-0001": 4248, + "02-72-7317c531e9f6b1561cb368cf0ee2-0001": 4249, + "02-72-7cd6766dcea2ad949d98648d00bb-0001": 4250, + "02-72-9c9759f55876d80a21076f7b56ee-0001": 4251, + "02-72-c06ce4486639ea98b0979728c9d7-0001": 4252, + "02-72-dc4cb8c833705a01b97851c7a630-0002": 4253, + "02-72-f4da5db303b2b50f112768e32a44-0001": 4254, + "02-73-0cf29ad0207095e470adc032315a-0001": 4255, + "02-73-360cf719537fd7e23278d79ee5e9-0002": 4256, + "02-73-42281f7ede73f0cba657c2478f00-0001": 4257, + "02-73-4c05f278ac3037522a01497d10c8-0002": 4258, + "02-73-5087650391bd7e10bd5b004d99c7-0002": 4259, + "02-73-517de62074821418e04bb040f1b0-0001": 4260, + "02-73-574f40ac09b09aec1314d3631886-0003": 4261, + "02-73-7a0256db3dd2bbb47e268318b101-0001": 4262, + "02-73-b13109321287f9d60547b5470073-0002": 4263, + "02-73-b3fdf9c7b562b896bac01eea5d7f-0003": 4264, + "02-73-c07123e9b1a0e7a6dee842ea4b2e-0003": 4265, + "02-73-e6aaac3784026f455a448e2accf0-0001": 4266, + "02-73-ff009fda21b80de1c8314df2db40-0001": 4267, + "02-74-05e80e4f7e5c2863caebc753fafd-0004": 4268, + "02-74-3a1f1a855399d9929dfd2786d48b-0002": 4269, + "02-74-584337ad79d9a9dd960652d14d72-0001": 4270, + "02-74-60b745a8720ce04cf52a52605e8c-0001": 4271, + "02-74-b21d204bb73b91dc59ce0ebb522f-0001": 4272, + "02-74-ce550d4f73a72e7ccaaa0ce50aab-0001": 4273, + "02-74-f5d91001d7d61c0629866fb3fe87-0002": 4274, + "02-75-084a4e95c4e9378042582b6c5739-0003": 4275, + "02-75-200c950c37d77dc6ef18f496f87c-0002": 4276, + "02-75-5e71acb15c9f28085519dff8dfc2-0001": 4277, + "02-75-8dd75a66ed54afc9a70e3f89c99e-0002": 4278, + "02-75-b024d3f13058bb5a5893c4a0b258-0001": 4279, + "02-75-bb9fb106058ee47312b12db534a3-0001": 4280, + "02-75-c50ce7b414225735ff3132821f88-0001": 4281, + "02-76-3150806452281f5077bbe1bab67f-0001": 4282, + "02-76-333aa75b4733ff25a6477461a0c5-0002": 4283, + "02-76-8c6033c6eec0c829f65a994d3bff-0003": 4284, + "02-76-b01a1a46facc7249e15a4218f830-0004": 4285, + "02-76-e7cbd9d9781a1d57946028d485e6-0002": 4286, + "02-77-0afce73f0ac9be24fd0d3eea3b83-0001": 4287, + "02-77-0ec52c5f3e93d766f39fa5df2523-0002": 4288, + "02-77-1797f986762528c483be67063995-0002": 4289, + "02-77-2f45d3fdd6a22c9282ef05959523-0002": 4290, + "02-77-8c9c8f8e2f7672c6065c04fa6551-0001": 4291, + "02-77-981410a4bcaf00eaba569a5f1f30-0001": 4292, + "02-77-e1163d7aed4239c1060e28ac1328-0001": 4293, + "02-77-e64d01ea658e3be5523444f1ab40-0001": 4294, + "02-78-19ab6a98f03e37f05c825738aca4-0001": 4295, + "02-78-2aac06c152f04430bb132f8d8214-0001": 4296, + "02-78-68bc7a2c1e4fb646cd29fcc84822-0002": 4297, + "02-78-99e79e91506ba35ddd098401f079-0001": 4298, + "02-78-a5ad9e19dc32825f6a1679f4f9a6-0001": 4299, + "02-78-bac34a4add2d9457be571447862c-0001": 4300, + "02-78-ca1a0124a44c4ba73869ce979cc7-0001": 4301, + "02-78-d0ee41f918c5f3f5489ca65a30c4-0001": 4302, + "02-78-e39c322920bdfaa5df0930390123-0002": 4303, + "02-79-26f47a58680759ec2b335c059d89-0001": 4304, + "02-79-29a568bbc64efc9c642f33ee2d03-0002": 4305, + "02-79-7ca1dc05bc0d93f793ce5486c44b-0002": 4306, + "02-79-8167ea91250fd05dd85c48ebd831-0001": 4307, + "02-79-adbc49332db9ff76f489c8bca7e6-0001": 4308, + "02-79-b5436a34c1ae2c5b522330207770-0002": 4309, + "02-79-b7a32cd5ce12e279c4ade0e99a9c-0003": 4310, + "02-79-d76b744b0b38f54c186ff586f5c1-0001": 4311, + "02-79-eaaccf948f02aa6021e6f3b169e4-0001": 4312, + "02-7a-04463cb2ba1d915ff24ae852bbb7-0001": 4313, + "02-7a-0d38a6bfb4c7fef6538625efa5f9-0001": 4314, + "02-7a-1da78c500c676eede993f8ca1806-0002": 4315, + "02-7a-21ba3cfe9c2e1e34a52179830cb9-0001": 4316, + "02-7a-258fca8694f077546ffca62d2696-0001": 4317, + "02-7a-2d386ba6fcad51d03709293bd280-0001": 4318, + "02-7a-348d6913cc64387988e1a7af7636-0003": 4319, + "02-7a-71a72d624569ced49ee03d5adb5a-0001": 4320, + "02-7a-722ab78256c49bf46083d65e9ca7-0001": 4321, + "02-7a-7d1620038a998bbeaa2c0988d01f-0001": 4322, + "02-7a-82c55a9e57d85bac95c36b7e7510-0001": 4323, + "02-7a-85e69ccc2a99df933e773cd24cc8-0001": 4324, + "02-7a-cb5aa69496b52b9092b662a053a4-0002": 4325, + "02-7a-cffd7ca8d79fdd61ec38afae38fa-0003": 4326, + "02-7b-0252ac8e1496c46acf0154faef85-0001": 4327, + "02-7b-1d2fcc4cf94aa911b2207271104a-0002": 4328, + "02-7b-266f38be2c260dd4f6c620927911-0001": 4329, + "02-7b-3c29c8ec187b21d9ca0984e48bcf-0001": 4330, + "02-7b-3f92fe076283303c835c0b3a0a75-0004": 4331, + "02-7b-6b7229652b14f36f17b0b1664472-0003": 4332, + "02-7b-7a4a598fd924008be368a3f95737-0002": 4333, + "02-7b-8bd123c48c53b139618b4ef38a10-0001": 4334, + "02-7b-a20c38b7b40dbe0c8e0c34704e9e-0002": 4335, + "02-7b-afe53c93d1c18ee480bbfd065131-0001": 4336, + "02-7b-f2bd3e638701e1920c381c09c706-0002": 4337, + "02-7c-09df0a51c40a3b2d2a45c98a454e-0001": 4338, + "02-7c-3b1a15c36f25a9b090460ad5b8d6-0002": 4339, + "02-7c-6163b5240f79d298241b609ed291-0001": 4340, + "02-7c-751223c31ed5e4ff0dda509c1141-0001": 4341, + "02-7c-840da37256ecb9a31754509f6b61-0001": 4342, + "02-7c-99cc8316c13eaaefaacddabbfd7c-0001": 4343, + "02-7c-b379c669a9a1426d0be2e51b494a-0002": 4344, + "02-7c-d3ab6d552b0cc3e208154442e901-0001": 4345, + "02-7d-1a5c9a74886b68d4699f84dcd20b-0001": 4346, + "02-7d-38b7943c365b712842fdbb3984da-0001": 4347, + "02-7d-43174cd9a2c6bd427c7bba49ad32-0001": 4348, + "02-7d-45a81d022260aeab852b5f3badff-0001": 4349, + "02-7d-4b5a6014f81f2c25850849729b19-0002": 4350, + "02-7d-9733ec5ed1b0a072a897026212b2-0002": 4351, + "02-7d-a7360238e8b4f8b342dc76778af8-0001": 4352, + "02-7d-b6a046476e1d034f20c497c65b68-0001": 4353, + "02-7d-b8df4b232015d2e322744dfd4f4c-0001": 4354, + "02-7d-e2aa0bfeeb486dc401d07a6c06dd-0004": 4355, + "02-7e-24be7c088ec25e664627c2a64306-0001": 4356, + "02-7e-2cc58cd9f992eba17d550e9942eb-0002": 4357, + "02-7e-3c89cb9fc719f0a2f705d68ead9f-0001": 4358, + "02-7e-75765a6513602c2ee6c18bce4daf-0001": 4359, + "02-7e-8cca7ab852295f8dd907e2d5eeb4-0001": 4360, + "02-7e-bff771cea62f88de95f33bd8ec4d-0001": 4361, + "02-7e-c157ce401ba816ad6592468bd8a4-0001": 4362, + "02-7e-db6f006dde5cb13550e01a73ce0f-0002": 4363, + "02-7f-1d7cfe9ad096391f05456f1a617a-0001": 4364, + "02-7f-204d20742488700c4a7d2d1afde6-0001": 4365, + "02-7f-4d4aeafba5bce578a9a56106e6e5-0002": 4366, + "02-7f-8e843b4e2eae24cd99d11fe7639f-0002": 4367, + "02-7f-9fb5c1901bf1b2dfa7d242a2b936-0001": 4368, + "02-7f-ab4b85e6c587d3efb5e616a0a967-0003": 4369, + "02-7f-b12425354c83f82c7d58676d06d5-0001": 4370, + "02-7f-bc2917ca0b82f6520071ff6b3ab8-0001": 4371, + "02-7f-bcefdf63db6dcf73e6f1d4042302-0002": 4372, + "02-7f-f24eac45b007189e5db30771b611-0001": 4373, + "02-7f-f3b52d57051aab33d15fbcc99063-0001": 4374, + "02-80-046ab63deaa0c8e7da4b4eb69f20-0002": 4375, + "02-80-85eef4c1d3b0a81707a9db580e36-0002": 4376, + "02-80-8dea9f4efe8e337ab26dc5de9533-0002": 4377, + "02-80-d8c3dc1f0a42596032e7442c0606-0001": 4378, + "02-80-fe5188c8136b4f3d02bf238ad321-0001": 4379, + "02-81-7b56908fab25dc645756746b075c-0001": 4380, + "02-81-fbf6a5ee439cc60000df538ccbf7-0002": 4381, + "02-82-4a98fe64f2dc9cc4b4d70bf8dc21-0001": 4382, + "02-82-58391c478fc8e83448ef1b1eceb7-0001": 4383, + "02-82-807be9ed3b9caec2a8560736ab20-0003": 4384, + "02-82-8753b858bcca6aa7f7f94626dc93-0001": 4385, + "02-82-fffbd6a8887ac6d80ea814b08be5-0002": 4386, + "02-83-07d18cc3afd0fa7c4456b54facca-0001": 4387, + "02-83-481a8e10dcb89c150f5e10bff366-0002": 4388, + "02-83-62336d508ef91d2d717df10c0656-0001": 4389, + "02-83-649a0c54002f858a79619f3dacb4-0002": 4390, + "02-84-27b43ef3faac4ba4edc8a4cb8f4b-0001": 4391, + "02-84-4bf61eabc7f4cb56bbce345e5679-0001": 4392, + "02-84-525e0ef0219a7dbb5460a6d0cf4e-0001": 4393, + "02-84-553b4e348f925622cedb82228892-0002": 4394, + "02-84-570f7fb3e5cdf762358c164552a6-0001": 4395, + "02-84-b7db0a9907a52f5dde7405441626-0001": 4396, + "02-84-bfeb39d24ec47f059b1bf676dacc-0001": 4397, + "02-84-c83bf683887e7c8e87e7c703a093-0001": 4398, + "02-84-cf7709b3d01eccda88190042fac3-0003": 4399, + "02-84-e5d29c6e1c247124d35f41dfa307-0001": 4400, + "02-84-f797cdd0a1e5cffeadd45ee45602-0001": 4401, + "02-85-031683e96211ead584bf1fd54640-0001": 4402, + "02-85-5b6148588b8901d568931529138e-0001": 4403, + "02-85-8f3b514af8391d27b69f0b3cc985-0001": 4404, + "02-85-abf4e1e0696804d4c7ba50b87843-0001": 4405, + "02-85-ad507cb0cd5ff6ea6e5f383748d8-0002": 4406, + "02-85-d104d58e6c60a6e78a1e89333189-0003": 4407, + "02-86-3ca59544d86ac7ab4878c4d80186-0001": 4408, + "02-86-4aa55943a1c21c576fb6a94cb912-0001": 4409, + "02-86-769ea599fc7cc6517edd6f4a077e-0001": 4410, + "02-86-7921648ff69ee4951d1aeba068ad-0001": 4411, + "02-86-c5e5858be8331505fba87a7c4f19-0001": 4412, + "02-86-d9e66df1cb59dfd9214c529b7cbf-0001": 4413, + "02-86-e08f2db6d4a282d6e0d58aab8c66-0002": 4414, + "02-87-0177b7023c70b5133ff12d3b07b3-0001": 4415, + "02-87-5006da74b649f0e227a2934c6a42-0001": 4416, + "02-87-51c1d4d4af97086a07a464c4353e-0001": 4417, + "02-87-788d648a5c7e726bd3863eaa9387-0001": 4418, + "02-87-81ce6e8d98a1c817655cd1abc0d5-0002": 4419, + "02-87-85b262773e6523f056de18928dda-0001": 4420, + "02-87-940995bd16148a4e7c5466524021-0002": 4421, + "02-87-988224a04d2b4104e0bee2aad8a4-0001": 4422, + "02-87-a7853ccd838d11161f0ac0e4deaa-0003": 4423, + "02-87-bcc567b408f9a257c773e58ffb1e-0002": 4424, + "02-87-d13904a8cec1eb72a145159e46aa-0003": 4425, + "02-87-d78e47a809168bd272b924ad9c04-0002": 4426, + "02-88-1cd9e09c2c9f7b55f0d9e4f5df65-0001": 4427, + "02-88-31bc13d967e7a30771ed9c96339a-0001": 4428, + "02-88-4f3cc351f688436b8810f048dc85-0001": 4429, + "02-88-6dd0761b36af138812c5d87cd47c-0001": 4430, + "02-88-7e1a2728fddd00e2e42c3c2f7baa-0001": 4431, + "02-88-9692e246521c1f87e4eeeefef903-0001": 4432, + "02-88-ae406d4529d562790c8e0ca77f4e-0001": 4433, + "02-88-bb013ad78f9ee57c84bc94702c2d-0001": 4434, + "02-88-d3cb4b74718b3318371acbed68b6-0001": 4435, + "02-88-e0d1fcb91266f814bf62b0440164-0002": 4436, + "02-88-ffbda0d0f1bb8834f20195f4a70e-0001": 4437, + "02-89-4045d06eaa2abc19589271e89418-0001": 4438, + "02-89-4a14c7759004f04a2b629859b720-0002": 4439, + "02-89-651007278d33ff84062b98c311bf-0001": 4440, + "02-89-99ce6d0640718e5b459b18c48302-0002": 4441, + "02-89-c45f26122cecae9a88aacd848e14-0001": 4442, + "02-8a-264bc187b6b4c70dc4b749e61b67-0001": 4443, + "02-8a-2ed35fb9c2a36dfe9df132c0b07c-0001": 4444, + "02-8a-43c7fbe90886b1cf802d432e46a2-0003": 4445, + "02-8a-5e3287fd71dff3f8c6f428851996-0004": 4446, + "02-8a-602d185f285d908487aa4a67d5f4-0002": 4447, + "02-8a-84d4922e451682cb90058b5da8dd-0001": 4448, + "02-8a-afc4fe7eeecfd43a7d4877a5f594-0003": 4449, + "02-8a-c36f3b51bc04d64935be9523aaba-0001": 4450, + "02-8a-ef080d517a915eb0a4a87d3bf341-0001": 4451, + "02-8b-0b690feac46cc6a8a0f1f6b9e2cf-0001": 4452, + "02-8b-15a218a910412b373ceca2e39150-0003": 4453, + "02-8b-20563aed672132107c8f758afa14-0001": 4454, + "02-8b-a93a2e4e964e0cc22d7359b2c993-0001": 4455, + "02-8b-ba5de503650f279c16a2b69e192e-0001": 4456, + "02-8b-c558fe52e28e2204dcb04234338f-0001": 4457, + "02-8b-d5741c7f322f149d4570c4142233-0001": 4458, + "02-8b-f604427723d786711af4ea6e77d1-0001": 4459, + "02-8c-2dba6c47d1182f320e0d1623b946-0002": 4460, + "02-8c-564262de94c2ecb556021303f334-0001": 4461, + "02-8c-58a7d311a4abc3d071de300cc56b-0001": 4462, + "02-8c-85b91c14c0ba925848659def0e88-0002": 4463, + "02-8c-943227e27e9fb87ebe26d93ba461-0001": 4464, + "02-8c-96414e034797d08fdb10e05b1f50-0002": 4465, + "02-8c-a336a58ba828464ea79e9c6596b9-0002": 4466, + "02-8c-b54ddcaca2e71d0aa76f54a74908-0001": 4467, + "02-8c-dc6866baebab0c71dc4781b1240b-0003": 4468, + "02-8d-0d1c3234f17b2d4fc0c0f93c5fa9-0002": 4469, + "02-8d-0eafcf690e4fa9aaa200bb2967c3-0001": 4470, + "02-8d-4bc9a42af33b0cf448c6da7f70b3-0001": 4471, + "02-8d-584e82e75bdef082998518c8407b-0002": 4472, + "02-8d-5b835907f80256f8a684b0df02be-0001": 4473, + "02-8d-6e0e20c253a7a008793824ef02a4-0001": 4474, + "02-8d-774ef0ee22d30103ff32bb42e47b-0002": 4475, + "02-8d-c1b7320adca14e9ee430a8dd2533-0001": 4476, + "02-8e-3084a9dd03973b4e8ef4f191d042-0001": 4477, + "02-8e-492978ed66533ae06ec12aec0c1a-0001": 4478, + "02-8e-6d1b643323e78644ebedde189377-0002": 4479, + "02-8e-6fcc80e0926394a1d7da2a59e6dd-0002": 4480, + "02-8e-796e31903bf1d63d493bf5793a8c-0001": 4481, + "02-8e-82a9acbac93c7317c98975f5d86e-0001": 4482, + "02-8f-09be3f31f0e6472f67f66d29fe94-0001": 4483, + "02-8f-7c070546136572f4ccf3425658b7-0002": 4484, + "02-8f-898d7b988dd783043dd74e6a06b4-0001": 4485, + "02-8f-c464c2b7765a666aaa655ec5d537-0002": 4486, + "02-90-057a1295ab6b0c600b37b5d7222d-0001": 4487, + "02-90-185b3a47c2539047d56369f55ae0-0001": 4488, + "02-90-3ba2ffd2dcf9532f2e7d0b7cce6f-0001": 4489, + "02-90-80c470c586d68c0cff62e5b60491-0001": 4490, + "02-90-bacc2ddc790113063efbdc30380c-0001": 4491, + "02-90-e167c7b71a06e6996e3a953ef3b2-0004": 4492, + "02-90-e5f032b0aabbb89b23255e0ec148-0001": 4493, + "02-90-fcd53c5f9c27289eaaa9fca489ad-0004": 4494, + "02-91-0af4989313a771c81e2bf99d3dab-0002": 4495, + "02-91-48a165a55c146cc3fac0c61a7072-0001": 4496, + "02-91-500e58f42cb8eee8a04d85e4ec46-0001": 4497, + "02-91-5887430703495510dcdef0e97663-0001": 4498, + "02-91-7b171187c3eb2983ad2dd1ce8543-0001": 4499, + "02-91-9300492b992eefc43fed30ca2cde-0002": 4500, + "02-91-b5468b2e81846eb61868c7771c4c-0001": 4501, + "02-92-7112a03df547f851fc78e8727ff1-0001": 4502, + "02-92-7375b89bd7aa7edb813801d3af7a-0002": 4503, + "02-92-81713cb2600ab7d85c43f5c7078b-0001": 4504, + "02-92-8a334ae16f11ac3c8def73533ace-0001": 4505, + "02-92-956de62fea61a9dd76651bf5e6ca-0001": 4506, + "02-92-9ff284c2fe5c76be1d0b2cf85ae9-0001": 4507, + "02-92-a12615657ec0abc779d1f66bea43-0004": 4508, + "02-92-a3a1b5367dc2e11e3376e3fb674d-0002": 4509, + "02-92-da81a23e880e2ded5bb96229d26e-0001": 4510, + "02-93-008c7722ad2e31b53171a353ff57-0002": 4511, + "02-93-0e4e99074bed87140c8ee4d297da-0002": 4512, + "02-93-271cd233d8497f7e223f56e6174f-0001": 4513, + "02-93-36380bc57de623af6875d850f71c-0001": 4514, + "02-93-36cc96cd0fc09753153af330a26c-0001": 4515, + "02-93-59d70ff15f8f83d0f75c2193bff9-0002": 4516, + "02-93-5c4f13acda2f413ee1dd1c4e2083-0002": 4517, + "02-93-6c5d1bf1a617e58c7afb0feb419c-0002": 4518, + "02-93-6d0205a55b2d84de0d0faa347628-0001": 4519, + "02-93-6eb17c5901463b89f2aa827939cd-0001": 4520, + "02-93-7a6b403784b615f63d822ce6f9a3-0002": 4521, + "02-93-7d41da4c6da3ba9ab4f8060198b9-0002": 4522, + "02-93-9e68d31fc4b80f96e18976b67808-0002": 4523, + "02-93-cd12d34622163c8ba548db471bca-0001": 4524, + "02-93-cd58fff16befaf4cceafaf68e5aa-0001": 4525, + "02-93-e6c03ac87312221a9ba3a85000cf-0002": 4526, + "02-94-17880b4cfd283e7a6fcebd5d5135-0001": 4527, + "02-94-4b6e5438a857018347460db442c2-0001": 4528, + "02-94-5c3859e905486d1b51ae8a3b760c-0001": 4529, + "02-94-8727b741b4acd329c078814b0c5b-0001": 4530, + "02-94-960b9f31b9eefa203982eb3b7d8f-0001": 4531, + "02-94-c8ec26751af8e0a8c9bea7b9b770-0002": 4532, + "02-94-de5b0e4af77a65ca2ed4508c1a4c-0001": 4533, + "02-95-026cb82a06f048c9e5db5efc907f-0004": 4534, + "02-95-0ae85626ae9593962d23b6dda76e-0002": 4535, + "02-95-16e09ced05cccf27e619459d5a0c-0002": 4536, + "02-95-4cc99debd70085601bf08855c85c-0001": 4537, + "02-95-4f6e88f94ac3b69c864919c39c7f-0003": 4538, + "02-95-7430a97f688ae730157c71b2782a-0001": 4539, + "02-95-83b0ed57191956e27b3cbc53fc1a-0003": 4540, + "02-95-83cbf47c76f9b8c9f5117e111d36-0001": 4541, + "02-95-88d00a3cdbda0193415ef4fafeb8-0003": 4542, + "02-95-d685f9ef2ed6b1828d5cb89e889f-0001": 4543, + "02-96-05da309ba6086d7cd7ccea8a8d74-0001": 4544, + "02-96-3290b4cf13f64ea0f2eb3d13a828-0004": 4545, + "02-96-343b706f69f946fa612073b4b729-0002": 4546, + "02-96-4030ee1f73368c3e5f7b490082e5-0001": 4547, + "02-96-4379f1b7d17f31a6e88f2cc3a0bb-0002": 4548, + "02-96-53b4b5e7e7478dc88fc468816dfd-0002": 4549, + "02-96-5d32eddf62eeb2a7a693584a8135-0002": 4550, + "02-96-88cbce1007fb300851adb26470ab-0001": 4551, + "02-96-94247091376614277f94834996df-0002": 4552, + "02-96-a5fefe75904e8a3bd5f893337ad2-0001": 4553, + "02-96-ca865599dbc73ce01862a188b4b4-0001": 4554, + "02-96-f0f4fa12495bf8addfc81b7957e5-0001": 4555, + "02-97-04f28e2f00cbf331e87aea0698c3-0001": 4556, + "02-97-2fa7ed56f9f18a0c66c9bca2b6f9-0001": 4557, + "02-97-46127adae9010bc274e4d9131871-0002": 4558, + "02-97-581732c0e40ef93593d2b0f7e95c-0001": 4559, + "02-97-6fcdca52b04210d1cf8260c6c14f-0002": 4560, + "02-97-708daac481869a1b8b44d0ce8f4c-0001": 4561, + "02-97-82368d5cea637b199cb759ebaae3-0001": 4562, + "02-97-a790fa439a6174cdf1baad6af55f-0001": 4563, + "02-97-b153f6283d10a6415a80061e855b-0001": 4564, + "02-97-b2600e45ade09bad20eda3ab8e98-0003": 4565, + "02-97-c3110a7413085e78ca4694e5713e-0001": 4566, + "02-97-eb80151cab3f22283fb0bec5e111-0001": 4567, + "02-98-0464618955d827a69f7ca8b77c10-0001": 4568, + "02-98-26bf3c6b5a7bdda8e427228f875c-0001": 4569, + "02-98-30803eef496ac4d5275cba96a84e-0001": 4570, + "02-98-41eff037c6a84aba022f1d224c8d-0002": 4571, + "02-98-5c0b7a0627345cfc1fb2422553bf-0001": 4572, + "02-98-69ee470572e6290bffd60a6a0c06-0001": 4573, + "02-98-6b3a6660410198d678311cc4f52a-0001": 4574, + "02-98-75c27fafee1d208705fb637d821e-0001": 4575, + "02-98-770bce07bd36ab53817f2b9d8f80-0001": 4576, + "02-98-bbc0fae1af1e246c64df38419308-0003": 4577, + "02-98-c18b171f8989c09e364926ac1187-0002": 4578, + "02-98-f9cd7091d2c1604e2cc4bed0ed86-0003": 4579, + "02-99-648577fb8e6c820107b288c0bbfb-0001": 4580, + "02-99-89373e6482b5225900e017128bc5-0002": 4581, + "02-99-9ae36de6ec80e37390294cf4bb04-0002": 4582, + "02-99-a265558f68f590076606e86ead33-0002": 4583, + "02-99-a43ecfa0f6b08281c2687b0f01ed-0001": 4584, + "02-99-a5b6af68fbbd7a75b29396849639-0001": 4585, + "02-99-d2303227910769da777c95d453a2-0001": 4586, + "02-9a-1803c102c9a3c2494b050c1ab425-0001": 4587, + "02-9a-8b8a1b3324ac0ab603bd130725a6-0001": 4588, + "02-9a-a36543d55495a534026dac726a6f-0001": 4589, + "02-9a-a6872b05473c81b4e31a5c2b69e6-0001": 4590, + "02-9b-0d4f947002673f5c266a16c13b8a-0004": 4591, + "02-9b-47cf0035f905220c1c4490a0f3fc-0001": 4592, + "02-9b-4e1335b2b3d8ccd06fe623b17275-0001": 4593, + "02-9b-9c48e8bacced10b2092e7aed6473-0001": 4594, + "02-9b-a1aad27f0b218e1594016124333f-0002": 4595, + "02-9b-ca731730fc792c3bb437b0ccdde1-0001": 4596, + "02-9b-d4906b083e6b9c4a6b9bdbb59934-0002": 4597, + "02-9b-f54cd2ef1efc43ee6c29d486d3f6-0001": 4598, + "02-9c-04dd7b01a4f3a3b1c07628f04c0a-0001": 4599, + "02-9c-0e97ed5d6f5e8b00d1b16488d5aa-0003": 4600, + "02-9c-824114fca787c7fd8b8186f404f0-0003": 4601, + "02-9c-c218749a2588d8dc38848d861621-0004": 4602, + "02-9d-104d1c172083081b8b6205a42d42-0001": 4603, + "02-9d-29d27f9c1e17bb2f0b6dbe040758-0002": 4604, + "02-9d-4f7e5dca7ded32ef87152c1f2d85-0001": 4605, + "02-9d-96cd0e83d36a0e696e08cf0565a4-0002": 4606, + "02-9d-b383248473a8007c147ece570df3-0002": 4607, + "02-9d-b8e2ba875667c60445c016fc2fa5-0002": 4608, + "02-9d-d7b3e390a1982660e76291526b9a-0006": 4609, + "02-9d-f2f75f70ca3082621986c01f4470-0001": 4610, + "02-9e-1e52be6c014415ce22bd97f94f24-0001": 4611, + "02-9e-398b0fcd5ab853733b560cd09c30-0001": 4612, + "02-9e-4c4053d931cd41d2e84c1b0a98bb-0002": 4613, + "02-9e-56b273cd23c6be879ed8fa908302-0003": 4614, + "02-9e-651af5bab464b23583f5d9682758-0002": 4615, + "02-9e-7c9e1397ed17ac85ae8ff6a3a05e-0002": 4616, + "02-9e-8613f7fd5a0d41f2c3d2b50eac7f-0002": 4617, + "02-9e-ac1a16356b4ff306ef86f4511d9a-0002": 4618, + "02-9e-bbf07ce2c0870ad870e777ed283c-0001": 4619, + "02-9e-bdadef5237ef27700d739fb54639-0002": 4620, + "02-9e-c07f2914a339ce7ea73eca3ef3c5-0001": 4621, + "02-9e-cd31d3c48d96371548e4cdc5abbd-0002": 4622, + "02-9e-d3c5e094ab2ffeefc30d374395b1-0001": 4623, + "02-9e-e9573e45a7b009630ccfcdbec339-0001": 4624, + "02-9f-0190ba41c463bebd9b45140c2148-0001": 4625, + "02-9f-020568d3af922aa689f15733b058-0002": 4626, + "02-9f-23e5bc8daefcad80defac1ec803e-0001": 4627, + "02-9f-2e85e3e90de74317dda391723212-0002": 4628, + "02-9f-5f9013c92717c7198caaa893619a-0001": 4629, + "02-9f-95c4e2d2daf116734fcfa7ece28f-0002": 4630, + "02-9f-a327c2ff796b27f358f88731dbd2-0001": 4631, + "02-a0-00c9ff89eeddc4d1ad9481c7a58a-0002": 4632, + "02-a0-1a4bfa6fc34129067838eb99ca5c-0001": 4633, + "02-a0-4b377937f896606211a144b5e047-0001": 4634, + "02-a0-4f5471a6980fecb206bf907d80d5-0002": 4635, + "02-a0-949611a617e1c0f7ba8e95946b9b-0001": 4636, + "02-a0-b49433ffad1cf2e9de94244d6415-0002": 4637, + "02-a1-09784ac971a4628ff3430fe8401f-0001": 4638, + "02-a1-6b52ed0e6daa7a91d65dbc758f6e-0001": 4639, + "02-a1-a0cd4d83417b3b138a8a72bd3f2a-0001": 4640, + "02-a1-b33a60b23415f7af5095bb5e72cf-0001": 4641, + "02-a1-d9ed2280285ab5a8d1dc781070dd-0002": 4642, + "02-a2-14d13936835d82781401a7da57f6-0002": 4643, + "02-a2-1d194b443fd3800d06fbd7951ca2-0001": 4644, + "02-a2-2745e5e5b0b1daa9391f743dc4a3-0002": 4645, + "02-a2-7e810a0caf19e22fc61c48a91079-0001": 4646, + "02-a2-8d025ea8befe18775bb94d735749-0001": 4647, + "02-a2-914877685c868dcd2bc561e68f2f-0001": 4648, + "02-a2-9702c8c71b785182d912b215c977-0002": 4649, + "02-a2-ccbfcd5a5734a1eb27df853b1dfe-0001": 4650, + "02-a2-d171a14e53767ca5ffed8836ca77-0002": 4651, + "02-a2-dddab0452a2149385352124810a2-0001": 4652, + "02-a2-e92a845da4395a43919160c08d97-0002": 4653, + "02-a3-351a0839361054e807c680822da2-0001": 4654, + "02-a3-356a0422b5975bebdb58b8bd26d3-0002": 4655, + "02-a3-3fdf9da036c2723a5898cfa3dcb7-0001": 4656, + "02-a3-5e664a6da8c48ba9328664bbd7db-0001": 4657, + "02-a3-5f07ab2623b89564e9432b5454a0-0002": 4658, + "02-a3-6a9b1252be5ee4a0c1da9a473aaa-0002": 4659, + "02-a3-76017cb9cce79b2ad353f7054a48-0001": 4660, + "02-a3-8688e4b0dade2325e574e31d0f1a-0001": 4661, + "02-a3-a36e13d5330f99b06961880e5bcb-0002": 4662, + "02-a3-a5eaf88875b133951ef81797b118-0001": 4663, + "02-a3-dfe7aba6f97616cd4c51c887b523-0001": 4664, + "02-a4-060f5595bbba91a53cf7a8e73924-0001": 4665, + "02-a4-1e2f2a6b8ed632d28d9f3c3e3445-0001": 4666, + "02-a4-288e0b252083ecb9884caddf57eb-0002": 4667, + "02-a4-3acff2aa57d2432beaa5ee8a09a5-0003": 4668, + "02-a4-43aa491b5f5a37d75e5d41bc7531-0001": 4669, + "02-a4-59a3b17bdeb4c6b5632c8a6670b8-0001": 4670, + "02-a4-5f1e178900d745fcd43db6449e67-0002": 4671, + "02-a4-770f7b653bf8185c0747f04ca03c-0001": 4672, + "02-a5-1da88b0ffaaa4f262a24a0211f2c-0001": 4673, + "02-a5-6046c48f2efb9777b9ca3e6c824d-0001": 4674, + "02-a5-cf37be7f23d60056a5fe05cb82b3-0001": 4675, + "02-a6-1ff0e096e79cb8c0b910168341f2-0002": 4676, + "02-a6-49f5065ec72b156bed535f16ffee-0002": 4677, + "02-a6-5cc368711d6880b8b46d37121487-0001": 4678, + "02-a6-e0949e37edcaadb5c58866bf99fc-0002": 4679, + "02-a6-f8b99326a4722d7befeb1a08b663-0001": 4680, + "02-a6-f937aecfd1cf8be2a57bdbc8647b-0001": 4681, + "02-a6-fc8d63c012caf071af266acf5fa7-0001": 4682, + "02-a6-fecd8efeccbc72180699b52fde20-0001": 4683, + "02-a7-04d47d7b12027ccc2c0f368fd25f-0001": 4684, + "02-a7-05b6f0c62526ea58a1550b2c531c-0001": 4685, + "02-a7-0874f06856cba75e9b1ba69c39fd-0002": 4686, + "02-a7-08b45f3fcae1ff3d97d10d58f174-0001": 4687, + "02-a7-1a45d7750947c6c565db8c913ca8-0002": 4688, + "02-a7-2b3b79889233676c7d285aaca169-0001": 4689, + "02-a7-3ccf9f6fbe48188647a50a41b4e8-0001": 4690, + "02-a7-b7d97dd721b642943021a8b85894-0001": 4691, + "02-a7-c4c7ed2c12c564e4b0e13c14f514-0002": 4692, + "02-a8-2d61a19a36a8f13a30e73a68c77e-0002": 4693, + "02-a8-30a7acc9f17bd0ed19da4b763309-0002": 4694, + "02-a8-3322523f4cd7ca240df68ae2f5f2-0001": 4695, + "02-a8-4876080392e2540c314912973493-0001": 4696, + "02-a8-6dc2aea59da10b00f07f0525941d-0001": 4697, + "02-a8-b5b531863b482f85f654a67a1264-0001": 4698, + "02-a8-f9f3ce991095a921f4abfbde359c-0001": 4699, + "02-a9-388affc6449b848eabd8116204f7-0001": 4700, + "02-a9-3ea77c67304f46457afe301c1652-0003": 4701, + "02-a9-4fd51f7b94834296002dc67de201-0001": 4702, + "02-a9-6472c1de803614442cfc3bc75df9-0002": 4703, + "02-a9-a17e32819152171c5320c49e225f-0001": 4704, + "02-a9-a72c8cb2a68e4f304d31a1c4e540-0002": 4705, + "02-a9-dac4bd37e2356c140021d7e5cc8b-0002": 4706, + "02-a9-dfe97190dd4664b6e8621b723188-0002": 4707, + "02-aa-17da9d487c7f902703545b15a40b-0002": 4708, + "02-aa-4be8b885acb2ea1082632e057960-0004": 4709, + "02-aa-5482cfe5a6d4d13225fe7935c038-0001": 4710, + "02-aa-6c86f8087459d553702145916079-0002": 4711, + "02-aa-8c60af5af4ebc102de26c62e476b-0002": 4712, + "02-aa-b4985aa1b514ecd875d28b0b419f-0001": 4713, + "02-ab-387f594b8f5a4e8fd212f951c570-0001": 4714, + "02-ab-95bb1b858633a8d7f8d394c5166d-0001": 4715, + "02-ab-ca42326638c6ebab776478ca3ec3-0003": 4716, + "02-ab-d28eec83b112bbd5c09a72896910-0001": 4717, + "02-ab-ed99ff52974540b0e8015ebb01b3-0001": 4718, + "02-ac-406d6750a478b87c62a6738a0f6f-0001": 4719, + "02-ac-5758b71dadb87e73ae6089d28c8a-0002": 4720, + "02-ac-625e09ae39b09739fc3fe0a4374d-0001": 4721, + "02-ac-7e52220f702455c6549deba8f60d-0001": 4722, + "02-ac-89d677c019eed2515dd7cca061ca-0002": 4723, + "02-ac-bbbda4e0d81fe04ca726bb7a464a-0001": 4724, + "02-ac-d1a34caef575fcca70cb9218175f-0004": 4725, + "02-ad-4dde0fd6468fc5cf3d35326e295d-0002": 4726, + "02-ad-5af832d63194c0053799a37ecb71-0001": 4727, + "02-ad-6586722f4b37927ec8415659f6a2-0002": 4728, + "02-ad-a7a306a37922a63485fa2d1d4452-0001": 4729, + "02-ad-c544fec6cd9ad21be60542a79ee7-0002": 4730, + "02-ad-d8a7f2a32c66e699a42f864d0641-0001": 4731, + "02-ad-f7418c6f7804f27b4e9fa45f449e-0002": 4732, + "02-ae-27300800a06dddf5ce4070b20669-0001": 4733, + "02-ae-2f2de88ac9e1918f0bac659b4c16-0003": 4734, + "02-ae-37c26830c58a44c6b22a01957647-0004": 4735, + "02-ae-3a399da4797319a64036cbe250b3-0002": 4736, + "02-ae-475bde22cfd7fc3f5cd72bcb9ce1-0001": 4737, + "02-ae-525ca7f875a0844e994c672ddbdd-0001": 4738, + "02-ae-62b67a75ec488942c40f4d2fb200-0001": 4739, + "02-ae-6dd55ec0315f135b290a8b37d2f0-0002": 4740, + "02-ae-70ddd87445209b2c21363373ca70-0001": 4741, + "02-ae-8a264362073b0a01deb75dd5d026-0002": 4742, + "02-af-62d12b668a7393667ab90d69e526-0004": 4743, + "02-af-75fbfd60740dae33911d379c1bb7-0006": 4744, + "02-af-950c325acde8084f0f1da70747f5-0002": 4745, + "02-af-e7291dab4443cf6b43314a296d4e-0001": 4746, + "02-b0-1aa16b5d8ccd3dba8d584d4b5924-0001": 4747, + "02-b0-9041fd3efdaf2e7bdda919582481-0002": 4748, + "02-b0-904d6138937b5ac3a88b2da631b2-0001": 4749, + "02-b0-94d218131b5b23d3e31aa125de2c-0002": 4750, + "02-b0-98a94c7fdfaf7810a1a9f267db10-0002": 4751, + "02-b0-e73f96585177d38f9ba9dd4e34f0-0001": 4752, + "02-b1-0eecbe47d847b326d13029439c18-0001": 4753, + "02-b1-6d80cb373a34a1597c48fc351e52-0001": 4754, + "02-b1-9ca7aabb2c5408af9e42ec60fc69-0001": 4755, + "02-b1-ba75e1bf8806d6b4007136986ce5-0001": 4756, + "02-b1-c70fe9641c8bae3db1ad9aaf035f-0001": 4757, + "02-b1-d7ea1c0aebab8ea2a69e4baa53b4-0001": 4758, + "02-b2-241874e1118bb242be0d983df5bc-0001": 4759, + "02-b2-8207d6fd49eb0d1d11b6439b5e83-0002": 4760, + "02-b2-c21787e49f590f5e6290e3a4c777-0001": 4761, + "02-b2-db1439a5390598ded3b60ad67093-0001": 4762, + "02-b3-1c846a30eb46fb5ef3352f486b2f-0002": 4763, + "02-b3-49e794bcaf7471cdd7419cfd04f8-0001": 4764, + "02-b3-5222892588aaf8949ffd8df4862c-0002": 4765, + "02-b3-611857e15da8d51320cccdfeb556-0002": 4766, + "02-b3-8b17bc3b4d850062fe12cac94ed4-0002": 4767, + "02-b3-b2b1a84f75ce2bb46cc2917a1e45-0001": 4768, + "02-b3-b844ef8123afe11570db2c515bcf-0001": 4769, + "02-b4-99142414d7874fcc28b8060dbcff-0002": 4770, + "02-b4-a53da2b6d9c7fd8d5a67bccc0686-0001": 4771, + "02-b4-c342ebbf72902a291a9b61569932-0002": 4772, + "02-b4-c74f8e0759b83fc83ce79f6e1a3b-0001": 4773, + "02-b4-e0640ae8f9d43d1b620779cee362-0002": 4774, + "02-b4-fb2a688e5746a50b95b446c93629-0003": 4775, + "02-b4-fff5e9151719f1603a78fe146972-0004": 4776, + "02-b5-044e9e2204482b11d7da23058a79-0002": 4777, + "02-b5-3336fc794d999ab918a82af0afb9-0001": 4778, + "02-b5-4e0ef59dd77b104d15435c28e614-0001": 4779, + "02-b5-bef8f320ac0909809961f86092e9-0001": 4780, + "02-b5-ca564399eb27c9ea622dd6dc6d7a-0001": 4781, + "02-b5-ceb4cca26ac94aec566a73e49233-0002": 4782, + "02-b5-f4fcdaab05e3aa999ea6a461a6c7-0001": 4783, + "02-b6-1d59d14b639b4443614c506f7dcb-0001": 4784, + "02-b6-350813ec3131958b5821f1882519-0001": 4785, + "02-b6-4c356fab7fe4247e00e557fe32bd-0002": 4786, + "02-b6-b4fee7b677bcbadc3ce83f9b61cd-0001": 4787, + "02-b6-c2e90586b5b3812327e63dacb948-0002": 4788, + "02-b6-d69d29f79ecad86dd09ca7794102-0001": 4789, + "02-b6-e42cbf20ce7b782d54ef7a7ed2db-0001": 4790, + "02-b6-e7a568fb5af374e2e28fa2096f9f-0001": 4791, + "02-b7-1d5b628b16b5674fff0bfd65fdad-0003": 4792, + "02-b7-3d48d74a1a569fe3b4c9afeed3ea-0001": 4793, + "02-b7-4608a5fe9eaa7433d7eb3f873f27-0003": 4794, + "02-b7-5be11d435a7f91a1b6d3b9f2bc38-0001": 4795, + "02-b7-5cfc0586de3b99f33ac04fb17404-0001": 4796, + "02-b7-606a5c6f1c9b2d1b76485f4362c0-0001": 4797, + "02-b7-8504b65ad947926ec527fcf01454-0002": 4798, + "02-b7-91aef172777da8dae371e9e0e15f-0001": 4799, + "02-b7-99595c53e9f42012fa79284f21c6-0002": 4800, + "02-b7-ae4ec32b7509b04c8e4732e371ce-0001": 4801, + "02-b8-52ac2940ff4a8414a38040f59f85-0001": 4802, + "02-b8-692045243945b1ca5c688ecd7b14-0001": 4803, + "02-b8-b78cebede27466c8d36ff66685df-0001": 4804, + "02-b8-f4e5e5c16535cf3b35da86c27b0a-0001": 4805, + "02-b8-f514f8652287ae8701736de40399-0001": 4806, + "02-b9-11c4b008e2f119086b18cb59d74b-0001": 4807, + "02-b9-478436fb4a6f47d2285b24ba4a22-0004": 4808, + "02-b9-78a35d723e91d77537d11a4ff695-0002": 4809, + "02-b9-836c965f0042ada3a61f7a736204-0001": 4810, + "02-b9-afe0dd3f94c459f061727fb4e167-0004": 4811, + "02-b9-b20e00bdbd104229baf978dc3437-0003": 4812, + "02-b9-bfe0dec668506491466bec5db684-0002": 4813, + "02-ba-8a1183b32c340b6e52881e347ce0-0001": 4814, + "02-ba-924e079aed4a94f10ec896d72d40-0001": 4815, + "02-ba-9b333de20401a1fdc3fabc939785-0001": 4816, + "02-ba-a347a341ed4ad6c1d6a23b3fd55d-0003": 4817, + "02-ba-a3a8124eb49521a628fc5a3d8cb4-0003": 4818, + "02-bb-9336c67ebd693b2843f11ecabc70-0001": 4819, + "02-bb-b57375869de87883d9bbe8cf3d6a-0002": 4820, + "02-bc-0728adb13a9cce46dd0adadc1c95-0001": 4821, + "02-bc-0e8f91f5dd6d5c70cfb83edea182-0001": 4822, + "02-bc-14d6d01bff1a83141b71bb7d9ce8-0001": 4823, + "02-bc-539401b75a75e96dde68beacb6b6-0002": 4824, + "02-bc-68b4b32ab50c5795da275c95a8a9-0002": 4825, + "02-bc-833054f09c5d1b4d590633b8033d-0001": 4826, + "02-bc-9104596fcd7c3b1109f981d09bf9-0003": 4827, + "02-bc-dc56e7e1d1141ea35749bdef0117-0001": 4828, + "02-bd-3f4631dd19188f3399f5f50369c2-0001": 4829, + "02-bd-43214dfecdb87c3d697d7f7a994c-0004": 4830, + "02-bd-6b3e719fec9b7f0d8c973d4dcf0a-0003": 4831, + "02-bd-7f1308bc4013c8e0746068063389-0001": 4832, + "02-bd-db81a68341a68e93011a5e020866-0001": 4833, + "02-be-04126046126a5d6581a0b34a1add-0001": 4834, + "02-be-24451f576f8874027c65b7114d66-0001": 4835, + "02-be-32ea18fa51dbc9684b28e6eb9323-0001": 4836, + "02-be-488e616b3ff26fe7d76f8714a977-0001": 4837, + "02-be-96a21ad6ad450703718353618f05-0002": 4838, + "02-be-bba4d0ae414f607f48d13c3a9c5b-0001": 4839, + "02-be-c23ad6c5dbd9f664068cb3b59c57-0001": 4840, + "02-be-f69621059c7bab27f1b8db856dd2-0002": 4841, + "02-bf-00c6007946fb10c6315b6751d3f1-0001": 4842, + "02-bf-15fbafa89ec35a6a0472c18ac51c-0001": 4843, + "02-bf-1929b33ae1786705795a13a3ee18-0001": 4844, + "02-bf-2e889b47db4a2ffc5894ab3226a3-0002": 4845, + "02-bf-3aeb375e8b36e8b519db7cc04b79-0002": 4846, + "02-bf-3ec4dfe1de7f9872e8c13c22d15f-0003": 4847, + "02-bf-56310b5dba626b13045118eff3d7-0002": 4848, + "02-bf-808313b1738f87cfee4061427ff2-0003": 4849, + "02-bf-8ae987b5d45739309e27f61c155a-0001": 4850, + "02-bf-8e1bf4f157a833cff896e5a32e59-0001": 4851, + "02-bf-98cdadd75bb6e47c9b8add03d8fd-0001": 4852, + "02-bf-9b4201caae7de02aff8288147d3d-0002": 4853, + "02-bf-d9eca6ae627a81f558874b02a2b9-0003": 4854, + "02-bf-f06a0d1facae500b822efd915b5d-0003": 4855, + "02-bf-fe5c1cfe425a45fb06a7b8efa05c-0001": 4856, + "02-bf-fef1f827c2624d138232ebe11251-0002": 4857, + "02-c0-29b7bd812890208fc861527124a3-0001": 4858, + "02-c0-8c65ea597d98ccae5564ae133b4f-0001": 4859, + "02-c0-9b8e752b9e1943510cfa009e6784-0001": 4860, + "02-c0-a3c3100ee9d49d16f2bf3c3ad3d8-0002": 4861, + "02-c0-b890112bc8ddd06b68d2274fae6a-0001": 4862, + "02-c0-c49b2febe2159effd217d4c45881-0002": 4863, + "02-c0-d6e610bc21fb6509357b665d8262-0002": 4864, + "02-c0-ec668d1b053b7eb946f6d737fd7e-0001": 4865, + "02-c0-f993973a9b4e31cb7f555ab1cf4b-0001": 4866, + "02-c1-03d3e5f4fc1e74c2419c0a04ecd8-0002": 4867, + "02-c1-0dfa6425e21ce4dc09789380441c-0003": 4868, + "02-c1-4838f9344ed45cacd0d7e062f78d-0004": 4869, + "02-c1-8c7296c56c7a2fe4503ca6b6f1ac-0002": 4870, + "02-c1-bd6e0cc67295c0f9cce747be330d-0001": 4871, + "02-c1-d099f9a34eb5019b97e38e799211-0001": 4872, + "02-c1-d362b1e883b34775f20d4d1cdd86-0002": 4873, + "02-c1-ed3c4a7ff612f55c311dd826fab6-0001": 4874, + "02-c1-f7813a81abc016d54878e64c62db-0002": 4875, + "02-c2-2fdb6ca0b46c264ab726c8f65798-0003": 4876, + "02-c2-3b72c64155c9c22d182d954ebe40-0001": 4877, + "02-c2-3e7b5e73287a852674d86e8208f4-0001": 4878, + "02-c2-594824a1856404ba0f57d0fe917e-0001": 4879, + "02-c2-5f7be9999ec510327ffe21bd6b00-0003": 4880, + "02-c2-64d03f0ee35717cb063cd47f3517-0001": 4881, + "02-c2-69f804f09c8ac63ff22178de7e11-0001": 4882, + "02-c2-72db4e34df5a5482bef407b2815b-0002": 4883, + "02-c2-75368ea1c10c1fe614144f52541a-0003": 4884, + "02-c2-ac1f26ee43854848b72497740536-0001": 4885, + "02-c2-b10c910eb02ea003a329611ea504-0001": 4886, + "02-c2-ba0361bdc2143537abc0eb3f8ccd-0002": 4887, + "02-c2-c46aa58167ce87234d6793c8e5c7-0001": 4888, + "02-c2-d6d42044087c14d8cd98632f5100-0002": 4889, + "02-c2-da82c584538b84d49441651c4dbf-0001": 4890, + "02-c2-db70fdfb972abdd4ce11612c35e9-0001": 4891, + "02-c3-42bfbfdfdc0f1e7f31f55d821788-0001": 4892, + "02-c3-8c7c2034805c7b7c646cc1da146a-0001": 4893, + "02-c3-a909435e61240b041445cbe357f7-0001": 4894, + "02-c3-d2c7793b81db1299c4af4f812e5f-0001": 4895, + "02-c3-ee90da83208071aa02a6e0e7ae8d-0001": 4896, + "02-c4-3aeff34e5e0011c778199a85553a-0001": 4897, + "02-c5-20901b214bb1fac595e1215bef93-0001": 4898, + "02-c5-812c19258e3113ae633433ba59a9-0002": 4899, + "02-c5-8c59ae5552346fa3211c3b2dc6cc-0001": 4900, + "02-c5-98ec02a4966c4fcb4b58f7adba7a-0002": 4901, + "02-c5-b7c2122e3c466938e717e52bdd61-0002": 4902, + "02-c5-e00f2bacc33ca75ff0ae98eca863-0002": 4903, + "02-c6-2980a0b362e07bff1397d4910b6c-0002": 4904, + "02-c6-95c6cd30e237123308e09f81126e-0002": 4905, + "02-c6-a7303b504fc43a53f4551442f068-0001": 4906, + "02-c6-bb5ddd7ca830145b751142d8bc3f-0001": 4907, + "02-c6-d2cba31bd364d7ecc6087ef9270e-0001": 4908, + "02-c6-fb57888c5b91e5cc6620703b03de-0002": 4909, + "02-c7-4948a84afe57744aef62396b2cfc-0002": 4910, + "02-c7-6a0c742d2d7e28a114c3d14a3d5e-0001": 4911, + "02-c7-735b649bcecc54ff68fa3ff36a9d-0001": 4912, + "02-c7-90b21faa3d54e6994aa84afd1c89-0002": 4913, + "02-c7-af4e6dbff5deaa5218021e511015-0002": 4914, + "02-c7-ca2cc91a2e248462d4602550633b-0002": 4915, + "02-c8-223bda56b9cc9e6d7911daf23538-0002": 4916, + "02-c8-41df3374e8dabf2f3c680800463f-0001": 4917, + "02-c8-4df8e93fe4235ae3432763a94def-0002": 4918, + "02-c8-69a346b9bebfa9caf85e49a780c2-0002": 4919, + "02-c8-80ef70ddb69295bc3ff491fb6a16-0002": 4920, + "02-c8-bf0d0b5f74f6b7dff19c97229fec-0001": 4921, + "02-c8-deb85d84106063b9158184eeaf6e-0001": 4922, + "02-c8-df8814e2113ca567cfd3b30b13de-0001": 4923, + "02-c8-e5bb8e6eba8af8841e9ab685819c-0002": 4924, + "02-c8-e6bc163891b2e6ed1a5b56c3b05c-0001": 4925, + "02-c9-0050193118b569bf48010e3b141e-0002": 4926, + "02-c9-74d0795c4594cc91603374379bb9-0001": 4927, + "02-c9-7b61aac569f2a4f3faf875e6adc9-0002": 4928, + "02-c9-9cf7bed848b335ec9ae45e3b2f98-0002": 4929, + "02-c9-d02639cc8c027b0675a0e20ff7a9-0001": 4930, + "02-c9-d585eae48249cb7b2689696aaab5-0001": 4931, + "02-c9-ec4a2e07fda8cd0bac46bf702040-0001": 4932, + "02-c9-ed7b424564ce2e58342f503833a2-0002": 4933, + "02-ca-0646a8a61fbbe59b74e390dda92e-0002": 4934, + "02-ca-0fcefacdc2c2295b043ac2c8fe39-0001": 4935, + "02-ca-294fc96047a85fee6900e39bd784-0001": 4936, + "02-ca-42fac29e4e1d79e38d2930994e20-0001": 4937, + "02-ca-85bf581fb0bd9483fca88a25a983-0001": 4938, + "02-ca-8d19b38063dbb860310b4da4cd73-0001": 4939, + "02-ca-9fc9e0e4be4919a8076851b30a1a-0001": 4940, + "02-ca-aa0e11f961d0ebfa43d037bfda05-0001": 4941, + "02-cb-7f0a46a637ea731a78a210798750-0001": 4942, + "02-cb-8951a8ef1d34cd7200387cf7e6bd-0001": 4943, + "02-cb-9484634495dc6ef63a5862716161-0002": 4944, + "02-cb-a0579fbad429e7f8d66e39a97ead-0001": 4945, + "02-cb-d2de5bc8fb51fff1580787f2e033-0001": 4946, + "02-cb-dfd6afed327e719f8fee1b036e36-0003": 4947, + "02-cc-117569f4248a5b86ae5e5b90eca5-0001": 4948, + "02-cc-3277b82d44f7d330118b5f376f60-0001": 4949, + "02-cc-5e498a0b3e7b29db5944710b5bea-0001": 4950, + "02-cc-647d32427849af279315b78acbd0-0003": 4951, + "02-cc-685f29b48d0c3dc68fceb69cb1d5-0001": 4952, + "02-cc-7c12eb7baea8d03c9406f5afe6b5-0002": 4953, + "02-cc-ace6bf424f3d9c15694e81df8b03-0002": 4954, + "02-cc-ed9e7888ecfa05e991d4e1b28ea9-0001": 4955, + "02-cc-f0b413ebc70718d942cfe81086cd-0006": 4956, + "02-cd-09ba8ab5d2ebf0c7d8ef2a0059dc-0001": 4957, + "02-cd-20f4ebc936587b6550a48600caeb-0001": 4958, + "02-cd-785fe61dc72cb1990314d54f7881-0001": 4959, + "02-cd-99c169693ee8f001e668a6fb2d93-0003": 4960, + "02-cd-b5d71f008fca26e718ceae830f4b-0001": 4961, + "02-cd-d4053be4606e9a9e1c2d8f3c408d-0002": 4962, + "02-ce-0022b946876b657757b9806ae8f5-0001": 4963, + "02-ce-3ce0789b9f900d03b1a15ffebbaf-0001": 4964, + "02-ce-53e8356a44ca8c097f15c61f5151-0001": 4965, + "02-ce-a969982f66b44006ac6c8c80fc68-0001": 4966, + "02-ce-bbd1839fe84f617ef569fabd2605-0002": 4967, + "02-cf-28afd9e930ccbcfc80a61dff54a5-0003": 4968, + "02-cf-4b34402549364586f7c1cef57e61-0001": 4969, + "02-cf-7dd1180d616521d5acb4315307fd-0001": 4970, + "02-cf-92d475b972ab9b4aa8ba6f9002f3-0001": 4971, + "02-cf-9e0ff16d7f4ed97078df4885c180-0001": 4972, + "02-cf-abd8d19039734105e7d9a8f253b3-0002": 4973, + "02-d0-0fb373ddf77143f0c30ac45533c3-0002": 4974, + "02-d0-2bdf314bc4d6e1844023293993d3-0001": 4975, + "02-d0-359999325e2f5a1fb98f8d9d38fd-0003": 4976, + "02-d0-38c0b21e2b9617455c54a6fa502c-0001": 4977, + "02-d0-45ecf56829b1797e3a2be5fa8eff-0001": 4978, + "02-d0-4f7358cac198215d6fb60a275b52-0001": 4979, + "02-d0-8585322481fe5e7756bd8967531e-0001": 4980, + "02-d1-0990664b561f8635312c98634ad2-0001": 4981, + "02-d1-2fb0a60613d09dd575f4446300d6-0001": 4982, + "02-d1-44dc660e4a27618ef20420b5b75d-0002": 4983, + "02-d1-b9b6d36629416a1dc1ed875eecdf-0001": 4984, + "02-d1-e0f6f070337c50023ad8f3c16681-0002": 4985, + "02-d2-1e23a37b1bc3f3ea7c71a8d5fef6-0001": 4986, + "02-d2-573b78cb94418ffd44c9a46afe57-0002": 4987, + "02-d2-710fa611edc4409a2d289713c9ee-0004": 4988, + "02-d2-7cf190461777e529690adfea70d9-0002": 4989, + "02-d2-845ca17145d93c55a5d0445fb8bc-0002": 4990, + "02-d2-9643c7b7bf1aec7d3db1a4e7aadd-0002": 4991, + "02-d2-e547260e641effe4ca59cc29320f-0002": 4992, + "02-d2-e610f51cf7fdba0f61b603a25930-0001": 4993, + "02-d3-0c4d266fa47aa597820b8547f0a1-0001": 4994, + "02-d3-13ba08934c1d788c023c16d9f5d5-0002": 4995, + "02-d3-5d81d83f5417afce6cfe9f9c4b2e-0003": 4996, + "02-d3-5dbc05d79c0ebc8ff857cc8dbc7a-0001": 4997, + "02-d3-7289b86738cf8c9f6c215ec83bca-0001": 4998, + "02-d3-73a5e01c6968094db76629d8fccf-0001": 4999, + "02-d3-9eb9e858a5404289e6e4d9198d08-0001": 5000, + "02-d3-a2b400f757f23bbf2df2a9510192-0002": 5001, + "02-d3-cd32caadfd8cee2ea7fdf6a4cf4b-0002": 5002, + "02-d3-e8a816af401cf6e3d5119a5d6159-0001": 5003, + "02-d3-edea9a4e4a41f28b07a94345882b-0001": 5004, + "02-d4-1d0094c930418545eeb0d16ef465-0002": 5005, + "02-d4-60b18f616bdaafe6fbaad613c27b-0001": 5006, + "02-d5-51144d7decc32989c9a6f1ce0455-0001": 5007, + "02-d5-6666248acb63a2b2a37dba110860-0001": 5008, + "02-d5-d2185181c0deb6f39c7c778342f6-0001": 5009, + "02-d6-62785e61ac797cf0c9cb8868a849-0001": 5010, + "02-d6-688cc6ba1bfe86fa8d11ae5a5a85-0002": 5011, + "02-d6-72b13ee924ac727cc1f400aef78f-0001": 5012, + "02-d6-b1a5321c68cde54087cfc0325774-0003": 5013, + "02-d6-b3165035b1ffdff87042bd2f943e-0001": 5014, + "02-d7-1668445be5f03bd47843d864b6d0-0001": 5015, + "02-d7-20015e45b38523e20823ef4b6195-0001": 5016, + "02-d7-70ae0d627de596e525b1028beddb-0001": 5017, + "02-d7-cc2e0e12711cd5cd7dbde328a07e-0001": 5018, + "02-d7-df93a9c7cf0221da6c3d8ab373b9-0004": 5019, + "02-d7-dfaaa5ebc853a9ed2160658e16f9-0001": 5020, + "02-d8-255147c92fedebe266c2ba8fd7db-0001": 5021, + "02-d8-29370adea243ce6c23ac37f152d6-0001": 5022, + "02-d8-4921ec76a75e8a51dff1610e9002-0004": 5023, + "02-d8-7171769d995fd283abe52974a580-0001": 5024, + "02-d8-746353ba4fe89975fee1237c1480-0001": 5025, + "02-d8-ccd120d90d0e4906bec4fcf918b3-0001": 5026, + "02-d8-d4392e8b6c6b8003dc6a6eaa49d8-0002": 5027, + "02-d8-ec430fabef5b3696246c05adf84f-0001": 5028, + "02-d8-f096b4bee5d35501183b013fe35d-0001": 5029, + "02-d9-0a31cd4f002e3dd4c8e0bb887e6a-0001": 5030, + "02-d9-1292983bd3c9807618d9a624e997-0002": 5031, + "02-d9-14bbfe5f8e837bd60a1997af6f32-0002": 5032, + "02-d9-1b22909c7edc3bb013110661ae7d-0002": 5033, + "02-d9-54974f39b877e512b47788ef65f6-0003": 5034, + "02-d9-683b6156fc41e02cab626b7ec9ff-0001": 5035, + "02-d9-6ecce4408adededb642d4b30761b-0001": 5036, + "02-d9-84d5f06a78b6f5bc269788ea7d51-0002": 5037, + "02-d9-994bd9ad6cb01896ad1072302ab5-0001": 5038, + "02-d9-9e2f213fa7339e09d8a3fe752e50-0001": 5039, + "02-d9-ce1fd2993d577b76152ea748b2ef-0004": 5040, + "02-d9-cfdf4a017f001f9270b1d2feba68-0001": 5041, + "02-d9-fa142d78dc0d4d62e841b54a0b45-0002": 5042, + "02-da-00997190254937953317ba0234b9-0001": 5043, + "02-da-07460a96d6cca5f3b8d59ecaed71-0001": 5044, + "02-da-419bf84bfaa10ee5defab074d8ec-0002": 5045, + "02-da-832bca56dc5176129a3b6b380337-0001": 5046, + "02-da-8b1b2f862fae46392fa86714c071-0002": 5047, + "02-da-aab7c12ad9164d553353bef8dd96-0004": 5048, + "02-da-b9124707cdf9e38f5b63ec56a570-0001": 5049, + "02-da-be0a97640682bcc3f320b4af4b25-0001": 5050, + "02-da-c9e964804bd7ac31e5d498a940e8-0001": 5051, + "02-da-fa6f82b3ce3884773576f6c25d99-0001": 5052, + "02-da-fdb775bdc6aff24c58261e453079-0001": 5053, + "02-db-5f355db3ba2620ab1a5c6b60bc4b-0001": 5054, + "02-db-9748d373e5e8e6a46dddd04c1878-0001": 5055, + "02-db-ccc82e12e4c8adb6f7dc9199a76b-0001": 5056, + "02-db-cedf68155c2a11f3fa77b4762a99-0001": 5057, + "02-db-fed3908ca9f9c0d0f9dcbf5eec51-0002": 5058, + "02-dc-0788db0b1957f20059edffff8d61-0002": 5059, + "02-dc-1ac4cb069f5f1400602b790e414f-0001": 5060, + "02-dc-2bce84522285db3936ac325145c3-0001": 5061, + "02-dc-81040575594131e69eff695937e9-0003": 5062, + "02-dc-92c351e7baa6f4f9a43fff367e0a-0001": 5063, + "02-dc-a0617a4f39f4500471b038a30167-0002": 5064, + "02-dc-b1d7057b0b58650f94233a68711f-0002": 5065, + "02-dc-d5ab7655a42a094fdcc702e296f1-0004": 5066, + "02-dd-1a4c7d3a097828047d76ba2f3ec2-0001": 5067, + "02-dd-2270898b6ef9691e6352e9b8a446-0001": 5068, + "02-dd-2a70afc962c0ee054c49c4bb43e3-0001": 5069, + "02-dd-32cb3b9c74ae49017ad7cd21cf1f-0002": 5070, + "02-dd-68bc95051a14a0cb6de662821f28-0001": 5071, + "02-dd-72ba917134195cea35b306956aaf-0001": 5072, + "02-dd-7583a98bcd1a1495f23f7dedd63f-0001": 5073, + "02-dd-7a7d26cbe318ddb414ac276c8240-0001": 5074, + "02-dd-ace70e323a5b846fddd4ebf1751d-0001": 5075, + "02-dd-f3c71d4e88277f7bac3882e83413-0001": 5076, + "02-dd-f4d8ba893284233264f00ce02aa0-0002": 5077, + "02-dd-f665e557fac2f28b8827f4690c04-0002": 5078, + "02-dd-f74698eae8a070a99db8d884d73b-0002": 5079, + "02-de-0216f8700e9405043bcc43c70aab-0002": 5080, + "02-de-105ea20980d89c4c38c394f7ce5b-0001": 5081, + "02-de-4534d88685926654b104c02f5e38-0002": 5082, + "02-de-56b00759e2f8a0279fea75f42ab8-0001": 5083, + "02-de-5ce709971b41b60ad5eae3e8bfee-0001": 5084, + "02-de-9178f1cb12124ad553bc74360f69-0001": 5085, + "02-de-a5c46960a63b6009cc92f80cee54-0001": 5086, + "02-de-adcfb354d9cda38eff445e24e763-0002": 5087, + "02-de-f3aef969f0217c942e61948b69cd-0002": 5088, + "02-df-041b64e477cd280bd8db7d8d1767-0001": 5089, + "02-df-04c564c735a9ce254441d1f7f6e0-0001": 5090, + "02-df-275d1ed13c7c7b3582ad898e9ecd-0001": 5091, + "02-df-91f0b8780d4d42943d9438a7894f-0001": 5092, + "02-df-97bc87f039b7937f5ab2ea4dd64c-0001": 5093, + "02-df-c5b8041a32ef5b8e7da11c02a1d1-0001": 5094, + "02-df-e4818678d682a6fa42a7c91e404a-0002": 5095, + "02-e0-2297a97baddc24f63f1567e40353-0001": 5096, + "02-e0-7a2c18e196dae8c6e3187523d6eb-0001": 5097, + "02-e0-862536577a2a34e7e1b5404028c4-0001": 5098, + "02-e0-8ef276abaeca21714aa1fc4a2811-0001": 5099, + "02-e0-992c079d421c1f5a0265becde509-0002": 5100, + "02-e0-d6f94d06c122796f5c2ee681a55a-0002": 5101, + "02-e0-e2a7df10621e041d31130af9629b-0002": 5102, + "02-e0-ea713782f316a899adf9ec35d522-0001": 5103, + "02-e0-eaf4e6b13e6faffb4d1003a3e6ab-0001": 5104, + "02-e1-1006f2a5074e867dcac2ac7343a9-0002": 5105, + "02-e1-105bb1ca1758796f1eb358a546eb-0001": 5106, + "02-e1-2cf37a1b40f0e5a3d216367b5f63-0002": 5107, + "02-e1-35995e5faf58ae2a560ce3012251-0001": 5108, + "02-e1-3c438acb320b23a7fe5274025c52-0002": 5109, + "02-e1-7cf90b6cf01c3609a9d9b8b31e18-0002": 5110, + "02-e1-92675700e5ff6d6b3055bd079609-0002": 5111, + "02-e1-974f46b72fed5af64f9125c1adfa-0002": 5112, + "02-e1-a71a518f98725c782bce1f6250f2-0002": 5113, + "02-e1-adcc1b4ae83edc768edc927e503f-0002": 5114, + "02-e1-b47568c0fa09a09a26636c381f21-0003": 5115, + "02-e1-f53544be5d20a436a1ca54369bf1-0001": 5116, + "02-e2-6f5da8058c23791f65ac5c623f53-0001": 5117, + "02-e2-aa2d43d371dce841ac9c87005cd2-0004": 5118, + "02-e2-be788927c3e785dbb7ce8011f427-0002": 5119, + "02-e3-0b4b89e898c01b701779122dad25-0002": 5120, + "02-e3-16d8d2a5da9b7da6b1706dc0855c-0003": 5121, + "02-e3-19aab31f438020606faa1bc50e67-0001": 5122, + "02-e3-4afbb9369b697628712db32d8ad0-0002": 5123, + "02-e3-61943311896b7545802c62ec7449-0002": 5124, + "02-e3-65f472e0a9961a2e321da7d997e6-0001": 5125, + "02-e3-6f50b06d180e0c92e1d5fd9fb671-0001": 5126, + "02-e3-827c41f23a106d092ac16cb2edea-0002": 5127, + "02-e3-926eb35afd57d84af8a729ad6f56-0002": 5128, + "02-e3-c9a5b8b67efcb9038b3fdc2e9749-0001": 5129, + "02-e3-e4f38a550f9e0acd89e51093c602-0001": 5130, + "02-e4-6363479f720c1fd6bd445687ba11-0002": 5131, + "02-e4-a4e2383f91d6e97205bce95e9ebf-0001": 5132, + "02-e4-b3e1a00d5d52a3c6e1626579ffb9-0001": 5133, + "02-e4-cedaca90e7db6539ffee21bb5adc-0001": 5134, + "02-e4-cf9a19d722ab2106131e0599e58d-0001": 5135, + "02-e4-d9ddf57d11c30acb4aaceeead6ea-0001": 5136, + "02-e5-0c01bbf5e3aa8b2f2f7edb13beb1-0001": 5137, + "02-e5-46d08d9a03afb82bbd05fafafd6e-0001": 5138, + "02-e5-4a77cf16e02e3cb7d28088d91320-0003": 5139, + "02-e5-78b3088121b45ac664266e407f0b-0002": 5140, + "02-e5-7b9a5a8f1511f5602878c7bc71bd-0001": 5141, + "02-e5-b7eb81aacf64c6f14f1e6d9308ee-0002": 5142, + "02-e5-cf2dbec3f5bf398a80b1a2971bb6-0002": 5143, + "02-e5-dce5888420f9c44935e136204af4-0001": 5144, + "02-e6-217261c4dd841a061aa4112969bd-0001": 5145, + "02-e6-3182b6045a465948990701dbc9b2-0001": 5146, + "02-e6-3b7a028b6ce2494826efdce4c8ae-0001": 5147, + "02-e6-6a6fc7466a2c2cc457e0d5f21168-0001": 5148, + "02-e6-7aa3b2298919378516dc5170a3ac-0001": 5149, + "02-e6-8afbc89b8d6843e06ba19be591b2-0001": 5150, + "02-e6-cc8ebe7ccbbca0e742bf9ad237b3-0002": 5151, + "02-e6-e1312ef5984dd88c643437c127d5-0001": 5152, + "02-e6-ff0ee06155081fe2539248388852-0002": 5153, + "02-e7-3432a74e27da065bf5e4dfeffba3-0001": 5154, + "02-e7-bf27f236f09f5769ab703926d400-0001": 5155, + "02-e7-cf3538cdb85ef296de7c1889da71-0001": 5156, + "02-e7-f30a4390341b5cb1eaa7ca79285d-0001": 5157, + "02-e8-48aa7d4b43b250dd87a2a7164690-0001": 5158, + "02-e8-49d58bb95fc48f4170d1082b8cab-0002": 5159, + "02-e8-4da602c7e0375fc55a6d41b1c6a9-0001": 5160, + "02-e8-ad01118725bc8cd234ceae8499d4-0001": 5161, + "02-e8-d132f6057dc9c0a1304c3df2da79-0001": 5162, + "02-e8-da5ab5d5dfdea3864297c6eefb59-0002": 5163, + "02-e9-17c7080c728e7a6238e8ee5c1cf1-0001": 5164, + "02-e9-204caceff8eb205e7a6ac1ba1582-0002": 5165, + "02-e9-4a65aa8a28f0fb85aff7e81177c2-0003": 5166, + "02-e9-cf0f2d703b293a0d68044cf4da8a-0001": 5167, + "02-e9-e2960c01847ae10a3c8ce11f74c3-0001": 5168, + "02-e9-f54fab7669a9005414a34f5485ea-0002": 5169, + "02-e9-fbe60d74ac032a0e9c7d28aec5f8-0001": 5170, + "02-ea-0ef1318c8e4fdeb6cb44950ed38e-0001": 5171, + "02-ea-2d30bb65785c6981af37c72d197b-0003": 5172, + "02-ea-3ca2f2ffe6d44ed679c014cbecd6-0001": 5173, + "02-ea-4d4bcceff0699b433d9a8acc5f8c-0001": 5174, + "02-ea-7f8e1650387359669e1ef86440ef-0001": 5175, + "02-ea-864183b218c872bbb05472785558-0001": 5176, + "02-ea-c930234839e1fe534d7bf6c10561-0001": 5177, + "02-eb-822d001cd0484641908f6bbf16d3-0001": 5178, + "02-eb-8d5de61b668c1e754228c3903d30-0004": 5179, + "02-eb-930f3af034d0e55bd751f6541127-0001": 5180, + "02-eb-e526f741afc5125ef5c073085b9d-0001": 5181, + "02-ec-084b6fceee5c73a5e31b51d0180c-0001": 5182, + "02-ec-915ba4d860837f906a0b868742cd-0002": 5183, + "02-ec-967387f40cc36f573a59a43e2ece-0001": 5184, + "02-ec-d7aa266bd4bd9cef1a33393301b8-0001": 5185, + "02-ed-3dee66d69dd33a8305aeee51f254-0002": 5186, + "02-ed-43db3f774d6513ef71592f44bafb-0001": 5187, + "02-ed-55122792fffadd2cc7c2eb4f7a5e-0001": 5188, + "02-ed-834b285ff0c2d2a2836696d1fee0-0002": 5189, + "02-ed-89804dd49ee4e37b66dab94d5700-0001": 5190, + "02-ed-8afcadb04e8dbdf1a39a0e8d5e91-0002": 5191, + "02-ed-90c7e2266ee0c797e840fba2fe5a-0002": 5192, + "02-ed-98e393aa03cc51210b8989351886-0002": 5193, + "02-ed-f84528fa88f8484a7ef93c8923f3-0002": 5194, + "02-ee-2ad45466b58928e830511c6f9fb0-0002": 5195, + "02-ee-4f6de5ae9aecbdc182d35294d761-0001": 5196, + "02-ee-580f812d5ea5a7a37b7b89db2f81-0001": 5197, + "02-ee-6ad9af431297faa76b4b6eaecef2-0001": 5198, + "02-ee-8b80030b2e8b426fdb8f247b8077-0002": 5199, + "02-ee-9fc6f3d281340c419292d8214984-0001": 5200, + "02-ee-d5feef133e1514f5fbf557f29fb4-0001": 5201, + "02-ee-fd2c04144a5bbb1b821d9ebec247-0001": 5202, + "02-ef-02a2d6401acc2f4c4616fe9932ee-0002": 5203, + "02-ef-174a8c60635a4d313a9637dd4997-0001": 5204, + "02-ef-4eb28e1ab9c2cacc66643cbca6d1-0002": 5205, + "02-ef-569ef293eda11270ecea8955dbea-0003": 5206, + "02-ef-590bdfbd494b434e945d537410b5-0001": 5207, + "02-ef-729dace25a028d63a5081598df9c-0002": 5208, + "02-ef-75af562de17bbcd8d1c4f09ae17c-0002": 5209, + "02-ef-86a0b433a98a7a9dd3dcdb77d658-0003": 5210, + "02-ef-93d53517d2dc329ae7296d0f48e4-0004": 5211, + "02-f0-0c7effe55f85ec4c8bc94297d8cc-0001": 5212, + "02-f0-2ddba6af4551f9a57515fbbbde02-0001": 5213, + "02-f0-3f695cd33dd78189dfadb81b871c-0003": 5214, + "02-f0-a19e1bf993d90a417d7ff60aae02-0001": 5215, + "02-f0-aa3a6d4186fc693af68c5115f9f3-0002": 5216, + "02-f0-b6351ae36fea5521b732385d3768-0001": 5217, + "02-f0-b820b5b54604e086388442fe7c31-0001": 5218, + "02-f0-cce6cae38e05f2733dae8eb7e4e4-0001": 5219, + "02-f0-e7d47c0e04a574999e5e6b388f33-0001": 5220, + "02-f1-00b7a19219b83965fdaded8d2d1c-0002": 5221, + "02-f1-1fd5d1c2b943a852cd73d5615abb-0002": 5222, + "02-f1-2098095d9ff0af1e2dd2ed014be6-0001": 5223, + "02-f1-473af48ba2687e3759ac41181bc7-0002": 5224, + "02-f1-766442766a7598ce01464c455432-0001": 5225, + "02-f1-786fe78b24363b4c80a53158992c-0001": 5226, + "02-f1-90a54e665fc5731cd446b82eccca-0001": 5227, + "02-f1-d04bb9d1e90db14907d2db4b7f75-0001": 5228, + "02-f1-dd8df2583f750f9c642d9091bc63-0001": 5229, + "02-f1-f9e1d2f3962f571486f53c39cd4c-0001": 5230, + "02-f2-0bb8c4dcf7da606f985411039f9b-0001": 5231, + "02-f2-27a8d25846d0d65f1f5a2dc885b4-0001": 5232, + "02-f2-37022e483a630fe33c3b221a0fe1-0002": 5233, + "02-f2-5a0da566f1da4c8095fd40a1fd3e-0001": 5234, + "02-f2-64cec7e37b1db1713f860014babc-0001": 5235, + "02-f2-6ce59fcc133b0aef966d193b0768-0002": 5236, + "02-f2-84a29e8e00ac27a1795cd866a357-0003": 5237, + "02-f2-98ba7a071971415fcfe9d37bb30d-0002": 5238, + "02-f2-9a77996f9b3f3a4928414225bbfc-0003": 5239, + "02-f3-080900465f4362bc8e5330fa2f8e-0001": 5240, + "02-f3-3cd8553407722d937b7742807586-0001": 5241, + "02-f3-6bce29755560320f1f05e9efd1c3-0001": 5242, + "02-f3-b893687df87c95c98201a6e8eeb0-0002": 5243, + "02-f3-c2744aa12a04a16deadaea0fd7a3-0001": 5244, + "02-f3-cb1d29b4513a575596a01da96904-0001": 5245, + "02-f3-d311b522f4a222cbce751ee6ec0d-0001": 5246, + "02-f4-117f865594767ac711880b019530-0002": 5247, + "02-f4-5dffb08042e482b78c54af018332-0001": 5248, + "02-f4-73fe0ce6fed117e0197a28d7e0fe-0001": 5249, + "02-f4-81e71706fb908f76413c4545f2fd-0003": 5250, + "02-f4-bc8a863e05e465be74e675d93ae6-0001": 5251, + "02-f4-df2944a6bd8842b8164558ee47d0-0001": 5252, + "02-f4-e3bc6be5299d4093fa3cb16f6058-0001": 5253, + "02-f5-37c5c33911d727d3eeaaed81eff2-0001": 5254, + "02-f5-74ed4edc7f62ebee75145224afed-0001": 5255, + "02-f5-c27f6904305765e85e94f0cbf942-0001": 5256, + "02-f5-ec5c2178e6c284ab15d890fa89a6-0002": 5257, + "02-f6-39c546e9b1a0879c2d3f26ec804d-0002": 5258, + "02-f6-48e11d479e415d6bef02478934cb-0002": 5259, + "02-f6-9c2975c5f48c6eb6dba53fbcd9a3-0002": 5260, + "02-f6-a48f5f598fa01d6594380b2fb182-0002": 5261, + "02-f6-d369d5291f65cd2e23b34212e000-0001": 5262, + "02-f6-db9576c18c3dee3c681e630f3e19-0001": 5263, + "02-f6-de06abbe7def96eaa538034d925d-0002": 5264, + "02-f7-093122c0d298574c4a2002b11cb1-0003": 5265, + "02-f7-2243ae9305ca6c07448ded01d29e-0001": 5266, + "02-f7-53857922c666bd0520df95cbb893-0001": 5267, + "02-f7-54eed5036a1521f723757d61bc1e-0001": 5268, + "02-f7-8b4a6135a9d9f5fc154226232bca-0001": 5269, + "02-f7-958b05bd16d6018e3dba521058b8-0001": 5270, + "02-f7-a3fe6937196139d6bbf792f274ec-0001": 5271, + "02-f7-b7496e56d78036e8ab2f59a3cd9e-0001": 5272, + "02-f7-df0d1e990ebec890bd60d0dc96e4-0001": 5273, + "02-f8-24569c8e2d491fbf558e76876f5d-0001": 5274, + "02-f8-4eed0ad2d017d3458c171fa8d38d-0001": 5275, + "02-f8-bb7b7bad97d1abe06bc95df4a9c1-0001": 5276, + "02-f8-bc66682e482e1bffcd8ed8e85d29-0001": 5277, + "02-f8-e2f3b84f343801e627a6b0b6e72f-0002": 5278, + "02-f9-1b6d6dca4852b5ee434a1b7ce83b-0002": 5279, + "02-f9-1c08b8beacf5b57f1458f9e51b6f-0001": 5280, + "02-f9-3f5c1d5a3e35e8223cb9136db37a-0001": 5281, + "02-f9-63efe3cae638efb880554ddaeb62-0001": 5282, + "02-f9-6f129cd67357e0b86cae48cbdb32-0002": 5283, + "02-f9-80a61c5bb00cf87672edc08c0521-0001": 5284, + "02-f9-97b6cdbeceee57ab2861d6aa4c40-0001": 5285, + "02-fa-288083185813c8d719db5e1b2742-0001": 5286, + "02-fa-73c66ea148f0440cf58cb2777232-0003": 5287, + "02-fa-74922e6585deeb1829057c68e207-0001": 5288, + "02-fa-c8837702910b969e4b3b055e91d4-0002": 5289, + "02-fa-efe66575794d99796c2a065c772c-0001": 5290, + "02-fb-07d2b818616058c592dd66ae1f06-0002": 5291, + "02-fb-184cc0b7d036abc44ec38db3a60d-0001": 5292, + "02-fb-448db1aecc5dc0249d8ae2779932-0001": 5293, + "02-fb-7b69c5b090eed04b83733264bea5-0001": 5294, + "02-fc-22462db17bd3685fb65f19ebb636-0001": 5295, + "02-fc-2d377fbe1db03e3dc65e7b9b1bf1-0001": 5296, + "02-fc-41e88724e454f599f502cd145a17-0001": 5297, + "02-fc-521acddf3460dd4723c0d46bb0a4-0002": 5298, + "02-fc-659291d7550e1df15d3d98fb91c6-0001": 5299, + "02-fc-7a4d82700ffe205b994ee545b177-0001": 5300, + "02-fc-7d14347255c3dc83ca393c97501d-0001": 5301, + "02-fc-7d66dcd4a67c85bf5b8eaea3cf78-0001": 5302, + "02-fc-997c02f2a6c70c185bbf61be7638-0001": 5303, + "02-fc-a6f0f8fb6c8af2ba3e041e777707-0001": 5304, + "02-fc-b675ff2e93145f901c238410ca40-0001": 5305, + "02-fc-dc6ecf1aafa394c3d27ad6800c77-0001": 5306, + "02-fc-f96ee3c5620d9b45181a1992f9f5-0001": 5307, + "02-fc-ff633ff9958fbc0e355930829802-0004": 5308, + "02-fd-403926f02db4f4db170e4d1a4215-0001": 5309, + "02-fd-477969728bad47891074fb6c8afb-0001": 5310, + "02-fd-a07b27cc6f6e6d9d527218c035cd-0001": 5311, + "02-fd-a83d096cdaf814fa88f9c440a824-0002": 5312, + "02-fd-febc3a08d4de771267d8e0fef0bc-0001": 5313, + "02-fe-1f653ef2b75a65ea45af4e4580bb-0001": 5314, + "02-fe-2d1c11b89b18e25016d5e7ff11e6-0001": 5315, + "02-fe-3212d6d384ecfc44751418074343-0002": 5316, + "02-fe-368d9ff074a42e3e797a8ff4ce3e-0001": 5317, + "02-fe-6291e0aabf40d744a358c23a5aa4-0002": 5318, + "02-fe-a57c76d96ef8983234b2215744c8-0003": 5319, + "02-fe-b91774b7b4a184dea873f58bbb2f-0002": 5320, + "02-fe-b922146eb83d8d34e237de97732a-0002": 5321, + "02-fe-c2590edc4c099f40435008c701dc-0001": 5322, + "02-ff-0fab5e2cf93015f60d9c26eb0586-0001": 5323, + "02-ff-b2194b8e769d299c9e02567285ed-0001": 5324, + "02-ff-b49b93908fa87d59e5232ade5c05-0002": 5325, + "02-ff-b76e4c87e29046daa871089dbeeb-0001": 5326, + "02-ff-ba0ae53ea1707d14dc404652070b-0001": 5327, + "02-ff-cf9f7a533915246ed2a8940f02c3-0001": 5328, + "02-ff-d7677f0bd790e45d52ff3a42736b-0002": 5329, + "02-ff-e101066e4528f1d14a61dc0d3c60-0002": 5330, + "03-00-2653aad7980f83d258d9d8062049-0001": 5331, + "03-00-532a276b845900211c936829d25d-0002": 5332, + "03-00-5f5f631077ca21f0f77ba14a22ac-0001": 5333, + "03-00-6d255efcb0320c7bf110ff942511-0001": 5334, + "03-00-83f1ea9d09de52fe84591eceed00-0002": 5335, + "03-00-91bac0dd2166bff3ce0d07d79ed7-0001": 5336, + "03-00-9568473a7a4a26a75a11a43a9ed3-0001": 5337, + "03-00-b57e05b6b27b31fcf0701cd702e9-0002": 5338, + "03-01-07a01d53817d364397a597b4718d-0001": 5339, + "03-01-33c163df5bbf74ef6aa722f785aa-0001": 5340, + "03-01-3b301e6ef389541ed59a8c243fb7-0001": 5341, + "03-01-53a1db35b67a81cc03635d8529eb-0001": 5342, + "03-01-726d71003cddd5eeea9a19449435-0001": 5343, + "03-02-13d3d4d34553e17360d5d2058146-0001": 5344, + "03-02-322bcad871281a648fb34c985fc7-0001": 5345, + "03-02-469f0f490bcbc6b9f36773a8eb6b-0001": 5346, + "03-02-6bb2ef56a3c0ed7ade860dbef304-0002": 5347, + "03-02-8adfcbbb8db9bdf23b0d4e6918c1-0001": 5348, + "03-02-9404cb4a0b05428e74acce60b319-0003": 5349, + "03-02-ada99d2f2e618b8a1d816c307e04-0002": 5350, + "03-02-c8f1d687709a21de2611abe76304-0003": 5351, + "03-03-56ec3bd8b89710ba6ded821768dd-0001": 5352, + "03-03-6997cd1bc990b2b79c2384889877-0003": 5353, + "03-03-ad5a22edaa6dac6710de2f2f0847-0001": 5354, + "03-03-bc2f6dd9e4304d48f83bec99bdea-0001": 5355, + "03-03-db6c95dade45a65e8c824830a833-0003": 5356, + "03-03-ec5e28bdf168a100fc7465360e41-0001": 5357, + "03-04-074e9aad9795eaf79dd54533b00e-0002": 5358, + "03-04-27eab9035bd864449bb6cb029cf3-0001": 5359, + "03-04-3ebe1cd6a5f8486e1849caafdae7-0003": 5360, + "03-04-3f6fc90b232c414889a8d0284520-0001": 5361, + "03-04-7917886b618c15e763cd982ad6d7-0001": 5362, + "03-04-7a789e728f47630da76803f1804d-0001": 5363, + "03-04-8aecca78641fe683755551aaa60c-0002": 5364, + "03-04-9c123b3017a28ed011c7476cb3d4-0002": 5365, + "03-05-1a7072fe4888aa2b78ddfa3b1d00-0002": 5366, + "03-05-300976e85cff08574e97c656efb2-0002": 5367, + "03-05-3de70b057416f74f03dcf6716d4d-0001": 5368, + "03-05-3e2447675e540905b793bf12bf2f-0001": 5369, + "03-05-424450b56cfdb8ad565d5a9957d9-0001": 5370, + "03-05-d132cf8cf051279f38be870a9eab-0002": 5371, + "03-06-00043f4e1c4255527cfc23b3d606-0002": 5372, + "03-06-5ae062f86b3abc15f48c6a3e97e4-0002": 5373, + "03-06-6628432269146657a21ad23681d9-0002": 5374, + "03-06-84e99bcd1ea0c6fe0ac1fa9553d8-0001": 5375, + "03-06-b26ecdfb14a683e66b050d20955c-0002": 5376, + "03-06-b6ffec832fcaf4a477d4c06bd6b8-0001": 5377, + "03-06-dd40b7cc5c1c000f979d7d6c2e43-0001": 5378, + "03-06-ddac81e483770bee8e30c3ba413a-0001": 5379, + "03-06-ed3d8b164ca24351ddc2845b5a08-0002": 5380, + "03-07-1bc2c9b781066b092d9cad336e40-0001": 5381, + "03-07-4c6be89806a76bc37efa8670594b-0002": 5382, + "03-07-7a6e8d8d22af6ec3cdf10049b7c8-0001": 5383, + "03-07-a475bbf35d731acf4034e18377f2-0001": 5384, + "03-07-d54f4ac92d86ed5f736158f5b398-0002": 5385, + "03-07-dd32cf32273f8fd58d2ee0b5d6c9-0001": 5386, + "03-07-fe60bead63ab9620f2dc5deeda5c-0001": 5387, + "03-08-14c4de20c7eef4cfab1c55a73b70-0002": 5388, + "03-08-201a23d6150d63e2432ec0c9a1bf-0002": 5389, + "03-08-265c442156961a570c6d124321bf-0001": 5390, + "03-08-3c6f7f84eb42fe163c9e4a1e0625-0001": 5391, + "03-08-4d1c3c6ef2a7b2d0bf55a0cece0a-0002": 5392, + "03-08-5a04f55cdf478c880b96c35610b6-0001": 5393, + "03-08-74e7ee4cc3e3c1e9c96d3bfc6dea-0002": 5394, + "03-08-7bbf15f9a66c46c77905fcf16604-0001": 5395, + "03-08-a9b06374a9b093538061d5cd0555-0001": 5396, + "03-08-c85fc7b2454804a0d0bdbe77c419-0001": 5397, + "03-08-e3ab9a8875894acb4439477e7d69-0002": 5398, + "03-08-faffac694732070beefff0cb05bc-0002": 5399, + "03-09-41e64ac75cbab55b7168b75d65ea-0002": 5400, + "03-09-7403668466679bfae2b00084f128-0002": 5401, + "03-09-960fef8267fe9a9c721e14002c23-0003": 5402, + "03-09-9cf31e254cc363db1e2f18b5b4d5-0001": 5403, + "03-09-a3bf15b6210f77b49106849390c2-0002": 5404, + "03-09-b5336b54a0bcdd95a0d8cb4fd13b-0001": 5405, + "03-09-e36230b2b0bf90bd1b174672f571-0001": 5406, + "03-09-e4bb570bfd8b3680b571f1e6eee8-0001": 5407, + "03-09-e7fe59f5ec582c3dd7d34845857b-0001": 5408, + "03-09-f584392038f7bf7a2f1951c47067-0001": 5409, + "03-0a-38f09ae2b3e94938d8346c4ebc4e-0001": 5410, + "03-0a-523b14133071946a2298ecf757ee-0001": 5411, + "03-0a-7a479a1a2698807c36544dd22fb4-0001": 5412, + "03-0a-7db1451826b3a00c44866cd7e084-0002": 5413, + "03-0a-946ffc00701592c4a2f7134ab086-0002": 5414, + "03-0a-d88a00c19f853f9cdcb092eaf751-0001": 5415, + "03-0a-fca70a3898056a5b4358f11e2d5d-0001": 5416, + "03-0b-0558f324cecf71fbbe30c6012ae6-0001": 5417, + "03-0b-1ce08424d6d768871648e23fd343-0001": 5418, + "03-0b-4478aa98cde3d666f2dd1951d6d4-0002": 5419, + "03-0b-ab4c8cce270f325092025a154c5d-0001": 5420, + "03-0b-ab8705f6c2f9bb3650eab8cde7c9-0001": 5421, + "03-0b-af9f7e416b63724affebb4d02e1c-0001": 5422, + "03-0b-e19d5a957b61244410a267d0805f-0001": 5423, + "03-0b-f62f622dd093ad307e54d0122a8b-0001": 5424, + "03-0c-36e7ae5151104e6db5c64ceb0ded-0001": 5425, + "03-0c-54fcb995b506d49f20c8ba216e69-0001": 5426, + "03-0c-643e9fcf5f16b6745191c2235a4f-0001": 5427, + "03-0c-744792cdb301e8a33b2e1632a9aa-0002": 5428, + "03-0c-9a87589c037429b5a1d4e0cb0c4f-0001": 5429, + "03-0c-c8fb26052a3147f00e181b131382-0002": 5430, + "03-0c-ceaf61a7a847bc8528006a31f5df-0001": 5431, + "03-0c-ef464be0597a5bd36acb403d2856-0002": 5432, + "03-0d-24d1384a688291fec8823165ef82-0002": 5433, + "03-0d-648d4141e6b3fffae9f2295a00da-0002": 5434, + "03-0d-66ae166c2e04fb7f249dcd5b9dfa-0002": 5435, + "03-0d-79e05b598e42794adc67f44f3088-0002": 5436, + "03-0d-84810d6710c005b2eeee85f68b7e-0001": 5437, + "03-0d-853e5814f0e383c5e1ddda37056d-0002": 5438, + "03-0d-f118d1bade9b6722ef96fc49a0b5-0002": 5439, + "03-0e-0ab8898563f44cc77e0e5b2867da-0001": 5440, + "03-0e-16c27ba2ec7a0a0698b30b0e7752-0004": 5441, + "03-0e-31ab1080343cd670653176e45fe2-0001": 5442, + "03-0e-35d7b05727ff19fb747dc6a3b003-0001": 5443, + "03-0e-484f8746181ec588abdecc982bee-0003": 5444, + "03-0e-52ed4478947c44e2d39d7540b33b-0001": 5445, + "03-0e-76f00217a5e38ae6fe0781ce3b5d-0001": 5446, + "03-0e-8f3cfe93ea70803502b0c2e771a3-0002": 5447, + "03-0e-b33ca2d8a40b0e0d9067e5f18ee8-0001": 5448, + "03-0e-bbc80a9faf913e07c592ce38da97-0001": 5449, + "03-0e-f8854fd0cdc14e6c767b9f5ea398-0001": 5450, + "03-0f-228601351d9ee146dc49d2623857-0002": 5451, + "03-0f-2d6bc78380530be0e1130622147d-0003": 5452, + "03-0f-4a2eeb9308c26fc6affed5ce45a2-0002": 5453, + "03-0f-61b4b16597d025815f0781116675-0002": 5454, + "03-0f-6b665a6a2c6ff43f1e46aa76728e-0001": 5455, + "03-0f-7d4be0f33ba4868874122c1783f3-0001": 5456, + "03-0f-8f412f5c2c314c28ddb110a02bd7-0001": 5457, + "03-0f-a34ce49ec5a1ea22fc38928903c5-0002": 5458, + "03-0f-b47a8e9b4d641ac04b775bfc1f89-0002": 5459, + "03-0f-ca0a545bfd66bc9463837f8df31e-0001": 5460, + "03-0f-d66f938839980996c3dd9f7c5895-0001": 5461, + "03-0f-e4d3823b18e5595778a34c3c3c96-0001": 5462, + "03-10-10963c23d464a7e4333cc18fe503-0001": 5463, + "03-10-26435d73fe24d1000ac20e5c11fd-0001": 5464, + "03-10-5831f69e927e8da818845a0a1788-0002": 5465, + "03-10-8af4357f4e97d8f4b5fb3107d247-0001": 5466, + "03-10-9d9a0eec5314e261aea9b5316571-0001": 5467, + "03-10-ffc17b20c42bd8247887e312074b-0001": 5468, + "03-11-092a7a20a80c0de337827e321556-0001": 5469, + "03-11-62f46dc119d6d5b13e19334070db-0002": 5470, + "03-11-8755bd0b6b443ef4c4072dba9608-0001": 5471, + "03-11-8c0231792f458e950679c44aa150-0002": 5472, + "03-11-8fb6740843a42a3eebffee0404e4-0002": 5473, + "03-11-92cc71be6e917fb9f2603777db47-0002": 5474, + "03-11-93f7b729c4869fb72dfe157e0f75-0001": 5475, + "03-11-a7a1cba6cc381033b217ab1449cf-0001": 5476, + "03-11-a7a672bea20a4eaa7925921df508-0003": 5477, + "03-12-2a1ba8d50c8bd04ff2047e769a03-0003": 5478, + "03-12-2d69347e8ae35ff9e8093ea3598b-0001": 5479, + "03-12-2f5af58e7390a64f10752bbca5bc-0002": 5480, + "03-12-47288877aba37b0550d1141b2379-0002": 5481, + "03-12-4da382e1d47e645bd9341370a32d-0003": 5482, + "03-12-5e3aef2e5ca453a83310e6dde7ab-0001": 5483, + "03-12-a5c73344b13bf029fcdbe36f46f3-0004": 5484, + "03-13-055ea1d07e5eedc2e9930cd218aa-0002": 5485, + "03-13-3a787d88d86424908a7cdff40127-0001": 5486, + "03-13-3c74ddd3ea6850b89d669b7e45b3-0002": 5487, + "03-13-429144114d22adad61e8fa8474d9-0001": 5488, + "03-13-7780ae6a0dc83c76b9f72339ca42-0001": 5489, + "03-13-809f9e78cb904d7e417dc102b961-0001": 5490, + "03-13-9954d44f52e8ef2b19a94e9d94d0-0001": 5491, + "03-13-aaa253d7d1a8a17c27636fa2311e-0001": 5492, + "03-13-c4d8f6c99ca628d707e7eda99112-0002": 5493, + "03-13-c84887cb881aeaf7aa5f107a5cae-0002": 5494, + "03-14-2847d115b6f43da8bc9f0329094e-0001": 5495, + "03-14-40012c9559cc1811917f70fe46aa-0001": 5496, + "03-14-57a5549ef0d0a89b28c4aaf7afd0-0003": 5497, + "03-14-8a7242ca12ad69d48d1757d12895-0001": 5498, + "03-15-6f2c7caa5e414790274eacf5c9ac-0001": 5499, + "03-15-74a65d1e56f1138e67acdb968ce6-0002": 5500, + "03-15-7c885d78d44baaaa08425d8e7947-0001": 5501, + "03-15-9271e04f1d659b0824685ef7ed32-0001": 5502, + "03-15-9a53bcd31d96f3ac1be1c66d036d-0001": 5503, + "03-15-de44752f7fe8e0984c65a14aeba8-0002": 5504, + "03-16-2406f93caef5fbc0dbf94ccc83e5-0002": 5505, + "03-16-3f47a95692eb7fb7f2a837daa1d7-0003": 5506, + "03-16-52e00acde6c1ee89f6788ff4e2d9-0001": 5507, + "03-16-58238ad958e66bfae82e7cb94a1c-0001": 5508, + "03-16-5ab89a346ad41245796f532c7776-0002": 5509, + "03-16-692930324153a95c76faf0ada9ff-0001": 5510, + "03-16-8f2fc46d859ccdc8aa86f440542b-0001": 5511, + "03-16-aecedb4735eff84ababc5c0d0e85-0002": 5512, + "03-16-bf7a2d47d119a4083bcb1728766f-0001": 5513, + "03-17-09b22cc8b7a4ebe0e29835d31bd2-0002": 5514, + "03-17-09f14f00b6d4cf69292cbd158c24-0002": 5515, + "03-17-0e36ff77c4c688956f6db3df5dda-0001": 5516, + "03-17-0e3df05631d73b8bfc9d5d6529fb-0001": 5517, + "03-17-1e7a03c10afe7c5c6b488e46b835-0002": 5518, + "03-17-94838e819dc6fc9aecbfd9199d03-0001": 5519, + "03-17-e455d3a3e4cdaa6f9a9046a57524-0001": 5520, + "03-17-f2536cfd3f521b67027f39f778ab-0001": 5521, + "03-18-300078ae4345b05a43db7c56669b-0003": 5522, + "03-18-60e6a90d3b98783065faef732392-0001": 5523, + "03-18-748e37a45fa1b487ca859f8749e5-0002": 5524, + "03-18-85c35732ae07a7ae56043c111f26-0001": 5525, + "03-18-8fbb6c10d01dce6db833fcf083f4-0001": 5526, + "03-18-997d67d77a428d4d78d9ab1170dd-0002": 5527, + "03-18-d53c07e17ec8aabf36757a2aec56-0001": 5528, + "03-18-e129a9a223022f1b239742fc8caf-0001": 5529, + "03-19-08e1ac58cc42f91c5da05f9842b3-0001": 5530, + "03-19-24b20b86f9266286ee6293048eca-0001": 5531, + "03-19-92d037a5fb95fb605251b0da7112-0001": 5532, + "03-19-c6978422a4572877110556c2e437-0001": 5533, + "03-19-c8221beb81f9e525af3731d1b9ce-0001": 5534, + "03-19-d1d73c5fd325bca6fcbc473dabee-0001": 5535, + "03-19-d4fc2b581d15a9c0578eaca063f5-0001": 5536, + "03-19-e0b31f9313c66eacb8eb91c416e6-0002": 5537, + "03-1a-07820a43ffe18cad3e526db5c413-0001": 5538, + "03-1a-5bb6c715acb7685b6017f6d103fd-0001": 5539, + "03-1a-5ea26e71ee29323a6cccac8d16e2-0002": 5540, + "03-1a-bd7cc1f7f4f128fb8fccaef0a652-0001": 5541, + "03-1a-c81bb9151ddb7012d615c4fd848e-0002": 5542, + "03-1a-d8f8243fd5925d19b74527e87fb7-0002": 5543, + "03-1a-e0f43316590dce3a3a236281f047-0001": 5544, + "03-1a-f2f53ab03f38f7f2b2b7c90bd7b0-0001": 5545, + "03-1b-002aa3b556a67ebf97eeb213d693-0001": 5546, + "03-1b-12aaf65f1a0dd0475aa887ed28d4-0001": 5547, + "03-1b-196fb33201512b2f7e4d8dd9847b-0001": 5548, + "03-1b-1d9f2708207f4ce40d866912b73e-0001": 5549, + "03-1b-4ef49bb53e1fb29cd45acd6c5474-0001": 5550, + "03-1b-74d8e3e13a8c1797794b601c39f3-0001": 5551, + "03-1b-761b0a1dceb1a4c1b1d87396abd8-0001": 5552, + "03-1b-8317affb5dca438ff41737df8287-0001": 5553, + "03-1b-91c403403f251fc5039139cfc72a-0001": 5554, + "03-1b-d91bfc4758fecab19bef698c0439-0002": 5555, + "03-1b-e3f80ed10eab293f9413bf0376ee-0001": 5556, + "03-1b-f4d58bd11150194fa6ec3bad2194-0003": 5557, + "03-1c-3446684744c7fce44a8ad05c587a-0003": 5558, + "03-1c-3d6c9d14bb63bd06f81a43d48893-0002": 5559, + "03-1c-3f397b0a74e851b97e4247ff0e22-0002": 5560, + "03-1c-a509ef6d159180c6a153320f4658-0001": 5561, + "03-1c-cdbf328b05739143b5452f6adb7e-0001": 5562, + "03-1d-1f698a08a91197b945c27de39203-0001": 5563, + "03-1d-27a69877a19a2e2849e9800ef84f-0003": 5564, + "03-1d-316cc527a7d1f218394a0f4c5528-0003": 5565, + "03-1d-4bec2355a8744433f1b15aabca9f-0001": 5566, + "03-1d-626f8af47da96871c69b85615a9c-0001": 5567, + "03-1d-78cc7337e0066b9ee654ba4e4091-0001": 5568, + "03-1e-069339270a7ff1683f893e9bab42-0002": 5569, + "03-1e-89787ed09f8cf7499959b0f16cbb-0002": 5570, + "03-1e-b1188e358363eb596e87decdbc86-0001": 5571, + "03-1e-ca15c93567d6d68a80a48ab472e7-0001": 5572, + "03-1e-d3d83b2e91a25054bb88585edeee-0001": 5573, + "03-1f-1100855b5e52696bb423cd5963e3-0001": 5574, + "03-1f-1839bda8ce4ddb9f3d0ab77cbe2c-0001": 5575, + "03-1f-2f61c1df4bec1a1f1bcae1d311b6-0001": 5576, + "03-1f-37cd81262b79748ff6ff02df0a8c-0001": 5577, + "03-1f-532b9920e0cc56f9c345dd64d51e-0003": 5578, + "03-1f-675c1e1f8468d5b3fa77edd4afc8-0003": 5579, + "03-1f-7d09793dca66b31caf3e48ed1a52-0001": 5580, + "03-1f-8a809288c63f7f3392ea349be8f7-0001": 5581, + "03-1f-c996a98b9a4e8619743a277755ea-0002": 5582, + "03-1f-cd5179de54374462e9aefebbaf72-0001": 5583, + "03-20-20f1d8421c6724caa1543b97acaa-0001": 5584, + "03-20-2a9aa2ae226f7017992937d71c6c-0002": 5585, + "03-20-322e2c81a669d5c12802b2a08555-0002": 5586, + "03-20-36b4466d795f3fdfc72e9624432e-0001": 5587, + "03-20-36f56dd4f68d5f307323b7a34162-0001": 5588, + "03-20-4ae36d4a4b5791629eb974eb7f48-0003": 5589, + "03-20-778ad0b11eb0f667e20aa60de5a1-0001": 5590, + "03-20-8441803656b1702bea57e0ef7165-0003": 5591, + "03-20-ab9da551a6c3f10ba760e7d1b7bf-0002": 5592, + "03-20-bb0f2a056a0f24522a72d09446a2-0004": 5593, + "03-20-dba76b6da56ae105a7a1713b4794-0001": 5594, + "03-20-dcc46d5835c72149222806ec1ae1-0002": 5595, + "03-20-f67dd4dfe870489e6d7c607b60d6-0001": 5596, + "03-20-f70d5636b44ffeb2253554938f3a-0001": 5597, + "03-20-f7ff4daa7c7348768c4df539e24d-0001": 5598, + "03-21-0c7b33998049e6bc14273c74d7b9-0001": 5599, + "03-21-199ec74a28e61d42d6b14d46a82e-0001": 5600, + "03-21-388caa52432496baf5c6f3202828-0001": 5601, + "03-21-7b597eb27b07cac62b8ea9a4208f-0001": 5602, + "03-21-805bfb3357b2b8d6c0e29249ae4f-0001": 5603, + "03-21-9ff65d272a072c0fec7c769f5011-0002": 5604, + "03-21-e825f866714b3abfb922c94b3bfb-0001": 5605, + "03-21-f755079321d48ae0d4401fa6b13e-0002": 5606, + "03-21-f9e07ae770ff48a5955baf6b79fc-0001": 5607, + "03-22-38be5b176f0c88d334b4d19a249b-0002": 5608, + "03-22-8a57f94af3fcabbc093feeeab8bf-0001": 5609, + "03-22-b8e2c700a0fd312ae5c076dfd56b-0003": 5610, + "03-22-f41080be525566f6bb6085beeca1-0001": 5611, + "03-23-2f190db733d64b82a388ceb49187-0001": 5612, + "03-23-53a433c0fcd432b573d07724b141-0001": 5613, + "03-23-a88f1c0da7d6dd7c2b99899809f0-0003": 5614, + "03-23-c4e5bdd928ce6e5d7f42cf2d8d82-0001": 5615, + "03-23-cab6f0b44f0894d611fe7e40ab54-0001": 5616, + "03-23-f0d291e09fb3af66422981aee35c-0003": 5617, + "03-23-fd60e0da304f0b3025ff14fee8e2-0001": 5618, + "03-24-1aabc1bc2d8d3c58327c9df594da-0002": 5619, + "03-24-2e4b5a4805c57e7a7dcdc87ce7a0-0001": 5620, + "03-24-5b82b96abefdc124801e1a06153f-0002": 5621, + "03-24-6484a7a656a488a39d94315815eb-0001": 5622, + "03-24-aac8bc03270c4181119a236acbee-0001": 5623, + "03-24-bde6e1500f62a28d4aa02bd6b251-0002": 5624, + "03-24-dd5c575f8d7b111c0ac0c6b84a6c-0001": 5625, + "03-25-47d6e39b5bed005d7cc03f311330-0002": 5626, + "03-25-4aa0735e1577846ccafc61830e43-0002": 5627, + "03-25-59d2c7510dcc824a0fe03ba455fa-0002": 5628, + "03-25-6a483e3a0ed4129720d4a77deff8-0001": 5629, + "03-25-a20e8eb7e75484f57c988ae857a3-0001": 5630, + "03-25-dee0a254562263247aefa3d05d44-0001": 5631, + "03-26-26414e23046e1f840f4d6d2b5daf-0001": 5632, + "03-26-2cd1fa5f4e8b2a06b58c3a7aa7af-0003": 5633, + "03-26-518c29d09ccb10c800814b6d6579-0001": 5634, + "03-26-51b215d23d282a339c77e9a123f8-0001": 5635, + "03-26-5d3b5fee45a28956c943d7ecaa80-0001": 5636, + "03-26-5e636309779fb31156118b32301f-0001": 5637, + "03-26-79da3a22937125b78ece5427ad86-0002": 5638, + "03-26-adcfba301baf44a3a1862c0db4ba-0001": 5639, + "03-26-d883f4b713a87e541f4e02c762cf-0001": 5640, + "03-26-dcaeea21ee49e5bc829fb9ed38b3-0001": 5641, + "03-26-ddcd9b9cb264fee6518ac189613e-0001": 5642, + "03-26-efa1e65cbed2b9a6985406dc32d7-0001": 5643, + "03-26-f894e81d1092c2f6c10c7390af1e-0003": 5644, + "03-27-1da7f0e175c5d984566e81017f0c-0001": 5645, + "03-27-61fb62a2e900e21362208737a75e-0002": 5646, + "03-27-6f428c3de160453d6a6117c745b3-0002": 5647, + "03-27-a6054d2ad6429865a41a16b34f0b-0001": 5648, + "03-27-d60d16b89c77ed8efd0ce9ec9a0e-0001": 5649, + "03-27-dd327407a3521d20ce478ca3608c-0002": 5650, + "03-27-f5ff3350b4da9e65c8dc2fe563b2-0001": 5651, + "03-27-ff4330797879869b60d3767c18b1-0001": 5652, + "03-28-04d745a8cf8fa2d29527b900b691-0002": 5653, + "03-28-12aed4b4e4532e14cb2b7c58e337-0002": 5654, + "03-28-1c5ce4ade39c75272f3ed6a6ccf8-0001": 5655, + "03-28-488a7e3b2e93b30f7315ec17db7f-0001": 5656, + "03-28-5756488d794d9cf2b8193c86752a-0003": 5657, + "03-28-765f8ab6958b68ae3568f491feb4-0001": 5658, + "03-28-87228ec62fafcfabe4ee69a3c276-0001": 5659, + "03-28-9701b4b9a05fadf27272c1f03e65-0002": 5660, + "03-28-c669501332f1ccca857a25a89cb7-0002": 5661, + "03-28-d63ecc392d57d99c2704c3912891-0002": 5662, + "03-28-e42a912762d90e5f2aabd3f594df-0001": 5663, + "03-28-fd19e6e71020b5462bf7ce015c3f-0001": 5664, + "03-29-2ad6ab78f2c7dd35946b65b3d5d3-0001": 5665, + "03-29-2adda058524d5b5757a402b5c655-0002": 5666, + "03-29-57ba488597bd82d1f5d6bb5fbc34-0002": 5667, + "03-29-5997d8786f73f221b38a5f9add44-0003": 5668, + "03-29-7864c8d2c0b9955e32667bdfabd1-0001": 5669, + "03-29-7a5b39be803c15d7b0c791411a11-0001": 5670, + "03-29-b5d6f8c7d44a0664c321a8db238f-0002": 5671, + "03-29-f05f58b84927c6aefab428160e11-0001": 5672, + "03-2a-1c4eb5e1454a568643b5ffc44469-0001": 5673, + "03-2a-353e456c87801d7bf23cd6010017-0001": 5674, + "03-2a-4669b2ef0974c50542a1f80d9246-0001": 5675, + "03-2a-5ebdb3175a50a001a9b5e0dbb616-0002": 5676, + "03-2a-b2163b125c3ef6363edd00dc5ec1-0001": 5677, + "03-2a-d723b6f999bcbeeac36bafe952d6-0001": 5678, + "03-2a-d9127704b8c9fcde32367dbcebe8-0001": 5679, + "03-2a-fe5e07dd918435ba4c2709a6ca4b-0002": 5680, + "03-2b-09a7fdc68ebd7fc33973a7308670-0003": 5681, + "03-2b-144ab74b72ee57adb63e5dd0bc28-0002": 5682, + "03-2b-6a4343f0a5aa2aff4599c26cd107-0001": 5683, + "03-2b-9096e1febff38a4c2f1b81741ae5-0001": 5684, + "03-2b-a49361b60a5554e8828c5f2f61ca-0001": 5685, + "03-2b-acf8a2358d7b2ac44304f52f8c8c-0001": 5686, + "03-2b-b2dec2dd5289ac979de739177132-0002": 5687, + "03-2b-c6fa71a95f34c2636b2a7ea51cab-0001": 5688, + "03-2b-fcf5fdab6adcbf03ee0f793bf171-0001": 5689, + "03-2b-fde54929ee04e5a7586eb157d089-0002": 5690, + "03-2c-10c7914cb4c0f49542da4e02045b-0002": 5691, + "03-2c-191c29607316ce88caf8ca7a13e0-0001": 5692, + "03-2c-1dd1925235435b1e916bca4e5ec5-0001": 5693, + "03-2c-4205420a43b894b3eb3f44c63815-0001": 5694, + "03-2c-666eb142d97ac1887c86009c0f09-0001": 5695, + "03-2c-b6278ed045b67e418e7dceea294b-0001": 5696, + "03-2c-d61d831120a59f0e6baf3c1bf7f9-0001": 5697, + "03-2c-d65f2db81d3087092ff0ee8660a4-0001": 5698, + "03-2c-dd8364231e498d5f143946d58d90-0004": 5699, + "03-2c-fdf5014bdcf6ddda4812e23423c4-0001": 5700, + "03-2d-1398901061ad1608902d3909ab6f-0001": 5701, + "03-2d-39d19993799ebef25b08f41bb48e-0002": 5702, + "03-2d-5402aec40cd226ad2ac783301e70-0002": 5703, + "03-2d-7d36cbb00e2bff797c18aecdea42-0001": 5704, + "03-2d-7d9d94561c5581326fd229703907-0004": 5705, + "03-2d-cca9a5c5bd5a566a74ce4136082b-0001": 5706, + "03-2d-e24e1dd69e7adad655dd1862e80a-0002": 5707, + "03-2e-5aee83602d27815ed3b993f8d1f5-0001": 5708, + "03-2e-6740cda6329042e086a2c4c11b06-0001": 5709, + "03-2e-79d675010d4c34c2d41eb79789ed-0001": 5710, + "03-2e-a0eb06981bb00f6e37fbcd0805cc-0001": 5711, + "03-2e-ab82fe3238aedd17cad66083e084-0001": 5712, + "03-2e-b89437fa450eb96a21b337d1efed-0001": 5713, + "03-2e-e054928bd05c68948bd291267b80-0001": 5714, + "03-2f-4bef71ba3cb8d3c98671a57f6ae0-0001": 5715, + "03-2f-59597a4ca6d73e19bdea40883f9d-0001": 5716, + "03-2f-595e6fa4747bd92a8cd3e20377f9-0001": 5717, + "03-2f-9404dbe2a0dbae1bba92ac7c5ac7-0001": 5718, + "03-2f-bccef721446495e254352be6fff4-0001": 5719, + "03-2f-d3ab5b52c203594edc24789234f6-0001": 5720, + "03-30-5d5cc2d5d7f3dc8b78c4dabdc843-0001": 5721, + "03-31-2465151c2539777ef8e45b319b2e-0001": 5722, + "03-31-2696657527ff714f67a5701c36b5-0002": 5723, + "03-31-4e28f25c7f5cc3c43860db4fb121-0001": 5724, + "03-31-64cfac7acccffdff56571ab65fb9-0003": 5725, + "03-31-7671ee8def2b87cff156211825b3-0001": 5726, + "03-31-a2cb0fdef8859a893718bfbe4dd9-0001": 5727, + "03-31-ac152c5d3c9567705a166eaa460a-0001": 5728, + "03-31-c1cdf5c4ce2a72ddba9c144ca690-0001": 5729, + "03-31-c945be9dc843fb588a4573b0b302-0001": 5730, + "03-31-e12bc92d3166d3121968097921fc-0001": 5731, + "03-32-0424393f37a0a404daa943160160-0001": 5732, + "03-32-058cf66ea59a3a13008f45da8763-0001": 5733, + "03-32-2db20baa84aa87113679e912efac-0001": 5734, + "03-32-5c61fbdca7e8283f2e68946dae91-0001": 5735, + "03-32-6dd914c939438af3383432b72c69-0002": 5736, + "03-32-71b8147027efc94f3dfeae3f51e2-0001": 5737, + "03-32-73a835629707e41d539ed68fd7e3-0001": 5738, + "03-32-75f5bf90ac3dd776806e29f70650-0001": 5739, + "03-32-8c0dca966029cf1dff1d4d9f5868-0002": 5740, + "03-32-b6341575d2d8211d2d56fcb54ce8-0001": 5741, + "03-32-fbcd6a56fc8227ec507767bad9d1-0001": 5742, + "03-33-02357fedb5e1eb109da365bf85ab-0002": 5743, + "03-33-0e9304c7849bbfe7bc630d536f3a-0001": 5744, + "03-33-2eb4bdfe3056bd10e511920f443f-0003": 5745, + "03-33-4105ce5ec78a728566cf7e147969-0002": 5746, + "03-33-73239abc04ba984b21ca18c7f265-0001": 5747, + "03-33-a91a17e7dfa55e0450af71cbd509-0001": 5748, + "03-33-c49f748153ef9b7e71b194c42442-0001": 5749, + "03-33-ccbbf9df5c734e5bf291dab7ad78-0002": 5750, + "03-34-0083fef4583d1321aaef72119770-0001": 5751, + "03-34-09bec6a5b27d446bb22e355761ac-0002": 5752, + "03-34-587a2030c9fd11c06e815d64aeaa-0001": 5753, + "03-34-60725589e569c9fef7bc84729460-0003": 5754, + "03-34-8301802ce8601334aa22c5b6088f-0001": 5755, + "03-34-86518446946b71ed73fa0f1714f7-0003": 5756, + "03-34-96f874e8d7602c9c0e302d4ca161-0001": 5757, + "03-34-b16e2c9f462cd9ce918c470aea5f-0002": 5758, + "03-34-bf5767e6d450e34b409dbde6385d-0001": 5759, + "03-34-cf448a13164ee721e472fa4a4841-0002": 5760, + "03-34-d68dde9d40892aab225276efc6bf-0001": 5761, + "03-34-e9d12dc9e0b714357f9bfbf369ab-0001": 5762, + "03-35-14a83629072373f2dd4f05371efb-0001": 5763, + "03-35-164e4de8de5ed03f213ed790fc75-0001": 5764, + "03-35-35611dc6e998f6c13ad0e13e1195-0001": 5765, + "03-35-58da86bb1ed80d52fc51da602254-0001": 5766, + "03-35-6b2c7c59a76175687ec42abe13cf-0002": 5767, + "03-35-b02a99c14201ac32eddb71ffe3dd-0001": 5768, + "03-35-bcf9846359a7f50c78203cd56128-0001": 5769, + "03-35-c12bda9c27ce243930f3b0fd88dc-0001": 5770, + "03-35-c1e58cb9f78404412adf3578801a-0001": 5771, + "03-35-ec06d9fff5511e83d9fe7fb6f738-0002": 5772, + "03-36-069bd9f9ec9f693366d359a32a28-0001": 5773, + "03-36-5b34890c8a850b3413747fb4044e-0001": 5774, + "03-36-667a408daaa2aa74358709b01974-0001": 5775, + "03-36-c3362712cc4d52a95d58ea13fa58-0001": 5776, + "03-37-036f0dfe54f65e81489ca6975967-0002": 5777, + "03-37-1e4354a8e6c3ddb9917b223da113-0001": 5778, + "03-37-4437d19ae4d143ffd2f9c3407b33-0001": 5779, + "03-37-f39893cbe606cf074d7ad00cadc4-0001": 5780, + "03-38-011f6652581a4aace5a033217c12-0002": 5781, + "03-38-22e6015d2e62ca7f0cba44cb94e4-0001": 5782, + "03-38-2e627b2f33e10972c7494022bc22-0001": 5783, + "03-38-3fbd97e0bdbf2296cb234b823229-0001": 5784, + "03-38-9adac5c7444af24b7c9454f5d69a-0001": 5785, + "03-38-df5047da83e0d2c69d466313c49b-0003": 5786, + "03-38-ff75008ecce443d17ed8746fc8ff-0001": 5787, + "03-39-167f6398829876ed0373527fd254-0002": 5788, + "03-39-1e33c0390e64a31f1cead23e7ad4-0003": 5789, + "03-39-444035a280a0ffbf47e3e8170aab-0002": 5790, + "03-39-5357fe1c531f0992b2f1839f2cc8-0002": 5791, + "03-39-8335cfc15e52587b66d1afca86f7-0001": 5792, + "03-39-8e6f69e410fb8db65d97ac5713d9-0002": 5793, + "03-39-b9decd0bf32b788c875ce0cadbf4-0002": 5794, + "03-3a-0147b7184fbe5cf46bdebaecf44c-0002": 5795, + "03-3a-1343b7461f6170c53b509e82a06b-0001": 5796, + "03-3a-4d4b90d5f681be884d334c5746ad-0001": 5797, + "03-3a-62dc8ae2c0fd8e70c8379839ab80-0001": 5798, + "03-3a-b67d76a4f96f191e9f19370de995-0001": 5799, + "03-3a-b8b82a8560038466690166fe8f85-0002": 5800, + "03-3b-2a1aea910bb5e929128dd69c548c-0001": 5801, + "03-3b-3032ff42f8e92019f764d5b2e6bf-0001": 5802, + "03-3b-708211f01decaa978d8494ee61aa-0001": 5803, + "03-3b-a282fcc5806df1f7f3e0612b700b-0001": 5804, + "03-3b-b99b7ac3d727f594d4811417e3ed-0002": 5805, + "03-3c-15c13792d94b620b02625b0e6483-0001": 5806, + "03-3c-17afaf88074a5a6cebe881292cc7-0002": 5807, + "03-3c-30f04439e6e9381cbd7a95089d7f-0001": 5808, + "03-3c-4e942225d2bd12bdef81d15ee849-0001": 5809, + "03-3c-4f2355222f56d0dff921534741ac-0001": 5810, + "03-3c-531e6f8c1656469aea2dfa7f7341-0001": 5811, + "03-3c-6450dfbac4aaada734fd47a84509-0001": 5812, + "03-3c-70a2adae0b1b97909e4c22c21ab4-0002": 5813, + "03-3c-8032bb04a1e7fca0396f024a379b-0001": 5814, + "03-3c-b0a4b76b8ebf902f6932678d492e-0001": 5815, + "03-3c-e08918723259340c95886689dc8c-0001": 5816, + "03-3c-efedafb01083f9ef6950ef5c97a5-0001": 5817, + "03-3c-f69b340332196f4eee6c075eec44-0001": 5818, + "03-3d-003a44d821b0c01717d23ad3f713-0003": 5819, + "03-3d-1b6d8d5ab8913dadf15942408ec0-0002": 5820, + "03-3d-1cb41d08076d28a89b5ef47b46d1-0001": 5821, + "03-3d-2a17c932ce30daa6b59732edc056-0001": 5822, + "03-3d-2f06525c520fdbdff9561487d5a2-0002": 5823, + "03-3d-4778753ad77205062a49a16ddeee-0001": 5824, + "03-3d-c5fc0098cacd2d4fb13af2942203-0001": 5825, + "03-3d-e1f9da8324963160bfdb3772391f-0004": 5826, + "03-3d-e5ce13678478dcfb044b0503847f-0001": 5827, + "03-3e-44e70071c3f09b050140fc4916e9-0001": 5828, + "03-3e-5b871c6f5038b399dd99e0fa24d8-0001": 5829, + "03-3e-656fb9f347d85d187adc36b47753-0001": 5830, + "03-3e-87c9071423b2398de3d83f71134e-0001": 5831, + "03-3e-8b3d3354f9f04186870bd0a38886-0002": 5832, + "03-3e-9a208c8135a3ee5611f9696cf3f7-0002": 5833, + "03-3e-9ece79e80e269ebe2480647cdfdd-0001": 5834, + "03-3e-aed9b92aa952b8c19b7fb06b5cfe-0002": 5835, + "03-3e-bbcc360ed0cae7c30aa4a4fc3175-0001": 5836, + "03-3e-f7fdb0611c584623159ef7d1c828-0001": 5837, + "03-3f-0e88b72b5d079f54f67311747fe6-0002": 5838, + "03-3f-19533a9f17929c94f3b59e611d4e-0001": 5839, + "03-3f-3f863fa4a8cd86adbe2294b482d7-0002": 5840, + "03-3f-55bb0faeca5cd4ecff3025087ed3-0001": 5841, + "03-3f-694bb463eb152fb89652ebfea3fd-0002": 5842, + "03-3f-929bcc70d940ae2895692357de3f-0002": 5843, + "03-3f-c3866bbf8ede6f84b32ad3112d3d-0002": 5844, + "03-3f-c54ecb0e3a2358948997c24a72d6-0001": 5845, + "03-3f-c96f16bfed9b453bd70bd9796bb1-0002": 5846, + "03-3f-c9fd30724468d4cc7702316620ff-0001": 5847, + "03-3f-cb78ebd08dd5c1377e9e9a7febdf-0001": 5848, + "03-3f-e6a5af8536f0e779539f3b804855-0001": 5849, + "03-3f-f481b22cb50f5a963d507595d950-0002": 5850, + "03-40-0e205f73d03263e2db4c0ac1ea52-0001": 5851, + "03-40-10c9db65f94ea70d07ec78eec580-0002": 5852, + "03-40-10cc31da42a013339e35edcad595-0004": 5853, + "03-40-50b6e28d9336a27d76692237a714-0003": 5854, + "03-40-9f26490b23afdca0cd8b5b73c0d8-0001": 5855, + "03-41-0782bd9ec11c0e2af18854aaad1c-0002": 5856, + "03-41-2a87b70db77b65d91f2fd7442288-0001": 5857, + "03-41-2bc3223c58d883e1a7870a6be807-0001": 5858, + "03-41-6951c3277252d5462ff5b581db03-0001": 5859, + "03-41-a1cbef32a71154c7ebab6cc44c4b-0001": 5860, + "03-41-a3e1504d4f718a7625b97de9baea-0001": 5861, + "03-41-a7f45774c20d66bedffa27fcd3af-0002": 5862, + "03-41-d382fb39d386b0ad5da2523e2da4-0001": 5863, + "03-41-e210580b29afaf74d2101ba8ca16-0001": 5864, + "03-41-f3023659bc39fcd5383dcab04c2a-0001": 5865, + "03-42-19f732b2b092c1173ecf914ae5bf-0001": 5866, + "03-42-33a2f698ed768df2f6f704baf353-0001": 5867, + "03-42-4700af210c716ce7b5b05e84782a-0001": 5868, + "03-42-57bb8be0fdd835abf21a61fe3d6a-0001": 5869, + "03-42-5b0b0f3c3e275bae497ef8d3aa81-0001": 5870, + "03-42-aeba432724109a4199108cc1c0b5-0001": 5871, + "03-44-6f38a9bc55eb9160f13c089aaf45-0001": 5872, + "03-47-b27e45bad8dd2992df5ed46cec4b-0002": 5873, + "03-4a-11424e691c6f25094851c1b11ba9-0002": 5874, + "03-4a-48799451cd630822307c00c02ba6-0003": 5875, + "03-4a-7008e2ff067779ea7d60dff56d18-0001": 5876, + "03-4a-b0ff79da1e105aa4c55d96a25adc-0001": 5877, + "03-4a-b33241614ea20a2d797fd5b2931c-0002": 5878, + "03-4a-d241e8b8536b188c9483a5c66efb-0001": 5879, + "03-4a-d978e59feebbb1ad1a5533d4f42f-0001": 5880, + "03-4a-fb4b4b8f8682c4047fbb6bfe31dd-0001": 5881, + "03-4b-01630fe7ad0222decbec5768a069-0002": 5882, + "03-4b-1d0c60b4568b1b873afd0819135e-0001": 5883, + "03-4b-2a55027fe4a42a6ca84f47af9e80-0002": 5884, + "03-4b-2a8d73023e0e1aa771e198af65a4-0001": 5885, + "03-4b-45d5c243a8edc238255b8e0fd345-0001": 5886, + "03-4b-56f2d961015f78013cef0e22d771-0001": 5887, + "03-4b-66e66df72f94006929d109450b3f-0001": 5888, + "03-4b-73b6c1993414ba9fdcb0dfc1f448-0002": 5889, + "03-4b-7c0e630a707ed89edd39034f5e22-0003": 5890, + "03-4b-aebe7291fcb09b6bb63bb56942a4-0001": 5891, + "03-4b-bcfe0a23845581a9ddd810695ae9-0001": 5892, + "03-4b-c1319b103220669f825af8238402-0002": 5893, + "03-4c-049f3f7d8e012a35b59260696196-0001": 5894, + "03-4c-1d5ae848e8550c5fc8df9e60d335-0003": 5895, + "03-4c-28e9268d5f7e33982beb0e02ed88-0001": 5896, + "03-4c-379362d50a71a4a53b105dd0494c-0002": 5897, + "03-4c-4bb78f1f42b499ac0f45f26a7435-0001": 5898, + "03-4c-7a5e6b69da3c03dab54fea0930f8-0001": 5899, + "03-4c-934a46d5857c0b0000eb86dba77c-0002": 5900, + "03-4c-a1e293e062b0ea66f07cf30f909c-0001": 5901, + "03-4c-bdf656fdaa329ca757aaba178ac6-0001": 5902, + "03-4d-0f05d3c09158fc8b6451de49757a-0002": 5903, + "03-4d-153def42a9f89429045c1bf638ca-0003": 5904, + "03-4d-38e00c09cead41d599951d5232eb-0003": 5905, + "03-4d-5688d46703b661b82954b8c825a0-0002": 5906, + "03-4d-8af749aeed2e58326684c7f958c4-0001": 5907, + "03-4d-ae24cc1c56c6a4c14ac4fa2bc5f3-0003": 5908, + "03-4d-b9c414d093cd332c5de073a38b6b-0001": 5909, + "03-4d-c9ac5b64dff1de3a0473cb47fb57-0001": 5910, + "03-4d-e632098a469d2605d9ea2a567bab-0002": 5911, + "03-4d-ec28eb1339eb7426af5483a8408e-0002": 5912, + "03-4d-ef6478ca45af27e84fb676fa34fd-0002": 5913, + "03-4d-ff283b031757ecca2847ba50836d-0001": 5914, + "03-4e-66132e5d7cee3f9e72bfe3ffa960-0001": 5915, + "03-4e-86aa4b2019d51970b847537e0789-0003": 5916, + "03-4e-8f9213546db7dbdbc313442424df-0001": 5917, + "03-4e-9b665279007aa3d44c4b1ba65b43-0001": 5918, + "03-4e-c6aae0bcabfa5f3f2916d94f0e09-0002": 5919, + "03-4e-ed7f19fe865cf51c2f4c31f2dcd6-0001": 5920, + "03-4f-0f54cfcef42392a1edfefb7432cd-0001": 5921, + "03-4f-a9638f7f0e63f6ad46a445c608d6-0001": 5922, + "03-4f-adc9ad6c74f2d645468950014b3c-0001": 5923, + "03-4f-b0419fa51f6569bfa6224c41d82e-0001": 5924, + "03-4f-dbdaae48dd5ab56fcd80d6dc38fe-0001": 5925, + "03-4f-ecc267bf3212463832f2285dab15-0003": 5926, + "03-5a-05041d3763a29bcee2b1c4d4ef50-0001": 5927, + "03-5a-0aed5a46470d4b4a297b6e66b56b-0001": 5928, + "03-5a-448738afd59cb379a6ef8a0d6a62-0001": 5929, + "03-5a-472cb0e4e551d95c01ccabe5e3ca-0001": 5930, + "03-5a-6b3146ecbdcf7759a15084bcf000-0002": 5931, + "03-5a-6bc3b7f212226fd30462c2ae77a8-0002": 5932, + "03-5a-dd2766965aca21d03997af55810c-0001": 5933, + "03-5b-13f2140ad90386194c0e72161ebd-0001": 5934, + "03-5b-157bfa7706240b70b05f0c0842e2-0001": 5935, + "03-5b-39acc554adcc36eb904f589cc7d0-0002": 5936, + "03-5b-491cda46c4be0077ed7fb7aa68ce-0001": 5937, + "03-5b-546bfb1a2f71f0f29bb806e68d1b-0001": 5938, + "03-5b-787a44e3a82e7e1be9bb66d6cd49-0001": 5939, + "03-5b-93e4b9ba3c33659eba82ed47c62c-0001": 5940, + "03-5b-ac7333bccfcab831fea3627057a9-0002": 5941, + "03-5b-bbe3910274d4c6958734c0a2431b-0002": 5942, + "03-5b-e6feb00eef3ca4dba5c213d62f74-0001": 5943, + "03-5b-ff9e0fe03df28a9aad3544527977-0002": 5944, + "03-5c-04257cd18c1377c4ea2d31e0e3b0-0001": 5945, + "03-5c-164a823cd26f483e9692f3333012-0001": 5946, + "03-5c-2e86f92b0bed23768148b4e9db9f-0001": 5947, + "03-5c-32607ecb9a1c419ca4443069be59-0003": 5948, + "03-5c-37698c1cdadb984712044df2a729-0001": 5949, + "03-5c-a7795c13faf82b581e449f361ecd-0002": 5950, + "03-5d-111297687f0fd48aafd9ea0afffb-0001": 5951, + "03-5d-2087c2411caaf730b09220cb1835-0001": 5952, + "03-5d-823e629e5d2723eaeda1f3a838a5-0002": 5953, + "03-5d-97facafa072fcca7804018c7ff15-0001": 5954, + "03-5d-a058adb7d2b276091c0f94838afb-0001": 5955, + "03-5d-cae7b0fe995e40828ae3adbebbb4-0001": 5956, + "03-5d-d61b9415e4b5d10e2ebbe9e369fc-0001": 5957, + "03-5d-e23d5e9f7a49be108bd91ea6ef75-0001": 5958, + "03-5d-f0af9519b3d9683bacb64ab825b6-0003": 5959, + "03-5e-0915cd278ef6e7711fac4eaf9fef-0001": 5960, + "03-5e-0e7ebad67c81628e4995a235c75c-0001": 5961, + "03-5e-3c7b49a2a732f754a2c00e7c84a2-0001": 5962, + "03-5e-6bea9f87329772ecd256537aa27e-0001": 5963, + "03-5e-9209c25c9024e4ab151b78244094-0001": 5964, + "03-5e-92926d51798527796234837b42da-0001": 5965, + "03-5e-de48b44ce21c47e61ee4ac0cdc9b-0002": 5966, + "03-5e-e75e381bcd79f24138cb3b43b9c6-0001": 5967, + "03-5f-32310859b73ac69d77c272f8dd04-0002": 5968, + "03-5f-37e1b26d74a262811f81eaf07d51-0002": 5969, + "03-5f-3f9bd52b8408190d4c5820293d1c-0001": 5970, + "03-5f-4b75d88147ccf7298d52460cf475-0001": 5971, + "03-5f-70345895fe806cf7f1fe79e9b554-0002": 5972, + "03-5f-710be7760254897ed1c1bda4527a-0001": 5973, + "03-5f-772f7b089d6721c74ae5ecbc8edd-0003": 5974, + "03-5f-ad3c4a86674eb631ed0871772b64-0001": 5975, + "03-5f-b0abe5ded3fa8277ab0e955f780a-0002": 5976, + "03-5f-b587f02a79ebac82ec5ad067325e-0001": 5977, + "03-5f-f7b0834f6d390ef82e6adb53a543-0001": 5978, + "03-5f-fd01be61ff9688dce330945d3f39-0001": 5979, + "03-6a-2d7c0bed6c1d988187e54c629656-0001": 5980, + "03-6a-2f9208cb211a81409d831d8198e1-0002": 5981, + "03-6a-4626e98912f47f920619c326f909-0001": 5982, + "03-6a-492a3c9f9924101e1510042d7a16-0001": 5983, + "03-6a-67cdddb7b57499cdc7be3604323c-0001": 5984, + "03-6a-6a016b681c4e11d10bd278f47f42-0004": 5985, + "03-6a-6d616da225b24702d60f490c6e46-0002": 5986, + "03-6a-af2ab43cba0c1fefec789138b30c-0001": 5987, + "03-6a-b8b8ced4614ab7f218f9398301bf-0001": 5988, + "03-6a-de7183c2969f9ac3d2203cd4201e-0001": 5989, + "03-6a-f8274c8e014da7d5d824df371217-0001": 5990, + "03-6b-47682bf14f4a2bf3c4cf3aa079c0-0001": 5991, + "03-6b-4a0a5cac4094d36ef3e205712344-0001": 5992, + "03-6b-5f36e08b767d956c429598f076e5-0001": 5993, + "03-6b-69770b791e40d909d278cfd975a8-0001": 5994, + "03-6b-da917a3f8939708267b356b36dd0-0001": 5995, + "03-6b-db0c8cd962294c9ec096a132f7bc-0002": 5996, + "03-6b-f324ffc6eeb5ea5d8d3cdb4db592-0001": 5997, + "03-6c-20b95aaabc02ffb6eb7e0426ad9d-0001": 5998, + "03-6c-2b45b0eaebec42d9416dac5d2b38-0002": 5999, + "03-6c-36d815a31452afc87027cc9b9485-0001": 6000, + "03-6c-42aefbd358a19a55783407bfcd9c-0001": 6001, + "03-6c-63b5209a512a24fd247c6b2983c3-0002": 6002, + "03-6c-70da8edd50bb644dd1b8069a1cb0-0002": 6003, + "03-6c-9e8f32bbe35382e7309610f057aa-0002": 6004, + "03-6c-e12c93b6920620bd7cbc0702abd5-0002": 6005, + "03-6c-f478d894d3c553924c5eb317b32e-0002": 6006, + "03-6d-2beac3db1a51f99ac839ea5db208-0001": 6007, + "03-6d-35969d9b63169f31a1c431bd6da5-0002": 6008, + "03-6d-52929a338c78d373a2db858e16c8-0002": 6009, + "03-6d-9085a1d9465e9cb92e634062d7e3-0001": 6010, + "03-6d-978cc254ec08a1fbdda40f0f13f4-0002": 6011, + "03-6d-a8b4c74b02e0984b47e31bfb4aeb-0001": 6012, + "03-6d-c38f9a097058b745c31df6266455-0002": 6013, + "03-6e-25637a1c2bf0628ed248b9fdbfd5-0001": 6014, + "03-6e-76e4325123f6dad27322eda0d6cb-0001": 6015, + "03-6e-b81f5a093b1a2263b01d45dee740-0002": 6016, + "03-6e-c4a1eb4d14cdc75be90f1ef3542c-0001": 6017, + "03-6e-ddbe9aea6715170a67f35937b278-0001": 6018, + "03-6f-257ceb7b7880eb5198b1f561c45a-0002": 6019, + "03-6f-36ce3a43865470a370e5a1bcc7b8-0002": 6020, + "03-6f-6f2e77903ac8db4103db7288fe28-0004": 6021, + "03-6f-8311b01382b99f1c5dd55d6c51f4-0002": 6022, + "03-6f-87d74d5817a9623d2c7f649dee45-0001": 6023, + "03-6f-8bbf1cb8c655a5df162854e96fdc-0001": 6024, + "03-6f-bd85eae641cf07acccee5f9a2214-0002": 6025, + "03-77-6a49f129cf0d2f366512909c8248-0001": 6026, + "03-79-c65322c9df499e3be8b7f5cc5d6b-0001": 6027, + "03-7a-0c5173dd4fa41d5dd002a015726a-0001": 6028, + "03-7a-2b45ca90a8e5700f06fe66d32f00-0001": 6029, + "03-7a-2d1503de59b7e31685790bec835b-0001": 6030, + "03-7a-2fce17a644646145f543be455022-0004": 6031, + "03-7a-ba27c5f584f5b40a6eac96f07a1b-0001": 6032, + "03-7a-ba9bd6afa34c6e3a646204bc3451-0002": 6033, + "03-7a-c106f3bb4e4ab4114d4fc4c74c08-0001": 6034, + "03-7a-dee7bfd83bd81c235015a29ed3d7-0001": 6035, + "03-7b-11318471b0bac8a65632a0b0da63-0001": 6036, + "03-7b-2520bcef920b2dc7a903fda7dc02-0001": 6037, + "03-7b-417a6046745e566b3d7b20d2a8f5-0001": 6038, + "03-7b-5009c7d1ac9f74fbf049c4459d84-0002": 6039, + "03-7b-718b7d800c64d185a585e2c067ac-0001": 6040, + "03-7b-7d2459a32ec04162dc239ede4a14-0001": 6041, + "03-7b-8f2e5a3111518ff307e1765d91a9-0001": 6042, + "03-7b-8fdd74a05303becbb526e2848ad8-0001": 6043, + "03-7b-cc42cd11795a5b1e46ad4fe9a2a2-0001": 6044, + "03-7b-e172b9a8f21b0f707efe8c8eadcd-0003": 6045, + "03-7c-056b057853d2138b2226087e9808-0001": 6046, + "03-7c-112da07f93c17264c027f784fa79-0001": 6047, + "03-7c-19cfa070fa31d6aeb0f890a149a8-0003": 6048, + "03-7c-32394440c4280a0a0b5de31074e6-0002": 6049, + "03-7c-4be926673042ac0f9a57217fe69c-0002": 6050, + "03-7c-5334c2ea805c3592e27110f6ea18-0001": 6051, + "03-7c-5d31daf7236cbc0a3c0a6e176045-0001": 6052, + "03-7c-5d47d9a236dd84bb453a697b8882-0001": 6053, + "03-7c-6b0b1abe0aae41e13e64cf15d915-0001": 6054, + "03-7c-b4076de24abdf434ffa0f41f7347-0002": 6055, + "03-7c-b602bca2803ade3a8917aeffc0eb-0003": 6056, + "03-7c-c60b71134170f98d81068dcd0c31-0003": 6057, + "03-7c-e534f2b9dcaa1ba3e0a2ef580760-0001": 6058, + "03-7d-1a727b0c53b8dd774f587de2c87f-0002": 6059, + "03-7d-2fcbea487c4e3dbda58bd505ca53-0001": 6060, + "03-7d-6c50d46ad5f50ccd0390987fd4b9-0001": 6061, + "03-7d-8168d0f15dedde13bd26b41bf92e-0002": 6062, + "03-7d-efa695618b7226af160e88fc185e-0001": 6063, + "03-7e-02edbcfc1df0d587394d0cbdc969-0003": 6064, + "03-7e-07ded0bd4d1be0c065f85fd9c25c-0001": 6065, + "03-7e-0e7bc5c19d40ae88728d9852fc17-0002": 6066, + "03-7e-181c961c26439fc60d0c0af24cd4-0001": 6067, + "03-7e-19016bdf7113a34a415f1a84da02-0001": 6068, + "03-7e-426478055ac85c32e8c86549dfdf-0001": 6069, + "03-7e-475efc306b9458366684617e4c99-0002": 6070, + "03-7e-76e5d5a4ae25af22b09eeeaee41a-0002": 6071, + "03-7e-99121aadd09a6a47dd2ecc4143b9-0001": 6072, + "03-7e-a840cd30040d965fdcce011c3c02-0001": 6073, + "03-7f-061bb21cc7b6d6eaa747983161d7-0002": 6074, + "03-7f-2351672973d6beafacbbe081a55c-0001": 6075, + "03-7f-4d1c9737ceef68afccba329bdb1f-0002": 6076, + "03-7f-91b6b76ebf19521d88185cb1519e-0001": 6077, + "03-7f-aa941edffba4b1b50e20899947d6-0001": 6078, + "03-7f-ede52413c008b50d6f53ed4bc31b-0001": 6079, + "03-81-df518eb604d0a151309eb821fce5-0002": 6080, + "03-82-0ddfea1647368646dbcb68ef8e1c-0003": 6081, + "03-8a-471dde98acdf57b1590767605451-0001": 6082, + "03-8a-4f3fc4bf12cc3451f1ef71ddb8f9-0001": 6083, + "03-8a-63a9b41f1a589baa83073d191320-0001": 6084, + "03-8a-64d2e3dd8c9182f44e88fe9c6974-0001": 6085, + "03-8a-a0d10a1acb79764ee8661ee11c67-0002": 6086, + "03-8a-c5d59ec0cad36b62b5673da4a9b5-0002": 6087, + "03-8a-dfbcddf3a63abaa318b09d649978-0001": 6088, + "03-8a-f686446a64a9265a001c1f0af6a8-0002": 6089, + "03-8a-feee41e31564857d4c4d520096fd-0002": 6090, + "03-8b-0c75e1f73a9962ed1e4ea50bfecb-0001": 6091, + "03-8b-4476eca254d2b36361c57da003e1-0001": 6092, + "03-8b-6cf6bf1d25b0d86394e8184d6b26-0001": 6093, + "03-8b-76c8a17668e78b831c5bb2147a8c-0001": 6094, + "03-8b-81068a44112229a75ba7d2486231-0001": 6095, + "03-8b-c3486089e1ededfc2a5d0890e91a-0001": 6096, + "03-8b-c4cfa03d70ad5165538bec5f609a-0002": 6097, + "03-8b-ca139921a5463e22bba0ac16558f-0001": 6098, + "03-8b-e9528da75cbb2114e024f51da5b0-0002": 6099, + "03-8b-f26a172793719eb29c322b831f32-0001": 6100, + "03-8b-fbf35f1f4b703d0d943c445ec8ac-0002": 6101, + "03-8b-ff15a189cbc5812d4fb4016354c2-0002": 6102, + "03-8c-0a30df70859b480dd5af1b70092e-0001": 6103, + "03-8c-189e1f892de742aae3820aff7a35-0003": 6104, + "03-8c-1aa31eb16a8cfb09cd3a805ab173-0001": 6105, + "03-8c-225e0874b4eb675046dd1507703f-0003": 6106, + "03-8c-6365f7a3da3ad835b02affc9b507-0002": 6107, + "03-8c-7c127de31878f2bd45f1cf56d8f5-0001": 6108, + "03-8c-b359f6e5fde8cb36021e6e22c2ad-0002": 6109, + "03-8c-dcedc4da6542aef93a92e686400f-0002": 6110, + "03-8c-f3c95d695546a11670c2828115df-0002": 6111, + "03-8d-0f1a1914fb286dc15c1d774259d0-0002": 6112, + "03-8d-118bcf79051fef7a723a687e7fc1-0002": 6113, + "03-8d-1d28cf3762a3ef9a7340998cc7c0-0002": 6114, + "03-8d-3135464196703d66f54ef95b4e83-0001": 6115, + "03-8d-7c3aea9df7dad1267100898c3090-0001": 6116, + "03-8d-8f5bb8ad74504bd6fdd8dbe36c0a-0001": 6117, + "03-8d-a58b90007a84c00a77c2bbd85a28-0001": 6118, + "03-8d-ba02743e71a83179c76851989d76-0002": 6119, + "03-8d-c4c4f2deaf84e63ebc23282646b9-0001": 6120, + "03-8d-de7ae40c00ef9d5564b97147f1ed-0001": 6121, + "03-8d-df83ad96a67e62c629a7895bcb02-0001": 6122, + "03-8d-e68f7b7cfdceebf027bf06afd8c8-0002": 6123, + "03-8d-f309715414693acae519455dcd9f-0002": 6124, + "03-8e-4e81744002cce0338121899daec6-0001": 6125, + "03-8e-7a7dd1309f3202c90cfdb5331a36-0002": 6126, + "03-8e-91bcceff0aff1b6b1fc8a0642904-0002": 6127, + "03-8e-a3c138911acfb37a7c7688ca4e41-0001": 6128, + "03-8e-ac93d7ccd2f4a15cacb1bc7617e6-0001": 6129, + "03-8e-b2128b8c6804c6d2ccc52ad18359-0003": 6130, + "03-8e-b5173ad332b7dbd93a18fc8201c2-0001": 6131, + "03-8e-c5b4257295aecc505efd85c40e35-0002": 6132, + "03-8e-e2f6b566a2ff25244da10e365d53-0002": 6133, + "03-8e-eba5402ccfcefee9f4421c276d34-0002": 6134, + "03-8e-fc31995c37af727c9643df25a6ab-0001": 6135, + "03-8f-0b6ff9971077b0070b988031fafa-0001": 6136, + "03-8f-21a280ade9f0e0dff9b2bfe58e7d-0001": 6137, + "03-8f-22974807ebcf891d833050b812d8-0001": 6138, + "03-8f-4eec0bce494ae27f7beda7eb885a-0001": 6139, + "03-8f-9143589f60d64e9d581d0e4b0ea8-0001": 6140, + "03-8f-af2d104773d3c03eef0dfb8b8e65-0001": 6141, + "03-8f-cc0e31c654519fa90d12f3ecccb8-0002": 6142, + "03-8f-fb1416c7c3d680a8e413319488fb-0001": 6143, + "03-95-3742f011e92f8b9e587772872a84-0001": 6144, + "03-95-3bfdaf901015295e4170cee202a5-0003": 6145, + "03-95-9f2087f4f9860d66065dbb13ec46-0002": 6146, + "03-95-c590301421593e10be7e4d3add9a-0001": 6147, + "03-95-ca76aa173445dedaa6cb33893eff-0003": 6148, + "03-95-df75e806632fd2ed1f3124a3b82f-0001": 6149, + "03-96-29c037a883e9e1079c99fe885c2e-0001": 6150, + "03-96-2e60faddb6c22d91fa9dbfbbca70-0003": 6151, + "03-96-3e6db42d013060585d1ece798a3a-0001": 6152, + "03-96-4593cc043b2502e5357e2177b230-0003": 6153, + "03-96-50490a02eb70a7902205880c4194-0001": 6154, + "03-96-898b38596e7abd14c8b134695c64-0002": 6155, + "03-96-96f5bbfb0dd88913a71c8c947334-0002": 6156, + "03-96-bbfc0dd6e1c5a004e1534c00d98f-0003": 6157, + "03-96-ee75b9cab25525b9c0058e8b4a5c-0002": 6158, + "03-96-ffb552eea252c664fc7775f0b32a-0001": 6159, + "03-97-45e4a4acb330dbb9cba0f84df6c3-0001": 6160, + "03-97-583e2f2afba6c9ccf661c15fa268-0002": 6161, + "03-97-80413bfdba44f266f1959049c623-0002": 6162, + "03-97-a2149e9d3b09297c9c61740135e5-0002": 6163, + "03-97-a9d855ad8a6b612b89acde2c088a-0001": 6164, + "03-98-03ee83d52b855a9053e9a1918d58-0001": 6165, + "03-98-0a3aee512e950aad3460b559583d-0004": 6166, + "03-98-196242ec76071b2b63e285fcca95-0001": 6167, + "03-98-43ac0f04a6f4acfdf355e8769b24-0001": 6168, + "03-98-4520f74d62f9c273dacc0137ce97-0001": 6169, + "03-98-63ed6c66455c6fc4de49256eef2b-0001": 6170, + "03-98-9e536a0fe5d5b77ad3b4cb1ea180-0001": 6171, + "03-98-d6d7dbf6765b134278e22481789d-0001": 6172, + "03-98-e0b87c9d021aef0a83c0f496b519-0001": 6173, + "03-99-29b8901b313e9f0efb5060f33a98-0002": 6174, + "03-99-7469c4e267fd11eb7f653959b767-0002": 6175, + "03-99-9bdd06b23e43c5758dbd3ca29b90-0002": 6176, + "03-99-a10f3dac7f7eb57807f307e8699e-0001": 6177, + "03-99-b7dec745e91c2dcccac2132109ae-0002": 6178, + "03-99-e3b388cbdc3bde59e6474b9644bb-0001": 6179, + "03-9a-3cfcc380cbebc4e6c7923ffeefdf-0001": 6180, + "03-9a-654eaeed5c76be30181c35c1f8d9-0001": 6181, + "03-9a-6b711c4757739b9a81ba3adac58b-0001": 6182, + "03-9a-7bd415400f412e85df86d1ef3720-0001": 6183, + "03-9a-8fd9eefe6fd357cef58ce5724a09-0001": 6184, + "03-9a-b0777aada65b83f59f2c01337545-0001": 6185, + "03-9a-c736a38b184f78a33ff6f1a41a2a-0002": 6186, + "03-9a-cfd37eec3a01e1ce82627e0df3a0-0001": 6187, + "03-9a-ee54a3dff3a900d505fb9f183714-0002": 6188, + "03-9b-0626eb0a62adf09eba21831bf55f-0001": 6189, + "03-9b-12c0c4d2307fe0941dbe3ad20fcb-0006": 6190, + "03-9b-57d8e97db8e30bda80bdd2a5e000-0001": 6191, + "03-9b-741bcff1d688318c8374a62458fb-0001": 6192, + "03-9b-887e975b818eae3c0251d43958b0-0001": 6193, + "03-9b-9cfb3e8c56cfb9e18a26207d3632-0001": 6194, + "03-9c-0d4a8216eec57802f7a5b2e74d33-0001": 6195, + "03-9c-364dc4f2dd12207e656ecd005487-0002": 6196, + "03-9c-534e8b3f978a396762f952f1f034-0001": 6197, + "03-9c-6d7bbc8bc05bcce22b7ae70d23e8-0003": 6198, + "03-9c-7eae1d8b802d2dd581be56264023-0002": 6199, + "03-9c-ac3d9c9742e6e6d0fafeb265c3ce-0001": 6200, + "03-9c-af5cfb262963a12ae09fa4cadd3a-0002": 6201, + "03-9c-c42a5b9622039f48ce41c90d97a0-0002": 6202, + "03-9c-d69ff28a00c79f0e4ddb82910c7a-0001": 6203, + "03-9c-dcb571bf61fb594344997458f0e3-0002": 6204, + "03-9c-f15ce8c76689f54f90b81e6096b1-0001": 6205, + "03-9d-225fd32f2bea6f31c8736396892d-0002": 6206, + "03-9d-2913c640b2b20aa86e18d2f6c8ba-0001": 6207, + "03-9d-428c6685b40f76e2e88f35416efd-0003": 6208, + "03-9d-7ed94719f288f07659c9c14767d4-0001": 6209, + "03-9d-8b2ebdc3ea6fcc567417ff13bca9-0003": 6210, + "03-9d-8f66113f3170d3212624fc8e1e8f-0003": 6211, + "03-9d-92d932efbe87572cf1e297f284e4-0001": 6212, + "03-9d-a1ae4fb8dac4ed7d3cf7780f1738-0001": 6213, + "03-9d-f58e41c090c665c2fdc402270775-0001": 6214, + "03-9d-f7c239c67aca32beb459e5632a34-0001": 6215, + "03-9e-074d1aa559dbe5fa3b8e3dd3edb7-0001": 6216, + "03-9e-135338dd1bd4bea97202ccdfbc64-0002": 6217, + "03-9e-34046991620d1977c53e3b214d73-0001": 6218, + "03-9e-41e8477bb4c5e68760ddc399b343-0002": 6219, + "03-9e-6214be47a3a3d210b423f1174107-0001": 6220, + "03-9e-78024ca750f6443e94cc711c8d21-0001": 6221, + "03-9e-a03acbd05752a2aa0c026aa2f638-0001": 6222, + "03-9e-aee59dc4e5317c7c52b571ef4448-0002": 6223, + "03-9e-b47a86ae46b11e1d6ff2db2f37e2-0001": 6224, + "03-9e-bbe6d04978cee0ca34f6ad0712bc-0001": 6225, + "03-9e-ff9df6e1c7f3ef7708d81d59b95a-0001": 6226, + "03-9f-46bc5b6aad3ce5635751738c7758-0001": 6227, + "03-9f-48013e831496e906c432ad560185-0001": 6228, + "03-9f-59a46d042582f387facf61d5d02d-0001": 6229, + "03-9f-829d343d929d5055896ca361e5bb-0001": 6230, + "03-9f-9a425132153c4b775ed9fb8e9a36-0002": 6231, + "03-9f-a60064ed2762ed063583c7000c31-0001": 6232, + "03-9f-af725e6e83c8bd0e9017848d6d0a-0001": 6233, + "03-9f-c19ec7953feb7b2a294a70a12c86-0001": 6234, + "03-9f-c658d0979796ec56c3edaa5340ca-0002": 6235, + "03-9f-e6d58123206a69ef0fd47f03bf20-0001": 6236, + "03-9f-f2a0bbed8326dd2514d565bf66a1-0001": 6237, + "03-a0-1bd8a45052fd56d9356c170644da-0001": 6238, + "03-a0-6527b29cc011b2979987262f827d-0001": 6239, + "03-a0-7da24ae5270f7c9e15ebc80e33d2-0001": 6240, + "03-a0-7eb2a6421641f2d05d6a959c05f2-0001": 6241, + "03-a0-b2acd6bf3b0855f09773ab65ecf6-0003": 6242, + "03-a0-c005bedaa45451bd1a1d2175d856-0001": 6243, + "03-a1-412dbcad873ce04f3d0757153c83-0002": 6244, + "03-a1-47a63ada89ac9282f8f56dfe159e-0001": 6245, + "03-a1-9aefa1926160eec595c46da27c29-0003": 6246, + "03-a1-a0659b5baa32001cee0f1cdfaea2-0001": 6247, + "03-a1-c61822d6a4240c31d6877792e83c-0001": 6248, + "03-a1-cc9b48dfd1af13568d4a6923903d-0002": 6249, + "03-a1-d519739c199f5a0988d508a4fa12-0001": 6250, + "03-a1-dba4d65bf45082caea921ec2a9e2-0001": 6251, + "03-a1-f055797ba4153005d61b11fb9989-0001": 6252, + "03-a2-56f084a1e88bf2b0351642909a17-0001": 6253, + "03-a2-6437b7c32075e70aa9d134d44027-0001": 6254, + "03-a3-045102a1c6546617053813660cda-0002": 6255, + "03-a3-1ce7e98cad6b21613786d74542f1-0001": 6256, + "03-a3-39b8f049e67e7546f7c242277ac0-0002": 6257, + "03-a3-62bba990662f936a88f288007b74-0002": 6258, + "03-a3-70585492b84bcc5219e7114f463d-0001": 6259, + "03-a3-b55bde1fae245e4b146a83b2eeb4-0002": 6260, + "03-a3-b9aab7a90ec3aaa33ed5ccf0619f-0002": 6261, + "03-a3-ec669162470d95a63cad8493b8c1-0001": 6262, + "03-a4-0820370611a2fb7afde5cfd79748-0001": 6263, + "03-a4-104eb14033cb084b1c4184145b78-0001": 6264, + "03-a4-6722f53ff625bef1efdbbf2e2fd7-0001": 6265, + "03-a4-8fc8a48aecb2c2da9b2d2cb52d51-0003": 6266, + "03-a4-8ff6ff49d6e918794ef362d4be75-0002": 6267, + "03-a4-9d55d183d92f1fc3c4d999d98c96-0002": 6268, + "03-a4-c9a57db2eed9661abb302662a265-0002": 6269, + "03-a4-e3bbd2ea8cbd4b57815d7a44a0bb-0001": 6270, + "03-a4-e711fe35859e70d72c7046e65247-0001": 6271, + "03-a5-090ecfcdbff09436b80f25fc0e9e-0001": 6272, + "03-a5-24394b3c12bdc9cc7d225c25df3e-0001": 6273, + "03-a5-3bf1a8619fc5b3d3281bbd655f72-0002": 6274, + "03-a5-6fc1983f0dd4a0d3fa8766b084e1-0002": 6275, + "03-a5-c3c996bd80d63c6c915f4c54742e-0001": 6276, + "03-a5-f42c4fb5e5a93229b6ca2d74c1cc-0002": 6277, + "03-a6-10d5b81c92c3b1cf2f7e0b5c0d48-0002": 6278, + "03-a6-3e356b4677e2a01b7e03288bff24-0001": 6279, + "03-a6-57428b717a8c4dd0782f87ae48ce-0001": 6280, + "03-a6-7f1cac4b7445e784abb8c6972538-0001": 6281, + "03-a6-9b6a1e18edda2f42f7b0ab547de4-0002": 6282, + "03-a6-c0032584c56d26876c186e7abc6a-0001": 6283, + "03-a6-c30b958577c0b9af1369994c1f9b-0001": 6284, + "03-a7-3671a125444a701b11644be1bc6b-0002": 6285, + "03-a7-5d37286a351fc67a21dfe72468f2-0002": 6286, + "03-a7-a3c25d351264489c6fdafddd1240-0001": 6287, + "03-a7-b4189314efc637fd713d22b5822d-0001": 6288, + "03-a7-ba350f54ae337f22f5699170c20a-0001": 6289, + "03-a7-bfc7986bbf902058498b7ab6da08-0001": 6290, + "03-a7-df9588e3d9957ce92f29ea142c73-0001": 6291, + "03-a7-f7912665ca7851088943e53db4d8-0002": 6292, + "03-a8-0b88c6cf9cb09bd2e84c9a27c53c-0001": 6293, + "03-a8-1fba6f39bca6b3a316e28294e2e1-0003": 6294, + "03-a8-48b226d4343145aa6b371abe9765-0001": 6295, + "03-a8-4e70921b9d44e1f824db6421e385-0001": 6296, + "03-a8-7782614b909b2ea16395b06c4345-0001": 6297, + "03-a8-b00e5192ed1d0895535bc8d51199-0001": 6298, + "03-a8-b932c6fec860d7458a0402aa381c-0001": 6299, + "03-a8-db2f7e98dc04d15bbccec9b25340-0001": 6300, + "03-a8-e974df718fe2204ff55e6332eb78-0001": 6301, + "03-a9-0b029537fe66904bef0904749a2b-0001": 6302, + "03-a9-399af22d587655ed625f1d238bcc-0001": 6303, + "03-a9-438e8e6eff6b03c2efcb3f5a76a0-0001": 6304, + "03-a9-6768e7e914be631d291a7088fde5-0002": 6305, + "03-a9-7107c00bdf59bd0b908059486f34-0001": 6306, + "03-a9-7baee7647bd62a35b04ec27eb33d-0001": 6307, + "03-a9-8fbda050ae63806e2e2f81c0a7c7-0001": 6308, + "03-a9-a3db964d48da150a5eaf6ef92970-0003": 6309, + "03-a9-a592e5620865c8204d8504d56725-0002": 6310, + "03-a9-aa46424f1221b3e1c7e32df9697f-0001": 6311, + "03-a9-d67fb49f063a57d4f69c77693f0b-0001": 6312, + "03-a9-f6b989ae5e87eadaf3f9064637ef-0002": 6313, + "03-aa-0ced1093587d47f4cb03537f571a-0001": 6314, + "03-aa-4a74c261e59719cb8c103994140a-0002": 6315, + "03-aa-67fcc3bfd021cde49bee911a5a1a-0001": 6316, + "03-aa-7ac8193fffda33eaf339fcbf8f9d-0001": 6317, + "03-aa-894710784f613f15f157ec7ffee0-0001": 6318, + "03-aa-898b3d17ed17a61b4275e6ead7a1-0001": 6319, + "03-aa-95077a92a35f40428d46d775ca0a-0001": 6320, + "03-ab-07d771450497f04f0fa09e84f40d-0001": 6321, + "03-ab-07dfebe8b633d59950db4f6db026-0001": 6322, + "03-ab-0ffb72ecc32e98e9cf4208b325b6-0002": 6323, + "03-ab-29aadcba034a71ab8e850bd580f5-0001": 6324, + "03-ab-5e32d3b63414fe9ff7ff7b71d473-0001": 6325, + "03-ab-79d338af01faf732e7898a068188-0003": 6326, + "03-ab-82a2f1ec875e57ce758d26c35ac3-0002": 6327, + "03-ab-8ccdd2c0c1fe4de0b6f4df171804-0001": 6328, + "03-ab-acf6fbd04368881821aeecda5cc9-0001": 6329, + "03-ab-cd89cea571b9da125cdfc92ee035-0001": 6330, + "03-ab-d9218874bbcd545f65dc50e019ae-0001": 6331, + "03-ab-d980361f4e8db883c06dc47980a1-0003": 6332, + "03-ab-e3b82500dc50e43aca6014141c5c-0003": 6333, + "03-ab-e6f949c035ccf9f3b8fe885a78d1-0001": 6334, + "03-ab-fd9d41ac4c0ad15c8c57e0d300cf-0027": 6335, + "03-ac-1034e16c17d1ceaf0d947d58e88e-0001": 6336, + "03-ac-1f293a7429cada852e86b5f3fd0e-0003": 6337, + "03-ac-2445f9b224377ec01d87964cfa2e-0002": 6338, + "03-ac-3d8b8c9e4041f6905fe7118a7dc3-0001": 6339, + "03-ac-5ff83e65b909da5dd1a4f4bd11ab-0002": 6340, + "03-ac-93a24728186b06fb0d3807811a07-0002": 6341, + "03-ac-a818f86fd0775284451df1b0bec9-0002": 6342, + "03-ac-b8e7e59d13e2a1dd10844a726574-0001": 6343, + "03-ac-c03e789c757b8f7fbeff97c3b60a-0001": 6344, + "03-ac-c223d75d09415bd5fdd62e21490f-0002": 6345, + "03-ac-d63a81b85b5940aa0c41d2fc7863-0001": 6346, + "03-ac-d6e6b4f7f826e9624d69165b5643-0001": 6347, + "03-ad-1623316c6c3a116fc3e2784b7b01-0001": 6348, + "03-ad-416c5525f4f782a87e9340d4e94b-0001": 6349, + "03-ad-493cc5defbb2e72f0566ce1cd86d-0001": 6350, + "03-ad-8ebf578a9d95411fb8d2d1a0634b-0003": 6351, + "03-ad-cbe70790f3e0bd30a0a9f6b2fd52-0001": 6352, + "03-ae-01f4701e5ffe5942aa7d47d215a5-0001": 6353, + "03-ae-4cd988770fb4dd108f710ac93e1e-0002": 6354, + "03-ae-5a0efbf76477d1a4404a426f31ef-0001": 6355, + "03-ae-5e16a3bbcb0992d30618ab00fcaf-0003": 6356, + "03-ae-671b36fc9cd9dcfeef812cd11ec6-0001": 6357, + "03-ae-8e9575a4e06e2a13c49fc5b1d057-0002": 6358, + "03-ae-948cea14d62f7655cee89aa4cfbe-0002": 6359, + "03-ae-b38cda6b964a102c02241eb0f8fd-0001": 6360, + "03-ae-de1d4dc7f61aab790b7902e1a914-0001": 6361, + "03-ae-ec22583895761ef720b7a1c3afc0-0002": 6362, + "03-ae-f5321216d760a155f573dec1a76c-0001": 6363, + "03-af-067e314a76019027248ce46d3923-0002": 6364, + "03-af-0afbbca4d1c12532fd11f2d44123-0001": 6365, + "03-af-3e152fee0b2431338063207bd8f8-0003": 6366, + "03-af-50a785f96db935fc4cb2d9bd153d-0001": 6367, + "03-af-68e334915d607e7a3611f43559fa-0002": 6368, + "03-af-cdf2209c62f2926bc61b06887787-0001": 6369, + "03-af-e0b5db91c0358f7a8de53b19e6ee-0001": 6370, + "03-af-eb30a04388be485daab5129e4472-0004": 6371, + "03-b0-3b0a05ad7738f91fedf1b133ea95-0001": 6372, + "03-b0-40fe1ba65ad66fbebdbaed775f92-0001": 6373, + "03-b0-46de3784150a5b18a70fb626ae1e-0001": 6374, + "03-b0-49ba72929255c51c618850b9dd4b-0001": 6375, + "03-b0-6ecac38fa2159f1b6a784cc7f78b-0002": 6376, + "03-b0-9b6745ef6a64618990938c4bf499-0001": 6377, + "03-b0-b75e025b1223e014dada4fa459e3-0002": 6378, + "03-b0-d0dbe3e862a55063befe442c94b9-0001": 6379, + "03-b0-fe9c2903a0fc0a3f07795fbd5336-0001": 6380, + "03-b0-ff2fe1c6a784b7682c0822282949-0001": 6381, + "03-b1-2042a353ebf8e743b65755283535-0002": 6382, + "03-b1-34b18e12f5afd90e901994d09d09-0002": 6383, + "03-b1-7fc6d7a292d7fb35cfdf77d49f1a-0003": 6384, + "03-b1-a6a6daae98620aba56789d119afd-0002": 6385, + "03-b1-ac39f9013bef20b8800419e1f878-0003": 6386, + "03-b1-b2ed4400c2d840c8037587ef9d1d-0003": 6387, + "03-b1-cd7ad4d730ec938027ab3c051cc1-0001": 6388, + "03-b1-e77e08f41ad1e661632fe563ad6c-0001": 6389, + "03-b2-18f42be5707e2a2dc79efbf317a9-0001": 6390, + "03-b2-2ab8dacd771b34335dc48e0a3600-0002": 6391, + "03-b2-6bd281fda845d44dd5abcaeaa9df-0001": 6392, + "03-b2-78b81211c87dc1be7ffa094a6231-0001": 6393, + "03-b2-79c1f2e5c90911573fb0127bcb87-0002": 6394, + "03-b2-cba124143dac2660b1ea6c144c2b-0001": 6395, + "03-b2-e0b5090a7a9a7ca5a0c9bf3b15aa-0001": 6396, + "03-b2-e75ac4f8b1b320808f56b1170435-0003": 6397, + "03-b2-fc81d4229f86abdef3f79a1198e8-0001": 6398, + "03-b3-05a206ff6122633d99a2af158a99-0002": 6399, + "03-b3-5597bde0c3cf9cc5fa9f49ddf6ff-0002": 6400, + "03-b3-88a8f88acc6695bbd71cbb02c5db-0002": 6401, + "03-b3-94fc62d3e42677127b83602ba6f4-0001": 6402, + "03-b3-a86774090e173cf906268934548b-0001": 6403, + "03-b3-b41f6f0d2091f8f7d946e07de1c8-0001": 6404, + "03-b4-129293a30917a3713876949aa2d6-0001": 6405, + "03-b4-1db69fb9d17d01a3b8c0ff4b7b4a-0001": 6406, + "03-b4-1e273b6d66acf99d85a3a0643046-0002": 6407, + "03-b4-2f9e746e5092e229d71150626b70-0001": 6408, + "03-b4-66c6c1943089becf395b16036e20-0003": 6409, + "03-b4-6f444de82a5e41b9571ba176e895-0002": 6410, + "03-b4-7ff6133bdf04c01587dc259ff86f-0001": 6411, + "03-b4-c30f00b69dd2bc82b75b83842787-0001": 6412, + "03-b4-e02b28833e826f41c6fea1e519aa-0001": 6413, + "03-b4-f92f5ee83d6dc1b3b55e77783e05-0001": 6414, + "03-b5-0e7b8c6cc77dc1ff152955205662-0001": 6415, + "03-b5-3b196f03ea3276fd6d2d522c6f4e-0001": 6416, + "03-b5-4c88dd71e498d2232adf729c2b61-0001": 6417, + "03-b5-91335d55fc38dc8087b8758dc609-0001": 6418, + "03-b5-b812c9b88ca0c0789df1fbb81a56-0001": 6419, + "03-b6-3151b7649c31785d530df667e874-0001": 6420, + "03-b6-3cb3c169741a48934a1f65844441-0001": 6421, + "03-b6-3d7af935675143aee58d4b210440-0002": 6422, + "03-b6-5d0c4d3e0077bcc0ddd86512af46-0001": 6423, + "03-b6-88d207a0731e76f19823c4f7f642-0001": 6424, + "03-b6-8f79a833906084dc3bac4045c3b2-0002": 6425, + "03-b6-cedd36cd33ee34d4f1609bfea23a-0002": 6426, + "03-b7-146677f141d40fc3891ac17c76ec-0001": 6427, + "03-b7-31a304c8fdae013c3b62b7da31bd-0003": 6428, + "03-b7-ebdd2cb7759033d785f27140a0db-0001": 6429, + "03-b7-f1042bacdf472e47d0aef28ad44e-0001": 6430, + "03-b8-345b07a003241de2290a09ba2382-0002": 6431, + "03-b8-6bb63a012c59d759e6496d1a9409-0002": 6432, + "03-b8-9af11a09317353a2d5acead251fc-0001": 6433, + "03-b8-c997c8708f9d83feba3c1f495e5c-0001": 6434, + "03-b8-e5a1702299033eef696d831ef0c6-0001": 6435, + "03-b8-f9ed72cf041d6582d6c54d35e1af-0002": 6436, + "03-b9-006946859551d90d102bfdd32353-0001": 6437, + "03-b9-0664a088c4c3b8470e81b33135d3-0003": 6438, + "03-b9-17f93e8575347dd54cfd7d9f97bd-0001": 6439, + "03-b9-8a865d2c01ff8f310f639489f676-0002": 6440, + "03-b9-a3989a1c7943f3b835d3a1ad6541-0002": 6441, + "03-b9-c0e2f6d09ab4eedd00365909a011-0002": 6442, + "03-b9-c1d49ff7c2e16236cf0909d8af8a-0001": 6443, + "03-b9-cbd50ae7d3bde6be7520d8b0b5cc-0001": 6444, + "03-b9-dfcd166df97a3ac6eacd9778f384-0001": 6445, + "03-b9-f702ac02028964851829f37bbc18-0003": 6446, + "03-ba-14d4edeff8dccb56c4e1f286aed9-0001": 6447, + "03-ba-2214bdb8087ddf972e88c432e6cf-0001": 6448, + "03-ba-293c8df1055a8d1e7228c4e6030e-0001": 6449, + "03-ba-8702345891bb8e61faa84e1a85c3-0002": 6450, + "03-ba-94fe566e6a17155e15d62db8e467-0001": 6451, + "03-ba-95a1a90fb97040ee9fc5034bbf1b-0001": 6452, + "03-ba-a68d9a72161bccaa5306db0ee977-0002": 6453, + "03-ba-a7f58b199d0e1b87a32d38674ab9-0001": 6454, + "03-ba-ac5c3b151d91bc7181f3667f7e25-0001": 6455, + "03-ba-cbcf30a2a7e76388954461125d76-0001": 6456, + "03-bb-085736f5860d93348b73b068b437-0001": 6457, + "03-bb-09bf216af4f05620dd2f532c0301-0003": 6458, + "03-bb-0aad9a0372e4e6ced7154dd75ce0-0001": 6459, + "03-bb-35144d6aa6e1689c7e5a32a62620-0002": 6460, + "03-bb-58d2b01b0cda6ea96858fca44d04-0001": 6461, + "03-bb-a4fee024ba095ad846430de0c9a0-0001": 6462, + "03-bb-a9c86673b91bb7dc844413188493-0001": 6463, + "03-bb-e3ec4a7fad9485bdba83a4553604-0001": 6464, + "03-bb-edc2fd9c264ba13a97e426abc756-0001": 6465, + "03-bb-f5b1c08d507b13be64a4783f36b4-0002": 6466, + "03-bb-fa5792d75429b1e04db2cab07668-0001": 6467, + "03-bc-279f9436e426839975961f5e0e93-0003": 6468, + "03-bc-4a7d5d695e248981d149bd426886-0002": 6469, + "03-bc-4fb5f56f2bd9fda1075968e83ae5-0001": 6470, + "03-bc-55ed63a1fd4a1edbcc78c30c384e-0004": 6471, + "03-bc-5c4a3c944eb0c3c9682a0a4ca0c4-0002": 6472, + "03-bc-5f1aea0b5a736a04c3f5b4664508-0002": 6473, + "03-bc-6aab5f0cbcc2f2b7548c5f856b22-0002": 6474, + "03-bc-8062fbd6aa1002663abddea3ab71-0002": 6475, + "03-bc-c763ccf7ae7fdd22c72c91ff67ca-0002": 6476, + "03-bc-f8a7ca6a5f084264174bb7d0b8fc-0003": 6477, + "03-bc-f9edf922938694104473472e80ba-0002": 6478, + "03-bd-0968b02e7704353d30492dcb240b-0001": 6479, + "03-bd-39340d8f9f2c64aa3d398c2de015-0001": 6480, + "03-bd-8ddd6127d9a70b2eafd4facff034-0002": 6481, + "03-bd-939920e46361c336771f4a9002df-0001": 6482, + "03-bd-97375d5325f784700cf9a87f5444-0001": 6483, + "03-bd-9adda03ea4a918f7a7be28f2406b-0002": 6484, + "03-bd-bd2c1a3f48382ae3c083555d2b28-0003": 6485, + "03-bd-c11c037310b7d03140b5ae26a519-0002": 6486, + "03-bd-d913bc9a11502146cda14c56df2d-0001": 6487, + "03-bd-ebbab2f01e5d7e6e2d69f9a73854-0001": 6488, + "03-bd-ec1dc862da9abb6ac73635bf7304-0001": 6489, + "03-be-0e4a68544fe3467bf25ef8aff8fe-0001": 6490, + "03-be-11a5a803716ccc5e90bf835f1a49-0002": 6491, + "03-be-3010935d07e00c7891f2be7ee609-0001": 6492, + "03-be-3b7ccafb198c7db280f39e3d2153-0003": 6493, + "03-be-428a7991213b30be44084bd829de-0001": 6494, + "03-be-448b95e9d29c683b3d98bbc20131-0002": 6495, + "03-be-64af08e661020bb64649dec55d8a-0002": 6496, + "03-be-7c3b079d9e61b4183acc6ff46371-0003": 6497, + "03-be-b4053daa24241056e571ebd3a8d3-0001": 6498, + "03-be-defc766232926504be8620aa5457-0001": 6499, + "03-be-e719d9464534b49f7a9b74b34d68-0001": 6500, + "03-bf-4d32d639c761b2fd65cc2e0f4df7-0001": 6501, + "03-bf-63cbb82f805cfddb39a13d8805ff-0001": 6502, + "03-bf-9daaa18bdad2cba09e21f050f327-0001": 6503, + "03-bf-ddee8e14b5e39efd0b466c3ffb86-0002": 6504, + "03-bf-e4b8aed48a9ee13a727c465d56ba-0003": 6505, + "03-bf-ff5f9b038557ce68dc6f5d12aaa6-0001": 6506, + "03-c0-2b2a0b93e23b99fe8a21646c93ed-0001": 6507, + "03-c0-2c32cb8e9d6505846178969780aa-0001": 6508, + "03-c0-3ab6e001628f069a1c5a25cfaac5-0003": 6509, + "03-c0-7296fb8f29b8ae2f4080f0bda58f-0002": 6510, + "03-c0-89c429cfb94c356b19c835bfa1a8-0001": 6511, + "03-c0-a7f6e81fb47cf7c7c3c83c5c89f9-0001": 6512, + "03-c0-ab149e41a205e113865c1bf8452c-0002": 6513, + "03-c0-ad8e42682db7e024225d78a77402-0002": 6514, + "03-c0-b8f1e23aff170edb75f99ba50590-0001": 6515, + "03-c1-21a3e8acc5c514ab85132f221f1b-0001": 6516, + "03-c1-418b9b83dd8da1db927dbf6d50e3-0001": 6517, + "03-c1-6054e15f697aa416629279d26de1-0002": 6518, + "03-c1-81d34be6bb99ce15432223ad7a64-0001": 6519, + "03-c1-8aace69f444e673a2a65b04493e2-0001": 6520, + "03-c1-f4e1daac90d24d65482621cb4f12-0002": 6521, + "03-c1-fce16fd20cad2b25451d290b8689-0001": 6522, + "03-c2-216ed420f396c0ec8a07fe7fb7b5-0001": 6523, + "03-c2-4b171eb43c1cc921b2731f40bb7f-0002": 6524, + "03-c2-654810ebce017fe8ab5ba5b95cb0-0001": 6525, + "03-c2-7f683c91b3d91d58ede260eb4edc-0002": 6526, + "03-c2-85680368b9b35869cf0a6e476674-0001": 6527, + "03-c2-9ee8d1bb6b13a1847fbd0eb21298-0002": 6528, + "03-c2-adef533a24907810e2a186d705ff-0001": 6529, + "03-c2-b54495f99ff481db119334edcb16-0001": 6530, + "03-c2-fb4cfca7cd8c545fdc975c61e69d-0003": 6531, + "03-c2-fb70f03ea9f674d041d2733538f9-0001": 6532, + "03-c2-fbd4dc94006c2e1f75adc0e3986c-0002": 6533, + "03-c2-fe35bae9eab8861e34ab5506fd08-0001": 6534, + "03-c3-512fc6cbdc7a29cce503f3dec67a-0001": 6535, + "03-c3-53492ceedaf6aa8dd063299997b4-0001": 6536, + "03-c3-8f697e604fa65c20deddff50d44d-0002": 6537, + "03-c3-9b13f3740af7c03590f6e68b1a76-0001": 6538, + "03-c3-a6c467a356fe5e6264fe7c0bdca8-0002": 6539, + "03-c3-af8e329a19b3ab73469240e5d525-0004": 6540, + "03-c3-ceff05ec3a247969b55a8079804b-0001": 6541, + "03-c3-dea642ed67f81dc4a6a044d1c9c7-0001": 6542, + "03-c4-123d71c4aa40cfc9780164a7112c-0002": 6543, + "03-c4-3e8f91e8154b6e729448d4e42fad-0001": 6544, + "03-c4-80565853bd4d6ce7cea322dc3d6a-0001": 6545, + "03-c4-af4aabfffd36b1242cf6df16a40d-0001": 6546, + "03-c5-46c2c7ba828bff3536f9e192e220-0001": 6547, + "03-c5-4a4ba6f9fb28cb17e5eb1f4dfab5-0002": 6548, + "03-c5-6f8cec0942cc95180525b305e5a3-0001": 6549, + "03-c5-8db03f346c83f737fb464ee5115e-0001": 6550, + "03-c5-c4a3b2626bdd810b9e367ccf947c-0002": 6551, + "03-c5-f6ccddfcaff9f8570ae8ab6b46f6-0003": 6552, + "03-c6-41876d4687214c3dccdc6338c20a-0002": 6553, + "03-c6-7385d5d585fef4194feddeff9cb2-0002": 6554, + "03-c6-aa8b555afb2c93ca6e075c92f3ef-0002": 6555, + "03-c6-b0a223fbd21040d09af8c2d9eb3c-0002": 6556, + "03-c6-b8acf891eaf4ab8f8137ef2bfec0-0001": 6557, + "03-c6-b972cd96c336b3561682c2328a18-0002": 6558, + "03-c6-eb138d06e29228b2852985f9b23f-0001": 6559, + "03-c6-f83d39a4a466818a15d53a117c7a-0002": 6560, + "03-c7-1d154668cecdcf5916c5957448a4-0001": 6561, + "03-c7-500173bee12c10a5063004a68962-0001": 6562, + "03-c7-8f5fd78ae62af0b4589c528f4e6f-0003": 6563, + "03-c7-abd08c2203152fa2425ce3ef6df1-0003": 6564, + "03-c7-caf2c1099ed28b18b56c330071d0-0001": 6565, + "03-c8-0319c03e098f3869de9377a39f58-0001": 6566, + "03-c8-38ec382cee1c3b1424db66ef51c6-0001": 6567, + "03-c8-50dac6c474f22c91d8b541773bca-0001": 6568, + "03-c8-5c9bafc8a0088c40c301dbf41906-0003": 6569, + "03-c8-ac3c08d5dc49f07a430c5aefc116-0001": 6570, + "03-c8-bab65289fbefa5af844b0f664073-0001": 6571, + "03-c8-e138cd96098110a79dc0318dc042-0001": 6572, + "03-c8-e890a73ad502cb331e627f6e7946-0003": 6573, + "03-c8-f3a3cba5cbb090e611605c830ad7-0001": 6574, + "03-c8-f93956f5a338145ccaee0abaf137-0002": 6575, + "03-c9-07cab7a36599d236a127133a4b5d-0001": 6576, + "03-c9-4f5948a2185b8dbfa197a8822c76-0001": 6577, + "03-c9-4fa74d43a5d6b371c19afd8a3c14-0001": 6578, + "03-c9-8c3f9c150bd6664d3b68d24c3460-0002": 6579, + "03-ca-0735d2561566f5571b118f19a4b2-0001": 6580, + "03-ca-1b1f16415efabfdc7f78171407fb-0001": 6581, + "03-ca-3f9f7e5745cb7b701a0c4ca298ff-0002": 6582, + "03-ca-461df31a27a333be59e6f029f6b7-0001": 6583, + "03-ca-9503c4d624d010f450fd95f97f52-0001": 6584, + "03-ca-9a4ba238f649bfec734f44192035-0001": 6585, + "03-ca-a8f56bc251b4623cf94ac6374dd2-0003": 6586, + "03-ca-e6904b19c05634e4b90e1f9cf378-0001": 6587, + "03-ca-f1b747f8970ea7c7419ebd02533e-0001": 6588, + "03-ca-f553530a6a5e96617ae5f9ba0cf6-0001": 6589, + "03-cb-000a0aadb373b5ec749735ad5b07-0001": 6590, + "03-cb-05b34bfab36f11800497f2d8b7c4-0001": 6591, + "03-cb-1149e12d6377d58a4fc76b4261ce-0001": 6592, + "03-cb-138c6324d0a02729a20ae70f7620-0001": 6593, + "03-cb-204354390d14f0de6f0998ca0434-0001": 6594, + "03-cb-326d3bca3cc8d3eef00f1cd1b185-0001": 6595, + "03-cb-5a77b637598d361c0ce95fcba3e2-0001": 6596, + "03-cb-604e78be168e2f3e74711d8e2d99-0003": 6597, + "03-cb-bc60036b7d0e7ffd65c0d5a495a9-0003": 6598, + "03-cc-0d7126fdba28b26d91f353277ff2-0001": 6599, + "03-cc-5bb3d5d3bfe5a8e0564a8d572536-0001": 6600, + "03-cc-6b6b7f1a6b905096f32665f52f99-0003": 6601, + "03-cc-6e5508fd91e703d6de489c92ce6d-0001": 6602, + "03-cc-95cef57d328701fbbd79afbd32d3-0001": 6603, + "03-cc-d120fdb913099d75b588fb393806-0001": 6604, + "03-cc-e6f6a2fe9063b6357da5240b936e-0001": 6605, + "03-cd-2ce1c9c22b9a092c0d4c967a0f4a-0001": 6606, + "03-cd-4635a2f94e3c5d9c1fc91ec4c971-0001": 6607, + "03-cd-4f5739c359fe81845763e65c8eae-0001": 6608, + "03-cd-57dad425f3697690190f6fb07b54-0002": 6609, + "03-cd-836ff145e05bff92cf1454883807-0001": 6610, + "03-cd-9e4be6a5cc886f2a018c5532f163-0002": 6611, + "03-cd-a31cbbff1d377d9878407caf4551-0003": 6612, + "03-cd-ca32b3d2b36f202951ef3ca9dc6c-0001": 6613, + "03-cd-f3d33737115130884590208efeb8-0001": 6614, + "03-ce-18e5a7f995f3e8eb4f7297aa9a39-0001": 6615, + "03-ce-1fcf378bd241bc85a9c9bfde030f-0001": 6616, + "03-ce-25e5354f73b6ddc2984b9c5f621a-0001": 6617, + "03-ce-4d18c1ee0c87f8bbb3c33086675d-0004": 6618, + "03-ce-72ebcd3d73709a51e407d02d4a07-0001": 6619, + "03-ce-c12594004809a313da949b49833f-0001": 6620, + "03-ce-c4acb250f706b65eeb9538666d3b-0001": 6621, + "03-ce-d193458b19d62597fc93e4300fc9-0001": 6622, + "03-ce-d2bb5def455d06f3f1946a953a9a-0002": 6623, + "03-ce-d2eb1f46343f4bec560c51a9d191-0001": 6624, + "03-ce-fa07a0f1120e0d573aef21036eae-0002": 6625, + "03-cf-2b2fde1166d582ab07678b3b68d2-0001": 6626, + "03-cf-384ff74d3473c0e4972462a755ce-0002": 6627, + "03-cf-3c5efa159d6cfa45afb7a9f67156-0001": 6628, + "03-cf-429f3453f3018b4f7aeb7b3db0e2-0002": 6629, + "03-cf-4b26f90f860ecf77c600c44afcc8-0001": 6630, + "03-cf-7e73fb3fa7c7d9a294a88f1c8f4d-0001": 6631, + "03-d1-642802d2242e48b8230f0093404b-0004": 6632, + "03-d4-3900a48f13423e8197e574c03702-0002": 6633, + "03-d5-1407388007df7599d24456d8b49d-0001": 6634, + "03-e1-b8dd3ef6e2987cb0f2bb5b5b2fb7-0002": 6635, + "03-ec-e29f26379c462419dc0ec066d4cd-0003": 6636, + "03-ee-735d89466c9e3580e1216032c362-0003": 6637, + "03-fc-9799762cede9471318ccb90c5090-0001": 6638, + "04-04-30f571e872d9937a40e2d2ebc99f-0001": 6639, + "04-07-8b3a35a7cbd5a0fcf0e71e3a444e-0001": 6640, + "04-08-e86b4dec79263962d09476b37c6d-0002": 6641, + "04-0d-03fb6ae2717595f685b0a747411b-0001": 6642, + "04-10-3dcd2c09205756dbb0df8ea39d69-0001": 6643, + "04-1b-1276b28c987f29824c9a29147ba1-0001": 6644, + "04-26-db22eab05be5e8d64599357c3ceb-0001": 6645, + "04-37-bbe093d7b922dc8920fb4bf0611a-0001": 6646, + "04-49-838f6befa14ce26b0cab69d509cd-0002": 6647, + "04-4d-1a51ebbf3334cb19c44cf3b99023-0003": 6648, + "04-4d-5089089d5d794270f9202f5c657d-0002": 6649, + "04-6a-15dcfc83adf9a7074e37b4dc795b-0001": 6650, + "04-73-6a8db4283cb6431b1f1049e82279-0002": 6651, + "04-74-68495767564859d4a3f0a750bc61-0001": 6652, + "04-7d-733ab00b65ffe85cdc21b370ca4e-0001": 6653, + "04-7e-81af99d038fe613747802a789d77-0001": 6654, + "04-88-1badc19186f7b5cabe11ba5af8cd-0001": 6655, + "04-94-d94476c98b1ae4f4ef86df9d0173-0002": 6656, + "04-bb-12a1baab66d8835eee59ad928748-0001": 6657, + "04-bf-3923093cc8dae93048609f5be910-0002": 6658, + "04-cb-572fb5c98f64ec61317331d4b1fb-0002": 6659, + "04-d2-61001795802270a09fa1233d07fa-0002": 6660, + "04-db-1d11d85230d2f01ae795d5dff253-0001": 6661, + "04-e5-5e5400e8fad94326e65fcc4b70c5-0001": 6662, + "04-ea-b04fc0d140bf259d16fc5dc0c6ab-0001": 6663, + "04-ec-23ecb22ed85445e08fc7b340d9d1-0002": 6664, + "04-fb-2b64f1218e54274cef2de618d5c2-0001": 6665, + "04-fd-9bec718f20c9e95a3ee372f3b9ec-0001": 6666, + "05-12-2425a7eaaac35bb058ef183f1772-0001": 6667, + "05-35-11911f308db3cfc7121b6567aa02-0004": 6668, + "05-3a-df5da22daa370eab4cf20fbeee55-0003": 6669, + "05-46-792a254bf611d48a028f650f9bb6-0002": 6670, + "05-69-dfe65b787e2fcb6169bf3b8926a8-0001": 6671, + "05-75-6abad30954133758143206d5fdd6-0002": 6672, + "05-75-cb2292ea009c91346fcaefe0f627-0001": 6673, + "05-85-59e19f606a1df76cbfc918ee0daa-0001": 6674, + "05-87-aca01b30d853de14f17dc47bff26-0001": 6675, + "05-90-0868442a7bcbd8f6d16ced2d6649-0001": 6676, + "05-9e-758a25df4d2c744aa48a3644eda8-0001": 6677, + "05-a1-41ba3188a764b11abaa28ad86846-0001": 6678, + "05-ab-94bd2b9bc0073e697b76fa651411-0001": 6679, + "05-bc-873d7672685834b351b81ea92bb2-0001": 6680, + "05-c1-cec0dccda37cdc89ac731b044fd8-0001": 6681, + "05-c5-8db62157ae337570bc2ff83e868b-0001": 6682, + "05-fb-967c900a7aeb570719f1175a85a3-0001": 6683, + "06-3d-5dfeb988fdbfa5b302880629c2be-0001": 6684, + "06-48-717d58163d93c81d3f60981dbee1-0001": 6685, + "06-4f-28800b60363f247a3ee0706f1073-0001": 6686, + "06-50-779554632689a26cabc1e9f08638-0001": 6687, + "06-66-647551f6498f9cb970cef09f0ed5-0002": 6688, + "06-68-a7ca89fa3a77625600bab0d7e066-0001": 6689, + "06-69-82a81996b5da3634d043672bad3d-0001": 6690, + "06-6b-a9b95de2ea3fa9de76d03d2ac95f-0001": 6691, + "06-70-7fed526abcdf946258d8dcbdfd35-0003": 6692, + "06-72-c729204e76fb86504351ad3744cf-0001": 6693, + "06-75-a02f22967bbe35d78ec932410dfe-0003": 6694, + "06-7f-da8ab37d5a2b159805cfb8336023-0002": 6695, + "06-88-f4993c3cfb47bab0a51ad04c81ac-0002": 6696, + "06-8c-502e91b6056ab1bb1a1e8fa2221e-0002": 6697, + "06-9d-ea946858781903fb20a6d931eb26-0001": 6698, + "06-a6-1357971eb9b0dc3ebb946af6b695-0001": 6699, + "06-ab-8fd45745b84519e7579a41b252a8-0001": 6700, + "06-b3-b7e902b260f79b73c375ae0c8ac9-0002": 6701, + "06-be-3f7401171a71489928050a0fcb21-0002": 6702, + "06-c1-a17649ce2365a588d94188cfe22b-0003": 6703, + "06-c4-89612e0365c3c9f60a1c028b101a-0001": 6704, + "06-cc-22fb2a27c922de45d10f7660826e-0001": 6705, + "06-e2-ec76f9564456280dd2f5efadf534-0002": 6706, + "06-e9-1760c53bcd8c2a77beec85d520ec-0002": 6707, + "06-ef-9a37733233963ca28afdae0d8807-0002": 6708, + "06-f0-360b15505448c4a74607ddba7142-0001": 6709, + "06-f6-81043c4ada7c065871bf01f6bf94-0001": 6710, + "06-f6-868f0d470633f3a5242781cd1bd8-0001": 6711, + "06-fa-9896fe36990fed86241e4f32dd53-0001": 6712, + "07-0e-cbffe81c5780232e27075022a085-0001": 6713, + "07-1f-ecd3f404cc58782b6df30081b1e8-0001": 6714, + "07-20-ae3e4b247f82b277cff337079028-0002": 6715, + "07-24-57f257194328b2c08b752d3ef525-0001": 6716, + "07-35-25b5cec7e305856f2051d63628f0-0002": 6717, + "07-46-2c73453c94df937779f483b2d6bd-0003": 6718, + "07-4c-96721d339d130fc43f051f83f764-0002": 6719, + "07-4d-122f5d9cf0f83cdf1f933bebd2ad-0001": 6720, + "07-4d-cb5426dc7569486ad33f81ef288f-0001": 6721, + "07-51-ea89f5b59cd6d52dfb87a5c45d89-0001": 6722, + "07-5d-93254aa2510deb75702be2ad57db-0002": 6723, + "07-62-d7ef48af135ec999781084253ccb-0001": 6724, + "07-7f-f1d6c81507cb2bfb3e164b4f418a-0002": 6725, + "07-87-e92ac5cb21146c504867a91612db-0001": 6726, + "07-92-a2212f2f62bec319f1764ab0aa1c-0001": 6727, + "07-9a-f773f7645a8ce1affcdfb208e9a7-0001": 6728, + "07-a7-199b85c1afe7b75ce8f2f7b05c1d-0003": 6729, + "07-aa-b4675c6eaa10a3f1669c10abf28f-0001": 6730, + "07-b4-14b13fceb85eac0a01392e494a09-0001": 6731, + "07-bf-52c8be5d151753d73029a4c62bc2-0002": 6732, + "07-c4-9b8fba8cd1c9afe3700e9988a9c5-0002": 6733, + "07-d5-927bbfdd0d9e2fcb4c612de6e8b9-0004": 6734, + "07-d8-ce50bf1ed0bc2b86fd22505fa779-0001": 6735, + "07-e1-84737853445f94e988b75ad2eb1c-0004": 6736, + "07-f5-cdaf48e36956875eacc24a7341ee-0001": 6737, + "08-05-7932b69fc5113c66833bb93f111f-0001": 6738, + "08-0c-634330d276865e070c1ddf9bb4ec-0001": 6739, + "08-12-add1c467fa521d285be49f7e1b69-0002": 6740, + "08-17-a049c6a81ec94fe71d0bf8dda811-0001": 6741, + "08-1c-9fb212647e25f2f585835ade2c6b-0001": 6742, + "08-20-59719a31544754e14fc3f555010b-0002": 6743, + "08-32-4cf27f5814e5d3293146714c67ed-0002": 6744, + "08-3b-f090908a67098e3b76c026e75373-0002": 6745, + "08-3c-4144463ce4d677723e9032358408-0002": 6746, + "08-3c-b1658f6e2b433c8037b57b7a15a0-0002": 6747, + "08-49-bf2507c6b912dab4c860659d8cb1-0002": 6748, + "08-4d-86eea8729293fbaa091bc0f1e2a3-0001": 6749, + "08-51-98ee5c0db4d4503e8b3a8bbc51dd-0002": 6750, + "08-61-9140e17d7885b7d8e3b07934899e-0002": 6751, + "08-63-14ec2217cb50a7b6c5b4de4b8fa4-0001": 6752, + "08-74-6e07886c5fe5ea8cc4197d8ba306-0001": 6753, + "08-77-b905fa1e9a3e38c1e00d15e69cd9-0001": 6754, + "08-7e-55443068a3b4376a39fb662803f8-0003": 6755, + "08-87-04e5dd15532a656fce62a06c80f0-0001": 6756, + "08-8a-77cca949984bbb49b308d59e452e-0002": 6757, + "08-8f-2ffd79a8cc123db3e54b06754c09-0003": 6758, + "08-95-0e0165840673d0e6b8091ce6888e-0001": 6759, + "08-97-66588b651a7fa94341ab555eabf0-0002": 6760, + "08-9e-aff51b0297c53d8d602bcdd04d71-0002": 6761, + "08-a8-dd045f1eaefc537e697cd62d7a02-0001": 6762, + "08-ad-7088112eefb4981180734b8bf544-0001": 6763, + "08-b5-9e476443e06469aac0035f0688c0-0001": 6764, + "08-b8-66867f8036b4a6b752ea446b9944-0003": 6765, + "08-c3-3c50e53df3ae7394e686cfad4821-0001": 6766, + "08-c8-df77e47600361320b53bd5050f1d-0001": 6767, + "08-e7-ff6043e3cecb42da61f3c5fb9e97-0002": 6768, + "08-fd-44b7d3d16c71f0c2c7096e699b0d-0001": 6769, + "08-fe-cb1ded96be76713683c50e731f3f-0001": 6770, + "09-02-f54372a564eca0f0643b78370057-0003": 6771, + "09-03-0a1e4b45e77de7f6f0eea512f983-0001": 6772, + "09-03-415496024e2623c06233cbdd7405-0003": 6773, + "09-03-49a291cd3a905c6fb9aca82a482e-0001": 6774, + "09-03-5a22e07c8a311665b9dbd5c31281-0001": 6775, + "09-03-808447c91bb3d99761cd58b02409-0002": 6776, + "09-03-8a118ed86236c6a8e5bdf920c03f-0001": 6777, + "09-03-a24d83164371832670f90790b60d-0001": 6778, + "09-03-a5e48b1491b0257dc68b8afa2f9e-0001": 6779, + "09-03-ac1e56a246c946bdff8219b2af97-0002": 6780, + "09-03-e9a2ba745e9be70985ab9144ea33-0001": 6781, + "09-03-edd22036ea63a5e27e337b8f8375-0002": 6782, + "09-03-f73689d2613f9a68ccda0bf87641-0001": 6783, + "09-04-1f3630baf1fbe8389ade8f6ca5d1-0002": 6784, + "09-04-25c2dd47c9b999bea812b0c084dc-0001": 6785, + "09-04-52cbb352bc386ca66cea78fe52ed-0001": 6786, + "09-04-5b3fc184b6242423d1c9826aa43f-0001": 6787, + "09-04-8bf5d64eae50a2f85e250b472488-0001": 6788, + "09-04-cbf7e491a9cfaf5748ad51051864-0001": 6789, + "09-04-db92c9d66ae96591b924b9a4507c-0002": 6790, + "09-04-dfebfae5c69cb3b2d091ac386cb7-0001": 6791, + "09-05-02b25796d49f75222ce205fe186a-0002": 6792, + "09-05-1436c97e08e9d2cd94d5f8d16871-0001": 6793, + "09-05-166523b7c36d8bf26f80d3c92d1a-0002": 6794, + "09-05-4ff3ad005541bcefb3912b8c69dc-0001": 6795, + "09-05-58c5cd9eb81672623974b2004130-0001": 6796, + "09-05-6fe45a0cbc8e34392772b79981d5-0002": 6797, + "09-05-7179d222149daba7fde87fad93ff-0002": 6798, + "09-05-9aa98c8362a4c9b0b34dad3fdd5d-0003": 6799, + "09-05-f3bd3f6213df3cf5e7bef602ffd2-0002": 6800, + "09-06-01ecc8593152697878f0d03a8ab6-0003": 6801, + "09-06-3525b01a4069232cef7003d5f397-0001": 6802, + "09-06-970c7d3134b547cf1beca4192834-0001": 6803, + "09-06-e4e155b7b93ebff0bd2250ac5f5f-0002": 6804, + "09-07-29034c7065ea92e7f3a4c999c5d7-0001": 6805, + "09-07-3ff156b83d84f1f4c4de9f1088af-0002": 6806, + "09-07-46352b4d16c22ce66f907adf8256-0004": 6807, + "09-07-6826b33475e23abd21e120f449ce-0002": 6808, + "09-07-717c86a5e23aeb963ba84e47eaa8-0002": 6809, + "09-07-7db0000a47fbc04e6780d0fa26a1-0001": 6810, + "09-07-ad9a2a6796e5c9150a916eb248af-0002": 6811, + "09-07-bdf63cb17353a83d62673a707044-0001": 6812, + "09-07-c6efdff01afd939f2ca4fbeb5b26-0001": 6813, + "09-08-63fdc10b57095d936c97a3a84b96-0002": 6814, + "09-08-6a15878cad9ed7684c09c586ddb1-0001": 6815, + "09-08-9d0cb3083fcc643f565a9f33e945-0002": 6816, + "09-08-b5e1dd4d0e848b13153e9e960823-0001": 6817, + "09-08-bd2ef416b03f20c19536cb055d20-0002": 6818, + "09-08-e11474204f0eb2b26e35e0c0e865-0001": 6819, + "09-08-e6c7455cfa7ab3f55a10467a51b1-0001": 6820, + "09-09-1acd3df01771721ed22ae4b90865-0002": 6821, + "09-09-44690ca30656413bf66b5ff29246-0001": 6822, + "09-09-4c48d2238ae59e335ef97e9094d6-0002": 6823, + "09-09-b91150f121d1d7909acb99f59842-0001": 6824, + "09-09-c1831d0e365b7186e9ff375094cf-0002": 6825, + "09-09-e92a8452c72743f70553535e19e8-0001": 6826, + "09-0f-3f1dfc6c28f322768e9e844c434a-0003": 6827, + "09-10-0c2ed9fdafa1b8ab2faf69b7fc39-0002": 6828, + "09-10-3cea595289afba3eae81526081db-0001": 6829, + "09-10-5bbb115c6cb50adf8b1eb3ac0284-0001": 6830, + "09-10-94e4f9805e69ebe77332978b19c4-0003": 6831, + "09-10-9a4113d7cc6a6ed452ce45cc70cf-0001": 6832, + "09-10-a598f4ab8ad7390f54b16d932c1e-0001": 6833, + "09-10-b79248f61d8dd12c86e9e6633112-0001": 6834, + "09-10-ca5c5a224d37a6a2a5af65cfc15d-0001": 6835, + "09-11-01ce350a3153643a04f7af23076a-0001": 6836, + "09-11-24a9724ce7372f08c17ecf87efcd-0001": 6837, + "09-11-31737aebf42c2aa4db390b9c7392-0001": 6838, + "09-11-51af1161f1e640f7e73b4428b156-0001": 6839, + "09-11-5fd4c84ca329c1abe5928c109fca-0001": 6840, + "09-11-7262260a1eb0d22252c4a0cb46d4-0002": 6841, + "09-11-7f3e3009376fbba8d3fa372d64b9-0001": 6842, + "09-11-87421bb24ee5c57f4bd202e56255-0001": 6843, + "09-11-c327ff49bb183b4eddf2a7309710-0001": 6844, + "09-11-c43062260f1a30a1157e1824d955-0001": 6845, + "09-11-d93d946c7b5471bbf550da023991-0003": 6846, + "09-12-1a1fc993203b443145fe0ed695f1-0001": 6847, + "09-12-28f25297af746f314a54ad5f4028-0001": 6848, + "09-12-2fbb3c116be68521cc50aa180cf3-0001": 6849, + "09-12-624f1c09eb1adccf3fb4c4a40d13-0001": 6850, + "09-12-6561fca0dbbf9b84753e482bec47-0002": 6851, + "09-12-a2d5dd1d4f31985ae02e08a9b39d-0001": 6852, + "09-12-d74182f8ec85eebdaf51d479cd06-0001": 6853, + "09-12-fbb12a7dce818d9a2b4598faabc8-0003": 6854, + "09-13-3e03e7808b40ba88155227842125-0001": 6855, + "09-13-4fa5ea60f16ae23cd0bc77d42a52-0003": 6856, + "09-13-564567897203d7bce2ac77a92b16-0001": 6857, + "09-13-5cee2017047f10186a20ff3ea5b0-0003": 6858, + "09-13-a31c5177a10adc17a43cb52c94d7-0001": 6859, + "09-13-b75ec6de8055f68fcca8112e0f08-0001": 6860, + "09-13-bd1e0881cc550d9c5dc1f1b0b4ba-0001": 6861, + "09-13-dbb1599e8acc06d6b79da68d6fac-0001": 6862, + "09-13-de2d733e8f6a2e45ef2028330034-0003": 6863, + "09-14-6a9cde3e2cc902361c9450e4e974-0001": 6864, + "09-14-c5a3449c86e57e246c9303353db4-0001": 6865, + "09-14-d5204236ecd72a06852c013e116b-0002": 6866, + "09-15-009c9eccd947bc295e8be58e62a7-0001": 6867, + "09-15-25d51204c109de20f05c11ebb5a2-0003": 6868, + "09-15-58d807dd2770961db93b0cb2b417-0001": 6869, + "09-15-74f82edf0fe3aa1ba0d6df8452b1-0003": 6870, + "09-15-d1a0d1a97ac5c375b2fecad9f2d2-0001": 6871, + "09-15-f2762645d042d0ed2d922dbc68cb-0002": 6872, + "09-15-f5f3e9d88615efa43b7318f2916c-0001": 6873, + "09-16-06c87ea3a94eb7f83a85d9b4597c-0002": 6874, + "09-16-2a8351bb6034f92464f5e2e300a5-0002": 6875, + "09-16-5b572f8297210fe8532ccd592190-0001": 6876, + "09-16-93d2e51f2183b9ab9ded76659244-0002": 6877, + "09-16-ce707561bad9aed76ac83c742817-0002": 6878, + "09-16-d4138f902358dd5c44e60f9e4e3c-0001": 6879, + "09-16-d5c26e5699031ce588c44206da55-0001": 6880, + "09-16-d619db829ef6431e8837d39478f3-0002": 6881, + "09-16-e22c5ad5ff2f1eb33981d37487ca-0001": 6882, + "09-17-27b35af82a9d6d965acb500e3343-0001": 6883, + "09-17-2d46e7938ad9b9c85418e3db2fd2-0001": 6884, + "09-17-3954549d00c283f7dc17636f9c31-0002": 6885, + "09-17-79bff4e53fb7e407a628bf6a65cd-0001": 6886, + "09-17-825ed3bf880de36f620c14e1c0dc-0001": 6887, + "09-17-88aa25a9b26bf5da99b24092e792-0003": 6888, + "09-17-a00ff7ebcf3a49c044245d3f3d18-0001": 6889, + "09-17-a3721556d73a1d9e1848c32e8810-0001": 6890, + "09-17-afb47a5f36b944f633dce18430ed-0002": 6891, + "09-17-cb803cc563821f043039b246a895-0002": 6892, + "09-17-df2a2c50e1be4adef5db8c751042-0001": 6893, + "09-17-efbf7c4dc485bd48d365d70e72c5-0001": 6894, + "09-18-0482de4919b1d4d6b0dad6462607-0001": 6895, + "09-18-12756b71902f948ae7ca72c35b6a-0001": 6896, + "09-18-1d9dff44fb9aca01a3945ee14959-0001": 6897, + "09-18-23412aa9bbf24d7a7336ec9c92ff-0004": 6898, + "09-18-481bcd95739e2629c29b26f93954-0002": 6899, + "09-18-4c0a225da934906b63092d3de03d-0002": 6900, + "09-18-6cad07da601403146a9eb36d47aa-0001": 6901, + "09-18-9c048bbe25f779674a7bb9c9a785-0001": 6902, + "09-18-f877cf96bfe19b2605d92b41fe6a-0002": 6903, + "09-19-88bcaa7ce3b6b6475847af942c40-0001": 6904, + "09-19-961ed7e3197c58a3427f371fb076-0002": 6905, + "09-19-b87c994f55224d91c2e270fd7ed6-0001": 6906, + "09-19-e7a7a901e7714cdc1cce079b8855-0001": 6907, + "09-19-f8166818a467876ef6ba996d824f-0001": 6908, + "09-1e-d6ecc0111fdef79bd6d92d857f1e-0004": 6909, + "09-20-00b32a4913f21a462216d833899d-0002": 6910, + "09-20-03b16c2d414eeaef22b106158ba6-0001": 6911, + "09-20-132ba55b615bc7335ffc217cfc38-0001": 6912, + "09-20-7f9571c05d6f037193c623ceddb8-0001": 6913, + "09-20-b71d20239d49f50db88f874d9602-0001": 6914, + "09-20-c127097529688a310c5895724465-0001": 6915, + "09-20-eeb027ff9494d61a9a61ec04ac61-0002": 6916, + "09-20-ff3d53fba0c9d73adced972c7189-0001": 6917, + "09-21-299fff0a5f2e98a1af8ef80a82f0-0001": 6918, + "09-21-5492aafbfde64c01d1eaf6baa3fc-0001": 6919, + "09-21-58881f4c4b1d7c8e0fa0b6d4bd4d-0001": 6920, + "09-21-5aa426f55cced84b17a8c5945f1a-0001": 6921, + "09-21-7e7e6802cc1d8882310690e1e0d8-0002": 6922, + "09-21-88da5a394af3e8b9bae9acea7614-0001": 6923, + "09-21-8a9b47a302aee01f721376dcbad4-0001": 6924, + "09-21-9ea83ab2318ae4d928d87b3cf5a7-0001": 6925, + "09-21-9fb06d2cd60ded2bef081817057f-0001": 6926, + "09-21-affa24e3b9818f5ab73eb2a0762a-0001": 6927, + "09-21-dcdfd4d748e58f79385bdd3236a8-0001": 6928, + "09-21-f57405a0c5e9e85c0915a51d45eb-0002": 6929, + "09-22-16a8d4448dbb245b5ae2ba452cc4-0002": 6930, + "09-22-30795ddf00e46726c59b0f6fec9b-0001": 6931, + "09-22-356db195a994e2b957297ae72ff7-0001": 6932, + "09-22-4c7d443479f63a5e85e2547a888a-0001": 6933, + "09-22-5d75f951f499b15a4c9aa233ca6f-0002": 6934, + "09-22-6d850e4966c43a3e9f600a2b741e-0001": 6935, + "09-22-78d6d77fae74e06c9e4eaed1c33c-0002": 6936, + "09-22-7b9a5b5fe72cb4c1be48c44da48e-0003": 6937, + "09-22-923144636fc1959b902c04137207-0002": 6938, + "09-22-94e4b0465c712e23529d334c4b74-0002": 6939, + "09-22-b9bd969fff7820dbdbc660b6df26-0001": 6940, + "09-22-d2ed9f9623e34b46aec002bf6783-0001": 6941, + "09-23-1bd0a92c078e7480c20c66aab279-0001": 6942, + "09-23-9fc2aae489b93ae7d22b65cbba8c-0001": 6943, + "09-23-a8095be79c2edf79bbc7ee2dba99-0001": 6944, + "09-23-daf160c90705f0cc963b7e6fbe6b-0003": 6945, + "09-23-dd2c075c3a7f82008917bd4629fa-0001": 6946, + "09-24-0e4cbbeec2756d03dc3be9b9ceea-0001": 6947, + "09-24-0fb5652ae4734e6352a67f304203-0001": 6948, + "09-24-185b084035942ba277378b0d9fac-0001": 6949, + "09-24-293787852a9f5b7ee516b0c2694e-0002": 6950, + "09-24-3a0f4206d581b88bec1defb5ecf3-0001": 6951, + "09-24-55c92bf6416b873d8ff8c4750ef0-0002": 6952, + "09-24-5e81cbda9984ede45e1225d14267-0002": 6953, + "09-24-cb9020e39877596394ce3a807736-0001": 6954, + "09-25-03aff59bf08eb1c8b7335d834385-0001": 6955, + "09-25-2b7fb43056c785b1827b5525c6cf-0001": 6956, + "09-25-37c02eae3c4c62232e2aff0e25d9-0001": 6957, + "09-25-4ccc3a3b1c5b7d555b34dc094e6a-0003": 6958, + "09-25-82c63a9c15830da88ad7cd05eab2-0002": 6959, + "09-25-8b91a9ba41298a55a48676b6fe30-0001": 6960, + "09-25-9cd36b5a65c5895e5c7dc68c4b66-0002": 6961, + "09-25-eb06328f061a0c515b1a107889a4-0002": 6962, + "09-26-00d5e01f2bd906c618db573c7b6f-0001": 6963, + "09-26-037360a5b3ba0b771e484c16dea2-0002": 6964, + "09-26-11cba7d302536f1a0200f1905f1b-0001": 6965, + "09-26-2421861b2d39c5ca4edd6382a5fd-0003": 6966, + "09-26-460c21bd27472e6b79534ce8c505-0001": 6967, + "09-26-4e0d7a62b0438c7df66eeba7d409-0001": 6968, + "09-26-7940a4315b6ba790c5ffe95b0871-0001": 6969, + "09-26-83ff38d0bffd9246f7e5c675598e-0001": 6970, + "09-26-8a86010a4de78e6b864e7a5d8096-0001": 6971, + "09-26-94f1b2a33bae0e9a01deb44995a1-0001": 6972, + "09-26-96e34de5ae8029a6bc830decd1b6-0002": 6973, + "09-26-9c49efb2ea605b7cc1b69918a87d-0001": 6974, + "09-26-a6f8c64c71f7d592bf5d8f52081b-0001": 6975, + "09-26-bc0c91724247e3762dc812b6d20b-0002": 6976, + "09-26-e5c3bcb754e3d575982eeb18019c-0002": 6977, + "09-26-f2247f5064b90881b52d63e840c1-0002": 6978, + "09-27-0c2e3be50bda085bbffe8a86768c-0001": 6979, + "09-27-207311783d183d3cc2520ac9c989-0002": 6980, + "09-27-310716d5b47ba9edc8a36d213f4f-0001": 6981, + "09-27-47b4c2930986b59a250f89c4682e-0002": 6982, + "09-27-4aed6bc63b280e034bba87ccce8c-0004": 6983, + "09-27-87496ea77d0bf448d5f53eedee51-0001": 6984, + "09-27-91631930b21c3c78fb9586b64a16-0001": 6985, + "09-27-9597573bdbe1fd2169b06bb46ab1-0002": 6986, + "09-27-b3322b16d70d8d77be58461ebc7d-0001": 6987, + "09-27-b3a453c1d935030f7fbcde80a150-0001": 6988, + "09-27-b6b43474eb34de6c8f095f5a9641-0001": 6989, + "09-27-ba98ea3b08070869dc6595ccd5db-0002": 6990, + "09-27-c74881b1cf1a1fc9cb684de4ef66-0003": 6991, + "09-27-dc45b0516fcab2a1ac6696b380e4-0002": 6992, + "09-27-e5a57b7fe960820eaf6e5a8f06a0-0002": 6993, + "09-28-07109c01c9b79dfc8b3af96b2004-0001": 6994, + "09-28-301e07012a5f9bf232ecadae031d-0002": 6995, + "09-28-4b44dc734fc5e0dc65b9fbdf8df9-0001": 6996, + "09-28-870ce733aefe780595b9278265f3-0001": 6997, + "09-28-b81f41285edee5d85c61c28f9aea-0001": 6998, + "09-28-c124d8c304db5202e43857a10aef-0003": 6999, + "09-28-d54a7806cad35ee3b57c6bacd629-0002": 7000, + "09-28-eb1dc8dd0d6f50a9183c914ce7d7-0001": 7001, + "09-28-ec759c6c5d9253443787e7bef74c-0001": 7002, + "09-29-0d7d20ab90d0726447411798da8a-0004": 7003, + "09-29-40873fe3159388d07ce42fbc5499-0001": 7004, + "09-29-6766909bc4b678d03f778c49c80f-0002": 7005, + "09-29-703c43d11f71e407f3e4480e3492-0001": 7006, + "09-29-a07c5249f6273a0789065b5f3269-0001": 7007, + "09-29-ab54cbbbab54b90cf65829c9b13b-0001": 7008, + "09-29-abd1c2c7962ef52efac12edb5c56-0001": 7009, + "09-29-ddb43cec5cad4edcf66b00727a88-0001": 7010, + "09-2a-121eb0adb0daf8881540c12b86c8-0001": 7011, + "09-2a-6f57495d062d86e73e4c14cbf221-0002": 7012, + "09-2a-74cd637becb9a6a4520870e75645-0001": 7013, + "09-2a-9f49bd338d6e39425f3d191b9021-0001": 7014, + "09-2a-bd24bb528161668fc0f616d95a43-0001": 7015, + "09-2a-e88869c807f6b0c77a10cf479224-0002": 7016, + "09-2b-326bb4e544b72368c7e92cb9e7ee-0002": 7017, + "09-2b-53b81a56655aba5a8be6803de274-0003": 7018, + "09-2b-5b5505a1546048dcf9e171795c3f-0001": 7019, + "09-2b-c8ae098956ca76ae9dbcb87d3eed-0001": 7020, + "09-2b-deaecdc5a1e9f29a328c5db0c33e-0001": 7021, + "09-2b-f0612b3ed8505981250fba725a83-0004": 7022, + "09-2b-f099e1a1b92142906366e32abda3-0001": 7023, + "09-2c-21762548f5f1810c05b941d1f3b9-0002": 7024, + "09-2c-2bd5f02dc6e924a3a9951fb38f08-0001": 7025, + "09-2c-b1aaabf893fabcec01c41fcb52a2-0004": 7026, + "09-2c-ba17b3982343d32aab2d86920da1-0002": 7027, + "09-2d-12170e2a02d79a2f6186b1dd73d9-0001": 7028, + "09-2d-41e7992bdf25998741819fb388fc-0001": 7029, + "09-2d-6ac348ce9c53da793786519c15b6-0001": 7030, + "09-2d-6c484685638b247a50323027cbfb-0003": 7031, + "09-2d-77f988720513ff2c0eb0c6b52d86-0002": 7032, + "09-2d-c36ccbb6cc1de467c6b226a429ca-0002": 7033, + "09-2d-c4cb7b03956135bb8f34b7bae73a-0001": 7034, + "09-2d-cbf10ebe31692398b33e41d24dd8-0001": 7035, + "09-2d-d6317335097f43d969f1b705b47a-0001": 7036, + "09-2d-ebdf0c93dcf350b40f89d1f95e98-0002": 7037, + "09-2e-2b9e797782b80b2ca4212c891b28-0001": 7038, + "09-2e-4e0b9cb77d144e1ad18af9922ff9-0001": 7039, + "09-2e-5164c56d5bb031da4c1c3f8b23ce-0001": 7040, + "09-2e-799e68706e00a07df827aea53297-0002": 7041, + "09-2e-897a66c823d78d6e8a2eaac319b8-0001": 7042, + "09-2e-b5ed2fee2c288aaaef55fde6d47a-0002": 7043, + "09-2e-bec80dc936f50172c47e84a98fa4-0001": 7044, + "09-2e-f5c568ad4bc2e025c0ac7a94cd93-0001": 7045, + "09-2f-01673db25369fd2573e2a939ebfa-0002": 7046, + "09-2f-0412ed07a54cb17608ad773e7790-0001": 7047, + "09-2f-0cbbf20e2735ebfe855513d6d7e7-0001": 7048, + "09-2f-5d155e2c0c36ee4804ba06844d71-0001": 7049, + "09-2f-72bdf01c9e5cc06ca4c3a4d68607-0001": 7050, + "09-2f-995e916c9400a35d300867d8ee42-0002": 7051, + "09-2f-a51f19c35506b0d5b590f6eda7a1-0001": 7052, + "09-2f-a9a2e446894b7c2cc811428674c7-0001": 7053, + "09-30-2bbfe67c5db311c97b332100c27c-0001": 7054, + "09-30-361d9dcba79c702a7beebfae4461-0002": 7055, + "09-30-3e3b86f86c03b7f8c402c06dd942-0002": 7056, + "09-30-3e8e78bbc5c2006d9f491f8ade5a-0001": 7057, + "09-30-7dea28e2c3fa1207aae79cc62731-0002": 7058, + "09-30-8a5fb12deb4e65d654fcdd5f4219-0001": 7059, + "09-30-8f955ff910139c9ffc00b48d85d6-0001": 7060, + "09-30-9d6fb87e52d80f8b99a81c0df107-0001": 7061, + "09-30-ba6a8e529024dbb6c2f8fb6f5f9d-0002": 7062, + "09-30-c376acbe2dd2437ae1b7ed8fd34d-0003": 7063, + "09-30-d89547537c28c0b853be402bf33c-0001": 7064, + "09-30-e9a129dc188a967604c99a93110b-0001": 7065, + "09-30-f0d02b61d130b502c03d3e073dd7-0001": 7066, + "09-31-0394047de359f3d59bbd6c18029f-0001": 7067, + "09-31-099b6f23a2152828e89fa2ecf3d4-0001": 7068, + "09-31-39f365d7e300195c8efcf4521533-0002": 7069, + "09-31-6bedcb5a4d5c57f696ac3c09971b-0002": 7070, + "09-31-84fb76010c0f067c0c6cf73de20d-0002": 7071, + "09-31-99c7d53762f4c178ac1dd448016f-0002": 7072, + "09-31-bb3a56525d7148018b4982f615d7-0001": 7073, + "09-32-0a2e83cf08fe2330d8920335cd5d-0001": 7074, + "09-32-4b5c236c5f22b32a21424ff0800b-0001": 7075, + "09-32-8d9e4060e61f39b834077bb93676-0001": 7076, + "09-32-9830214290244347ebb7b066790f-0001": 7077, + "09-32-98eac6168488bff5280111949652-0002": 7078, + "09-32-b796bd587e7f12a8ada3706b6d15-0001": 7079, + "09-32-d7110f47ecda30a50c87ba28c315-0001": 7080, + "09-32-f140360015dc8e67cb5f9c78445a-0001": 7081, + "09-32-fc32711412b8eb1129e930c74ef4-0001": 7082, + "09-33-4b731766746138e9f5554cdced91-0002": 7083, + "09-33-587816b87bf963bb82495d0238de-0001": 7084, + "09-33-5a2584cd8f022ec04f2cc0b79a04-0001": 7085, + "09-33-af77bd7bafffb7241d48b33b8b18-0003": 7086, + "09-33-c58c705959bd6d1cdd412afab911-0001": 7087, + "09-33-cf8cd57850c514b6e75781ee6606-0001": 7088, + "09-33-f67384ea9416b86441a8e4c8fae7-0001": 7089, + "09-34-5fdb8c5563bfcd5fb694ff2f19ab-0001": 7090, + "09-34-6af70ced4fc8161a2bc0655f9213-0001": 7091, + "09-34-9522ac5e20ebcb126a638690ec5f-0002": 7092, + "09-34-9d192877f550aeb9280c55b30f81-0001": 7093, + "09-34-c09823c45610fba4ee8c04eab344-0001": 7094, + "09-34-eb0be2241584a7b81ed62d9531ea-0002": 7095, + "09-35-1cf0c03484023b5ec1f7b79c46f7-0003": 7096, + "09-35-1ec4839a05c3cdcac17f8f088b77-0001": 7097, + "09-35-4c63ad2a3c638252bd342e6f0211-0002": 7098, + "09-35-ab0c01c558d97f1a4fcbe2787145-0001": 7099, + "09-35-bf956c8060a0f31d72b5aa3c1a78-0002": 7100, + "09-35-d12cbe0b12587f436f8bf5e98f5e-0002": 7101, + "09-35-e05a51fbed5357a1400709074c0a-0002": 7102, + "09-35-eb0d1172bd1d26a24980454c1fba-0001": 7103, + "09-36-057436c15ce1d933dff0acee2aa6-0001": 7104, + "09-36-1bd7dce242bd30f380e7454e051d-0001": 7105, + "09-36-4e8d034476c88a7f9d28a7d19826-0001": 7106, + "09-36-547477c4b468f0f115359625b572-0001": 7107, + "09-36-5487525309d97f16cfa33673f6eb-0001": 7108, + "09-36-5ab35496cd20426e68c2244211a1-0002": 7109, + "09-36-5c16e87fea60517fec5870348efe-0002": 7110, + "09-36-a87468c86e4814acb00789757097-0001": 7111, + "09-36-f3cae3659a763fa263134b912195-0001": 7112, + "09-37-86094ed081a8a1b698e49ff3eeef-0001": 7113, + "09-37-ad4ec119f1c42fcd64b3344840f6-0002": 7114, + "09-37-ada16fe5e2f2c5f9f77e79dc9d9f-0001": 7115, + "09-37-d724206027c405b3fcdc162a7da1-0001": 7116, + "09-38-770a824bcf00f754dc7095115810-0001": 7117, + "09-38-7c68fb7d51439f3c928d431a9d1c-0001": 7118, + "09-38-824abadb4996f06a51088290b02c-0001": 7119, + "09-38-9078365cb1b88534dc68b6f0a625-0001": 7120, + "09-38-d10cb5e70d6d8aa0b7842cee1e5b-0002": 7121, + "09-38-d3ebf34308eba0ec9cee10f62e96-0002": 7122, + "09-39-15a85647065fe0ffba27a652d4af-0002": 7123, + "09-39-1d4d6ab0b5a5057777457f33b204-0001": 7124, + "09-39-62436c1fae4dc955f7459494bba1-0001": 7125, + "09-39-7bf66a158e2dfe676000697dd4a6-0001": 7126, + "09-39-8500040ac660aad21cac426c11e2-0001": 7127, + "09-39-9eba186e4308d3bf46c4ee89f46c-0001": 7128, + "09-39-ccb2708277f1ba7c985f7cf13416-0002": 7129, + "09-3a-0b22d0f03b1058986b37f1e0f853-0007": 7130, + "09-3a-11434fc96842a31413d0370439f5-0001": 7131, + "09-3a-121dd0e59a6347bbd327a57b9250-0001": 7132, + "09-3a-1d9e1aadf19fcbd8e83ac9af5900-0001": 7133, + "09-3a-443c261088876ef2e3e3c8ac8bce-0002": 7134, + "09-3a-7531c2894bccc4d93cdcca7cc7aa-0002": 7135, + "09-3a-aa026840f719c6c8bae75a160327-0002": 7136, + "09-3a-bae580585fa2df9b84b00c01054a-0001": 7137, + "09-3a-f814471e2cbda9af2bc97bdf5730-0002": 7138, + "09-3a-fa6ec95fe7e67d8dd48f50ad9b78-0001": 7139, + "09-3a-fc5984265d5c18501cf597631153-0002": 7140, + "09-3b-07513bf2afd473af64e9ee1f8d3a-0001": 7141, + "09-3b-269b54b3b2469b12fe9fbb423b8e-0001": 7142, + "09-3b-375660193085850dd0f4445b1c6c-0001": 7143, + "09-3b-477b63857a881fa92f2d030e9f75-0002": 7144, + "09-3b-56f5aa681675c88c43e4c07c279d-0002": 7145, + "09-3b-de754c69d83ca4c43a7fc491427a-0003": 7146, + "09-3b-ecaeb1cc12b719fdafe0276fd731-0001": 7147, + "09-3b-f5c236209bd139ed5d807c41c8ad-0002": 7148, + "09-3c-305b9a1eced92f8fd528981212b6-0001": 7149, + "09-3c-5276960bd8a0866e1fc60cce96a5-0001": 7150, + "09-3c-5cd4a20639ecee8f4762a424cdf2-0001": 7151, + "09-3c-823eb1b9eea663f9789904abeb21-0002": 7152, + "09-3c-8933c7052dc21b113b7e668a967c-0001": 7153, + "09-3c-cc84938e9ade101f9f5471c8e9d7-0001": 7154, + "09-3c-cfd720d588053c9b7a556f188fbb-0001": 7155, + "09-3c-df569acb6fc18046e85ee9931ac7-0001": 7156, + "09-3c-e0dde50d0fc14935010f3a3f987e-0002": 7157, + "09-3c-e6a14d48b626e5e2f71910026460-0003": 7158, + "09-3c-edad06e4d88b95c703638d626332-0001": 7159, + "09-3d-0d85af261a27ee75957f65e615aa-0002": 7160, + "09-3d-16a0d78c385c5cbec7aeb5c29352-0001": 7161, + "09-3d-43c7c7d644015daa22cff30a5ad1-0001": 7162, + "09-3d-482c921492ee9562648dfba63a44-0002": 7163, + "09-3d-59fac124bcc109ca54176ee6d80d-0001": 7164, + "09-3d-64c86400dabb4484582b50972365-0002": 7165, + "09-3d-67c3d89ed0acd3714dc15bcb56a6-0001": 7166, + "09-3d-92207eeb511677172b9a1ef27983-0001": 7167, + "09-3d-ae07dce96aadc0e7213119e38be1-0002": 7168, + "09-3d-ee528ba56c42074f6395e2d24ac6-0001": 7169, + "09-3e-47b73ffaf65d664ea0ea861207ef-0002": 7170, + "09-3e-4d9c752a0293238020e5581e5a92-0001": 7171, + "09-3e-4ddbf67d417bcfc72bff2c39c8de-0003": 7172, + "09-3e-58ac0ca67a400fd38adc2c33faf0-0001": 7173, + "09-3e-627f141aa39ea852870575f03c67-0003": 7174, + "09-3e-abfa0031a1ae526eceb1a7a47c5d-0001": 7175, + "09-3e-e3dd989eddd5eef5b58c65dc18e4-0001": 7176, + "09-3e-e90a87f5c9630cf008f99c9fbb63-0001": 7177, + "09-3e-ea2071cec195d59bba44b4924c84-0001": 7178, + "09-3e-ed60a0d364fcbd8fd9416d607239-0001": 7179, + "09-3f-23e40e5ebc3b535d3e0adbad6294-0001": 7180, + "09-3f-53a508dce55fc6084611db10b4a1-0001": 7181, + "09-3f-8a0b1f566dcafaaa417e14e9d78c-0001": 7182, + "09-3f-a3825974ea717d9294a35801cb4e-0001": 7183, + "09-3f-a6c38b7b58b3a8520d286d706f7f-0003": 7184, + "09-3f-d88e17e011f18fbd39d2698c1be8-0002": 7185, + "09-40-0f7dce8d90bd5a5c5f214ceb0712-0003": 7186, + "09-40-33e558d1e8fef28e4fa447a9799e-0002": 7187, + "09-40-67dc372a68b930292f736714e408-0001": 7188, + "09-40-71dd9deeb11276862ae4bc56c8ba-0002": 7189, + "09-40-78860e52d6be885cc95e410c8afd-0003": 7190, + "09-40-9a5a0c9dc3744f6a8791af9f2d21-0001": 7191, + "09-41-0e5f42ae708a1b4f42f039a52c76-0001": 7192, + "09-41-65ccd05f3c7c029f06cdc7b5f4e9-0001": 7193, + "09-41-6b174ef96f5619daae7b0af8dec6-0002": 7194, + "09-41-89537432f39db9d8ec0492e570c5-0002": 7195, + "09-41-a029bc2b503b0fb9759fb52c9ba8-0001": 7196, + "09-41-affdefeccb46b7d8f41664ca2277-0002": 7197, + "09-41-bf84818e173e914a675fc8c40f89-0001": 7198, + "09-41-c5b068558987dd35681ab198e753-0002": 7199, + "09-41-fc0fb12bef4ac0e2fce0522699c7-0001": 7200, + "09-42-578d0ec49b9e9eecdf7a317e5dd9-0001": 7201, + "09-42-be03d69263099aea96a41f888536-0002": 7202, + "09-43-136400944fc1224d9788da7d3f44-0001": 7203, + "09-43-457a6fa1842728e276a907349699-0001": 7204, + "09-43-750069c4347564e2ae5458f80d41-0002": 7205, + "09-43-7b37ea23d2d7bababd553e644e4b-0001": 7206, + "09-43-8459a4aa88e2633e4a99fef61253-0001": 7207, + "09-43-85aa38969aeaada363c304d609ac-0002": 7208, + "09-43-986ac7d71eb69ecf3e300c3c2df0-0001": 7209, + "09-43-ad428f80b21c7bd25fb52f941084-0001": 7210, + "09-43-b9851c918eeb9379417149c8f6b1-0003": 7211, + "09-43-be5e35d8564ed34642944ce7e935-0002": 7212, + "09-43-c02be46e8fd4bf9d61992404852c-0001": 7213, + "09-43-d70a86c856ef0e22256e73138046-0002": 7214, + "09-44-11be73cc50eef6299bd2506d4713-0001": 7215, + "09-44-547b00f0e59ebb9b0c81976da862-0001": 7216, + "09-44-b39464007e7d429dcbc45a2f3989-0001": 7217, + "09-44-d10d74c4458c38c0d642df88b25f-0003": 7218, + "09-45-0abbf1d9b8fe96748dbfe1535141-0003": 7219, + "09-45-0c5fc0c3a0e1499495585c2befb7-0002": 7220, + "09-45-0e6cfe4881764475b3b287efc66e-0001": 7221, + "09-45-0ee0b5eea6f84d5249a9f8f62dbf-0002": 7222, + "09-45-8a91125418506d821871f84e0d53-0001": 7223, + "09-45-91535f472679f48bf2528b510f75-0002": 7224, + "09-45-b5479ca44c380af7209d8346d036-0002": 7225, + "09-45-c09322dde2c711f3aa33e8b03eb0-0003": 7226, + "09-45-f056efd46b6f0bf4f0dc1200a783-0002": 7227, + "09-46-51605c3794589dfd9ed0ace90936-0001": 7228, + "09-46-60448864f64a2960e4eb30e55a4b-0001": 7229, + "09-46-68b19c8f3251b1f55a4659aececf-0001": 7230, + "09-46-80ec1400eaacd5dd0a79b365b05b-0001": 7231, + "09-46-97d41ec3f89862d0380b48c4d0ac-0001": 7232, + "09-46-c49cfd7e35994dee5bb6a0c94ee7-0004": 7233, + "09-46-e1cfce0141f02226c85c8f571c68-0001": 7234, + "09-46-e232c39f8154e6d7a5371952c003-0002": 7235, + "09-46-ef4c4223aff5bac23ed76875f3f1-0001": 7236, + "09-47-166fdfb7ee70aa036d527154f72c-0001": 7237, + "09-47-2ab0e242343c9de0482ed05d479a-0001": 7238, + "09-47-5d8597ebc172574e192c6e03631e-0002": 7239, + "09-47-642ac3d1a74ddcf813769e967695-0002": 7240, + "09-47-aab892be7d34c12685f30392d918-0001": 7241, + "09-47-b01c3fa6ca2331827cc0ddd1fe42-0003": 7242, + "09-47-b3fcd0318fdb07d6bce4194f464b-0001": 7243, + "09-47-d174ca0e72d0d3f44752eac0520e-0002": 7244, + "09-47-dcf4e4097fdeb77d79cdbd340319-0001": 7245, + "09-47-e8f807f03e87ea12fe8e14d864c9-0001": 7246, + "09-48-7022653ff14b5fbac316e73c396f-0003": 7247, + "09-48-c09ddef3ca1fa10c313a87784c1b-0002": 7248, + "09-48-cdefaa774bcce6b657a358a4bb29-0001": 7249, + "09-48-d0913f62af59db9b45f3823b3bfd-0001": 7250, + "09-48-da90972d0255f0c213df950228db-0001": 7251, + "09-48-e98b94623ee176d8e29bca9b2b63-0002": 7252, + "09-49-43a68b2adda20fff4c1009c9aeb6-0001": 7253, + "09-49-5b5134ccb3a84a7962f84e155740-0001": 7254, + "09-49-85fec7a9f0ab8bfaa51fdd7972cc-0002": 7255, + "09-49-b1b214dc26ec831667fc35fc699b-0003": 7256, + "09-49-cc70b9e6df4565033ed5268b8057-0001": 7257, + "09-4a-83e2919a5e361f320668218577af-0003": 7258, + "09-4a-98167d3e0d121c48e999e3f669e8-0001": 7259, + "09-4a-99163286029c6cdb55fd6e322924-0001": 7260, + "09-4b-15974b30e2aaf73b68ee56b81811-0001": 7261, + "09-4b-4847c7f902e15bad9a8af777f797-0001": 7262, + "09-4b-578ad55c2b96266538b38b7b51fe-0002": 7263, + "09-4b-6b167492ca6867c34455656e0039-0002": 7264, + "09-4b-81e9eebfe152e32d8b9970273f81-0002": 7265, + "09-4b-9018e496ffa0f1ff8bef534e6174-0002": 7266, + "09-4b-dbceacfccaf4b61f1489d3e82818-0001": 7267, + "09-4c-2bbda264336953a1219206be4f28-0001": 7268, + "09-4c-61072ec159942b8f450392b4cc7d-0002": 7269, + "09-4c-99c1cb1b8c35a3d6d39eb35ffdb0-0002": 7270, + "09-4c-b098f2d42cb16c5efdaaa4af4d30-0001": 7271, + "09-4c-bf251891dbe0be0bf6d0307ef2b6-0001": 7272, + "09-4c-cb705313a40a6d72cdde2a9c1c82-0001": 7273, + "09-4c-e29c425b471ffa0d7be2ca1c8530-0002": 7274, + "09-4c-e61a37bf540c134a587cb53fb678-0001": 7275, + "09-4c-e64986684cab93b90b99e0c90ba2-0001": 7276, + "09-4c-eb804d51c87ca24e589f2b8dc207-0001": 7277, + "09-4d-231a689ca7f041d4966460333b46-0001": 7278, + "09-4d-5640a3c31cfc7fe97f229e7d7358-0001": 7279, + "09-4d-afcb3f20c1122573afdcaff498a0-0001": 7280, + "09-4d-e0970e13610fc2affa61b2e7f6d1-0001": 7281, + "09-4d-fb1e28dfddb23441018055ebb484-0001": 7282, + "09-4e-11aed266d8a5d88243d55ff53acb-0001": 7283, + "09-4e-1798cedd1c9303991e49eae9f612-0001": 7284, + "09-4e-1e88e86fd95abf58e2514e4d5aad-0001": 7285, + "09-4e-284ced73754f78f8ce3abbedd05b-0001": 7286, + "09-4e-5cad0a82aa252e692e2808a16cb6-0001": 7287, + "09-4e-8a6ad83b15ad14ba7dd592c4111d-0001": 7288, + "09-4e-8c956b6b9a1c40112a4d45097973-0001": 7289, + "09-4e-aaebb0f3ce711ee2d819f3ff7aff-0001": 7290, + "09-4e-c0eeb8851cdddff7ce1a8cbd1e1e-0001": 7291, + "09-4e-d3bcfaf9b2c1c42fe45f254f3c5a-0006": 7292, + "09-4e-daf36b1bd5c755af2665a2e4322e-0001": 7293, + "09-4f-05ee523cb040e8a11e52f2924fdf-0001": 7294, + "09-4f-3112eeb95cdfe98791c43a149c32-0003": 7295, + "09-4f-3b34472fa1388b90bacd740d7393-0001": 7296, + "09-4f-7d4f293c8d342b618a0d8b8d5bad-0001": 7297, + "09-4f-7ea953c136e1099bcdb1120f8db2-0001": 7298, + "09-4f-855a9c00a86cbfa33eca982c5988-0001": 7299, + "09-4f-ae8cdceef62568ac842113ff67ab-0002": 7300, + "09-4f-bd1478d42269d04c57e63a291020-0001": 7301, + "09-4f-cd29fd2a20494a0d12b11aba8ee2-0003": 7302, + "09-4f-cfa62844d125efb9a3891d97daf8-0001": 7303, + "09-4f-f6b2ab07070cccb856c3adc63366-0001": 7304, + "09-50-0b33d77a6338a0503ccd223d08d0-0001": 7305, + "09-50-111eb58fa1c63fc470094c5342b6-0001": 7306, + "09-50-14b44c36c12b115b00bc72eddb18-0001": 7307, + "09-50-353fdd1d683af7994b9eb07a91f7-0001": 7308, + "09-50-3eae22535189a4e966de439ebf4f-0004": 7309, + "09-50-569fae618031b7122fcdfcd16453-0002": 7310, + "09-50-6b5b3a503cf93207a1f8988b4b90-0002": 7311, + "09-50-b8ef5fbe2974be6d5e6a6d41b84c-0001": 7312, + "09-50-c6d03b08c339ff066fdf4f6b7fa8-0001": 7313, + "09-50-ebeb07d30d4377afd5d1ec1ce977-0002": 7314, + "09-50-effab73cf2705f63b7fa50b04a6f-0001": 7315, + "09-50-f89357d0044bda44d306d3e4ce5d-0002": 7316, + "09-50-fa042c12d5d786cc1610bead9f97-0001": 7317, + "09-51-056e2860f49765cac6a0b9259d12-0002": 7318, + "09-51-46ecc171cc25756d20003c6aaa4e-0001": 7319, + "09-51-5790be0cb7479290cd9ff10944cb-0001": 7320, + "09-51-689e0265bc3f81af7c38043cafe4-0001": 7321, + "09-51-80bdf819f7c42368a5a3bff6c5eb-0002": 7322, + "09-51-ecd1e4ad7418992c026cc3cd178c-0001": 7323, + "09-52-0176192832d9dcd389493681e916-0001": 7324, + "09-52-23596b9b10367a42a2bb41b9d76c-0001": 7325, + "09-52-37b4fd3c9671abab330ab0f6e6d9-0001": 7326, + "09-52-51fa4f7fd07db1548cfc1afa22df-0002": 7327, + "09-52-578263a79b689d68bf5083ac9c26-0001": 7328, + "09-52-5d19186c62ae86ddf305b7249cc7-0001": 7329, + "09-52-921fae2bf9eee4545c2f9b7bbd87-0001": 7330, + "09-52-b1cdea7eb3bd61802ff7536d44c5-0001": 7331, + "09-52-cef1657422b2a78a8fabbbe77e3e-0001": 7332, + "09-52-d4d086e4d7fd41fdf1bdd82f4753-0014": 7333, + "09-52-e8ca6ca9c583ecb980dcccc38fe5-0001": 7334, + "09-53-0e939ad1f85189f59a46938ae029-0001": 7335, + "09-53-206986c2baf265df1d53b152b3b8-0001": 7336, + "09-53-2797254de2a17ce46e83aa50bb4c-0001": 7337, + "09-53-47be20729f5822fd89cf9729b020-0001": 7338, + "09-53-764dd913aef57898fed92a1fae3e-0001": 7339, + "09-53-8f1b7729467da644f61ba851bb4d-0002": 7340, + "09-53-e84df0f70f542bfb87668011444e-0001": 7341, + "09-53-ef04eab3f2fbfc187002438fded3-0001": 7342, + "09-54-16ab598335e38cda6f67d7660b08-0001": 7343, + "09-54-42e6b64fadf5218a88415a2256ce-0001": 7344, + "09-54-503fbd3f1ce5efcd19460c74dd2d-0001": 7345, + "09-54-63bd8e03e70de3bedc4a46d7a7f8-0001": 7346, + "09-54-66a588af5d5216efd20aece82e06-0001": 7347, + "09-54-748eebe6df6e4e4603606d3f47bd-0003": 7348, + "09-54-7fb234c997986cb58f7be082ac51-0001": 7349, + "09-54-8871f5b87fb523563b1ab2b8359c-0001": 7350, + "09-54-970136041d405d8e2d610fa3e4d6-0001": 7351, + "09-54-9f15c35e7fda3c328bfd055d3e97-0001": 7352, + "09-54-e08387c045c13ade08dfbd812d44-0002": 7353, + "09-55-999c1472319270ff3509ac0a4361-0001": 7354, + "09-55-c74c42a8595a90da9456c3e3509c-0001": 7355, + "09-56-137cbcb13c8d754f96d7562749c7-0001": 7356, + "09-56-3001f234d388f7cbff2bc1f952a8-0003": 7357, + "09-56-7a7fea1493dcb493410fe211eac6-0001": 7358, + "09-56-999c045fe22ffc00bb751bc96054-0001": 7359, + "09-56-a14d86a3b3b7027c7c7972b84c2b-0001": 7360, + "09-56-c968be02bca02ef7cd6112a8c93e-0001": 7361, + "09-56-d5718974ce2aa2b3972757da8103-0002": 7362, + "09-56-e10d4a964dd2d780cac5c98d2089-0002": 7363, + "09-56-ee1595598b4984f4b066406667bd-0001": 7364, + "09-57-07fb175d7359c890a4c7ad15e1ea-0001": 7365, + "09-57-3de1d1660c5afbf42b961200d43e-0002": 7366, + "09-57-65636891b1d83063bfb1b3b51138-0002": 7367, + "09-57-7c0ea0abc3802db7d7d6b22f7911-0002": 7368, + "09-57-7c894f8b9b9d30bda1c67c7cedf6-0002": 7369, + "09-57-a8339e07fe847fa632e68d07bb91-0001": 7370, + "09-57-b3e98f2f57beae9563b5749675ff-0001": 7371, + "09-57-c15eaefd84ff95f1339203c2d5a9-0001": 7372, + "09-57-f6b3fcc0252b3ad75168966ce843-0001": 7373, + "09-57-ff117c421666b435d84000c48f03-0002": 7374, + "09-58-15043f78846d5633f572eaa7281e-0002": 7375, + "09-58-2f9dc0decfa675f0fba09a8232ba-0001": 7376, + "09-58-51703cafdc8075f800056507d551-0001": 7377, + "09-58-59ad2688fe7afde008b3352693fe-0003": 7378, + "09-58-673801cbc73120a3ed3dfa38fc16-0001": 7379, + "09-58-73c75b8e8955ea56a77a2aef44cf-0003": 7380, + "09-58-8c5ff83b3afdb90c0aa67c7d766d-0002": 7381, + "09-58-a8f1d5e7c1c0c29ae8e93729387f-0001": 7382, + "09-58-b9bb257a04d9292e1e8197e1d221-0002": 7383, + "09-58-b9efbb6f64920d2f560a043ba394-0001": 7384, + "09-58-eafd5dda736f3b25c2df4c51e868-0003": 7385, + "09-58-ebc3bf4df227e58c39a816809d74-0004": 7386, + "09-59-6e4b7e276a7f07e276cd4c606bd4-0001": 7387, + "09-59-891cb44b5f47898a36e34dbc2882-0001": 7388, + "09-59-89ee41df815ab1181196e859b90b-0002": 7389, + "09-59-d23dc43f3fa6087a0b9d57a1923d-0002": 7390, + "09-59-da44ad36eb6c44ab22b731b17760-0004": 7391, + "09-5a-03a441941306a39d531c7ff83ee9-0003": 7392, + "09-5a-34b83987fc4b0f22e260ee12431d-0001": 7393, + "09-5a-3aafb8139ab943d13e883712a216-0002": 7394, + "09-5a-5726acaabe39afd7a214d585fdd0-0003": 7395, + "09-5a-69390a44cacb1e3286349111450a-0002": 7396, + "09-5a-69e83acb888c10b132b318ccca1d-0001": 7397, + "09-5a-6bdf14559a5c5e1f94ab0c6f57ed-0002": 7398, + "09-5a-b8b46c2f11d473336c39113340c5-0002": 7399, + "09-5a-d06f777d67c1c3da652de0bb60f3-0001": 7400, + "09-5a-d1089dde0bb868fd24563a236431-0002": 7401, + "09-5a-db4bff2bd64ffc08a5cc5bcd2ffd-0002": 7402, + "09-5b-05472eb3e777129e80ed57910886-0001": 7403, + "09-5b-0c544da7c49e0b95b0cb3d7eb657-0002": 7404, + "09-5b-19321097ac772b79ccf933a6b4ba-0001": 7405, + "09-5b-4297d62dc2581fc8e4da2bde82b1-0002": 7406, + "09-5b-455381bf83ff52b2fc18354e76b0-0001": 7407, + "09-5b-498dfd2fb7acddc0fbee63f4e2ee-0004": 7408, + "09-5b-5fb610f7adefde2666857f37e834-0002": 7409, + "09-5b-b35ecea1e705dd53b68ef5d8deaf-0001": 7410, + "09-5b-e7f013f77c481729f23cd0200f76-0001": 7411, + "09-5b-fa0ff545bcff7306cf7c27e38e35-0001": 7412, + "09-5c-045c9acc52cd8e44c5067f2d494e-0001": 7413, + "09-5c-62720a07294f0dc3dab6d1b78222-0001": 7414, + "09-5c-7353d519b205b9aba6d74921e001-0001": 7415, + "09-5c-8c60c6fb8b5b33007cdbc614c5c9-0001": 7416, + "09-5c-98bbead95853936857e08f54f2fa-0003": 7417, + "09-5c-e6037bfd26675510c630b9859137-0003": 7418, + "09-5d-0b89ce5dad2ede91d630f75d6616-0001": 7419, + "09-5d-3149fc679402fdc9b6eca5f543d5-0001": 7420, + "09-5d-43753246eec946a967e36426f636-0002": 7421, + "09-5d-52354aed5004d498063bbb167462-0002": 7422, + "09-5d-b429c6339308fc3764c8be855d97-0002": 7423, + "09-5d-da790afcce764504e62387534834-0001": 7424, + "09-5d-e132bcd6a50c9170f171bad04cc8-0001": 7425, + "09-5d-fa60e535a016dee0bf308d013efa-0001": 7426, + "09-5e-17a91a581ea0dc67d26e897b4d4d-0002": 7427, + "09-5e-24b7cca15a51feeaed87e61359fb-0002": 7428, + "09-5e-3941845159b2dc5006079628697b-0002": 7429, + "09-5e-3f1ccc37db457e5cb5bf91d36353-0001": 7430, + "09-5e-4a07879da5290d2fffe998e17b8d-0001": 7431, + "09-5e-7c8324417a3dcddcc06858f834ef-0001": 7432, + "09-5e-9806a0c74c93aacd66b4d2e8c62d-0001": 7433, + "09-5e-bece116ee18d9150dda39e6920f9-0002": 7434, + "09-5e-fe9b690c1246167b947e563d4cd2-0001": 7435, + "09-5f-02b693e1644e294cf069bba35c05-0002": 7436, + "09-5f-1c0013d801b3b065cff25114cac7-0001": 7437, + "09-5f-391b960c78d9e0efc38b86a7fe2f-0001": 7438, + "09-5f-6362a1c561149225dbd814b17929-0001": 7439, + "09-5f-6ac660c49d7848f250e8f9ec9668-0002": 7440, + "09-5f-6cd53043ebaffa7dd49a11e56942-0001": 7441, + "09-5f-7d414c968f0f4ccf3111bba5bdb0-0002": 7442, + "09-5f-7e8d22a3777a393318e72535a1f8-0001": 7443, + "09-5f-90a4c0311ad3868cc12b6920cfab-0001": 7444, + "09-5f-b496d3a9b8da265dfc3974e0dc16-0001": 7445, + "09-5f-c4e9ccb015eab523d5149bbf8a81-0002": 7446, + "09-5f-f5634ec1c211c88ffb9bbe566817-0001": 7447, + "09-60-0aef9cee9cf81b4046d6164390c1-0002": 7448, + "09-60-624ec57d0d0fe6f2ad0768125584-0001": 7449, + "09-60-7b97963854f041e96b4a5d000862-0002": 7450, + "09-60-f597dc2d17ea176ee1a2138c7d57-0001": 7451, + "09-61-5cd062a8ab92a8fb3bf4c30113dc-0001": 7452, + "09-61-5e81aa28f2596eb76aa3847931bb-0002": 7453, + "09-61-b83b2b430a924de03d7e72164d51-0001": 7454, + "09-61-bd4dc246d494c74669f80cd08bfc-0003": 7455, + "09-61-dc32d57b9c5b0025ece6b5198c5b-0002": 7456, + "09-62-2b3c8d195b0e6927c9ffe37fe68b-0001": 7457, + "09-62-34030261e7d50aea5ec68b849f28-0002": 7458, + "09-62-3d95048174cfcc0a7337dc1c9a4f-0002": 7459, + "09-62-4278731c13cda5ce6072c5afcede-0003": 7460, + "09-62-43eec6f9ae2829f24b5717c54f6f-0001": 7461, + "09-63-1c48ade652f7c8eecb0c4287971f-0002": 7462, + "09-63-507df2af4fadf32aecfa6dcedfa8-0002": 7463, + "09-63-5ff3d218326ab602fe52f080af91-0001": 7464, + "09-63-81d9f78f410cc3608f77a59df573-0001": 7465, + "09-63-8d1a1ff5ac600ada44fad921e177-0001": 7466, + "09-63-b20631b9829b101c19830fdb8069-0003": 7467, + "09-64-15a6dd2e52d8e201f4401622d5a3-0001": 7468, + "09-64-5607430d963302942c4d849bf354-0001": 7469, + "09-64-8ed7013b4fa5e29e0c2dcf48ad9a-0002": 7470, + "09-64-cd21a3f642ef14c051fb76e42145-0002": 7471, + "09-64-d3ac2af895a8e4396d2fb9bf9f9d-0001": 7472, + "09-64-d9eefde3b3320f33ec1b4fbc1620-0001": 7473, + "09-64-de19fa4ee6f88c8678dc5773951a-0002": 7474, + "09-64-fe2907a9c62108b2e18005b037f7-0001": 7475, + "09-65-b23c12d7f56cde4659a6ebfb8400-0001": 7476, + "09-65-ca834395f240bb1b3cb31d951d75-0002": 7477, + "09-65-df7b926235629fc65506f436fa22-0001": 7478, + "09-65-e464ad9a7f302fd8a852b0c1ea23-0001": 7479, + "09-65-f10e3abc8c5a1d6efb043cf56c44-0001": 7480, + "09-66-020206e880fc468ada53403e6a9a-0002": 7481, + "09-66-05c1f9699db1372656e01e6538cd-0001": 7482, + "09-66-64a59cb309e54c2fdb4d5705dc4c-0002": 7483, + "09-66-658039cad9199cc32680ef25e22b-0001": 7484, + "09-66-6ee36a24af8157f713c7b584e55f-0001": 7485, + "09-66-76fa2eeccb7c9c85e64c296c7c99-0001": 7486, + "09-66-8f471cf91a0c0e6828a3f2d6647e-0002": 7487, + "09-66-97ed5804b52f076be131c679ffbd-0001": 7488, + "09-66-9df9db3786b47db2e4f96da9c00b-0001": 7489, + "09-66-a7f4e5020cf60b40860ee29ba92d-0002": 7490, + "09-67-0f16b887c231f5a41ca2a34bb637-0001": 7491, + "09-67-10fda2f079a4cd2ef73f15a7036f-0001": 7492, + "09-67-12045d3b9132582a0b48b019eb69-0001": 7493, + "09-67-29bad7fe215be73a0d71b1fef130-0001": 7494, + "09-67-3fc3f3710db2d749a887fdf0a5dd-0001": 7495, + "09-67-55136b81664fdb282a86a657f3f8-0002": 7496, + "09-67-ae29a68e007720802aabd4551811-0001": 7497, + "09-67-afccd4d17538216c8002dd9b2548-0001": 7498, + "09-67-cdf1ea77f945694c7d2fadd7810a-0002": 7499, + "09-67-e08386aaf3e78cad7ffdd93cc043-0001": 7500, + "09-67-f99c80904a5edb33aa1757b05b90-0001": 7501, + "09-67-fa0e0ffc5afcc5cdee2a61ca187b-0001": 7502, + "09-68-1970edc1c40f8fa69f380a464738-0001": 7503, + "09-68-32f706d4fbfdff313c42a9fb8f02-0001": 7504, + "09-68-5a618464fe39f7af64102fb22971-0001": 7505, + "09-68-8cf7b9b243719a00129cbfe007ee-0001": 7506, + "09-68-c44f11c7d877b2be703b497c1f3b-0002": 7507, + "09-68-d0de857dd122f5bb512f1c855e89-0001": 7508, + "09-69-4791647d61d33967a4c2d3e869e9-0002": 7509, + "09-69-8761723470fce4754713cb1485c5-0001": 7510, + "09-69-cd02fba2da13903aed3ac054c465-0001": 7511, + "09-6a-1736824aa23a676f134b3f2ebcbd-0001": 7512, + "09-6a-1c9948528874225376800c2d0eb4-0001": 7513, + "09-6a-1fabb62553a86108b005df4e1454-0001": 7514, + "09-6a-3cb2cb68acc48ae1a6b3a9c9b69a-0001": 7515, + "09-6a-8c99c02439309239e9c068c24389-0001": 7516, + "09-6a-ae85de1b0cc112c6b1edf3eb206e-0002": 7517, + "09-6a-d3330c76825b1f7c5daf1cc65d80-0001": 7518, + "09-6a-d9f172c4dae2515253e1047ec122-0001": 7519, + "09-6a-e43faf46c77f33cb881e0e0a2f0c-0001": 7520, + "09-6b-02995009daaf8aa1a7e020a34d05-0001": 7521, + "09-6b-1d45fc94569d210f94058ae6bd2b-0001": 7522, + "09-6b-30c2785cfcaef0e07f9e5bdee3a5-0002": 7523, + "09-6b-3aaaa5ff2d4331a3e31effd7fdec-0002": 7524, + "09-6b-62d645a2b0d7436bf91305592e3c-0001": 7525, + "09-6b-69d63d2c99d56e224d994c73b1fd-0001": 7526, + "09-6b-6f98451b77aeca6c79d6db7d9a67-0002": 7527, + "09-6b-a29c4b4631d3bd2d59962a3e307d-0006": 7528, + "09-6b-b97e5acec20cd6ce378ec556899c-0001": 7529, + "09-6b-c4171c4b6b1c5c74ce2ea07bdb37-0002": 7530, + "09-6b-dd772e3148b3d59838ab71b2a063-0002": 7531, + "09-6b-ea3d6cd678de691d80b61e63ef74-0003": 7532, + "09-6c-00b836d9cda544169175890286e4-0003": 7533, + "09-6c-1d83531475c67f6ce6ebc705c896-0001": 7534, + "09-6c-2990c1b0efcec188c385e662da09-0001": 7535, + "09-6c-4ebeb9af71798195f90b81fc98fa-0001": 7536, + "09-6c-51106c40ff4275600030495102a9-0001": 7537, + "09-6c-8307856073e719055634720ecae3-0001": 7538, + "09-6c-a325f12608a58165006157e01687-0002": 7539, + "09-6d-58f7af35cac50a10c2bc45b21233-0001": 7540, + "09-6d-7c08bf1f6786d54c3b16c78c04d6-0001": 7541, + "09-6d-7f2d780f36a689aa5407f1423937-0001": 7542, + "09-6d-946b6974b070d292f0baed243f50-0001": 7543, + "09-6d-e04f8751ef4293eb7208a8bf4130-0002": 7544, + "09-6e-174ee7bb1ac91473a022d8432f06-0001": 7545, + "09-6e-30b73d317ce19d801e56382f1dd4-0001": 7546, + "09-6e-419646336bfe15befda4d6f7ff5a-0001": 7547, + "09-6e-429a0f63bfb4f87239041b533629-0003": 7548, + "09-6e-d4173d3fe8c1bcf97798496cbf7d-0001": 7549, + "09-6e-e53db99188131a1212476c4404bc-0002": 7550, + "09-6e-e750e8d0c29293598605f70a351c-0001": 7551, + "09-6f-446c0f2b9ddc87cd6a0cb781f7cd-0001": 7552, + "09-6f-4818cb1eba2ee17543b5c9a5644e-0001": 7553, + "09-6f-8aab2b9f8024a7275a75a252ac52-0001": 7554, + "09-6f-aad71f9511ea8c886b5b65bb65bb-0002": 7555, + "09-6f-ad91684e73456a791ee46ec335de-0003": 7556, + "09-6f-b63faf45967dc30f25e04213d4a3-0001": 7557, + "09-6f-bd4c6042c5437b3a7e0c7ae1f6a5-0001": 7558, + "09-70-1185238c063349bdce4e8f68bdd0-0001": 7559, + "09-70-3e5195b72a2a32b3582faf535eef-0001": 7560, + "09-70-bb83d4d13b2e3d9ee963970d1298-0002": 7561, + "09-70-c2bfecd8d2d545d5da8174ecd1b5-0002": 7562, + "09-70-d6f87a69f0077e0985d51ec7de6c-0001": 7563, + "09-70-de8ef904d2a634709f8f7f0df78e-0001": 7564, + "09-70-df71c5959c0df3a6cc8c954bdf53-0001": 7565, + "09-71-07dffae5ba9cf8b90a7cbcf6a1b7-0002": 7566, + "09-71-7c46538979399d54311b1cb9c6cc-0002": 7567, + "09-71-9fc8f7922b13ed2bec74c52b6aaa-0001": 7568, + "09-71-aca904812e9dd8e166cd260f8e2c-0001": 7569, + "09-71-e9a2a7891c464f729247dcb00639-0002": 7570, + "09-72-374e911103abcd5995c2fe6cd75e-0001": 7571, + "09-72-58952e8131457287176497976cc8-0001": 7572, + "09-72-62a3b9c3065d37abe5bdd3f29de6-0002": 7573, + "09-72-7f1482fcc57e5fa17879fb1ec121-0001": 7574, + "09-72-8ea54b32e884cf2422dac84d215d-0001": 7575, + "09-72-a1763c63a5e7e5be5a2e083e97ad-0003": 7576, + "09-72-dcc1d3b29cffb13dce742b1b53a6-0002": 7577, + "09-73-157585d9038eb73eb4fb983a0caf-0001": 7578, + "09-73-2be3bb59126f502c3781c19ab811-0001": 7579, + "09-73-bbfc70513932f7f19940027a06a3-0002": 7580, + "09-73-c7a19a52b4ab5012b3d24c18bc2f-0002": 7581, + "09-73-de94073fbc20293e1806d6ca7526-0002": 7582, + "09-74-005b90eb7dedf7baf4cabcfbf80c-0001": 7583, + "09-74-225e57f3579f30d19f713600ada7-0001": 7584, + "09-74-5eee0576da8c111d868991de058c-0002": 7585, + "09-74-77f76519f27cc3b16ff11e89dc8f-0002": 7586, + "09-74-917de9ea58cd8ee3cb5731983db8-0001": 7587, + "09-74-a4e8eae01aa49664d3a5e88f12f3-0001": 7588, + "09-74-fe64e99b3fcc9d7d274cadc0edc4-0001": 7589, + "09-75-05267f813e28faae7633b3ea484c-0002": 7590, + "09-75-13bab369a34486f00afb504fb482-0001": 7591, + "09-75-3a2c3f8c0313cf78eb4ca99ff682-0001": 7592, + "09-75-45656f2ed982987cca8b366c0dc5-0002": 7593, + "09-75-a11c98c6d809e3fe684430920dee-0001": 7594, + "09-75-c5214badd7230066815e3412c4d3-0001": 7595, + "09-75-caf11e88aceaeee8a047a081f7cd-0002": 7596, + "09-75-e1f354360bd1c4c67357a0037b48-0002": 7597, + "09-75-f39d6e657739510e26943f24b709-0002": 7598, + "09-75-f6e2b36bfe2b8f49db424416b176-0002": 7599, + "09-76-4714de14758055f95759849caf40-0002": 7600, + "09-76-51f6f7faee7c3405749fea3c380b-0002": 7601, + "09-76-6987cfb2beb5b703efb8ddaba239-0001": 7602, + "09-76-6df957c3e48c840b4ba0196e79d4-0003": 7603, + "09-76-6eb205d2e69532baffbc1fc8fc3b-0001": 7604, + "09-76-c31986e6ad383d8a46bd660213d5-0001": 7605, + "09-76-d2bc479231180e105cda7033828f-0001": 7606, + "09-76-d4cef8dfe99430e70a2c1f3c7617-0003": 7607, + "09-76-eb2175e57f4d1e07a0ebab8ce995-0002": 7608, + "09-77-10c78cdb265c47437c206c07414e-0001": 7609, + "09-77-35fe43e533e9534528a9a863d6be-0002": 7610, + "09-77-52c2259705aa6c7d686413ee0817-0002": 7611, + "09-77-5c23eeb5e46447f9408d6d37c24c-0001": 7612, + "09-77-7a3ffe159a851a846c9a0b340248-0002": 7613, + "09-77-9173c055c177d822a8e481b45751-0002": 7614, + "09-77-9406f661bd913f16836ef6af0cc2-0001": 7615, + "09-77-94db501ba0bb87f9fe1c62d663cd-0001": 7616, + "09-77-be54b698bfd77ec5cc789448f52f-0001": 7617, + "09-77-ef8e336ac25f856b902ba3ee46ec-0002": 7618, + "09-78-2021be65eadda7f6c37c2c20d6b9-0001": 7619, + "09-78-24545e8bcc4d7af5b975bae79081-0001": 7620, + "09-78-5405bbd8871a5c7fdfe4ec66f921-0001": 7621, + "09-78-68e09553488072ce53dca8dec49f-0002": 7622, + "09-78-7b2e15a8dc08ceb6184a70e3b0e9-0001": 7623, + "09-78-8eb949eaf139480d4e76a6dc7b1c-0001": 7624, + "09-78-9a35315de03122d98436a682a009-0001": 7625, + "09-78-be474aa3398ed4dc5e77c1ac8caf-0001": 7626, + "09-78-f5f4696504cc782db34bd7ed2bf5-0001": 7627, + "09-79-68e5edf1143bda82a2772e489417-0001": 7628, + "09-79-801c77aef8ccc304b967545eee81-0002": 7629, + "09-79-b4a405d0c2a24d3a2d0c56858d07-0001": 7630, + "09-79-e00ff3c28c2576631bd4bfba6bb1-0002": 7631, + "09-79-f706624615822b026e1f7991b8ec-0001": 7632, + "09-7a-239046489180616a17dc172fbb78-0003": 7633, + "09-7a-25c1cc53eabf183b51b59c7f3ebb-0002": 7634, + "09-7a-31ac921efd10db1ccac1f7b62de1-0002": 7635, + "09-7a-45268d0a851666b40cb9e6456407-0001": 7636, + "09-7a-880d5fd0bc4d89e19d4266e9ca3d-0001": 7637, + "09-7a-8f83c3a5ab7d641a60cece2159c8-0001": 7638, + "09-7a-c7d6d3e5978ba4d0e5b33ac32544-0001": 7639, + "09-7a-d0ef837235bfcd3513b414f54ee4-0001": 7640, + "09-7a-d290b87cd791c3c588349c161f66-0001": 7641, + "09-7a-d8e9e8f5049a3f74d54ae90e0f1d-0004": 7642, + "09-7b-0b2b83fd71a62ffab84794c52243-0001": 7643, + "09-7b-2afff1d4ee2565a57a64d431cd3c-0001": 7644, + "09-7b-69cb0789db07fe072c7e5a5973ee-0002": 7645, + "09-7b-6b1b35bd20d2c4ceb2ff8292d773-0001": 7646, + "09-7b-85031f0f60e9831b96f6206e63f4-0001": 7647, + "09-7b-b084edafc82c3b3766b07a88029b-0002": 7648, + "09-7b-e17c098e75990f7db5cd957efc75-0003": 7649, + "09-7b-fcfe0322216478d1b463920b758a-0002": 7650, + "09-7c-079526eb26797a5dcbb154f657b4-0002": 7651, + "09-7c-465c36a8e35ed5ce4bdad2f37922-0001": 7652, + "09-7c-48f210aa4301e37c60557e0d03af-0001": 7653, + "09-7c-52750a41d44c48cd7033de84a901-0004": 7654, + "09-7c-5495b8c0b1a1ecf072b6d88a4682-0001": 7655, + "09-7c-6214db1b158079459cfd4d19bb33-0004": 7656, + "09-7c-65725d617f09989cdabe90661a0f-0002": 7657, + "09-7c-77ba3768a15638a3fe0a1b462e4f-0002": 7658, + "09-7c-88ad369868b4dc899be4ed24cb3a-0001": 7659, + "09-7c-b84db363a6c6ea1db4a9b18bc5c4-0001": 7660, + "09-7c-c618597249f43d706758803e9c24-0001": 7661, + "09-7d-211d526815bc51c2acd18a504643-0002": 7662, + "09-7d-28c832381951ed0ba3b37b80e56b-0003": 7663, + "09-7d-3652c0423a1f783c4b668bb1a51b-0002": 7664, + "09-7d-40145da981e03b63328debed61d7-0002": 7665, + "09-7d-4919950844bf03c28ddc2a08049c-0001": 7666, + "09-7d-63553006db460f95e1d62a235ed6-0002": 7667, + "09-7d-68f4931fe70be672c9213aa5dc10-0001": 7668, + "09-7d-80dae519b61f9cd7d25f5c3ffd7c-0001": 7669, + "09-7d-9cd6a14e4238f75147f13a9aab52-0003": 7670, + "09-7d-9d3153e229693800e29a10767b7b-0003": 7671, + "09-7d-b6f9e5585d590d45a7bfca94325f-0002": 7672, + "09-7d-b872cb5c096d8875bd5de1e01fdc-0001": 7673, + "09-7d-dc38920cdd66570b3288bd963276-0001": 7674, + "09-7e-0e023cd224e7c084663dd9d0d090-0002": 7675, + "09-7e-4958e62b170677eadee331fdd78b-0002": 7676, + "09-7e-6aa2e976b3cab0377ce89feb7645-0001": 7677, + "09-7e-75601c0084f88a3c462439635731-0002": 7678, + "09-7e-80409af4c66b6e816a70db647fac-0002": 7679, + "09-7e-94cbe4d9a8742457271a4ad10346-0001": 7680, + "09-7e-aede04020558d82b1807c908c624-0001": 7681, + "09-7e-b30dd77cd6ca77f000ba5922c1d3-0001": 7682, + "09-7e-c47a56f785bc26fd10e3fddee9de-0001": 7683, + "09-7e-cfc95dfa3f78b78a49224378808d-0003": 7684, + "09-7e-f3836de483e071bb1ae474f82a71-0002": 7685, + "09-7f-407e33b33fa09341e132e56c67e2-0001": 7686, + "09-7f-d724a964315f4e3069f61c140a75-0001": 7687, + "09-7f-e28faeed504e866094a1dcb24e99-0003": 7688, + "09-80-3d5f9aec3d305b49a7d0747b62f7-0001": 7689, + "09-80-74f906f4c94d793486628c10492c-0001": 7690, + "09-80-ce938335b9b976b4e54ac9ef3ce4-0003": 7691, + "09-80-e4f120e482b5899fa75714149b74-0001": 7692, + "09-81-102feb7fa0dd58efc97a840843a9-0001": 7693, + "09-81-1c98d267fc551e00c23d50f04928-0001": 7694, + "09-81-23837c93fdcdb209f7dccfa3f0ce-0003": 7695, + "09-81-23c4a1e2370effaf4daaf1accdb9-0002": 7696, + "09-81-319922c81e6b6dcfb621cfe64a4f-0001": 7697, + "09-81-64e67b7a8f17973e298494eed993-0002": 7698, + "09-81-6fc8ff710c17d913c2d6601c1102-0001": 7699, + "09-81-85bb930994bcdbf0153dd1367f28-0002": 7700, + "09-81-a51d81f5d43d9e32647f58a5a9f8-0001": 7701, + "09-81-ed1762a4c3a691dfe63cf03653c5-0001": 7702, + "09-82-009616723288e77d84a0cd74a809-0001": 7703, + "09-82-19f7e45aebf6a306472d36ef6c6e-0002": 7704, + "09-82-485596b4ef89b60b3383d1ec8588-0001": 7705, + "09-82-5b17a1588981695b63316045299e-0002": 7706, + "09-82-7b10decc72ebc69e4239bf6cffe4-0002": 7707, + "09-82-a411a8f1d2f5abffb7585ad6e2d6-0003": 7708, + "09-83-2198e01283d2e4586a0c6f26be76-0002": 7709, + "09-83-3d3962b9f37ee0b1d666a3c9d81e-0001": 7710, + "09-83-5e4f35f305c114d14ee0293b68f6-0002": 7711, + "09-83-7fee2217bc1a1a77c7bd519956ef-0002": 7712, + "09-83-88e3ab9c11784b319303200902da-0002": 7713, + "09-83-8a122a25a85588f697d5d59576cd-0001": 7714, + "09-83-aac0c9bfe62b2631212dcc9c9f0d-0002": 7715, + "09-83-c3231ff4affff8de2982e3b24b10-0002": 7716, + "09-83-c40914af6ea2c32c8b5ff6f2db4b-0002": 7717, + "09-83-d4efd9827f5d632a275d203d8642-0002": 7718, + "09-83-d75149abbca28f49a65dd50f3ae7-0001": 7719, + "09-84-51c005e673075571535b993be1ff-0002": 7720, + "09-84-659bb6271c1b6974f7de9f98d8b0-0001": 7721, + "09-84-8912fa14f99026e2657320efa06d-0001": 7722, + "09-85-09b735976c14afef89883577df8b-0001": 7723, + "09-85-0db2ad51b73a3323cb85b017902f-0002": 7724, + "09-85-12bf7d919747959a7e005e5bbe9b-0002": 7725, + "09-85-41962cf63ee62f0ba26a502760f8-0001": 7726, + "09-85-435eeda67ecb3e391548300da349-0001": 7727, + "09-85-44dc612bbeaaa8a3816150cd5485-0001": 7728, + "09-85-6716d11b2f0225d441a985b7ae8a-0001": 7729, + "09-85-94859aa622a4ec9ab75c7110cc58-0002": 7730, + "09-85-ed207ff8803f19b88be1b149ce5f-0001": 7731, + "09-86-0e881324fb8bb2d2d8499f866fe0-0001": 7732, + "09-86-1a41a3ef008c3bb92d9906c577e3-0001": 7733, + "09-86-3999dbdf52c8075bb98b0256a7cc-0002": 7734, + "09-86-5a9a2bbbfec163a47d2d1d59f7c2-0003": 7735, + "09-86-78467221602f895bd0d7e44f85ce-0002": 7736, + "09-86-a439b73b8b72014d45df4c9311ec-0001": 7737, + "09-87-02fdb27c2af944ec6deb54b248df-0001": 7738, + "09-87-26816a8854ba581e612c79a7c669-0002": 7739, + "09-87-413429f6adc12e8ce16f9c6a3533-0001": 7740, + "09-87-54168171fb8cdf43d2e6e49264c4-0001": 7741, + "09-87-9926a8d27289cee7b78dd9cb9c99-0001": 7742, + "09-87-ab825652939729e5f22a7f2fabc0-0001": 7743, + "09-87-f06bcee5e12c92989beea7b6683a-0002": 7744, + "09-88-181b3e371b44a05580c705473f77-0003": 7745, + "09-88-2829a99de30bbb4fa5ea245a9123-0001": 7746, + "09-88-43c22b6f5d062e4a6e54f41839b7-0001": 7747, + "09-88-5a952a39def4b81e743f24675f6a-0002": 7748, + "09-88-637864debd3b2c70b25c75d5ddee-0002": 7749, + "09-88-81b274acf9ebebddeb2491b45ba7-0001": 7750, + "09-88-848f1c36777dd5bcdf7c3ff678b4-0001": 7751, + "09-88-9b4f1a99b143680759ebd1884e8e-0001": 7752, + "09-88-a6ec524b43697856b4f43be8b174-0002": 7753, + "09-88-badab446c35d530f3d1a02740929-0001": 7754, + "09-88-d55504aa076ce8ff83e173b66460-0001": 7755, + "09-88-e56732b00c5d6d561a2a9de8af58-0001": 7756, + "09-88-ff2095a9b4be90efe44b216e23c0-0002": 7757, + "09-89-08473b03bd2e680469412e6a24e7-0001": 7758, + "09-89-1950216a82cd01e9d7c91d0ef0cd-0002": 7759, + "09-89-32894c21a1a3ee965201b168a807-0001": 7760, + "09-89-5972fb8c3495ef7cbc444b324f10-0006": 7761, + "09-89-5983d13bc13e6658436957982c58-0001": 7762, + "09-89-7142993fec8483c9d0192ae0ef12-0001": 7763, + "09-89-ab551033e2ff32e927b70bae089d-0001": 7764, + "09-89-f22bec4184aed5713e9cfba24fb5-0001": 7765, + "09-8a-33ce374fe382ea537f9a32ca7578-0001": 7766, + "09-8a-834b2f2ce3ac442e09347cb10abf-0001": 7767, + "09-8a-890dfecda54818de8b40a86638ad-0003": 7768, + "09-8a-a640a4b27aa128845f94d734401c-0004": 7769, + "09-8a-a9dee55b2285e10bfb4ebb3d7a38-0001": 7770, + "09-8a-e893acf991b34ddeca8395997978-0002": 7771, + "09-8a-f1f4e5a29d6739dee0a32c270e0c-0001": 7772, + "09-8a-f79ce88799a28a79ab172f3ec685-0004": 7773, + "09-8b-722f48c7761d3365655a3d132e55-0003": 7774, + "09-8b-a27d3c1f6eae68124c98bba3e596-0002": 7775, + "09-8b-ca56cb9b84f4c55c729583884439-0001": 7776, + "09-8b-d64962daac3ee7fc3e5ed1152902-0002": 7777, + "09-8c-25878dd6a93e9b68b8a1a76819ff-0001": 7778, + "09-8c-2702bd0ce653748f158a0cefe6c0-0001": 7779, + "09-8c-35c26ace798e840e9ff2fe7766ac-0002": 7780, + "09-8c-3ea657446d70fbe7db5d2e706294-0001": 7781, + "09-8c-68538cb59ff911777e53f015be53-0001": 7782, + "09-8c-7fc0f6e9f9154418a1f4c9573d9d-0001": 7783, + "09-8c-c5d449aa83e7b745dbec9fe480d7-0001": 7784, + "09-8d-24494221775d3a31f75f637e31b3-0001": 7785, + "09-8d-2491e98c53a77bb07349778d3a68-0003": 7786, + "09-8d-5a0e4d36f9439dec0593585acdec-0004": 7787, + "09-8d-8102d10306b99bcac08a8df9d1a4-0001": 7788, + "09-8d-840406acbed11367886eed3f0930-0001": 7789, + "09-8d-97205144e1b9b7e1718d9ac51a68-0001": 7790, + "09-8d-beefce6035036cb6734b0a760be3-0002": 7791, + "09-8d-c4822e804d2b6c5c952a679b86c3-0001": 7792, + "09-8d-e9bb5bf61a2f1ce9d84204b415d6-0001": 7793, + "09-8d-f7a2bf57d421ec4df13cbc502cc3-0001": 7794, + "09-8e-2b955507173b57c3e5b44acb2d3a-0001": 7795, + "09-8e-3fba1ef3ef58dfd1500e7df04c3f-0001": 7796, + "09-8e-7a4da6538029b3fe8dd04ef669e0-0001": 7797, + "09-8e-8cdfff34badf865772b071687e28-0002": 7798, + "09-8e-e7d5d18b66dcad05e7d3c0c19a30-0001": 7799, + "09-8f-2ea5816d514c5e9b33dc7903de68-0001": 7800, + "09-8f-351c1c01c921a50879397163adc3-0001": 7801, + "09-8f-53d616d5f57db1b6cacbbcbadc59-0001": 7802, + "09-8f-843b4cb397dde88b9c083a696978-0001": 7803, + "09-8f-85cb0c2fd30184b376624f193e3e-0002": 7804, + "09-8f-9ee688fa372935fd3eecdea9a76c-0001": 7805, + "09-8f-ad991c39d0c8ac9f1be652bfbab8-0001": 7806, + "09-8f-b8850ef432d71b4305ec00278cfe-0001": 7807, + "09-8f-f949c8c6b9bb9eec0e7099a301be-0001": 7808, + "09-8f-fcb4c1eade65e584386f1a180f1c-0004": 7809, + "09-90-1cea1b2998bdc015b4c279ad3c78-0002": 7810, + "09-90-25a71d4257eb21ce726c6d1c749d-0001": 7811, + "09-90-40614044d39bfb0138553c2940c5-0001": 7812, + "09-90-4883032cd745af742b959d5e2ef0-0001": 7813, + "09-90-61b2c5e50b6e7fff92ed2ae226af-0001": 7814, + "09-90-6ede7952472b31bbfbdfbee2aa41-0001": 7815, + "09-90-8ae09561b04567be08ed0704f273-0001": 7816, + "09-90-994951f97dd6fd47f415ded91395-0001": 7817, + "09-90-d5d0fbb388d6a043db91126202fd-0001": 7818, + "09-90-f4c64d0195f3d21da86c81f7655f-0001": 7819, + "09-91-402bc4cb9e7ce1168fdeceb5b245-0002": 7820, + "09-91-42cb7389958b9d714e1faf49330b-0001": 7821, + "09-91-4575679f276057fc79e437f3f5a5-0001": 7822, + "09-91-4c9aa434f7a8b6e1ccae1d4456c7-0002": 7823, + "09-91-6042daf3b249e655c4cee7702a55-0001": 7824, + "09-92-0c68834c47fb46dbe805e972b476-0002": 7825, + "09-92-0ef788fa7208dd0b1d723d3bb222-0001": 7826, + "09-92-3250dada777a6f6ea260b47ba0a9-0003": 7827, + "09-92-8bf0424c28c3aacbb3a7361043b3-0002": 7828, + "09-93-01355dea9cb3952bb3b414471841-0002": 7829, + "09-93-105d573264d8d6863f873ccf5a47-0003": 7830, + "09-93-129216c72e071a7b59c4470b18c3-0001": 7831, + "09-93-2168fb4db591d7d59d77484f422e-0004": 7832, + "09-93-31890eacfe8aef0f4cdce7818ac9-0004": 7833, + "09-93-35f7739518deb2da6d571bf8e209-0002": 7834, + "09-93-7485d1c9718ddda444541f08b3d5-0002": 7835, + "09-93-84be63da237658a540c7565b8985-0001": 7836, + "09-93-a41d476ac32e0fd582d2d587da4d-0001": 7837, + "09-93-e2678a14ab0efad51360cf74b71a-0001": 7838, + "09-93-e3a292f40bc88bcbf84024146c5a-0002": 7839, + "09-93-f63fa8929e9cab81dc9f143992ba-0002": 7840, + "09-93-f6df66807cd3c814e84ab61fb51f-0001": 7841, + "09-94-4c609937eab44d3b4c8037ea2f72-0002": 7842, + "09-94-4d2f1984566be71f0b4894f66b46-0002": 7843, + "09-94-9d14129d5c1e87238f87bf211c26-0001": 7844, + "09-94-af62a90c77776d7cabf195da97c3-0001": 7845, + "09-94-ca3360b1e9b1daa80ee9ae387ff3-0001": 7846, + "09-94-cf43cf2be89e93827bd0f165c4d8-0002": 7847, + "09-94-db3ff65b98f28d05a181edd2ff39-0001": 7848, + "09-94-e5d7478f04008be9cd2b65e02756-0001": 7849, + "09-94-f9dd80582b6c48f287c2f1d48cfb-0001": 7850, + "09-95-0a4a37da28134b81f7c140a4be5e-0001": 7851, + "09-95-8aac3eb2255b6f934d8ca23cf5ad-0002": 7852, + "09-95-d0ca623b4b4bd6f995eda0feac80-0004": 7853, + "09-95-dc9d2f646b7c911b292339791923-0002": 7854, + "09-95-dfb614c27bdbaac5babe57bbbb13-0002": 7855, + "09-95-e33156a15f0ecd91fc0300bd97f5-0002": 7856, + "09-95-e7b6af8c021728c006b0913506b5-0001": 7857, + "09-96-11c1d05db4582a4133a51402e520-0001": 7858, + "09-96-376fb83b0033b3fabd0587736b3c-0002": 7859, + "09-96-7ac466c2dd45d66bde1d2e3a7c3c-0001": 7860, + "09-96-8cd3f67cdfd64aaca22c3747cdd0-0001": 7861, + "09-96-c29cde90801a3c73e22ea46812e7-0004": 7862, + "09-96-cd045aaafdeab3700676bbc2aa44-0001": 7863, + "09-97-ab72e96e32e7a4597b03dce0fd73-0001": 7864, + "09-98-043b7877b840eec0aa7e2e3049a7-0002": 7865, + "09-98-0939dcfd7243484d15d5721dc65b-0002": 7866, + "09-98-0becacca83aec152f04853df73a0-0001": 7867, + "09-98-32e3ac00decd24feb4a170d30c57-0002": 7868, + "09-98-3e2dca2aa84fe4fb16fc2e43517f-0001": 7869, + "09-98-4707ae7ee4f5f9ecf243b320490c-0001": 7870, + "09-98-9878ccf46a8cc3877906e36ce5a1-0001": 7871, + "09-98-cc42cb8d451bc554ef33817c3a73-0002": 7872, + "09-98-da0523ba12fe3bfa7fcaddacf21b-0002": 7873, + "09-99-5f080ec334b14b6cfe63eb435deb-0003": 7874, + "09-99-786e7ca037b68671aaf005a76a00-0001": 7875, + "09-99-e3a08f00074f9d8ba33277f0b5c9-0001": 7876, + "09-9a-00d7728410c1ca8451474e6be8c5-0001": 7877, + "09-9a-247d5834d0ed586ace1053747492-0004": 7878, + "09-9a-28be5370ccc44cb984bf62b7bbca-0001": 7879, + "09-9a-47a76aca7feb93778a29492aa42e-0006": 7880, + "09-9a-c778028e2f6717375c74df205a57-0001": 7881, + "09-9a-c9cf8c27741d825b122e7a711b3f-0001": 7882, + "09-9a-caade5534983dd140fbf7dc5fef0-0002": 7883, + "09-9a-dee19e8035381ecd776f32d4198a-0001": 7884, + "09-9a-e43fee6690267390efa0e771adba-0001": 7885, + "09-9b-3ff60b56fd29d0672d6c1a9df208-0001": 7886, + "09-9b-41a4a5d4b940744a7f310d7a1af4-0002": 7887, + "09-9b-48366ac702faba9719e1dd430a53-0004": 7888, + "09-9b-6ffb50cbd71a8dca37b119c77fa2-0003": 7889, + "09-9b-775c7032c4739c9de2eea2cb08bf-0003": 7890, + "09-9b-a19eb198db3ee7f30c63fa01b536-0001": 7891, + "09-9b-b1268ff0af1728770d59e9fc2ab6-0002": 7892, + "09-9b-badddd622cb85d9730980d52467e-0001": 7893, + "09-9b-f9a248c1bf8e8017b75d7abac845-0001": 7894, + "09-9c-1acc5bffa6c851b81fa7459d5e13-0001": 7895, + "09-9c-2e1c45b3ce0eb0d06812372d0e46-0001": 7896, + "09-9c-33ea956a137051cd9ee07331e448-0001": 7897, + "09-9c-35db691282e5f273fe86ec567777-0001": 7898, + "09-9c-3913a8c6dde0fb48136d7cac7af4-0002": 7899, + "09-9c-9165b9512b6d54c0825d95531054-0001": 7900, + "09-9c-93a8ece7965ec697b081e402402d-0001": 7901, + "09-9c-b76d61b0832df3b551b7b6ff1fb1-0001": 7902, + "09-9c-bd0307ceb59b77c7056c50346373-0002": 7903, + "09-9c-eabe2045c55e29023ad35cf6532c-0003": 7904, + "09-9d-407c9c5d0168e1db1027abe48dd5-0001": 7905, + "09-9d-4773f324730d14e1c22e4019314e-0001": 7906, + "09-9d-4a728fec8194810ddcc37f2a6cb8-0001": 7907, + "09-9d-4cd9620d778be04b17ecb3a1144f-0001": 7908, + "09-9d-6035939bcc5f55fe1e6e2fe248a0-0001": 7909, + "09-9d-76b5d71acf4626c65a378de7b58d-0001": 7910, + "09-9d-d42c01587c5f0236b1e17e64ee9c-0001": 7911, + "09-9e-02746e5a1b398ea5aa9b998cae8c-0001": 7912, + "09-9e-0cadcbfca7c336a2066249e1453e-0002": 7913, + "09-9e-5649a20ef9c17e68706a984b4ff6-0001": 7914, + "09-9e-5fa781bd2e21e2a4f7e2c6471015-0002": 7915, + "09-9e-6ad93b5e1d03eeb27ba9717288fa-0002": 7916, + "09-9e-709b9a29b5766590dbad481890e1-0002": 7917, + "09-9f-0340cbd6bde4681069472733d963-0001": 7918, + "09-9f-3bf4befe8fae02aeade8d44e9aa1-0002": 7919, + "09-9f-44a0d060ee9c9cc89234a04ca9ae-0001": 7920, + "09-a0-688f9bb94ad27a560f79bb48dcf2-0003": 7921, + "09-a0-72b61aee02b104ddb638ad31a14d-0001": 7922, + "09-a0-825a7e1d0c0478c88e081142aefe-0002": 7923, + "09-a0-b1107bdc02b0ae47fe939a92bc61-0004": 7924, + "09-a0-d2b0333835d484d5f05c6c16800c-0001": 7925, + "09-a1-215f29dd8df919a7268dbae16b60-0001": 7926, + "09-a1-2743628192493c3957567d4f1a33-0002": 7927, + "09-a1-4520c7aa1528e6a5941a660190d6-0002": 7928, + "09-a1-482cec7b94363cbf2fd9514a7094-0001": 7929, + "09-a1-4c20601f519153d9c5ce4ba5b824-0001": 7930, + "09-a1-52b579d1a1954446da9b7fbbea9a-0001": 7931, + "09-a1-7154df1ab38598b5d5b81f647701-0001": 7932, + "09-a1-b74e284131302c5c91230c0e9133-0003": 7933, + "09-a1-c1739e6b7b8f932bc2a170390a12-0002": 7934, + "09-a1-e6573e33f6fb1ccca5be33f8de04-0001": 7935, + "09-a2-01744829018c3483d6dcca272dfd-0001": 7936, + "09-a2-06068074247f27db778592b1ca6b-0002": 7937, + "09-a2-4005bf008431913150fd68383745-0001": 7938, + "09-a2-a69463b17835146fd54f536b32ea-0001": 7939, + "09-a2-b1f11e82e3531bb9786e68aecaf4-0001": 7940, + "09-a2-bbdecf567cbb79c36bf635552b51-0002": 7941, + "09-a2-be7a27612e95d5821a81e07fb9d8-0001": 7942, + "09-a2-e97ae035e5e31025593eac33b9a2-0002": 7943, + "09-a2-f8522affd837dfdf32af97813809-0001": 7944, + "09-a3-11c156541b7cac446f3ea8d6b968-0001": 7945, + "09-a3-1ce1c05804e3551fa595eafd37e2-0001": 7946, + "09-a3-2e9871f69f7cf22f92e338d0f59a-0001": 7947, + "09-a3-328b4c39f6a2a48230b9e7920911-0002": 7948, + "09-a3-3a9cca52e1457e9e20c3af3da58e-0003": 7949, + "09-a3-ac9f153ecc1daca054847c91f609-0002": 7950, + "09-a3-b56a6cb86ecb24ba9c490f1b96cc-0001": 7951, + "09-a3-c2f276fe7d07d4443c7eb1d5384a-0002": 7952, + "09-a3-f3a38b7ec0843b9d6316b02b6319-0002": 7953, + "09-a4-0ba5a08f1960c6e6bbd29f38dea5-0001": 7954, + "09-a4-18e447b059c422de897ab563b8c5-0001": 7955, + "09-a4-508e809b0dd76b79e10fc1d25183-0001": 7956, + "09-a4-523012f1d35adb88056ab8b309ed-0001": 7957, + "09-a4-6e42d7b56a6c607ed132aef4cfab-0001": 7958, + "09-a4-7a5118d9bc0c3389e6e84f7aeb03-0001": 7959, + "09-a4-7e9e261d9869c372e09be0f4b1ce-0001": 7960, + "09-a4-b5b776cdc6e873cd4db49527d002-0002": 7961, + "09-a4-d1d079f4d35d631f8f9bb1bb267e-0002": 7962, + "09-a4-d3b6f3fa87b76b971d4ca55f3bfa-0002": 7963, + "09-a5-0ddbb10beddf125aca8aa8a12126-0002": 7964, + "09-a5-13234be4f4149a783923ef488a48-0003": 7965, + "09-a5-40ee2815ce7024c32564b5c739ed-0001": 7966, + "09-a5-41ee4bb56efdb3d0c82d6cc27ecf-0003": 7967, + "09-a5-567ce9faeb51e35c33e82247da18-0001": 7968, + "09-a5-60c71f21157768139e59e01b2740-0001": 7969, + "09-a5-941815ead7974e7234fb5c3f4331-0001": 7970, + "09-a5-f33a2f46fc6bfde7a1bf17ee7bd6-0002": 7971, + "09-a6-158eb2f1095b6e5f96d597e06353-0002": 7972, + "09-a6-36ea0fcd1d28be81c1b831c74526-0001": 7973, + "09-a6-b78556f4f221f6fe2ee1927a0795-0001": 7974, + "09-a6-daecee6c215b3f345a79ca858ddc-0001": 7975, + "09-a7-29199a06f518e7c0a58f26677943-0001": 7976, + "09-a7-521f97b3bff42b34bf1f645cb4d9-0001": 7977, + "09-a7-8773d6fc8a1bf64198f50b49351d-0002": 7978, + "09-a7-8ca34f7e32c321b0bbe8a4fc077e-0002": 7979, + "09-a7-b39692cca7c4c590b88ec8713931-0002": 7980, + "09-a7-b674c52eb1c350320172fda5abc0-0001": 7981, + "09-a7-bd92eb622ba01af80f6289dbdfec-0001": 7982, + "09-a7-d6d976a66a20e7b3725e537d49df-0001": 7983, + "09-a7-f8d5c59ff445b6ca7f548bc914bf-0001": 7984, + "09-a8-bb1f52520979f0a128793a5aa7a5-0001": 7985, + "09-a8-ce20e2da22280bd23629df5dc255-0001": 7986, + "09-a8-e5001de31a49cc2dbdd624453ccd-0002": 7987, + "09-a9-219a6eb03e9e4491f38f7c865966-0001": 7988, + "09-a9-262e42e5d63ae0e83381c3aefc67-0001": 7989, + "09-a9-b96b96bd01897433471c5ce37167-0003": 7990, + "09-a9-cf23c85408fc10fe22e4222b776a-0001": 7991, + "09-aa-0c96af1d2f547573968497f7abc7-0002": 7992, + "09-aa-3cd67bc67238919d6eebe149c38d-0001": 7993, + "09-aa-3f9543b95eeae190fcc023c8084f-0001": 7994, + "09-aa-5720d628aa47f1d5302a36e79cd9-0001": 7995, + "09-aa-8890beffb5be0416fbcecd760c48-0001": 7996, + "09-aa-999723f5626ae6f391fc7a7dec52-0001": 7997, + "09-aa-bfbd866b66590208483a1a7fef60-0001": 7998, + "09-aa-d187c7b3cc08d9e01c789b34bd35-0002": 7999, + "09-aa-d74e2b94a3f1761b1d31254f5788-0002": 8000, + "09-aa-f1c39284f63046285ef0af62160e-0002": 8001, + "09-aa-f2c3bea74af3f4200ec892e9c2be-0001": 8002, + "09-aa-ffa0a9e1674d4fd8acd1cf709b14-0001": 8003, + "09-ab-081c8eceb60984ad4176bcc84908-0001": 8004, + "09-ab-39a096ece88381bc570d46eb7c79-0002": 8005, + "09-ab-6029502a9fa3291d781d7e80f29c-0002": 8006, + "09-ab-87d8f65f40fe0cf16dce172a0984-0001": 8007, + "09-ab-9bf3b30f70a0204b95ef1d750db4-0001": 8008, + "09-ab-a92eb6b2939502eb54ccd846ced3-0001": 8009, + "09-ab-c5bec06a011844ea4834d1ee4fbb-0001": 8010, + "09-ab-d742171ba308f08eefb7dac96586-0002": 8011, + "09-ab-dfaf254d65a5023f2ce69284e120-0001": 8012, + "09-ab-ed362e1364bc75382048c0a27d4b-0002": 8013, + "09-ac-3ab6af48b7aca6cdc473b6daf731-0002": 8014, + "09-ac-543967bfe1aa9c87a6317e981a6a-0001": 8015, + "09-ac-abd6200b4d8e2c20781a07ef794d-0001": 8016, + "09-ac-b1aeaba20ef8755915d8db323fbb-0002": 8017, + "09-ac-c495d6fbf3a155e3245f9843695f-0001": 8018, + "09-ac-ce8e10544f450a86a456c213d19a-0001": 8019, + "09-ac-e0b7b985416854dd7afb6dc7e625-0003": 8020, + "09-ac-f0ccaf5aede3a7ef03ae4d4a2fa9-0001": 8021, + "09-ad-12bd386ace5c0e3e80176de02a68-0001": 8022, + "09-ad-1575901e84a7ad42cc53bc8b982b-0003": 8023, + "09-ad-5f96ba4353edbf9ee598aec2e110-0003": 8024, + "09-ad-955df948a50dfa8c576d71d77976-0002": 8025, + "09-ad-a942081a870ef5ddeb03a471a6b8-0001": 8026, + "09-ad-aff27a34e45b3a8b44c79c09094d-0002": 8027, + "09-ad-b407cee7964f7d8ebbbc603b095d-0002": 8028, + "09-ad-d6235aee0ec5572c280de4966abe-0002": 8029, + "09-ae-0025c8f86fb129f4a413254dbfd6-0001": 8030, + "09-ae-123c800c1c3aa90454ee02ae2a5b-0001": 8031, + "09-ae-389ccd956debffd443f2cbaa873b-0002": 8032, + "09-ae-492abe1287461d8cbf776d87dd1c-0001": 8033, + "09-ae-a044852bb04ad60102281fdb1f63-0001": 8034, + "09-ae-ed13a1655871d80509509a90ed54-0002": 8035, + "09-af-064a3832c16fde9b5209c4c10215-0002": 8036, + "09-af-07d3bf916db631e80927ab7fc152-0001": 8037, + "09-af-2a709af4988aa4c1a7f8f9f53eec-0003": 8038, + "09-af-796df053ad03d768990ec4f8e9f9-0003": 8039, + "09-af-87897d0c2470bfe985f0216202fd-0004": 8040, + "09-af-9c8eaf7df61b7aa50a5f487c96cc-0002": 8041, + "09-af-b0955c1678d4fc77dd4418f95270-0002": 8042, + "09-af-f51b3c3e7d1ab921c7eccf08fbf0-0001": 8043, + "09-b0-256705064e1a49359b9e04ccf8b2-0001": 8044, + "09-b0-2cfc3a6339b4132ab298257be04f-0001": 8045, + "09-b0-2e5de10a714a09f297935ea47264-0003": 8046, + "09-b0-ee014ba31cbc18b563770db87668-0002": 8047, + "09-b0-f0e8a85af2795382c9057b04f52b-0001": 8048, + "09-b1-1aa4196f28648076f69ec57675b5-0001": 8049, + "09-b1-6b3e68a36f54b9304c93947c1af1-0001": 8050, + "09-b1-72291b286eb042d091ab68a5dc4e-0001": 8051, + "09-b1-8ae1b9518f2238277cf4f1c7570e-0001": 8052, + "09-b1-b2ef87259cbcd3daed8e041c4189-0001": 8053, + "09-b1-d1e91240c2114fbf004ec506e668-0001": 8054, + "09-b1-d71356deae9b08b6bd4c17145122-0001": 8055, + "09-b1-dad6584dc4a6eedfe0a80290fe75-0001": 8056, + "09-b1-f608e088ccc99b09be7c05a7f612-0001": 8057, + "09-b1-f652131e9804f65418728eca520c-0002": 8058, + "09-b2-12ef3d7b2d5bf8e4b6e22b3a642a-0001": 8059, + "09-b2-3031a2cb6bcc89c2d50c6d260a3f-0001": 8060, + "09-b2-47ff4f79fe12feb8e3c7c30b8985-0001": 8061, + "09-b2-4f1aff0342a60708bd952504ebb8-0002": 8062, + "09-b2-7a67a1b301aef287c44304da34dc-0002": 8063, + "09-b2-8ec20a4ea9afb39ed93fd1b06caa-0003": 8064, + "09-b2-e94d6a6bcdf8edf2b07b1544f05a-0001": 8065, + "09-b3-09065d66a08e2681300b69652d93-0001": 8066, + "09-b3-5a645bf962df8e6f7b738181bc82-0001": 8067, + "09-b3-5e77c205ffac901978b8785a58de-0002": 8068, + "09-b3-5f3d2b416a86e9c882560a2605c4-0003": 8069, + "09-b3-95a95f694a286135f870ac4bc262-0002": 8070, + "09-b3-9e1d0f982ca4d1b9b3950e7288c6-0002": 8071, + "09-b3-a5b9e66281e617b3823d0f1d00a2-0002": 8072, + "09-b3-d82e30892388293d7e48ed144de9-0002": 8073, + "09-b3-ddd23624c0298483dc86ccdcaf75-0001": 8074, + "09-b3-de6458799a1ca9f1a7afc95e5629-0001": 8075, + "09-b3-f7b73e37f60ac0283c9bf52b76c7-0001": 8076, + "09-b4-068b31a3aa7eff2fa9cc7572686a-0002": 8077, + "09-b4-089d730e5da98c504d59030fec88-0002": 8078, + "09-b4-0da28f3a93c0bac82b2fbc2fa35f-0001": 8079, + "09-b4-160e2acf01dd9b31610735018ae3-0002": 8080, + "09-b4-1fc60b3955559305a46859aeccd2-0002": 8081, + "09-b4-2716a8297f75622d72320b8f1a7f-0001": 8082, + "09-b4-360c5011315941e6f183bc54a2c9-0002": 8083, + "09-b4-497ae3699d65234292f205e0d6af-0003": 8084, + "09-b4-4ae700a63dcd7a2b65123ef2d2ec-0001": 8085, + "09-b4-4ff04fdd76515d8c66e883e8b2bb-0001": 8086, + "09-b4-6be951b6ce98388222ffe59cc2d2-0003": 8087, + "09-b4-7b614c26c07671e51ad9ed08e877-0001": 8088, + "09-b4-8712518913693de955f558558b20-0001": 8089, + "09-b4-a4dfd61972bccc3b07ce6167f042-0001": 8090, + "09-b4-cdf4d7b4ae26ecdcc081a8dbe9b3-0003": 8091, + "09-b4-e4c8da99b3f215e06a2da4951997-0004": 8092, + "09-b5-11e467a21e290e20fff82bf6d43b-0003": 8093, + "09-b5-15d0d7467c2d9c78db131db6276d-0001": 8094, + "09-b5-1f83d4645a9f4a09321e6758f6aa-0002": 8095, + "09-b5-3ccf2749b3e696fd2a49784b093a-0003": 8096, + "09-b5-6d20706455105c3b2e29465b8d16-0001": 8097, + "09-b5-bded55f16ddad33ded00265f4431-0002": 8098, + "09-b5-fc872a0d04b86c46e33c2ba9f482-0001": 8099, + "09-b6-403fd1183a95bb5121f9c8dad778-0001": 8100, + "09-b6-6067580cdd82fdae055e87a08364-0002": 8101, + "09-b6-674d4a259cbc50e53092f9bc8586-0001": 8102, + "09-b6-bad6c7ec7376010f78382c35058c-0001": 8103, + "09-b6-e0d25e7bd866c1610d0100f028b1-0001": 8104, + "09-b6-e8ed4209cd5f676d8f56102897b6-0001": 8105, + "09-b7-09bb74cb88e395334b335a3aa4ff-0002": 8106, + "09-b7-1217e5fae37e3159052bea346727-0002": 8107, + "09-b7-3ed894bb2f0ccecb5c448873ca9a-0001": 8108, + "09-b7-64a8df1317538e4cc1fef1e2a8ce-0002": 8109, + "09-b7-6af88b79be2140f546e6f73c38f4-0002": 8110, + "09-b7-910f8c630d43ca28fde25489c004-0001": 8111, + "09-b7-db92983e424a1d5a901775eff205-0002": 8112, + "09-b7-e682c5d3653d050381926f89155c-0001": 8113, + "09-b7-e6d670fedc7a38b4d5f9dc1293bd-0002": 8114, + "09-b7-fbd716a2e436d096a08b6abf5cde-0001": 8115, + "09-b8-16899d4778f7854c9cf1fa1f9556-0004": 8116, + "09-b8-23bb9c84fa835986d5094504f82b-0003": 8117, + "09-b8-34d1ab40207088051a4ffa21f4d5-0001": 8118, + "09-b8-7a0c5ab0e79a09284dbc915ad1ce-0002": 8119, + "09-b8-8dbc774530dbcdd885cd92b68b43-0002": 8120, + "09-b8-911d469fc4ce4d1e0f0e6cc12e77-0002": 8121, + "09-b8-c2963d5a14ea1c73a642d66b925e-0001": 8122, + "09-b8-d3f9ae9813ea4d787fd4812d17c5-0002": 8123, + "09-b8-e6079019ba41512560f8d77b8c73-0001": 8124, + "09-b8-f6e8226b5eb23466d091dc1eedc1-0002": 8125, + "09-b9-679b64a3197a9513d6fcd7a158b4-0001": 8126, + "09-b9-763b3f1cc96388bc4d612d2980b5-0001": 8127, + "09-b9-b130726d070de00866e1dabf0bb8-0002": 8128, + "09-b9-f61cc053eed7f25947d3db1b1e1c-0002": 8129, + "09-ba-0db409f371acee09530a8f3e76ba-0001": 8130, + "09-ba-50a6751015df78fec99265f4c908-0003": 8131, + "09-ba-59225dc5754fef091ce7884ec992-0001": 8132, + "09-ba-71505bc6fea06be9016031440815-0001": 8133, + "09-ba-7697c4111b5895cd15b8823658c7-0002": 8134, + "09-ba-76c12ceeb7676f37f1bb2c8956b6-0001": 8135, + "09-ba-8a55b60e2c1985ecf0bec1655849-0001": 8136, + "09-ba-c11457ccfa58236c791c29f85c8d-0001": 8137, + "09-ba-c3796b0c510e59fa4e613bf3f341-0002": 8138, + "09-ba-d12397acd027c3f037939566dfa4-0001": 8139, + "09-bb-6a20b0d732a7c66204fdb894a39b-0002": 8140, + "09-bb-9baef01d057623f50136e0c361dd-0002": 8141, + "09-bb-e8051c05584c9ee3c5cad635081a-0001": 8142, + "09-bc-10837765543238eb2793d918aee0-0001": 8143, + "09-bc-215fbe766b419fb35ed118da23f7-0001": 8144, + "09-bc-6cbfe6fff951ea0b1f0e52fcd93f-0003": 8145, + "09-bc-785056070586f0ecd72f19434046-0001": 8146, + "09-bc-ee2c98b6268ddd8ee39d735252cc-0001": 8147, + "09-bd-06ff66047f03e802c3a2e421b78a-0001": 8148, + "09-bd-180fc3e1fcc2cd2514a8bd19f731-0003": 8149, + "09-bd-1a622593639dbc38cc30ccb86fa2-0001": 8150, + "09-bd-261647890da0c383f744609aa759-0003": 8151, + "09-bd-2a0bc272805122b23c552cd17c40-0001": 8152, + "09-bd-515fff5f40f83d650955adcbff41-0001": 8153, + "09-bd-57654f8576f0307d6a07f1a869a3-0002": 8154, + "09-bd-beaa7676c0821762a2bf3a81682a-0001": 8155, + "09-be-72e42aaf4128e8d11012f745c711-0002": 8156, + "09-be-ba9618acf1330cce085802c57e81-0001": 8157, + "09-be-c0865c81741ebec43866e79420f3-0002": 8158, + "09-be-efcbb01d144a8b570143b5d51b5a-0001": 8159, + "09-be-f5c48781d4ef9ff6ff338ee1d3c3-0002": 8160, + "09-bf-0c11778e795a5d52e898049c8149-0001": 8161, + "09-bf-1b6a372e4c791a6ec1850d2bea9a-0001": 8162, + "09-bf-25057431a2ad47cada9712652f55-0002": 8163, + "09-bf-28927b2feb8dcdd231209f07e6d4-0002": 8164, + "09-bf-39f6c5f74681a98ddbec5bf4732c-0001": 8165, + "09-bf-48584cb6b95685cb41df94ea771a-0001": 8166, + "09-bf-544cd488ba0eb9645599204c9936-0001": 8167, + "09-bf-55d6985e3ea9c08d6fc9f8e470ad-0001": 8168, + "09-bf-6bb3da5a1cbbe46200ded12893de-0009": 8169, + "09-bf-875e7a01e6427c0156933f7683c9-0001": 8170, + "09-c0-08ba72bc9e2739490e7d0c4edce0-0001": 8171, + "09-c0-18cc52d6809a942b1c03164d685c-0001": 8172, + "09-c0-8b386a29317040da440c407b19ea-0001": 8173, + "09-c0-b2f67d9a20a6f3d10804abad76e5-0002": 8174, + "09-c0-b3d5d4962c48eee9a7cb653feaea-0002": 8175, + "09-c5-ef43c84287c1e7692e2c094b8582-0001": 8176, + "09-cc-3f5dccf70833c7e982a79928a21f-0002": 8177, + "09-df-07ede5312178eaeacba800d2746d-0001": 8178, + "09-ee-837233af39c5616d8500d3512d01-0002": 8179, + "09-f0-ce500883781c35771cf408619bf0-0002": 8180, + "09-f2-142e91ed252f7b8d641bc744f76e-0002": 8181, + "0a-00-196af24123791a175dc712c2ebf0-0003": 8182, + "0a-00-3de8451baf603496750f707df6ac-0002": 8183, + "0a-00-5a6d8787b18c8dd2a8bcb5f137af-0001": 8184, + "0a-00-7718a47a769505563d8b0e7cf7fa-0002": 8185, + "0a-00-84c826317b3e60b4d073bafaffdb-0003": 8186, + "0a-00-9e3c320bcccd5fcb59c12e9c4d64-0002": 8187, + "0a-00-eae735c58cf069d3a8908c1dcab7-0001": 8188, + "0a-00-fdfa286e245986caf42a7831fd2e-0001": 8189, + "0a-01-2d9d0a108e45a805bbac4501a2bc-0001": 8190, + "0a-01-2e6e4d8a27127970f05d7fbda22e-0001": 8191, + "0a-01-78dd4e9c7e9253c1e5f976c99945-0002": 8192, + "0a-01-89e514f8dacab5c2b8f8ee254ea8-0001": 8193, + "0a-01-daf5f3242926f2c9cfd5f685df3c-0001": 8194, + "0a-01-f00ac6e39d19523b4933f37ff182-0003": 8195, + "0a-02-03da8d4586cde5e8e4c3e0b4d2b7-0002": 8196, + "0a-02-0735dc7c85a9b8d295343a6b95c8-0001": 8197, + "0a-02-099f98d3a5589620b2375caa659a-0001": 8198, + "0a-02-36a0a7608e3755e5fd7d2e4fd889-0002": 8199, + "0a-02-3db193b57b35f462290e276a6a06-0001": 8200, + "0a-02-49d8a77c57877fb03399bb59966d-0002": 8201, + "0a-02-832d1d432bc6ad0c626c9deb6113-0001": 8202, + "0a-02-95d939ec3a4e9988c2f9e0682630-0002": 8203, + "0a-02-b38f6d6bd6688f4d927eeb0a916e-0002": 8204, + "0a-02-c13df2fccf34810b6aed5c67a66e-0002": 8205, + "0a-03-097afe1385765ca6abbfb4835c7a-0001": 8206, + "0a-03-1ad20726e6257061eca042d07af1-0001": 8207, + "0a-03-3cc116df9ebbc049e480ce82448f-0001": 8208, + "0a-03-577d5c82271b794cbef29c90e4fb-0001": 8209, + "0a-03-b83c74ad056dd55563f2d50383a8-0004": 8210, + "0a-03-d3bf4f4333f3f5666c78387b1088-0003": 8211, + "0a-03-e3892f398ad02ab3d3f79c80b2c6-0001": 8212, + "0a-03-f8f3f757b0b9233a553166a66aae-0001": 8213, + "0a-04-7a3516142ac14e8d91cf90ed6ef4-0001": 8214, + "0a-04-bce357c4def0b542169ebc95618a-0001": 8215, + "0a-04-c9876a802fd1b8caf23a3e0ce8e2-0001": 8216, + "0a-04-ca8b4b27a955fbb1d2eafce74737-0001": 8217, + "0a-04-e9375fd862513db82c9d015eeeb0-0002": 8218, + "0a-04-fff777d22e43464ffe2fe2d1e472-0001": 8219, + "0a-05-19c5cf8555a45d0f958b4b8dcdf1-0001": 8220, + "0a-05-3a607e5c351e27a8d2b01e5b5eb8-0002": 8221, + "0a-05-5e88ba1680a6e0f4941d3d8edef9-0003": 8222, + "0a-05-b3fa22f831cf93e0c1d49207df65-0001": 8223, + "0a-05-bffc8126b48e421b102c9673c992-0002": 8224, + "0a-06-74bf4818f553f0ac6408bd4c7ecc-0001": 8225, + "0a-06-a0142c36fff5b365fa6191454036-0001": 8226, + "0a-06-af86fcb23774db2fbf7e1f54963e-0003": 8227, + "0a-06-ed0f317e58f19617bbefa25556f8-0001": 8228, + "0a-07-0755fbdff23c62c5f6306ed91bbc-0002": 8229, + "0a-07-a35acf7b22729002eef14978c264-0001": 8230, + "0a-07-a549ab1922daf163411c3d93bc73-0001": 8231, + "0a-07-d37f960143c085254ce2d48a8d7c-0001": 8232, + "0a-07-e77b64a5998a9920663a44a12f6d-0001": 8233, + "0a-07-fe72fdf67268c08ce56bc4cf86b0-0001": 8234, + "0a-08-22b45d4a19e1c5f5d6a15880f9e9-0001": 8235, + "0a-08-5a9e28fd94ef3108d3d2771d3d95-0002": 8236, + "0a-08-a6651f4081e1f1b864badc952e84-0003": 8237, + "0a-08-c12cf11f49c03a70ac257b137af5-0001": 8238, + "0a-08-d8667943baa3783bb73bf3e18369-0001": 8239, + "0a-08-e2a20e0d328434c5f1dc181f4e17-0001": 8240, + "0a-08-ed72b991911f4a0487e95c0fe935-0001": 8241, + "0a-09-40df5c1c259b41bfec3b43209b75-0001": 8242, + "0a-09-8ef3104260c6c803f8edde362496-0002": 8243, + "0c-00-0ad2911a2f520a88739802ca62ab-0001": 8244, + "0c-00-0e55fc740ec0574ecf88f0e3c4a1-0001": 8245, + "0c-00-4cecb7abdac0edee445641fa1838-0002": 8246, + "0c-00-57da8ece21e830e02dd04ab6806a-0001": 8247, + "0c-00-6a61bf708c70ad6c17888d8b9551-0002": 8248, + "0c-00-6d2f0315d960210dc722b0580f3a-0002": 8249, + "0c-00-71b8c3049fbef811750e64b4addd-0003": 8250, + "0c-00-7eeee791d7143697d80911f56921-0001": 8251, + "0c-00-87fcae9a0d64ec7d486b298bd11a-0001": 8252, + "0c-00-dde51c3642857ce4b7a31d3d225a-0001": 8253, + "0c-00-e42fe3e229d28001ad1fca46a812-0003": 8254, + "0c-00-fe6f8011990cae830648e044098a-0001": 8255, + "0c-06-251a6170a2bcb8c470bbe0772594-0003": 8256, + "0c-06-3f8e3f3228da893038ba8c0b4a67-0001": 8257, + "0c-06-468217546c1ea021a9c5f65906f8-0003": 8258, + "0c-06-5aa59a18b7192688a7e027c32131-0002": 8259, + "0c-06-a63e8a57c944b7151e9b86952baf-0001": 8260, + "0c-06-cae9aedc96d12336263d827fea8b-0002": 8261, + "0c-06-cfa86ebe2bee4fd22b2f03f2459a-0001": 8262, + "0c-06-e1f572a195ebbf10c0aa2736658a-0001": 8263, + "0c-07-05022e2c85d0c66638c7a82fc178-0002": 8264, + "0c-07-2b2fa1dc2172a624869c3ed5da8c-0001": 8265, + "0c-07-35b8cdde4e4fe20dd8346a39042d-0001": 8266, + "0c-07-5072aafb4c8a15e01103a13ced96-0001": 8267, + "0c-07-54031ff76703be3f26b2b2451722-0001": 8268, + "0c-07-5d63c38ca464a24a0866388c5fde-0001": 8269, + "0c-07-6984b50d264815d16b0e80a55f90-0001": 8270, + "0c-07-7c74d2619bb3a8f46d214ad0687b-0001": 8271, + "0c-07-7d34bbb6071b4fa7b7abc9037434-0002": 8272, + "0c-07-9899278c5792b69fea6e3e5a095b-0002": 8273, + "0c-07-99533e015dc57d8146b68084e490-0001": 8274, + "0c-07-9babadef2b1ca9c475eb933b2638-0002": 8275, + "0c-07-9c09b9c3cd2eb2283be947186c95-0002": 8276, + "0c-07-9c819df3ba1e63e560ae10842e2a-0001": 8277, + "0c-07-e7ee5683414811d5ab264359733b-0003": 8278, + "0c-07-e977df1bef351e548d1e2d470295-0001": 8279, + "0c-08-0f5c87dd29b23a89dd846333a370-0001": 8280, + "0c-08-23d19fa6216fa4e457cbb8fa09ae-0002": 8281, + "0c-08-2f54511fe16f4eac16fa6505d7a5-0001": 8282, + "0c-08-51b2dea3584871f84c032c70f9c4-0001": 8283, + "0c-08-5f056be0e0756853571fddb1195a-0003": 8284, + "0c-08-625c052152c41e45fb1a6b26addf-0001": 8285, + "0c-08-64f996127a4b70dfec95171927aa-0001": 8286, + "0c-08-6cf92b83edf3d2d1e5ab6326b330-0001": 8287, + "0c-08-738edbc1622bac000a03e4798ab5-0001": 8288, + "0c-08-8cd3eeeb1e7bbb45d518d4c740f2-0003": 8289, + "0c-08-8e9b749314002d30246544a70120-0001": 8290, + "0c-08-99102d846ad788149c7192d8c291-0001": 8291, + "0c-08-9eb32c0343bd76c46ea4daa12f44-0001": 8292, + "0c-08-a56ae8b1fbe6a58365d010f1767d-0003": 8293, + "0c-08-a7b4b3e28068385fd99853227bd5-0001": 8294, + "0c-08-d5631558a6ad2d3fa220d18eaa1f-0001": 8295, + "0c-08-f42cd622170aa74433cf1fbf60de-0002": 8296, + "0c-09-057d75f78878d58a8bcc7a14d279-0002": 8297, + "0c-09-0ac8f3eee9ba0a26f052b8fd2dbf-0002": 8298, + "0c-09-15b8019e3bbe6ba51aab9996330d-0002": 8299, + "0c-09-3cc6992707a241b411d626d5559e-0001": 8300, + "0c-09-4068a37cc998b5ae06e1d402b1f7-0001": 8301, + "0c-09-4b3edffd02e419787cc4bee61349-0003": 8302, + "0c-09-67a4329ab20e4c6da15c3b713941-0002": 8303, + "0c-09-6e3c62b157c0d103ba0ddfcaca71-0001": 8304, + "0c-09-9b15170ae148b1fedbacafa07496-0001": 8305, + "0c-09-9f807780b9eb25bd2ca27664fe72-0002": 8306, + "0c-09-a5b67e336b69389a15a7f4c1f85c-0001": 8307, + "0c-09-a9a0f18be24e28d0729c7a6e8e1c-0002": 8308, + "0c-09-b2868eea02b235016510144e7669-0001": 8309, + "0c-09-b5615e2b51990be0f34cb1014b6e-0001": 8310, + "0c-09-ce9ea16fece04c3547d77e28400d-0003": 8311, + "0c-09-d161adaad69ebd27f579fe62c9c8-0002": 8312, + "0c-09-e5581377f47a5f9c61546163e6b4-0001": 8313, + "0c-09-f062ef23002c24ca788d683de872-0001": 8314, + "0c-09-f21aba2d4f099f13c234e4ad6c36-0002": 8315, + "0c-0b-161435ab7bbb5f61915888ea9a9b-0001": 8316, + "0c-0b-1f6c80fbe5cdd57cba39fa6bd59f-0002": 8317, + "0c-0b-3d64112eddbe3f087d66251ad1aa-0001": 8318, + "0c-0b-5318500fa72a9b7f2ded92775b8a-0001": 8319, + "0c-0b-835c683b044b8b1ae9688227f4af-0002": 8320, + "0c-0b-96dabcfa6a00598b117d1e1ddaf2-0003": 8321, + "0c-0b-b2bff69d31db934c3dad54057db9-0001": 8322, + "0c-0b-cdd09106bef5a03202d119413e45-0001": 8323, + "0c-0b-f4f385c169b36733c0f8be0e982e-0001": 8324, + "0c-0b-f7617d0fd5af50944478bdbd37e7-0001": 8325, + "0c-0b-fab7beb79dee338a318a940a9b8f-0002": 8326, + "0c-0c-27b7e5d93ae4fd07713a0e6e5c9b-0001": 8327, + "0c-0c-380ee13c0932d3d5205657fce5b3-0001": 8328, + "0c-0c-58b135edaf334a0015ff656be550-0002": 8329, + "0c-0c-5a4060ead0cb2e9604badb04bc65-0001": 8330, + "0c-0c-5c8fa2b865308fda61a6312cd393-0001": 8331, + "0c-0c-5fa52038e7bfcb38d964943cf488-0001": 8332, + "0c-0c-6bfbc298f3689a6ad370642c1b4c-0001": 8333, + "0c-0c-6d6beaf365192d2ea6b65e8f0fa6-0001": 8334, + "0c-0c-70b7e4f2f44cb244573231b43ac7-0001": 8335, + "0c-0c-7bf94086d9c5846e49c11f8854a3-0001": 8336, + "0c-0c-8479288312334d6055053a5e02df-0001": 8337, + "0c-0c-adff9f34a41c74c086481d725fd5-0001": 8338, + "0c-0c-b521c533e65b56d377e68b0435ee-0002": 8339, + "0c-0c-ecf196a31905970919ffb07fd7e3-0003": 8340, + "0c-0c-f5926fd280730ff3ed4b7adad062-0001": 8341, + "0c-0e-0a4ac231d2f12946656c0a920f5f-0003": 8342, + "0c-0e-24d1cda5abf75bbb366d1b5f5429-0001": 8343, + "0c-0e-6f0d2085b6e697271b56b4e680ab-0001": 8344, + "0c-0e-9aaca8a3f9bd27bb01cab58251b1-0001": 8345, + "0c-0e-9df03b8c303fd049d5c3462cab5e-0001": 8346, + "0c-0e-a535760acb88cb3caff89902e0a2-0001": 8347, + "0c-0e-d799a7700133ecd43dca447d8b04-0001": 8348, + "0c-0e-dd38d5694da1d65b1220dd138535-0002": 8349, + "0c-0e-f1e5312d83428e7f3b43c8179a7f-0001": 8350, + "0c-10-0efd47615c51f799d38ac245dde7-0002": 8351, + "0c-10-1ecc636dfcb3c1528b8e2faac872-0003": 8352, + "0c-10-2758e12742687bbcc1716e2ee6d7-0001": 8353, + "0c-10-3b1bf4388b780eed5eae6bac1f72-0003": 8354, + "0c-10-3e68c2ecc4c4b585e9138d2819f3-0001": 8355, + "0c-10-59cf48aeaaa9775bea40ce99ab92-0001": 8356, + "0c-10-78ba5a37e52fab2e16cdd80706db-0001": 8357, + "0c-10-8486f08035ba152d5244ac54099c-0001": 8358, + "0c-10-84d31a0331417308df042e80fcb1-0001": 8359, + "0c-10-b3531fb60cacb2cf5024e03cfec2-0001": 8360, + "0c-10-bcbd4e14ae143ac15059e3c4d7f1-0001": 8361, + "0c-10-bcc2d4d2130fbd8de772bfc023a8-0001": 8362, + "0c-10-bf96a0642ddc8aea3f8b961de42d-0002": 8363, + "0c-10-c468a57377ff8ef63d3b26a6d1fa-0002": 8364, + "0c-10-ca65709dc41cfbd85dca0508539c-0002": 8365, + "0c-10-df4c252fc8da8bbd93019f748dd3-0002": 8366, + "0c-10-ee6bfacbd537d191d4f993c08ac3-0001": 8367, + "0c-10-fc0869ac7cb46e6a210731414c7c-0001": 8368, + "0c-10-fce09b4e97c9e1981603de292ef1-0002": 8369, + "0c-12-043eb2d0c36d9080037a69e21ed6-0001": 8370, + "0c-12-113793012883c24400ae0d9e6dcf-0002": 8371, + "0c-12-1ccffb1e4313521d39b4da4ebdfc-0001": 8372, + "0c-12-35bbd2c0fc1284a06b6d21dc204e-0002": 8373, + "0c-12-61df363935a9029f48170101c2f8-0001": 8374, + "0c-12-651cfb742d69c8fd4c6d1cef3700-0002": 8375, + "0c-12-7151cebac6e986b3e0b4bec37086-0001": 8376, + "0c-12-89db2b59d184a838367def4ef8c8-0001": 8377, + "0c-12-a231f9cc98878f7e4b1ce4115c93-0001": 8378, + "0c-12-a47176a68737d6bba782893a3ea9-0002": 8379, + "0c-12-a71adb403c3d963fbd08a3403fa0-0001": 8380, + "0c-12-b2af00c9275202d8988b1c834556-0001": 8381, + "0c-12-b49e8e8dc5a923f91b66077ab2d7-0001": 8382, + "0c-12-b8e47e9ec28cbdff595c17981d44-0003": 8383, + "0c-12-cf96b1ddc48d5633958979429819-0002": 8384, + "0c-12-de294028be6cce19aa6519c0be7a-0001": 8385, + "0c-14-03d8f31d1408833e890ccd877d2b-0003": 8386, + "0c-14-0663763e19c5cc710703b77b71e6-0001": 8387, + "0c-14-19d728dd9d42ff8a984aedcef5f1-0002": 8388, + "0c-14-421ed63655f9d80c6990cd392d7b-0001": 8389, + "0c-14-4da9f7abe02fba8ddea6a3ba5487-0001": 8390, + "0c-14-53dd1ea7683f239c13b5645d1427-0002": 8391, + "0c-14-61a8058bab1d1d94e7b24e4c1163-0002": 8392, + "0c-14-75bb1beff04aba54e71b40fd4878-0001": 8393, + "0c-14-8296810bff572989978ed934251e-0003": 8394, + "0c-14-90061a9038d01a63b9f41c87cb87-0001": 8395, + "0c-14-9c4580534edfa0e01e0af18878e5-0002": 8396, + "0c-14-a06bce7178876ffa2151e3d91cb8-0002": 8397, + "0c-14-c270fe94902d7ffba9df658bdda2-0002": 8398, + "0c-14-d1102dbc6d0d61473adf9dd5fa90-0002": 8399, + "0c-14-d799663d566b937d336e74f154f7-0003": 8400, + "0c-14-e8c5649cd8ead1b0a5570af0a634-0002": 8401, + "0c-14-f2b46dc20310b229424dcdbd0793-0001": 8402, + "0c-14-fe9900a0be8476db194719423b6b-0004": 8403, + "0c-15-00ecfdbea4e3a94b3411bbfa1e3b-0001": 8404, + "0c-15-325b9e925064ea017692e434c7b1-0003": 8405, + "0c-15-39c2e2b5788b6c02c830f5a6e8d0-0001": 8406, + "0c-15-55f3cbe0af7c4b85d60e96caa668-0002": 8407, + "0c-15-5aee236b9ad2fbac730505edf4f9-0001": 8408, + "0c-15-624bc263c2ed4aa331e68fc6d718-0002": 8409, + "0c-15-69b4ef761a16712816deed174d05-0004": 8410, + "0c-15-70ddb5d5d92ed7a607810e714ab1-0002": 8411, + "0c-15-8a0320cec9e4304a1be2704b9408-0002": 8412, + "0c-15-8ea22dd822fd18d1a6aa1b91a1b3-0001": 8413, + "0c-15-aebc34624a3d57977c23ee8851b4-0001": 8414, + "0c-15-b0d2e3eafc95a0f6ddfa02f09625-0001": 8415, + "0c-15-b1314fc9bacd9d6ee8a56157c981-0004": 8416, + "0c-15-d9f1b05af77c97392359e8baf2e9-0002": 8417, + "0c-15-e6650674804cbb3248f2bc9e4856-0002": 8418, + "0c-15-fd0d5457dbf13f2a7ce5df7de904-0001": 8419, + "0c-15-fee178344b0ac3f633932a9ebc46-0001": 8420, + "0c-17-0b23148bb0e30d1fba95cec884a6-0001": 8421, + "0c-17-144079770cd85c4c52d48a0679ed-0001": 8422, + "0c-17-1be3a8f4b8e6aa34cedb4469c6e8-0001": 8423, + "0c-17-29aef64fb3501658f51604cba439-0003": 8424, + "0c-17-401aa9e3c20f4b9265f4665227ab-0001": 8425, + "0c-17-40ba3bf44639edbe9966bf1c36ab-0001": 8426, + "0c-17-49cd044fe65552a3122cfa0f5e4d-0001": 8427, + "0c-17-56c9f506a568e71ac361078052b0-0002": 8428, + "0c-17-6efd272d61b0fbad44747550db7a-0001": 8429, + "0c-17-7c6ac18af5c728bbae36228bff95-0001": 8430, + "0c-17-8f130c91c0ccde0da9bb917ba420-0001": 8431, + "0c-17-93fd046ee04a909e44015c86f367-0001": 8432, + "0c-17-9428e1bf8b1b28b11ed790e29fae-0001": 8433, + "0c-17-b0c044fcb477dae6c9073c3394f3-0001": 8434, + "0c-17-b6c25f839bdbfdd25f4b37438c92-0002": 8435, + "0c-17-bed5c00b5bcd7a92341c296b0c41-0001": 8436, + "0c-17-ced4925404c496d18b020d37a07f-0001": 8437, + "0c-17-d80a115039f089b65f009bff6e92-0001": 8438, + "0c-18-01cb8d3fa8b861c7256e60b82f39-0001": 8439, + "0c-18-080869277818eddd4b8e46d06693-0001": 8440, + "0c-18-454ee8978141f4cd4da075594366-0001": 8441, + "0c-18-6a09b8dd5cf472aa8ea2b2927a3b-0001": 8442, + "0c-18-6c3e80cba08632fe9c4a512ce789-0001": 8443, + "0c-18-74016e4072fd9d15a5fa276abdfe-0004": 8444, + "0c-18-7555a7b645e6fd23db4c4360f2b6-0001": 8445, + "0c-18-7d3817a5a2e42715d70bd04d3738-0004": 8446, + "0c-18-842bb63a21e513da58ce3f581dad-0001": 8447, + "0c-18-896597966cd725bfa12cd5ea30e6-0002": 8448, + "0c-18-9de1b66e29ba2b4f28e8b58bc241-0001": 8449, + "0c-18-a589c77b3799d86917ca72b9d06e-0001": 8450, + "0c-18-c284245f07e3373e30c27fbd0323-0002": 8451, + "0c-18-c658946c8f749aa07eb3621f0044-0001": 8452, + "0c-18-c8061f2ecb6db6b9e9b0992ced19-0001": 8453, + "0c-18-e490898091657c8b88878b594a98-0001": 8454, + "0c-18-e64df0629d64951042b334fade1e-0001": 8455, + "0c-18-ebb2fe85640815e25d5cf48b59e9-0001": 8456, + "0c-18-ecbbca46da5a15317e27ee0810e0-0002": 8457, + "0c-18-f20934fa83c1062975490c1793f1-0001": 8458, + "0c-18-f864a9b196ed723042623752da82-0002": 8459, + "0c-1a-042291b042fd4bd4cbfea100c44f-0003": 8460, + "0c-1a-15ee424d9211d3f37f66822f65a0-0001": 8461, + "0c-1a-2a1446ab42f724443c51578f9e44-0002": 8462, + "0c-1a-389fd42852487e6e382eb705c303-0001": 8463, + "0c-1a-4ed7109882b339c35090117c3f86-0001": 8464, + "0c-1a-5e194fdff312355df9724f7b1efe-0004": 8465, + "0c-1a-6ec1ff49e81010f0c4c89ba9bb12-0001": 8466, + "0c-1a-b54278ef3d0790821e1c9a12d094-0001": 8467, + "0c-1a-bac21dd877c98be2c741da2204c6-0002": 8468, + "0c-1a-d201d4352de5b16a90c4b396a32a-0001": 8469, + "0c-1a-d83d07714ff1b6dd068696ab184c-0001": 8470, + "0c-1a-dd9c4f78680b59c03d8668dc270b-0002": 8471, + "0c-1a-e7d51d03f1d56a5af57137e03e1d-0002": 8472, + "0c-1a-ecb4bae2b0d37bcd08fe15ebe5de-0002": 8473, + "0c-1a-fd51b02e47dd9628b0ea1f766ae3-0002": 8474, + "0c-1a-ffcf90b1cd3f9d8441f62bfb684d-0001": 8475, + "0c-1c-105602f44505c9228df8d883a0b4-0001": 8476, + "0c-1c-2c703ef1dfd13328664d3533e6e4-0001": 8477, + "0c-1c-2f06a427ed6bf4e13c7831e00d8c-0001": 8478, + "0c-1c-51027107a239c546ae32847a5398-0001": 8479, + "0c-1c-7d8699b0d8fc73bfd1000c5568eb-0001": 8480, + "0c-1c-7edb8319d1a024f3dd0778586db4-0001": 8481, + "0c-1c-9cf9b3844c677d43ef322f8c1666-0002": 8482, + "0c-1c-c60f36ef795873e1bb484aea516b-0001": 8483, + "0c-1c-dfb63923a87c16ff658b04634469-0001": 8484, + "0c-1c-fa557393d609bb6941c01eb3bd76-0004": 8485, + "0c-1e-19c5f86208faefe636af9fae6ce1-0001": 8486, + "0c-1e-242d3b1dcba66a43c8113e30aa9d-0001": 8487, + "0c-1e-328c89233b91a022e70937030226-0001": 8488, + "0c-1e-332ae895d8f2c0b864e62e3f2cbf-0002": 8489, + "0c-1e-37cfa95660348da2b3acebd3e424-0001": 8490, + "0c-1e-5e1ff636005b7f43e47c8a3b26f9-0002": 8491, + "0c-1e-7993e710e48cf54dab7370f4d8a6-0004": 8492, + "0c-1e-82db14290f6279d1b5f50aea5239-0001": 8493, + "0c-1e-853dc5e698658e2582e10789e24b-0003": 8494, + "0c-1e-8a42f7a1ba0fe8e5e79dc93903d4-0001": 8495, + "0c-1e-8f6c7ff05e50495c06a48c83ab3d-0001": 8496, + "0c-1e-96ccbae6eb1db60d4e8505f8a4da-0002": 8497, + "0c-1e-b6f6c00ddef88d47eb9f7397a4c1-0001": 8498, + "0c-1e-d39487861cc673ef27f034577694-0001": 8499, + "0c-20-03378649131c6b5792560429d98e-0002": 8500, + "0c-20-0b8361997477dbd1ce60a8d9f733-0001": 8501, + "0c-20-25dab22100f69159f0c2ff53b4c6-0001": 8502, + "0c-20-263708ac43e37ad6ec316433fcb9-0002": 8503, + "0c-20-4d5a2c2dbba3cb7b933e74248f8c-0001": 8504, + "0c-20-4e381880ddc8b8396893f49d393e-0002": 8505, + "0c-20-555830050e349d0e4db7c5d73dd0-0001": 8506, + "0c-20-58f0208406fa88a33aa1966493ee-0001": 8507, + "0c-20-7b17a5a5f10ec8f59bae287b6e2e-0001": 8508, + "0c-20-8a5949fada4cec5f88a0fb7c4ced-0001": 8509, + "0c-20-a407a0eb9da408e7860e07766db4-0002": 8510, + "0c-20-aaaa069e42ee92a7312a80cac66b-0002": 8511, + "0c-20-d398130e5c0e638e86728ae521e6-0003": 8512, + "0c-20-d8b56b5ddb7126a52a7a485050b4-0001": 8513, + "0c-20-f301c0f26eabf15033ece12a2dc3-0001": 8514, + "0c-22-083aed017c171c09223f08acba0d-0001": 8515, + "0c-22-0efe35837c88b23610f587b2d7fc-0001": 8516, + "0c-22-1101fda0ea4d02421922c57d6453-0002": 8517, + "0c-22-13a05fc1c3de1872863598f63ee7-0001": 8518, + "0c-22-1e6fac01698322c6b30cf5dec408-0001": 8519, + "0c-22-37349538c14f2515e0d2d447795c-0003": 8520, + "0c-22-43535721cd45d46a681ba92c3f8c-0001": 8521, + "0c-22-50bdd8c5a820b1e6021a4fd80a9e-0001": 8522, + "0c-22-5ffc9da9a02e7daf04db5de0a648-0002": 8523, + "0c-22-60086e96fc06bea9f2771ddb0dd9-0001": 8524, + "0c-22-643625a193e36506b4b4d7cce179-0001": 8525, + "0c-22-8e1c85631d481e8b8e24dff2db34-0004": 8526, + "0c-22-a88c98c550e82ebc637ef088d98f-0001": 8527, + "0c-22-a9aa8bb7141fec0c2e6a55d95997-0001": 8528, + "0c-22-acd299bc90707fd9a05172eb5343-0001": 8529, + "0c-22-b8b93f6df1f51e1aa97fcaf2807e-0001": 8530, + "0c-22-c8f95f7f81e8306b1a2203b50604-0003": 8531, + "0c-22-dd65550ee260739ca894f51e4090-0001": 8532, + "0c-22-f098b50043c4dbfee0f86f11ba28-0001": 8533, + "0c-25-02831ab10b3b9c0486ea9463f8bd-0002": 8534, + "0c-25-02b66c1b47fd5f43d1e9381c9234-0002": 8535, + "0c-25-0775dca8c6d21c52c70d44e34d39-0001": 8536, + "0c-25-16dbbfaa08baeb7d426a23331339-0001": 8537, + "0c-25-1d0bc89e63337538ed0a853181a9-0001": 8538, + "0c-25-26c345e520f80f14c0780ff82439-0001": 8539, + "0c-25-2d624368d139c81d29bd7bbbbf1a-0001": 8540, + "0c-25-2dcaab5d814aeeb99f078f9e6896-0003": 8541, + "0c-25-4a7f061a82644b6c5134c421765a-0002": 8542, + "0c-25-4ae4999060a6871190ade328b451-0001": 8543, + "0c-25-8f4d96b41957810b2e8ba62db304-0003": 8544, + "0c-25-b181f07b9692f2c332c9fcd741f7-0001": 8545, + "0c-25-b8208ac7cfa7e3bf6c6c299e9930-0001": 8546, + "0c-25-e3ed1a5a851c043d77a878c7ad48-0001": 8547, + "0c-25-e566e10e0370ae7c0cad7411e805-0001": 8548, + "0c-25-e791a0f130742b503a3056c1f2d0-0001": 8549, + "0c-26-2540fa3cab55d2b9af9eed2f26b6-0001": 8550, + "0c-26-4d7d070b3ee592ac917c0609bec0-0001": 8551, + "0c-26-64ee24c2f5e24846649d1c4963da-0002": 8552, + "0c-26-736295f765786ad0da32998225b2-0001": 8553, + "0c-26-7377af8c36732abc7525631e2c1f-0001": 8554, + "0c-26-892eb40347e4c4861ac33891e5f3-0001": 8555, + "0c-26-964f48e64a1ab263ccb47da547e8-0001": 8556, + "0c-26-9977738e29d51eb6088d1cad685c-0001": 8557, + "0c-26-aeedaeef8cb23934c1be585b7589-0001": 8558, + "0c-26-b0f9d692ed2bc92d17021cb560b0-0001": 8559, + "0c-26-d65895154ce9971ff06b8588f6eb-0001": 8560, + "0c-26-d6c9028ba50bcc58f1bcc8828159-0001": 8561, + "0c-26-e84e84df191a06eccb4fed491d7c-0001": 8562, + "0c-26-fab35fe6d3bcc782ee7c08ba9ae9-0001": 8563, + "0c-27-084aa63fe36574c47a2adb8b5315-0001": 8564, + "0c-27-1f5d0ae75e203f23aa4e5a9e5a5c-0002": 8565, + "0c-27-1f5d0ae75e203f23aa4e5a9e5a5c-0021": 8566, + "0c-27-21299a9e78c37ea1b6b50a641d95-0001": 8567, + "0c-27-2f62fff635d378ac339ab81e0e9c-0002": 8568, + "0c-27-31652a7ca9f271ad29afb613ae11-0002": 8569, + "0c-27-496dbf50f4bec896140f0deafb2c-0002": 8570, + "0c-27-61534e97fad61e6b19c4ab803723-0001": 8571, + "0c-27-74a3bb490f24d9944f44ca1c34a3-0001": 8572, + "0c-27-7758eaa9b24ff6ebb6cd2d7ab246-0001": 8573, + "0c-27-b1d51b103f8238eb06c55f1da7ee-0002": 8574, + "0c-27-cdbe38ed178fbf605e4bdaf8e5e5-0003": 8575, + "0c-27-ce1fea95180c69a5f02a21719bd1-0002": 8576, + "0c-27-e2f54173eead77bdc487afdb9689-0002": 8577, + "0c-27-e390f232c2e19a57c2a5b80d82d8-0003": 8578, + "0c-27-f0c600e63a7a5cda6b5be6959373-0003": 8579, + "0c-2b-372b22921e451cdebfb2e3820bf4-0002": 8580, + "0c-2b-42e4f4d25e6e71e5ab8e831ae2a3-0003": 8581, + "0c-2b-557ebdbeb57bdfd581ac46c8d876-0002": 8582, + "0c-2b-664f9bbc34ff6bf32f580fedc0fc-0001": 8583, + "0c-2b-88733b89d63b98012b1efd97524b-0003": 8584, + "0c-2b-8f46189851b3f6f07c962e2cf12a-0001": 8585, + "0c-2b-9ef619f9ceff94b43bd45b2084d2-0004": 8586, + "0c-2b-a75fecb6f37fba8aa70e21cb35c6-0001": 8587, + "0c-2b-c85a5af5c32de3c098913d5d5858-0001": 8588, + "0c-2b-ce06c3a48785b4e2e2e9c40f0931-0001": 8589, + "0c-2b-d336d0cb57b45124c44ba6522144-0001": 8590, + "0c-2d-08966ffe4f1d4fba77de72eda1b7-0001": 8591, + "0c-2d-481cb70f54160294265ba7ffbcf9-0001": 8592, + "0c-2d-54ea50f01aecb27723e11c2665c7-0002": 8593, + "0c-2d-683cc3714eedfe0c008fca4aa6b2-0002": 8594, + "0c-2d-78341a2eb752d8bb80cb164e7346-0002": 8595, + "0c-2d-a173ead16ed387116539f22581fc-0002": 8596, + "0c-2d-c49e4dfd9a1135cb48e35bafb5a0-0001": 8597, + "0c-2d-d442d5ccbfed14afe62d90964531-0003": 8598, + "0c-2d-db1f32cfdeb9685a87baf50686ea-0001": 8599, + "0c-2d-e2dfcc7fb17f38277d46d74280db-0002": 8600, + "0c-2d-e6667c0b5523d5fe337f0e6ef17a-0001": 8601, + "0c-2d-f5a2f2ed5db074af4c7c6aa82630-0002": 8602, + "0c-2f-01b5c66b841d4b5111753b676b0e-0001": 8603, + "0c-2f-06af8b81e066fdc31cb7bffbadee-0002": 8604, + "0c-2f-1e1f5dbe217f7f257a78b2889668-0001": 8605, + "0c-2f-3cfe1024d5453c3b7cb9e284e00b-0001": 8606, + "0c-2f-59647da39938585926ec7df1c383-0004": 8607, + "0c-2f-637b2db5c3f8d0b79c6d5c3de1ee-0001": 8608, + "0c-2f-6430de53d4634c29ce2870a2f3a1-0001": 8609, + "0c-2f-683b2ea6735f05e9b805e842fba6-0001": 8610, + "0c-2f-814e9afed5fd8ad3dec0c1424397-0001": 8611, + "0c-2f-88acd0b5234575e7feff82f6a0b3-0001": 8612, + "0c-2f-c58e56c0d2c36a8528dd59204fcb-0001": 8613, + "0c-2f-d8f6b3c706dfcfb5119bc0451edd-0001": 8614, + "0c-2f-fcc943c3d99585629b6876943368-0002": 8615, + "0c-31-1f3c451a59742c569f7ac79969a4-0002": 8616, + "0c-31-29755d0204d6d2cda81e3a834e58-0001": 8617, + "0c-31-2b275ccd9581790bb241b5ff5ff7-0003": 8618, + "0c-31-4b55400a5402605db0d0729abe44-0001": 8619, + "0c-31-6a0025045c8147b1466e39366358-0002": 8620, + "0c-31-8d09ea394af9473970e915c9b31b-0001": 8621, + "0c-31-a10789964d8807ef1a99bb755d5a-0002": 8622, + "0c-31-ca7d3cdb4cc105d9669ffaf4611a-0002": 8623, + "0c-31-d8430d4fbc161dd89bdef2808f7f-0001": 8624, + "0c-31-f9139ca65fad56b37ca68518a049-0001": 8625, + "0c-31-ff4bee83ba3bb17eba31d58e9ed7-0001": 8626, + "0c-33-10fc4f83369c306b63beb9e838e7-0001": 8627, + "0c-33-128eca8942f7484ca832b47a4fb2-0001": 8628, + "0c-33-15a2971cfad60a6c6a13479dc4c7-0001": 8629, + "0c-33-19479ed29f47b74a30de1082245f-0002": 8630, + "0c-33-1a646a5900377bfeb6a2a71c47f5-0001": 8631, + "0c-33-2776cda0019a6e4f07114d112d6e-0002": 8632, + "0c-33-2a6aa6945461b5bcb54d1dccf095-0002": 8633, + "0c-33-3fce3a420ae30df06cf9201ad5b5-0001": 8634, + "0c-33-57a24977c1e627ba2022c214e259-0002": 8635, + "0c-33-60216af2c37721683cb7ad99c2e8-0001": 8636, + "0c-33-62146c5869b59fb39282641141c8-0001": 8637, + "0c-33-664ef8ed85b19a5f7117a31cc230-0001": 8638, + "0c-33-6c18ea2c8189a3e0487d0c2b3b4e-0001": 8639, + "0c-33-703089376badbd1a1e06453e43a8-0002": 8640, + "0c-33-893f7a998864401c5212bfde1412-0001": 8641, + "0c-33-8da0c5a2e02aa66312024fa500f8-0004": 8642, + "0c-33-8eb63d8e74a33482ef20d32ad2a1-0002": 8643, + "0c-33-96894562b4587b475acde5b1a1e9-0001": 8644, + "0c-33-a1e6f359a091f725158b1ccf64f0-0001": 8645, + "0c-33-a5e6bc0921b6cf9094f045410741-0003": 8646, + "0c-33-a977189e8f33debaa1f817eeed20-0004": 8647, + "0c-33-c5c8ce538137eb0a1d34e5d20e04-0001": 8648, + "0c-33-e4637a5c64df3c0d8e3596a684ed-0002": 8649, + "0c-33-e5ff6df0b6b813bcd8c69d887ab2-0002": 8650, + "0c-33-e92b1c7f40368a892a686a55fc18-0002": 8651, + "0c-35-1e9a3ca5a8a0d0e4553c06c8cb8d-0001": 8652, + "0c-35-1f3fd1ffed607dd3d949dd71ecaa-0003": 8653, + "0c-35-241426d76c0dc2f468ed71aee703-0001": 8654, + "0c-35-25d08eefcddf570bfe0f192b4d61-0001": 8655, + "0c-35-51ac617382e3cca6b358d2a03366-0001": 8656, + "0c-35-51bea33bd60a415827b7af4bdfdf-0003": 8657, + "0c-35-5d3ad13925b5a353c9747144aed7-0001": 8658, + "0c-35-5dacd763f823ddf22c0973713d9b-0003": 8659, + "0c-35-68a7c8c3fb7aca6e8c9d9b4aaff0-0002": 8660, + "0c-35-697bf70907108025d75f887e7dee-0001": 8661, + "0c-35-73f4aae8dbb5624fee34595fc453-0001": 8662, + "0c-35-7aed6ba678f6781e047956c850a4-0001": 8663, + "0c-35-7f1a13e437a001db015eb45be0e3-0002": 8664, + "0c-35-8787138fb1f3aea6387838d263b9-0002": 8665, + "0c-35-a246f38bc7136d7cec826bcd59d5-0001": 8666, + "0c-35-a9a1c618de3365b8e383b8cf336f-0003": 8667, + "0c-35-b9734d13c9a9775fca86698ccb90-0001": 8668, + "0c-35-c164766d34cfe4778e4a7e778867-0003": 8669, + "0c-35-f2af17dc3ec43dcf1022d574a3f7-0001": 8670, + "0c-35-ffbc9bd79228d87c19a22b8ae6b5-0003": 8671, + "0c-37-5db66cef7752da1152e22aa23329-0001": 8672, + "0c-37-798394ab408d1159c59ed8219e65-0002": 8673, + "0c-37-af4c4d4aede4eda85380893d96ea-0001": 8674, + "0c-37-da7f8d58e784d701ac08c099026e-0001": 8675, + "0c-39-16f56bea0bb9877e2527fac89957-0003": 8676, + "0c-39-22b94eef5a5c0a009c5c779b2168-0001": 8677, + "0c-39-30af08da492a88d38fd99034fcd7-0001": 8678, + "0c-39-389197db4f09ba16b69ed7f03975-0001": 8679, + "0c-39-49d10b38d1057d6291d77516be41-0001": 8680, + "0c-39-50ec5b316bd572f8b12898b86196-0001": 8681, + "0c-39-7808265002f37ad385651ef6571f-0002": 8682, + "0c-39-b64124046ad607899aef6f822e65-0001": 8683, + "0c-39-c103df844ce856b1ebc787f04012-0001": 8684, + "0c-39-cac5cca11676c21bad4c08d8ab1c-0002": 8685, + "0c-39-dd1252c09ff768fbdc47ad11d55b-0001": 8686, + "0c-39-e7121ba919e0bbdbff40d8696da0-0003": 8687, + "0c-39-f90fda6d73954b84459ffd83ae1a-0001": 8688, + "0c-39-ffcbb25cbda790fda315ab45839c-0002": 8689, + "0c-3a-0d0db3e3c2734e09e1cb26378c61-0001": 8690, + "0c-3a-206eff7b16df0421cd822949969d-0002": 8691, + "0c-3a-34def978ab4dd416104250f53948-0001": 8692, + "0c-3a-3bbc76b2c7f18f6475fd00444b33-0001": 8693, + "0c-3a-436e28ea3adff99471886e879bf2-0002": 8694, + "0c-3a-57dba3c1d41c77669908dc1b6d4b-0001": 8695, + "0c-3a-5d9672b94f7359228bb8ba46539c-0003": 8696, + "0c-3a-6098c40576c39192d1c2b6942a91-0001": 8697, + "0c-3a-662ed6772dfe7b30e0996801617b-0001": 8698, + "0c-3a-7fa1dbda6818c788342de348ce88-0002": 8699, + "0c-3a-88dc14365ae342fb9befb2cef4bd-0001": 8700, + "0c-3a-a9d7ed8dc00d5f83da0e11a2b2dd-0001": 8701, + "0c-3a-bd69f5a9a2cb3796a829be74e608-0002": 8702, + "0c-3a-cb75fb3f201cde660e1d86506516-0002": 8703, + "0c-3a-dd74c50436d0ec4ba6c6d9661fda-0002": 8704, + "0c-3a-e0c042b0c84d63b169024b329d91-0002": 8705, + "0c-3b-1ba3870ad0f365143915f693aef2-0001": 8706, + "0c-3b-2155ee0f8084bfbb74529a5eeeb4-0002": 8707, + "0c-3b-235a2a053e604c45f3c2ca8bbc7e-0001": 8708, + "0c-3b-270d678b45ef043f0bbb91912db7-0001": 8709, + "0c-3b-277d1d692112ecfe89198f1a98b0-0001": 8710, + "0c-3b-2bd12fbea814a2d41cedac328dba-0001": 8711, + "0c-3b-2f287c883de1e48c825967f22ee0-0001": 8712, + "0c-3b-3b21dfedcf6932ee619b8152447d-0001": 8713, + "0c-3b-4f8f17875a5751cd861586df8df5-0001": 8714, + "0c-3b-5d0263aa0c3b2413969d73776b08-0002": 8715, + "0c-3b-6e115ccc19d80615d6196b4958f8-0001": 8716, + "0c-3b-7307db10b797edfb3aa18459693a-0003": 8717, + "0c-3b-8eb1a7545ec7d5be89343171482a-0001": 8718, + "0c-3b-9701ec507d991cc76f51f00b2a53-0001": 8719, + "0c-3b-970f0a85e0323bd71c24239a7fa8-0002": 8720, + "0c-3b-a6ba242b3fdba0e3634dc185fa67-0001": 8721, + "0c-3b-a9a599f36a9ae70de717c2c8836d-0002": 8722, + "0c-3b-c96345dc185665cdd95b57fe4810-0001": 8723, + "0c-3b-d0fbe35bdf3a5c711217ac93f7cb-0001": 8724, + "0c-3b-d487e8353c514291f25ad1b518ab-0001": 8725, + "0c-3b-d9cbb3ed9ba4fa50338424a56a80-0001": 8726, + "0c-3b-ed8c41537c06badf4daf23ed987b-0001": 8727, + "0c-3b-f289ed526e1b884420001fe8c660-0001": 8728, + "0c-3c-00a8675753299c946b300ca6f78c-0003": 8729, + "0c-3c-09e1def19d6285356c04cb5e74a1-0002": 8730, + "0c-3c-1bcfd28ec610949254508df498f5-0003": 8731, + "0c-3c-500897c44dc62668270c44261872-0001": 8732, + "0c-3c-63f741668f4e837a5c445e473f04-0001": 8733, + "0c-3c-645294d044e93b39a9e779b74abb-0002": 8734, + "0c-3c-728be3ed4bd274be3c82298c7f52-0001": 8735, + "0c-3c-7e5778b3ea381b7fadefea5b770b-0001": 8736, + "0c-3c-8461a34694877461931bc2464714-0003": 8737, + "0c-3c-85ec4fdcdae4c12a4a666532311d-0001": 8738, + "0c-3c-904280f520bf129d08dc6a0e745b-0001": 8739, + "0c-3c-9e713ba1e555d953dab80cc7fe36-0001": 8740, + "0c-3c-a3a38ef5aee6d4636a15840323ae-0001": 8741, + "0c-3c-acba5959360b94c16ccffae175f2-0001": 8742, + "0c-3c-bc3ad9d825be5ac75b8195864802-0002": 8743, + "0c-3c-c319a1ab52aef7be8a2709aa422d-0003": 8744, + "0c-3c-d2ac7292c8236416a8b961c1bb87-0001": 8745, + "0c-3c-e57939660cc456c89a7ca5a10b9e-0001": 8746, + "0c-3c-e77b1de3472f837db0761366ac84-0001": 8747, + "0c-3d-0fffa9f590b80f0b0910d39b9296-0001": 8748, + "0c-3d-1e4e72a2295d42b1a00b303144e9-0001": 8749, + "0c-3d-435b047dcc7389fab55e3fb34097-0002": 8750, + "0c-3d-63ad2bb47336d6c5ffec73867186-0001": 8751, + "0c-3d-658a7d72a59f8779ffb3ef4994eb-0002": 8752, + "0c-3d-6894753ff0405ff3f1dca20228c9-0002": 8753, + "0c-3d-789833da11d9152999d621c89c8f-0002": 8754, + "0c-3d-7d25b2c96ca70a503ea184f599e7-0001": 8755, + "0c-3d-899de101c835dc84b9b4321e7098-0001": 8756, + "0c-3d-9a614b9843c5c28cf9a7ee61d552-0001": 8757, + "0c-3d-9ca0106439fa8d2411fe3335b9a9-0001": 8758, + "0c-3d-b7a850c61d6fcde6c642658a8c85-0001": 8759, + "0c-3d-bccc792f015e6aac3ea7e5270f65-0002": 8760, + "0c-3d-c7c55242a9806b3a12ab08a20616-0001": 8761, + "0c-3d-cb8cdc5590a3278eb285fd27cf3b-0004": 8762, + "0c-3d-cf9003f6a01cc4e2946f1fc63daa-0001": 8763, + "0c-3d-d1b5125df176a74f0a055374d2be-0001": 8764, + "0c-3d-d78f4eb399c43a856ce2059779d4-0002": 8765, + "0c-3d-e17319c0eecc8ea53296b20f3ed9-0001": 8766, + "0c-3d-e80e22a29f9da5a06425498901b5-0001": 8767, + "0c-3d-eb6de116f4aa79b24f0e142f69c3-0004": 8768, + "0c-3e-01abea1e85161e7c9acada0e6529-0001": 8769, + "0c-3e-0fd4f29fb1bcf49d9bf6fb27d06f-0001": 8770, + "0c-3e-13144454a0e1c6049bcde2e8d753-0001": 8771, + "0c-3e-137752d550565f28e5d558ab86b8-0003": 8772, + "0c-3e-1f784b5de249dbe528959e2b0b97-0001": 8773, + "0c-3e-2b4e3767d0042f6730c0e39be54b-0002": 8774, + "0c-3e-335ccab57a91000ea03f034e8ea0-0001": 8775, + "0c-3e-54158c77c1f60a44574c58ccabb3-0001": 8776, + "0c-3e-5a60f94591a0decf9127c122ee05-0001": 8777, + "0c-3e-62f79facb75a5dc486d429fd1823-0002": 8778, + "0c-3e-91176ba89e9202c0b7d13c3742bd-0001": 8779, + "0c-3e-be03ece9b643633342cf34f86a12-0003": 8780, + "0c-3e-d8804db926914b6b18ae722cf370-0002": 8781, + "0c-3e-ea72e1224030ba427e5c6a729e22-0002": 8782, + "0c-3e-f3db8d557dc1aeb455f6c86b9957-0001": 8783, + "0c-3f-04991bcd5bbece1c533049ea1694-0001": 8784, + "0c-3f-0d34d0295e4787e87b95fa0e11b8-0003": 8785, + "0c-3f-1b97806f20c73de48ece2dcb1742-0001": 8786, + "0c-3f-1f71f721e77ca9e5b1548d38087c-0001": 8787, + "0c-3f-3867c968affa6714c27e4e7e90e4-0001": 8788, + "0c-3f-3ac832743874294f37dd285661f7-0001": 8789, + "0c-3f-3d3de77cfbcd1ca84c3b9d45085a-0002": 8790, + "0c-3f-778f0ad1d1bfb4e1b21a0d703e6b-0003": 8791, + "0c-3f-8c0e6b12e33eff1ed76635b6dfe0-0001": 8792, + "0c-3f-bf31c72fb54d90400e972b9b2a3a-0001": 8793, + "0c-3f-c575e2f514900aad13c1d1ce83b0-0001": 8794, + "0c-3f-c6796862d02521dfca19d8708d64-0001": 8795, + "0c-3f-d14e888419bdef1e23f8a7ef8e1e-0001": 8796, + "0c-3f-df00eff92e262551b0f1bad83157-0003": 8797, + "0c-3f-e151e518d8a2aa191a26a7163581-0001": 8798, + "0c-3f-e1da71d3845091386eb88164a4b1-0001": 8799, + "0c-40-0062693c54638f6bdb89ec31651b-0001": 8800, + "0c-40-04bab591fc1d21298734dabef446-0001": 8801, + "0c-40-10a6d0aaf499eb5d3720dc12bec0-0001": 8802, + "0c-40-1f66ba80e3bbc4495ebc05759d83-0002": 8803, + "0c-40-2fe406d9b32643b374ddd8163cb0-0001": 8804, + "0c-40-3c446398f4a533403f0aa3ebb291-0001": 8805, + "0c-40-4fde731ac8114134277c73cfedaf-0001": 8806, + "0c-40-682cf84a1b6bb028e53c5554294a-0003": 8807, + "0c-40-6a916b5585581cc6fcc5394918ab-0001": 8808, + "0c-40-7c377eacce353b10edcd1007281a-0002": 8809, + "0c-40-83d200bb2912820ec06b204cc96c-0003": 8810, + "0c-40-83f199d1ce3ddf1a5ad6874b6be9-0001": 8811, + "0c-40-919f10e69cd76b0ef0ef9979e1e0-0001": 8812, + "0c-40-923a8ce71c555052622e547164de-0001": 8813, + "0c-40-9c2bdb6addb0e0d6a728074b1482-0001": 8814, + "0c-40-b54c615456d69df9ad906f483be6-0002": 8815, + "0c-40-c3a299ec53723775e27f2388c87c-0001": 8816, + "0c-40-dba96891d8f81f1e906fa957aeea-0004": 8817, + "0c-40-dfbe055e3a2b426692b6eab218ec-0001": 8818, + "0c-40-e0a46bf0703c809d7b23a67a12c0-0002": 8819, + "0c-40-e1c8e635154dd3eeada1ff57e87c-0002": 8820, + "0c-40-f67c57f242163ecdcd5f4835855c-0003": 8821, + "0c-42-063d75eb930c0885822f2cacfa1c-0001": 8822, + "0c-42-1050615acc89a2758e9ecbc4b48b-0001": 8823, + "0c-42-17ab23d277b01d60871487effc47-0001": 8824, + "0c-42-2e2f356900770d4c2d068c73938b-0001": 8825, + "0c-42-48115182948ca4f34f7699d614cb-0001": 8826, + "0c-42-62f3dbda0a2514b56d80957e48e3-0003": 8827, + "0c-42-700221fbc792088efad9a1d7dde6-0001": 8828, + "0c-42-7c51c8717d0b1d588a4b20516580-0001": 8829, + "0c-42-9095b6ddd15f75e5f19f74ef6dac-0001": 8830, + "0c-42-9e426f7e61de24f70547fabb469c-0001": 8831, + "0c-42-a081505d8fd073686e348aafe174-0001": 8832, + "0c-42-a34f7359889854d45da6946d5a54-0001": 8833, + "0c-42-b14d3e23f4b8c5bdb73abd03335e-0001": 8834, + "0c-42-fa9c3531acecc9b5192ee6925763-0002": 8835, + "0c-42-ff8c73b89d4eb878dfe46096b3b4-0001": 8836, + "0c-44-00a7ab478d004adbd81f274cc345-0001": 8837, + "0c-44-1b6fd36f8393b1dadd21d11a7d53-0001": 8838, + "0c-44-1fda29addadc5b453d4085bea247-0002": 8839, + "0c-44-42ee4a6f7680c2f383aef8ff8f2e-0001": 8840, + "0c-44-451c1ccd412a2ff389353f725cf1-0001": 8841, + "0c-44-7c7fe0d646b8b954e767cc03a206-0001": 8842, + "0c-44-8b518774eec96c709fa792ba6247-0001": 8843, + "0c-44-991ff8765907ab36128544368491-0001": 8844, + "0c-44-9d9335036bfc8419465676b1abaf-0001": 8845, + "0c-44-bafa8caecc7b709e5898d9d51bc7-0001": 8846, + "0c-44-c7db0a6466279395b77aad5b814b-0001": 8847, + "0c-44-d078725ce9a1d5b2fec272e20306-0001": 8848, + "0c-44-dd5c8b4fe95bb6bd90d8f949cb2b-0001": 8849, + "0c-44-defd24cc85d9468c10fa199904a6-0001": 8850, + "0c-44-f1e0a1cc46b7ba6c1ab1b6a293e4-0004": 8851, + "0c-44-facc231cad5d021d5e10d24fed33-0003": 8852, + "0c-49-0a39933515cdcb3589c63d42148e-0001": 8853, + "0c-49-20a95873602ec2aaad2326792fd1-0001": 8854, + "0c-49-27c7c6a8fa99279637b5a35fb2b9-0001": 8855, + "0c-49-2d0d55dc676f7c403f3bdfd281c6-0002": 8856, + "0c-49-47ead36ab6604632333296af0270-0002": 8857, + "0c-49-5eaece245eacec7598b10217fa7c-0001": 8858, + "0c-49-6d8c82e1b5db6f2396191150c936-0001": 8859, + "0c-49-7068c0627a0caa567d99a740429f-0001": 8860, + "0c-49-71203ddaa857653ff0665fa21c23-0002": 8861, + "0c-49-717216f134419a5f2e6b715b07d2-0002": 8862, + "0c-49-91f96d5341f5cc6fe9f077d21390-0001": 8863, + "0c-49-9564cff49e8c2c0673325a28fc50-0001": 8864, + "0c-49-9da3896773555af9a6ffcf521263-0001": 8865, + "0c-49-b6b3143cce5f5b4a846bf169dd23-0001": 8866, + "0c-49-c14762c0763fb1d1ce08cba6587f-0003": 8867, + "0c-49-cd18e574b4124d881ee95aaf91ea-0001": 8868, + "0c-49-db7cf9e6af1f96089aded573b1a7-0001": 8869, + "0c-49-e024fab572c4728798e8152c3a34-0003": 8870, + "0c-49-e05442b60ab3567259a5dc76f64f-0001": 8871, + "0c-49-e3905fef6fae95c2094f96dde611-0001": 8872, + "0c-4a-0640e9937a34850d5b4a52976c75-0002": 8873, + "0c-4a-3053a4705ae5acc4d1eb57a4466c-0002": 8874, + "0c-4a-3b1827d5ce7aee71e318328045c3-0001": 8875, + "0c-4a-69ae86705ddd627070003a42b7fa-0001": 8876, + "0c-4a-792ef0bdf00dfe4e23fe93002111-0001": 8877, + "0c-4a-98a0945d3fbdb95448d79ea302ec-0001": 8878, + "0c-4a-a907bb86bb82b5357879aedd800d-0004": 8879, + "0c-4a-af4fce9e1d6b95cd5dbf9ac33733-0001": 8880, + "0c-4a-d3b239b58d9e9d9aba587f298286-0001": 8881, + "0c-4a-dbfab1b53f650f02eae2edf3405b-0001": 8882, + "0c-4a-e734bf1799e15006dc767589dfbd-0001": 8883, + "0c-4a-fe8c3d0633d2b6458144ceaa6c0f-0002": 8884, + "0c-4b-18fc465c742c57e9891c7d756c19-0003": 8885, + "0c-4b-1a4167ddecfa32821a43d5ca4ce3-0001": 8886, + "0c-4b-37455c7783f3e1cbcdb2364265fa-0001": 8887, + "0c-4b-3a7956695431c1f4f4f6b1ba5c94-0003": 8888, + "0c-4b-3be6254f8e7c124d0d48c920bf6a-0003": 8889, + "0c-4b-5517eccff1152bd6aa74527d9295-0002": 8890, + "0c-4b-57f6865d7a31065798ea4dfc750c-0003": 8891, + "0c-4b-738b8cccc38dc9892994a211d9ce-0001": 8892, + "0c-4b-7c0afc9a6a3b96dd249cdf01a9d0-0001": 8893, + "0c-4b-8999885a756ea36b2321b6158aa8-0001": 8894, + "0c-4b-9292079c60e577fff3ed7d21c27b-0002": 8895, + "0c-4b-968086dca0fc8180b822e0409abd-0003": 8896, + "0c-4b-c099644633091824eb223b8cfeb9-0001": 8897, + "0c-4b-ccf0e30bb31259c7a28654ed0908-0001": 8898, + "0c-4b-ff1d12015fb984954eccb02397f1-0002": 8899, + "0c-4c-12f2b957fa9d743eb7b781402316-0001": 8900, + "0c-4c-14d0fe80ecf1ff8d0c4dd577cc11-0002": 8901, + "0c-4c-2b7070ccb4a3c8009f6ec48bc46c-0001": 8902, + "0c-4c-2cffab0e47d0a6ca4518d169cc08-0001": 8903, + "0c-4c-35759887810a0fbe0968f71b5c0e-0002": 8904, + "0c-4c-39dd2b987a5f79051b184b869260-0003": 8905, + "0c-4c-3fd3c1ef3d189fe60f374b3c6e7a-0002": 8906, + "0c-4c-4ba4663edda87ac90b163a8908ba-0002": 8907, + "0c-4c-540adb207222ea5998c8b1e618bb-0002": 8908, + "0c-4c-62e77b9d62b2b51b4a0c17f8e72f-0001": 8909, + "0c-4c-72556bf51dad0fd0aafc1cdd927d-0001": 8910, + "0c-4c-7b63c5d53be3da292013b4b06625-0001": 8911, + "0c-4c-8d5f20d6a4b64f408e687a1cda1b-0001": 8912, + "0c-4c-8fab5bfd13f890d0fb3d9c96a5c0-0001": 8913, + "0c-4c-9d287a482b8deaf7cd6b39e82c2b-0001": 8914, + "0c-4c-a90569cd37960f2a5353b5ee160a-0001": 8915, + "0c-4c-a962df7c427ab8f3c0608540e60a-0001": 8916, + "0c-4c-b82b3a63998f7710e0a8b86f7e32-0001": 8917, + "0c-4c-bf69885fce54d40bac421e8e76e8-0001": 8918, + "0c-4c-c4c2cd562262ee34d40de86917ba-0001": 8919, + "0c-4c-c93231ea3fc8569d4a3231c94e7b-0003": 8920, + "0c-4c-d9778aca199d94603ff48d205f5a-0001": 8921, + "0c-4c-ec670ac272ab65cd45fc1e3cc62d-0001": 8922, + "0c-4c-ee8313297520a272036d68c51730-0001": 8923, + "0c-4c-f27360dfe11537f6291606f331c4-0003": 8924, + "0c-4c-f3f15d4f660ac08d1afe347bd631-0001": 8925, + "0c-4c-faef5bf8204b5aaa73ad84370f55-0002": 8926, + "0c-50-0331c72845141c63d99f126064c6-0003": 8927, + "0c-50-1001f906afffbc0f15612d209fee-0001": 8928, + "0c-50-1155474bfb1ebccaadbc1a530046-0002": 8929, + "0c-50-181bd95978a83cbcbe7600ebf463-0002": 8930, + "0c-50-212cf14a192225f0d0dea778bf0b-0001": 8931, + "0c-50-225e1360e3c3b04a6e2eef9044f8-0001": 8932, + "0c-50-2415cd9fd952c786ca672a9de652-0002": 8933, + "0c-50-263877b1544073dae37b490de797-0001": 8934, + "0c-50-36a5ee25fe7c084bf080590de30c-0001": 8935, + "0c-50-53156cd822122dead2813181d892-0002": 8936, + "0c-50-57f0dc9d64832d8e41b94fb057de-0001": 8937, + "0c-50-8486234251577c6c5d34bae2b401-0001": 8938, + "0c-50-9adda8f0c8e85c5ff2425706003d-0002": 8939, + "0c-50-ad310f3ed5cd45e29e284d2c41ae-0001": 8940, + "0c-50-afbf0a794957a894454647708844-0004": 8941, + "0c-50-b3d6e2e1bd7016ae3110acf58e31-0001": 8942, + "0c-50-b791c0cb01f8e4503630a9324bdb-0001": 8943, + "0c-50-c1cfbc56ad2f622acc02a275cc64-0002": 8944, + "0c-50-cebedc7ddaa1845661f412ffc63c-0002": 8945, + "0c-50-f2f4289d79f748eef7e5e1614ec7-0001": 8946, + "0c-50-f408622d98931ce5a4357faa0827-0001": 8947, + "0c-50-f6ec7e2864a58e808f7e70362727-0001": 8948, + "0c-52-078096a47ca5cae2ea93b31997c0-0001": 8949, + "0c-52-0c68e2ffeb998a34cb07957f0384-0001": 8950, + "0c-52-0ead48dc9b9f08a402569e5c7bd4-0001": 8951, + "0c-52-168fb967c15e9095b685736232b7-0002": 8952, + "0c-52-1c7b63af9977b4179f2176cde5de-0002": 8953, + "0c-52-3888f72c6372cceda12e98b71b31-0001": 8954, + "0c-52-5163b24fa7fedc1e7ee4ed85c39f-0003": 8955, + "0c-52-63f5eeb87f87e64b92d090949d58-0001": 8956, + "0c-52-6d5c76b32e337399fe225fdd76ce-0002": 8957, + "0c-52-8aa5e64dfff7ae25f5e47b465971-0002": 8958, + "0c-52-8e7d0454f174ac3b6cfdc714f0e1-0001": 8959, + "0c-52-958c3b21629aa44b3bca80c30e49-0001": 8960, + "0c-52-ad3e17b59fbf4fb63954ef47e75c-0001": 8961, + "0c-52-d262849d31d1f90483c3e34675b0-0002": 8962, + "0c-52-f89a8ac9dfa2af828fffa5a789ef-0001": 8963, + "0c-53-01c06869202d7957d2d2fa0b01f4-0001": 8964, + "0c-53-04f2ad0b668426a1f2bfaff37264-0002": 8965, + "0c-53-0e8e197f21c4c07bceae5a7bd2c0-0001": 8966, + "0c-53-306e7ffe0fd077d500abe28b0a90-0001": 8967, + "0c-53-37a1125af6dcf36b6475adac2481-0001": 8968, + "0c-53-3ed90b23ae200db1f571ea794108-0001": 8969, + "0c-53-524c765f429058fea170b2184938-0001": 8970, + "0c-53-806c7a00fa9e409cad2aea8880f4-0001": 8971, + "0c-53-966647f1b1cfc69253e4fe80f5c0-0002": 8972, + "0c-53-a5c378d0d64a46cb536a5a3b0e20-0001": 8973, + "0c-53-d609bc493363f00152205da4c949-0001": 8974, + "0c-53-eadbed72853e363317da80814fed-0001": 8975, + "0c-53-ef353509b9aff4ef9d06fa53462f-0001": 8976, + "0c-53-fcb740929bfb7609e224d838349c-0002": 8977, + "0c-54-0169ed749214b661ce7f3666f5db-0001": 8978, + "0c-54-05c01d8c9129885c11a4e940b788-0003": 8979, + "0c-54-0f970eff3efbd795b40363b3406f-0001": 8980, + "0c-54-194883674d19525ab46592feb6c2-0002": 8981, + "0c-54-3659c97848a54d37cc42c36a7149-0002": 8982, + "0c-54-383c74172bc76b96754268118ca6-0001": 8983, + "0c-54-3d5862d9914ecf172ccc071c5ef3-0001": 8984, + "0c-54-6438998d6e602e3dcc85585f9fb3-0002": 8985, + "0c-54-8ebe6d7e52e94d5f098849fe684a-0001": 8986, + "0c-54-9cb2d710f9544d81c02ca4e6a28f-0001": 8987, + "0c-54-b2daeed8f399b256f93be6928132-0001": 8988, + "0c-54-c7cc8d7a6c101a8d68ee470f060a-0002": 8989, + "0c-54-d51b37d04f89ffea74fc4e224003-0002": 8990, + "0c-55-000ce21134b0c9ef2617e364a797-0001": 8991, + "0c-55-086af2567fcf4e4ac3bc8d063372-0001": 8992, + "0c-55-117f0fa29bbd426dbaa156fb7eec-0003": 8993, + "0c-55-1697bae0455100405e0744e157cf-0001": 8994, + "0c-55-16bef5b07c711339d017f2e6e65a-0001": 8995, + "0c-55-1f8ac9d1b0d597830b80f0ca7ef2-0001": 8996, + "0c-55-2f17f4d805110204919eef91d48c-0002": 8997, + "0c-55-3d59789f1e394709f37369748406-0001": 8998, + "0c-55-3fedb77ac5d50dc7106af341747e-0001": 8999, + "0c-55-70efbbafd9010382b6d4bd5a10e4-0001": 9000, + "0c-55-7bb1297a2d1c7e424c3eb2315707-0001": 9001, + "0c-55-928868769b1cebf37db2b942e25a-0001": 9002, + "0c-55-b3540bfad23a62e8fd6749aaae48-0001": 9003, + "0c-55-e80af13c07d8641ec3214e739b05-0001": 9004, + "0c-55-fcd22768fe9ef5a715bec7d13f87-0001": 9005, + "0c-58-0e6037004a00600d908605fea14c-0001": 9006, + "0c-58-1c8139dba1fba11ad39fcb44e079-0001": 9007, + "0c-58-23c6f6e97019b743e6b0693f1ffa-0001": 9008, + "0c-58-292082357bf277f8e90024070933-0003": 9009, + "0c-58-5065ec2157f98c290f30d45e4eb2-0001": 9010, + "0c-58-577ad3489bfd67a822f1924493a1-0001": 9011, + "0c-58-c6e507ee6cb6990a23a55f61d8e7-0002": 9012, + "0c-58-d5fb6826473f43a3989fe6d978c6-0001": 9013, + "0c-58-d7006821fea12475e748c50e4575-0001": 9014, + "0c-58-d7cff233bd0ea98a15e924f4e150-0002": 9015, + "0c-58-fdee98222388f57e49793989e203-0001": 9016, + "0c-5c-03879a2ed5fbb46428758006eae5-0001": 9017, + "0c-5c-0bba6dd45b3c68f1c69903c48284-0001": 9018, + "0c-5c-12c4976f20d4f6b471f31f4f4149-0003": 9019, + "0c-5c-37de8a28f1d15fc84fb80cd99914-0002": 9020, + "0c-5c-38f4c5c82937184f778636d847b4-0001": 9021, + "0c-5c-3e87f25a8570151bdbfa9f7c8a70-0001": 9022, + "0c-5c-5ebc03200f6b117780b206ca375a-0002": 9023, + "0c-5c-664d19f2cd50af09a12e0b447ba9-0003": 9024, + "0c-5c-ab7d625d7b224082d754489c2346-0001": 9025, + "0c-5c-c3b173da19f3d5ab1c0026aff6bd-0002": 9026, + "0c-5d-02afd20876d21eb43ade75b57b2f-0002": 9027, + "0c-5d-0f1a6e5d77cfc3fb454747ee0b6f-0002": 9028, + "0c-5d-3320f0aacd90d61fd1aa69f92d7d-0001": 9029, + "0c-5d-33bfc7ce580f6c1f2e834799d868-0002": 9030, + "0c-5d-459817a125e4e194b0508165c5da-0001": 9031, + "0c-5d-6245133d0176af7a677014f4fd2d-0001": 9032, + "0c-5d-6cc3ac73f6fa1c1597bd3a1542dd-0004": 9033, + "0c-5d-766b42099d7fce181cb3e296da00-0003": 9034, + "0c-5d-7cec70c6ba15248b917761537948-0001": 9035, + "0c-5d-7d4cfd4490e73a672d495e35ceba-0003": 9036, + "0c-5d-7dfada49a64fb52f35247d50acfd-0002": 9037, + "0c-5d-8b8e78d7fdc84c07c631fa4292e7-0002": 9038, + "0c-5d-a43175257e212d1fc39e3c985bda-0002": 9039, + "0c-5d-c03501b93ae7e67bca68f0de1ccb-0001": 9040, + "0c-5d-c0370f76e44645a5f7d18ea6da71-0001": 9041, + "0c-5d-c0a95b7246f96b2201f2fe30f991-0002": 9042, + "0c-5d-c46c61ac47ed412ff73e79e40960-0001": 9043, + "0c-5d-d8c50111ac4c3901c9be833fccef-0001": 9044, + "0c-5d-d9294dfe0c48555347106ce7b453-0002": 9045, + "0c-5d-df139a7a7c3a2653d714b1e09a36-0001": 9046, + "0c-5d-e08b843953e0dc32c441b2af1bc3-0001": 9047, + "0c-5d-e9496d7513a84421cbdb6638ee2f-0001": 9048, + "0c-5d-ed8e49901e129a3d458f4755153d-0001": 9049, + "0c-5e-00a439369464184ab18cd5014ba0-0003": 9050, + "0c-5e-121c796ccc2f33fba98e9f7bf447-0001": 9051, + "0c-5e-1602259bcc64d6e0f655befcd10d-0001": 9052, + "0c-5e-198e54ad9bce4417ed8facb67c5a-0001": 9053, + "0c-5e-1b9580604f7408329fb65300e873-0001": 9054, + "0c-5e-2f78cb612ec0fddf4e0714767c2a-0001": 9055, + "0c-5e-64b00c8e1a0dd5ed2cc3036f61aa-0001": 9056, + "0c-5e-6644ef8003fa68c6d11e26520cf9-0002": 9057, + "0c-5e-a2c0db6a6a2b4e3011e92220b7cb-0001": 9058, + "0c-5e-a9bee89bbb325625790a3bcc13f1-0003": 9059, + "0c-5e-b14fb831655da2fce1ffcf20e6a3-0001": 9060, + "0c-5e-c77cb099f57f3bc60320671a4160-0002": 9061, + "0c-5e-cb0f92e435a1db1df8e4d1ed474a-0001": 9062, + "0c-5e-d222f59e4a243f2f0c71399b0bd8-0001": 9063, + "0c-5e-d632203b78261b3e90f58fe18691-0002": 9064, + "0c-5e-f79b47091883366f3d48e46c0cbc-0001": 9065, + "0c-5e-ff97dcf42f1a153ce8aa44c4aef3-0001": 9066, + "0c-5f-3f19e5ab900321314d861d9bc220-0001": 9067, + "0c-5f-4d68ec7904598c4489311e76e4bb-0001": 9068, + "0c-5f-51952306879884c3c4d340e94e85-0001": 9069, + "0c-5f-6cccada264ae1117d048d55422a0-0001": 9070, + "0c-5f-6ebeb3125e6ffa274d2100b4c68f-0001": 9071, + "0c-5f-7625d401bb626edf8377a580f833-0001": 9072, + "0c-5f-8dce8e401f6c75634895c8181464-0001": 9073, + "0c-5f-9016fa34e4f1198c4a82d88c9048-0002": 9074, + "0c-5f-979cecf53d825f31236ab7d234b2-0002": 9075, + "0c-5f-9e19fa72a2e35adc2dca30d279fd-0005": 9076, + "0c-5f-9efb363483cad660b0711e2bf0e3-0002": 9077, + "0c-5f-9fb92bc77b84b4a9276e612d7c49-0002": 9078, + "0c-5f-cea35fbd5900ad56fb52412bf531-0001": 9079, + "0c-5f-e0a28cc75da29bfa90916a345047-0001": 9080, + "0c-5f-f6b846766cc7d0ea52d5bde9fb39-0001": 9081, + "0c-5f-ff7ee03a45a99893d123873ccb61-0001": 9082, + "0c-61-0205c07f56f91c6e1a54b055fbf2-0001": 9083, + "0c-61-15932808353700cac21605cd0468-0001": 9084, + "0c-61-18e17cb7d2cc19572ef2e32387f4-0001": 9085, + "0c-61-2b20b19240cb3d28cc94f9fbe892-0001": 9086, + "0c-61-3054163e3bbea88d85f400c1a703-0003": 9087, + "0c-61-34a693f67e420339e6e0bfe840af-0002": 9088, + "0c-61-659b361888db56bb0dd714fcf3d5-0001": 9089, + "0c-61-7919d948b1ce61c5af8aab82b04a-0002": 9090, + "0c-61-812a5cd5416ab0e129711835849f-0001": 9091, + "0c-61-9a70ffd39eb8b5ddea8251ae64d2-0001": 9092, + "0c-61-a3a390e2bd1d5ec5e3cb7f728426-0002": 9093, + "0c-61-e482a70951b705518881386cf3a2-0001": 9094, + "0c-61-e5bb444656cfefa04778ee46fe41-0001": 9095, + "0c-61-e84427f1c412e184373b9070755c-0001": 9096, + "0c-63-0bfabd9c138173b65b02153de2fa-0001": 9097, + "0c-63-17ac82846556df636e1a6bc028aa-0001": 9098, + "0c-63-4bb5031be219cf035e1897840b4f-0001": 9099, + "0c-63-4cf5fb50695236fff746fafaa96e-0001": 9100, + "0c-63-510edeed50dc89ac197e4da39ae5-0001": 9101, + "0c-63-6b5af21f012db92f4188016577cf-0003": 9102, + "0c-63-6cfbd7d3479d66be520c3a99e087-0001": 9103, + "0c-63-7cc1ca9573520cecab22387dd8ec-0001": 9104, + "0c-63-80e7277170e5640dbea80d3249e1-0003": 9105, + "0c-63-852467ddf45818448c0ba251d51d-0001": 9106, + "0c-63-9445e53e4bd277539d03ae12e0d9-0002": 9107, + "0c-63-a335a7eb83608e2ba539239834e6-0003": 9108, + "0c-63-ad3152e2b65a6e3f088cc0edbef9-0001": 9109, + "0c-63-b53400dffa197dd5549689496841-0001": 9110, + "0c-63-bf955d9acdbfa41cf88e9ddae562-0002": 9111, + "0c-63-c5fe49cdd33adbb8320b1b5f9822-0002": 9112, + "0c-63-d5edae492f3eecc27758b8cb1167-0001": 9113, + "0c-63-d99b1a47dc62c978460bc9cc726d-0001": 9114, + "0c-63-e190740d2371b9ab8309c7508c7e-0002": 9115, + "0c-65-0257e8aca0aa9fcefa7b58e50ca3-0001": 9116, + "0c-65-157b3b29990236ff26b7dd3a6c75-0002": 9117, + "0c-65-1a5827037e64cc559f0cb526b5e0-0001": 9118, + "0c-65-241411098b3e31523921ac92c6f4-0002": 9119, + "0c-65-49c6792746faa7f124c9124b77b6-0004": 9120, + "0c-65-4f0328330bd2aead9a45becfc388-0001": 9121, + "0c-65-5aac55990106ed0cb4778a9edcaf-0002": 9122, + "0c-65-5c907352a07d0fbe24cff38775a9-0002": 9123, + "0c-65-6b407335736703882c935c7cb00e-0002": 9124, + "0c-65-7a10823fe5802659a7ac636728bb-0003": 9125, + "0c-65-7a4447fb7e21e8ff0245f527b282-0001": 9126, + "0c-65-7af8d2c5483601bad262cbb54db2-0003": 9127, + "0c-65-a8e8cfa2395a943121c77fd38194-0002": 9128, + "0c-65-c0b344af2f5f68bf0fc4a860e52f-0002": 9129, + "0c-65-c5cbabbf2a57337047e2e71c4133-0001": 9130, + "0c-65-df467ba365931ac60c8f4fca5399-0002": 9131, + "0c-65-e88e03ba843fa18d9d58395e56a2-0001": 9132, + "0c-65-fa57d078cd30d00b51bdfa0e22f2-0001": 9133, + "0c-66-0437d010ac2337c44b1f3ad3b85a-0003": 9134, + "0c-66-28d0f061324181744d080de178fe-0001": 9135, + "0c-66-3c9771b689aafecd5d432eba6c72-0002": 9136, + "0c-66-4736d6c13e43a66d35ab0be074d3-0001": 9137, + "0c-66-5250631f4d68b062b2ab9fc70f0f-0002": 9138, + "0c-66-5877068a8dc28546d5b0cbb35775-0001": 9139, + "0c-66-5fe6f1548ee4543ca8a3546889e6-0001": 9140, + "0c-66-686078bef2d575521f80e83d2eda-0001": 9141, + "0c-66-69aab0ccc9afe5a3cdc7472ca726-0001": 9142, + "0c-66-88f10f56b4bceb298fe738a601a9-0003": 9143, + "0c-66-8b9e3974a3ac1c34447268cf1cf1-0002": 9144, + "0c-66-933c36e3b28ee6fe15916f1119f1-0018": 9145, + "0c-66-a60942a535efefb1880327f72901-0001": 9146, + "0c-66-a8f71e4fea7f9f4836bd13769b4c-0001": 9147, + "0c-66-abf873033da10c053e5f173761bc-0001": 9148, + "0c-66-be4c9d8b815fdf2baaff21148057-0003": 9149, + "0c-66-c35e87fd00a8dee57fc52d6ac881-0002": 9150, + "0c-66-c5004ed0af8123d01133bc77f03f-0001": 9151, + "0c-66-d05fb48caab40dab49d77a2561ad-0001": 9152, + "0c-66-d48bb0f2157ddbbcc1782ee04d3b-0001": 9153, + "0c-66-ec52f589abeb10e92b6a2c85c682-0002": 9154, + "0c-66-fa1ecd372d763d9c09a3885afde5-0002": 9155, + "0c-66-fde4cc21e7a8c73f9b4f43c44d55-0001": 9156, + "0c-67-0780aeecb29c875ebdeb39d39107-0001": 9157, + "0c-67-28add00236890bb095a314f2b30d-0002": 9158, + "0c-67-37d0b23ebf8c5e7414f035aeec5e-0001": 9159, + "0c-67-44c6a8537b1aaffc6ad3a007d923-0001": 9160, + "0c-67-4f1f2ef88aa23fd6d84acf063c2b-0001": 9161, + "0c-67-59105215cb554a958ad2e58f971f-0001": 9162, + "0c-67-6211895d0c8f734bd2fbbe63df5a-0002": 9163, + "0c-67-6b3cf7185f79952757a684276d13-0001": 9164, + "0c-67-7bc57a4b762de72f743e53dbb1c6-0002": 9165, + "0c-67-833cda74727c2a3514f46fd58e84-0001": 9166, + "0c-67-86247cf20df973f44e3733f6a877-0001": 9167, + "0c-67-8ac7e77607b68b473023e6eb0217-0001": 9168, + "0c-67-9a5417ff4810fdda907da426097c-0003": 9169, + "0c-67-a36a4e3fd01b12c184da7ace5035-0003": 9170, + "0c-67-adc3ef34253b0e2f4d4a8cf2f2ee-0001": 9171, + "0c-67-bb49530166a184bca9339073dee4-0001": 9172, + "0c-67-c3d93112068af87a969219e0dedc-0002": 9173, + "0c-67-c905b0889c9ef534bdced725dddc-0002": 9174, + "0c-67-d98e5d5371bee5916f075262db1f-0001": 9175, + "0c-67-e40134d8b8998b8f54641db6f2a3-0001": 9176, + "0c-67-f8fc9f81eff64a76df686d93ca92-0001": 9177, + "0c-68-1f2ff53825dae2f5426b251ac16d-0002": 9178, + "0c-68-2be087c0dd23f566e9881544bf29-0001": 9179, + "0c-68-3fac8faad82f587df0df268726f4-0001": 9180, + "0c-68-61c0ed3357329db714b78bff608a-0001": 9181, + "0c-68-8326e3df26bb7767bad9d4dea474-0002": 9182, + "0c-68-8565633015e734055e0e8c79338a-0003": 9183, + "0c-68-8b88a8eb8e125102096ecfe9f0a6-0001": 9184, + "0c-68-8f3a6d994d2b8997d584305884ba-0001": 9185, + "0c-68-913bbd3e3bfaeff459f100fc586a-0001": 9186, + "0c-68-93cedc9a020b4004c0381a24a457-0001": 9187, + "0c-68-a504d15d345e501ab00dc2e39d82-0002": 9188, + "0c-68-b1bf9be70878f0a6feb7ec2018db-0001": 9189, + "0c-68-b3e2cf1fe55bfd637d15bd5b6049-0004": 9190, + "0c-68-c81c0d4364fa7b8cec0bc8881d79-0002": 9191, + "0c-68-ccce5088e7cc5554956bbcb5b5c2-0001": 9192, + "0c-68-dc0cde4f20a049666716593137d8-0001": 9193, + "0c-68-e417938fd515a08b671fa1882372-0003": 9194, + "0c-68-ed262c0f8fc8fdc2db3eeb2e04d7-0002": 9195, + "0c-68-f60af990af44107f98c38d40a188-0002": 9196, + "0c-68-fce254ad86ce93a88742087e26b6-0004": 9197, + "0c-69-076456dd0070acd2b330195f2f15-0002": 9198, + "0c-69-0b5561707cf34fe738a3e90cfbb1-0003": 9199, + "0c-69-0f81fcb1111642272f8f265632eb-0001": 9200, + "0c-69-1544e2c7e6569d9e710357c826c6-0001": 9201, + "0c-69-15851ae3a10f20c975c658f16742-0003": 9202, + "0c-69-24e9e43259f09d8d31f2ca8c86b0-0001": 9203, + "0c-69-25f8d4828baad7669d380198fee9-0002": 9204, + "0c-69-2d2b55a088996d066e053829a077-0001": 9205, + "0c-69-2d8d398eeacc37747e91ca4c81da-0001": 9206, + "0c-69-2d98fb38189a11e053cfb8401fd9-0001": 9207, + "0c-69-5101d4cfebd3d9de389b0d460ae8-0002": 9208, + "0c-69-691c4f221f5250bbdbea82a6a383-0001": 9209, + "0c-69-811b535c128c78fb3df510367c11-0001": 9210, + "0c-69-89abdb0ca58dbec1227f358c19ba-0002": 9211, + "0c-69-8dbf8bebe66814f0098980ccd00b-0011": 9212, + "0c-69-9a0bb0ecbf4f4144363c082099d7-0001": 9213, + "0c-69-beadaa23e1bd01c95693d1a59f8f-0001": 9214, + "0c-69-c79188a8e0677af61d8b5bae96af-0001": 9215, + "0c-69-cd008fcf3131b640a20d5f21eabd-0001": 9216, + "0c-69-d01eb640be6b631faf425eb94a52-0003": 9217, + "0c-69-da08d2053bf520fcea2006efceac-0001": 9218, + "0c-69-dc7064ec914072419ed59b4e1b17-0001": 9219, + "0c-6a-0f0e6bae2b82e8a42e0d510f65c0-0003": 9220, + "0c-6a-10ffcd8317a70d25d3d4427100b1-0001": 9221, + "0c-6a-2716590dd386852d1bef4e44aea6-0001": 9222, + "0c-6a-2966dc8733d5ef89f1edffa5cba9-0001": 9223, + "0c-6a-2b2ab1503f116cac4faf0da7e87c-0002": 9224, + "0c-6a-432d402ffbe7d514bd02fcd97e85-0001": 9225, + "0c-6a-46b00206c99d4d16888e2564f81b-0001": 9226, + "0c-6a-5287477d8a6dc32c05be955d015f-0002": 9227, + "0c-6a-8968eeb0b8ebebad24b4ccc8ba1b-0003": 9228, + "0c-6a-aeb5f60f37db2c64ef7ee4c35846-0001": 9229, + "0c-6a-c694d87ed0bd6988742e559295c8-0001": 9230, + "0c-6a-e4bb266a4f5862cacc3eabe19977-0001": 9231, + "0c-6a-e949492d06285771d6371b4b7229-0001": 9232, + "0c-6e-0c05cdf298c978162290532171cf-0001": 9233, + "0c-6e-229020d6c424b89bff8594a7a17a-0001": 9234, + "0c-6e-355c7623ea90d4c51dba73d0e938-0001": 9235, + "0c-6e-3d3bf0f6ba9530aa8292ca7a48a4-0001": 9236, + "0c-6e-447e2bc5a6fcda68eea94fc6f542-0001": 9237, + "0c-6e-44b27008570f0cb80788d760e59f-0002": 9238, + "0c-6e-4a538a01f4c1e91692a360d29e05-0001": 9239, + "0c-6e-63ead73d59ed43f00bcb945e0a61-0003": 9240, + "0c-6e-733bc1ccafd1f45020e0687689e2-0001": 9241, + "0c-6e-8172a74ff4bfa41901710f2b1e47-0004": 9242, + "0c-6e-8a2cc13adf3b6329687fdf5e38ef-0002": 9243, + "0c-6e-94ff2d8c02743be436dd939e0f2b-0001": 9244, + "0c-6e-9a288a3f2882772b0f16a103a7e3-0001": 9245, + "0c-6e-a6081665512b259bf341839b721b-0002": 9246, + "0c-6e-cf59eaaef9a894c0d3e4931571a2-0001": 9247, + "0c-6e-de807025cd8dd12329fae4ee33d6-0001": 9248, + "0c-6e-e0f1e20408fdd9fa1bd8ed15fca7-0002": 9249, + "0c-6e-f02a745695c5179e79d30f9e5f7d-0001": 9250, + "0c-70-00785d1dd1198e3fe795f6a558b4-0001": 9251, + "0c-70-026d4cb761b7835c961b9eb11576-0001": 9252, + "0c-70-03e6eb0f93e8dc87bafeb3d0b092-0001": 9253, + "0c-70-0e76573352c6d30e4b8fddb9403d-0002": 9254, + "0c-70-139da617a08780398e33abbd9ea1-0001": 9255, + "0c-70-18cc556f6fdf19a3f29ffbb6a98a-0001": 9256, + "0c-70-48b5f85ce8d75762a5fe525a3999-0001": 9257, + "0c-70-773e307d53b216475735a26e5b4a-0001": 9258, + "0c-70-7c0c0db1e153747c4a60c4a86a76-0001": 9259, + "0c-70-8d4efc6fe362ae08f82413c6ddd1-0002": 9260, + "0c-70-a2b86a5a1aac1a2a41a62d14e58d-0001": 9261, + "0c-70-bde536523d0c59ee7d8785d0d24a-0002": 9262, + "0c-70-bee63e1f2b52ee8575abd706f318-0001": 9263, + "0c-70-bfa5174c3a92e22a089eb84afa68-0001": 9264, + "0c-70-d966720c206ed7f6ada4eb8a8260-0001": 9265, + "0c-70-dfdfcc501126e40633d6ab3e5e0d-0001": 9266, + "0c-70-e3daf49ffc0237e3fd43e24590e5-0002": 9267, + "0c-70-e4184b26f9033c5fa5c7c6c7f91c-0002": 9268, + "0c-70-f2648b8f20ac522ddd9bfcbe3ad0-0001": 9269, + "0c-71-023d55c3fd37ecadd85bb40360df-0001": 9270, + "0c-71-03cfb4981da139f3bf76e2a8a4bc-0001": 9271, + "0c-71-1ec5aeceecad43238204f6790a66-0002": 9272, + "0c-71-2498c0c171b41d25831e2c8b6619-0001": 9273, + "0c-71-25bf6ddc12b13a3b629e935e93a2-0001": 9274, + "0c-71-26c8df74933248c7d1501f2a7f18-0001": 9275, + "0c-71-366a6ac786fc774558b6dab31ef6-0001": 9276, + "0c-71-5594e8c25e08494db39853d0aba9-0003": 9277, + "0c-71-6b0ab27e810b80f076037496da06-0001": 9278, + "0c-71-99f895ad720ca68bdb65dab6a609-0002": 9279, + "0c-71-aa775701cedb4ebf4306e0279deb-0004": 9280, + "0c-71-c8865d07aa0043b9f8194661819b-0001": 9281, + "0c-71-dc693fcefbdfff6b692c1b976888-0001": 9282, + "0c-71-f38e20305a7947359104e2c15716-0001": 9283, + "0c-74-106acfa21c9b3fb336a72bb0ae4e-0001": 9284, + "0c-74-1149245e980b68cfc869bab61d1a-0001": 9285, + "0c-74-1bb1d97f5afbd805503dd86f75c6-0001": 9286, + "0c-74-25eb9fc6d46ff7889c9d9b8a034c-0002": 9287, + "0c-74-39de08a17b39de166f074c02ce5b-0001": 9288, + "0c-74-452097938b2b8954ab31f6d5877b-0001": 9289, + "0c-74-6a4b2a2a33f7e1c252d3ae972ece-0001": 9290, + "0c-74-7fec836733de6fa22eeaeea1e9c9-0002": 9291, + "0c-74-82439095a1997fd8e9aff8acd0ac-0001": 9292, + "0c-74-8bb8b4afda55d89d2b37f40767a9-0001": 9293, + "0c-74-a6502933229df77e3d36719e5495-0001": 9294, + "0c-74-ead41fe3017ca4e9038e7e008edd-0001": 9295, + "0c-74-ec3ea1004a196a8724fd15233f66-0001": 9296, + "0c-74-efbb2d0a1178f5b6369388992649-0001": 9297, + "0c-74-fc07721d7409c56bb78ce9c08e5d-0001": 9298, + "0c-76-0218790bea103c15ced592e4af0a-0001": 9299, + "0c-76-094c75f1e8fe8aa5bc1b74baa851-0001": 9300, + "0c-76-1ff51e0438852f749a6a73d103e0-0001": 9301, + "0c-76-34a63d15ac028b04331f4d4a2d00-0001": 9302, + "0c-76-3d6438743515ec75eddb7452b5c6-0001": 9303, + "0c-76-3eac7b15cf3d9862961645cc32e4-0001": 9304, + "0c-76-4eb8e3f9b997a13169b998218af2-0002": 9305, + "0c-76-513d12cd4dbfb1fce9661a776cea-0002": 9306, + "0c-76-5a453df9e626c02ccbdc43032dd9-0003": 9307, + "0c-76-5af725243a3ab59120ae703b3399-0001": 9308, + "0c-76-72db7fadbfc42dd92a248317103f-0001": 9309, + "0c-76-87c7e54504f0f8dbd3955aa848f9-0002": 9310, + "0c-76-8e054e56450b413800ac6b02740b-0002": 9311, + "0c-76-9ffbd823bef3bed201604017216d-0001": 9312, + "0c-76-b982453d8f51cbf3fcbf1fdd7d6b-0001": 9313, + "0c-76-bce9663d161a6166e93103a9354f-0002": 9314, + "0c-76-cf23c8c0f553a4229c21263a4265-0003": 9315, + "0c-76-ec81b4a07e317c8911b4d62f5ff0-0001": 9316, + "0c-76-f6ae14d71294aa5aea9a48d92627-0010": 9317, + "0c-77-094d51b8f16988439de1d62b17f8-0004": 9318, + "0c-77-0f349ed584ee9d829d9965abe8fe-0003": 9319, + "0c-77-4e6f8bdfc4156847af9435cc3b27-0001": 9320, + "0c-77-51e81699f39bf8ecfc792e9c238a-0001": 9321, + "0c-77-67d3e31bf1cbc5932765b7529d7d-0001": 9322, + "0c-77-75752c7aaf5ebf4d29cc753324a4-0002": 9323, + "0c-77-759ed355ec64dd699bbd1bbc7ca9-0001": 9324, + "0c-77-8c3b810864449792594875422189-0001": 9325, + "0c-77-989ba3d466ee2426b62851760c70-0001": 9326, + "0c-77-ab5f0e1b89b2e64bf0772d0f4fd0-0001": 9327, + "0c-77-bf578aafbbd17d5e939436d5e54b-0001": 9328, + "0c-77-cf4e9c8a30df56d6065dcd78f2c4-0001": 9329, + "0c-77-e5316eb2257ad16ef47b71cd3f3e-0002": 9330, + "0c-77-e9d932ceb51dc8b713a42b5b6325-0001": 9331, + "0c-78-029f5fab1048ebcac5ead625c256-0001": 9332, + "0c-78-04d9d588c55175a8fee3e22dd774-0001": 9333, + "0c-78-0c6dafe5943644efa9ba4d7e4c73-0002": 9334, + "0c-78-1385afb28f742bb938d80b5d9f13-0002": 9335, + "0c-78-26e950370522e871b08d678864e4-0001": 9336, + "0c-78-2e2b4dc17b74559757b02c1d3348-0001": 9337, + "0c-78-539c95f95c0104b52cb0fdd02aee-0001": 9338, + "0c-78-7e664208735ddffc41d2f2a0e1c5-0001": 9339, + "0c-78-ad3362af715ee23482b328d16a76-0002": 9340, + "0c-78-b7aab53ee7061c04ff95ef54226f-0001": 9341, + "0c-78-d58fc5b9ea434b0197929d30153a-0002": 9342, + "0c-78-dbbbd2b48fb31d18f8dfc0dd67d8-0001": 9343, + "0c-78-e7daaa0a695b119b0dd56a389831-0001": 9344, + "0c-78-ee45f8b93edb638dfa44ce504ae9-0003": 9345, + "0c-78-f0a5a4de5fbabbc8c4633018754b-0001": 9346, + "0c-78-f73c9be9c97e1a48f000e14ed08e-0002": 9347, + "0c-79-089f6860f8c3cbc76709043c7e94-0001": 9348, + "0c-79-3c9d0bfacedad7376aff78ad2824-0001": 9349, + "0c-79-5434efc726e557cdcd91c98175eb-0001": 9350, + "0c-79-58414f9ceb82f79ab836732df87d-0001": 9351, + "0c-79-5d3c183715fca8b36664fcac3085-0001": 9352, + "0c-79-5e1609b5646b498346c1f5df5850-0002": 9353, + "0c-79-5f7966cb68ac8cec40883190e648-0001": 9354, + "0c-79-8703298c85372910ef2ae1032eff-0002": 9355, + "0c-79-aa5fa3f3ce8f47ed6c7058654ac8-0001": 9356, + "0c-79-b539ecb078f1323d80b3e009ebfd-0001": 9357, + "0c-79-b54b87a0729e8dac2480f94a54c6-0001": 9358, + "0c-79-b9c026f5a6903e1590025d5a54c7-0003": 9359, + "0c-79-be06c409365a80371cf31a51946e-0001": 9360, + "0c-79-d0ea51ed97fa66cf146d58c60e53-0001": 9361, + "0c-79-d6dc814085850e73680c958a0c6b-0002": 9362, + "0c-79-f1c5d6400ade7c2f3d859b5c7470-0001": 9363, + "0c-79-fb699593bda9b6b4ffd17e77e557-0001": 9364, + "0c-7a-0042ec13a6cdb629ee6453920bfe-0003": 9365, + "0c-7a-04407a917870cc67d3a91ce2957b-0001": 9366, + "0c-7a-0d53707978895f6b473b2ebdc929-0002": 9367, + "0c-7a-399f50589d98e5ed08a931a88fc8-0001": 9368, + "0c-7a-42c553bd965342b6eb628b3f796e-0001": 9369, + "0c-7a-55167e14013e06cc96a6462cd687-0003": 9370, + "0c-7a-896efc43da4fa93578312b4256db-0001": 9371, + "0c-7a-a3ba2cdfe4fa48cf8c6ba28d25db-0001": 9372, + "0c-7a-a5604b562c4c08fb49ccd4a75a10-0002": 9373, + "0c-7a-d22447bc146108795d1a73a93201-0002": 9374, + "0c-7a-ef27a00a475025d76feb12c5cbe7-0001": 9375, + "0c-7a-f8113b17f5a18105fd67f034ab50-0002": 9376, + "0c-7a-fc13e6d3f9bc08800e083f5b8998-0001": 9377, + "0c-7b-64218356292652388417ca688a99-0002": 9378, + "0c-7b-8f4c6779c3f9fbf8ca7fb7779e17-0002": 9379, + "0c-7b-8f5154a70c315eefae138f64d078-0001": 9380, + "0c-7b-c908528fd6a566aeec3664274f66-0001": 9381, + "0c-7b-e016afef8afda6a895b80cf4b6f6-0001": 9382, + "0c-7b-f2c38c70ae2b5ac9fcd0faae7868-0001": 9383, + "0c-7b-fb7786c85a2752e3aa3c849b4d9c-0001": 9384, + "0c-7d-16b0ff0024da3a78b4def515d582-0001": 9385, + "0c-7d-2f7d2a3cd64109f2d8ba345fee0a-0001": 9386, + "0c-7d-342a25a792c4f9bb645f6bebd782-0002": 9387, + "0c-7d-4b5e9d5e6488975dfddfe710bedd-0001": 9388, + "0c-7d-6f899315b337968776c5e5759a6b-0001": 9389, + "0c-7d-75e9d06c7630def116d30752ac93-0001": 9390, + "0c-7d-80cdfbf8be177b778587904ad5cc-0002": 9391, + "0c-7d-954a624fe8997b2490c2e93e7b11-0001": 9392, + "0c-7d-9fdbb488b49f93354b9bbe3e3cf2-0001": 9393, + "0c-7d-b2ebe2b732d9574bed860f3f3fb7-0002": 9394, + "0c-7d-bcaf5e5ebd14f2b84f616e30675c-0001": 9395, + "0c-7d-e58dd8c5e54187ce51a98d391859-0001": 9396, + "0c-7d-f1a2580523ea82910d8c4149f866-0002": 9397, + "0c-7d-f22c60e42075475f1d42e949898b-0001": 9398, + "0c-7e-2a28b2e2a03e32ae0087c9ef7774-0003": 9399, + "0c-7e-42943d5838ee978ee46b5e46cf8d-0002": 9400, + "0c-7e-53d40954918f8dc62b59fd7c18ca-0002": 9401, + "0c-7e-53d4e6a87183d8aa6ce8bc8feffd-0001": 9402, + "0c-7e-57a4a6e669fba76fc735a8003f9f-0002": 9403, + "0c-7e-6f79fdfd11c0919c978fac806762-0001": 9404, + "0c-7e-7a3423acac2e1aac5264e14ae914-0001": 9405, + "0c-7e-8477e396b32f73439f649fc9b260-0002": 9406, + "0c-7e-85d84a87e9a2a8ed0344694114a2-0001": 9407, + "0c-7e-85e416c49b555fd757d0c8cb4a88-0001": 9408, + "0c-7e-a62eb7c2eaedb90e4dffac6a7143-0003": 9409, + "0c-7e-b1abbeaa127c47cc13d2c5dbda30-0001": 9410, + "0c-7e-ba73fead3ced5355eb2081b7dd12-0003": 9411, + "0c-7e-c6e0c042d577d5b80706ec1e8e39-0001": 9412, + "0c-7e-d7bf1d672eb4fb478219109cf1fc-0003": 9413, + "0c-7e-dde84c02237e83d51c1879e2911f-0001": 9414, + "0c-7e-e2a68dd387664ec5ee91e6a2a8b8-0001": 9415, + "0c-7e-e9e8e9f1badd7faac542ce97b52f-0001": 9416, + "0c-7e-fa1a1da019834d97959938245a67-0002": 9417, + "0c-7f-07cfd409611c7d98f812f6230e22-0002": 9418, + "0c-7f-0ba985d4a96f0afd1c912144f306-0001": 9419, + "0c-7f-1f1463dfeada5508e79e23449ea1-0001": 9420, + "0c-7f-8530da958a7f5db2a9dcc71314e0-0001": 9421, + "0c-7f-8628898ff598d396cd65edaafd12-0002": 9422, + "0c-7f-90cfc1eb2f5f18e8a084955f539f-0003": 9423, + "0c-7f-a583e56522200c624f907fb8e82a-0001": 9424, + "0c-7f-b73a822dd4b33f31f5e729f3cb2c-0001": 9425, + "0c-7f-d031962b6ea64969d385f97d6a8d-0001": 9426, + "0c-80-0bedf73e98ce09e403663a1f6126-0001": 9427, + "0c-80-23970e79d5e8a9c18dca3788ad9f-0001": 9428, + "0c-80-24159ea1f7df01ee4054c51dbbeb-0002": 9429, + "0c-80-3a61312fdac997d6733ec82392f6-0004": 9430, + "0c-80-3f0d3330c694a780e044c0a8f25d-0002": 9431, + "0c-80-53d423a835976b563d879f5bac1d-0001": 9432, + "0c-80-59ba66acaabf05490945e19e5bc5-0001": 9433, + "0c-80-606f6483609a72b5ad329dbe57fa-0004": 9434, + "0c-80-6add7c50f3856f2f695815ba201e-0002": 9435, + "0c-80-81ce059b0d4b3079ff7d9a44d6bb-0001": 9436, + "0c-80-8acc3cabc7f286fcac2e210345d6-0001": 9437, + "0c-80-8de793fe6461de5a88c281eb3edd-0001": 9438, + "0c-80-9779fee625fe5cf6a13a9615f12f-0001": 9439, + "0c-80-a0cde90fd204fa476edcba14c3c7-0001": 9440, + "0c-80-a89d389b23287c72be36e0706200-0001": 9441, + "0c-80-e6464ca1912027a9326e16c674d0-0001": 9442, + "0c-81-0f07e234156d6b560f2bced64e0e-0001": 9443, + "0c-81-698f5b117c57051485c68bed69a4-0001": 9444, + "0c-81-7da9d50ca88ce967e64e80792019-0001": 9445, + "0c-81-d333d4d1e5a6b04683c6b2b40414-0001": 9446, + "0c-81-dbd223977aef6b42fd0e787dbad9-0002": 9447, + "0c-82-01e05e12f043ddc7ac5c34108b55-0001": 9448, + "0c-82-02ceef15dabb499fec87a0fbbb69-0001": 9449, + "0c-82-0840ea7a2bf9586c39d83a780b17-0001": 9450, + "0c-82-1f234f5fedb5cedcf50376d141d3-0002": 9451, + "0c-82-2e20de6b727e1b000f98524d6971-0003": 9452, + "0c-82-42e571869bd4bb4e4f1d9b683633-0001": 9453, + "0c-82-4300891bdfc3224f86b7c89e9913-0002": 9454, + "0c-82-5a25856422f8a68b00f24096b286-0001": 9455, + "0c-82-605c5fca91d8059bd890d764d847-0001": 9456, + "0c-82-6d16119cd1a60a2cf375a03644d3-0001": 9457, + "0c-82-8f9c1f23873404990fab8ae0dac9-0001": 9458, + "0c-82-c19515a0ac2e7fd046886b253859-0001": 9459, + "0c-82-db21d8a3bfbaeebcd1dcd6a0db93-0001": 9460, + "0c-82-dd3900aa0a4f422bb46521837b02-0001": 9461, + "0c-82-f8e97e8852cf620de2e965de9e96-0002": 9462, + "0c-83-0522bcec81fa5a991dca34ac5f72-0001": 9463, + "0c-83-0b60109b2a9c66c36a90451bbd17-0002": 9464, + "0c-83-12a6867977922a289f373fc0ed7b-0002": 9465, + "0c-83-1e8cd5303995ccdb29271d2f28b2-0002": 9466, + "0c-83-43110871e179523a9ae42f4cc1ea-0003": 9467, + "0c-83-6b3e3ed29617a99642728c98c429-0001": 9468, + "0c-83-78ed5b34c2c0db5fcbd3a8732359-0001": 9469, + "0c-83-a2daef74876cad666256cff5aab1-0002": 9470, + "0c-83-a8854ae452967fd53ff0b43b19d4-0002": 9471, + "0c-83-a8ad196126f728eddb73834bf9f1-0001": 9472, + "0c-83-bcad2bbafc900e31597e4fe0d86f-0001": 9473, + "0c-83-ca98da4ee07ec44cd301121cf874-0001": 9474, + "0c-83-d431d09d709e9dd7b39c2001eae4-0003": 9475, + "0c-83-ed093e7957de08df98ee452fd77d-0002": 9476, + "0c-83-f112fbbb2aa81b1f3589910cb87e-0001": 9477, + "0c-85-061b92db3c59bf4906d8e147bb4b-0002": 9478, + "0c-85-08fb78529f36e1b1ada8c4619f4f-0004": 9479, + "0c-85-349b0b7c65e16ef6a22d42f829fe-0001": 9480, + "0c-85-44c3c396ebbb803c32cbb8f999be-0001": 9481, + "0c-85-4a65849333521b596ea6223702da-0001": 9482, + "0c-85-5e064e0271bf8a126f5f5461941d-0001": 9483, + "0c-85-7b3205aa572c4eb90aa7636a58c2-0001": 9484, + "0c-85-895f7b7da97f71a2fe1cfe9f12bf-0002": 9485, + "0c-85-8ea04781600de74c7bb4f81a936c-0002": 9486, + "0c-85-bf1225a8ff3b3b2e98c965349582-0001": 9487, + "0c-85-c49b67d6839af785097abd640530-0004": 9488, + "0c-85-c92288acb5a5be7758e44d483bbd-0001": 9489, + "0c-85-c9ff567cec56d5638e62027caf34-0001": 9490, + "0c-85-dc65e6af8ac26252873a44ec9269-0001": 9491, + "0c-86-0b10f75f42f247cc68fe3c2cc421-0003": 9492, + "0c-86-0f203ebce3290fd9a89446c84998-0002": 9493, + "0c-86-262cff2a637e9e7c8da024b0fafe-0003": 9494, + "0c-86-2f13f3741a7e31d59c58624128fc-0001": 9495, + "0c-86-2fe1468ce114290f17fe198b9bb0-0002": 9496, + "0c-86-3761f4c84aae110ab28db6b4222d-0003": 9497, + "0c-86-3b4563296af563997a2a84e60ea0-0001": 9498, + "0c-86-52a55964779c1002cf4e84308fe0-0001": 9499, + "0c-86-5697a703a1492416b2f0bc2e8d20-0002": 9500, + "0c-86-5d720e5510de9499486f53f02d41-0001": 9501, + "0c-86-6bf4d31897a38bb1e27eecd2ff85-0001": 9502, + "0c-86-7fc923ec777a1b3fcf5cb71179b0-0001": 9503, + "0c-86-917470e4c30ca7396d0220ba96fa-0001": 9504, + "0c-86-91f101fbf04d8a13fba3efc575c7-0001": 9505, + "0c-86-b753551fc5afa3e6ee9b1bc79dec-0001": 9506, + "0c-86-f9954f08e62c28ee9ab3ddc0fb4e-0001": 9507, + "0c-87-1cd155feb92230b7b82f5a8e2e40-0001": 9508, + "0c-87-3de3881ddd0d54cfa590d230061f-0003": 9509, + "0c-87-3fd5dab4f42d93b1ddf6a37f68e5-0001": 9510, + "0c-87-447f485e4aae5afa5a5cfa746e8e-0001": 9511, + "0c-87-5809a7c7c9655444cd76ff043da2-0002": 9512, + "0c-87-5a3657229d60df03502f3223a095-0001": 9513, + "0c-87-68a9818418f19087d5bc779ae523-0003": 9514, + "0c-87-7c7350f0d2717303527b194d03e1-0001": 9515, + "0c-87-91020f2cb0652655ba166dbd0ca0-0002": 9516, + "0c-87-9e03cf654ee0dc7b05c8510e0723-0001": 9517, + "0c-87-af036bd32ae14b5e86515af6d73c-0002": 9518, + "0c-87-bd15989a9b533ffa69b4a28969b9-0001": 9519, + "0c-87-cb7f11a6baa0de8c133bac591c20-0001": 9520, + "0c-87-e6d9479936d1312d698c81948edf-0001": 9521, + "0c-87-ff03ca31d77f251ba3e24b43acb6-0001": 9522, + "0c-89-0b599ba57e74a98d88820356b35b-0002": 9523, + "0c-89-17fdeed6aa99bb751a844b4b127d-0001": 9524, + "0c-89-1b13e7eff42f86d0eaf676a18ee5-0001": 9525, + "0c-89-292ab604b150d439527204f445a8-0001": 9526, + "0c-89-33481c1772821277c5881489b878-0002": 9527, + "0c-89-477eef73bff304c7053fab0cdd58-0002": 9528, + "0c-89-4a0e0e1ecbc06895571e39d476da-0002": 9529, + "0c-89-4b73fb90e322495f125bdbeb3488-0001": 9530, + "0c-89-5530ee9a0e305d0d48f1bb6e94a2-0001": 9531, + "0c-89-65f5ec06021956a3e4cc71568688-0001": 9532, + "0c-89-70161c1f2d49115b5a8589433a63-0001": 9533, + "0c-89-79701481fdefcf89f027722aedf6-0001": 9534, + "0c-89-7993ab88cbaa754b699926085a11-0001": 9535, + "0c-89-86b8567272e96293cd60352ea70b-0001": 9536, + "0c-89-87568681898ab5f950b4c6297fa2-0001": 9537, + "0c-89-88adb2c881c74a6b4b15ce7ffbe8-0002": 9538, + "0c-89-8d539421d5d7f56a4c038fadca2a-0002": 9539, + "0c-89-8d7a96dcee03ff9531c089819519-0001": 9540, + "0c-89-9d8ad21d958cd0feaa433f2b4cee-0001": 9541, + "0c-89-9e0bd8978a1e7570bd16d6623af6-0001": 9542, + "0c-89-a09d8cfcaa2de1f13a1c8b2f86c4-0003": 9543, + "0c-89-a327cc83302b8f3fd638b1a3349a-0001": 9544, + "0c-89-c4aa6db085dc430a5cf81f25f4bc-0001": 9545, + "0c-89-cdeb6345eb91be89ded69618d4e5-0001": 9546, + "0c-89-d4915c309edfd77a78232e013c8f-0001": 9547, + "0c-89-e413e27b97c564e3503635d2e890-0003": 9548, + "0c-89-fc941433d4c8f2537fa7a54d86a0-0003": 9549, + "0c-8b-02ea5bb75a3834408e1a3df3e10d-0001": 9550, + "0c-8b-0d99e138a12e1c29baa12d1e99d2-0001": 9551, + "0c-8b-1a95c03493046819f2d7c4a98aed-0001": 9552, + "0c-8b-21bf4499fc5e3503f1c367038a85-0001": 9553, + "0c-8b-3d3ed04dbe5248ecb546f9a7679e-0001": 9554, + "0c-8b-7313ac6589bcbe281e6dae375ac7-0001": 9555, + "0c-8b-846a137f5b1f66a04b10ad046c2c-0002": 9556, + "0c-8b-90e125e2f62019d7a847679389f2-0002": 9557, + "0c-8b-91fd66b1fc7af5d2eed84b4fd87e-0004": 9558, + "0c-8b-9b912d719d1572c020341592a799-0002": 9559, + "0c-8b-a813903793b342bcc4ee44124720-0001": 9560, + "0c-8b-b229c15d5c2b0ca1ddaa200dc38e-0002": 9561, + "0c-8b-bf113eeca5dca37ed88667641126-0001": 9562, + "0c-8b-c23258934fd2efcf155dca8ffc7e-0003": 9563, + "0c-8b-d4f12b8e8d42bc600731b90d4457-0001": 9564, + "0c-8b-e98618a1c3b7ffc994e50b4b15f5-0002": 9565, + "0c-8b-ead0356caf80fe9f3687b2616fad-0002": 9566, + "0c-8b-fab8fbe83f38d2aeb5fea8e2bc08-0003": 9567, + "0c-8d-0d9fd531ed071501cc638b8e2d7b-0004": 9568, + "0c-8d-0f8017e24f804d75a5f11e396a76-0001": 9569, + "0c-8d-13d4200b92163c19b00d38daaeae-0001": 9570, + "0c-8d-1e29a22933b58bf12510401d8442-0002": 9571, + "0c-8d-2a848cfff655d7150797f7d50af8-0001": 9572, + "0c-8d-535034cff52313e4fa3dc85a7545-0001": 9573, + "0c-8d-58cc6bdd1c240b5b7f352118f866-0001": 9574, + "0c-8d-722ca5f5baf5bc72179f4c75281b-0001": 9575, + "0c-8d-99486584bd10015c1150d1d32e93-0001": 9576, + "0c-8d-9ccc7ec993bf640c9a74b34cd0bc-0001": 9577, + "0c-8d-a1fac19c31fb991fe6e3f286a741-0003": 9578, + "0c-8d-ac2abb258433bbd7586aea1f514d-0002": 9579, + "0c-8d-c6920572fcb7d2def115648805a7-0001": 9580, + "0c-8d-db25781a313476fe7e011a1268c3-0001": 9581, + "0c-8d-db3d3d8fc03d99e89d169d4ed379-0001": 9582, + "0c-8d-fa3c322c2e98cf501805b2f27aa8-0001": 9583, + "0c-8e-03140c279e2f8d3ac0a46e2423c8-0001": 9584, + "0c-8e-604d4c2a63513ab99a555a6f7046-0001": 9585, + "0c-8e-692305e469fc1b4127b3aee42a57-0001": 9586, + "0c-8e-942ef961b4da95dfc00e5a75ec47-0001": 9587, + "0c-8e-9ed3b702fd05db710521bfc39140-0001": 9588, + "0c-8e-a1050691f07dec39e833bf88a261-0001": 9589, + "0c-8e-a31602614d49be015f05ff1a2baa-0001": 9590, + "0c-8e-aa8d12bf6d0a33c24ab22fdd7903-0002": 9591, + "0c-8e-ab1e70eba5a3f5bbcfb3d7a1c650-0001": 9592, + "0c-8e-ad83c393e78b3bf8d3d0fe0ab2d9-0003": 9593, + "0c-8e-aeacb6b211961e38ee666c132cf6-0002": 9594, + "0c-8e-b9eee60a8b3b4ff0a0239904fbf7-0003": 9595, + "0c-8e-d49b1067482b6c1b5165a5e43966-0001": 9596, + "0c-8e-e8d34ec8db89c3167cc9d484d86f-0002": 9597, + "0c-8e-eacb6348a31c23ecec868c60ab44-0001": 9598, + "0c-90-2f17d6bd3a4e6ccc6ebc31f3d312-0002": 9599, + "0c-90-3c4b8daee7c484c2ff427e8a010e-0001": 9600, + "0c-90-41ad94bfc6ac541c5ef7ccc9d263-0001": 9601, + "0c-90-446196c03d80fd40365506d04790-0001": 9602, + "0c-90-54035003eefb9f77d6ac29dec670-0003": 9603, + "0c-90-772a8f7fb2471bd22cb74b65a3a9-0003": 9604, + "0c-90-8138d96dcfb877dfbbd1744e0e12-0002": 9605, + "0c-90-8a48d94d37d5cc44359d88ae3ad4-0002": 9606, + "0c-90-9001d37a16831776d095cde5732f-0002": 9607, + "0c-90-a8f5cd5b46f84ea6475c13d89fb8-0001": 9608, + "0c-90-cdad12ea18fbd710211e8764db9d-0003": 9609, + "0c-90-dae4566123269aabfbf44d48fdd5-0002": 9610, + "0c-90-ea2e6350fd079fc2e50a19920971-0001": 9611, + "0c-91-025c39e719f321b87f01f866ba44-0001": 9612, + "0c-91-02aab0476c28399ab4123e8b1e47-0004": 9613, + "0c-91-0cde5ae928d7eff221553a0c5924-0002": 9614, + "0c-91-1ae99a4a7c1e033af2e431ea4906-0002": 9615, + "0c-91-22b11a1f8e01de2263aaa3394676-0002": 9616, + "0c-91-5a51a5d5e636ace8e130d4db83ac-0002": 9617, + "0c-91-65941f81be9babeb5f77d9a889ff-0002": 9618, + "0c-91-814e47fa2a7738ed5ed28e7b2e01-0003": 9619, + "0c-91-8278c49e1845546bde721b76a7dd-0001": 9620, + "0c-91-921f1c6db094a3f48d4b218675fb-0001": 9621, + "0c-91-d80c820affebc96b80d1ad344044-0001": 9622, + "0c-92-1c634142a89145f4e0f6f2231323-0002": 9623, + "0c-92-2c2b453d6003d4ece06274554e64-0001": 9624, + "0c-92-2e71debf48e310e49febab5a9353-0001": 9625, + "0c-92-3e267aba940011fe4cbf97c975c4-0001": 9626, + "0c-92-46a60cae03f428c1a6d8e5c6eac3-0001": 9627, + "0c-92-49800afaef5fb3adff562383bafc-0001": 9628, + "0c-92-509c89ba3f073a82e1b845bf0e51-0002": 9629, + "0c-92-5381cf935311a790f5758aa9402c-0001": 9630, + "0c-92-93a53cd5c87c15142afc08b74d2d-0003": 9631, + "0c-92-c1c6770f1469d75a53e19489a15d-0001": 9632, + "0c-92-c5a5914892b6e02f44bff9e9b571-0001": 9633, + "0c-92-cad79b32bc5be75e1d478e4b2b69-0002": 9634, + "0c-92-cfe557aac4e45050c2e7b9077709-0001": 9635, + "0c-92-df4a3ab4f78ff0112ed796efc499-0001": 9636, + "0c-92-e67d953f4eeda6eb304ca2064336-0004": 9637, + "0c-92-edec67f6719bf2d30c2483d5a53c-0001": 9638, + "0c-93-169b7475e59fb9ae90c2857f00bd-0001": 9639, + "0c-93-36bb5b3fe16d54bb906243ad4fb5-0001": 9640, + "0c-93-517538fc63d2a5ef196e75ca6fa8-0001": 9641, + "0c-93-5d6f133bdd189280610cd1fe4b68-0002": 9642, + "0c-93-854b878032d7c2d5fe6f2ebe3820-0001": 9643, + "0c-93-8ad4a44cf8241be7d1bc9bb851c3-0001": 9644, + "0c-93-94972c207cb71efcd14d3b0a284b-0002": 9645, + "0c-93-b7e284ee9f4dd840376048501cce-0001": 9646, + "0c-93-bddad03e6563bf2365c7f0265b3b-0001": 9647, + "0c-93-c9054612577662c74a17a96f8857-0001": 9648, + "0c-93-e609de55f1195294d4983c733d74-0001": 9649, + "0c-93-ec46a650e33d541931cf90ca80a0-0001": 9650, + "0c-93-f335a6328f1af705255358eb59d4-0001": 9651, + "0c-93-f40726397d0edc57427cf7674271-0001": 9652, + "0c-94-1b25900424863400d001b704b6e1-0001": 9653, + "0c-94-4eb2616288f15c52326fccaf25cc-0002": 9654, + "0c-94-52415b3316df2c29192a1d771a66-0002": 9655, + "0c-94-654a440a907258eb42186851c4e9-0001": 9656, + "0c-94-738dfc3591709c91a8daacc70d3d-0002": 9657, + "0c-94-743b9266881b75e585e0277088c8-0002": 9658, + "0c-94-9440e8ddb12f5d1729e572ca3cca-0001": 9659, + "0c-94-9b57c517745d255b5d63bc5e0ad2-0001": 9660, + "0c-94-9cc0ca107dc53a3333b0ea7bf9d1-0002": 9661, + "0c-94-aed549128c1c94498b65e4358cde-0001": 9662, + "0c-94-af02b89668e87c2dc2422fd5ea27-0001": 9663, + "0c-94-c7854f082386a53fb42fea1bf516-0002": 9664, + "0c-94-da068fd2b8a1337b9a6dd4ba7fb6-0002": 9665, + "0c-94-ea456ba846f8ce41daf79cc05168-0001": 9666, + "0c-94-f6d8cf3eb0f267b5a8d59cbd674f-0002": 9667, + "0c-96-19b55202bb77c0f890c7ac59c3f2-0001": 9668, + "0c-96-1d3d219e40f501c615f353e17931-0001": 9669, + "0c-96-401951792c3dd2980ec23321b927-0001": 9670, + "0c-96-42980637231a3e12ce97e5fe510c-0001": 9671, + "0c-96-4d27cc41f8ac52e7b4d766278cee-0001": 9672, + "0c-96-5d0c5d9d917563e6fba55c50a658-0002": 9673, + "0c-96-670663b690c75ce80567a83e54a7-0001": 9674, + "0c-96-76d1b5b849eb82fa698394619350-0001": 9675, + "0c-96-78ddb1745edea92ac5a2972a0a44-0001": 9676, + "0c-96-7a5dc2876ca40a2f2134328a78d4-0001": 9677, + "0c-96-7d44d0d2a4c703029f107440d8fb-0001": 9678, + "0c-96-7fb3cb38020742770facc2a9908f-0001": 9679, + "0c-96-812421c1d0d14d1f56ade757fa7b-0001": 9680, + "0c-96-85825e002fb7df56ce1743f807ef-0001": 9681, + "0c-96-a893e2a8e42a77d855fd25c93b95-0002": 9682, + "0c-96-ad1152f51008d9ec34a0b612c8d3-0001": 9683, + "0c-96-ade5831ac81ac30b78fda2fd1188-0003": 9684, + "0c-96-c543c406b2c233a1e24d6a24a8b8-0001": 9685, + "0c-96-d62537f886bad6bbf1ef3ef017e3-0001": 9686, + "0c-96-d840ecbaf4ade120e375b4a61a6d-0001": 9687, + "0c-96-e5baa842e2502b544958f4c10ae4-0001": 9688, + "0c-97-280af7f7dc2cbff4f21769268409-0001": 9689, + "0c-97-30eb4aa346ab2960f97313f50036-0001": 9690, + "0c-97-311f43edb242495cb21d7de67518-0002": 9691, + "0c-97-38ad0ca98f7fe8bfd66a67b8e00e-0002": 9692, + "0c-97-42712ae2c5236c88cbbd9aaf12dd-0001": 9693, + "0c-97-445f691a953f13a4e47e21bdbc3f-0002": 9694, + "0c-97-47624ebd2ef0ca7b6317de474617-0001": 9695, + "0c-97-4abbceeb5527532c7a2134b67a90-0001": 9696, + "0c-97-59a165b315a1056d2b4e8b66e38a-0001": 9697, + "0c-97-6dd271df27d8e5c513e6cb81cceb-0002": 9698, + "0c-97-744f49c9a383267bcca0f022ea65-0001": 9699, + "0c-97-9e2ff7f1fe6792dec5f1c526eba6-0001": 9700, + "0c-97-ba901a9a90d19a0bcf6fe5cbd733-0002": 9701, + "0c-97-d1e6c10aeb9de7b4122d2fa3b09d-0001": 9702, + "0c-97-d924ba80bcd34b6dbea752eae698-0001": 9703, + "0c-97-ed05e3632761124a549375ca4925-0001": 9704, + "0c-97-ff4e2bcb1db62da3840598e8e6b6-0001": 9705, + "0c-99-0c1ffd7bffc7b358981ed079225a-0001": 9706, + "0c-99-26a20482168bb008f56ca078d219-0002": 9707, + "0c-99-45e49289bdf0939e9c539a3fba14-0002": 9708, + "0c-99-6849be69bc88e53432a06b485050-0001": 9709, + "0c-99-86ddf4b8c0bd5b8328c10e90f86e-0001": 9710, + "0c-99-933c48cefaa2e8fcc115c7064c55-0002": 9711, + "0c-99-c1531f7b3d73e5fa5d5bc0c5b629-0002": 9712, + "0c-99-c44f6ef71d02082df27b7353d680-0002": 9713, + "0c-99-c9a1790327140bb53c931876f1b6-0001": 9714, + "0c-99-e01194da4411cf46b0bd639c1b9a-0001": 9715, + "0c-99-e71db49d8a184cbfda82a5762c7c-0002": 9716, + "0c-99-f031c29a82b643e85160bf7577ba-0004": 9717, + "0c-99-f563ca40cc5a2718c99329153113-0003": 9718, + "0c-99-fe3a7023618ed2c4cffdb4fd2c57-0002": 9719, + "0c-9b-1abf04d5ae583db7325b9ec3f523-0001": 9720, + "0c-9b-27ec5056d38c4c16a8ce914c51ce-0002": 9721, + "0c-9b-3c30a5ffa7b7324de7148211b252-0001": 9722, + "0c-9b-3cb510c89aef065d399c4445db35-0002": 9723, + "0c-9b-4a397f6b9dc9453345c603e33a8e-0001": 9724, + "0c-9b-5b2eebbf9cb5800ca69574e13c2a-0001": 9725, + "0c-9b-5cd9ecd15cbea2d5c2077a72009e-0001": 9726, + "0c-9b-62336bac0ac32e583f24dd33e8dd-0001": 9727, + "0c-9b-786fbe9f8be63b13e848cd8c9d2d-0001": 9728, + "0c-9b-93c835115c46263a3ecc35ffa4ee-0001": 9729, + "0c-9b-a2b24888e4417ff2dd9d19cf6546-0002": 9730, + "0c-9b-ab6063d2cb12c48a4357c986daf5-0001": 9731, + "0c-9b-bad58bf4f05a5cebd6c5dee19f69-0001": 9732, + "0c-9b-be8931aa39a5f494b044181bdf4e-0001": 9733, + "0c-9b-c314baa6c9cfd5e05c01b3f4e37d-0001": 9734, + "0c-9b-c7efc622dd78a75273bdbfafc645-0001": 9735, + "0c-9b-cb4b7f063337f679a1362f65b641-0001": 9736, + "0c-9b-f084091a0977a665ac9463ee95c5-0001": 9737, + "0c-9b-fadbd8f9329eb20987c3f0d6ce00-0001": 9738, + "0c-9b-fd09b6c66e1f993ed0514781f82b-0001": 9739, + "0c-9d-058f4ae0358a1f494df63838298d-0001": 9740, + "0c-9d-0e9565ee00230291c386c91d09d1-0001": 9741, + "0c-9d-19781e2ca6fee42c134a87869bbf-0003": 9742, + "0c-9d-212c7aaa7f7027c22d726b08c364-0001": 9743, + "0c-9d-223cb7f24f1098592c53f75dcfa2-0001": 9744, + "0c-9d-295831b6e22fe6fe0ef55bd6b136-0001": 9745, + "0c-9d-2b286dd1be501238e0137ecb2ff5-0002": 9746, + "0c-9d-452ee57085da438869b18d3d2aa2-0001": 9747, + "0c-9d-5697b26fd2f8e0735973825bd558-0003": 9748, + "0c-9d-6da189b225bfa2e1ec376ff9aa90-0002": 9749, + "0c-9d-7d26b731c21fbde8cb2e90f2aa73-0001": 9750, + "0c-9d-a3ee6169b7d61f9a049b0553938a-0001": 9751, + "0c-9d-a60c195dcd02ee0b24ba17dd7002-0001": 9752, + "0c-9d-ba4cc7eb9262be1c0e35d56a8b87-0002": 9753, + "0c-9d-ccf8b79f851d92f2aaf107e58adb-0002": 9754, + "0c-9d-e18cdde957f512a3c6959643cc74-0001": 9755, + "0c-9d-f0f5598e8e5f5cf48c37ccd18f18-0001": 9756, + "0c-9f-01b86f43d569a3043a8a25fbd50a-0001": 9757, + "0c-9f-0d1001984b4c19de01b2a475dad8-0001": 9758, + "0c-9f-26e3a6ede5f14d5a539db2b30b12-0001": 9759, + "0c-9f-2f117edd50565381d9f6b5c7e23d-0001": 9760, + "0c-9f-3955e05711d7b51d79d826376029-0001": 9761, + "0c-9f-6b779ce4dae077dc6d9d22022425-0002": 9762, + "0c-9f-72b5d2bd504df882c5bcd9191d92-0001": 9763, + "0c-9f-777810228a44dbd7d6adf757d63a-0001": 9764, + "0c-9f-84d874c842aed938e8ffe6dd3c6e-0002": 9765, + "0c-9f-85ad42f0af66822a75aa476d95bf-0003": 9766, + "0c-9f-9bec135c217379322cfe145090d1-0002": 9767, + "0c-9f-b296f86e60d8b9ac7b58f38f3002-0001": 9768, + "0c-9f-bd55bba1b81262dbda13801ef8f3-0001": 9769, + "0c-9f-c37f250503a0a25e28559baaa465-0001": 9770, + "0c-9f-cbd3400b1db9edb9ddaf5e6d67ee-0001": 9771, + "0c-9f-cf1b65b8fb0d6e13b3623da8d518-0001": 9772, + "0c-9f-d5370d6f942cb80d285a7d4022f1-0002": 9773, + "0c-9f-e20fa35d764f8e60973b7585a854-0001": 9774, + "0c-9f-ec4d8c9a771288e77905a2a8ba9a-0002": 9775, + "0c-9f-fbd8766dea7b6a76bcb8f3a15bc3-0001": 9776, + "0c-a0-00c7cde271c8a901f9f3a9e0bbe4-0001": 9777, + "0c-a0-1bb7de50498397e296d5af8a0545-0001": 9778, + "0c-a0-1fce81777083496715830a334359-0001": 9779, + "0c-a0-20c213bfcc29e4d988d46f952c34-0001": 9780, + "0c-a0-25278959ff3007f1ed1625ba5980-0001": 9781, + "0c-a0-488faa42d2a24cae9b604d272c96-0003": 9782, + "0c-a0-4ac2b8df9340e2b447a2abc93f65-0003": 9783, + "0c-a0-5b6c622148cdd3e7c882926476ce-0001": 9784, + "0c-a0-643dd0574afc97937b93e4c2f0e3-0001": 9785, + "0c-a0-71b14ec706d63c7ca91ae3f18bca-0001": 9786, + "0c-a0-7b583838cad1ecda0857dd2349e1-0004": 9787, + "0c-a0-7fea2b0695f92e0b60c8152da2a2-0002": 9788, + "0c-a0-89d860f707e17fe899ef009d5c61-0003": 9789, + "0c-a0-949452bd96311063d53253fda7b3-0001": 9790, + "0c-a0-985279fbd3dd406e8acb84bb62e9-0001": 9791, + "0c-a0-9afb5c1dc6081f66f22280747e85-0001": 9792, + "0c-a0-9c8a5c82c8caf7c3a3b02fbfc595-0001": 9793, + "0c-a0-ab6ea1ae7e4e33da267409c7d451-0001": 9794, + "0c-a0-b650e7ca6fae1c90a91fd9950ced-0001": 9795, + "0c-a0-d4b3c9cd2bc825b59168186cbea1-0001": 9796, + "0c-a0-df23ad21e86a535a506452319b7c-0001": 9797, + "0c-a0-e14834eba5b679a9c979f196c8ac-0002": 9798, + "0c-a0-e5afc8239a670c01e29eac2d524e-0001": 9799, + "0c-a0-f619d034df99802dbdb6576714f8-0001": 9800, + "0c-a2-4121b288282f6a36f59af5c3e554-0002": 9801, + "0c-a2-4188efb858cf7f4dafe99144ea7c-0001": 9802, + "0c-a2-4513114cfe1cd64cf41e10dfef21-0001": 9803, + "0c-a2-6482bbca6a347e363c00e3fcdf7a-0001": 9804, + "0c-a2-a3a69d76a0954f8fa664fc759a95-0002": 9805, + "0c-a2-aeab30390ca85e17b869f008a420-0001": 9806, + "0c-a2-c82f729930cafcd03cf55e77782b-0003": 9807, + "0c-a2-ca8142def543d132d22713fe495d-0004": 9808, + "0c-a2-cb9485e113f760747a3037b97514-0004": 9809, + "0c-a2-d2cafff5e7ff57212e3b7208b587-0003": 9810, + "0c-a2-f33451543291ac3b4b64c6f1b566-0001": 9811, + "0c-a3-0c6f9421a884bdf0a372a6bfa011-0004": 9812, + "0c-a3-20bff1756a37d976bf4f80fae5b3-0002": 9813, + "0c-a3-32ed2571232ecaa1d5a8c3264459-0001": 9814, + "0c-a3-3dc7f7b8e8c31c95081a98bbd14f-0002": 9815, + "0c-a3-643537e338ea55f50b378c1dff93-0001": 9816, + "0c-a3-67c8120a3fc78e174fde61469c5a-0001": 9817, + "0c-a3-7b54b6792798ae00e14f9d509b03-0001": 9818, + "0c-a3-8672c7d8165966811fd1c3238a8d-0002": 9819, + "0c-a3-907ddfce43a52fb9ed7c5d083c9c-0002": 9820, + "0c-a3-b94b3473d6e96725d521b164d18a-0001": 9821, + "0c-a3-c55123dc1abd4749e5074b47b087-0001": 9822, + "0c-a3-d048c02e92344fe70d3ccab6f042-0002": 9823, + "0c-a3-e5b327dec818716591f88aaf866c-0002": 9824, + "0c-a3-ee3f8d79e343c9a85c24c1905f15-0003": 9825, + "0c-a3-efb8155ab8676748d3891e1960e4-0004": 9826, + "0c-a4-0224de171106a1457d56390dcac5-0002": 9827, + "0c-a4-292e0b3aff0d38e39c1e46f56254-0001": 9828, + "0c-a4-2ab270c7c40c636a40b7c9874e6d-0001": 9829, + "0c-a4-368e9c592d2d7fc4cf4c8e9a2fd2-0003": 9830, + "0c-a4-60f0ec8515f777155e55edd1cb73-0002": 9831, + "0c-a4-6af7bf3f7d9a2104c99cdcee6049-0002": 9832, + "0c-a4-6d8e40350f024cec36c49bcd86b9-0001": 9833, + "0c-a4-797d55b9e22cf3b6e68d1a0a4a5b-0001": 9834, + "0c-a4-ae696308a64f331f1f0af7394f1c-0002": 9835, + "0c-a4-bddc5d991e7d997fe17b1801ddb3-0001": 9836, + "0c-a4-c0f058562921aa11a5589990f59d-0002": 9837, + "0c-a4-e34e39b1bdfb439bfa63470c56b5-0001": 9838, + "0c-a4-ef77a6385fd16e898795c4e26714-0003": 9839, + "0c-a4-f26f8635434615c3b65895cddcd2-0002": 9840, + "0c-a5-10ccc3d055f50e1e7ed6d9e11d2b-0001": 9841, + "0c-a5-269ddf5a891270b2b441702d8b63-0002": 9842, + "0c-a5-3170a345a8b8c9a91348f9de64e0-0001": 9843, + "0c-a5-56fa73de95c38ca1db1df76e6a02-0001": 9844, + "0c-a5-6c465a3b157be3f51c21256224fa-0003": 9845, + "0c-a5-6fb0a26274f2299af3653f292b0b-0002": 9846, + "0c-a5-6fb0a26274f2299af3653f292b0b-0020": 9847, + "0c-a5-8c4b171daaa65bb230ccef2418ae-0001": 9848, + "0c-a5-a3525529accdcdd42c7ad46f97c5-0001": 9849, + "0c-a5-ad2bcbfed43be4e7f00fc89e9217-0001": 9850, + "0c-a5-bdadff40fc706163d38244299ad8-0001": 9851, + "0c-a5-d8e4228e32a09d7075f427172bbf-0002": 9852, + "0c-a5-de6779b479c187666a63565cbba3-0002": 9853, + "0c-a5-e498f99659a28c66150fca7744e9-0001": 9854, + "0c-a5-e56704bbd52fc447dc2e4b3bf6f7-0002": 9855, + "0c-a5-edf0b7fdf45ff1649a12c5b59c22-0001": 9856, + "0c-a6-2426a06e27f6ab7bd6833c354114-0002": 9857, + "0c-a6-2f7a5bfbd4b5523d05b30922f2cb-0001": 9858, + "0c-a6-37cbd8901ee2a6ea5c62ec3a5b16-0001": 9859, + "0c-a6-3a7dc523117b564261377f24e146-0001": 9860, + "0c-a6-4b9cf5b2a4155591631b1172187d-0002": 9861, + "0c-a6-5f47359b781b1f01b4757b0cf6c3-0001": 9862, + "0c-a6-64c767666ae53cc2bf0375bd0951-0002": 9863, + "0c-a6-7c42271323dcaf3f2f2b30f82e43-0001": 9864, + "0c-a6-9aca8fe6de90bbc6d22d3706a9dd-0001": 9865, + "0c-a6-9e592d604185318a8ae5af099792-0001": 9866, + "0c-a6-a18b58cf444e849486c29df6bc95-0002": 9867, + "0c-a6-a3a49e43d6d29c413f2912452251-0002": 9868, + "0c-a6-a8c91435ad1efa30fe97b01f173a-0003": 9869, + "0c-a6-ae0bce1ec1299b76417d959ccaf7-0001": 9870, + "0c-a6-b8b016dcf75adbf6dc9705e1dec5-0001": 9871, + "0c-a6-d5c96116b800cfc9c3cc34b525e3-0001": 9872, + "0c-a6-f3c9b32a28bb59527cc86af4b976-0001": 9873, + "0c-a7-0f48f7fe7a804f162a3bc3f10610-0002": 9874, + "0c-a7-1ac470727e9cf1939ea395c845fc-0001": 9875, + "0c-a7-3a95ee54c5dab9d5f04738e741e3-0001": 9876, + "0c-a7-5bd9e66e1892904bef88a49dd188-0001": 9877, + "0c-a7-85309880f192ce402734dc79f0fc-0001": 9878, + "0c-a7-8e4294cd286067556064f9195f3f-0001": 9879, + "0c-a7-9c70e4cd7c6d9d9dcbfc4472a848-0002": 9880, + "0c-a7-b450a31b4153a2c12c3452f14bc6-0004": 9881, + "0c-a7-bc7a0ff8c365ac92d4448dbd6b2c-0001": 9882, + "0c-a7-c697d9ce6176da17f34375da7fb1-0001": 9883, + "0c-a7-eac1b01708ea7682358465f397ea-0001": 9884, + "0c-a9-007506173337ac76586eaebf42fd-0001": 9885, + "0c-a9-1b220e4b7dd2af821bd8b6125d8a-0001": 9886, + "0c-a9-33547b556dc043ad255aaf3153d6-0001": 9887, + "0c-a9-3bc2eb6c12bfc02bb67434bbc674-0001": 9888, + "0c-a9-63deb3fafaff08801dbb7130af20-0001": 9889, + "0c-a9-6f5ecf3dce58e2b647cf2333340b-0001": 9890, + "0c-a9-8f274992a59ee27d43edfd79ba08-0001": 9891, + "0c-a9-a0cba214855ce140e2293b982a59-0001": 9892, + "0c-a9-b403ee0cc23b1ea5d97a43cf1e70-0001": 9893, + "0c-a9-d35528d9e47301f2ea3920dbc734-0002": 9894, + "0c-a9-d42a365d36880b1ca133aa3a731b-0001": 9895, + "0c-a9-e5c4166cc6901c2dde13f4146bcc-0001": 9896, + "0c-a9-e96d7ed3ea409f590f5a42e44680-0001": 9897, + "0c-a9-f2deed19ee2ace773a52222c1873-0001": 9898, + "0c-aa-0a38cd9a9e19e25dc4b677c89236-0001": 9899, + "0c-aa-0d88918edc0459e7390493f83a3d-0001": 9900, + "0c-aa-1414fbae3a5d5fa035b2aa2cd329-0002": 9901, + "0c-aa-153a560ea50b644af0b700f1a884-0001": 9902, + "0c-aa-1996b5dc7a2fe3182c031ad3378d-0003": 9903, + "0c-aa-1b365bece08b6190f01ad097a2eb-0001": 9904, + "0c-aa-1b420511f981322c2c6453a68a40-0001": 9905, + "0c-aa-4733adffaf7685375ef6aa289ecd-0001": 9906, + "0c-aa-4fa894efc3aa99726749b649a08a-0002": 9907, + "0c-aa-545f6f06fc898f0971cbf132fb58-0002": 9908, + "0c-aa-572299e3469155cfcc5ddfb774df-0001": 9909, + "0c-aa-5981a808a72edf6586f6bca1240e-0002": 9910, + "0c-aa-61cd4dc982e6447312cb91173fa8-0001": 9911, + "0c-aa-62c486f7eee0f9efae37b1c4ff83-0002": 9912, + "0c-aa-840a34cc2b7d228c2180a07df886-0001": 9913, + "0c-aa-8f8c33d15b28fd91cb7f43a690e4-0001": 9914, + "0c-aa-a40cf15584b1ef97d1e7c6e5fba3-0002": 9915, + "0c-aa-a6bad72adc581eafacd7ff9b6e1b-0001": 9916, + "0c-aa-ab50dc31733fb06523258f14fbb0-0002": 9917, + "0c-aa-bae181f319c7ad49339bb987892c-0001": 9918, + "0c-aa-d7c189329ea3c6748f383217ee31-0001": 9919, + "0c-aa-e9f966ae2f804432720587728371-0003": 9920, + "0c-aa-ea14fce28037e0fb46a3dd76f9d0-0002": 9921, + "0c-aa-f86ba41eb029a0369ef7a8e3d721-0001": 9922, + "0c-ab-1880dff860a2872327e1461a609e-0001": 9923, + "0c-ab-2e1fcc1187e4264ebcfe55c5a6de-0001": 9924, + "0c-ab-4e94203666255d70b4365b9513d9-0001": 9925, + "0c-ab-60b1cbe1d7a2a67cccb0bf5ad475-0001": 9926, + "0c-ab-6b0d3e597a2acebfef05409bc565-0002": 9927, + "0c-ab-74652336fbe8b2d2a12f4e83857b-0001": 9928, + "0c-ab-8165a45675fac03539ed28534eed-0001": 9929, + "0c-ab-86cf67155e3594b6ba7ae92fa045-0002": 9930, + "0c-ab-a092c66fc719b37734016c560c63-0001": 9931, + "0c-ab-afda0fc013d37c08072acadd309e-0002": 9932, + "0c-ab-bba5f36f16f5e49f615273a296ec-0003": 9933, + "0c-ab-bfa42059a3e04fc941b1cce9d6cb-0001": 9934, + "0c-ab-c20c7ecbc696ad3277ec8b8c7447-0001": 9935, + "0c-ab-cc6ba48d8ee96e22e514ee23b9b7-0001": 9936, + "0c-ab-cc960fc0d7c57437b2a4f7fd632f-0001": 9937, + "0c-ab-d0ffc0af078245c1fb5d9b5c3c55-0002": 9938, + "0c-ab-f9304d3f4372e03a05fa3eedd419-0001": 9939, + "0c-ac-01a890b88fcc0590da09406b70f8-0001": 9940, + "0c-ac-264bd9dabdaacf594525b7fe165f-0001": 9941, + "0c-ac-4d24f10df1ac75554c6810a3e7be-0003": 9942, + "0c-ac-5284a71718f27c1a368a9895a88a-0001": 9943, + "0c-ac-76b7e21b8f5a1d735e0eb7e54d98-0001": 9944, + "0c-ac-8d56073464178ed6ce55aef525fd-0002": 9945, + "0c-ac-9aaa0a58fa16d568489b7fd8834b-0001": 9946, + "0c-ac-b3b96492f04c2ab601f3538cc2c2-0001": 9947, + "0c-ac-b964e0bdee152f967de21f21cc18-0001": 9948, + "0c-ac-bbcdc11beca4808d5c0983e8ed92-0001": 9949, + "0c-ac-bc22772837228bcbcca03461d67d-0001": 9950, + "0c-ac-c346f9be82231ade6ecbe26d5000-0004": 9951, + "0c-ac-c6cb1948b8304360e4a96f7b6d20-0002": 9952, + "0c-ac-c6d5c5c4e91f05f8bf85fe44e4cb-0003": 9953, + "0c-ac-e30d304d93e9ad35b71c7118b26b-0002": 9954, + "0c-ac-faeb6758eb0943da7b710b80b7f8-0002": 9955, + "0c-ac-fe9ff2ebec0d6f27437983e29d6a-0004": 9956, + "0c-ae-0076595a0ce1b493f77c95552d4d-0002": 9957, + "0c-ae-0ba65ac4b1338752ba3ac58e44b8-0002": 9958, + "0c-ae-0e47ebde3024000452bf5c1d9df3-0002": 9959, + "0c-ae-133d4090af3e4a95d523bde4409f-0002": 9960, + "0c-ae-197b4b4276fa27765f56481fb629-0003": 9961, + "0c-ae-2192f384a504918f0537d26a0642-0001": 9962, + "0c-ae-3edb897d73ae040388a1bb2017de-0001": 9963, + "0c-ae-40bb84882292587e2581aabe78a8-0001": 9964, + "0c-ae-56fa1beb6649910d278616d51dde-0002": 9965, + "0c-ae-5d5d81cdffad4ffe732cdad9ea53-0002": 9966, + "0c-ae-5e38a484025d0d4ad261e0c6e03e-0002": 9967, + "0c-ae-68157969970304ab3c323b519c31-0001": 9968, + "0c-ae-8dd0214fd594893c5f4bb1edd845-0002": 9969, + "0c-ae-9ca4f65e26167e4aa71a8faadab5-0003": 9970, + "0c-ae-a18ef9a68b8a7faeffc4fafe4ac1-0001": 9971, + "0c-ae-abfe7601c534d845fe64ab5b4c55-0003": 9972, + "0c-ae-c23d8c9bf242371c0705d4465ac4-0001": 9973, + "0c-ae-d0453f2b9afd082928504a3eecc6-0001": 9974, + "0c-ae-d8adbafc3f259bb5828209c44078-0002": 9975, + "0c-ae-e2cad6a0176408efd2dc60869aa4-0001": 9976, + "0c-ae-e4725818cf8bb33b881870d9f46a-0002": 9977, + "0c-ae-e4f5c2b4ab97f9d6166ffee33e27-0001": 9978, + "0c-ae-eb2281fa133dec2d74a2fa9af0cc-0001": 9979, + "0c-ae-efd3db86381012928397a6133dba-0001": 9980, + "0c-ae-f8a020e1a00405a2b54bb2f51572-0001": 9981, + "0c-ae-fbe7d636be9ee05817a971c3d092-0001": 9982, + "0c-af-23979dd64d914797d8133d963cfa-0003": 9983, + "0c-af-2c8474dbe26f15d18226418b98e6-0001": 9984, + "0c-af-2f97fe13ef82ae607244b234b160-0001": 9985, + "0c-af-4030f9066848c7d78a39552ae7f2-0002": 9986, + "0c-af-437f90004c29fc518e4568858ca4-0001": 9987, + "0c-af-4bcf4384808e2e5b1a410a0a2fb3-0002": 9988, + "0c-af-4df1652085ab131aba53249bdabf-0003": 9989, + "0c-af-6c3b5e5cea58ec63997387f103a4-0001": 9990, + "0c-af-894fe00673026fb7ec753a25c0f2-0001": 9991, + "0c-af-8b3cfcfdc69087d7d349bbf71512-0001": 9992, + "0c-af-96849226e867446378d0c9648f0c-0002": 9993, + "0c-af-adaa74d666227ecb0f7068b987bb-0001": 9994, + "0c-af-c69132d3ed2c628f57e7ac4e5ea9-0001": 9995, + "0c-af-c7453c67d645aacae6f0038a3c88-0001": 9996, + "0c-af-d29bfea80231ad5b84b9dc1f33d2-0001": 9997, + "0c-af-de1af3ee47ce31490bd36503cf47-0001": 9998, + "0c-af-e9da57cc6588c0ceb6a6840941b4-0001": 9999, + "0c-b0-0617be26a3a666c5aaa9760eebe5-0001": 10000, + "0c-b0-133c3e61c173028b3e8573b3a47b-0001": 10001, + "0c-b0-20308a943ce6f5e629dd13be5dec-0002": 10002, + "0c-b0-32c956750b877ffa8f5b48a595cd-0003": 10003, + "0c-b0-63c8bfda85e6c1551c9d6590481c-0001": 10004, + "0c-b0-6b1d12a8faeb899ba8ada42db49f-0001": 10005, + "0c-b0-6f748a4ae4557fb66adcfd1cdaaa-0001": 10006, + "0c-b0-74c4ee4506a9ee7eca9975113105-0001": 10007, + "0c-b0-7f60dbb6947a3fb2c50815db437b-0001": 10008, + "0c-b0-9e1126fe872a2c6ccca7d16118bb-0001": 10009, + "0c-b0-9fd0dfd54fda6927e24cebbc47f8-0002": 10010, + "0c-b0-a57572c4f67922f5f2e9ad045368-0001": 10011, + "0c-b0-aa381f9ddc80a7416c3e8de23a6f-0003": 10012, + "0c-b0-cc682daf3a7be78694a272a075d2-0001": 10013, + "0c-b1-08637a416973637649579ec84053-0003": 10014, + "0c-b1-208b3acd4e4724c74f0a4c83ce7b-0001": 10015, + "0c-b1-2448a0b57224aedb14c7efd2d72f-0002": 10016, + "0c-b1-44f2956aa7319362d5cff83118da-0001": 10017, + "0c-b1-44f4d5c8ee3b2abed8103dc82432-0001": 10018, + "0c-b1-5fda72ec70dc53ed28e51c789ec1-0001": 10019, + "0c-b1-82cdd938aa48dee5d0aa6025f62c-0001": 10020, + "0c-b1-8baff458d86532497d17678513fd-0002": 10021, + "0c-b1-908e349d83722c43919a4ff59c4f-0002": 10022, + "0c-b1-96a77d108ba0a41cd01accdd0fe3-0001": 10023, + "0c-b1-99f12343a64da82d1eca65975879-0004": 10024, + "0c-b1-a36471e662391c1c04751d2f30a9-0003": 10025, + "0c-b1-b60475f191221ae58dd55a9219a6-0001": 10026, + "0c-b1-bff46e94ca2c5ec86758ae91a379-0002": 10027, + "0c-b1-cb4d25b5a2212559d460d7767169-0001": 10028, + "0c-b1-cd1fa23957fbba6e4220bfe74f00-0001": 10029, + "0c-b1-db09be410f17762706fbfd2d967f-0001": 10030, + "0c-b1-de068e8995de629f405eb8c92660-0001": 10031, + "0c-b1-e574e829f3990f1053f2e8505ede-0001": 10032, + "0c-b1-edbf4c875ea82fe35bb7e2f6492b-0002": 10033, + "0c-b1-f953c2c11d1bf86a7573115c6e0e-0002": 10034, + "0c-b1-fb4a48d9095e64785b31d9e84b0c-0002": 10035, + "0c-b1-ff65a70a6548171a4240f7e9a4a7-0001": 10036, + "0c-b2-19d5dff47294e8039c7d7903cf8d-0001": 10037, + "0c-b2-3a04c6fea8b9da4055a19bb4d077-0001": 10038, + "0c-b2-4e411e94980286d5574e56c3cb94-0001": 10039, + "0c-b2-65d026fea562ba72207ac2e03bc6-0001": 10040, + "0c-b2-690cd74b159602b20f71781411dc-0001": 10041, + "0c-b2-76616dd373e1d53195717df15c31-0002": 10042, + "0c-b2-9b21eb0decf1a3bf13f5d16f4b78-0001": 10043, + "0c-b2-a20abfae47c61304a07ceff0cc06-0001": 10044, + "0c-b2-a7a11440662dc2f7a40ad7dc65d3-0001": 10045, + "0c-b2-b7484ff1c2ad2b75341fca934744-0002": 10046, + "0c-b2-c9b8dc6a0d02385965c28ed50889-0001": 10047, + "0c-b4-12c4c04a0b04bbe4d9c05d19100a-0001": 10048, + "0c-b4-1ffc178b9edb574fc4d58abb824d-0001": 10049, + "0c-b4-25d71ab475bbedd2f7997dcc4915-0002": 10050, + "0c-b4-3e14ab2abdebd46a40919f1e187e-0001": 10051, + "0c-b4-538a1229f1b6b8b879c549701531-0001": 10052, + "0c-b4-5697649be54fdad34ee10cda7a32-0002": 10053, + "0c-b4-5de1202bc0455248b7b8161fb768-0002": 10054, + "0c-b4-84ae2e8debfb466f3cca1db22050-0001": 10055, + "0c-b4-877d49d4b368336cc7d0adc744c4-0001": 10056, + "0c-b4-95f59b60a59be0c04fd76a06dd40-0002": 10057, + "0c-b4-96e4951b1f509709da184e779a7a-0001": 10058, + "0c-b4-b118f3ea76c76ee53c0d2a10a165-0002": 10059, + "0c-b4-b931de778afc336f45262a81f312-0001": 10060, + "0c-b4-c392d7af16b97fc030a1e8018648-0001": 10061, + "0c-b4-ec3a3019f7a30abdbf0f150becbc-0001": 10062, + "0c-b5-02d108f140e1810a80c20f8e594f-0003": 10063, + "0c-b5-03b36e8e2cf25f000064f810a4ce-0001": 10064, + "0c-b5-07fe3940cb8d34267175eaeecef7-0001": 10065, + "0c-b5-14f497c8bece6d6302078ef2b4ed-0002": 10066, + "0c-b5-25822ad30ef21da0beb95b4ce48d-0001": 10067, + "0c-b5-2d05c481d74f28968a68b39f3c67-0001": 10068, + "0c-b5-3872237cb512495a0589317dbb3d-0001": 10069, + "0c-b5-49829660a1f9a932b5e42453d616-0001": 10070, + "0c-b5-4f99eef21f6f9579fa6b9440d49a-0001": 10071, + "0c-b5-55878c202e155b85c1ade3979578-0002": 10072, + "0c-b5-6954d64115b52732361e45ce8bd0-0001": 10073, + "0c-b5-81e3e6d0f062fbdc67a05af5f8fc-0001": 10074, + "0c-b5-8921bae38879941a1ae1f301b66e-0001": 10075, + "0c-b5-ab47d0a2706c89672b99e7bed483-0001": 10076, + "0c-b5-b980dbc4e6056204639ad0efc6bb-0001": 10077, + "0c-b5-c0cd3e46787da0b2accd89997e11-0002": 10078, + "0c-b5-c9ae8458e679b8380a498785834e-0002": 10079, + "0c-b5-e6b8b24e3db31a16c13c4161103b-0001": 10080, + "0c-b5-e7d139b76904646bd42f270bf2af-0004": 10081, + "0c-b5-fdad268d525eecb84907a689a1bb-0002": 10082, + "0c-b8-07e2470eaf1eea80473a83d945a3-0001": 10083, + "0c-b8-0cfd60f63be73b53074639db08f2-0003": 10084, + "0c-b8-295b2562b07534a84ea259d39297-0001": 10085, + "0c-b8-3900a67fb94e26ce7c3d42c5cd2b-0001": 10086, + "0c-b8-3b646ef2b8c92a42534413bb58ec-0002": 10087, + "0c-b8-54ca7dac3cc38107b69a6b948d31-0001": 10088, + "0c-b8-98cb03f614197bb070575660a5f4-0001": 10089, + "0c-b8-c195eeb875e43630dc780ad99cef-0001": 10090, + "0c-b8-c225190f9328fe9be5cfe3db1acc-0001": 10091, + "0c-b8-f5b34aeaf7b752441c777758e9f5-0002": 10092, + "0c-b8-fb0714769ed14f4e39dcdf85543a-0002": 10093, + "0c-b9-0411e44b05d6aa4e0714a0bcb58e-0001": 10094, + "0c-b9-1a27ef0e566c3c992aa28aff8bf4-0001": 10095, + "0c-b9-25628dfd56e9522fd5f9a6b12cc6-0002": 10096, + "0c-b9-3bc72b9e0e2b196ec86968bd9ad7-0001": 10097, + "0c-b9-3ef6ac2052de9804bf25d6051de4-0002": 10098, + "0c-b9-465c0f36389dbb4570767381a138-0001": 10099, + "0c-b9-4d15e5a2e3997286c442dab3d977-0002": 10100, + "0c-b9-5028208b5667173a77cd94e5855e-0003": 10101, + "0c-b9-587de6a41b8b736a08abe85902d2-0001": 10102, + "0c-b9-923968984a64dcc55b5490773777-0001": 10103, + "0c-b9-9a558e0465e6c5159a60debd2176-0002": 10104, + "0c-b9-9ab9ec4636ca8c612234c0f6e658-0001": 10105, + "0c-b9-d040e14de04156a9897658a9c644-0002": 10106, + "0c-b9-d1e9252e5f03ec6c1aeca37202e2-0001": 10107, + "0c-b9-e1beda2a74aa4cfd5369fd092b3f-0001": 10108, + "0c-bb-2650cc7eec217f699e37856799f8-0002": 10109, + "0c-bb-2a89500c9c45381cb39c8316fbb5-0002": 10110, + "0c-bb-2e492572f0d418649c26ad42f3a4-0001": 10111, + "0c-bb-55fb729bec6c09f9688bfdaad54f-0001": 10112, + "0c-bb-6640d345c5ae574145f43c53a71b-0001": 10113, + "0c-bb-6b5eea29aae39a03f3f4df44108e-0001": 10114, + "0c-bb-7d6b84cee76073f6e00776c9bc52-0002": 10115, + "0c-bb-8e1b09e2ee423578eca714b36183-0002": 10116, + "0c-bb-9fbfd487cd5d001284cdbd94fa6b-0001": 10117, + "0c-bb-ca72cd26149a10c9e4ac73ed9617-0001": 10118, + "0c-bb-cc3c1e9eecf20a32d9fd82e337d9-0001": 10119, + "0c-bb-d23e3db26d2f15b6468b44236b09-0003": 10120, + "0c-bb-e0e707ead628a619d42428e4c3ee-0002": 10121, + "0c-bb-e5b0cb5b988029612dbc9bdb449b-0002": 10122, + "0c-bc-1f70c1334e7a28d001bb247e412e-0001": 10123, + "0c-bc-3a3f935277f4bb05e5b4b09d4e13-0001": 10124, + "0c-bc-574bf5679720a024a40f0bc3ac76-0002": 10125, + "0c-bc-67e32093c1a4c0f2d74d707f3e75-0001": 10126, + "0c-bc-6961c4359024dae95dc409e17a98-0001": 10127, + "0c-bc-74b4ffa6f3b095feb1efa58da29c-0002": 10128, + "0c-bc-785a1a8c0f8234bca2cc349c3289-0004": 10129, + "0c-bc-7865fbd38535d14e17db1099b429-0001": 10130, + "0c-bc-79ce10dbbbb2183659a78c605b42-0002": 10131, + "0c-bc-8bf1a9fd54ff6c4cce68ee53a8c8-0002": 10132, + "0c-bc-8f61e85a2058a38f9072b14c85b9-0002": 10133, + "0c-bc-9004f8c3d51f0e7c8ac293301ee1-0001": 10134, + "0c-bc-a8f1d1caef11c05e9114f2065ede-0001": 10135, + "0c-bc-c097138e683c818a5566f74367e7-0001": 10136, + "0c-bc-cc5ef9a830f2a5e1e9a6bcfd4640-0001": 10137, + "0c-bc-d8f2589b78be9fc8b95c721f77c0-0001": 10138, + "0c-bc-de790a9ea2afabce3cfd129fe41a-0001": 10139, + "0c-bc-e64ce67779bec5174cfa61702b24-0003": 10140, + "0c-bc-ff6afa4997faad8e4780127de060-0004": 10141, + "0c-bd-0e08105d7dcbf562285e7b8006ad-0001": 10142, + "0c-bd-0f91f0e3a1d0411972f74a7fd9e2-0002": 10143, + "0c-bd-16b51ccdaf223ae6748fa3a60877-0001": 10144, + "0c-bd-1a145a0ca224d43af2bc93bc6f13-0001": 10145, + "0c-bd-3daa6cdd5d8c68ebb58eb9356a92-0001": 10146, + "0c-bd-40d99f620fcc46fd72a166af8698-0001": 10147, + "0c-bd-46a96d5412ae8c901a95392e5bbd-0002": 10148, + "0c-bd-46f9aa67f5bf32a3a0bc5b8ae97e-0001": 10149, + "0c-bd-4ada212950e7fa508c055063893e-0003": 10150, + "0c-bd-5e76967bb14539c37124ae8e99d1-0002": 10151, + "0c-bd-6142b7509b86f37321d82aa367c9-0003": 10152, + "0c-bd-6f98ffd0542f78d394b34ec5532c-0001": 10153, + "0c-bd-8dcbeaab74dc107ce24130b61ae2-0001": 10154, + "0c-bd-9938d95e474f2e46bc2f7ba79760-0003": 10155, + "0c-bd-a262b7c437163cc39ba0dd0ce754-0001": 10156, + "0c-bd-b3baf00363cc83168b0bc429542b-0002": 10157, + "0c-bd-c743fd09e54d8456cbaaac83aaa2-0004": 10158, + "0c-bd-cb6078896d2981adba5911bed81b-0001": 10159, + "0c-bd-ea0e09cf1ca27fa19a9b757ea02e-0001": 10160, + "0c-bd-f3a40a8f8960c59f02f369da9e85-0001": 10161, + "0c-be-07733d9eb524b834bf52691a2508-0001": 10162, + "0c-be-2b25b70f39615820d3c41fd7b918-0001": 10163, + "0c-be-4ab74b7645c30bbf3f68969dccec-0001": 10164, + "0c-be-4afec507b12d5ba20290f5884c41-0001": 10165, + "0c-be-4efcbfb579b6c9c6c2307c1ca123-0001": 10166, + "0c-be-56b7758391c6ba00b8aec5b8e3db-0002": 10167, + "0c-be-5c00fd54ce4414ea4179b0c9c1d7-0001": 10168, + "0c-be-5f34d9f0d878e06a297cf67932b3-0001": 10169, + "0c-be-66f045e4a74a6a22180e7e608292-0001": 10170, + "0c-be-8a6a0aee9cb85fb7765ce77d78d1-0001": 10171, + "0c-be-96274c57394febe7499d0634336f-0001": 10172, + "0c-be-9ca21e0c58aee5914b3369d47b86-0001": 10173, + "0c-be-cdd04cd5964e1d0d7bdd664dad55-0001": 10174, + "0c-be-d867fdddfade49edb4ccede6aff4-0001": 10175, + "0c-be-ea5a7287b38fa01b1f8ac3cc9ceb-0002": 10176, + "0c-be-fc0d90786c912508298e7d95a9c6-0001": 10177, + "0c-c0-12f97085938ba691a9c1ce470c21-0001": 10178, + "0c-c0-16eedaa08e723bd9913184c21128-0001": 10179, + "0c-c0-17547fca02e882cfe563ac30bfe8-0001": 10180, + "0c-c0-1806e111a974f7ea4c8951932fa1-0001": 10181, + "0c-c0-2c346c71df9f125c5456f1e70e43-0002": 10182, + "0c-c0-300b51d930903fab28acef9191a5-0001": 10183, + "0c-c0-483e56b159cfcc227ce0ab496923-0002": 10184, + "0c-c0-53c92deceef95cc87ea3c9af7292-0001": 10185, + "0c-c0-53dea5b4cd38a75ef68eb7c157d9-0001": 10186, + "0c-c0-5574142055ccbdb44c8b25137149-0001": 10187, + "0c-c0-5987efe816ebc1d5466af38c7cdc-0001": 10188, + "0c-c0-6bd5b7427c475fd0e31b7250949b-0001": 10189, + "0c-c0-6d9174d7e1c02c473d2a6d754eb9-0001": 10190, + "0c-c0-71516a8fffc59058ed6148a824c5-0001": 10191, + "0c-c0-86654b4be19bca43f387ad5a9c2e-0001": 10192, + "0c-c0-9406b25dae88b0f4bfcbd6231d46-0001": 10193, + "0c-c0-96d9e7e2ebd53ab3ba7575a7168b-0001": 10194, + "0c-c0-c7d4cff6a426a78c3cccb2ac8089-0001": 10195, + "0c-c0-c810cc494cde203dfc2ba3bbe3f8-0001": 10196, + "0c-c0-e531cbc3e2275796a9a09397132e-0002": 10197, + "0c-c0-fdab0c665ad6f79dd312999f7160-0001": 10198, + "0c-c4-048f3fed1cf5ab244642a89d3881-0001": 10199, + "0c-c4-316fdba9ba7ce9906bf1d4bfd807-0001": 10200, + "0c-c4-3f27f62fbd1552c7178509f5b563-0001": 10201, + "0c-c4-40877d4b8c5383f93b67496cbd26-0003": 10202, + "0c-c4-45f8fc73181390a37aa7a6073b3a-0001": 10203, + "0c-c4-465cdba09595a4d9517c2de64b72-0001": 10204, + "0c-c4-486e74284c236f57cd7622b27ea6-0001": 10205, + "0c-c4-565f751f44c88e44bd4e9ae904bf-0004": 10206, + "0c-c4-61b40d288931099ad51b67f5901f-0001": 10207, + "0c-c4-64325b751d5409ba4cddb23eea64-0001": 10208, + "0c-c4-6630deb1e061186ec083ff2a5f50-0001": 10209, + "0c-c4-7317259251b1713ab312d4561294-0001": 10210, + "0c-c4-731f47ec93329c4d3a259af784fb-0002": 10211, + "0c-c4-7a94395a54d1419d2d4a4eedbdaa-0002": 10212, + "0c-c4-80ef2b83ba621bc1b5640427ead5-0001": 10213, + "0c-c4-8e3b4dffd42943311f177d711aa4-0002": 10214, + "0c-c4-b82dd9fa99fe7afc4ad1c77224cd-0002": 10215, + "0c-c4-bde3de83327bc5cf604783172b82-0001": 10216, + "0c-c4-bf65642bca6a875fa93f8f4ea2e1-0002": 10217, + "0c-c4-c2ee44d76f960970d26db1bc95f9-0001": 10218, + "0c-c4-c9e5ef1ce1f3cdc78ddbe698f33e-0001": 10219, + "0c-c4-d1a4e291f86649a2f3c000897e3b-0001": 10220, + "0c-c4-d6937e8daf28466c6a091b30d1ec-0001": 10221, + "0c-c4-d84d48628590f9d46a085d8150a4-0001": 10222, + "0c-c4-fdb12337342ba066922378317c48-0001": 10223, + "0c-c7-02ea8bac74afd664582ba1a5f0c2-0002": 10224, + "0c-c7-266dbd93794d52d008937142364e-0001": 10225, + "0c-c7-2cde333c3817bd6331a2f16df8e3-0001": 10226, + "0c-c7-3c79f4133d8873627c4f0f567e6b-0002": 10227, + "0c-c7-41cb0465573836e01d1fd03944c1-0001": 10228, + "0c-c7-601c6465327f9a62917540854219-0002": 10229, + "0c-c7-7431fad8ae85173f19972db38145-0001": 10230, + "0c-c7-7fb9057434a02f23554ba4d6d5d8-0002": 10231, + "0c-c7-99f497a99095d689271f94e549ad-0002": 10232, + "0c-c7-a79f0339381b54f646f971ca11de-0002": 10233, + "0c-c7-bd36458bb44bb3ddb363b2b499e5-0001": 10234, + "0c-c7-d8066b89badaf210542731733ae8-0001": 10235, + "0c-c7-e9b002a0201a0bd2e086a54fe37d-0001": 10236, + "0c-c7-ef98cc278d10891db378d6c5e892-0001": 10237, + "0c-c9-02a64cdce8f029e32312847bdf61-0002": 10238, + "0c-c9-0f0b57546f4310e9965ccf5a7394-0001": 10239, + "0c-c9-201a607be6e334d5386d7319bcee-0002": 10240, + "0c-c9-255128a1f193cc95cdf9cbb0b181-0001": 10241, + "0c-c9-294df9158b3a4749d77a15e22a6e-0001": 10242, + "0c-c9-368c910d8574f1d184fb313d70fb-0003": 10243, + "0c-c9-56678a7f29e43623c28f7bfdd4fd-0001": 10244, + "0c-c9-7203c30270f285965545d40f3d05-0002": 10245, + "0c-c9-741432fd0adc7660a35d920b563d-0003": 10246, + "0c-c9-94ddbb0cd63da2cd48699ecfbd05-0003": 10247, + "0c-c9-a867fcc690d720aa021bcd15eb9d-0001": 10248, + "0c-c9-bbb4524d55d535b7ab0746ef5273-0001": 10249, + "0c-c9-bcdddd45fe38cc1cffdba9ed5e2c-0001": 10250, + "0c-c9-bda8f331374488d2b516fa0a1468-0001": 10251, + "0c-c9-c7033b2172fff2c637a31696bd62-0002": 10252, + "0c-c9-f59d1532a4d04f404cf877f5f25c-0001": 10253, + "0c-c9-fd7e92357ba029f0ed2cbc39bd90-0003": 10254, + "0c-ca-180addad7f2f96339f855336ceac-0003": 10255, + "0c-ca-21aad208ee0ee4c2b7050b4a3c1a-0003": 10256, + "0c-ca-30237408d4e0357f9b8051f024d1-0001": 10257, + "0c-ca-365b7adee3ec839cda5764750726-0001": 10258, + "0c-ca-4bee208c92493ac287bcb23c1459-0002": 10259, + "0c-ca-535fab98769a97c7f72161bbebd1-0001": 10260, + "0c-ca-55c616ccde06f39bcae1127f19a4-0001": 10261, + "0c-ca-5d7d8fd0ef17ecd36624899c3bb0-0001": 10262, + "0c-ca-7a7b276e2cd3a352412bbe76520d-0002": 10263, + "0c-ca-8de03a5b82c664f76c8c9747e292-0001": 10264, + "0c-ca-968029d1b00186b1639bf7828913-0001": 10265, + "0c-ca-b29aa25abe1674b3bbd0bd6a9895-0001": 10266, + "0c-ca-b3f7b436d2609e834a97a291e8f4-0001": 10267, + "0c-ca-cbd4a4a6c4525f3868e236f72a87-0001": 10268, + "0c-ca-dbde561d3fc4181c55535123d0fd-0001": 10269, + "0c-ca-e05371242948255c269dbe607e86-0001": 10270, + "0c-ca-e0b8db93d856d2271443f8223f03-0001": 10271, + "0c-ca-ea65abe15bc4194ef4acd90a4d49-0001": 10272, + "0c-ca-eaf34a67113b2c6fa802e4dd6924-0002": 10273, + "0c-ca-f71c89d9239a36a93f074bd8b81c-0001": 10274, + "0c-cc-07367a2338b40f10fec58dce58cd-0001": 10275, + "0c-cc-0d3569a7172d714ecd9da10b008f-0003": 10276, + "0c-cc-17360fb2a3c0c91ed60189be9710-0001": 10277, + "0c-cc-198790caf147490f5357451850fa-0001": 10278, + "0c-cc-365fa34cca12ee78968dfbae0e22-0001": 10279, + "0c-cc-566722c045377e5deb9a95da92da-0001": 10280, + "0c-cc-5b5c55f9ee8b83bc2f0d6a62eb3d-0001": 10281, + "0c-cc-65a80fcf227b0a8b4989dcc30a07-0001": 10282, + "0c-cc-8f24e6f162b61e6fecf22d9b02bb-0002": 10283, + "0c-cc-9058c9f670d62ea2a980b9e863ac-0001": 10284, + "0c-cc-9142019647f1534678062a85e368-0001": 10285, + "0c-cc-9f291424f6604d632924af68ac7f-0001": 10286, + "0c-cc-acd7cb2c0e46fa55cd865f840069-0002": 10287, + "0c-cc-cecdb6e6999f354e711f4000fa82-0003": 10288, + "0c-cc-d78603d5532147c817a7c46dfda8-0001": 10289, + "0c-cc-fe6c8aa7afe41eb20e5e5cefba62-0001": 10290, + "0c-cd-005a4d3742a62de38ec8110c0d71-0001": 10291, + "0c-cd-065cf75431fa256e1d1295ef942a-0001": 10292, + "0c-cd-2f5b45b7683f6d9b1a85b21b6808-0001": 10293, + "0c-cd-3660e079d7f08c036c90882ffa40-0001": 10294, + "0c-cd-49a2359fd18f669a779c6071780f-0003": 10295, + "0c-cd-5168c5b4078687b597a1757f4d64-0001": 10296, + "0c-cd-7229f0fb3b522a59e517159e7f17-0002": 10297, + "0c-cd-76bcfd783f716a32c5526bdbb518-0002": 10298, + "0c-cd-77662e81d025a8407f1f5b1e69c1-0001": 10299, + "0c-cd-c9667a268d046bda807a96666342-0002": 10300, + "0c-cd-d4cd28aaf68c235400505e249b29-0002": 10301, + "0c-cd-f75175823f85dcd2dd93592389bb-0001": 10302, + "0c-cd-fb2daa096fc96dc9ec557b7da3bc-0002": 10303, + "0c-ce-109d7900504bf7974d452e5c2146-0002": 10304, + "0c-ce-131a4ad662b5b465eca561735e8c-0002": 10305, + "0c-ce-1a0dfd6521749ee694c4a560d0c1-0001": 10306, + "0c-ce-3d465497bb9ad9c809bbdb732a0e-0003": 10307, + "0c-ce-5422fb7c623add0e3119316083ba-0003": 10308, + "0c-ce-6ab5a3bd00518b06dfeacd3c62d5-0002": 10309, + "0c-ce-6f01f5f8036ca3bba6009adea2d3-0002": 10310, + "0c-ce-7c1f8afcefaa41f94fa272f3fa08-0002": 10311, + "0c-ce-a43c1dfa5dde318287e2a6a3fdc8-0001": 10312, + "0c-ce-bd9a4e2b497dacf830f22e4bd5a2-0001": 10313, + "0c-ce-ddb30377dae324e911c481ee6a90-0001": 10314, + "0c-ce-df8eb2785486d4eb51f1006db9ef-0001": 10315, + "0c-ce-e38a464af6dd5a8d94dc87bd4eba-0003": 10316, + "0c-ce-f84a2697daf74b5b7e9bed771a2f-0003": 10317, + "0c-ce-fcf39b1262156eaad4b5e1c4624b-0001": 10318, + "0c-ce-fdf0f4c708142c15d9acfd8b9c78-0001": 10319, + "0c-cf-184fbf1f1a014246416356e7a9f3-0001": 10320, + "0c-cf-28578cfce82d16bb85d4e51cea41-0001": 10321, + "0c-cf-59e9bcfb845eee5a7f921962fe8b-0001": 10322, + "0c-cf-93ea0a48dd593fe071730894403f-0001": 10323, + "0c-cf-9ae7dc4f834ec7b6874c63ce6591-0002": 10324, + "0c-cf-b2bf3daa9614c9c3e17d1b036180-0001": 10325, + "0c-cf-b4d45511d7d39fd5a4758ef42632-0001": 10326, + "0c-cf-b7b33e97559e828d31529e08b13e-0001": 10327, + "0c-cf-c635484197a7a740870cd8984bc5-0001": 10328, + "0c-cf-fcefcbaad6f853c733cb4e6359ba-0001": 10329, + "0c-d3-00cc3920a242fded7ea2d9568811-0001": 10330, + "0c-d3-070c85a59197257c7ffa60ebafe2-0001": 10331, + "0c-d3-0a98de8264d2dcf366a9dfdb38f3-0001": 10332, + "0c-d3-0d03a720e64189fac5a24bc63e6d-0002": 10333, + "0c-d3-0f2ddb10e2d98a0591b790e67487-0001": 10334, + "0c-d3-25822239487b34979e23b8328480-0002": 10335, + "0c-d3-498528ef7cc11829c0d89f36e0b5-0002": 10336, + "0c-d3-518d87d1e5f028a8e48934e1f05a-0001": 10337, + "0c-d3-52e0525db2e446db70ed80f6db60-0002": 10338, + "0c-d3-7e4f2a13f231ee39c284ba5ef0cc-0002": 10339, + "0c-d3-825eafa46184fd306373f9a2c826-0001": 10340, + "0c-d3-8a7cc2cc3b8549d17daeeaa0df2a-0001": 10341, + "0c-d3-947be4e6cb7bd630b77d24a430a7-0002": 10342, + "0c-d3-a1a65eca7e2692c100615af61e2f-0001": 10343, + "0c-d3-a6b2c0c8f77aa4ff3a8deba0b95d-0001": 10344, + "0c-d3-a7a83261e94b6913d9f736997c36-0002": 10345, + "0c-d3-aa3ee19fc2362b838e67553c10e5-0002": 10346, + "0c-d3-be544f45535624164a7fc820c11a-0004": 10347, + "0c-d3-c1ef7b0e79b798f6367dc879c8cf-0001": 10348, + "0c-d3-d8369fa18f277a79ab785902e944-0002": 10349, + "0c-d3-dc114229a7eda0c7c94b4c17bcbf-0001": 10350, + "0c-d4-06dd22ee98b14c9a4674e6e2d7bc-0002": 10351, + "0c-d4-08772691363dd62128f2e5cd47ee-0001": 10352, + "0c-d4-0b65918efc6f3ff27df7b2ef779a-0001": 10353, + "0c-d4-2132c60acea918dcb15a5d1198e2-0004": 10354, + "0c-d4-2714bb7d1dc458218b195ea061ee-0001": 10355, + "0c-d4-2988b81f3a676cda4de2ba269f88-0002": 10356, + "0c-d4-3adc1eedfc27fbef30ebb12580b1-0001": 10357, + "0c-d4-3e0b7f4bf0fdeb424ba8996e3158-0001": 10358, + "0c-d4-5a932a957c504be7616925060b29-0001": 10359, + "0c-d4-609d83157ece1afe5a4c5fa26ca8-0002": 10360, + "0c-d4-6ed9f56226e37502d8c87ef341ad-0001": 10361, + "0c-d4-6ffec2349d9611e48a560416c01d-0002": 10362, + "0c-d4-8696a0b85b82545716cc86cd6702-0003": 10363, + "0c-d4-b0075c912b6fd622a06aaa017eb6-0001": 10364, + "0c-d4-b9c4c2f36c720cb97631bb5a4d2c-0001": 10365, + "0c-d4-ce5db91384dc914dabfceefd46a8-0001": 10366, + "0c-d4-d7c916a9b259d8b542e21fb59e57-0001": 10367, + "0c-d4-dce33fc95d886fec235db96e9316-0004": 10368, + "0c-d4-f42be3af15cb4e4853e88b4e67ea-0001": 10369, + "0c-d4-fcdaede5f4be46c2a8078d93527c-0002": 10370, + "0c-d5-063c7591ec28eaf718e40374420f-0001": 10371, + "0c-d5-1dc15c0705547f2851fdbdc69940-0001": 10372, + "0c-d5-34282129c61058275ee625b4c82d-0002": 10373, + "0c-d5-40745a3eb61c4a1bf77c05e03fc8-0001": 10374, + "0c-d5-46651b890f6f21c5ab6cab9d4b33-0001": 10375, + "0c-d5-50c30d78aa7b8a5f9ecaec9c4ea3-0001": 10376, + "0c-d5-5a49e3b319baf04a145f0ee00694-0001": 10377, + "0c-d5-5c77064025f993fb69626e9b7206-0001": 10378, + "0c-d5-637e6422502a14c5da7e860b793b-0001": 10379, + "0c-d5-74b0ce58dd7670a97b8b2d6f4401-0003": 10380, + "0c-d5-8a8b1b0a04f738539bca9c7a6250-0002": 10381, + "0c-d5-9e5ca362915590924aee4ff6d029-0001": 10382, + "0c-d5-a9624f1850c7fff1d64ebccc47f5-0002": 10383, + "0c-d5-aa859e6196cbd1f4a4fa2c6cd792-0001": 10384, + "0c-d5-b937e304804087c2792df90afaf8-0001": 10385, + "0c-d5-beb3f42f7da15e7ae31f96ad46c4-0001": 10386, + "0c-d5-c39ee9f6cec918023b2a4406b781-0003": 10387, + "0c-d5-d0581446c1543860eadf7d7f4d8e-0001": 10388, + "0c-d5-dd315c693706fbc819d8b97b6c43-0002": 10389, + "0c-d5-df78f3611c3b7654d01f77f434dd-0001": 10390, + "0c-d5-e53092588eaea39226cefd347207-0001": 10391, + "0c-d5-f8eb5f15ed6308d6a72ccfeb8e4d-0003": 10392, + "0c-d7-15014c0f616a5c36d9a908832385-0002": 10393, + "0c-d7-2828f8b0bd3ea0303f310cc30381-0003": 10394, + "0c-d7-2d4c7de58b3f908a54b5fc1249ff-0001": 10395, + "0c-d7-3355b1f6dbc1d07d46a545b44cca-0001": 10396, + "0c-d7-42cc97a7c69e4a00ba311721b5fa-0001": 10397, + "0c-d7-65fdfb2f19da2d98fe159b4606a3-0001": 10398, + "0c-d7-6a4025ff9fb85c3af8c266e9bf08-0003": 10399, + "0c-d7-750589e3c887633957b63d6544c9-0002": 10400, + "0c-d7-9562a275060b6ee8680df5d02f48-0002": 10401, + "0c-d7-9a96128641a3c972bab1ca36faab-0002": 10402, + "0c-d7-b4086cb86d693a10aa35fc208451-0001": 10403, + "0c-d7-be72cfc03b0cee93433647e2d205-0002": 10404, + "0c-d7-c9b85908121fd1ed1be71d525a2d-0001": 10405, + "0c-d7-d4bbe6e184d9982934b6d5bb6479-0001": 10406, + "0c-d7-dc8a2ef0e00c7bbc490a2495dfa1-0002": 10407, + "0c-d7-ed7cd688c84ce2d3db6ce96fc05f-0003": 10408, + "0c-d7-fe45cea0c80703aabceb98427c7b-0003": 10409, + "0c-d8-09c7c2e1b14db5d9e7977265295e-0001": 10410, + "0c-d8-136f9bee7e717ca452bd13c29d2a-0001": 10411, + "0c-d8-14046d49bf10eee494c693b8fe43-0001": 10412, + "0c-d8-198108cc294cd5a954931cd8e3d8-0001": 10413, + "0c-d8-24d40d2fd8564c7178b78d3602c6-0005": 10414, + "0c-d8-2bab669d421e3b6cfd40e58db17e-0003": 10415, + "0c-d8-388086579091454799b282dbb65e-0001": 10416, + "0c-d8-856bf447ce0eb35347aadeea3372-0002": 10417, + "0c-d8-8ad3923de7b6f5432ec600636da1-0002": 10418, + "0c-d8-97018a27630a5b1e93b1289d8184-0001": 10419, + "0c-d8-b3ca46cb83e715ed79034b243cec-0001": 10420, + "0c-d8-f20ed693ce187a25c18effdb5a1c-0001": 10421, + "0c-d8-fb98432e734caab37ec0811111b9-0004": 10422, + "0c-da-0f0f525fb9edee2d6890e490d9e5-0002": 10423, + "0c-da-11d2ba13133d0e6c368fa6b5de78-0004": 10424, + "0c-da-214245c4937203f6c18e9cd1aa84-0001": 10425, + "0c-da-289ee61f90c51707b8a53bedb406-0002": 10426, + "0c-da-35b1f6e112e8362f4402092d88c9-0001": 10427, + "0c-da-512fbbadc59188e77bc6aafbdd78-0003": 10428, + "0c-da-620ec61f50c062d99ffb50fb00d5-0002": 10429, + "0c-da-7f616cc426b363a1b9bc6340a321-0001": 10430, + "0c-da-804048c4e6c3a4a9c8710a4f4c97-0001": 10431, + "0c-da-84252da9df5e370bb7a1b2bf8aa6-0001": 10432, + "0c-da-92ce9deb5c78c8eb4ac042ee41e9-0002": 10433, + "0c-da-95005cf89ba1721b71ddef41e706-0001": 10434, + "0c-da-988536ba55f404c77b0f3905f539-0001": 10435, + "0c-da-a1fc47e14b359c06b5bb445922f6-0001": 10436, + "0c-da-a1fd4937fca83fe4885b5951ce8e-0001": 10437, + "0c-da-a97a6d215c55c9f4505e9a3fd0a0-0001": 10438, + "0c-da-aaae457e72b128e15f6b47778ea1-0001": 10439, + "0c-da-b82c42dbdbb572965094562ae267-0001": 10440, + "0c-da-c768764f578525b6559fe646d940-0001": 10441, + "0c-da-c9881c4aa883060b93a25bbf6355-0001": 10442, + "0c-da-de52f4ff0b990b3ec69715c75fc6-0002": 10443, + "0c-da-eb30d14b42cb3d7f393d66408f9f-0002": 10444, + "0c-da-f833affca0cacba6f37bdb44ce86-0002": 10445, + "0c-da-f861dabf12279e06d34acda61d89-0001": 10446, + "0c-de-0453d9440b60a88faa995bdfa26c-0001": 10447, + "0c-de-12056574e08c682e5cf8dc9221bb-0001": 10448, + "0c-de-14310c0e3e78dca5c8f25ab2b545-0004": 10449, + "0c-de-328be288cfa1cb49bb4009ed73b5-0001": 10450, + "0c-de-3751f55141674669bdb48dd897ba-0001": 10451, + "0c-de-51405d4f1a2120ca17082f854862-0001": 10452, + "0c-de-8f66b15f6e07fb7ae07ba0dc654d-0001": 10453, + "0c-de-93e4d9028154baf2473f9554af7c-0001": 10454, + "0c-de-a513a1ed0afd8b0510b7d7de910f-0001": 10455, + "0c-de-a76d98ad3cad8bea6d6d21d8fd7d-0001": 10456, + "0c-de-a812e9dd89cc4be24f1f1e9a55a2-0002": 10457, + "0c-de-a8b42618076e118462f6fa3665a1-0002": 10458, + "0c-de-b39c1cfcc10409434a630fd133ba-0002": 10459, + "0c-de-d9dd3a6f69a093e6be2609a583f9-0002": 10460, + "0c-de-fa3e239b2a188d6fb133b2a81e7c-0002": 10461, + "0c-df-0793c962920343730e14c8d69289-0002": 10462, + "0c-df-0b0cff30be293c2c5a4ad8c7f6df-0002": 10463, + "0c-df-186ba6c34c4de94066da9f70f87b-0002": 10464, + "0c-df-218630c984a14b8726d5f7fd4326-0002": 10465, + "0c-df-3d4293638490b0a55f4a60ae6455-0001": 10466, + "0c-df-40813cc865338c39d3786c75bda0-0001": 10467, + "0c-df-53d347a42126029f50e91677069c-0001": 10468, + "0c-df-7060a280dc800abf32cf7a2814f2-0002": 10469, + "0c-df-7a82dcd3821c49072697a51aafb8-0001": 10470, + "0c-df-a2eb7d972d30cf9e148d6699402e-0001": 10471, + "0c-df-f0c31103979c277bc67d2e6ae0f5-0002": 10472, + "0c-e0-0bcb02ece2cd1e5983194cfc46a5-0001": 10473, + "0c-e0-0fe945645348c84c682799290e20-0001": 10474, + "0c-e0-1ecf7bb41550fd7694cd55dfee6e-0002": 10475, + "0c-e0-42f3e881cbe1382c1500588676ba-0003": 10476, + "0c-e0-5df2c85016e641fed718c8136d97-0001": 10477, + "0c-e0-5f3d1d04b18d6775bedc8fa7f9ff-0001": 10478, + "0c-e0-79cb5f1d99118a1406383494960a-0001": 10479, + "0c-e0-81d2053493b7f4792cb82164b7a1-0001": 10480, + "0c-e0-9b4b7e00352cbcafbf0c140c0638-0001": 10481, + "0c-e0-b4b3d8df42d236d6142816d8bc8e-0001": 10482, + "0c-e0-b9c0ad595e1184f89665bb838238-0002": 10483, + "0c-e0-bf0ec2997b3446ffabcef64e3fd9-0001": 10484, + "0c-e0-c0ca45feca04974bc564233a4732-0002": 10485, + "0c-e0-d19982e3a1fbb73d32a8220fac1a-0001": 10486, + "0c-e0-e4b801c61683c316484fb41600b1-0001": 10487, + "0c-e2-21bd70b2c424d3fb1a4b4f5dedfa-0003": 10488, + "0c-e2-415411274a638fbe7bfd3fd16bab-0001": 10489, + "0c-e2-418dd985f7fac5645382cfabeadd-0001": 10490, + "0c-e2-48d521e155844b4f99c25368bfeb-0001": 10491, + "0c-e2-5a76fc3f39ba5fd00ac5de9ab77b-0003": 10492, + "0c-e2-8321de809c2e271a7b250a2e408f-0002": 10493, + "0c-e2-a582678cdbe64840bd2deaf3327f-0001": 10494, + "0c-e2-aac46214793bfa34a1cb0701374a-0002": 10495, + "0c-e2-c47e34e8adf5db8a9d4701b27ed8-0001": 10496, + "0c-e2-c56ddd4b658b2cd22d6ae28838f8-0003": 10497, + "0c-e2-cd34053a05219b1486873f1f13aa-0001": 10498, + "0c-e2-ceed82afc75a368c98c9705426a5-0003": 10499, + "0c-e2-d02f61e18363a7f4ec4f36277e89-0001": 10500, + "0c-e2-e157856a63c028ba62d4e399165b-0001": 10501, + "0c-e2-e88a915ebfc6051530a7bd956e4e-0002": 10502, + "0c-e2-fd58bf1c0660a57064432f4165b3-0002": 10503, + "0c-e3-00ecd3aa4132b49a0bf286b87144-0002": 10504, + "0c-e3-05b837e6d6e7e888a9fe80854c2e-0001": 10505, + "0c-e3-0906e2ad70797f08dcb3f60afb1d-0001": 10506, + "0c-e3-1a63ac54c988ef7cf323c0baf963-0001": 10507, + "0c-e3-24fcc04a0bdc77f7df1ec42c3e1b-0001": 10508, + "0c-e3-2cf5bb876db7b4124faab51c91f2-0001": 10509, + "0c-e3-4067add614dd74d94c1dfdd8d2ad-0002": 10510, + "0c-e3-47662b1ae71815e3ec8bbbd119ee-0002": 10511, + "0c-e3-4dd65ead9431bbde6d91c361f0ca-0001": 10512, + "0c-e3-59ef05efff8d38a39acfed24c1b7-0001": 10513, + "0c-e3-631a50a6fabe962eb91ffba276d7-0001": 10514, + "0c-e3-6cf7ff922cdfb6087dfef7eb105b-0001": 10515, + "0c-e3-93120e43ccd3f4d7d5a916baeff4-0001": 10516, + "0c-e3-b199e9584500dc0786be101a43a4-0004": 10517, + "0c-e3-b32d82dd2751e82c275b89c8285a-0001": 10518, + "0c-e3-c2bdffb477623d860f21c322d843-0001": 10519, + "0c-e3-d8e3be0d429976abf05dce5dd379-0001": 10520, + "0c-e3-df04360f84c182a85044072e63a1-0001": 10521, + "0c-e3-e672be8e12fee1979fcebd4e4632-0002": 10522, + "0c-e3-f41037e79d4675fa44d23dbb7b93-0001": 10523, + "0c-e3-fad15d0f5944c3836c25d652f087-0003": 10524, + "0c-e6-04d71bcb6fb843443e98b4c1271e-0001": 10525, + "0c-e6-13b5cf008b01fd0121ceba4b4f1b-0002": 10526, + "0c-e6-16c1c97838b9b6a6e8394d316ecb-0004": 10527, + "0c-e6-2156c0950f1f33daf6ec1bbc5405-0001": 10528, + "0c-e6-253f384b64a02bc4d8211e43a59b-0001": 10529, + "0c-e6-3c5109b03683a22ce0a77b9677c5-0001": 10530, + "0c-e6-40ea295da477a2fbd7139faa3eeb-0001": 10531, + "0c-e6-447929d8a432d1b7802310542fb2-0002": 10532, + "0c-e6-45f849a4ab80c5f1087672fe12a7-0001": 10533, + "0c-e6-5ceb0b16acc303e4cd224d705f78-0001": 10534, + "0c-e6-6475654f747c85f101b9c69e1307-0001": 10535, + "0c-e6-79db51569e84f570c6a029c32213-0001": 10536, + "0c-e6-7be67d1e16cda4c16394436077b3-0001": 10537, + "0c-e6-80fd457fa9fca1d2db73216e4da8-0001": 10538, + "0c-e6-84b64af56693f66276fbdf9c80a7-0003": 10539, + "0c-e6-aa1b24d79eee81546c0bde0fc6f7-0001": 10540, + "0c-e6-bbaf3326a2c3176ddbb78b0553e0-0001": 10541, + "0c-e6-bd2360e6a15763f88de73dc3a9d6-0001": 10542, + "0c-e6-d1a0266e34bdcdb9800b51253261-0002": 10543, + "0c-e6-fb415857f39a5f7f6efcfdf292ab-0003": 10544, + "0c-e6-ffd51dc53454a15e5af1c90d131e-0002": 10545, + "0c-e8-09fed0bb2357b8611d8cd179e7e4-0001": 10546, + "0c-e8-0a66223493378ef3774825e7df89-0001": 10547, + "0c-e8-0e5021eaaa4573db125076c77744-0001": 10548, + "0c-e8-225aaaf1700b7d5d3c651a6e6a8b-0001": 10549, + "0c-e8-249a82e7a5b0729dc1aefc3a3a1c-0001": 10550, + "0c-e8-2a8be6250830c37546bb2b3043ed-0001": 10551, + "0c-e8-3152de28929bed351cb803774896-0001": 10552, + "0c-e8-500dc7a2dee4c755250dbb0b6ddf-0001": 10553, + "0c-e8-58407606b8f41a21512103649413-0001": 10554, + "0c-e8-70b79eaef76d7e91424759c6f57f-0001": 10555, + "0c-e8-7a33960175ecd67970d5243994c6-0001": 10556, + "0c-e8-8388c85633c0c9012c47b294f6d5-0013": 10557, + "0c-e8-855f1bd6cb632c9a5f8815d469ba-0002": 10558, + "0c-e8-891a47478dd022d49d8f159f99ba-0001": 10559, + "0c-e8-a4d4992a2778b23a1e9e21d75d7e-0001": 10560, + "0c-e8-aa86a8e693a481906eaa0143f683-0002": 10561, + "0c-e8-af55c7eac036080b1ab15bfd2650-0001": 10562, + "0c-e8-b0ba5dc65f4bb354426656d9c9b3-0001": 10563, + "0c-e8-b336f5b8e459f3d625af41747168-0002": 10564, + "0c-e8-c643216363ffc19b8fca9ca8190a-0001": 10565, + "0c-e8-e306f594e896eff34c1269dc1fc0-0002": 10566, + "0c-e8-f527477a1acf1ee14245d5324be9-0002": 10567, + "0c-e8-fdcd8a20bb3fc7f2bb03cb67166f-0003": 10568, + "0c-e8-ff04c30eaa43d06e87128c4e2aae-0001": 10569, + "0c-eb-0ab59c37d5ed6b13a354e9916d11-0001": 10570, + "0c-eb-0d990660f5695a4df259d9f881b2-0001": 10571, + "0c-eb-10abf0bb00664cee85f1b8add723-0004": 10572, + "0c-eb-21b94e85ebca5b28feb598479050-0001": 10573, + "0c-eb-2470019b360fc9b580342f37cc01-0001": 10574, + "0c-eb-35a5b056aaa9637fc61f94a36f7f-0002": 10575, + "0c-eb-3c7ed0935d486aafbff4fb9812d9-0001": 10576, + "0c-eb-4087be55578c472bd1fb275da609-0002": 10577, + "0c-eb-42b8606a1105e276c8e21b5c35c4-0001": 10578, + "0c-eb-447f482c8224c403e160bdda88f7-0002": 10579, + "0c-eb-45896ddcd4ace1ce129c6fb14aec-0002": 10580, + "0c-eb-675909119d6423429d9d9caf4a5b-0004": 10581, + "0c-eb-7876c254cb279e8db3a174a08843-0001": 10582, + "0c-eb-8180e77bda4f920bd1c375d21d8a-0001": 10583, + "0c-eb-90043034792e32a309f74245b045-0002": 10584, + "0c-eb-9768369d951e079c813c8e6f5af5-0001": 10585, + "0c-eb-99de056b93df690ef26e7627327f-0001": 10586, + "0c-eb-9f29b7b34cbc61d679ece47c1da9-0001": 10587, + "0c-eb-b2463b4eaefffda9b7a55aa0fe6a-0002": 10588, + "0c-eb-bba7c60ff84f802001f390b8cf42-0001": 10589, + "0c-eb-e76cb95c2354935616b15fd08929-0002": 10590, + "0c-ec-06f03bfacfd02f8de883251bfff6-0001": 10591, + "0c-ec-16b7a794a8161f7c0f17ab151bf3-0001": 10592, + "0c-ec-1a5d7f2b0a2d7c0e6d9ec91a62da-0001": 10593, + "0c-ec-311312af8bde59c373003cb944f5-0001": 10594, + "0c-ec-36d4ce91a66ef6e73a9d2ce2453c-0001": 10595, + "0c-ec-3c869894d0c72f04d0dd09af8de4-0001": 10596, + "0c-ec-3f670579a08a7e0b29548ab25521-0003": 10597, + "0c-ec-4d35291208e349f913f8ff419170-0001": 10598, + "0c-ec-4dc9485944f2f02c340cdc43c7ed-0001": 10599, + "0c-ec-64c394e2e913f834e59343365bb0-0001": 10600, + "0c-ec-6d2ccbf90da49d41fc3778a96c07-0001": 10601, + "0c-ec-6dbeb8414980112f20fd03a4a38a-0001": 10602, + "0c-ec-6f39d986d644adc6cf78ded2a851-0001": 10603, + "0c-ec-71ef1c15d0fa32b4168a3323a266-0001": 10604, + "0c-ec-8b7e79886b06d6ca94739fa880ef-0004": 10605, + "0c-ec-8e9e9e6260c401d964072021cf20-0001": 10606, + "0c-ec-95778e7c2ca98a9670a648834bf5-0001": 10607, + "0c-ec-99f729c58419f0d0b3c6dc07f85c-0001": 10608, + "0c-ec-b15f01177e0ba309cb11479b0519-0001": 10609, + "0c-ec-b779b6f9f7550c491174ce21e071-0002": 10610, + "0c-ec-bc1f910968ad496f731dfe92ab01-0002": 10611, + "0c-ec-cb28cffc3f94dbefbdafb28323dc-0001": 10612, + "0c-ec-cdf0c6c31299e923ab64f6bfb755-0001": 10613, + "0c-ec-f4e49f57e19a10c655154dfe83c1-0001": 10614, + "0c-ec-ff506424c5dc9b8cf39cede76fc2-0001": 10615, + "0c-ec-ff7d4d15bc787a32c64feaa0f3d9-0002": 10616, + "0c-ed-19a0db925b52a349203f836d80ff-0001": 10617, + "0c-ed-1eb3a1c1a5511e4625dda03d8d43-0001": 10618, + "0c-ed-21a37e6e3517f8d958c051240805-0002": 10619, + "0c-ed-38ccadf1a2d8f4d301879478853b-0002": 10620, + "0c-ed-405448c96b81ffe5b41224b8c7a8-0001": 10621, + "0c-ed-5579cee8cc5d8c06466dddd4c6d8-0002": 10622, + "0c-ed-58f586d7d20c1e7e1052c34c2461-0003": 10623, + "0c-ed-59e332c9207547d31c2808535756-0001": 10624, + "0c-ed-5deb6cf0be4783ed247be1dd1a24-0001": 10625, + "0c-ed-6a5085cd47f6b73e88d0af7b839f-0002": 10626, + "0c-ed-6fd7396b72fa0d0830df7797877d-0001": 10627, + "0c-ed-7443abb313d3063f4ecd680575a3-0001": 10628, + "0c-ed-79e523f134483bdd99282ad1954a-0001": 10629, + "0c-ed-8b20834425a0faddd6cefb0d3680-0001": 10630, + "0c-ed-b4433c247518f2acc319338ac34f-0001": 10631, + "0c-ed-ba99b21adb8ea44617ac13dadb8a-0001": 10632, + "0c-ed-c28b5f656f13dea075df98775c98-0001": 10633, + "0c-ed-c967956db9e3ef21e621421ceb7e-0002": 10634, + "0c-ed-cef0b43dd4b822be695d0f4d2e0b-0001": 10635, + "0c-ed-d026f63aa776367fa26b0b604c71-0001": 10636, + "0c-ed-d2b81dbf3d65a368840bd084df25-0001": 10637, + "0c-ed-dc8a8f2ea64c1db016074236c12a-0004": 10638, + "0c-ed-e239c7bf8cf625cb05c5dd7284f3-0002": 10639, + "0c-ed-e25e4b6c727daeccf69d128cc338-0002": 10640, + "0c-ed-e6b2a4c9bb114ce2c4ff286c3f4e-0001": 10641, + "0c-ed-e797b2991ffc4046245d20ac0c02-0002": 10642, + "0c-ee-15a10b079cafdd16afc09b41a37f-0002": 10643, + "0c-ee-1fbeea0826d6ddbd836feaffc37f-0001": 10644, + "0c-ee-20bca523c55c84646021257e1fd3-0001": 10645, + "0c-ee-42e42876f5a36f15f7172b545508-0002": 10646, + "0c-ee-4e31f4e1641bd9d889023c853335-0001": 10647, + "0c-ee-5748a953f22a49562bac6fb7223b-0002": 10648, + "0c-ee-5ba297b7b4180d7084f3d63eab26-0001": 10649, + "0c-ee-5fcef323665a17117fbf28b896bb-0001": 10650, + "0c-ee-6291130b08a21ab0de181698a921-0002": 10651, + "0c-ee-667956f4696d53b09ef984e10349-0002": 10652, + "0c-ee-7ed95f8726a188b87bf21fc316ca-0001": 10653, + "0c-ee-819708dbfa19a9e4eb87bb0d3d98-0001": 10654, + "0c-ee-9261e69bcea8c72293e353fd8192-0002": 10655, + "0c-ee-98e9e90621b399192c9f1c901a00-0001": 10656, + "0c-ee-9f805bc27b27a84612a8abdee96f-0003": 10657, + "0c-ee-a3ce1c9835aefbdba044e1ee0f9d-0001": 10658, + "0c-ee-ac1212caa13f14a5be5c6d60510b-0001": 10659, + "0c-ee-b4e3eb8260751c7c6141ef25bed9-0001": 10660, + "0c-ee-b78a043f519f644021e4475696f6-0004": 10661, + "0c-ee-bde4da6d966cc6aacb629825af29-0001": 10662, + "0c-ee-d28cff74b2ebc5015f904abb8482-0001": 10663, + "0c-ee-d67b1337cd0e723e3f68184d3634-0001": 10664, + "0c-ee-eb60852c79f2d569b4dddbd7a5e1-0001": 10665, + "0c-ee-ee24d32815d9cdefbc645c51f737-0001": 10666, + "0c-ee-f5738d25ad42c876a61eaf31d1bc-0003": 10667, + "0c-f1-029b42190c482cd7993ee8dbfec5-0001": 10668, + "0c-f1-038061afa766302813280555e36b-0001": 10669, + "0c-f1-08e6dc1c832a4453d9fe9d89f197-0001": 10670, + "0c-f1-0d1a2465fddf71c1b062fa9ed584-0001": 10671, + "0c-f1-2ad01c51652ede7a5e6c13ea6aac-0001": 10672, + "0c-f1-56a522bb2e4e849254ecf7c87f4d-0002": 10673, + "0c-f1-5a4d39855d9378a9698b0fbf8dfe-0001": 10674, + "0c-f1-5f925e6508362f9db9d1db308941-0002": 10675, + "0c-f1-60dd4d393df6914b18259d10d519-0001": 10676, + "0c-f1-63d08cbcac73fdb3ed50c7225bc4-0002": 10677, + "0c-f1-7b950e0f024ee1a29b206e5f5e2a-0001": 10678, + "0c-f1-858669b2745834fa8530876e4230-0002": 10679, + "0c-f1-8e864a8c6bdca79830ca69c29838-0002": 10680, + "0c-f1-90ad5e189504fd9494ba5170b2a1-0001": 10681, + "0c-f1-9cfc35f3c618a1362646d62ebacd-0001": 10682, + "0c-f1-a4ab9419dce1077313b595fb5730-0003": 10683, + "0c-f1-a6e1476c202a658e41ce86b776c3-0001": 10684, + "0c-f1-b22d976250a58999e4418e914ba6-0001": 10685, + "0c-f1-b3ee07fe6490b090d0e86d911ec1-0001": 10686, + "0c-f1-b5a1e938ef2b04be524208bea889-0001": 10687, + "0c-f1-cf3032135dc17fa691027ed0386c-0001": 10688, + "0c-f1-d78e1ff08dca059ab2dca4652ee3-0001": 10689, + "0c-f1-dfdfa54e5d06478aa0d0e7def1ff-0001": 10690, + "0c-f1-f5ac58d62302aafb0dcb288ce452-0001": 10691, + "0c-f2-071dbc5afb1a35a4962d87aefa71-0003": 10692, + "0c-f2-0c240f82385df470639d66a49c68-0001": 10693, + "0c-f2-188cca384f52ba8018eb877821b7-0001": 10694, + "0c-f2-22bfdfabbaee5fea52265f43c9e0-0002": 10695, + "0c-f2-3a1d6c7dbe2235415e4da11c3563-0001": 10696, + "0c-f2-3f6eeaa6b42449b465393402a0ab-0001": 10697, + "0c-f2-598e43966dd7a4bc978cf1f59722-0001": 10698, + "0c-f2-605c7dc3d30c546c7cc4a2a0f295-0001": 10699, + "0c-f2-6661b74d8157d452a0fe83252a03-0002": 10700, + "0c-f2-91d9bdcca482e0c505df23b559c8-0001": 10701, + "0c-f2-927fbfbd27e482a9dcda61bb802a-0003": 10702, + "0c-f2-94d949b3fb96ff760a9c5ad84db0-0004": 10703, + "0c-f2-a7602bd03b3004e6ea32d377922c-0002": 10704, + "0c-f2-ab1aa41990e22c45d84a2a2716e5-0001": 10705, + "0c-f2-ab58f20f8a4caa0f6680233d4d1f-0002": 10706, + "0c-f2-c94f68826d9057c1c482d7357d7b-0001": 10707, + "0c-f2-d041fcdbb67ac4f2d25c8390dea8-0001": 10708, + "0c-f2-ed640133bb56be39ad6f121f4235-0001": 10709, + "0c-f2-ee9a861f8247c82d4d1b8fe72bf5-0001": 10710, + "0c-f2-efb44c2431f2add67c952d71b0ba-0001": 10711, + "0c-f2-f435f4a73050e23188c3da18066c-0003": 10712, + "0c-f7-1557f6df6df8b565709a35f64068-0001": 10713, + "0c-f7-2c91ee36727daf77c870c3c12edf-0002": 10714, + "0c-f7-2cf9e26a02ee58e0ac686174a4f2-0001": 10715, + "0c-f7-3c277b55f84a7292838c7f4b12ce-0001": 10716, + "0c-f7-3d799fc8e356e6c13c8a56b1ff37-0001": 10717, + "0c-f7-4b39f546db9d83fdc9c00c8c7dea-0001": 10718, + "0c-f7-642a24380ed78aa8cee94de5352e-0001": 10719, + "0c-f7-65f735550c07a6c0ff1092a77d42-0001": 10720, + "0c-f7-6bbebee973a935d1ca17b95ae23e-0001": 10721, + "0c-f7-8eab9abb9ab5eacdedd6cf195f57-0001": 10722, + "0c-f7-93c18e7d8eb5fd57f1331e702f2e-0001": 10723, + "0c-f7-a3a586eb7710102969b269fdb27c-0003": 10724, + "0c-f7-b36aa8741c7a165ac4711f8fc0b0-0001": 10725, + "0c-f7-b3b9f67b18dcc0a6692040534edd-0001": 10726, + "0c-f7-ba3a920dc36991bb4d2af9e9dc0d-0003": 10727, + "0c-f7-c7970a0af1eafe847a7ad72cca4a-0002": 10728, + "0c-f7-d856def460ab58f633c70b87fd9c-0003": 10729, + "0c-f7-e24bd50632e3769381c643a31755-0001": 10730, + "0c-f7-e52b0ce1ecaff6a2d85acbe7e7d0-0001": 10731, + "0c-f7-e8accecc0f38636bffa656a24bef-0004": 10732, + "0c-fa-068a08942c625ad9a27f872db038-0002": 10733, + "0c-fa-0c5d55b4eb5fa86fa457f2431386-0002": 10734, + "0c-fa-13c2ce65ff141f7671c19c2922f4-0001": 10735, + "0c-fa-3340750fd07af4eb16a1d7b6a2d3-0001": 10736, + "0c-fa-4d5f59aa16b8d08ec8e54bd12e0f-0003": 10737, + "0c-fa-548ee542cd1f03149ae85661b0c1-0001": 10738, + "0c-fa-6a5cfbf495c9600940ffe95a171f-0002": 10739, + "0c-fa-826edef5115b74a2bb4a160bbc5c-0003": 10740, + "0c-fa-8af19e88044bfe6077c4301ada6c-0001": 10741, + "0c-fa-945868f8353e2014a95fbca9503d-0001": 10742, + "0c-fa-96cee1087e4540ae98d37d74196c-0002": 10743, + "0c-fa-9d296fa585d5e3cc3ae6b7a64807-0003": 10744, + "0c-fa-a7bf1edb3ebd867e2063b9a4bfa1-0001": 10745, + "0c-fa-c831e1d7a46bb56db3541a86e21a-0002": 10746, + "0c-fa-d1dc6ba133bb6a99a61f98784c7e-0001": 10747, + "0c-fa-e7fe44b5b64fa7e614061b2b274f-0002": 10748, + "0c-fa-f59b238faf32878dc37472a7261e-0001": 10749, + "0c-fb-076c366df0eea920031236121c1e-0001": 10750, + "0c-fb-148b23bbcc5b4118490604dbc674-0001": 10751, + "0c-fb-14d16919a8615696fc2927015217-0002": 10752, + "0c-fb-32e38b0a66219b9d32470e73516b-0001": 10753, + "0c-fb-33cfc4ec56c77550f0b2a5561dbb-0002": 10754, + "0c-fb-36033bb7df5e0d11dc6286fd17fb-0001": 10755, + "0c-fb-3833f90022e1344e0177ed4405a1-0001": 10756, + "0c-fb-3f16233b0801b4d5ae3ff6f3145b-0001": 10757, + "0c-fb-59e699291955df81ed34c6511d63-0001": 10758, + "0c-fb-84dc444797e0f9e0aa4a0b9e0b2f-0001": 10759, + "0c-fb-99d48efda783d2f8c2914f5d523f-0002": 10760, + "0c-fb-9bd59bec4dacaa152486ed2235a0-0002": 10761, + "0c-fb-a76b2e9be05dd613236880ed4552-0001": 10762, + "0c-fb-a93cdd56d7015db4cc388991aec3-0002": 10763, + "0c-fb-aae741db5a2ebf72537573b63ca7-0002": 10764, + "0c-fb-c790ba60e5c4aad22c740c563b9f-0001": 10765, + "0c-fb-d001c795b940a9024d998a85fabe-0001": 10766, + "0c-fb-d25800f762dafd4df47a575788ee-0001": 10767, + "0c-fb-d44b7e759d213368daef22990dd5-0001": 10768, + "0c-fb-d48477533fe1d6e6129330b8e006-0001": 10769, + "0c-fb-e5060a575677a5e5cafc4d57b475-0001": 10770, + "0c-fb-e7604686e1fbef5bd93658d4fff1-0002": 10771, + "0c-fd-076cb6d0b444eed77b63a848df0a-0002": 10772, + "0c-fd-11d7dd62cb2de02ed65dcd29e2b9-0002": 10773, + "0c-fd-1a6b540e9400056b78f0f68de98a-0001": 10774, + "0c-fd-207f202fe4ad7c2fb0ccfd6943e5-0001": 10775, + "0c-fd-2468c3b64e55554c5853ad0f4ab3-0001": 10776, + "0c-fd-2b29c2fe3afb6cfabe994ff42ba1-0002": 10777, + "0c-fd-2e4de56f689b580d92b1f8ec88c8-0001": 10778, + "0c-fd-2ebfdf2773b6ce80b4483b183c07-0003": 10779, + "0c-fd-346c39b0a3428e10ebb36b6735ba-0002": 10780, + "0c-fd-3b1a472fd2ca32f5c603967f5fe0-0001": 10781, + "0c-fd-3d2ed1a54a0ce41a018712662d8f-0001": 10782, + "0c-fd-46e1f5e13821b07883330f355218-0001": 10783, + "0c-fd-4b7ca601019f810e2c59d219940b-0001": 10784, + "0c-fd-4c18453502e4348995ba08eab1c1-0001": 10785, + "0c-fd-55627b1672fcfe398e5a7fc63371-0002": 10786, + "0c-fd-5f7da0f4eadb056c6e4c796cbc95-0001": 10787, + "0c-fd-702b9082ae890a62d0cc04bc23d6-0002": 10788, + "0c-fd-854f9c9f36c130cbf9c37b7fdf70-0003": 10789, + "0c-fd-8f19039a9c4d95afa148b752f546-0002": 10790, + "0c-fd-9708dd399acf632848ff0ab41162-0001": 10791, + "0c-fd-a5440b9cb084282e9066a51c77fb-0001": 10792, + "0c-fd-af97390a380d15e0083ecb8ce6c9-0001": 10793, + "0c-fd-b3ab99c73207f85a5871fc26c2bf-0001": 10794, + "0c-fd-f7a45b19989ffe781e2b18a55127-0002": 10795, + "0c-fd-fe9828dc272094bdbb56b4778ce2-0001": 10796, + "0c-fe-1032adbd9d2cf109826a11603d96-0002": 10797, + "0c-fe-1f10e0b258938db41738f9e3de5b-0002": 10798, + "0c-fe-2d543987fd130cb831a2c8eb068e-0001": 10799, + "0c-fe-4155158c10be059cfb9d30cf0767-0003": 10800, + "0c-fe-5a18c86817abd450e32a6d4e9caf-0002": 10801, + "0c-fe-ab37168b2a6c54c7bbaedac76f70-0001": 10802, + "0c-fe-abf150ba13c08e7ecb7f8ca0f599-0001": 10803, + "0c-fe-ba22aa2a8f12d441a33465443aa9-0002": 10804, + "0c-fe-ba5cfa4c3c25a469fe4b74f78fa9-0002": 10805, + "0c-fe-bc0bef4620268a2f26b082b487f3-0001": 10806, + "0c-fe-e79768019304658fdc4f576c2655-0002": 10807, + "0c-fe-edcada2d1bc11d8468b7888ae7f5-0004": 10808, + "0c-ff-311b3a7c43941f7527f9ff8f4ce1-0002": 10809, + "0c-ff-3803b830b26c930b338b89cd1f4d-0002": 10810, + "0c-ff-3e21f336072443f74a179bd60d1e-0001": 10811, + "0c-ff-406f2c3c9195e44ed66c88dc3c5e-0001": 10812, + "0c-ff-4b4e6cef1593d53ad34cb7c04bf9-0001": 10813, + "0c-ff-5151acfc7c35c6e3d91da922113d-0001": 10814, + "0c-ff-5c73d898650a414a8aebd51b8e94-0001": 10815, + "0c-ff-60a08c8b4c99a05845df8913b80f-0001": 10816, + "0c-ff-963cad8336bbd1d0e5d1a45d6846-0001": 10817, + "0c-ff-9c3fcb0933a91ab6a68ba5cc5b5a-0001": 10818, + "0c-ff-9e037534de14f213e7d80bf8c2ab-0001": 10819, + "0c-ff-9ee7c3c935582a30012cd86e4d11-0001": 10820, + "0c-ff-a74969a674984d30e741e7e1ba05-0004": 10821, + "0c-ff-c917ffb1a3d507023e50cb373533-0001": 10822, + "0c-ff-ca918e7870869138c0d331ae60e2-0002": 10823, + "0c-ff-ed28a0f2992946725cc63d3913ca-0001": 10824, +} diff --git a/data_preprocess/raster2graph/util/math_utils.py b/data_preprocess/raster2graph/util/math_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..559dd28f980f5e2e87a53376cf96d111bab0bcbd --- /dev/null +++ b/data_preprocess/raster2graph/util/math_utils.py @@ -0,0 +1,7 @@ +def clip(number, _min, _max): + if number <= _min: + return _min + elif number >= _max: + return _max + else: + return number diff --git a/data_preprocess/raster2graph/util/mean_std.py b/data_preprocess/raster2graph/util/mean_std.py new file mode 100644 index 0000000000000000000000000000000000000000..9c16d5d12bcb09f8a967a7c47da5dd506cf70dda --- /dev/null +++ b/data_preprocess/raster2graph/util/mean_std.py @@ -0,0 +1,2 @@ +mean = [0.920, 0.913, 0.891] +std = [0.214, 0.216, 0.228] diff --git a/data_preprocess/raster2graph/util/metric_utils.py b/data_preprocess/raster2graph/util/metric_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7962a508d26764254dd5f6758853c07004dbd801 --- /dev/null +++ b/data_preprocess/raster2graph/util/metric_utils.py @@ -0,0 +1,338 @@ +import copy +import math + +from shapely.geometry import Polygon +from util.geom_utils import poly_iou + + +def calculate_AP(valid_results, ground_truths, confidence_final): + ground_truths_copy = copy.deepcopy(ground_truths) + all_preds = [] + for image_id, image_pred in valid_results.items(): + for i in range(len(image_pred["points"])): + pred = {} + pred["score"] = image_pred["scores"][i].item() + pred["point"] = tuple(image_pred["points"][i].tolist()) + pred["size"] = tuple(image_pred["size"].tolist()) + pred["image_id"] = image_id.item() + all_preds.append(pred) + all_preds = sorted(all_preds, key=lambda x: x["score"], reverse=True) + + all_preds = [pred for pred in all_preds if pred["score"] > confidence_final] + + all_metrics = [] + for n in range(1, len(all_preds) + 1): + ground_truths = copy.deepcopy(ground_truths_copy) + + sub_preds = all_preds[0:n] + + TP = 0 + FP = 0 + FN = 0 + for pred in sub_preds: + pred_point = pred["point"] + img_size = (pred["size"][1], pred["size"][0]) + img_id = pred["image_id"] + dist_threshold = (img_size[0] * 0.01, img_size[1] * 0.01) + gt = [tuple(gt_point) for gt_point in ground_truths[img_id]["points"].tolist()] + gt_copy = copy.deepcopy(gt) + euc_dists = {} + dists = {} + for gt_point in gt_copy: + if gt_point[2] == 0: + dist = (abs(pred_point[0] - gt_point[0]), abs(pred_point[1] - gt_point[1])) + euc_dist = math.sqrt(dist[0] ** 2 + dist[1] ** 2) + euc_dists[gt_point] = euc_dist + dists[gt_point] = dist + euc_dists = sorted(euc_dists.items(), key=lambda x: x[1]) + if len(euc_dists) == 0: + FP += 1 + continue + nearest_gt_point = euc_dists[0][0] + min_dist = dists[nearest_gt_point] + if min_dist[0] < dist_threshold[0] and min_dist[1] < dist_threshold[1]: + gtip = ground_truths[img_id]["points"] + for i, p in enumerate(gtip): + if ( + p[0].item() == nearest_gt_point[0] + and p[1].item() == nearest_gt_point[1] + and p[2].item() == nearest_gt_point[2] + ): + # print('qqq', p, nearest_gt_point) + gtip[i, 2] = 1 + break + ground_truths[img_id]["points"] = gtip + # print('rrr', ground_truths[img_id]['points']) + TP += 1 + continue + FP += 1 + for img_id, points in ground_truths.items(): + points = points["points"] + for point in points: + if point[2] == 0: + FN += 1 + precision = TP / (TP + FP) + recall = TP / (TP + FN) + # print(n, TP, FP, FN, precision, recall) + all_metrics.append((precision, recall)) + + all_metrics = sorted(all_metrics, key=lambda x: (x[1], x[0])) + p_r_curve_points = {} + for point in all_metrics: + p_r_curve_points[point[1]] = point[0] + p_r_curve_points[0] = 1 + p_r_curve_points = sorted(p_r_curve_points.items(), key=lambda d: d[0]) + AP = 0 + for i, rp in enumerate(p_r_curve_points): + r = rp[0] + p = rp[1] + if i > 0: + small_rectangular_area = (r - p_r_curve_points[i - 1][0]) * p + AP += small_rectangular_area + + return AP + + +def get_results(best_result): + if 1: + preds = best_result[2] + output_points = [] + output_edges = [] + for triplet in preds: + this_preds = triplet[0] + last_edges = triplet[1] + this_edges = triplet[2] + for this_pred in this_preds: + point = tuple(this_pred["points"].int().tolist()) + output_points.append(point) + for last_edge in last_edges: + point1 = tuple(last_edge[0]["points"].int().tolist()) + point2 = tuple(last_edge[1]["points"].int().tolist()) + edge = (point1, point2) + output_edges.append(edge) + for this_edge in this_edges: + point1 = tuple(this_edge[0]["points"].int().tolist()) + point2 = tuple(this_edge[1]["points"].int().tolist()) + edge = (point1, point2) + output_edges.append(edge) + return output_points, output_edges + + +def get_results_visual(best_result): + if 1: + preds = best_result[2] + output_points = [] + output_edges = [] + for layer_index, triplet in enumerate(preds): + this_preds = triplet[0] + last_edges = triplet[1] + this_edges = triplet[2] + for this_pred in this_preds: + point = tuple(this_pred["points"].int().tolist()) + output_points.append([layer_index, point]) + for last_edge in last_edges: + point1 = tuple(last_edge[0]["points"].int().tolist()) + point2 = tuple(last_edge[1]["points"].int().tolist()) + edge = (point1, point2) + output_edges.append([layer_index, edge]) + for this_edge in this_edges: + point1 = tuple(this_edge[0]["points"].int().tolist()) + point2 = tuple(this_edge[1]["points"].int().tolist()) + edge = (point1, point2) + output_edges.append([layer_index, edge]) + return output_points, output_edges, len(preds) + + +def get_results_float_with_semantic(best_result): + preds = best_result[2] + output_points = [] + output_edges = [] + for triplet in preds: + this_preds = triplet[0] + last_edges = triplet[1] + this_edges = triplet[2] + for this_pred in this_preds: + point = ( + this_pred["points"].tolist()[0], + this_pred["points"].tolist()[1], + this_pred["semantic_left_up"].item(), + this_pred["semantic_right_up"].item(), + this_pred["semantic_right_down"].item(), + this_pred["semantic_left_down"].item(), + ) + output_points.append(point) + for last_edge in last_edges: + point1 = ( + last_edge[0]["points"].tolist()[0], + last_edge[0]["points"].tolist()[1], + last_edge[0]["semantic_left_up"].item(), + last_edge[0]["semantic_right_up"].item(), + last_edge[0]["semantic_right_down"].item(), + last_edge[0]["semantic_left_down"].item(), + ) + point2 = ( + last_edge[1]["points"].tolist()[0], + last_edge[1]["points"].tolist()[1], + last_edge[1]["semantic_left_up"].item(), + last_edge[1]["semantic_right_up"].item(), + last_edge[1]["semantic_right_down"].item(), + last_edge[1]["semantic_left_down"].item(), + ) + edge = (point1, point2) + output_edges.append(edge) + for this_edge in this_edges: + point1 = ( + this_edge[0]["points"].tolist()[0], + this_edge[0]["points"].tolist()[1], + this_edge[0]["semantic_left_up"].item(), + this_edge[0]["semantic_right_up"].item(), + this_edge[0]["semantic_right_down"].item(), + this_edge[0]["semantic_left_down"].item(), + ) + point2 = ( + this_edge[1]["points"].tolist()[0], + this_edge[1]["points"].tolist()[1], + this_edge[1]["semantic_left_up"].item(), + this_edge[1]["semantic_right_up"].item(), + this_edge[1]["semantic_right_down"].item(), + this_edge[1]["semantic_left_down"].item(), + ) + edge = (point1, point2) + output_edges.append(edge) + + return output_points, output_edges + + +def calculate_single_sample( + best_result, graph, target_d_rev, target_simple_cycles, target_results, d_rev, simple_cycles, results +): + output_points, output_edges = get_results(best_result) + gt_points = [k for k, v in graph.items()] + gt_edges = [] + for k, v in graph.items(): + for adj in v: + if adj != (-1, -1): + gt_edge = (k, adj) + if (adj, k) not in gt_edges: + gt_edges.append(gt_edge) + + points_TP = 0 + points_FP = 0 + points_FN = 0 + dist_error_x = 0 + dist_error_y = 0 + dist_error_l2 = 0 + gt_points_copy = copy.deepcopy(gt_points) + threshold = 5 + for output_point in output_points: + matched = False + for gt_point in gt_points: + if (abs(output_point[0] - gt_point[0]) <= threshold) and (abs(output_point[1] - gt_point[1]) <= threshold): + if gt_point in gt_points_copy: + points_TP += 1 + dist_error_x += abs(output_point[0] - gt_point[0]) + dist_error_y += abs(output_point[1] - gt_point[1]) + dist_error_l2 += ( + abs(output_point[0] - gt_point[0]) ** 2 + abs(output_point[1] - gt_point[1]) ** 2 + ) ** 0.5 + matched = True + gt_points_copy.remove(gt_point) + break + if not matched: + points_FP += 1 + points_FN = len(gt_points) - points_TP + + edges_TP = 0 + edges_FP = 0 + edges_FN = 0 + gt_edges_copy = copy.deepcopy(gt_edges) + threshold = 5 + for output_edge in output_edges: + matched = False + for gt_edge in gt_edges: + if ( + ( + (abs(output_edge[0][0] - gt_edge[0][0]) <= threshold) + and (abs(output_edge[0][1] - gt_edge[0][1]) <= threshold) + ) + and ( + (abs(output_edge[1][0] - gt_edge[1][0]) <= threshold) + and (abs(output_edge[1][1] - gt_edge[1][1]) <= threshold) + ) + ) or ( + ( + (abs(output_edge[0][0] - gt_edge[1][0]) <= threshold) + and (abs(output_edge[0][1] - gt_edge[1][1]) <= threshold) + ) + and ( + (abs(output_edge[1][0] - gt_edge[0][0]) <= threshold) + and (abs(output_edge[1][1] - gt_edge[0][1]) <= threshold) + ) + ): + if gt_edge in gt_edges_copy: + edges_TP += 1 + matched = True + gt_edges_copy.remove(gt_edge) + break + if not matched: + edges_FP += 1 + edges_FN = len(gt_edges) - edges_TP + + regions_TP = 0 + regions_FP = 0 + regions_FN = 0 + rooms_TP = 0 + rooms_FP = 0 + rooms_FN = 0 + gt_regions = [] + output_regions = [] + + for target_simple_cycle in target_simple_cycles: + target_polyg = [(point_i[0], point_i[1]) for point_i in target_simple_cycle] + gt_regions.append(target_polyg) + + for simple_cycle in simple_cycles: + polyg = [(point_i[0], point_i[1]) for point_i in simple_cycle] + polyg.pop(-1) + output_regions.append(polyg) + gt_regions_copy = copy.deepcopy(gt_regions) + iou_threshold = 0.7 + for output_region_i, output_region in enumerate(output_regions): + matched = False + for gt_region_i, gt_region in enumerate(gt_regions): + if poly_iou(Polygon(gt_region), Polygon(output_region)) >= iou_threshold: + if gt_region in gt_regions_copy: + regions_TP += 1 + if target_results[gt_region_i] == results[output_region_i]: + rooms_TP += 1 + else: + rooms_FP += 1 + matched = True + gt_regions_copy.remove(gt_region) + break + if not matched: + regions_FP += 1 + rooms_FP += 1 + regions_FN = len(gt_regions) - regions_TP + rooms_FN = len(gt_regions) - rooms_TP + # print(regions_TP, regions_FP, regions_FN) + # print(rooms_TP, rooms_FP, rooms_FN) + + dist_error = (0, 0, 0) + if points_TP > 0: + dist_error = (dist_error_x, dist_error_y, dist_error_l2) + return ( + points_TP, + points_FP, + points_FN, + edges_TP, + edges_FP, + edges_FN, + dist_error, + regions_TP, + regions_FP, + regions_FN, + rooms_TP, + rooms_FP, + rooms_FN, + ) diff --git a/data_preprocess/raster2graph/util/semantics_dict.py b/data_preprocess/raster2graph/util/semantics_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..ab9e4325d54d763c583aafff0ff62105555bd5f7 --- /dev/null +++ b/data_preprocess/raster2graph/util/semantics_dict.py @@ -0,0 +1,45 @@ +semantics_dict = { + "living_room": 1, + "kitchen": 2, + "bedroom": 3, + "bathroom": 4, + "restroom": 5, + "balcony": 6, + "closet": 7, + "corridor": 8, + "washing_room": 9, + "PS": 10, + "outside": 11, + "wall": 12, + "no_type": 0, +} +semantics_dict_rev = { + 0: "no_type", + 1: "living_room", + 2: "kitchen", + 3: "bedroom", + 4: "bathroom", + 5: "restroom", + 6: "balcony", + 7: "closet", + 8: "corridor", + 9: "washing_room", + 10: "PS", + 11: "outside", + 12: "wall", +} +semantics_dict_color = { + "living_room": (0, 0, 220), + "kitchen": (0, 220, 220), + "bedroom": (0, 220, 0), + "bathroom": (220, 220, 0), + "restroom": (220, 0, 0), + "balcony": (220, 0, 220), + "closet": (110, 0, 110), + "corridor": (110, 0, 0), + "washing_room": (0, 0, 110), + "PS": (0, 110, 110), + "outside": (0, 0, 0), + "wall": (110, 110, 110), + "no_type": (20, 20, 20), +} diff --git a/data_preprocess/stru3d/PointCloudReaderPanorama.py b/data_preprocess/stru3d/PointCloudReaderPanorama.py new file mode 100644 index 0000000000000000000000000000000000000000..c8a208e41c2e92f24666480c4be44da1091f6572 --- /dev/null +++ b/data_preprocess/stru3d/PointCloudReaderPanorama.py @@ -0,0 +1,253 @@ +import os + +import cv2 +import numpy as np +import open3d as o3d + +NUM_SECTIONS = -1 + + +class PointCloudReaderPanorama: + def __init__(self, path, resolution="full", random_level=0, generate_color=False, generate_normal=False): + self.path = path + self.random_level = random_level + self.resolution = resolution + self.generate_color = generate_color + self.generate_normal = generate_normal + sections = [p for p in os.listdir(os.path.join(path, "2D_rendering"))] + self.depth_paths = [ + os.path.join(*[path, "2D_rendering", p, "panorama", self.resolution, "depth.png"]) for p in sections + ] + self.rgb_paths = [ + os.path.join(*[path, "2D_rendering", p, "panorama", self.resolution, "rgb_coldlight.png"]) + for p in sections + ] + self.normal_paths = [ + os.path.join(*[path, "2D_rendering", p, "panorama", self.resolution, "normal.png"]) for p in sections + ] + self.camera_paths = [os.path.join(*[path, "2D_rendering", p, "panorama", "camera_xyz.txt"]) for p in sections] + self.camera_centers = self.read_camera_center() + self.point_cloud = self.generate_point_cloud( + self.random_level, color=self.generate_color, normal=self.generate_normal + ) + + def read_camera_center(self): + camera_centers = [] + for i in range(len(self.camera_paths)): + with open(self.camera_paths[i], "r") as f: + line = f.readline() + center = list(map(float, line.strip().split(" "))) + camera_centers.append(np.asarray([center[0], center[1], center[2]])) + return camera_centers + + def generate_point_cloud(self, random_level=0, color=False, normal=False): + coords = [] + colors = [] + points = {} + # normals = [] + + # Getting Coordinates + for i in range(len(self.depth_paths)): + depth_img = cv2.imread(self.depth_paths[i], cv2.IMREAD_ANYDEPTH | cv2.IMREAD_ANYCOLOR) + x_tick = 180.0 / depth_img.shape[0] + y_tick = 360.0 / depth_img.shape[1] + + rgb_img = cv2.imread(self.rgb_paths[i]) + rgb_img = cv2.cvtColor(rgb_img, code=cv2.COLOR_BGR2RGB) + # normal_img = cv2.imread(self.normal_paths[i]) + + for x in range(0, depth_img.shape[0]): + for y in range(0, depth_img.shape[1]): + # need 90 - -09 + alpha = 90 - (x * x_tick) + beta = y * y_tick - 180 + + depth = depth_img[x, y] + np.random.random() * random_level + + if depth > 500.0: + z_offset = depth * np.sin(np.deg2rad(alpha)) + xy_offset = depth * np.cos(np.deg2rad(alpha)) + x_offset = xy_offset * np.sin(np.deg2rad(beta)) + y_offset = xy_offset * np.cos(np.deg2rad(beta)) + point = np.asarray([x_offset, y_offset, z_offset]) + coords.append(point + self.camera_centers[i]) + colors.append(rgb_img[x, y]) + # normals.append(normalize(normal_img[x, y].reshape(-1, 1)).ravel()) + + coords = np.asarray(coords) + colors = np.asarray(colors) / 255.0 + # normals = np.asarray(normals) + + coords[:, :2] = np.round(coords[:, :2] / 10) * 10.0 + coords[:, 2] = np.round(coords[:, 2] / 100) * 100.0 + unique_coords, unique_ind = np.unique(coords, return_index=True, axis=0) + + coords = coords[unique_ind] + colors = colors[unique_ind] + # normals = normals[unique_ind] + + points["coords"] = coords + points["colors"] = colors + # points['normals'] = normals + + print("Pointcloud size:", points["coords"].shape[0]) + return points + + def get_point_cloud(self): + return self.point_cloud + + def generate_density(self, width=256, height=256): + + ps = self.point_cloud["coords"] * -1 + ps[:, 0] *= -1 + ps[:, 1] *= -1 + + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(ps) + pcd.estimate_normals() + + # zs = np.round(ps[:,2] / 100) * 100 + # zs, zs_ind = np.unique(zs, return_index=True, axis=0) + # ps_ind = ps[:, :2] == + # print("Generate density...") + + image_res = np.array((width, height)) + + max_coords = np.max(ps, axis=0) + min_coords = np.min(ps, axis=0) + max_m_min = max_coords - min_coords + + max_coords = max_coords + 0.1 * max_m_min + min_coords = min_coords - 0.1 * max_m_min + + normalization_dict = {} + normalization_dict["min_coords"] = min_coords + normalization_dict["max_coords"] = max_coords + normalization_dict["image_res"] = image_res + + # coordinates = np.round(points[:, :2] / max_coordinates[None,:2] * image_res[None]) + coordinates = np.round( + (ps[:, :2] - min_coords[None, :2]) / (max_coords[None, :2] - min_coords[None, :2]) * image_res[None] + ) + coordinates = np.minimum(np.maximum(coordinates, np.zeros_like(image_res)), image_res - 1) + + density = np.zeros((height, width), dtype=np.float32) + + unique_coordinates, counts = np.unique(coordinates, return_counts=True, axis=0) + # print(np.unique(counts)) + # counts = np.minimum(counts, 1e2) + + unique_coordinates = unique_coordinates.astype(np.int32) + + density[unique_coordinates[:, 1], unique_coordinates[:, 0]] = counts + density = density / np.max(density) + # print(np.unique(density)) + + normals = np.array(pcd.normals) + normals_map = np.zeros((density.shape[0], density.shape[1], 3)) + + import time + + start_time = time.time() + for i, unique_coord in enumerate(unique_coordinates): + # print(normals[unique_ind]) + normals_indcs = np.argwhere(np.all(coordinates[::10] == unique_coord, axis=1))[:, 0] + normals_map[unique_coordinates[i, 1], unique_coordinates[i, 0], :] = np.mean( + normals[::10][normals_indcs, :], axis=0 + ) + + print("Time for normals: ", time.time() - start_time) + + normals_map = (np.clip(normals_map, 0, 1) * 255).astype(np.uint8) + + # plt.figure() + # plt.imshow(normals_map) + # plt.show() + + return density, normals_map, normalization_dict + + def visualize(self, export_path=None): + pcd = o3d.geometry.PointCloud() + + points = self.point_cloud["coords"] + + print(np.max(points, axis=0)) + indices = np.where(points[:, 2] < 2000) + + points = points[indices] + points[:, 1] *= -1 + points[:, :] /= 1000 + pcd.points = o3d.utility.Vector3dVector(points) + + if self.generate_normal: + normals = self.point_cloud["normals"] + normals = normals[indices] + pcd.normals = o3d.utility.Vector3dVector(normals) + if self.generate_color: + colors = self.point_cloud["colors"] + colors = colors[indices] + pcd.colors = o3d.utility.Vector3dVector(colors) + + # wireframe_geo_list = visualize_wireframe(annos, vis=False, ret=True) + # o3d.visualization.draw_geometries([pcd] + wireframe_geo_list) + # o3d.visualization.draw_geometries([pcd]) + + pcd.estimate_normals() + + # radii = 0.01 + # mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting(pcd, radii) + + # alpha = 0.1 + # tetra_mesh, pt_map = o3d.geometry.TetraMesh.create_from_point_cloud(pcd) + # mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_alpha_shape(pcd, alpha, tetra_mesh, pt_map) + + o3d.visualization.draw_geometries([pcd]) + + if export_path is not None: + o3d.io.write_point_cloud(export_path, pcd) + + # o3d.visualization.draw_geometries([pcd]) + + def export_ply(self, path): + """ + ply + format ascii 1.0 + comment Mars model by Paul Bourke + element vertex 259200 + property float x + property float y + property float z + property uchar r + property uchar g + property uchar b + property float nx + property float ny + property float nz + end_header + """ + with open(path, "w") as f: + f.write("ply\n") + f.write("format ascii 1.0\n") + f.write("element vertex %d\n" % self.point_cloud["coords"].shape[0]) + f.write("property float x\n") + f.write("property float y\n") + f.write("property float z\n") + if self.generate_color: + f.write("property uchar red\n") + f.write("property uchar green\n") + f.write("property uchar blue\n") + if self.generate_normal: + f.write("property float nx\n") + f.write("property float ny\n") + f.write("property float nz\n") + f.write("end_header\n") + for i in range(self.point_cloud["coords"].shape[0]): + normal = [] + color = [] + coord = self.point_cloud["coords"][i].tolist() + if self.generate_color: + color = list(map(int, (self.point_cloud["colors"][i] * 255).tolist())) + if self.generate_normal: + normal = self.point_cloud["normals"][i].tolist() + data = coord + color + normal + f.write(" ".join(list(map(str, data))) + "\n") diff --git a/data_preprocess/stru3d/generate_coco_stru3d.py b/data_preprocess/stru3d/generate_coco_stru3d.py new file mode 100644 index 0000000000000000000000000000000000000000..67058bae644c5223150e462ea75e213dad18ead7 --- /dev/null +++ b/data_preprocess/stru3d/generate_coco_stru3d.py @@ -0,0 +1,199 @@ +import argparse +import json +import os +import sys + +from stru3d_utils import generate_coco_dict, generate_density, normalize_annotations, parse_floor_plan_polys +from tqdm import tqdm + +sys.path.append("../.") +from common_utils import export_density, read_scene_pc + +### Note: Some scenes have missing/wrong annotations. These are the indices that you should additionally exclude +### to be consistent with MonteFloor and HEAT: +invalid_scenes_ids = [ + 76, + 183, + 335, + 491, + 663, + 681, + 703, + 728, + 865, + 936, + 985, + 986, + 1009, + 1104, + 1155, + 1221, + 1282, + 1365, + 1378, + 1635, + 1745, + 1772, + 1774, + 1816, + 1866, + 2037, + 2076, + 2274, + 2334, + 2357, + 2580, + 2665, + 2706, + 2713, + 2771, + 2868, + 3156, + 3192, + 3198, + 3261, + 3271, + 3276, + 3296, + 3342, + 3387, + 3398, + 3466, + 3496, +] + +type2id = { + "living room": 0, + "kitchen": 1, + "bedroom": 2, + "bathroom": 3, + "balcony": 4, + "corridor": 5, + "dining room": 6, + "study": 7, + "studio": 8, + "store room": 9, + "garden": 10, + "laundry room": 11, + "office": 12, + "basement": 13, + "garage": 14, + "undefined": 15, + "door": 16, + "window": 17, +} + + +def config(): + a = argparse.ArgumentParser(description="Generate coco format data for Structured3D") + a.add_argument( + "--data_root", default="Structured3D_panorama", type=str, help="path to raw Structured3D_panorama folder" + ) + a.add_argument("--output", default="coco_stru3d", type=str, help="path to output folder") + + args = a.parse_args() + return args + + +def main(args): + data_root = args.data_root + data_parts = os.listdir(data_root) + + ### prepare + outFolder = args.output + if not os.path.exists(outFolder): + os.mkdir(outFolder) + + annotation_outFolder = os.path.join(outFolder, "annotations") + if not os.path.exists(annotation_outFolder): + os.mkdir(annotation_outFolder) + + train_img_folder = os.path.join(outFolder, "train") + val_img_folder = os.path.join(outFolder, "val") + test_img_folder = os.path.join(outFolder, "test") + + for img_folder in [train_img_folder, val_img_folder, test_img_folder]: + if not os.path.exists(img_folder): + os.mkdir(img_folder) + + coco_train_json_path = os.path.join(annotation_outFolder, "train.json") + coco_val_json_path = os.path.join(annotation_outFolder, "val.json") + coco_test_json_path = os.path.join(annotation_outFolder, "test.json") + + coco_train_dict = {"images": [], "annotations": [], "categories": []} + coco_val_dict = {"images": [], "annotations": [], "categories": []} + coco_test_dict = {"images": [], "annotations": [], "categories": []} + + for key, value in type2id.items(): + type_dict = {"supercategory": "room", "id": value, "name": key} + coco_train_dict["categories"].append(type_dict) + coco_val_dict["categories"].append(type_dict) + coco_test_dict["categories"].append(type_dict) + + ### begin processing + instance_id = 0 + for part in tqdm(data_parts): + scenes = os.listdir(os.path.join(data_root, part, "Structured3D")) + for scene in tqdm(scenes): + scene_path = os.path.join(data_root, part, "Structured3D", scene) + scene_id = scene.split("_")[-1] + + if int(scene_id) in invalid_scenes_ids: + print("skip {}".format(scene)) + continue + + # load pre-generated point cloud + ply_path = os.path.join(scene_path, "point_cloud.ply") + points = read_scene_pc(ply_path) + xyz = points[:, :3] + + ### project point cloud to density map + density, normalization_dict = generate_density(xyz, width=256, height=256) + + ### rescale raw annotations + normalized_annos = normalize_annotations(scene_path, normalization_dict) + + ### prepare coco dict + img_id = int(scene_id) + img_dict = {} + img_dict["file_name"] = scene_id + ".png" + img_dict["id"] = img_id + img_dict["width"] = 256 + img_dict["height"] = 256 + + ### parse annotations + polys = parse_floor_plan_polys(normalized_annos) + polygons_list = generate_coco_dict(normalized_annos, polys, instance_id, img_id, ignore_types=["outwall"]) + + instance_id += len(polygons_list) + + ### train + if int(scene_id) < 3000: + coco_train_dict["images"].append(img_dict) + coco_train_dict["annotations"] += polygons_list + export_density(density, train_img_folder, scene_id) + + ### val + elif int(scene_id) >= 3000 and int(scene_id) < 3250: + coco_val_dict["images"].append(img_dict) + coco_val_dict["annotations"] += polygons_list + export_density(density, val_img_folder, scene_id) + + ### test + else: + coco_test_dict["images"].append(img_dict) + coco_test_dict["annotations"] += polygons_list + export_density(density, test_img_folder, scene_id) + + print(scene_id) + + with open(coco_train_json_path, "w") as f: + json.dump(coco_train_dict, f) + with open(coco_val_json_path, "w") as f: + json.dump(coco_val_dict, f) + with open(coco_test_json_path, "w") as f: + json.dump(coco_test_dict, f) + + +if __name__ == "__main__": + main(config()) diff --git a/data_preprocess/stru3d/generate_point_cloud_stru3d.py b/data_preprocess/stru3d/generate_point_cloud_stru3d.py new file mode 100644 index 0000000000000000000000000000000000000000..7cf223cf73196b304f355a4dd3787ae097c60398 --- /dev/null +++ b/data_preprocess/stru3d/generate_point_cloud_stru3d.py @@ -0,0 +1,32 @@ +import argparse +import os + +from PointCloudReaderPanorama import PointCloudReaderPanorama +from tqdm import tqdm + + +def config(): + a = argparse.ArgumentParser(description="Generate point cloud for Structured3D") + a.add_argument( + "--data_root", default="Structured3D_panorama", type=str, help="path to raw Structured3D_panorama folder" + ) + args = a.parse_args() + return args + + +def main(args): + print("Creating point cloud from perspective views...") + data_root = args.data_root + data_parts = os.listdir(data_root) + + for part in tqdm(data_parts): + scenes = os.listdir(os.path.join(data_root, part, "Structured3D")) + for scene in tqdm(scenes): + scene_path = os.path.join(data_root, part, "Structured3D", scene) + reader = PointCloudReaderPanorama(scene_path, random_level=0, generate_color=True, generate_normal=False) + save_path = os.path.join(data_root, part, "Structured3D", scene, "point_cloud.ply") + reader.export_ply(save_path) + + +if __name__ == "__main__": + main(config()) diff --git a/data_preprocess/stru3d/stru3d_utils.py b/data_preprocess/stru3d/stru3d_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2e45ffc04e2d7141dd377037f8209a23b3e63854 --- /dev/null +++ b/data_preprocess/stru3d/stru3d_utils.py @@ -0,0 +1,244 @@ +""" +This code is an adaptation that uses Structured 3D for the code base. + +Reference: https://github.com/bertjiazheng/Structured3D +""" + +import json +import os +import sys + +import numpy as np +from shapely.geometry import Polygon + +sys.path.append("../data_preprocess") +from common_utils import resort_corners + +type2id = { + "living room": 0, + "kitchen": 1, + "bedroom": 2, + "bathroom": 3, + "balcony": 4, + "corridor": 5, + "dining room": 6, + "study": 7, + "studio": 8, + "store room": 9, + "garden": 10, + "laundry room": 11, + "office": 12, + "basement": 13, + "garage": 14, + "undefined": 15, + "door": 16, + "window": 17, +} + + +def generate_density(point_cloud, width=256, height=256): + + ps = point_cloud * -1 + ps[:, 0] *= -1 + ps[:, 1] *= -1 + + image_res = np.array((width, height)) + + max_coords = np.max(ps, axis=0) + min_coords = np.min(ps, axis=0) + max_m_min = max_coords - min_coords + + max_coords = max_coords + 0.1 * max_m_min + min_coords = min_coords - 0.1 * max_m_min + + normalization_dict = {} + normalization_dict["min_coords"] = min_coords + normalization_dict["max_coords"] = max_coords + normalization_dict["image_res"] = image_res + + # coordinates = np.round(points[:, :2] / max_coordinates[None,:2] * image_res[None]) + coordinates = np.round( + (ps[:, :2] - min_coords[None, :2]) / (max_coords[None, :2] - min_coords[None, :2]) * image_res[None] + ) + coordinates = np.minimum(np.maximum(coordinates, np.zeros_like(image_res)), image_res - 1) + + density = np.zeros((height, width), dtype=np.float32) + + unique_coordinates, counts = np.unique(coordinates, return_counts=True, axis=0) + # print(np.unique(counts)) + # counts = np.minimum(counts, 1e2) + + unique_coordinates = unique_coordinates.astype(np.int32) + + density[unique_coordinates[:, 1], unique_coordinates[:, 0]] = counts + density = density / np.max(density) + + return density, normalization_dict + + +def normalize_point(point, normalization_dict): + + min_coords = normalization_dict["min_coords"] + max_coords = normalization_dict["max_coords"] + image_res = normalization_dict["image_res"] + + point_2d = np.round((point[:2] - min_coords[:2]) / (max_coords[:2] - min_coords[:2]) * image_res) + point_2d = np.minimum(np.maximum(point_2d, np.zeros_like(image_res)), image_res - 1) + + point[:2] = point_2d.tolist() + + return point + + +def normalize_annotations(scene_path, normalization_dict): + annotation_path = os.path.join(scene_path, "annotation_3d.json") + with open(annotation_path, "r") as f: + annotation_json = json.load(f) + + for line in annotation_json["lines"]: + point = line["point"] + point = normalize_point(point, normalization_dict) + line["point"] = point + + for junction in annotation_json["junctions"]: + point = junction["coordinate"] + point = normalize_point(point, normalization_dict) + junction["coordinate"] = point + + return annotation_json + + +def parse_floor_plan_polys(annos): + planes = [] + for semantic in annos["semantics"]: + for planeID in semantic["planeID"]: + if annos["planes"][planeID]["type"] == "floor": + planes.append({"planeID": planeID, "type": semantic["type"]}) + + # if semantic["type"] == "outwall": + # outerwall_planes = semantic["planeID"] + + # extract hole vertices + lines_holes = [] + for semantic in annos["semantics"]: + if semantic["type"] in ["window", "door"]: + for planeID in semantic["planeID"]: + lines_holes.extend(np.where(np.array(annos["planeLineMatrix"][planeID]))[0].tolist()) + lines_holes = np.unique(lines_holes) + + ## junctions on the floor + # junctions = np.array([junc["coordinate"] for junc in annos["junctions"]]) + + # construct each polygon + polygons = [] + for plane in planes: + lineIDs = np.where(np.array(annos["planeLineMatrix"][plane["planeID"]]))[0].tolist() + junction_pairs = [np.where(np.array(annos["lineJunctionMatrix"][lineID]))[0].tolist() for lineID in lineIDs] + polygon = convert_lines_to_vertices(junction_pairs) + polygons.append([polygon[0], plane["type"]]) + + return polygons + + +def convert_lines_to_vertices(lines): + """ + convert line representation to polygon vertices + + """ + polygons = [] + lines = np.array(lines) + + polygon = None + while len(lines) != 0: + if polygon is None: + polygon = lines[0].tolist() + lines = np.delete(lines, 0, 0) + + lineID, juncID = np.where(lines == polygon[-1]) + vertex = lines[lineID[0], 1 - juncID[0]] + lines = np.delete(lines, lineID, 0) + + if vertex in polygon: + polygons.append(polygon) + polygon = None + else: + polygon.append(vertex) + + return polygons + + +def generate_coco_dict(annos, polygons, curr_instance_id, curr_img_id, ignore_types): + + junctions = np.array([junc["coordinate"][:2] for junc in annos["junctions"]]) + + coco_annotation_dict_list = [] + + for poly_ind, (polygon, poly_type) in enumerate(polygons): + if poly_type in ignore_types: + continue + + polygon = junctions[np.array(polygon)] + + poly_shapely = Polygon(polygon) + area = poly_shapely.area + + # assert area > 10 + # if area < 100: + if poly_type not in ["door", "window"] and area < 100: + continue + if poly_type in ["door", "window"] and area < 1: + continue + + rectangle_shapely = poly_shapely.envelope + + ### here we convert door/window annotation into a single line + if poly_type in ["door", "window"]: + assert polygon.shape[0] == 4 + midp_1 = (polygon[0] + polygon[1]) / 2 + midp_2 = (polygon[1] + polygon[2]) / 2 + midp_3 = (polygon[2] + polygon[3]) / 2 + midp_4 = (polygon[3] + polygon[0]) / 2 + + dist_1_3 = np.square(midp_1 - midp_3).sum() + dist_2_4 = np.square(midp_2 - midp_4).sum() + if dist_1_3 > dist_2_4: + polygon = np.row_stack([midp_1, midp_3]) + else: + polygon = np.row_stack([midp_2, midp_4]) + + coco_seg_poly = [] + poly_sorted = resort_corners(polygon) + + for p in poly_sorted: + coco_seg_poly += list(p) + + # Slightly wider bounding box + bound_pad = 2 + bb_x, bb_y = rectangle_shapely.exterior.xy + bb_x = np.unique(bb_x) + bb_y = np.unique(bb_y) + bb_x_min = np.maximum(np.min(bb_x) - bound_pad, 0) + bb_y_min = np.maximum(np.min(bb_y) - bound_pad, 0) + + bb_x_max = np.minimum(np.max(bb_x) + bound_pad, 256 - 1) + bb_y_max = np.minimum(np.max(bb_y) + bound_pad, 256 - 1) + + bb_width = bb_x_max - bb_x_min + bb_height = bb_y_max - bb_y_min + + coco_bb = [bb_x_min, bb_y_min, bb_width, bb_height] + + coco_annotation_dict = { + "segmentation": [coco_seg_poly], + "area": area, + "iscrowd": 0, + "image_id": curr_img_id, + "bbox": coco_bb, + "category_id": type2id[poly_type], + "id": curr_instance_id, + } + + coco_annotation_dict_list.append(coco_annotation_dict) + curr_instance_id += 1 + + return coco_annotation_dict_list diff --git a/data_preprocess/tools/plot_data.sh b/data_preprocess/tools/plot_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..241e3cab795523dbd57128ae3cd7ab49514fe3ce --- /dev/null +++ b/data_preprocess/tools/plot_data.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# Additional useful arguments: +# --crop_white_space: remove redundant whitespace from the rendering +# --one_color: use single color for every room (i.e. yellow) +# --compute_stats: compute statistics of the dataset (e.g. max_num_pts, max_num_polys) +# and plot histogram for counting number of Points, Rooms, Corners +# --drop_wd: disable Windor & Door in the plots +# --image_scale: adjust rendering resolution of the plots + +SPLIT=test +python plot_floor.py --dataset_name=stru3d \ + --dataset_root=data/coco_s3d_bw/ \ + --eval_set=${SPLIT} \ + --output_dir=data_plots/output_gt_s3dbw/${SPLIT} \ + --semantic_classes=19 \ + --input_channels 3 \ + --disable_image_transform \ + --poly2seq \ + --image_size 256 \ + --image_scale 1 \ + --compute_stats \ + --plot_gt \ + --plot_gt_image \ + --plot_polys \ + --plot_density + + +SPLIT=test +python plot_floor.py --dataset_name=r2g \ + --dataset_root=data/R2G_hr_dataset_processed_v1/ \ + --eval_set=${SPLIT} \ + --output_dir=output_gt_r2g/${SPLIT} \ + --semantic_classes=13 \ + --input_channels 3 \ + --poly2seq \ + --disable_image_transform \ + --image_size 256 \ + --image_scale 1 \ + --compute_stats \ + --plot_gt \ + --plot_polys \ + --plot_density + + +SPLIT=test +python plot_floor.py --dataset_name=cubicasa \ + --dataset_root=data/coco_cubicasa5k_nowalls_v4-1_refined \ + --eval_set=${SPLIT} \ + --output_dir=data_plots/output_gt_cc5k/${SPLIT} \ + --semantic_classes=12 \ + --input_channels 3 \ + --disable_image_transform \ + --poly2seq \ + --image_size 256 \ + --image_scale 1 \ + --compute_stats \ + --plot_gt \ + --plot_polys \ + --plot_density \ No newline at end of file diff --git a/data_preprocess/tools/run_cc5k.sh b/data_preprocess/tools/run_cc5k.sh new file mode 100644 index 0000000000000000000000000000000000000000..b326eb54d5feae1ef66a500ae4e5f3f4cb6152b1 --- /dev/null +++ b/data_preprocess/tools/run_cc5k.sh @@ -0,0 +1,15 @@ +# create COCO-style dataset for CubiCasa5k +python -m data_preprocess.cubicasa5k.create_coco_cc5k --data_root=data/cubicasa5k/ \ + --output=data/coco_cubicasa5k_nowalls_v4/ \ + --disable_wd2line + +# Split example has more than 1 floorplan into separate samples +python -m data_preprocess.cubicasa5k.create_coco_cc5k.floorplan_extraction \ + --data_root data/coco_cubicasa5k_nowalls_v4/ \ + --output data/coco_cubicasa5k_nowalls_v4-1_refined/ + +# Merge individual JSONs into single JSON file per split (train/val/test) +# This must be done after floorplan_extraction.py +python -m data_preprocess.cubicasa5k.combine_json \ + --input data/coco_cubicasa5k_nowalls_v4-1_refined/ \ + --output data/coco_cubicasa5k_nowalls_v4-1_refined/annotations/ \ \ No newline at end of file diff --git a/data_preprocess/tools/run_r2g.sh b/data_preprocess/tools/run_r2g.sh new file mode 100644 index 0000000000000000000000000000000000000000..f098d55a12f0e758a026f75648fc1914419f7908 --- /dev/null +++ b/data_preprocess/tools/run_r2g.sh @@ -0,0 +1,12 @@ +# preprocess raw Raster2Graph high-resolution dataset +python -m data_preprocess.raster2graph.image_process --data_root=data/R2G_hr_dataset/ + +# convert to COCO-style dataset +python -m data_preprocess.raster2graph.convert_to_coco --dataset_path data/R2G_hr_dataset/ --output_dir data/R2G_hr_dataset_processed/ + +# combine JSON files into single JSON file per split +python -m data_preprocess.raster2graph.combine_json \ + --input data/R2G_hr_dataset_processed/ \ + --output data/R2G_hr_dataset_processed_v1/ \ + +rm -rf data/R2G_hr_dataset_processed/ \ No newline at end of file diff --git a/data_preprocess/tools/run_s3d.sh b/data_preprocess/tools/run_s3d.sh new file mode 100644 index 0000000000000000000000000000000000000000..b3164f96eb59ca0b709cb9ada5e0c0aed41c150a --- /dev/null +++ b/data_preprocess/tools/run_s3d.sh @@ -0,0 +1,22 @@ +## Assume the Structured3D density dataset are downloaded +DATA=data/coco_s3d + +for split in train val test; do + python plot_floor.py --dataset_name=stru3d \ + --dataset_root=${DATA} \ + --eval_set=${split} \ + --output_dir=data/coco_s3d_bw/${split}/ \ + --semantic_classes=19 \ + --input_channels 3 \ + --disable_image_transform \ + --poly2seq \ + --image_size 256 \ + --image_scale 1 \ + --plot_gt \ + --is_bw \ + --plot_engine matplotlib + +done + +# Reuse the annotations +cp -r data/coco_s3d/annotations data/coco_s3d_bw/ \ No newline at end of file diff --git a/data_preprocess/tools/run_waffle.sh b/data_preprocess/tools/run_waffle.sh new file mode 100644 index 0000000000000000000000000000000000000000..f8583ff55dd5aa50b8573dde7c14fd1883736a6f --- /dev/null +++ b/data_preprocess/tools/run_waffle.sh @@ -0,0 +1,3 @@ +python -m data_preprocess.waffle.create_coco_waffle_benchmark \ + --data_root data/waffle/benchmark/ \ + --output data/waffle_benchmark_processed/ \ No newline at end of file diff --git a/data_preprocess/waffle/create_coco_waffle_benchmark.py b/data_preprocess/waffle/create_coco_waffle_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..b1952c21f2d8517453f71e434d2dd495b1165b8c --- /dev/null +++ b/data_preprocess/waffle/create_coco_waffle_benchmark.py @@ -0,0 +1,290 @@ +import argparse +import json +import os +import sys +from glob import glob +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image +from shapely.geometry import Polygon + +sys.path.append(str(Path(__file__).resolve().parent.parent)) +from common_utils import resort_corners + + +def draw_polygon_on_image(image, polygons, class_to_color): + """ + Draws polygons on the image based on the COLOR_TO_CLASS mapping. + + Args: + image (numpy.ndarray): The image on which to draw. + polygons (list of list of tuple): List of polygons, where each polygon is a list of (x, y) points. + + Returns: + numpy.ndarray: The image with polygons drawn. + """ + # Draw each polygon on the image + for polygon, polygon_class in polygons: + # Convert polygon points to numpy array + pts = np.array(polygon, dtype=np.int32).reshape(-1, 2) + color = class_to_color[polygon_class] + bgr = (color[2], color[1], color[0]) # Convert RGB to BGR for OpenCV + # Draw filled polygon + cv2.fillPoly(image, [pts], bgr) + + return image + + +def fill_mask(segmentation_mask): + filled_mask = np.zeros_like(segmentation_mask, dtype=np.uint8) + + # Iterate over each class index in the segmentation mask + for class_index in np.unique(segmentation_mask): + if class_index == 0: # Skip the background + continue + + # Create a binary mask for the current class + binary_mask = (segmentation_mask == class_index).astype(np.uint8) + + # Find contours for the current class + contours, _ = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + # Fill each contour with white color in the single-channel mask + cv2.drawContours(filled_mask, contours, -1, 255, thickness=cv2.FILLED) + + return filled_mask + + +def to_bw_image(input_image): + # Convert the input image to grayscale + gray_image = cv2.cvtColor(input_image, cv2.COLOR_BGR2GRAY) + + # Apply a binary threshold to convert the grayscale image to black and white + _, bw_image = cv2.threshold(gray_image, 127, 255, cv2.THRESH_BINARY) + return bw_image + + +def create_coco_bounding_box(bb_x, bb_y, image_width, image_height, bound_pad=2): + bb_x = np.unique(bb_x) + bb_y = np.unique(bb_y) + bb_x_min = np.maximum(np.min(bb_x) - bound_pad, 0) + bb_y_min = np.maximum(np.min(bb_y) - bound_pad, 0) + + bb_x_max = np.minimum(np.max(bb_x) + bound_pad, image_width - 1) + bb_y_max = np.minimum(np.max(bb_y) + bound_pad, image_height - 1) + + bb_width = bb_x_max - bb_x_min + bb_height = bb_y_max - bb_y_min + + coco_bb = [bb_x_min, bb_y_min, bb_width, bb_height] + return coco_bb + + +def prepare_dict(categories_dict): + save_dict = {"images": [], "annotations": [], "categories": []} + for key, value in categories_dict.items(): + type_dict = {"supercategory": "room", "id": value, "name": key} + save_dict["categories"].append(type_dict) + return save_dict + + +def convert_numpy_to_python(obj): + if isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, np.floating): + return float(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + else: + return obj + + +def config(): + a = argparse.ArgumentParser(description="Generate coco format data for WAFFLE BENCHMARK SET") + a.add_argument("--data_root", default="data/waffle/benchmark/", type=str, help="path to WAFFLE BENCHMARK folder") + a.add_argument("--output", default="data/waffle_benchmark_processed/", type=str, help="path to output folder") + + args = a.parse_args() + return args + + +if __name__ == "__main__": + LABEL_NOTATIONS = { + "Background": (0, 0, 0), # Black + "Interior": (255, 255, 255), # White + "Walls": (255, 0, 0), # Red + "Doors": (0, 0, 255), # Blue + "Windows": (0, 255, 255), # Cyan + } + + CLASS2INDEX = { + "Background": 0, # Black + "Interior": 1, # White + # "Walls": 2, # Red + "Doors": 3, # Blue + "Windows": 4, # Cyan + } + + # Create a mapping from RGB values to class indices + COLOR_TO_CLASS = { + (0, 0, 0): 0, # Background + (255, 255, 255): 1, # Interior + (255, 0, 0): 2, # Walls + (0, 0, 255): 3, # Doors + (0, 255, 255): 4, # Windows + } + + NEW_CLASS_MAPPING = { + 1: 0, + 3: 1, + 4: 2, + } + + CLASS_TO_COLOR = { + 0: (255, 255, 255), # Interior + 1: (0, 0, 255), # Doors + 2: (0, 255, 255), # Windows + } + + args = config() + + root = args.data_root + image_dir = f"{root}/pngs" + label_dir = f"{root}/segmented_descrete_pngs" + input_paths = sorted(glob(f"{label_dir}/*.png")) + + output_dir = args.output + output_aux_dir = f"{output_dir}/aux" + output_image_dir = f"{output_dir}/test/" + output_annot_dir = f"{output_dir}/annotations/" + fn_mapping_log = f"{output_annot_dir}/test_image_id_mapping.json" + os.makedirs(output_dir, exist_ok=True) + os.makedirs(output_aux_dir, exist_ok=True) + os.makedirs(output_image_dir, exist_ok=True) + os.makedirs(output_annot_dir, exist_ok=True) + + instance_count = 0 + + save_dict = prepare_dict(CLASS2INDEX) + output_mappings = [] + + for i, path in enumerate(input_paths): + # if i > 5: + # exit(0) + mask = Image.open(path).convert("RGB") + fn = os.path.basename(path).replace("_seg_colors.png", "") + new_fn = str(i).zfill(5) + + mask = np.array(mask) + image = Image.open(os.path.join(image_dir, f"{fn}.png")).convert("RGB") + image_width, image_height = image.size + + # Initialize an empty segmentation mask with the same height and width as the input mask + segmentation_mask = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint8) + + img_id = i + img_dict = {} + img_dict["file_name"] = str(img_id).zfill(5) + ".png" + img_dict["id"] = img_id + img_dict["width"] = image_width + img_dict["height"] = image_height + + output_polygons = [] + coco_annotation_dict_list = [] + # Iterate over each pixel in the mask and assign the corresponding class index + for color, class_index in COLOR_TO_CLASS.items(): + # Create a boolean mask for the current color + color_mask = (mask == color).all(axis=-1) + color_mask_uint8 = color_mask.astype(np.uint8) + + # Assign the class index to the segmentation mask + segmentation_mask[color_mask] = class_index + + if class_index not in NEW_CLASS_MAPPING: + continue + class_index = NEW_CLASS_MAPPING[class_index] + + # Find contours for the current color mask + contours, _ = cv2.findContours(color_mask_uint8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + new_contours = [] + for cnt in contours: + peri = cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, 0.001 * peri, True) + new_contours.append(approx) + + # Convert contours to polygon coordinates + polygons = [contour.reshape(-1, 2) for contour in new_contours] + + for polygon in polygons: + # Convert the polygon to a Shapely Polygon object + if polygon.shape[0] < 3: + continue + + shapely_polygon = Polygon(polygon) + area = shapely_polygon.area + rectangle_shapely = shapely_polygon.envelope + bb_x, bb_y = rectangle_shapely.exterior.xy + coco_bb = create_coco_bounding_box(bb_x, bb_y, image_width, image_height, bound_pad=2) + + if class_index in [3, 4] and area < 1: + continue + if class_index not in [3, 4] and area < 100: + continue + + coco_seg_poly = [] + poly_sorted = resort_corners(polygon) + # image = draw_polygon_on_image(image, poly_shapely, "test_poly.jpg") + + for p in poly_sorted: + coco_seg_poly += list(p) + + # Create a dictionary for the COCO annotation + coco_annotation_dict = { + "segmentation": [coco_seg_poly], + "area": area, + "iscrow": 0, + "image_id": i, + "bbox": coco_bb, + "category_id": class_index, + "id": instance_count, + } + coco_annotation_dict_list.append(coco_annotation_dict) + instance_count += 1 + output_polygons.append([coco_seg_poly, class_index]) + + save_dict["images"].append(img_dict) + save_dict["annotations"] += coco_annotation_dict_list + + # Print the unique class indices in the segmentation mask to verify + print(path) + print(np.unique(segmentation_mask)) + + filled_mask = fill_mask(segmentation_mask) + + clean_image = np.array(image) + filled_mask_resized = cv2.resize( + filled_mask, (clean_image.shape[1], clean_image.shape[0]), interpolation=cv2.INTER_NEAREST + ) + cv2.imwrite(f"{output_aux_dir}/{fn}_fg_mask.png", filled_mask_resized) + + clean_image = clean_image * np.array(filled_mask_resized[:, :, np.newaxis] / 255.0).astype(bool) + clean_image[filled_mask_resized == 0] = 255 + clean_image = cv2.cvtColor(clean_image, cv2.COLOR_RGB2BGR) + # clean_image = to_bw_image(clean_image) + cv2.imwrite(f"{output_image_dir}/{new_fn}.png", clean_image) + + image_with_polygons = draw_polygon_on_image(np.zeros_like(clean_image), output_polygons, CLASS_TO_COLOR) + cv2.imwrite(f"{output_aux_dir}/{fn}_polylines.png", image_with_polygons) + + output_mappings.append(f"{fn} {new_fn}") + + with open(fn_mapping_log, "w") as f: + for mapping in output_mappings: + f.write(f"{mapping}\n") + + # Serialize save_dict to JSON + json_path = f"{output_annot_dir}/test.json" + with open(json_path, "w") as f: + json.dump(save_dict, f, default=convert_numpy_to_python) diff --git a/datasets/__init__.py b/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..956e355ee4ae050d8346d05e9828964047b28eec --- /dev/null +++ b/datasets/__init__.py @@ -0,0 +1,67 @@ +from .poly_data import build as build_poly + + +def build_dataset(image_set, args): + if args.dataset_name in ["stru3d", "cubicasa", "waffle", "r2g"]: + print(f"Build {args.dataset_name} {image_set} dataset") + return build_poly(image_set, args) + raise ValueError(f"dataset {args.dataset_name} not supported") + + +def get_dataset_class_labels(dataset_name): + semantics_label = None + + if dataset_name == "stru3d": + semantics_label = { + 0: "Living Room", + 1: "Kitchen", + 2: "Bedroom", + 3: "Bathroom", + 4: "Balcony", + 5: "Corridor", + 6: "Dining room", + 7: "Study", + 8: "Studio", + 9: "Store room", + 10: "Garden", + 11: "Laundry room", + 12: "Office", + 13: "Basement", + 14: "Garage", + 15: "Misc.", + 16: "Door", + 17: "Window", + } + elif dataset_name == "cubicasa": + semantics_label = { + "Outdoor": 0, + "Kitchen": 1, + "Living Room": 2, + "Bed Room": 3, + "Bath": 4, + "Entry": 5, + "Storage": 6, + "Garage": 7, + "Undefined": 8, + "Window": 9, + "Door": 10, + } + elif dataset_name == "r2g": + semantics_label = { + "unknown": 0, + "living_room": 1, + "kitchen": 2, + "bedroom": 3, + "bathroom": 4, + "restroom": 5, + "balcony": 6, + "closet": 7, + "corridor": 8, + "washing_room": 9, + "PS": 10, + "outside": 11, + } + + id2class = {v: k for k, v in semantics_label.items()} if semantics_label else None + + return semantics_label, id2class diff --git a/datasets/data_utils.py b/datasets/data_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fa18eb9e6eaa4cbe4380744b3fbf6055627ac797 --- /dev/null +++ b/datasets/data_utils.py @@ -0,0 +1,60 @@ +import matplotlib.pyplot as plt +import numpy as np + + +def compute_centroid(polygon): + """Compute centroid of a polygon given as list of (x, y).""" + polygon = np.array(polygon) + x = np.mean(polygon[:, 0]) + y = np.mean(polygon[:, 1]) + return (x, y) + + +def get_top_left(polygon): + return min(polygon, key=lambda p: (p[1], p[0])) # y ascending, x ascending + + +def sort_polygons(polygons, tolerance=20, reverse=False): + # Step 1: Get top-left corner and original index + indexed = [(i, get_top_left(p), p) for i, p in enumerate(polygons)] + + # Step 2: Sort by Y (top to bottom) + indexed.sort(key=lambda x: x[1][1]) + + # Step 3: Group into rows + rows = [] + for idx, corner, poly in indexed: + y = corner[1] + added = False + for row in rows: + if abs(row[0][1][1] - y) <= tolerance: + row.append((idx, corner, poly)) + added = True + break + if not added: + rows.append([(idx, corner, poly)]) + + # Step 4: Sort each row left-to-right + for row in rows: + row.sort(key=lambda x: x[1][0]) # sort by x + + # Step 5: Flatten and return indices + sorted_indices = [idx for row in rows for idx, _, _ in row] + if reverse: + sorted_indices = sorted_indices[::-1] + sorted_polygons = [polygons[idx] for idx in sorted_indices] + + return sorted_polygons, sorted_indices + + +def plot_polygons(polygons, save_path): + plt.figure(figsize=(6, 6)) + for i, poly in enumerate(polygons): + poly = np.array(poly) + plt.fill(poly[:, 0], poly[:, 1], alpha=0.5, label=f"Polygon {i + 1}") + centroid = compute_centroid(poly) + plt.text(centroid[0], centroid[1], f"C{i + 1}", fontsize=10, ha="center") + # plt.title(title) + # plt.legend() + plt.gca().set_aspect("equal", adjustable="box") + plt.savefig(save_path) \ No newline at end of file diff --git a/datasets/discrete_tokenizer.py b/datasets/discrete_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..f8f226f6ce996a5d557b0ae49e64871a8a1b4ff9 --- /dev/null +++ b/datasets/discrete_tokenizer.py @@ -0,0 +1,60 @@ +import numpy as np +import torch + + +class DiscreteTokenizer(object): + def __init__(self, num_bins, seq_len, add_cls=False): + self.num_bins = num_bins + vocab_size = num_bins * num_bins + self.seq_len = seq_len + self.add_cls = add_cls + + self.bos = vocab_size + 0 + self.eos = vocab_size + 1 + self.sep = vocab_size + 2 + self.pad = vocab_size + 3 + if add_cls: + self.cls = vocab_size + 4 + self.vocab_size = vocab_size + 5 + else: + self.vocab_size = vocab_size + 4 + + def __len__(self): + return self.vocab_size + + def _padding(self, seq, pad_value, dtype): + if self.seq_len > len(seq): + seq.extend([pad_value] * (self.seq_len - len(seq))) + return torch.tensor(np.array(seq), dtype=dtype) + + def __call__(self, seq, add_bos, add_eos, dtype, return_indices=False): + out = [] + if add_bos: + out = [self.bos] + num_extra = 1 if not self.add_cls else 2 # cls and sep + indices = [] + for i, sub in enumerate(seq): + cur_len = len(out) + # Append sub only if it doesn't exceed seq_len + if cur_len + len(sub) + num_extra <= self.seq_len: + out.extend(sub) + indices.append(i) + else: + continue + # Append cls and sep tokens only if it doesn't exceed seq_len + if self.add_cls: + out.append(self.cls) # cls token + out.append(self.sep) + # Remove last separator token if present + if out and out[-1] == self.sep: + out.pop(-1) # remove last separator token + + if self.seq_len > len(out): + out.extend([self.pad] * (self.seq_len - len(out))) + + if add_eos: + out[-1] = self.eos + + if return_indices: + return torch.tensor(out, dtype=dtype), indices + return torch.tensor(out, dtype=dtype) diff --git a/datasets/poly_data.py b/datasets/poly_data.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f03c37b354ada4cf9ff8fe2f7f27592770de2b --- /dev/null +++ b/datasets/poly_data.py @@ -0,0 +1,590 @@ +import math +import os +from enum import Enum +from pathlib import Path + +import numpy as np +import torch +import torch.utils.data +import torchvision +from PIL import Image +from pycocotools.coco import COCO +from torch.utils.data import Dataset + +from datasets.data_utils import sort_polygons +from datasets.discrete_tokenizer import DiscreteTokenizer +from datasets.transforms import ResizeAndPad +from detectron2.data import transforms as T +from detectron2.data.detection_utils import annotations_to_instances, transform_instance_annotations +from detectron2.structures import BoxMode +from util.poly_ops import resort_corners + + +class TokenType(Enum): + """0 for , 1 for , 2 for , 3 for """ + + coord = 0 + sep = 1 + eos = 2 + cls = 3 + + +WD_INDEX = { + "stru3d": [16, 17], + "cubicasa": [9, 10], + "waffle": [], + "r2g": [], +} + + +class MultiPoly(Dataset): + def __init__( + self, + img_folder, + ann_file, + transforms, + semantic_classes, + dataset_name="", + image_norm=False, + poly2seq=False, + converter_version="v1", + random_drop_rate=0.0, + **kwargs, + ): + super(MultiPoly, self).__init__() + + self.root = img_folder + self._transforms = transforms + self.semantic_classes = semantic_classes + self.dataset_name = dataset_name + + self.coco = COCO(ann_file) + self.ids = list(sorted(self.coco.imgs.keys())) + + self.poly2seq = poly2seq + self.prepare = ConvertToCocoDictWithOrder_plus( + self.root, + self._transforms, + image_norm, + poly2seq, + semantic_classes=semantic_classes, + order_type=["l2r", "r2l"][converter_version == "v3_flipped"], + random_drop_rate=random_drop_rate, + **kwargs, + ) + + def get_image(self, path): + return Image.open(os.path.join(self.root, path)) + + def get_vocab_size(self): + if self.poly2seq: + return len(self.prepare.tokenizer) + return None + + def get_tokenizer(self): + if self.poly2seq: + return self.prepare.tokenizer + return None + + def __len__(self): + return len(self.ids) + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + dict: COCO format dict + """ + coco = self.coco + img_id = self.ids[index] + + ann_ids = coco.getAnnIds(imgIds=img_id) + target = coco.loadAnns(ann_ids) + + ### Note: here is a hack which assumes door/window have category_id 16, 17 in structured3D + if self.semantic_classes == -1: + if self.dataset_name == "stru3d": + target = [t for t in target if t["category_id"] not in WD_INDEX["stru3d"]] + # elif self.dataset_name == 'rplan': + # target = [t for t in target if t['category_id'] not in [9, 11]] + elif self.dataset_name == "cubicasa": + target = [t for t in target if t["category_id"] not in WD_INDEX["cubicasa"]] + + path = coco.loadImgs(img_id)[0]["file_name"] + + record = self.prepare(img_id, path, target) + + return record + + +class MultiPolyWD(MultiPoly): + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + dict: COCO format dict + """ + coco = self.coco + img_id = self.ids[index] + + ann_ids = coco.getAnnIds(imgIds=img_id) + target = coco.loadAnns(ann_ids) + + ### Note: here is a hack which assumes door/window have category_id 16, 17 in structured3D + # if self.semantic_classes == -1: + # if self.dataset_name == 'stru3d': + # target = [t for t in target if t['category_id'] not in [16, 17]] + # elif self.dataset_name == 'rplan': + # target = [t for t in target if t['category_id'] not in [9, 11]] + # elif self.dataset_name == 'cubicasa': + # target = [t for t in target if t['category_id'] not in [9, 10]] + + if self.dataset_name == "stru3d": + target = [t for t in target if t["category_id"] in [16, 17]] + elif self.dataset_name == "rplan": + target = [t for t in target if t["category_id"] in [9, 11]] + elif self.dataset_name == "cubicasa": + target = [t for t in target if t["category_id"] in [9, 10]] + + path = coco.loadImgs(img_id)[0]["file_name"] + record = self.prepare(img_id, path, target) + + return record + + +class ConvertToCocoDict(object): + def __init__( + self, + root, + augmentations, + image_norm, + poly2seq=False, + semantic_classes=-1, + add_cls_token=False, + per_token_class=False, + mask_format="polygon", + **kwargs, + ): + self.root = root + self.augmentations = augmentations + if image_norm: + self.image_normalize = torchvision.transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + else: + self.image_normalize = None + + self.semantic_classes = semantic_classes + self.poly2seq = poly2seq + if poly2seq: + self.tokenizer = DiscreteTokenizer(add_cls=add_cls_token, **kwargs) + self.add_cls_token = add_cls_token + self.per_token_class = per_token_class + self.mask_format = mask_format + + def _expand_image_dims(self, x): + if len(x.shape) == 2: + exp_img = np.expand_dims(x, 0) + else: + exp_img = x.transpose((2, 0, 1)) # (h,w,c) -> (c,h,w) + return exp_img + + def __call__(self, img_id, path, target): + + file_name = os.path.join(self.root, path) + + img = np.array(Image.open(file_name)) + + #### NEW + if len(img.shape) >= 3: + if img.shape[-1] > 3: # drop alpha channel + img = img[:, :, :3] + w, h = img.shape[:-1] + else: + # print(img.shape, file_name) + w, h = img.shape + #### NEW + + record = {} + record["file_name"] = file_name + record["height"] = h + record["width"] = w + record["image_id"] = img_id + + for obj in target: + obj["bbox_mode"] = BoxMode.XYWH_ABS + + record["annotations"] = target + + if self.augmentations is None: + record["image"] = (1 / 255) * torch.as_tensor(np.ascontiguousarray(self._expand_image_dims(img))) + record["instances"] = annotations_to_instances(target, (h, w), mask_format=self.mask_format) + else: + aug_input = T.AugInput(img) + transforms = self.augmentations(aug_input) + image = aug_input.image + record["image"] = (1 / 255) * torch.as_tensor(np.array(self._expand_image_dims(image))) + h, w = image.shape[:2] # update size + + annos = [ + transform_instance_annotations(obj, transforms, image.shape[:2]) + for obj in record.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + # resort corners after augmentation: so that all corners start from upper-left counterclockwise + for anno in annos: + anno["segmentation"][0] = resort_corners(anno["segmentation"][0]) + + record["instances"] = annotations_to_instances(annos, (h, w), mask_format=self.mask_format) + + #### NEW #### + if self.image_normalize is not None: + record["image"] = self.image_normalize(record["image"]) + + # convert polygons to sequences + if self.poly2seq: + # only happend for wdonly + if not hasattr(record["instances"], "gt_masks"): + polygons = [np.array([[0.0, 0.0]])] + polygons_label = [self.semantic_classes - 1] # dummy class + else: + polygons = [ + np.clip(np.array(inst).reshape(-1, 2) / (w - 1), 0, 1) + for inst in record["instances"].gt_masks.polygons + ] + polygons_label = [inst.item() for inst in record["instances"].gt_classes] + record.update( + self._get_bilinear_interpolation_coeffs( + polygons, polygons_label, self.add_cls_token, self.per_token_class + ) + ) + + return record + + def _get_bilinear_interpolation_coeffs(self, polygons, polygons_label, add_cls_token=False, per_token_class=False): + num_bins = self.tokenizer.num_bins + quant_poly = [poly * (num_bins - 1) for poly in polygons] + index11 = [[math.floor(p[0]) * num_bins + math.floor(p[1]) for p in poly] for poly in quant_poly] + index21 = [[math.ceil(p[0]) * num_bins + math.floor(p[1]) for p in poly] for poly in quant_poly] + index12 = [[math.floor(p[0]) * num_bins + math.ceil(p[1]) for p in poly] for poly in quant_poly] + index22 = [[math.ceil(p[0]) * num_bins + math.ceil(p[1]) for p in poly] for poly in quant_poly] + + seq11 = self.tokenizer(index11, add_bos=True, add_eos=False, dtype=torch.long) + seq21 = self.tokenizer(index21, add_bos=True, add_eos=False, dtype=torch.long) + seq12 = self.tokenizer(index12, add_bos=True, add_eos=False, dtype=torch.long) + seq22 = self.tokenizer(index22, add_bos=True, add_eos=False, dtype=torch.long) + + # in real values insteads + target_seq = [] + token_labels = [] # 0 for , 1 for , 2 for , 3 for + num_extra = 1 if not add_cls_token else 2 # cls and sep + count_polys = 0 + for poly in polygons: + cur_len = len(token_labels) + if cur_len + len(poly) + num_extra > self.tokenizer.seq_len: + break # INFO: change from break to continue + token_labels.extend([TokenType.coord.value] * len(poly)) + if add_cls_token: + token_labels.append(TokenType.cls.value) # cls token + token_labels.append(TokenType.sep.value) # separator token + target_seq.extend(poly) + if add_cls_token: + target_seq.append([0, 0]) # padding for cls token + target_seq.append([0, 0]) # padding for sep/end token + count_polys += 1 + # remove last separator token + if len(token_labels) > 0: + token_labels[-1] = TokenType.eos.value + mask = torch.ones(self.tokenizer.seq_len, dtype=torch.bool) + if len(token_labels) < self.tokenizer.seq_len: + mask[len(token_labels) :] = 0 + target_seq = self.tokenizer._padding(target_seq, [0, 0], dtype=torch.float32) + token_labels = self.tokenizer._padding(token_labels, -1, dtype=torch.long) + + delta_x1 = [0] # [0] for bos token + for polygon in quant_poly[:count_polys]: + delta = [poly_point[0] - math.floor(poly_point[0]) for poly_point in polygon] + delta_x1.extend(delta) + if add_cls_token: + delta_x1.extend([0]) # for cls token + delta_x1.extend([0]) # for separator token + delta_x1 = delta_x1[:-1] # there is no separator token in the end + delta_x1 = self.tokenizer._padding(delta_x1, 0, dtype=torch.float32) + delta_x2 = 1 - delta_x1 + + delta_y1 = [0] # [0] for bos token + for polygon in quant_poly[:count_polys]: + delta = [poly_point[1] - math.floor(poly_point[1]) for poly_point in polygon] + delta_y1.extend(delta) + if add_cls_token: + delta_y1.extend([0]) # for cls token + delta_y1.extend([0]) # for separator token + delta_y1 = delta_y1[:-1] # there is no separator token in the end + delta_y1 = self.tokenizer._padding(delta_y1, 0, dtype=torch.float32) + delta_y2 = 1 - delta_y1 + + if not per_token_class: + target_polygon_labels = polygons_label[:count_polys] + else: + target_polygon_labels = [] + for poly, poly_label in zip(quant_poly[:count_polys], polygons_label[:count_polys]): + target_polygon_labels.extend([poly_label] * len(poly)) + target_polygon_labels.append(self.semantic_classes - 1) # undefined class for and token + + max_label_length = self.tokenizer.seq_len + if len(polygons_label) < max_label_length: + target_polygon_labels.extend([-1] * (max_label_length - len(target_polygon_labels))) + + target_polygon_labels = torch.tensor(target_polygon_labels, dtype=torch.long) + + return { + "delta_x1": delta_x1, + "delta_x2": delta_x2, + "delta_y1": delta_y1, + "delta_y2": delta_y2, + "seq11": seq11, + "seq21": seq21, + "seq12": seq12, + "seq22": seq22, + "target_seq": target_seq, + "token_labels": token_labels, + "mask": mask, + "target_polygon_labels": target_polygon_labels, + } + + +class ConvertToCocoDictWithOrder_plus(ConvertToCocoDict): + def __init__( + self, + root, + augmentations, + image_norm, + poly2seq=False, + semantic_classes=-1, + add_cls_token=False, + per_token_class=False, + mask_format="polygon", + dataset_name="stru3d", + order_type="l2r", + random_drop_rate=0.0, + **kwargs, + ): + super().__init__( + root, + augmentations, + image_norm, + poly2seq, + semantic_classes, + add_cls_token, + per_token_class, + mask_format, + **kwargs, + ) + self.dataset_name = dataset_name + self.order_type = order_type # l2r, r2l + self.random_drop_rate = random_drop_rate + self.tokenizer = DiscreteTokenizer(add_cls=add_cls_token, **kwargs) + + def _get_bilinear_interpolation_coeffs(self, polygons, polygons_label, add_cls_token=False, per_token_class=False): + num_bins = self.tokenizer.num_bins + room_indices = [ + poly_idx + for poly_idx, poly_label in enumerate(polygons_label) + if poly_label not in WD_INDEX[self.dataset_name] + ] + wd_indices = [ + poly_idx for poly_idx, poly_label in enumerate(polygons_label) if poly_label in WD_INDEX[self.dataset_name] + ] + + _, room_sorted_indices = sort_polygons( + [polygons[poly_idx] for poly_idx in room_indices], reverse=(self.order_type == "r2l") + ) + _, wd_sorted_indices = sort_polygons( + [polygons[poly_idx] for poly_idx in wd_indices], reverse=(self.order_type == "r2l") + ) + room_indices = [room_indices[_idx] for _idx in room_sorted_indices] + wd_indices = [wd_indices[_idx] for _idx in wd_sorted_indices] + + #### NEW #### + combined_indices = room_indices + wd_indices # room first + if self.random_drop_rate > 0 and len(combined_indices) > 2: + keep_indices = np.where(np.random.rand(len(combined_indices)) >= self.random_drop_rate)[0].tolist() + if len(keep_indices) > 0: # Only apply drop if we have something left + combined_indices = [combined_indices[i] for i in keep_indices] + #### NEW #### + + polygons = [polygons[i] for i in combined_indices] + polygons_label = [polygons_label[i] for i in combined_indices] + + quant_poly = [poly * (num_bins - 1) for poly in polygons] + index11 = [[math.floor(p[0]) * num_bins + math.floor(p[1]) for p in poly] for poly in quant_poly] + index21 = [[math.ceil(p[0]) * num_bins + math.floor(p[1]) for p in poly] for poly in quant_poly] + index12 = [[math.floor(p[0]) * num_bins + math.ceil(p[1]) for p in poly] for poly in quant_poly] + index22 = [[math.ceil(p[0]) * num_bins + math.ceil(p[1]) for p in poly] for poly in quant_poly] + + seq11 = self.tokenizer(index11, add_bos=True, add_eos=False, dtype=torch.long) + seq21 = self.tokenizer(index21, add_bos=True, add_eos=False, dtype=torch.long) + seq12 = self.tokenizer(index12, add_bos=True, add_eos=False, dtype=torch.long) + seq22, poly_indices = self.tokenizer( + index22, add_bos=True, add_eos=False, dtype=torch.long, return_indices=True + ) + + # in real values insteads + target_seq = [] + token_labels = [] # 0 for , 1 for , 2 for , 3 for + + for i in poly_indices: + token_labels.extend([TokenType.coord.value] * len(polygons[i])) + if add_cls_token: + token_labels.append(TokenType.cls.value) # cls token + token_labels.append(TokenType.sep.value) # separator token + target_seq.extend(polygons[i]) + if add_cls_token: + target_seq.append([0, 0]) # padding for cls token + target_seq.append([0, 0]) # padding for sep/end token + # remove last separator token + token_labels[-1] = TokenType.eos.value + + mask = torch.ones(self.tokenizer.seq_len, dtype=torch.bool) + if len(token_labels) < self.tokenizer.seq_len: + mask[len(token_labels) :] = 0 + target_seq = self.tokenizer._padding(target_seq, [0, 0], dtype=torch.float32) + token_labels = self.tokenizer._padding(token_labels, -1, dtype=torch.long) + + delta_x1 = [0] # [0] for bos token + for i in poly_indices: + polygon = quant_poly[i] + delta = [poly_point[0] - math.floor(poly_point[0]) for poly_point in polygon] + delta_x1.extend(delta) + if add_cls_token: + delta_x1.extend([0]) # for cls token + delta_x1.extend([0]) # for separator token + delta_x1 = delta_x1[:-1] # there is no separator token in the end + delta_x1 = self.tokenizer._padding(delta_x1, 0, dtype=torch.float32) + delta_x2 = 1 - delta_x1 + + delta_y1 = [0] # [0] for bos token + for i in poly_indices: + polygon = quant_poly[i] + delta = [poly_point[1] - math.floor(poly_point[1]) for poly_point in polygon] + delta_y1.extend(delta) + if add_cls_token: + delta_y1.extend([0]) # for cls token + delta_y1.extend([0]) # for separator token + delta_y1 = delta_y1[:-1] # there is no separator token in the end + delta_y1 = self.tokenizer._padding(delta_y1, 0, dtype=torch.float32) + delta_y2 = 1 - delta_y1 + + if not per_token_class: + target_polygon_labels = [polygons_label[i] for i in poly_indices] # polygons_label[:count_polys] + input_polygon_labels = torch.tensor(target_polygon_labels.copy(), dtype=torch.long) + else: + target_polygon_labels = [] + for i in poly_indices: + poly, poly_label = quant_poly[i], polygons_label[i] + target_polygon_labels.extend([poly_label] * len(poly)) + target_polygon_labels.append(self.semantic_classes - 1) # undefined class for and token + input_polygon_labels = torch.tensor( + [self.semantic_classes - 1] + target_polygon_labels.copy()[:-1], dtype=torch.long + ) # right shift by one: , ..., + + max_label_length = self.tokenizer.seq_len + if len(polygons_label) < max_label_length: + target_polygon_labels.extend([-1] * (max_label_length - len(target_polygon_labels))) + + target_polygon_labels = torch.tensor(target_polygon_labels, dtype=torch.long) + + return { + "delta_x1": delta_x1, + "delta_x2": delta_x2, + "delta_y1": delta_y1, + "delta_y2": delta_y2, + "seq11": seq11, + "seq21": seq21, + "seq12": seq12, + "seq22": seq22, + "target_seq": target_seq, + "token_labels": token_labels, + "mask": mask, + "target_polygon_labels": target_polygon_labels, + "input_polygon_labels": input_polygon_labels, + } + + +def make_poly_transforms(dataset_name, image_set, image_size=256, disable_image_transform=False): + + trans_list = [] + if dataset_name in ["cubicasa", "waffle"] or (dataset_name == "r2g" and image_size != 512): + trans_list = [ResizeAndPad((image_size, image_size), pad_value=255)] + + if image_set == "train": + if not disable_image_transform: + trans_list.extend( + [ + T.RandomFlip(prob=0.5, horizontal=True, vertical=False), + T.RandomFlip(prob=0.5, horizontal=False, vertical=True), + T.RandomRotation([0.0, 90.0, 180.0, 270.0], expand=False, center=None, sample_style="choice"), + ] + ) + return T.AugmentationList(trans_list) + + if image_set == "val" or image_set == "test": + return None if len(trans_list) == 0 else T.AugmentationList(trans_list) + + raise ValueError(f"unknown {image_set}") + + +def build(image_set, args): + root = Path(args.dataset_root) + assert root.exists(), f"provided data path {root} does not exist" + + PATHS = { + "train": (root / "train", root / "annotations" / "train.json"), + "val": (root / "val", root / "annotations" / "val.json"), + "test": (root / "test", root / "annotations" / "test.json"), + } + + img_folder, ann_file = PATHS[image_set] + image_transform = make_poly_transforms( + args.dataset_name, + image_set, + image_size=args.image_size, + disable_image_transform=getattr(args, "disable_image_transform", False), + ) + + if args.wd_only: + dataset = MultiPolyWD( + img_folder, + ann_file, + transforms=image_transform, + semantic_classes=args.semantic_classes, + dataset_name=args.dataset_name, + image_norm=args.image_norm, + poly2seq=args.poly2seq, + num_bins=args.num_bins, + seq_len=args.seq_len, + add_cls_token=args.add_cls_token, + per_token_class=args.per_token_sem_loss, + mask_format=getattr(args, "mask_format", "polygon"), + ) + else: + dataset = MultiPoly( + img_folder, + ann_file, + transforms=image_transform, + semantic_classes=args.semantic_classes, + dataset_name=args.dataset_name, + image_norm=args.image_norm, + poly2seq=args.poly2seq, + num_bins=args.num_bins, + seq_len=args.seq_len, + add_cls_token=args.add_cls_token, + per_token_class=args.per_token_sem_loss, + mask_format=getattr(args, "mask_format", "polygon"), + converter_version=getattr(args, "converter_version", "v1"), + random_drop_rate=getattr(args, "random_drop_rate", 0.0), + ) + + return dataset diff --git a/datasets/room_dropout.py b/datasets/room_dropout.py new file mode 100644 index 0000000000000000000000000000000000000000..6e6a7014b88d765d0ccdfdfb45289e86f84bc106 --- /dev/null +++ b/datasets/room_dropout.py @@ -0,0 +1,237 @@ +import random +from typing import List, Optional, Tuple + +import cv2 +import numpy as np +from skimage.draw import polygon + + +class RoomDropoutStrategy: + """ + Strategy for randomly dropping rooms from a density map using ground truth coordinates. + + Density map: grayscale image where foreground (rooms) are white points and background is black + GT room coordinates: list of 2D points defining each room's boundary + """ + + def __init__(self, density_map: np.ndarray, room_coordinates: List[List[Tuple[int, int]]]): + """ + Initialize the dropout strategy. + + Args: + density_map: Grayscale image (H, W) where white pixels represent rooms + room_coordinates: List of rooms, each room is a list of (x, y) coordinate tuples + """ + self.original_density_map = density_map.copy() + self.room_coordinates = room_coordinates + self.num_rooms = len(room_coordinates) + + def create_room_masks(self) -> List[np.ndarray]: + """ + Create binary masks for each room using their GT coordinates. + + Returns: + List of binary masks, one for each room + """ + h, w = self.original_density_map.shape + room_masks = [] + + for room_coords in self.room_coordinates: + mask = np.zeros((h, w), dtype=np.uint8) + + if len(room_coords) >= 3: # Need at least 3 points for a polygon + # Convert coordinates to numpy array + coords = np.array(room_coords) + x_coords = coords[:, 0] + y_coords = coords[:, 1] + + # Create polygon mask using skimage + rr, cc = polygon(y_coords, x_coords, shape=(h, w)) + mask[rr, cc] = 1 + + room_masks.append(mask) + + return room_masks + + def drop_rooms_random(self, dropout_rate: float = 0.3, seed: Optional[int] = None) -> Tuple[np.ndarray, List[int]]: + """ + Randomly drop rooms from the density map. + + Args: + dropout_rate: Fraction of rooms to drop (0.0 to 1.0) + seed: Random seed for reproducibility + + Returns: + Tuple of (modified_density_map, list_of_dropped_room_indices) + """ + if seed is not None: + random.seed(seed) + np.random.seed(seed) + + # Determine number of rooms to drop + num_to_drop = int(self.num_rooms * dropout_rate) + + # Randomly select room indices to drop + room_indices = list(range(self.num_rooms)) + dropped_indices = random.sample(room_indices, num_to_drop) + + return self._apply_dropout(dropped_indices), dropped_indices + + def drop_rooms_by_indices(self, room_indices: List[int]) -> np.ndarray: + """ + Drop specific rooms by their indices. + + Args: + room_indices: List of room indices to drop + + Returns: + Modified density map with specified rooms removed + """ + return self._apply_dropout(room_indices) + + def drop_rooms_by_area( + self, min_area: Optional[int] = None, max_area: Optional[int] = None + ) -> Tuple[np.ndarray, List[int]]: + """ + Drop rooms based on their area constraints. + + Args: + min_area: Minimum area threshold (drop rooms smaller than this) + max_area: Maximum area threshold (drop rooms larger than this) + + Returns: + Tuple of (modified_density_map, list_of_dropped_room_indices) + """ + room_masks = self.create_room_masks() + dropped_indices = [] + + for i, mask in enumerate(room_masks): + area = np.sum(mask) + + should_drop = False + if min_area is not None and area < min_area: + should_drop = True + if max_area is not None and area > max_area: + should_drop = True + + if should_drop: + dropped_indices.append(i) + + return self._apply_dropout(dropped_indices), dropped_indices + + def _apply_dropout(self, room_indices_to_drop: List[int]) -> np.ndarray: + """ + Apply dropout by removing specified rooms from the density map. + + Args: + room_indices_to_drop: List of room indices to remove + + Returns: + Modified density map with rooms removed + """ + modified_map = self.original_density_map.copy() + room_masks = self.create_room_masks() + + # Remove each specified room + for room_idx in room_indices_to_drop: + if 0 <= room_idx < len(room_masks): + mask = room_masks[room_idx] + # Set pixels in the room area to background (black/0) + modified_map[mask == 1] = 0 + + return modified_map + + def visualize_dropout( + self, original_map: np.ndarray, modified_map: np.ndarray, dropped_indices: List[int] + ) -> np.ndarray: + """ + Create a visualization showing the dropout effect. + + Args: + original_map: Original density map + modified_map: Modified density map after dropout + dropped_indices: Indices of dropped rooms + + Returns: + Visualization image with original and modified maps side by side + """ + h, w = original_map.shape + + # Create side-by-side comparison + vis = np.zeros((h, w * 2), dtype=np.uint8) + vis[:, :w] = original_map + vis[:, w:] = modified_map + + # Highlight dropped rooms in red on the original map + if len(dropped_indices) > 0: + room_masks = self.create_room_masks() + vis_color = cv2.cvtColor(vis, cv2.COLOR_GRAY2BGR) + + for idx in dropped_indices: + if 0 <= idx < len(room_masks): + mask = room_masks[idx] + # Highlight in red on the left (original) side + vis_color[mask == 1, 0] = 0 # Blue channel + vis_color[mask == 1, 1] = 0 # Green channel + vis_color[mask == 1, 2] = 255 # Red channel + + return vis_color + + return cv2.cvtColor(vis, cv2.COLOR_GRAY2BGR) + + +# Example usage and testing +def example_usage(): + """ + Example of how to use the RoomDropoutStrategy class. + """ + # Create a sample density map (200x200 image) + density_map = np.zeros((200, 200), dtype=np.uint8) + + # Create some sample room coordinates (rectangles and polygons) + room_coordinates = [ + # Room 1: Rectangle + [(20, 20), (80, 20), (80, 60), (20, 60)], + # Room 2: Another rectangle + [(100, 30), (180, 30), (180, 80), (100, 80)], + # Room 3: L-shaped room + [(30, 100), (90, 100), (90, 130), (60, 130), (60, 160), (30, 160)], + # Room 4: Triangle + [(120, 120), (160, 120), (140, 160)], + # Room 5: Pentagon + [(50, 180), (70, 170), (90, 180), (80, 195), (40, 195)], + ] + + # Fill the density map with white pixels for each room + for room_coords in room_coordinates: + coords = np.array(room_coords) + x_coords = coords[:, 0] + y_coords = coords[:, 1] + + from skimage.draw import polygon + + rr, cc = polygon(y_coords, x_coords, shape=density_map.shape) + density_map[rr, cc] = 255 # White pixels for rooms + + # Initialize the dropout strategy + dropout_strategy = RoomDropoutStrategy(density_map, room_coordinates) + + # Example 1: Random dropout + print("Example 1: Random dropout (30% of rooms)") + modified_map1, dropped_indices1 = dropout_strategy.drop_rooms_random(dropout_rate=0.3, seed=42) + print(f"Dropped rooms: {dropped_indices1}") + + # Example 2: Drop specific rooms + print("\nExample 2: Drop specific rooms (indices 0 and 2)") + modified_map2 = dropout_strategy.drop_rooms_by_indices([0, 2]) + + # Example 3: Drop rooms by area + print("\nExample 3: Drop rooms with area > 3000 pixels") + modified_map3, dropped_indices3 = dropout_strategy.drop_rooms_by_area(max_area=3000) + print(f"Dropped rooms by area: {dropped_indices3}") + + return density_map, modified_map1, modified_map2, modified_map3 + + +if __name__ == "__main__": + example_usage() diff --git a/datasets/transforms.py b/datasets/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..b24590979e9ff0440b34a2a261598ccdfbb3f732 --- /dev/null +++ b/datasets/transforms.py @@ -0,0 +1,46 @@ +from PIL import Image + +from detectron2.data import transforms as T + + +class Resize(T.Augmentation): + """Resize image to a fixed target size""" + + def __init__(self, shape, interp=Image.BICUBIC): + """ + Args: + shape: (h, w) tuple or a int + interp: PIL interpolation method + """ + if isinstance(shape, int): + shape = (shape, shape) + shape = tuple(shape) + self._init(locals()) + + def get_transform(self, image): + return T.ResizeTransform(image.shape[0], image.shape[1], self.shape[0], self.shape[1], self.interp) + + +# Custom transform that resizes and then pads to fixed size +class ResizeAndPad(T.Augmentation): + def __init__(self, target_size, pad_value=0, interp=Image.BICUBIC): + super().__init__() + self.target_size = target_size # (height, width) + self.interp = interp + self.pad_value = pad_value + + def get_transform(self, img): + h, w = img.shape[:2] + scale = min(self.target_size[0] / h, self.target_size[1] / w) + new_h, new_w = int(h * scale), int(w * scale) + + # First resize preserving aspect ratio + resize_t = T.ResizeTransform(h, w, new_h, new_w, self.interp) + + # Then pad to target size + pad_h, pad_w = self.target_size[0] - new_h, self.target_size[1] - new_w + top = pad_h // 2 + left = pad_w // 2 + pad_t = T.PadTransform(left, top, pad_w - left, pad_h - top, new_h, new_w, pad_value=self.pad_value) + + return T.TransformList([resize_t, pad_t]) diff --git a/detectron2/__init__.py b/detectron2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bdd994b49294485c27610772f97f177741f5518f --- /dev/null +++ b/detectron2/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .utils.env import setup_environment + +setup_environment() + + +# This line will be programatically read/write by setup.py. +# Leave them at the bottom of this file and don't touch them. +__version__ = "0.6" diff --git a/detectron2/checkpoint/__init__.py b/detectron2/checkpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..348bea123815c3c793ae33094309d1679b76cc54 --- /dev/null +++ b/detectron2/checkpoint/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +# File: + + +from fvcore.common.checkpoint import Checkpointer, PeriodicCheckpointer + +from . import catalog as _UNUSED # register the handler +from .detection_checkpoint import DetectionCheckpointer + +__all__ = ["Checkpointer", "PeriodicCheckpointer", "DetectionCheckpointer"] diff --git a/detectron2/checkpoint/c2_model_loading.py b/detectron2/checkpoint/c2_model_loading.py new file mode 100644 index 0000000000000000000000000000000000000000..e12424300303b9899d068929cd9d405c83c7ec08 --- /dev/null +++ b/detectron2/checkpoint/c2_model_loading.py @@ -0,0 +1,387 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import re +from typing import Dict, List + +import torch +from tabulate import tabulate + + +def convert_basic_c2_names(original_keys): + """ + Apply some basic name conversion to names in C2 weights. + It only deals with typical backbone models. + + Args: + original_keys (list[str]): + Returns: + list[str]: The same number of strings matching those in original_keys. + """ + layer_keys = copy.deepcopy(original_keys) + layer_keys = [ + {"pred_b": "linear_b", "pred_w": "linear_w"}.get(k, k) for k in layer_keys + ] # some hard-coded mappings + + layer_keys = [k.replace("_", ".") for k in layer_keys] + layer_keys = [re.sub("\\.b$", ".bias", k) for k in layer_keys] + layer_keys = [re.sub("\\.w$", ".weight", k) for k in layer_keys] + # Uniform both bn and gn names to "norm" + layer_keys = [re.sub("bn\\.s$", "norm.weight", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.bias$", "norm.bias", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.rm", "norm.running_mean", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.running.mean$", "norm.running_mean", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.riv$", "norm.running_var", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.running.var$", "norm.running_var", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.gamma$", "norm.weight", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.beta$", "norm.bias", k) for k in layer_keys] + layer_keys = [re.sub("gn\\.s$", "norm.weight", k) for k in layer_keys] + layer_keys = [re.sub("gn\\.bias$", "norm.bias", k) for k in layer_keys] + + # stem + layer_keys = [re.sub("^res\\.conv1\\.norm\\.", "conv1.norm.", k) for k in layer_keys] + # to avoid mis-matching with "conv1" in other components (e.g. detection head) + layer_keys = [re.sub("^conv1\\.", "stem.conv1.", k) for k in layer_keys] + + # layer1-4 is used by torchvision, however we follow the C2 naming strategy (res2-5) + # layer_keys = [re.sub("^res2.", "layer1.", k) for k in layer_keys] + # layer_keys = [re.sub("^res3.", "layer2.", k) for k in layer_keys] + # layer_keys = [re.sub("^res4.", "layer3.", k) for k in layer_keys] + # layer_keys = [re.sub("^res5.", "layer4.", k) for k in layer_keys] + + # blocks + layer_keys = [k.replace(".branch1.", ".shortcut.") for k in layer_keys] + layer_keys = [k.replace(".branch2a.", ".conv1.") for k in layer_keys] + layer_keys = [k.replace(".branch2b.", ".conv2.") for k in layer_keys] + layer_keys = [k.replace(".branch2c.", ".conv3.") for k in layer_keys] + + # DensePose substitutions + layer_keys = [re.sub("^body.conv.fcn", "body_conv_fcn", k) for k in layer_keys] + layer_keys = [k.replace("AnnIndex.lowres", "ann_index_lowres") for k in layer_keys] + layer_keys = [k.replace("Index.UV.lowres", "index_uv_lowres") for k in layer_keys] + layer_keys = [k.replace("U.lowres", "u_lowres") for k in layer_keys] + layer_keys = [k.replace("V.lowres", "v_lowres") for k in layer_keys] + return layer_keys + + +def convert_c2_detectron_names(weights): + """ + Map Caffe2 Detectron weight names to Detectron2 names. + + Args: + weights (dict): name -> tensor + + Returns: + dict: detectron2 names -> tensor + dict: detectron2 names -> C2 names + """ + logger = logging.getLogger(__name__) + logger.info("Renaming Caffe2 weights ......") + original_keys = sorted(weights.keys()) + layer_keys = copy.deepcopy(original_keys) + + layer_keys = convert_basic_c2_names(layer_keys) + + # -------------------------------------------------------------------------- + # RPN hidden representation conv + # -------------------------------------------------------------------------- + # FPN case + # In the C2 model, the RPN hidden layer conv is defined for FPN level 2 and then + # shared for all other levels, hence the appearance of "fpn2" + layer_keys = [k.replace("conv.rpn.fpn2", "proposal_generator.rpn_head.conv") for k in layer_keys] + # Non-FPN case + layer_keys = [k.replace("conv.rpn", "proposal_generator.rpn_head.conv") for k in layer_keys] + + # -------------------------------------------------------------------------- + # RPN box transformation conv + # -------------------------------------------------------------------------- + # FPN case (see note above about "fpn2") + layer_keys = [k.replace("rpn.bbox.pred.fpn2", "proposal_generator.rpn_head.anchor_deltas") for k in layer_keys] + layer_keys = [ + k.replace("rpn.cls.logits.fpn2", "proposal_generator.rpn_head.objectness_logits") for k in layer_keys + ] + # Non-FPN case + layer_keys = [k.replace("rpn.bbox.pred", "proposal_generator.rpn_head.anchor_deltas") for k in layer_keys] + layer_keys = [k.replace("rpn.cls.logits", "proposal_generator.rpn_head.objectness_logits") for k in layer_keys] + + # -------------------------------------------------------------------------- + # Fast R-CNN box head + # -------------------------------------------------------------------------- + layer_keys = [re.sub("^bbox\\.pred", "bbox_pred", k) for k in layer_keys] + layer_keys = [re.sub("^cls\\.score", "cls_score", k) for k in layer_keys] + layer_keys = [re.sub("^fc6\\.", "box_head.fc1.", k) for k in layer_keys] + layer_keys = [re.sub("^fc7\\.", "box_head.fc2.", k) for k in layer_keys] + # 4conv1fc head tensor names: head_conv1_w, head_conv1_gn_s + layer_keys = [re.sub("^head\\.conv", "box_head.conv", k) for k in layer_keys] + + # -------------------------------------------------------------------------- + # FPN lateral and output convolutions + # -------------------------------------------------------------------------- + def fpn_map(name): + """ + Look for keys with the following patterns: + 1) Starts with "fpn.inner." + Example: "fpn.inner.res2.2.sum.lateral.weight" + Meaning: These are lateral pathway convolutions + 2) Starts with "fpn.res" + Example: "fpn.res2.2.sum.weight" + Meaning: These are FPN output convolutions + """ + splits = name.split(".") + norm = ".norm" if "norm" in splits else "" + if name.startswith("fpn.inner."): + # splits example: ['fpn', 'inner', 'res2', '2', 'sum', 'lateral', 'weight'] + stage = int(splits[2][len("res") :]) + return "fpn_lateral{}{}.{}".format(stage, norm, splits[-1]) + elif name.startswith("fpn.res"): + # splits example: ['fpn', 'res2', '2', 'sum', 'weight'] + stage = int(splits[1][len("res") :]) + return "fpn_output{}{}.{}".format(stage, norm, splits[-1]) + return name + + layer_keys = [fpn_map(k) for k in layer_keys] + + # -------------------------------------------------------------------------- + # Mask R-CNN mask head + # -------------------------------------------------------------------------- + # roi_heads.StandardROIHeads case + layer_keys = [k.replace(".[mask].fcn", "mask_head.mask_fcn") for k in layer_keys] + layer_keys = [re.sub("^\\.mask\\.fcn", "mask_head.mask_fcn", k) for k in layer_keys] + layer_keys = [k.replace("mask.fcn.logits", "mask_head.predictor") for k in layer_keys] + # roi_heads.Res5ROIHeads case + layer_keys = [k.replace("conv5.mask", "mask_head.deconv") for k in layer_keys] + + # -------------------------------------------------------------------------- + # Keypoint R-CNN head + # -------------------------------------------------------------------------- + # interestingly, the keypoint head convs have blob names that are simply "conv_fcnX" + layer_keys = [k.replace("conv.fcn", "roi_heads.keypoint_head.conv_fcn") for k in layer_keys] + layer_keys = [k.replace("kps.score.lowres", "roi_heads.keypoint_head.score_lowres") for k in layer_keys] + layer_keys = [k.replace("kps.score.", "roi_heads.keypoint_head.score.") for k in layer_keys] + + # -------------------------------------------------------------------------- + # Done with replacements + # -------------------------------------------------------------------------- + assert len(set(layer_keys)) == len(layer_keys) + assert len(original_keys) == len(layer_keys) + + new_weights = {} + new_keys_to_original_keys = {} + for orig, renamed in zip(original_keys, layer_keys): + new_keys_to_original_keys[renamed] = orig + if renamed.startswith("bbox_pred.") or renamed.startswith("mask_head.predictor."): + # remove the meaningless prediction weight for background class + new_start_idx = 4 if renamed.startswith("bbox_pred.") else 1 + new_weights[renamed] = weights[orig][new_start_idx:] + logger.info( + "Remove prediction weight for background class in {}. The shape changes from " + "{} to {}.".format(renamed, tuple(weights[orig].shape), tuple(new_weights[renamed].shape)) + ) + elif renamed.startswith("cls_score."): + # move weights of bg class from original index 0 to last index + logger.info( + "Move classification weights for background class in {} from index 0 to " + "index {}.".format(renamed, weights[orig].shape[0] - 1) + ) + new_weights[renamed] = torch.cat([weights[orig][1:], weights[orig][:1]]) + else: + new_weights[renamed] = weights[orig] + + return new_weights, new_keys_to_original_keys + + +# Note the current matching is not symmetric. +# it assumes model_state_dict will have longer names. +def align_and_update_state_dicts(model_state_dict, ckpt_state_dict, c2_conversion=True): + """ + Match names between the two state-dict, and returns a new chkpt_state_dict with names + converted to match model_state_dict with heuristics. The returned dict can be later + loaded with fvcore checkpointer. + If `c2_conversion==True`, `ckpt_state_dict` is assumed to be a Caffe2 + model and will be renamed at first. + + Strategy: suppose that the models that we will create will have prefixes appended + to each of its keys, for example due to an extra level of nesting that the original + pre-trained weights from ImageNet won't contain. For example, model.state_dict() + might return backbone[0].body.res2.conv1.weight, while the pre-trained model contains + res2.conv1.weight. We thus want to match both parameters together. + For that, we look for each model weight, look among all loaded keys if there is one + that is a suffix of the current weight name, and use it if that's the case. + If multiple matches exist, take the one with longest size + of the corresponding name. For example, for the same model as before, the pretrained + weight file can contain both res2.conv1.weight, as well as conv1.weight. In this case, + we want to match backbone[0].body.conv1.weight to conv1.weight, and + backbone[0].body.res2.conv1.weight to res2.conv1.weight. + """ + model_keys = sorted(model_state_dict.keys()) + if c2_conversion: + ckpt_state_dict, original_keys = convert_c2_detectron_names(ckpt_state_dict) + # original_keys: the name in the original dict (before renaming) + else: + original_keys = {x: x for x in ckpt_state_dict.keys()} + ckpt_keys = sorted(ckpt_state_dict.keys()) + + def match(a, b): + # Matched ckpt_key should be a complete (starts with '.') suffix. + # For example, roi_heads.mesh_head.whatever_conv1 does not match conv1, + # but matches whatever_conv1 or mesh_head.whatever_conv1. + return a == b or a.endswith("." + b) + + # get a matrix of string matches, where each (i, j) entry correspond to the size of the + # ckpt_key string, if it matches + match_matrix = [len(j) if match(i, j) else 0 for i in model_keys for j in ckpt_keys] + match_matrix = torch.as_tensor(match_matrix).view(len(model_keys), len(ckpt_keys)) + # use the matched one with longest size in case of multiple matches + max_match_size, idxs = match_matrix.max(1) + # remove indices that correspond to no-match + idxs[max_match_size == 0] = -1 + + logger = logging.getLogger(__name__) + # matched_pairs (matched checkpoint key --> matched model key) + matched_keys = {} + result_state_dict = {} + for idx_model, idx_ckpt in enumerate(idxs.tolist()): + if idx_ckpt == -1: + continue + key_model = model_keys[idx_model] + key_ckpt = ckpt_keys[idx_ckpt] + value_ckpt = ckpt_state_dict[key_ckpt] + shape_in_model = model_state_dict[key_model].shape + + if shape_in_model != value_ckpt.shape: + logger.warning( + "Shape of {} in checkpoint is {}, while shape of {} in model is {}.".format( + key_ckpt, value_ckpt.shape, key_model, shape_in_model + ) + ) + logger.warning("{} will not be loaded. Please double check and see if this is desired.".format(key_ckpt)) + continue + + assert key_model not in result_state_dict + result_state_dict[key_model] = value_ckpt + if key_ckpt in matched_keys: # already added to matched_keys + logger.error( + "Ambiguity found for {} in checkpoint!" + "It matches at least two keys in the model ({} and {}).".format( + key_ckpt, key_model, matched_keys[key_ckpt] + ) + ) + raise ValueError("Cannot match one checkpoint key to multiple keys in the model.") + + matched_keys[key_ckpt] = key_model + + # logging: + matched_model_keys = sorted(matched_keys.values()) + if len(matched_model_keys) == 0: + logger.warning("No weights in checkpoint matched with model.") + return ckpt_state_dict + common_prefix = _longest_common_prefix(matched_model_keys) + rev_matched_keys = {v: k for k, v in matched_keys.items()} + original_keys = {k: original_keys[rev_matched_keys[k]] for k in matched_model_keys} + + model_key_groups = _group_keys_by_module(matched_model_keys, original_keys) + table = [] + memo = set() + for key_model in matched_model_keys: + if key_model in memo: + continue + if key_model in model_key_groups: + group = model_key_groups[key_model] + memo |= set(group) + shapes = [tuple(model_state_dict[k].shape) for k in group] + table.append( + ( + _longest_common_prefix([k[len(common_prefix) :] for k in group]) + "*", + _group_str([original_keys[k] for k in group]), + " ".join([str(x).replace(" ", "") for x in shapes]), + ) + ) + else: + key_checkpoint = original_keys[key_model] + shape = str(tuple(model_state_dict[key_model].shape)) + table.append((key_model[len(common_prefix) :], key_checkpoint, shape)) + table_str = tabulate(table, tablefmt="pipe", headers=["Names in Model", "Names in Checkpoint", "Shapes"]) + logger.info( + "Following weights matched with " + + (f"submodule {common_prefix[:-1]}" if common_prefix else "model") + + ":\n" + + table_str + ) + + unmatched_ckpt_keys = [k for k in ckpt_keys if k not in set(matched_keys.keys())] + for k in unmatched_ckpt_keys: + result_state_dict[k] = ckpt_state_dict[k] + return result_state_dict + + +def _group_keys_by_module(keys: List[str], original_names: Dict[str, str]): + """ + Params in the same submodule are grouped together. + + Args: + keys: names of all parameters + original_names: mapping from parameter name to their name in the checkpoint + + Returns: + dict[name -> all other names in the same group] + """ + + def _submodule_name(key): + pos = key.rfind(".") + if pos < 0: + return None + prefix = key[: pos + 1] + return prefix + + all_submodules = [_submodule_name(k) for k in keys] + all_submodules = [x for x in all_submodules if x] + all_submodules = sorted(all_submodules, key=len) + + ret = {} + for prefix in all_submodules: + group = [k for k in keys if k.startswith(prefix)] + if len(group) <= 1: + continue + original_name_lcp = _longest_common_prefix_str([original_names[k] for k in group]) + if len(original_name_lcp) == 0: + # don't group weights if original names don't share prefix + continue + + for k in group: + if k in ret: + continue + ret[k] = group + return ret + + +def _longest_common_prefix(names: List[str]) -> str: + """ + ["abc.zfg", "abc.zef"] -> "abc." + """ + names = [n.split(".") for n in names] + m1, m2 = min(names), max(names) + ret = [a for a, b in zip(m1, m2) if a == b] + ret = ".".join(ret) + "." if len(ret) else "" + return ret + + +def _longest_common_prefix_str(names: List[str]) -> str: + m1, m2 = min(names), max(names) + lcp = [a for a, b in zip(m1, m2) if a == b] + lcp = "".join(lcp) + return lcp + + +def _group_str(names: List[str]) -> str: + """ + Turn "common1", "common2", "common3" into "common{1,2,3}" + """ + lcp = _longest_common_prefix_str(names) + rest = [x[len(lcp) :] for x in names] + rest = "{" + ",".join(rest) + "}" + ret = lcp + rest + + # add some simplification for BN specifically + ret = ret.replace("bn_{beta,running_mean,running_var,gamma}", "bn_*") + ret = ret.replace("bn_beta,bn_running_mean,bn_running_var,bn_gamma", "bn_*") + return ret diff --git a/detectron2/checkpoint/catalog.py b/detectron2/checkpoint/catalog.py new file mode 100644 index 0000000000000000000000000000000000000000..44a7ddae3a2dc3c0c666e4c4d8259a34a9f63fac --- /dev/null +++ b/detectron2/checkpoint/catalog.py @@ -0,0 +1,113 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging + +from detectron2.utils.file_io import PathHandler, PathManager + + +class ModelCatalog(object): + """ + Store mappings from names to third-party models. + """ + + S3_C2_DETECTRON_PREFIX = "https://dl.fbaipublicfiles.com/detectron" + + # MSRA models have STRIDE_IN_1X1=True. False otherwise. + # NOTE: all BN models here have fused BN into an affine layer. + # As a result, you should only load them to a model with "FrozenBN". + # Loading them to a model with regular BN or SyncBN is wrong. + # Even when loaded to FrozenBN, it is still different from affine by an epsilon, + # which should be negligible for training. + # NOTE: all models here uses PIXEL_STD=[1,1,1] + # NOTE: Most of the BN models here are no longer used. We use the + # re-converted pre-trained models under detectron2 model zoo instead. + C2_IMAGENET_MODELS = { + "MSRA/R-50": "ImageNetPretrained/MSRA/R-50.pkl", + "MSRA/R-101": "ImageNetPretrained/MSRA/R-101.pkl", + "FAIR/R-50-GN": "ImageNetPretrained/47261647/R-50-GN.pkl", + "FAIR/R-101-GN": "ImageNetPretrained/47592356/R-101-GN.pkl", + "FAIR/X-101-32x8d": "ImageNetPretrained/20171220/X-101-32x8d.pkl", + "FAIR/X-101-64x4d": "ImageNetPretrained/FBResNeXt/X-101-64x4d.pkl", + "FAIR/X-152-32x8d-IN5k": "ImageNetPretrained/25093814/X-152-32x8d-IN5k.pkl", + } + + C2_DETECTRON_PATH_FORMAT = "{prefix}/{url}/output/train/{dataset}/{type}/model_final.pkl" # noqa B950 + + C2_DATASET_COCO = "coco_2014_train%3Acoco_2014_valminusminival" + C2_DATASET_COCO_KEYPOINTS = "keypoints_coco_2014_train%3Akeypoints_coco_2014_valminusminival" + + # format: {model_name} -> part of the url + C2_DETECTRON_MODELS = { + "35857197/e2e_faster_rcnn_R-50-C4_1x": "35857197/12_2017_baselines/e2e_faster_rcnn_R-50-C4_1x.yaml.01_33_49.iAX0mXvW", # noqa B950 + "35857345/e2e_faster_rcnn_R-50-FPN_1x": "35857345/12_2017_baselines/e2e_faster_rcnn_R-50-FPN_1x.yaml.01_36_30.cUF7QR7I", # noqa B950 + "35857890/e2e_faster_rcnn_R-101-FPN_1x": "35857890/12_2017_baselines/e2e_faster_rcnn_R-101-FPN_1x.yaml.01_38_50.sNxI7sX7", # noqa B950 + "36761737/e2e_faster_rcnn_X-101-32x8d-FPN_1x": "36761737/12_2017_baselines/e2e_faster_rcnn_X-101-32x8d-FPN_1x.yaml.06_31_39.5MIHi1fZ", # noqa B950 + "35858791/e2e_mask_rcnn_R-50-C4_1x": "35858791/12_2017_baselines/e2e_mask_rcnn_R-50-C4_1x.yaml.01_45_57.ZgkA7hPB", # noqa B950 + "35858933/e2e_mask_rcnn_R-50-FPN_1x": "35858933/12_2017_baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml.01_48_14.DzEQe4wC", # noqa B950 + "35861795/e2e_mask_rcnn_R-101-FPN_1x": "35861795/12_2017_baselines/e2e_mask_rcnn_R-101-FPN_1x.yaml.02_31_37.KqyEK4tT", # noqa B950 + "36761843/e2e_mask_rcnn_X-101-32x8d-FPN_1x": "36761843/12_2017_baselines/e2e_mask_rcnn_X-101-32x8d-FPN_1x.yaml.06_35_59.RZotkLKI", # noqa B950 + "48616381/e2e_mask_rcnn_R-50-FPN_2x_gn": "GN/48616381/04_2018_gn_baselines/e2e_mask_rcnn_R-50-FPN_2x_gn_0416.13_23_38.bTlTI97Q", # noqa B950 + "37697547/e2e_keypoint_rcnn_R-50-FPN_1x": "37697547/12_2017_baselines/e2e_keypoint_rcnn_R-50-FPN_1x.yaml.08_42_54.kdzV35ao", # noqa B950 + "35998355/rpn_R-50-C4_1x": "35998355/12_2017_baselines/rpn_R-50-C4_1x.yaml.08_00_43.njH5oD9L", # noqa B950 + "35998814/rpn_R-50-FPN_1x": "35998814/12_2017_baselines/rpn_R-50-FPN_1x.yaml.08_06_03.Axg0r179", # noqa B950 + "36225147/fast_R-50-FPN_1x": "36225147/12_2017_baselines/fast_rcnn_R-50-FPN_1x.yaml.08_39_09.L3obSdQ2", # noqa B950 + } + + @staticmethod + def get(name): + if name.startswith("Caffe2Detectron/COCO"): + return ModelCatalog._get_c2_detectron_baseline(name) + if name.startswith("ImageNetPretrained/"): + return ModelCatalog._get_c2_imagenet_pretrained(name) + raise RuntimeError("model not present in the catalog: {}".format(name)) + + @staticmethod + def _get_c2_imagenet_pretrained(name): + prefix = ModelCatalog.S3_C2_DETECTRON_PREFIX + name = name[len("ImageNetPretrained/") :] + name = ModelCatalog.C2_IMAGENET_MODELS[name] + url = "/".join([prefix, name]) + return url + + @staticmethod + def _get_c2_detectron_baseline(name): + name = name[len("Caffe2Detectron/COCO/") :] + url = ModelCatalog.C2_DETECTRON_MODELS[name] + if "keypoint_rcnn" in name: + dataset = ModelCatalog.C2_DATASET_COCO_KEYPOINTS + else: + dataset = ModelCatalog.C2_DATASET_COCO + + if "35998355/rpn_R-50-C4_1x" in name: + # this one model is somehow different from others .. + type = "rpn" + else: + type = "generalized_rcnn" + + # Detectron C2 models are stored in the structure defined in `C2_DETECTRON_PATH_FORMAT`. + url = ModelCatalog.C2_DETECTRON_PATH_FORMAT.format( + prefix=ModelCatalog.S3_C2_DETECTRON_PREFIX, url=url, type=type, dataset=dataset + ) + return url + + +class ModelCatalogHandler(PathHandler): + """ + Resolve URL like catalog://. + """ + + PREFIX = "catalog://" + + def _get_supported_prefixes(self): + return [self.PREFIX] + + def _get_local_path(self, path, **kwargs): + logger = logging.getLogger(__name__) + catalog_path = ModelCatalog.get(path[len(self.PREFIX) :]) + logger.info("Catalog entry {} points to {}".format(path, catalog_path)) + return PathManager.get_local_path(catalog_path, **kwargs) + + def _open(self, path, mode="r", **kwargs): + return PathManager.open(self._get_local_path(path), mode, **kwargs) + + +PathManager.register_handler(ModelCatalogHandler()) diff --git a/detectron2/checkpoint/detection_checkpoint.py b/detectron2/checkpoint/detection_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..54c3337113d9d1164016ccffea6ed229a5bb54c2 --- /dev/null +++ b/detectron2/checkpoint/detection_checkpoint.py @@ -0,0 +1,115 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import os +import pickle + +import torch +from fvcore.common.checkpoint import Checkpointer +from torch.nn.parallel import DistributedDataParallel + +import detectron2.utils.comm as comm +from detectron2.utils.file_io import PathManager + +from .c2_model_loading import align_and_update_state_dicts + + +class DetectionCheckpointer(Checkpointer): + """ + Same as :class:`Checkpointer`, but is able to: + 1. handle models in detectron & detectron2 model zoo, and apply conversions for legacy models. + 2. correctly load checkpoints that are only available on the master worker + """ + + def __init__(self, model, save_dir="", *, save_to_disk=None, **checkpointables): + is_main_process = comm.is_main_process() + super().__init__( + model, + save_dir, + save_to_disk=is_main_process if save_to_disk is None else save_to_disk, + **checkpointables, + ) + self.path_manager = PathManager + + def load(self, path, *args, **kwargs): + need_sync = False + + if path and isinstance(self.model, DistributedDataParallel): + logger = logging.getLogger(__name__) + path = self.path_manager.get_local_path(path) + has_file = os.path.isfile(path) + all_has_file = comm.all_gather(has_file) + if not all_has_file[0]: + raise OSError(f"File {path} not found on main worker.") + if not all(all_has_file): + logger.warning(f"Not all workers can read checkpoint {path}. " "Training may fail to fully resume.") + # TODO: broadcast the checkpoint file contents from main + # worker, and load from it instead. + need_sync = True + if not has_file: + path = None # don't load if not readable + ret = super().load(path, *args, **kwargs) + + if need_sync: + logger.info("Broadcasting model states from main worker ...") + self.model._sync_params_and_buffers() + return ret + + def _load_file(self, filename): + if filename.endswith(".pkl"): + with PathManager.open(filename, "rb") as f: + data = pickle.load(f, encoding="latin1") + if "model" in data and "__author__" in data: + # file is in Detectron2 model zoo format + self.logger.info("Reading a file from '{}'".format(data["__author__"])) + return data + else: + # assume file is from Caffe2 / Detectron1 model zoo + if "blobs" in data: + # Detection models have "blobs", but ImageNet models don't + data = data["blobs"] + data = {k: v for k, v in data.items() if not k.endswith("_momentum")} + return {"model": data, "__author__": "Caffe2", "matching_heuristics": True} + elif filename.endswith(".pyth"): + # assume file is from pycls; no one else seems to use the ".pyth" extension + with PathManager.open(filename, "rb") as f: + data = torch.load(f) + assert ( + "model_state" in data + ), f"Cannot load .pyth file {filename}; pycls checkpoints must contain 'model_state'." + model_state = {k: v for k, v in data["model_state"].items() if not k.endswith("num_batches_tracked")} + return {"model": model_state, "__author__": "pycls", "matching_heuristics": True} + + loaded = super()._load_file(filename) # load native pth checkpoint + if "model" not in loaded: + loaded = {"model": loaded} + loaded["matching_heuristics"] = True + return loaded + + def _load_model(self, checkpoint): + if checkpoint.get("matching_heuristics", False): + self._convert_ndarray_to_tensor(checkpoint["model"]) + # convert weights by name-matching heuristics + checkpoint["model"] = align_and_update_state_dicts( + self.model.state_dict(), + checkpoint["model"], + c2_conversion=checkpoint.get("__author__", None) == "Caffe2", + ) + # for non-caffe2 models, use standard ways to load it + incompatible = super()._load_model(checkpoint) + + model_buffers = dict(self.model.named_buffers(recurse=False)) + for k in ["pixel_mean", "pixel_std"]: + # Ignore missing key message about pixel_mean/std. + # Though they may be missing in old checkpoints, they will be correctly + # initialized from config anyway. + if k in model_buffers: + try: + incompatible.missing_keys.remove(k) + except ValueError: + pass + for k in incompatible.unexpected_keys[:]: + # Ignore unexpected keys about cell anchors. They exist in old checkpoints + # but now they are non-persistent buffers and will not be in new checkpoints. + if "anchor_generator.cell_anchors" in k: + incompatible.unexpected_keys.remove(k) + return incompatible diff --git a/detectron2/config/__init__.py b/detectron2/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e6dcf181a0d29a2fa4f1f7f1c7c0431f5ab7c3d9 --- /dev/null +++ b/detectron2/config/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .compat import downgrade_config, upgrade_config +from .config import CfgNode, configurable, get_cfg, global_cfg, set_global_cfg +from .instantiate import instantiate +from .lazy import LazyCall, LazyConfig + +__all__ = [ + "CfgNode", + "get_cfg", + "global_cfg", + "set_global_cfg", + "downgrade_config", + "upgrade_config", + "configurable", + "instantiate", + "LazyCall", + "LazyConfig", +] + + +from detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/detectron2/config/compat.py b/detectron2/config/compat.py new file mode 100644 index 0000000000000000000000000000000000000000..726f49523d0d0d0f843220a49fd49d43384065b1 --- /dev/null +++ b/detectron2/config/compat.py @@ -0,0 +1,221 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Backward compatibility of configs. + +Instructions to bump version: ++ It's not needed to bump version if new keys are added. + It's only needed when backward-incompatible changes happen + (i.e., some existing keys disappear, or the meaning of a key changes) ++ To bump version, do the following: + 1. Increment _C.VERSION in defaults.py + 2. Add a converter in this file. + + Each ConverterVX has a function "upgrade" which in-place upgrades config from X-1 to X, + and a function "downgrade" which in-place downgrades config from X to X-1 + + In each function, VERSION is left unchanged. + + Each converter assumes that its input has the relevant keys + (i.e., the input is not a partial config). + 3. Run the tests (test_config.py) to make sure the upgrade & downgrade + functions are consistent. +""" + +import logging +from typing import List, Optional, Tuple + +from .config import CfgNode as CN +from .defaults import _C + +__all__ = ["upgrade_config", "downgrade_config"] + + +def upgrade_config(cfg: CN, to_version: Optional[int] = None) -> CN: + """ + Upgrade a config from its current version to a newer version. + + Args: + cfg (CfgNode): + to_version (int): defaults to the latest version. + """ + cfg = cfg.clone() + if to_version is None: + to_version = _C.VERSION + + assert cfg.VERSION <= to_version, "Cannot upgrade from v{} to v{}!".format(cfg.VERSION, to_version) + for k in range(cfg.VERSION, to_version): + converter = globals()["ConverterV" + str(k + 1)] + converter.upgrade(cfg) + cfg.VERSION = k + 1 + return cfg + + +def downgrade_config(cfg: CN, to_version: int) -> CN: + """ + Downgrade a config from its current version to an older version. + + Args: + cfg (CfgNode): + to_version (int): + + Note: + A general downgrade of arbitrary configs is not always possible due to the + different functionalities in different versions. + The purpose of downgrade is only to recover the defaults in old versions, + allowing it to load an old partial yaml config. + Therefore, the implementation only needs to fill in the default values + in the old version when a general downgrade is not possible. + """ + cfg = cfg.clone() + assert cfg.VERSION >= to_version, "Cannot downgrade from v{} to v{}!".format(cfg.VERSION, to_version) + for k in range(cfg.VERSION, to_version, -1): + converter = globals()["ConverterV" + str(k)] + converter.downgrade(cfg) + cfg.VERSION = k - 1 + return cfg + + +def guess_version(cfg: CN, filename: str) -> int: + """ + Guess the version of a partial config where the VERSION field is not specified. + Returns the version, or the latest if cannot make a guess. + + This makes it easier for users to migrate. + """ + logger = logging.getLogger(__name__) + + def _has(name: str) -> bool: + cur = cfg + for n in name.split("."): + if n not in cur: + return False + cur = cur[n] + return True + + # Most users' partial configs have "MODEL.WEIGHT", so guess on it + ret = None + if _has("MODEL.WEIGHT") or _has("TEST.AUG_ON"): + ret = 1 + + if ret is not None: + logger.warning("Config '{}' has no VERSION. Assuming it to be v{}.".format(filename, ret)) + else: + ret = _C.VERSION + logger.warning( + "Config '{}' has no VERSION. Assuming it to be compatible with latest v{}.".format(filename, ret) + ) + return ret + + +def _rename(cfg: CN, old: str, new: str) -> None: + old_keys = old.split(".") + new_keys = new.split(".") + + def _set(key_seq: List[str], val: str) -> None: + cur = cfg + for k in key_seq[:-1]: + if k not in cur: + cur[k] = CN() + cur = cur[k] + cur[key_seq[-1]] = val + + def _get(key_seq: List[str]) -> CN: + cur = cfg + for k in key_seq: + cur = cur[k] + return cur + + def _del(key_seq: List[str]) -> None: + cur = cfg + for k in key_seq[:-1]: + cur = cur[k] + del cur[key_seq[-1]] + if len(cur) == 0 and len(key_seq) > 1: + _del(key_seq[:-1]) + + _set(new_keys, _get(old_keys)) + _del(old_keys) + + +class _RenameConverter: + """ + A converter that handles simple rename. + """ + + RENAME: List[Tuple[str, str]] = [] # list of tuples of (old name, new name) + + @classmethod + def upgrade(cls, cfg: CN) -> None: + for old, new in cls.RENAME: + _rename(cfg, old, new) + + @classmethod + def downgrade(cls, cfg: CN) -> None: + for old, new in cls.RENAME[::-1]: + _rename(cfg, new, old) + + +class ConverterV1(_RenameConverter): + RENAME = [("MODEL.RPN_HEAD.NAME", "MODEL.RPN.HEAD_NAME")] + + +class ConverterV2(_RenameConverter): + """ + A large bulk of rename, before public release. + """ + + RENAME = [ + ("MODEL.WEIGHT", "MODEL.WEIGHTS"), + ("MODEL.PANOPTIC_FPN.SEMANTIC_LOSS_SCALE", "MODEL.SEM_SEG_HEAD.LOSS_WEIGHT"), + ("MODEL.PANOPTIC_FPN.RPN_LOSS_SCALE", "MODEL.RPN.LOSS_WEIGHT"), + ("MODEL.PANOPTIC_FPN.INSTANCE_LOSS_SCALE", "MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT"), + ("MODEL.PANOPTIC_FPN.COMBINE_ON", "MODEL.PANOPTIC_FPN.COMBINE.ENABLED"), + ( + "MODEL.PANOPTIC_FPN.COMBINE_OVERLAP_THRESHOLD", + "MODEL.PANOPTIC_FPN.COMBINE.OVERLAP_THRESH", + ), + ( + "MODEL.PANOPTIC_FPN.COMBINE_STUFF_AREA_LIMIT", + "MODEL.PANOPTIC_FPN.COMBINE.STUFF_AREA_LIMIT", + ), + ( + "MODEL.PANOPTIC_FPN.COMBINE_INSTANCES_CONFIDENCE_THRESHOLD", + "MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH", + ), + ("MODEL.ROI_HEADS.SCORE_THRESH", "MODEL.ROI_HEADS.SCORE_THRESH_TEST"), + ("MODEL.ROI_HEADS.NMS", "MODEL.ROI_HEADS.NMS_THRESH_TEST"), + ("MODEL.RETINANET.INFERENCE_SCORE_THRESHOLD", "MODEL.RETINANET.SCORE_THRESH_TEST"), + ("MODEL.RETINANET.INFERENCE_TOPK_CANDIDATES", "MODEL.RETINANET.TOPK_CANDIDATES_TEST"), + ("MODEL.RETINANET.INFERENCE_NMS_THRESHOLD", "MODEL.RETINANET.NMS_THRESH_TEST"), + ("TEST.DETECTIONS_PER_IMG", "TEST.DETECTIONS_PER_IMAGE"), + ("TEST.AUG_ON", "TEST.AUG.ENABLED"), + ("TEST.AUG_MIN_SIZES", "TEST.AUG.MIN_SIZES"), + ("TEST.AUG_MAX_SIZE", "TEST.AUG.MAX_SIZE"), + ("TEST.AUG_FLIP", "TEST.AUG.FLIP"), + ] + + @classmethod + def upgrade(cls, cfg: CN) -> None: + super().upgrade(cfg) + + if cfg.MODEL.META_ARCHITECTURE == "RetinaNet": + _rename(cfg, "MODEL.RETINANET.ANCHOR_ASPECT_RATIOS", "MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS") + _rename(cfg, "MODEL.RETINANET.ANCHOR_SIZES", "MODEL.ANCHOR_GENERATOR.SIZES") + del cfg["MODEL"]["RPN"]["ANCHOR_SIZES"] + del cfg["MODEL"]["RPN"]["ANCHOR_ASPECT_RATIOS"] + else: + _rename(cfg, "MODEL.RPN.ANCHOR_ASPECT_RATIOS", "MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS") + _rename(cfg, "MODEL.RPN.ANCHOR_SIZES", "MODEL.ANCHOR_GENERATOR.SIZES") + del cfg["MODEL"]["RETINANET"]["ANCHOR_SIZES"] + del cfg["MODEL"]["RETINANET"]["ANCHOR_ASPECT_RATIOS"] + del cfg["MODEL"]["RETINANET"]["ANCHOR_STRIDES"] + + @classmethod + def downgrade(cls, cfg: CN) -> None: + super().downgrade(cfg) + + _rename(cfg, "MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS", "MODEL.RPN.ANCHOR_ASPECT_RATIOS") + _rename(cfg, "MODEL.ANCHOR_GENERATOR.SIZES", "MODEL.RPN.ANCHOR_SIZES") + cfg.MODEL.RETINANET.ANCHOR_ASPECT_RATIOS = cfg.MODEL.RPN.ANCHOR_ASPECT_RATIOS + cfg.MODEL.RETINANET.ANCHOR_SIZES = cfg.MODEL.RPN.ANCHOR_SIZES + cfg.MODEL.RETINANET.ANCHOR_STRIDES = [] # this is not used anywhere in any version diff --git a/detectron2/config/config.py b/detectron2/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..945b786d7f72720b6545f5f0b878fa796cb67251 --- /dev/null +++ b/detectron2/config/config.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import functools +import inspect +import logging + +from fvcore.common.config import CfgNode as _CfgNode + +from detectron2.utils.file_io import PathManager + + +class CfgNode(_CfgNode): + """ + The same as `fvcore.common.config.CfgNode`, but different in: + + 1. Use unsafe yaml loading by default. + Note that this may lead to arbitrary code execution: you must not + load a config file from untrusted sources before manually inspecting + the content of the file. + 2. Support config versioning. + When attempting to merge an old config, it will convert the old config automatically. + + .. automethod:: clone + .. automethod:: freeze + .. automethod:: defrost + .. automethod:: is_frozen + .. automethod:: load_yaml_with_base + .. automethod:: merge_from_list + .. automethod:: merge_from_other_cfg + """ + + @classmethod + def _open_cfg(cls, filename): + return PathManager.open(filename, "r") + + # Note that the default value of allow_unsafe is changed to True + def merge_from_file(self, cfg_filename: str, allow_unsafe: bool = True) -> None: + """ + Load content from the given config file and merge it into self. + + Args: + cfg_filename: config filename + allow_unsafe: allow unsafe yaml syntax + """ + assert PathManager.isfile(cfg_filename), f"Config file '{cfg_filename}' does not exist!" + loaded_cfg = self.load_yaml_with_base(cfg_filename, allow_unsafe=allow_unsafe) + loaded_cfg = type(self)(loaded_cfg) + + # defaults.py needs to import CfgNode + from .defaults import _C + + latest_ver = _C.VERSION + assert ( + latest_ver == self.VERSION + ), "CfgNode.merge_from_file is only allowed on a config object of latest version!" + + logger = logging.getLogger(__name__) + + loaded_ver = loaded_cfg.get("VERSION", None) + if loaded_ver is None: + from .compat import guess_version + + loaded_ver = guess_version(loaded_cfg, cfg_filename) + assert loaded_ver <= self.VERSION, "Cannot merge a v{} config into a v{} config.".format( + loaded_ver, self.VERSION + ) + + if loaded_ver == self.VERSION: + self.merge_from_other_cfg(loaded_cfg) + else: + # compat.py needs to import CfgNode + from .compat import downgrade_config, upgrade_config + + logger.warning( + "Loading an old v{} config file '{}' by automatically upgrading to v{}. " + "See docs/CHANGELOG.md for instructions to update your files.".format( + loaded_ver, cfg_filename, self.VERSION + ) + ) + # To convert, first obtain a full config at an old version + old_self = downgrade_config(self, to_version=loaded_ver) + old_self.merge_from_other_cfg(loaded_cfg) + new_config = upgrade_config(old_self) + self.clear() + self.update(new_config) + + def dump(self, *args, **kwargs): + """ + Returns: + str: a yaml string representation of the config + """ + # to make it show up in docs + return super().dump(*args, **kwargs) + + +global_cfg = CfgNode() + + +def get_cfg() -> CfgNode: + """ + Get a copy of the default config. + + Returns: + a detectron2 CfgNode instance. + """ + from .defaults import _C + + return _C.clone() + + +def set_global_cfg(cfg: CfgNode) -> None: + """ + Let the global config point to the given cfg. + + Assume that the given "cfg" has the key "KEY", after calling + `set_global_cfg(cfg)`, the key can be accessed by: + :: + from detectron2.config import global_cfg + print(global_cfg.KEY) + + By using a hacky global config, you can access these configs anywhere, + without having to pass the config object or the values deep into the code. + This is a hacky feature introduced for quick prototyping / research exploration. + """ + global global_cfg + global_cfg.clear() + global_cfg.update(cfg) + + +def configurable(init_func=None, *, from_config=None): + """ + Decorate a function or a class's __init__ method so that it can be called + with a :class:`CfgNode` object using a :func:`from_config` function that translates + :class:`CfgNode` to arguments. + + Examples: + :: + # Usage 1: Decorator on __init__: + class A: + @configurable + def __init__(self, a, b=2, c=3): + pass + + @classmethod + def from_config(cls, cfg): # 'cfg' must be the first argument + # Returns kwargs to be passed to __init__ + return {"a": cfg.A, "b": cfg.B} + + a1 = A(a=1, b=2) # regular construction + a2 = A(cfg) # construct with a cfg + a3 = A(cfg, b=3, c=4) # construct with extra overwrite + + # Usage 2: Decorator on any function. Needs an extra from_config argument: + @configurable(from_config=lambda cfg: {"a: cfg.A, "b": cfg.B}) + def a_func(a, b=2, c=3): + pass + + a1 = a_func(a=1, b=2) # regular call + a2 = a_func(cfg) # call with a cfg + a3 = a_func(cfg, b=3, c=4) # call with extra overwrite + + Args: + init_func (callable): a class's ``__init__`` method in usage 1. The + class must have a ``from_config`` classmethod which takes `cfg` as + the first argument. + from_config (callable): the from_config function in usage 2. It must take `cfg` + as its first argument. + """ + + if init_func is not None: + assert ( + inspect.isfunction(init_func) and from_config is None and init_func.__name__ == "__init__" + ), "Incorrect use of @configurable. Check API documentation for examples." + + @functools.wraps(init_func) + def wrapped(self, *args, **kwargs): + try: + from_config_func = type(self).from_config + except AttributeError as e: + raise AttributeError("Class with @configurable must have a 'from_config' classmethod.") from e + if not inspect.ismethod(from_config_func): + raise TypeError("Class with @configurable must have a 'from_config' classmethod.") + + if _called_with_cfg(*args, **kwargs): + explicit_args = _get_args_from_config(from_config_func, *args, **kwargs) + init_func(self, **explicit_args) + else: + init_func(self, *args, **kwargs) + + return wrapped + + else: + if from_config is None: + return configurable # @configurable() is made equivalent to @configurable + assert inspect.isfunction(from_config), "from_config argument of configurable must be a function!" + + def wrapper(orig_func): + @functools.wraps(orig_func) + def wrapped(*args, **kwargs): + if _called_with_cfg(*args, **kwargs): + explicit_args = _get_args_from_config(from_config, *args, **kwargs) + return orig_func(**explicit_args) + else: + return orig_func(*args, **kwargs) + + wrapped.from_config = from_config + return wrapped + + return wrapper + + +def _get_args_from_config(from_config_func, *args, **kwargs): + """ + Use `from_config` to obtain explicit arguments. + + Returns: + dict: arguments to be used for cls.__init__ + """ + signature = inspect.signature(from_config_func) + if list(signature.parameters.keys())[0] != "cfg": + if inspect.isfunction(from_config_func): + name = from_config_func.__name__ + else: + name = f"{from_config_func.__self__}.from_config" + raise TypeError(f"{name} must take 'cfg' as the first argument!") + support_var_arg = any( + param.kind in [param.VAR_POSITIONAL, param.VAR_KEYWORD] for param in signature.parameters.values() + ) + if support_var_arg: # forward all arguments to from_config, if from_config accepts them + ret = from_config_func(*args, **kwargs) + else: + # forward supported arguments to from_config + supported_arg_names = set(signature.parameters.keys()) + extra_kwargs = {} + for name in list(kwargs.keys()): + if name not in supported_arg_names: + extra_kwargs[name] = kwargs.pop(name) + ret = from_config_func(*args, **kwargs) + # forward the other arguments to __init__ + ret.update(extra_kwargs) + return ret + + +def _called_with_cfg(*args, **kwargs): + """ + Returns: + bool: whether the arguments contain CfgNode and should be considered + forwarded to from_config. + """ + from omegaconf import DictConfig + + if len(args) and isinstance(args[0], (_CfgNode, DictConfig)): + return True + if isinstance(kwargs.pop("cfg", None), (_CfgNode, DictConfig)): + return True + # `from_config`'s first argument is forced to be "cfg". + # So the above check covers all cases. + return False diff --git a/detectron2/config/defaults.py b/detectron2/config/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..ea58627ef038f187a353122626742af08a298cd5 --- /dev/null +++ b/detectron2/config/defaults.py @@ -0,0 +1,646 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .config import CfgNode as CN + +# NOTE: given the new config system +# (https://detectron2.readthedocs.io/en/latest/tutorials/lazyconfigs.html), +# we will stop adding new functionalities to default CfgNode. + +# ----------------------------------------------------------------------------- +# Convention about Training / Test specific parameters +# ----------------------------------------------------------------------------- +# Whenever an argument can be either used for training or for testing, the +# corresponding name will be post-fixed by a _TRAIN for a training parameter, +# or _TEST for a test-specific parameter. +# For example, the number of images during training will be +# IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be +# IMAGES_PER_BATCH_TEST + +# ----------------------------------------------------------------------------- +# Config definition +# ----------------------------------------------------------------------------- + +_C = CN() + +# The version number, to upgrade from old configs to new ones if any +# changes happen. It's recommended to keep a VERSION in your config file. +_C.VERSION = 2 + +_C.MODEL = CN() +_C.MODEL.LOAD_PROPOSALS = False +_C.MODEL.MASK_ON = False +_C.MODEL.KEYPOINT_ON = False +_C.MODEL.DEVICE = "cuda" +_C.MODEL.META_ARCHITECTURE = "GeneralizedRCNN" + +# Path (a file path, or URL like detectron2://.., https://..) to a checkpoint file +# to be loaded to the model. You can find available models in the model zoo. +_C.MODEL.WEIGHTS = "" + +# Values to be used for image normalization (BGR order, since INPUT.FORMAT defaults to BGR). +# To train on images of different number of channels, just set different mean & std. +# Default values are the mean pixel value from ImageNet: [103.53, 116.28, 123.675] +_C.MODEL.PIXEL_MEAN = [103.530, 116.280, 123.675] +# When using pre-trained models in Detectron1 or any MSRA models, +# std has been absorbed into its conv1 weights, so the std needs to be set 1. +# Otherwise, you can use [57.375, 57.120, 58.395] (ImageNet std) +_C.MODEL.PIXEL_STD = [1.0, 1.0, 1.0] + + +# ----------------------------------------------------------------------------- +# INPUT +# ----------------------------------------------------------------------------- +_C.INPUT = CN() +# By default, {MIN,MAX}_SIZE options are used in transforms.ResizeShortestEdge. +# Please refer to ResizeShortestEdge for detailed definition. +# Size of the smallest side of the image during training +_C.INPUT.MIN_SIZE_TRAIN = (800,) +# Sample size of smallest side by choice or random selection from range give by +# INPUT.MIN_SIZE_TRAIN +_C.INPUT.MIN_SIZE_TRAIN_SAMPLING = "choice" +# Maximum size of the side of the image during training +_C.INPUT.MAX_SIZE_TRAIN = 1333 +# Size of the smallest side of the image during testing. Set to zero to disable resize in testing. +_C.INPUT.MIN_SIZE_TEST = 800 +# Maximum size of the side of the image during testing +_C.INPUT.MAX_SIZE_TEST = 1333 +# Mode for flipping images used in data augmentation during training +# choose one of ["horizontal, "vertical", "none"] +_C.INPUT.RANDOM_FLIP = "horizontal" + +# `True` if cropping is used for data augmentation during training +_C.INPUT.CROP = CN({"ENABLED": False}) +# Cropping type. See documentation of `detectron2.data.transforms.RandomCrop` for explanation. +_C.INPUT.CROP.TYPE = "relative_range" +# Size of crop in range (0, 1] if CROP.TYPE is "relative" or "relative_range" and in number of +# pixels if CROP.TYPE is "absolute" +_C.INPUT.CROP.SIZE = [0.9, 0.9] + + +# Whether the model needs RGB, YUV, HSV etc. +# Should be one of the modes defined here, as we use PIL to read the image: +# https://pillow.readthedocs.io/en/stable/handbook/concepts.html#concept-modes +# with BGR being the one exception. One can set image format to BGR, we will +# internally use RGB for conversion and flip the channels over +_C.INPUT.FORMAT = "BGR" +# The ground truth mask format that the model will use. +# Mask R-CNN supports either "polygon" or "bitmask" as ground truth. +_C.INPUT.MASK_FORMAT = "polygon" # alternative: "bitmask" + + +# ----------------------------------------------------------------------------- +# Dataset +# ----------------------------------------------------------------------------- +_C.DATASETS = CN() +# List of the dataset names for training. Must be registered in DatasetCatalog +# Samples from these datasets will be merged and used as one dataset. +_C.DATASETS.TRAIN = () +# List of the pre-computed proposal files for training, which must be consistent +# with datasets listed in DATASETS.TRAIN. +_C.DATASETS.PROPOSAL_FILES_TRAIN = () +# Number of top scoring precomputed proposals to keep for training +_C.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN = 2000 +# List of the dataset names for testing. Must be registered in DatasetCatalog +_C.DATASETS.TEST = () +# List of the pre-computed proposal files for test, which must be consistent +# with datasets listed in DATASETS.TEST. +_C.DATASETS.PROPOSAL_FILES_TEST = () +# Number of top scoring precomputed proposals to keep for test +_C.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST = 1000 + +# ----------------------------------------------------------------------------- +# DataLoader +# ----------------------------------------------------------------------------- +_C.DATALOADER = CN() +# Number of data loading threads +_C.DATALOADER.NUM_WORKERS = 4 +# If True, each batch should contain only images for which the aspect ratio +# is compatible. This groups portrait images together, and landscape images +# are not batched with portrait images. +_C.DATALOADER.ASPECT_RATIO_GROUPING = True +# Options: TrainingSampler, RepeatFactorTrainingSampler +_C.DATALOADER.SAMPLER_TRAIN = "TrainingSampler" +# Repeat threshold for RepeatFactorTrainingSampler +_C.DATALOADER.REPEAT_THRESHOLD = 0.0 +# Tf True, when working on datasets that have instance annotations, the +# training dataloader will filter out images without associated annotations +_C.DATALOADER.FILTER_EMPTY_ANNOTATIONS = True + +# ---------------------------------------------------------------------------- # +# Backbone options +# ---------------------------------------------------------------------------- # +_C.MODEL.BACKBONE = CN() + +_C.MODEL.BACKBONE.NAME = "build_resnet_backbone" +# Freeze the first several stages so they are not trained. +# There are 5 stages in ResNet. The first is a convolution, and the following +# stages are each group of residual blocks. +_C.MODEL.BACKBONE.FREEZE_AT = 2 + + +# ---------------------------------------------------------------------------- # +# FPN options +# ---------------------------------------------------------------------------- # +_C.MODEL.FPN = CN() +# Names of the input feature maps to be used by FPN +# They must have contiguous power of 2 strides +# e.g., ["res2", "res3", "res4", "res5"] +_C.MODEL.FPN.IN_FEATURES = [] +_C.MODEL.FPN.OUT_CHANNELS = 256 + +# Options: "" (no norm), "GN" +_C.MODEL.FPN.NORM = "" + +# Types for fusing the FPN top-down and lateral features. Can be either "sum" or "avg" +_C.MODEL.FPN.FUSE_TYPE = "sum" + + +# ---------------------------------------------------------------------------- # +# Proposal generator options +# ---------------------------------------------------------------------------- # +_C.MODEL.PROPOSAL_GENERATOR = CN() +# Current proposal generators include "RPN", "RRPN" and "PrecomputedProposals" +_C.MODEL.PROPOSAL_GENERATOR.NAME = "RPN" +# Proposal height and width both need to be greater than MIN_SIZE +# (a the scale used during training or inference) +_C.MODEL.PROPOSAL_GENERATOR.MIN_SIZE = 0 + + +# ---------------------------------------------------------------------------- # +# Anchor generator options +# ---------------------------------------------------------------------------- # +_C.MODEL.ANCHOR_GENERATOR = CN() +# The generator can be any name in the ANCHOR_GENERATOR registry +_C.MODEL.ANCHOR_GENERATOR.NAME = "DefaultAnchorGenerator" +# Anchor sizes (i.e. sqrt of area) in absolute pixels w.r.t. the network input. +# Format: list[list[float]]. SIZES[i] specifies the list of sizes to use for +# IN_FEATURES[i]; len(SIZES) must be equal to len(IN_FEATURES) or 1. +# When len(SIZES) == 1, SIZES[0] is used for all IN_FEATURES. +_C.MODEL.ANCHOR_GENERATOR.SIZES = [[32, 64, 128, 256, 512]] +# Anchor aspect ratios. For each area given in `SIZES`, anchors with different aspect +# ratios are generated by an anchor generator. +# Format: list[list[float]]. ASPECT_RATIOS[i] specifies the list of aspect ratios (H/W) +# to use for IN_FEATURES[i]; len(ASPECT_RATIOS) == len(IN_FEATURES) must be true, +# or len(ASPECT_RATIOS) == 1 is true and aspect ratio list ASPECT_RATIOS[0] is used +# for all IN_FEATURES. +_C.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.5, 1.0, 2.0]] +# Anchor angles. +# list[list[float]], the angle in degrees, for each input feature map. +# ANGLES[i] specifies the list of angles for IN_FEATURES[i]. +_C.MODEL.ANCHOR_GENERATOR.ANGLES = [[-90, 0, 90]] +# Relative offset between the center of the first anchor and the top-left corner of the image +# Value has to be in [0, 1). Recommend to use 0.5, which means half stride. +# The value is not expected to affect model accuracy. +_C.MODEL.ANCHOR_GENERATOR.OFFSET = 0.0 + +# ---------------------------------------------------------------------------- # +# RPN options +# ---------------------------------------------------------------------------- # +_C.MODEL.RPN = CN() +_C.MODEL.RPN.HEAD_NAME = "StandardRPNHead" # used by RPN_HEAD_REGISTRY + +# Names of the input feature maps to be used by RPN +# e.g., ["p2", "p3", "p4", "p5", "p6"] for FPN +_C.MODEL.RPN.IN_FEATURES = ["res4"] +# Remove RPN anchors that go outside the image by BOUNDARY_THRESH pixels +# Set to -1 or a large value, e.g. 100000, to disable pruning anchors +_C.MODEL.RPN.BOUNDARY_THRESH = -1 +# IOU overlap ratios [BG_IOU_THRESHOLD, FG_IOU_THRESHOLD] +# Minimum overlap required between an anchor and ground-truth box for the +# (anchor, gt box) pair to be a positive example (IoU >= FG_IOU_THRESHOLD +# ==> positive RPN example: 1) +# Maximum overlap allowed between an anchor and ground-truth box for the +# (anchor, gt box) pair to be a negative examples (IoU < BG_IOU_THRESHOLD +# ==> negative RPN example: 0) +# Anchors with overlap in between (BG_IOU_THRESHOLD <= IoU < FG_IOU_THRESHOLD) +# are ignored (-1) +_C.MODEL.RPN.IOU_THRESHOLDS = [0.3, 0.7] +_C.MODEL.RPN.IOU_LABELS = [0, -1, 1] +# Number of regions per image used to train RPN +_C.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 256 +# Target fraction of foreground (positive) examples per RPN minibatch +_C.MODEL.RPN.POSITIVE_FRACTION = 0.5 +# Options are: "smooth_l1", "giou", "diou", "ciou" +_C.MODEL.RPN.BBOX_REG_LOSS_TYPE = "smooth_l1" +_C.MODEL.RPN.BBOX_REG_LOSS_WEIGHT = 1.0 +# Weights on (dx, dy, dw, dh) for normalizing RPN anchor regression targets +_C.MODEL.RPN.BBOX_REG_WEIGHTS = (1.0, 1.0, 1.0, 1.0) +# The transition point from L1 to L2 loss. Set to 0.0 to make the loss simply L1. +_C.MODEL.RPN.SMOOTH_L1_BETA = 0.0 +_C.MODEL.RPN.LOSS_WEIGHT = 1.0 +# Number of top scoring RPN proposals to keep before applying NMS +# When FPN is used, this is *per FPN level* (not total) +_C.MODEL.RPN.PRE_NMS_TOPK_TRAIN = 12000 +_C.MODEL.RPN.PRE_NMS_TOPK_TEST = 6000 +# Number of top scoring RPN proposals to keep after applying NMS +# When FPN is used, this limit is applied per level and then again to the union +# of proposals from all levels +# NOTE: When FPN is used, the meaning of this config is different from Detectron1. +# It means per-batch topk in Detectron1, but per-image topk here. +# See the "find_top_rpn_proposals" function for details. +_C.MODEL.RPN.POST_NMS_TOPK_TRAIN = 2000 +_C.MODEL.RPN.POST_NMS_TOPK_TEST = 1000 +# NMS threshold used on RPN proposals +_C.MODEL.RPN.NMS_THRESH = 0.7 +# Set this to -1 to use the same number of output channels as input channels. +_C.MODEL.RPN.CONV_DIMS = [-1] + +# ---------------------------------------------------------------------------- # +# ROI HEADS options +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_HEADS = CN() +_C.MODEL.ROI_HEADS.NAME = "Res5ROIHeads" +# Number of foreground classes +_C.MODEL.ROI_HEADS.NUM_CLASSES = 80 +# Names of the input feature maps to be used by ROI heads +# Currently all heads (box, mask, ...) use the same input feature map list +# e.g., ["p2", "p3", "p4", "p5"] is commonly used for FPN +_C.MODEL.ROI_HEADS.IN_FEATURES = ["res4"] +# IOU overlap ratios [IOU_THRESHOLD] +# Overlap threshold for an RoI to be considered background (if < IOU_THRESHOLD) +# Overlap threshold for an RoI to be considered foreground (if >= IOU_THRESHOLD) +_C.MODEL.ROI_HEADS.IOU_THRESHOLDS = [0.5] +_C.MODEL.ROI_HEADS.IOU_LABELS = [0, 1] +# RoI minibatch size *per image* (number of regions of interest [ROIs]) during training +# Total number of RoIs per training minibatch = +# ROI_HEADS.BATCH_SIZE_PER_IMAGE * SOLVER.IMS_PER_BATCH +# E.g., a common configuration is: 512 * 16 = 8192 +_C.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512 +# Target fraction of RoI minibatch that is labeled foreground (i.e. class > 0) +_C.MODEL.ROI_HEADS.POSITIVE_FRACTION = 0.25 + +# Only used on test mode + +# Minimum score threshold (assuming scores in a [0, 1] range); a value chosen to +# balance obtaining high recall with not having too many low precision +# detections that will slow down inference post processing steps (like NMS) +# A default threshold of 0.0 increases AP by ~0.2-0.3 but significantly slows down +# inference. +_C.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.05 +# Overlap threshold used for non-maximum suppression (suppress boxes with +# IoU >= this threshold) +_C.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.5 +# If True, augment proposals with ground-truth boxes before sampling proposals to +# train ROI heads. +_C.MODEL.ROI_HEADS.PROPOSAL_APPEND_GT = True + +# ---------------------------------------------------------------------------- # +# Box Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_BOX_HEAD = CN() +# C4 don't use head name option +# Options for non-C4 models: FastRCNNConvFCHead, +_C.MODEL.ROI_BOX_HEAD.NAME = "" +# Options are: "smooth_l1", "giou", "diou", "ciou" +_C.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = "smooth_l1" +# The final scaling coefficient on the box regression loss, used to balance the magnitude of its +# gradients with other losses in the model. See also `MODEL.ROI_KEYPOINT_HEAD.LOSS_WEIGHT`. +_C.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_WEIGHT = 1.0 +# Default weights on (dx, dy, dw, dh) for normalizing bbox regression targets +# These are empirically chosen to approximately lead to unit variance targets +_C.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS = (10.0, 10.0, 5.0, 5.0) +# The transition point from L1 to L2 loss. Set to 0.0 to make the loss simply L1. +_C.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA = 0.0 +_C.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION = 14 +_C.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO = 0 +# Type of pooling operation applied to the incoming feature map for each RoI +_C.MODEL.ROI_BOX_HEAD.POOLER_TYPE = "ROIAlignV2" + +_C.MODEL.ROI_BOX_HEAD.NUM_FC = 0 +# Hidden layer dimension for FC layers in the RoI box head +_C.MODEL.ROI_BOX_HEAD.FC_DIM = 1024 +_C.MODEL.ROI_BOX_HEAD.NUM_CONV = 0 +# Channel dimension for Conv layers in the RoI box head +_C.MODEL.ROI_BOX_HEAD.CONV_DIM = 256 +# Normalization method for the convolution layers. +# Options: "" (no norm), "GN", "SyncBN". +_C.MODEL.ROI_BOX_HEAD.NORM = "" +# Whether to use class agnostic for bbox regression +_C.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG = False +# If true, RoI heads use bounding boxes predicted by the box head rather than proposal boxes. +_C.MODEL.ROI_BOX_HEAD.TRAIN_ON_PRED_BOXES = False + +# Federated loss can be used to improve the training of LVIS +_C.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False +# Sigmoid cross entrophy is used with federated loss +_C.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE = False +# The power value applied to image_count when calcualting frequency weight +_C.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT_POWER = 0.5 +# Number of classes to keep in total +_C.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CLASSES = 50 + +# ---------------------------------------------------------------------------- # +# Cascaded Box Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_BOX_CASCADE_HEAD = CN() +# The number of cascade stages is implicitly defined by the length of the following two configs. +_C.MODEL.ROI_BOX_CASCADE_HEAD.BBOX_REG_WEIGHTS = ( + (10.0, 10.0, 5.0, 5.0), + (20.0, 20.0, 10.0, 10.0), + (30.0, 30.0, 15.0, 15.0), +) +_C.MODEL.ROI_BOX_CASCADE_HEAD.IOUS = (0.5, 0.6, 0.7) + + +# ---------------------------------------------------------------------------- # +# Mask Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_MASK_HEAD = CN() +_C.MODEL.ROI_MASK_HEAD.NAME = "MaskRCNNConvUpsampleHead" +_C.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION = 14 +_C.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO = 0 +_C.MODEL.ROI_MASK_HEAD.NUM_CONV = 0 # The number of convs in the mask head +_C.MODEL.ROI_MASK_HEAD.CONV_DIM = 256 +# Normalization method for the convolution layers. +# Options: "" (no norm), "GN", "SyncBN". +_C.MODEL.ROI_MASK_HEAD.NORM = "" +# Whether to use class agnostic for mask prediction +_C.MODEL.ROI_MASK_HEAD.CLS_AGNOSTIC_MASK = False +# Type of pooling operation applied to the incoming feature map for each RoI +_C.MODEL.ROI_MASK_HEAD.POOLER_TYPE = "ROIAlignV2" + + +# ---------------------------------------------------------------------------- # +# Keypoint Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_KEYPOINT_HEAD = CN() +_C.MODEL.ROI_KEYPOINT_HEAD.NAME = "KRCNNConvDeconvUpsampleHead" +_C.MODEL.ROI_KEYPOINT_HEAD.POOLER_RESOLUTION = 14 +_C.MODEL.ROI_KEYPOINT_HEAD.POOLER_SAMPLING_RATIO = 0 +_C.MODEL.ROI_KEYPOINT_HEAD.CONV_DIMS = tuple(512 for _ in range(8)) +_C.MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS = 17 # 17 is the number of keypoints in COCO. + +# Images with too few (or no) keypoints are excluded from training. +_C.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE = 1 +# Normalize by the total number of visible keypoints in the minibatch if True. +# Otherwise, normalize by the total number of keypoints that could ever exist +# in the minibatch. +# The keypoint softmax loss is only calculated on visible keypoints. +# Since the number of visible keypoints can vary significantly between +# minibatches, this has the effect of up-weighting the importance of +# minibatches with few visible keypoints. (Imagine the extreme case of +# only one visible keypoint versus N: in the case of N, each one +# contributes 1/N to the gradient compared to the single keypoint +# determining the gradient direction). Instead, we can normalize the +# loss by the total number of keypoints, if it were the case that all +# keypoints were visible in a full minibatch. (Returning to the example, +# this means that the one visible keypoint contributes as much as each +# of the N keypoints.) +_C.MODEL.ROI_KEYPOINT_HEAD.NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS = True +# Multi-task loss weight to use for keypoints +# Recommended values: +# - use 1.0 if NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS is True +# - use 4.0 if NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS is False +_C.MODEL.ROI_KEYPOINT_HEAD.LOSS_WEIGHT = 1.0 +# Type of pooling operation applied to the incoming feature map for each RoI +_C.MODEL.ROI_KEYPOINT_HEAD.POOLER_TYPE = "ROIAlignV2" + +# ---------------------------------------------------------------------------- # +# Semantic Segmentation Head +# ---------------------------------------------------------------------------- # +_C.MODEL.SEM_SEG_HEAD = CN() +_C.MODEL.SEM_SEG_HEAD.NAME = "SemSegFPNHead" +_C.MODEL.SEM_SEG_HEAD.IN_FEATURES = ["p2", "p3", "p4", "p5"] +# Label in the semantic segmentation ground truth that is ignored, i.e., no loss is calculated for +# the correposnding pixel. +_C.MODEL.SEM_SEG_HEAD.IGNORE_VALUE = 255 +# Number of classes in the semantic segmentation head +_C.MODEL.SEM_SEG_HEAD.NUM_CLASSES = 54 +# Number of channels in the 3x3 convs inside semantic-FPN heads. +_C.MODEL.SEM_SEG_HEAD.CONVS_DIM = 128 +# Outputs from semantic-FPN heads are up-scaled to the COMMON_STRIDE stride. +_C.MODEL.SEM_SEG_HEAD.COMMON_STRIDE = 4 +# Normalization method for the convolution layers. Options: "" (no norm), "GN". +_C.MODEL.SEM_SEG_HEAD.NORM = "GN" +_C.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT = 1.0 + +_C.MODEL.PANOPTIC_FPN = CN() +# Scaling of all losses from instance detection / segmentation head. +_C.MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT = 1.0 + +# options when combining instance & semantic segmentation outputs +_C.MODEL.PANOPTIC_FPN.COMBINE = CN({"ENABLED": True}) # "COMBINE.ENABLED" is deprecated & not used +_C.MODEL.PANOPTIC_FPN.COMBINE.OVERLAP_THRESH = 0.5 +_C.MODEL.PANOPTIC_FPN.COMBINE.STUFF_AREA_LIMIT = 4096 +_C.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = 0.5 + + +# ---------------------------------------------------------------------------- # +# RetinaNet Head +# ---------------------------------------------------------------------------- # +_C.MODEL.RETINANET = CN() + +# This is the number of foreground classes. +_C.MODEL.RETINANET.NUM_CLASSES = 80 + +_C.MODEL.RETINANET.IN_FEATURES = ["p3", "p4", "p5", "p6", "p7"] + +# Convolutions to use in the cls and bbox tower +# NOTE: this doesn't include the last conv for logits +_C.MODEL.RETINANET.NUM_CONVS = 4 + +# IoU overlap ratio [bg, fg] for labeling anchors. +# Anchors with < bg are labeled negative (0) +# Anchors with >= bg and < fg are ignored (-1) +# Anchors with >= fg are labeled positive (1) +_C.MODEL.RETINANET.IOU_THRESHOLDS = [0.4, 0.5] +_C.MODEL.RETINANET.IOU_LABELS = [0, -1, 1] + +# Prior prob for rare case (i.e. foreground) at the beginning of training. +# This is used to set the bias for the logits layer of the classifier subnet. +# This improves training stability in the case of heavy class imbalance. +_C.MODEL.RETINANET.PRIOR_PROB = 0.01 + +# Inference cls score threshold, only anchors with score > INFERENCE_TH are +# considered for inference (to improve speed) +_C.MODEL.RETINANET.SCORE_THRESH_TEST = 0.05 +# Select topk candidates before NMS +_C.MODEL.RETINANET.TOPK_CANDIDATES_TEST = 1000 +_C.MODEL.RETINANET.NMS_THRESH_TEST = 0.5 + +# Weights on (dx, dy, dw, dh) for normalizing Retinanet anchor regression targets +_C.MODEL.RETINANET.BBOX_REG_WEIGHTS = (1.0, 1.0, 1.0, 1.0) + +# Loss parameters +_C.MODEL.RETINANET.FOCAL_LOSS_GAMMA = 2.0 +_C.MODEL.RETINANET.FOCAL_LOSS_ALPHA = 0.25 +_C.MODEL.RETINANET.SMOOTH_L1_LOSS_BETA = 0.1 +# Options are: "smooth_l1", "giou", "diou", "ciou" +_C.MODEL.RETINANET.BBOX_REG_LOSS_TYPE = "smooth_l1" + +# One of BN, SyncBN, FrozenBN, GN +# Only supports GN until unshared norm is implemented +_C.MODEL.RETINANET.NORM = "" + + +# ---------------------------------------------------------------------------- # +# ResNe[X]t options (ResNets = {ResNet, ResNeXt} +# Note that parts of a resnet may be used for both the backbone and the head +# These options apply to both +# ---------------------------------------------------------------------------- # +_C.MODEL.RESNETS = CN() + +_C.MODEL.RESNETS.DEPTH = 50 +_C.MODEL.RESNETS.OUT_FEATURES = ["res4"] # res4 for C4 backbone, res2..5 for FPN backbone + +# Number of groups to use; 1 ==> ResNet; > 1 ==> ResNeXt +_C.MODEL.RESNETS.NUM_GROUPS = 1 + +# Options: FrozenBN, GN, "SyncBN", "BN" +_C.MODEL.RESNETS.NORM = "FrozenBN" + +# Baseline width of each group. +# Scaling this parameters will scale the width of all bottleneck layers. +_C.MODEL.RESNETS.WIDTH_PER_GROUP = 64 + +# Place the stride 2 conv on the 1x1 filter +# Use True only for the original MSRA ResNet; use False for C2 and Torch models +_C.MODEL.RESNETS.STRIDE_IN_1X1 = True + +# Apply dilation in stage "res5" +_C.MODEL.RESNETS.RES5_DILATION = 1 + +# Output width of res2. Scaling this parameters will scale the width of all 1x1 convs in ResNet +# For R18 and R34, this needs to be set to 64 +_C.MODEL.RESNETS.RES2_OUT_CHANNELS = 256 +_C.MODEL.RESNETS.STEM_OUT_CHANNELS = 64 + +# Apply Deformable Convolution in stages +# Specify if apply deform_conv on Res2, Res3, Res4, Res5 +_C.MODEL.RESNETS.DEFORM_ON_PER_STAGE = [False, False, False, False] +# Use True to use modulated deform_conv (DeformableV2, https://arxiv.org/abs/1811.11168); +# Use False for DeformableV1. +_C.MODEL.RESNETS.DEFORM_MODULATED = False +# Number of groups in deformable conv. +_C.MODEL.RESNETS.DEFORM_NUM_GROUPS = 1 + + +# ---------------------------------------------------------------------------- # +# Solver +# ---------------------------------------------------------------------------- # +_C.SOLVER = CN() + +# Options: WarmupMultiStepLR, WarmupCosineLR. +# See detectron2/solver/build.py for definition. +_C.SOLVER.LR_SCHEDULER_NAME = "WarmupMultiStepLR" + +_C.SOLVER.MAX_ITER = 40000 + +_C.SOLVER.BASE_LR = 0.001 +# The end lr, only used by WarmupCosineLR +_C.SOLVER.BASE_LR_END = 0.0 + +_C.SOLVER.MOMENTUM = 0.9 + +_C.SOLVER.NESTEROV = False + +_C.SOLVER.WEIGHT_DECAY = 0.0001 +# The weight decay that's applied to parameters of normalization layers +# (typically the affine transformation) +_C.SOLVER.WEIGHT_DECAY_NORM = 0.0 + +_C.SOLVER.GAMMA = 0.1 +# The iteration number to decrease learning rate by GAMMA. +_C.SOLVER.STEPS = (30000,) + +_C.SOLVER.WARMUP_FACTOR = 1.0 / 1000 +_C.SOLVER.WARMUP_ITERS = 1000 +_C.SOLVER.WARMUP_METHOD = "linear" + +# Save a checkpoint after every this number of iterations +_C.SOLVER.CHECKPOINT_PERIOD = 5000 + +# Number of images per batch across all machines. This is also the number +# of training images per step (i.e. per iteration). If we use 16 GPUs +# and IMS_PER_BATCH = 32, each GPU will see 2 images per batch. +# May be adjusted automatically if REFERENCE_WORLD_SIZE is set. +_C.SOLVER.IMS_PER_BATCH = 16 + +# The reference number of workers (GPUs) this config is meant to train with. +# It takes no effect when set to 0. +# With a non-zero value, it will be used by DefaultTrainer to compute a desired +# per-worker batch size, and then scale the other related configs (total batch size, +# learning rate, etc) to match the per-worker batch size. +# See documentation of `DefaultTrainer.auto_scale_workers` for details: +_C.SOLVER.REFERENCE_WORLD_SIZE = 0 + +# Detectron v1 (and previous detection code) used a 2x higher LR and 0 WD for +# biases. This is not useful (at least for recent models). You should avoid +# changing these and they exist only to reproduce Detectron v1 training if +# desired. +_C.SOLVER.BIAS_LR_FACTOR = 1.0 +_C.SOLVER.WEIGHT_DECAY_BIAS = None # None means following WEIGHT_DECAY + +# Gradient clipping +_C.SOLVER.CLIP_GRADIENTS = CN({"ENABLED": False}) +# Type of gradient clipping, currently 2 values are supported: +# - "value": the absolute values of elements of each gradients are clipped +# - "norm": the norm of the gradient for each parameter is clipped thus +# affecting all elements in the parameter +_C.SOLVER.CLIP_GRADIENTS.CLIP_TYPE = "value" +# Maximum absolute value used for clipping gradients +_C.SOLVER.CLIP_GRADIENTS.CLIP_VALUE = 1.0 +# Floating point number p for L-p norm to be used with the "norm" +# gradient clipping type; for L-inf, please specify .inf +_C.SOLVER.CLIP_GRADIENTS.NORM_TYPE = 2.0 + +# Enable automatic mixed precision for training +# Note that this does not change model's inference behavior. +# To use AMP in inference, run inference under autocast() +_C.SOLVER.AMP = CN({"ENABLED": False}) + +# ---------------------------------------------------------------------------- # +# Specific test options +# ---------------------------------------------------------------------------- # +_C.TEST = CN() +# For end-to-end tests to verify the expected accuracy. +# Each item is [task, metric, value, tolerance] +# e.g.: [['bbox', 'AP', 38.5, 0.2]] +_C.TEST.EXPECTED_RESULTS = [] +# The period (in terms of steps) to evaluate the model during training. +# Set to 0 to disable. +_C.TEST.EVAL_PERIOD = 0 +# The sigmas used to calculate keypoint OKS. See http://cocodataset.org/#keypoints-eval +# When empty, it will use the defaults in COCO. +# Otherwise it should be a list[float] with the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. +_C.TEST.KEYPOINT_OKS_SIGMAS = [] +# Maximum number of detections to return per image during inference (100 is +# based on the limit established for the COCO dataset). +_C.TEST.DETECTIONS_PER_IMAGE = 100 + +_C.TEST.AUG = CN({"ENABLED": False}) +_C.TEST.AUG.MIN_SIZES = (400, 500, 600, 700, 800, 900, 1000, 1100, 1200) +_C.TEST.AUG.MAX_SIZE = 4000 +_C.TEST.AUG.FLIP = True + +_C.TEST.PRECISE_BN = CN({"ENABLED": False}) +_C.TEST.PRECISE_BN.NUM_ITER = 200 + +# ---------------------------------------------------------------------------- # +# Misc options +# ---------------------------------------------------------------------------- # +# Directory where output files are written +_C.OUTPUT_DIR = "./output" +# Set seed to negative to fully randomize everything. +# Set seed to positive to use a fixed seed. Note that a fixed seed increases +# reproducibility but does not guarantee fully deterministic behavior. +# Disabling all parallelism further increases reproducibility. +_C.SEED = -1 +# Benchmark different cudnn algorithms. +# If input images have very different sizes, this option will have large overhead +# for about 10k iterations. It usually hurts total time, but can benefit for certain models. +# If input images have the same or similar sizes, benchmark is often helpful. +_C.CUDNN_BENCHMARK = False +# The period (in terms of steps) for minibatch visualization at train time. +# Set to 0 to disable. +_C.VIS_PERIOD = 0 + +# global config is for quick hack purposes. +# You can set them in command line or config files, +# and access it with: +# +# from detectron2.config import global_cfg +# print(global_cfg.HACK) +# +# Do not commit any configs into it. +_C.GLOBAL = CN() +_C.GLOBAL.HACK = 1.0 diff --git a/detectron2/config/instantiate.py b/detectron2/config/instantiate.py new file mode 100644 index 0000000000000000000000000000000000000000..75231a4f1681b2af0da5a38a73c178ee429afad6 --- /dev/null +++ b/detectron2/config/instantiate.py @@ -0,0 +1,88 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import collections.abc as abc +import dataclasses +import logging +from typing import Any + +from detectron2.utils.registry import _convert_target_to_string, locate + +__all__ = ["dump_dataclass", "instantiate"] + + +def dump_dataclass(obj: Any): + """ + Dump a dataclass recursively into a dict that can be later instantiated. + + Args: + obj: a dataclass object + + Returns: + dict + """ + assert dataclasses.is_dataclass(obj) and not isinstance( + obj, type + ), "dump_dataclass() requires an instance of a dataclass." + ret = {"_target_": _convert_target_to_string(type(obj))} + for f in dataclasses.fields(obj): + v = getattr(obj, f.name) + if dataclasses.is_dataclass(v): + v = dump_dataclass(v) + if isinstance(v, (list, tuple)): + v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v] + ret[f.name] = v + return ret + + +def instantiate(cfg): + """ + Recursively instantiate objects defined in dictionaries by + "_target_" and arguments. + + Args: + cfg: a dict-like object with "_target_" that defines the caller, and + other keys that define the arguments + + Returns: + object instantiated by cfg + """ + from omegaconf import DictConfig, ListConfig, OmegaConf + + if isinstance(cfg, ListConfig): + lst = [instantiate(x) for x in cfg] + return ListConfig(lst, flags={"allow_objects": True}) + if isinstance(cfg, list): + # Specialize for list, because many classes take + # list[objects] as arguments, such as ResNet, DatasetMapper + return [instantiate(x) for x in cfg] + + # If input is a DictConfig backed by dataclasses (i.e. omegaconf's structured config), + # instantiate it to the actual dataclass. + if isinstance(cfg, DictConfig) and dataclasses.is_dataclass(cfg._metadata.object_type): + return OmegaConf.to_object(cfg) + + if isinstance(cfg, abc.Mapping) and "_target_" in cfg: + # conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all, + # but faster: https://github.com/facebookresearch/hydra/issues/1200 + cfg = {k: instantiate(v) for k, v in cfg.items()} + cls = cfg.pop("_target_") + cls = instantiate(cls) + + if isinstance(cls, str): + cls_name = cls + cls = locate(cls_name) + assert cls is not None, cls_name + else: + try: + cls_name = cls.__module__ + "." + cls.__qualname__ + except Exception: + # target could be anything, so the above could fail + cls_name = str(cls) + assert callable(cls), f"_target_ {cls} does not define a callable object" + try: + return cls(**cfg) + except TypeError: + logger = logging.getLogger(__name__) + logger.error(f"Error when instantiating {cls_name}!") + raise + return cfg # return as-is if don't know what to do diff --git a/detectron2/config/lazy.py b/detectron2/config/lazy.py new file mode 100644 index 0000000000000000000000000000000000000000..2104931b3556ed98e037807b54920d37adce27be --- /dev/null +++ b/detectron2/config/lazy.py @@ -0,0 +1,399 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import ast +import builtins +import collections.abc as abc +import importlib +import inspect +import logging +import os +import uuid +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import is_dataclass +from typing import List, Tuple, Union + +import cloudpickle +import yaml +from omegaconf import DictConfig, ListConfig, OmegaConf, SCMode + +from detectron2.utils.file_io import PathManager +from detectron2.utils.registry import _convert_target_to_string + +__all__ = ["LazyCall", "LazyConfig"] + + +class LazyCall: + """ + Wrap a callable so that when it's called, the call will not be executed, + but returns a dict that describes the call. + + LazyCall object has to be called with only keyword arguments. Positional + arguments are not yet supported. + + Examples: + :: + from detectron2.config import instantiate, LazyCall + + layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32) + layer_cfg.out_channels = 64 # can edit it afterwards + layer = instantiate(layer_cfg) + """ + + def __init__(self, target): + if not (callable(target) or isinstance(target, (str, abc.Mapping))): + raise TypeError(f"target of LazyCall must be a callable or defines a callable! Got {target}") + self._target = target + + def __call__(self, **kwargs): + if is_dataclass(self._target): + # omegaconf object cannot hold dataclass type + # https://github.com/omry/omegaconf/issues/784 + target = _convert_target_to_string(self._target) + else: + target = self._target + kwargs["_target_"] = target + + return DictConfig(content=kwargs, flags={"allow_objects": True}) + + +def _visit_dict_config(cfg, func): + """ + Apply func recursively to all DictConfig in cfg. + """ + if isinstance(cfg, DictConfig): + func(cfg) + for v in cfg.values(): + _visit_dict_config(v, func) + elif isinstance(cfg, ListConfig): + for v in cfg: + _visit_dict_config(v, func) + + +def _validate_py_syntax(filename): + # see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py + with PathManager.open(filename, "r") as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError(f"Config file {filename} has syntax error!") from e + + +def _cast_to_config(obj): + # if given a dict, return DictConfig instead + if isinstance(obj, dict): + return DictConfig(obj, flags={"allow_objects": True}) + return obj + + +_CFG_PACKAGE_NAME = "detectron2._cfg_loader" +""" +A namespace to put all imported config into. +""" + + +def _random_package_name(filename): + # generate a random package name when loading config files + return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename) + + +@contextmanager +def _patch_import(): + """ + Enhance relative import statements in config files, so that they: + 1. locate files purely based on relative location, regardless of packages. + e.g. you can import file without having __init__ + 2. do not cache modules globally; modifications of module states has no side effect + 3. support other storage system through PathManager + 4. imported dict are turned into omegaconf.DictConfig automatically + """ + old_import = builtins.__import__ + + def find_relative_file(original_file, relative_import_path, level): + cur_file = os.path.dirname(original_file) + for _ in range(level - 1): + cur_file = os.path.dirname(cur_file) + cur_name = relative_import_path.lstrip(".") + for part in cur_name.split("."): + cur_file = os.path.join(cur_file, part) + # NOTE: directory import is not handled. Because then it's unclear + # if such import should produce python module or DictConfig. This can + # be discussed further if needed. + if not cur_file.endswith(".py"): + cur_file += ".py" + if not PathManager.isfile(cur_file): + raise ImportError( + f"Cannot import name {relative_import_path} from " f"{original_file}: {cur_file} has to exist." + ) + return cur_file + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + if ( + # Only deal with relative imports inside config files + level != 0 + and globals is not None + and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME) + ): + cur_file = find_relative_file(globals["__file__"], name, level) + _validate_py_syntax(cur_file) + spec = importlib.machinery.ModuleSpec(_random_package_name(cur_file), None, origin=cur_file) + module = importlib.util.module_from_spec(spec) + module.__file__ = cur_file + with PathManager.open(cur_file) as f: + content = f.read() + exec(compile(content, cur_file, "exec"), module.__dict__) + for name in fromlist: # turn imported dict into DictConfig automatically + val = _cast_to_config(module.__dict__[name]) + module.__dict__[name] = val + return module + return old_import(name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + yield new_import + builtins.__import__ = old_import + + +class LazyConfig: + """ + Provide methods to save, load, and overrides an omegaconf config object + which may contain definition of lazily-constructed objects. + """ + + @staticmethod + def load_rel(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Similar to :meth:`load()`, but load path relative to the caller's + source file. + + This has the same functionality as a relative import, except that this method + accepts filename as a string, so more characters are allowed in the filename. + """ + caller_frame = inspect.stack()[1] + caller_fname = caller_frame[0].f_code.co_filename + assert caller_fname != "", "load_rel Unable to find caller" + caller_dir = os.path.dirname(caller_fname) + filename = os.path.join(caller_dir, filename) + return LazyConfig.load(filename, keys) + + @staticmethod + def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Load a config file. + + Args: + filename: absolute path or relative path w.r.t. the current working directory + keys: keys to load and return. If not given, return all keys + (whose values are config objects) in a dict. + """ + has_keys = keys is not None + filename = filename.replace("/./", "/") # redundant + if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]: + raise ValueError(f"Config file {filename} has to be a python or yaml file.") + if filename.endswith(".py"): + _validate_py_syntax(filename) + + with _patch_import(): + # Record the filename + module_namespace = { + "__file__": filename, + "__package__": _random_package_name(filename), + } + with PathManager.open(filename) as f: + content = f.read() + # Compile first with filename to: + # 1. make filename appears in stacktrace + # 2. make load_rel able to find its parent's (possibly remote) location + exec(compile(content, filename, "exec"), module_namespace) + + ret = module_namespace + else: + with PathManager.open(filename) as f: + obj = yaml.unsafe_load(f) + ret = OmegaConf.create(obj, flags={"allow_objects": True}) + + if has_keys: + if isinstance(keys, str): + return _cast_to_config(ret[keys]) + else: + return tuple(_cast_to_config(ret[a]) for a in keys) + else: + if filename.endswith(".py"): + # when not specified, only load those that are config objects + ret = DictConfig( + { + name: _cast_to_config(value) + for name, value in ret.items() + if isinstance(value, (DictConfig, ListConfig, dict)) and not name.startswith("_") + }, + flags={"allow_objects": True}, + ) + return ret + + @staticmethod + def save(cfg, filename: str): + """ + Save a config object to a yaml file. + Note that when the config dictionary contains complex objects (e.g. lambda), + it can't be saved to yaml. In that case we will print an error and + attempt to save to a pkl file instead. + + Args: + cfg: an omegaconf config object + filename: yaml file name to save the config file + """ + logger = logging.getLogger(__name__) + try: + cfg = deepcopy(cfg) + except Exception: + pass + else: + # if it's deep-copyable, then... + def _replace_type_by_name(x): + if "_target_" in x and callable(x._target_): + try: + x._target_ = _convert_target_to_string(x._target_) + except AttributeError: + pass + + # not necessary, but makes yaml looks nicer + _visit_dict_config(cfg, _replace_type_by_name) + + save_pkl = False + try: + dict = OmegaConf.to_container( + cfg, + # Do not resolve interpolation when saving, i.e. do not turn ${a} into + # actual values when saving. + resolve=False, + # Save structures (dataclasses) in a format that can be instantiated later. + # Without this option, the type information of the dataclass will be erased. + structured_config_mode=SCMode.INSTANTIATE, + ) + dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999) + with PathManager.open(filename, "w") as f: + f.write(dumped) + + try: + _ = yaml.unsafe_load(dumped) # test that it is loadable + except Exception: + logger.warning( + "The config contains objects that cannot serialize to a valid yaml. " + f"{filename} is human-readable but cannot be loaded." + ) + save_pkl = True + except Exception: + logger.exception("Unable to serialize the config to yaml. Error:") + save_pkl = True + + if save_pkl: + new_filename = filename + ".pkl" + try: + # retry by pickle + with PathManager.open(new_filename, "wb") as f: + cloudpickle.dump(cfg, f) + logger.warning(f"Config is saved using cloudpickle at {new_filename}.") + except Exception: + pass + + @staticmethod + def apply_overrides(cfg, overrides: List[str]): + """ + In-place override contents of cfg. + + Args: + cfg: an omegaconf config object + overrides: list of strings in the format of "a=b" to override configs. + See https://hydra.cc/docs/next/advanced/override_grammar/basic/ + for syntax. + + Returns: + the cfg object + """ + + def safe_update(cfg, key, value): + parts = key.split(".") + for idx in range(1, len(parts)): + prefix = ".".join(parts[:idx]) + v = OmegaConf.select(cfg, prefix, default=None) + if v is None: + break + if not OmegaConf.is_config(v): + raise KeyError( + f"Trying to update key {key}, but {prefix} " f"is not a config, but has type {type(v)}." + ) + OmegaConf.update(cfg, key, value, merge=True) + + from hydra.core.override_parser.overrides_parser import OverridesParser + + parser = OverridesParser.create() + overrides = parser.parse_overrides(overrides) + for o in overrides: + key = o.key_or_group + value = o.value() + if o.is_delete(): + # TODO support this + raise NotImplementedError("deletion is not yet a supported override") + safe_update(cfg, key, value) + return cfg + + @staticmethod + def to_py(cfg, prefix: str = "cfg."): + """ + Try to convert a config object into Python-like psuedo code. + + Note that perfect conversion is not always possible. So the returned + results are mainly meant to be human-readable, and not meant to be executed. + + Args: + cfg: an omegaconf config object + prefix: root name for the resulting code (default: "cfg.") + + + Returns: + str of formatted Python code + """ + import black + + cfg = OmegaConf.to_container(cfg, resolve=True) + + def _to_str(obj, prefix=None, inside_call=False): + if prefix is None: + prefix = [] + if isinstance(obj, abc.Mapping) and "_target_" in obj: + # Dict representing a function call + target = _convert_target_to_string(obj.pop("_target_")) + args = [] + for k, v in sorted(obj.items()): + args.append(f"{k}={_to_str(v, inside_call=True)}") + args = ", ".join(args) + call = f"{target}({args})" + return "".join(prefix) + call + elif isinstance(obj, abc.Mapping) and not inside_call: + # Dict that is not inside a call is a list of top-level config objects that we + # render as one object per line with dot separated prefixes + key_list = [] + for k, v in sorted(obj.items()): + if isinstance(v, abc.Mapping) and "_target_" not in v: + key_list.append(_to_str(v, prefix=prefix + [k + "."])) + else: + key = "".join(prefix) + k + key_list.append(f"{key}={_to_str(v)}") + return "\n".join(key_list) + elif isinstance(obj, abc.Mapping): + # Dict that is inside a call is rendered as a regular dict + return ( + "{" + + ",".join(f"{repr(k)}: {_to_str(v, inside_call=inside_call)}" for k, v in sorted(obj.items())) + + "}" + ) + elif isinstance(obj, list): + return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]" + else: + return repr(obj) + + py_str = _to_str(cfg, prefix=[prefix]) + try: + return black.format_str(py_str, mode=black.Mode()) + except black.InvalidInput: + return py_str diff --git a/detectron2/engine/__init__.py b/detectron2/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f95ac35d27afcc9bd7648a8cd1fbc7edcadc22fa --- /dev/null +++ b/detectron2/engine/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .launch import * +from .train_loop import * + +__all__ = [k for k in globals().keys() if not k.startswith("_")] + + +# prefer to let hooks and defaults live in separate namespaces (therefore not in __all__) +# but still make them available here +from .defaults import * +from .hooks import * diff --git a/detectron2/engine/defaults.py b/detectron2/engine/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..a8a2e83dc21fa8de64767955023c72b99123533c --- /dev/null +++ b/detectron2/engine/defaults.py @@ -0,0 +1,701 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +This file contains components with some default boilerplate logic user may need +in training / testing. They will not work for everyone, but many users may find them useful. + +The behavior of functions/classes in this file is subject to change, +since they are meant to represent the "common default behavior" people need in their projects. +""" + +import argparse +import logging +import os +import sys +import weakref +from collections import OrderedDict +from typing import Optional + +import torch +from fvcore.nn.precise_bn import get_bn_modules +from omegaconf import OmegaConf +from torch.nn.parallel import DistributedDataParallel + +import detectron2.data.transforms as T +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import CfgNode, LazyConfig +from detectron2.data import ( + MetadataCatalog, + build_detection_test_loader, + build_detection_train_loader, +) +from detectron2.evaluation import ( + DatasetEvaluator, + inference_on_dataset, + print_csv_format, + verify_results, +) +from detectron2.modeling import build_model +from detectron2.solver import build_lr_scheduler, build_optimizer +from detectron2.utils import comm +from detectron2.utils.collect_env import collect_env_info +from detectron2.utils.env import seed_all_rng +from detectron2.utils.events import CommonMetricPrinter, JSONWriter, TensorboardXWriter +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger + +from . import hooks +from .train_loop import AMPTrainer, SimpleTrainer, TrainerBase + +__all__ = [ + "create_ddp_model", + "default_argument_parser", + "default_setup", + "default_writers", + "DefaultPredictor", + "DefaultTrainer", +] + + +def create_ddp_model(model, *, fp16_compression=False, **kwargs): + """ + Create a DistributedDataParallel model if there are >1 processes. + + Args: + model: a torch.nn.Module + fp16_compression: add fp16 compression hooks to the ddp object. + See more at https://pytorch.org/docs/stable/ddp_comm_hooks.html#torch.distributed.algorithms.ddp_comm_hooks.default_hooks.fp16_compress_hook + kwargs: other arguments of :module:`torch.nn.parallel.DistributedDataParallel`. + """ # noqa + if comm.get_world_size() == 1: + return model + if "device_ids" not in kwargs: + kwargs["device_ids"] = [comm.get_local_rank()] + ddp = DistributedDataParallel(model, **kwargs) + if fp16_compression: + from torch.distributed.algorithms.ddp_comm_hooks import default as comm_hooks + + ddp.register_comm_hook(state=None, hook=comm_hooks.fp16_compress_hook) + return ddp + + +def default_argument_parser(epilog=None): + """ + Create a parser with some common arguments used by detectron2 users. + + Args: + epilog (str): epilog passed to ArgumentParser describing the usage. + + Returns: + argparse.ArgumentParser: + """ + parser = argparse.ArgumentParser( + epilog=epilog or f""" +Examples: + +Run on single machine: + $ {sys.argv[0]} --num-gpus 8 --config-file cfg.yaml + +Change some config options: + $ {sys.argv[0]} --config-file cfg.yaml MODEL.WEIGHTS /path/to/weight.pth SOLVER.BASE_LR 0.001 + +Run on multiple machines: + (machine0)$ {sys.argv[0]} --machine-rank 0 --num-machines 2 --dist-url [--other-flags] + (machine1)$ {sys.argv[0]} --machine-rank 1 --num-machines 2 --dist-url [--other-flags] +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--config-file", default="", metavar="FILE", help="path to config file") + parser.add_argument( + "--resume", + action="store_true", + help="Whether to attempt to resume from the checkpoint directory. " + "See documentation of `DefaultTrainer.resume_or_load()` for what it means.", + ) + parser.add_argument("--eval-only", action="store_true", help="perform evaluation only") + parser.add_argument("--num-gpus", type=int, default=1, help="number of gpus *per machine*") + parser.add_argument("--num-machines", type=int, default=1, help="total number of machines") + parser.add_argument("--machine-rank", type=int, default=0, help="the rank of this machine (unique per machine)") + + # PyTorch still may leave orphan processes in multi-gpu training. + # Therefore we use a deterministic way to obtain port, + # so that users are aware of orphan processes by seeing the port occupied. + port = 2**15 + 2**14 + hash(os.getuid() if sys.platform != "win32" else 1) % 2**14 + parser.add_argument( + "--dist-url", + default="tcp://127.0.0.1:{}".format(port), + help="initialization URL for pytorch distributed backend. See " + "https://pytorch.org/docs/stable/distributed.html for details.", + ) + parser.add_argument( + "opts", + help=""" +Modify config options at the end of the command. For Yacs configs, use +space-separated "PATH.KEY VALUE" pairs. +For python-based LazyConfig, use "path.key=value". + """.strip(), + default=None, + nargs=argparse.REMAINDER, + ) + return parser + + +def _try_get_key(cfg, *keys, default=None): + """ + Try select keys from cfg until the first key that exists. Otherwise return default. + """ + if isinstance(cfg, CfgNode): + cfg = OmegaConf.create(cfg.dump()) + for k in keys: + none = object() + p = OmegaConf.select(cfg, k, default=none) + if p is not none: + return p + return default + + +def _highlight(code, filename): + try: + import pygments + except ImportError: + return code + + from pygments.formatters import Terminal256Formatter + from pygments.lexers import Python3Lexer, YamlLexer + + lexer = Python3Lexer() if filename.endswith(".py") else YamlLexer() + code = pygments.highlight(code, lexer, Terminal256Formatter(style="monokai")) + return code + + +def default_setup(cfg, args): + """ + Perform some basic common setups at the beginning of a job, including: + + 1. Set up the detectron2 logger + 2. Log basic information about environment, cmdline arguments, and config + 3. Backup the config to the output directory + + Args: + cfg (CfgNode or omegaconf.DictConfig): the full config to be used + args (argparse.NameSpace): the command line arguments to be logged + """ + output_dir = _try_get_key(cfg, "OUTPUT_DIR", "output_dir", "train.output_dir") + if comm.is_main_process() and output_dir: + PathManager.mkdirs(output_dir) + + rank = comm.get_rank() + setup_logger(output_dir, distributed_rank=rank, name="fvcore") + logger = setup_logger(output_dir, distributed_rank=rank) + + logger.info("Rank of current process: {}. World size: {}".format(rank, comm.get_world_size())) + logger.info("Environment info:\n" + collect_env_info()) + + logger.info("Command line arguments: " + str(args)) + if hasattr(args, "config_file") and args.config_file != "": + logger.info( + "Contents of args.config_file={}:\n{}".format( + args.config_file, + _highlight(PathManager.open(args.config_file, "r").read(), args.config_file), + ) + ) + + if comm.is_main_process() and output_dir: + # Note: some of our scripts may expect the existence of + # config.yaml in output directory + path = os.path.join(output_dir, "config.yaml") + if isinstance(cfg, CfgNode): + logger.info("Running with full config:\n{}".format(_highlight(cfg.dump(), ".yaml"))) + with PathManager.open(path, "w") as f: + f.write(cfg.dump()) + else: + LazyConfig.save(cfg, path) + logger.info("Full config saved to {}".format(path)) + + # make sure each worker has a different, yet deterministic seed if specified + seed = _try_get_key(cfg, "SEED", "train.seed", default=-1) + seed_all_rng(None if seed < 0 else seed + rank) + + # cudnn benchmark has large overhead. It shouldn't be used considering the small size of + # typical validation set. + if not (hasattr(args, "eval_only") and args.eval_only): + torch.backends.cudnn.benchmark = _try_get_key(cfg, "CUDNN_BENCHMARK", "train.cudnn_benchmark", default=False) + + +def default_writers(output_dir: str, max_iter: Optional[int] = None): + """ + Build a list of :class:`EventWriter` to be used. + It now consists of a :class:`CommonMetricPrinter`, + :class:`TensorboardXWriter` and :class:`JSONWriter`. + + Args: + output_dir: directory to store JSON metrics and tensorboard events + max_iter: the total number of iterations + + Returns: + list[EventWriter]: a list of :class:`EventWriter` objects. + """ + PathManager.mkdirs(output_dir) + return [ + # It may not always print what you want to see, since it prints "common" metrics only. + CommonMetricPrinter(max_iter), + JSONWriter(os.path.join(output_dir, "metrics.json")), + TensorboardXWriter(output_dir), + ] + + +class DefaultPredictor: + """ + Create a simple end-to-end predictor with the given config that runs on + single device for a single input image. + + Compared to using the model directly, this class does the following additions: + + 1. Load checkpoint from `cfg.MODEL.WEIGHTS`. + 2. Always take BGR image as the input and apply conversion defined by `cfg.INPUT.FORMAT`. + 3. Apply resizing defined by `cfg.INPUT.{MIN,MAX}_SIZE_TEST`. + 4. Take one input image and produce a single output, instead of a batch. + + This is meant for simple demo purposes, so it does the above steps automatically. + This is not meant for benchmarks or running complicated inference logic. + If you'd like to do anything more complicated, please refer to its source code as + examples to build and use the model manually. + + Attributes: + metadata (Metadata): the metadata of the underlying dataset, obtained from + cfg.DATASETS.TEST. + + Examples: + :: + pred = DefaultPredictor(cfg) + inputs = cv2.imread("input.jpg") + outputs = pred(inputs) + """ + + def __init__(self, cfg): + self.cfg = cfg.clone() # cfg can be modified by model + self.model = build_model(self.cfg) + self.model.eval() + if len(cfg.DATASETS.TEST): + self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) + + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.MODEL.WEIGHTS) + + self.aug = T.ResizeShortestEdge([cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST) + + self.input_format = cfg.INPUT.FORMAT + assert self.input_format in ["RGB", "BGR"], self.input_format + + def __call__(self, original_image): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + image = self.aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + predictions = self.model([inputs])[0] + return predictions + + +class DefaultTrainer(TrainerBase): + """ + A trainer with default training logic. It does the following: + + 1. Create a :class:`SimpleTrainer` using model, optimizer, dataloader + defined by the given config. Create a LR scheduler defined by the config. + 2. Load the last checkpoint or `cfg.MODEL.WEIGHTS`, if exists, when + `resume_or_load` is called. + 3. Register a few common hooks defined by the config. + + It is created to simplify the **standard model training workflow** and reduce code boilerplate + for users who only need the standard training workflow, with standard features. + It means this class makes *many assumptions* about your training logic that + may easily become invalid in a new research. In fact, any assumptions beyond those made in the + :class:`SimpleTrainer` are too much for research. + + The code of this class has been annotated about restrictive assumptions it makes. + When they do not work for you, you're encouraged to: + + 1. Overwrite methods of this class, OR: + 2. Use :class:`SimpleTrainer`, which only does minimal SGD training and + nothing else. You can then add your own hooks if needed. OR: + 3. Write your own training loop similar to `tools/plain_train_net.py`. + + See the :doc:`/tutorials/training` tutorials for more details. + + Note that the behavior of this class, like other functions/classes in + this file, is not stable, since it is meant to represent the "common default behavior". + It is only guaranteed to work well with the standard models and training workflow in detectron2. + To obtain more stable behavior, write your own training logic with other public APIs. + + Examples: + :: + trainer = DefaultTrainer(cfg) + trainer.resume_or_load() # load last checkpoint or MODEL.WEIGHTS + trainer.train() + + Attributes: + scheduler: + checkpointer (DetectionCheckpointer): + cfg (CfgNode): + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): + """ + super().__init__() + logger = logging.getLogger("detectron2") + if not logger.isEnabledFor(logging.INFO): # setup_logger is not called for d2 + setup_logger() + cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size()) + + # Assume these objects must be constructed in this order. + model = self.build_model(cfg) + optimizer = self.build_optimizer(cfg, model) + data_loader = self.build_train_loader(cfg) + + model = create_ddp_model(model, broadcast_buffers=False) + self._trainer = (AMPTrainer if cfg.SOLVER.AMP.ENABLED else SimpleTrainer)(model, data_loader, optimizer) + + self.scheduler = self.build_lr_scheduler(cfg, optimizer) + self.checkpointer = DetectionCheckpointer( + # Assume you want to save checkpoints together with logs/statistics + model, + cfg.OUTPUT_DIR, + trainer=weakref.proxy(self), + ) + self.start_iter = 0 + self.max_iter = cfg.SOLVER.MAX_ITER + self.cfg = cfg + + self.register_hooks(self.build_hooks()) + + def resume_or_load(self, resume=True): + """ + If `resume==True` and `cfg.OUTPUT_DIR` contains the last checkpoint (defined by + a `last_checkpoint` file), resume from the file. Resuming means loading all + available states (eg. optimizer and scheduler) and update iteration counter + from the checkpoint. ``cfg.MODEL.WEIGHTS`` will not be used. + + Otherwise, this is considered as an independent training. The method will load model + weights from the file `cfg.MODEL.WEIGHTS` (but will not load other states) and start + from iteration 0. + + Args: + resume (bool): whether to do resume or not + """ + self.checkpointer.resume_or_load(self.cfg.MODEL.WEIGHTS, resume=resume) + if resume and self.checkpointer.has_checkpoint(): + # The checkpoint stores the training iteration that just finished, thus we start + # at the next iteration + self.start_iter = self.iter + 1 + + def build_hooks(self): + """ + Build a list of default hooks, including timing, evaluation, + checkpointing, lr scheduling, precise BN, writing events. + + Returns: + list[HookBase]: + """ + cfg = self.cfg.clone() + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 0 # save some memory and time for PreciseBN + + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + ( + hooks.PreciseBN( + # Run at the same freq as (but before) evaluation. + cfg.TEST.EVAL_PERIOD, + self.model, + # Build a new data loader to not affect training + self.build_train_loader(cfg), + cfg.TEST.PRECISE_BN.NUM_ITER, + ) + if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model) + else None + ), + ] + + # Do PreciseBN before checkpointer, because it updates the model and need to + # be saved by checkpointer. + # This is not always the best: if checkpointing has a different frequency, + # some checkpoints may have more precise statistics than others. + if comm.is_main_process(): + ret.append(hooks.PeriodicCheckpointer(self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD)) + + def test_and_save_results(): + self._last_eval_results = self.test(self.cfg, self.model) + return self._last_eval_results + + # Do evaluation after checkpointer, because then if it fails, + # we can use the saved checkpoint to debug. + ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD, test_and_save_results)) + + if comm.is_main_process(): + # Here the default print/log frequency of each writer is used. + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), period=20)) + return ret + + def build_writers(self): + """ + Build a list of writers to be used using :func:`default_writers()`. + If you'd like a different list of writers, you can overwrite it in + your trainer. + + Returns: + list[EventWriter]: a list of :class:`EventWriter` objects. + """ + return default_writers(self.cfg.OUTPUT_DIR, self.max_iter) + + def train(self): + """ + Run training. + + Returns: + OrderedDict of results, if evaluation is enabled. Otherwise None. + """ + super().train(self.start_iter, self.max_iter) + if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process(): + assert hasattr(self, "_last_eval_results"), "No evaluation results obtained during training!" + verify_results(self.cfg, self._last_eval_results) + return self._last_eval_results + + def run_step(self): + self._trainer.iter = self.iter + self._trainer.run_step() + + def state_dict(self): + ret = super().state_dict() + ret["_trainer"] = self._trainer.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self._trainer.load_state_dict(state_dict["_trainer"]) + + @classmethod + def build_model(cls, cfg): + """ + Returns: + torch.nn.Module: + + It now calls :func:`detectron2.modeling.build_model`. + Overwrite it if you'd like a different model. + """ + model = build_model(cfg) + logger = logging.getLogger(__name__) + logger.info("Model:\n{}".format(model)) + return model + + @classmethod + def build_optimizer(cls, cfg, model): + """ + Returns: + torch.optim.Optimizer: + + It now calls :func:`detectron2.solver.build_optimizer`. + Overwrite it if you'd like a different optimizer. + """ + return build_optimizer(cfg, model) + + @classmethod + def build_lr_scheduler(cls, cfg, optimizer): + """ + It now calls :func:`detectron2.solver.build_lr_scheduler`. + Overwrite it if you'd like a different scheduler. + """ + return build_lr_scheduler(cfg, optimizer) + + @classmethod + def build_train_loader(cls, cfg): + """ + Returns: + iterable + + It now calls :func:`detectron2.data.build_detection_train_loader`. + Overwrite it if you'd like a different data loader. + """ + return build_detection_train_loader(cfg) + + @classmethod + def build_test_loader(cls, cfg, dataset_name): + """ + Returns: + iterable + + It now calls :func:`detectron2.data.build_detection_test_loader`. + Overwrite it if you'd like a different data loader. + """ + return build_detection_test_loader(cfg, dataset_name) + + @classmethod + def build_evaluator(cls, cfg, dataset_name): + """ + Returns: + DatasetEvaluator or None + + It is not implemented by default. + """ + raise NotImplementedError(""" +If you want DefaultTrainer to automatically run evaluation, +please implement `build_evaluator()` in subclasses (see train_net.py for example). +Alternatively, you can call evaluation functions yourself (see Colab balloon tutorial for example). +""") + + @classmethod + def test(cls, cfg, model, evaluators=None): + """ + Evaluate the given model. The given model is expected to already contain + weights to evaluate. + + Args: + cfg (CfgNode): + model (nn.Module): + evaluators (list[DatasetEvaluator] or None): if None, will call + :meth:`build_evaluator`. Otherwise, must have the same length as + ``cfg.DATASETS.TEST``. + + Returns: + dict: a dict of result metrics + """ + logger = logging.getLogger(__name__) + if isinstance(evaluators, DatasetEvaluator): + evaluators = [evaluators] + if evaluators is not None: + assert len(cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format( + len(cfg.DATASETS.TEST), len(evaluators) + ) + + results = OrderedDict() + for idx, dataset_name in enumerate(cfg.DATASETS.TEST): + data_loader = cls.build_test_loader(cfg, dataset_name) + # When evaluators are passed in as arguments, + # implicitly assume that evaluators can be created before data_loader. + if evaluators is not None: + evaluator = evaluators[idx] + else: + try: + evaluator = cls.build_evaluator(cfg, dataset_name) + except NotImplementedError: + logger.warn( + "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " + "or implement its `build_evaluator` method." + ) + results[dataset_name] = {} + continue + results_i = inference_on_dataset(model, data_loader, evaluator) + results[dataset_name] = results_i + if comm.is_main_process(): + assert isinstance( + results_i, dict + ), "Evaluator must return a dict on the main process. Got {} instead.".format(results_i) + logger.info("Evaluation results for {} in csv format:".format(dataset_name)) + print_csv_format(results_i) + + if len(results) == 1: + results = list(results.values())[0] + return results + + @staticmethod + def auto_scale_workers(cfg, num_workers: int): + """ + When the config is defined for certain number of workers (according to + ``cfg.SOLVER.REFERENCE_WORLD_SIZE``) that's different from the number of + workers currently in use, returns a new cfg where the total batch size + is scaled so that the per-GPU batch size stays the same as the + original ``IMS_PER_BATCH // REFERENCE_WORLD_SIZE``. + + Other config options are also scaled accordingly: + * training steps and warmup steps are scaled inverse proportionally. + * learning rate are scaled proportionally, following :paper:`ImageNet in 1h`. + + For example, with the original config like the following: + + .. code-block:: yaml + + IMS_PER_BATCH: 16 + BASE_LR: 0.1 + REFERENCE_WORLD_SIZE: 8 + MAX_ITER: 5000 + STEPS: (4000,) + CHECKPOINT_PERIOD: 1000 + + When this config is used on 16 GPUs instead of the reference number 8, + calling this method will return a new config with: + + .. code-block:: yaml + + IMS_PER_BATCH: 32 + BASE_LR: 0.2 + REFERENCE_WORLD_SIZE: 16 + MAX_ITER: 2500 + STEPS: (2000,) + CHECKPOINT_PERIOD: 500 + + Note that both the original config and this new config can be trained on 16 GPUs. + It's up to user whether to enable this feature (by setting ``REFERENCE_WORLD_SIZE``). + + Returns: + CfgNode: a new config. Same as original if ``cfg.SOLVER.REFERENCE_WORLD_SIZE==0``. + """ + old_world_size = cfg.SOLVER.REFERENCE_WORLD_SIZE + if old_world_size == 0 or old_world_size == num_workers: + return cfg + cfg = cfg.clone() + frozen = cfg.is_frozen() + cfg.defrost() + + assert cfg.SOLVER.IMS_PER_BATCH % old_world_size == 0, "Invalid REFERENCE_WORLD_SIZE in config!" + scale = num_workers / old_world_size + bs = cfg.SOLVER.IMS_PER_BATCH = int(round(cfg.SOLVER.IMS_PER_BATCH * scale)) + lr = cfg.SOLVER.BASE_LR = cfg.SOLVER.BASE_LR * scale + max_iter = cfg.SOLVER.MAX_ITER = int(round(cfg.SOLVER.MAX_ITER / scale)) + warmup_iter = cfg.SOLVER.WARMUP_ITERS = int(round(cfg.SOLVER.WARMUP_ITERS / scale)) + cfg.SOLVER.STEPS = tuple(int(round(s / scale)) for s in cfg.SOLVER.STEPS) + cfg.TEST.EVAL_PERIOD = int(round(cfg.TEST.EVAL_PERIOD / scale)) + cfg.SOLVER.CHECKPOINT_PERIOD = int(round(cfg.SOLVER.CHECKPOINT_PERIOD / scale)) + cfg.SOLVER.REFERENCE_WORLD_SIZE = num_workers # maintain invariant + logger = logging.getLogger(__name__) + logger.info( + f"Auto-scaling the config to batch_size={bs}, learning_rate={lr}, " + f"max_iter={max_iter}, warmup={warmup_iter}." + ) + + if frozen: + cfg.freeze() + return cfg + + +# Access basic attributes from the underlying trainer +for _attr in ["model", "data_loader", "optimizer"]: + setattr( + DefaultTrainer, + _attr, + property( + # getter + lambda self, x=_attr: getattr(self._trainer, x), + # setter + lambda self, value, x=_attr: setattr(self._trainer, x, value), + ), + ) diff --git a/detectron2/engine/hooks.py b/detectron2/engine/hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..7d9fc4286d9d6502f1b6284245141ee53dcf9a02 --- /dev/null +++ b/detectron2/engine/hooks.py @@ -0,0 +1,672 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import datetime +import itertools +import logging +import math +import operator +import os +import tempfile +import time +import warnings +from collections import Counter + +import torch +from fvcore.common.checkpoint import Checkpointer +from fvcore.common.checkpoint import PeriodicCheckpointer as _PeriodicCheckpointer +from fvcore.common.param_scheduler import ParamScheduler +from fvcore.common.timer import Timer +from fvcore.nn.precise_bn import get_bn_modules, update_bn_stats + +import detectron2.utils.comm as comm +from detectron2.evaluation.testing import flatten_results_dict +from detectron2.solver import LRMultiplier +from detectron2.utils.events import EventStorage, EventWriter +from detectron2.utils.file_io import PathManager + +from .train_loop import HookBase + +__all__ = [ + "CallbackHook", + "IterationTimer", + "PeriodicWriter", + "PeriodicCheckpointer", + "BestCheckpointer", + "LRScheduler", + "AutogradProfiler", + "EvalHook", + "PreciseBN", + "TorchProfiler", + "TorchMemoryStats", +] + + +""" +Implement some common hooks. +""" + + +class CallbackHook(HookBase): + """ + Create a hook using callback functions provided by the user. + """ + + def __init__(self, *, before_train=None, after_train=None, before_step=None, after_step=None): + """ + Each argument is a function that takes one argument: the trainer. + """ + self._before_train = before_train + self._before_step = before_step + self._after_step = after_step + self._after_train = after_train + + def before_train(self): + if self._before_train: + self._before_train(self.trainer) + + def after_train(self): + if self._after_train: + self._after_train(self.trainer) + # The functions may be closures that hold reference to the trainer + # Therefore, delete them to avoid circular reference. + del self._before_train, self._after_train + del self._before_step, self._after_step + + def before_step(self): + if self._before_step: + self._before_step(self.trainer) + + def after_step(self): + if self._after_step: + self._after_step(self.trainer) + + +class IterationTimer(HookBase): + """ + Track the time spent for each iteration (each run_step call in the trainer). + Print a summary in the end of training. + + This hook uses the time between the call to its :meth:`before_step` + and :meth:`after_step` methods. + Under the convention that :meth:`before_step` of all hooks should only + take negligible amount of time, the :class:`IterationTimer` hook should be + placed at the beginning of the list of hooks to obtain accurate timing. + """ + + def __init__(self, warmup_iter=3): + """ + Args: + warmup_iter (int): the number of iterations at the beginning to exclude + from timing. + """ + self._warmup_iter = warmup_iter + self._step_timer = Timer() + self._start_time = time.perf_counter() + self._total_timer = Timer() + + def before_train(self): + self._start_time = time.perf_counter() + self._total_timer.reset() + self._total_timer.pause() + + def after_train(self): + logger = logging.getLogger(__name__) + total_time = time.perf_counter() - self._start_time + total_time_minus_hooks = self._total_timer.seconds() + hook_time = total_time - total_time_minus_hooks + + num_iter = self.trainer.storage.iter + 1 - self.trainer.start_iter - self._warmup_iter + + if num_iter > 0 and total_time_minus_hooks > 0: + # Speed is meaningful only after warmup + # NOTE this format is parsed by grep in some scripts + logger.info( + "Overall training speed: {} iterations in {} ({:.4f} s / it)".format( + num_iter, + str(datetime.timedelta(seconds=int(total_time_minus_hooks))), + total_time_minus_hooks / num_iter, + ) + ) + + logger.info( + "Total training time: {} ({} on hooks)".format( + str(datetime.timedelta(seconds=int(total_time))), + str(datetime.timedelta(seconds=int(hook_time))), + ) + ) + + def before_step(self): + self._step_timer.reset() + self._total_timer.resume() + + def after_step(self): + # +1 because we're in after_step, the current step is done + # but not yet counted + iter_done = self.trainer.storage.iter - self.trainer.start_iter + 1 + if iter_done >= self._warmup_iter: + sec = self._step_timer.seconds() + self.trainer.storage.put_scalars(time=sec) + else: + self._start_time = time.perf_counter() + self._total_timer.reset() + + self._total_timer.pause() + + +class PeriodicWriter(HookBase): + """ + Write events to EventStorage (by calling ``writer.write()``) periodically. + + It is executed every ``period`` iterations and after the last iteration. + Note that ``period`` does not affect how data is smoothed by each writer. + """ + + def __init__(self, writers, period=20): + """ + Args: + writers (list[EventWriter]): a list of EventWriter objects + period (int): + """ + self._writers = writers + for w in writers: + assert isinstance(w, EventWriter), w + self._period = period + + def after_step(self): + if (self.trainer.iter + 1) % self._period == 0 or (self.trainer.iter == self.trainer.max_iter - 1): + for writer in self._writers: + writer.write() + + def after_train(self): + for writer in self._writers: + # If any new data is found (e.g. produced by other after_train), + # write them before closing + writer.write() + writer.close() + + +class PeriodicCheckpointer(_PeriodicCheckpointer, HookBase): + """ + Same as :class:`detectron2.checkpoint.PeriodicCheckpointer`, but as a hook. + + Note that when used as a hook, + it is unable to save additional data other than what's defined + by the given `checkpointer`. + + It is executed every ``period`` iterations and after the last iteration. + """ + + def before_train(self): + self.max_iter = self.trainer.max_iter + + def after_step(self): + # No way to use **kwargs + self.step(self.trainer.iter) + + +class BestCheckpointer(HookBase): + """ + Checkpoints best weights based off given metric. + + This hook should be used in conjunction to and executed after the hook + that produces the metric, e.g. `EvalHook`. + """ + + def __init__( + self, + eval_period: int, + checkpointer: Checkpointer, + val_metric: str, + mode: str = "max", + file_prefix: str = "model_best", + ) -> None: + """ + Args: + eval_period (int): the period `EvalHook` is set to run. + checkpointer: the checkpointer object used to save checkpoints. + val_metric (str): validation metric to track for best checkpoint, e.g. "bbox/AP50" + mode (str): one of {'max', 'min'}. controls whether the chosen val metric should be + maximized or minimized, e.g. for "bbox/AP50" it should be "max" + file_prefix (str): the prefix of checkpoint's filename, defaults to "model_best" + """ + self._logger = logging.getLogger(__name__) + self._period = eval_period + self._val_metric = val_metric + assert mode in [ + "max", + "min", + ], f'Mode "{mode}" to `BestCheckpointer` is unknown. It should be one of {"max", "min"}.' + if mode == "max": + self._compare = operator.gt + else: + self._compare = operator.lt + self._checkpointer = checkpointer + self._file_prefix = file_prefix + self.best_metric = None + self.best_iter = None + + def _update_best(self, val, iteration): + if math.isnan(val) or math.isinf(val): + return False + self.best_metric = val + self.best_iter = iteration + return True + + def _best_checking(self): + metric_tuple = self.trainer.storage.latest().get(self._val_metric) + if metric_tuple is None: + self._logger.warning( + f"Given val metric {self._val_metric} does not seem to be computed/stored." + "Will not be checkpointing based on it." + ) + return + else: + latest_metric, metric_iter = metric_tuple + + if self.best_metric is None: + if self._update_best(latest_metric, metric_iter): + additional_state = {"iteration": metric_iter} + self._checkpointer.save(f"{self._file_prefix}", **additional_state) + self._logger.info(f"Saved first model at {self.best_metric:0.5f} @ {self.best_iter} steps") + elif self._compare(latest_metric, self.best_metric): + additional_state = {"iteration": metric_iter} + self._checkpointer.save(f"{self._file_prefix}", **additional_state) + self._logger.info( + f"Saved best model as latest eval score for {self._val_metric} is " + f"{latest_metric:0.5f}, better than last best score " + f"{self.best_metric:0.5f} @ iteration {self.best_iter}." + ) + self._update_best(latest_metric, metric_iter) + else: + self._logger.info( + f"Not saving as latest eval score for {self._val_metric} is {latest_metric:0.5f}, " + f"not better than best score {self.best_metric:0.5f} @ iteration {self.best_iter}." + ) + + def after_step(self): + # same conditions as `EvalHook` + next_iter = self.trainer.iter + 1 + if self._period > 0 and next_iter % self._period == 0 and next_iter != self.trainer.max_iter: + self._best_checking() + + def after_train(self): + # same conditions as `EvalHook` + if self.trainer.iter + 1 >= self.trainer.max_iter: + self._best_checking() + + +class LRScheduler(HookBase): + """ + A hook which executes a torch builtin LR scheduler and summarizes the LR. + It is executed after every iteration. + """ + + def __init__(self, optimizer=None, scheduler=None): + """ + Args: + optimizer (torch.optim.Optimizer): + scheduler (torch.optim.LRScheduler or fvcore.common.param_scheduler.ParamScheduler): + if a :class:`ParamScheduler` object, it defines the multiplier over the base LR + in the optimizer. + + If any argument is not given, will try to obtain it from the trainer. + """ + self._optimizer = optimizer + self._scheduler = scheduler + + def before_train(self): + self._optimizer = self._optimizer or self.trainer.optimizer + if isinstance(self.scheduler, ParamScheduler): + self._scheduler = LRMultiplier( + self._optimizer, + self.scheduler, + self.trainer.max_iter, + last_iter=self.trainer.iter - 1, + ) + self._best_param_group_id = LRScheduler.get_best_param_group_id(self._optimizer) + + @staticmethod + def get_best_param_group_id(optimizer): + # NOTE: some heuristics on what LR to summarize + # summarize the param group with most parameters + largest_group = max(len(g["params"]) for g in optimizer.param_groups) + + if largest_group == 1: + # If all groups have one parameter, + # then find the most common initial LR, and use it for summary + lr_count = Counter([g["lr"] for g in optimizer.param_groups]) + lr = lr_count.most_common()[0][0] + for i, g in enumerate(optimizer.param_groups): + if g["lr"] == lr: + return i + else: + for i, g in enumerate(optimizer.param_groups): + if len(g["params"]) == largest_group: + return i + + def after_step(self): + lr = self._optimizer.param_groups[self._best_param_group_id]["lr"] + self.trainer.storage.put_scalar("lr", lr, smoothing_hint=False) + self.scheduler.step() + + @property + def scheduler(self): + return self._scheduler or self.trainer.scheduler + + def state_dict(self): + if isinstance(self.scheduler, torch.optim.lr_scheduler._LRScheduler): + return self.scheduler.state_dict() + return {} + + def load_state_dict(self, state_dict): + if isinstance(self.scheduler, torch.optim.lr_scheduler._LRScheduler): + logger = logging.getLogger(__name__) + logger.info("Loading scheduler from state_dict ...") + self.scheduler.load_state_dict(state_dict) + + +class TorchProfiler(HookBase): + """ + A hook which runs `torch.profiler.profile`. + + Examples: + :: + hooks.TorchProfiler( + lambda trainer: 10 < trainer.iter < 20, self.cfg.OUTPUT_DIR + ) + + The above example will run the profiler for iteration 10~20 and dump + results to ``OUTPUT_DIR``. We did not profile the first few iterations + because they are typically slower than the rest. + The result files can be loaded in the ``chrome://tracing`` page in chrome browser, + and the tensorboard visualizations can be visualized using + ``tensorboard --logdir OUTPUT_DIR/log`` + """ + + def __init__(self, enable_predicate, output_dir, *, activities=None, save_tensorboard=True): + """ + Args: + enable_predicate (callable[trainer -> bool]): a function which takes a trainer, + and returns whether to enable the profiler. + It will be called once every step, and can be used to select which steps to profile. + output_dir (str): the output directory to dump tracing files. + activities (iterable): same as in `torch.profiler.profile`. + save_tensorboard (bool): whether to save tensorboard visualizations at (output_dir)/log/ + """ + self._enable_predicate = enable_predicate + self._activities = activities + self._output_dir = output_dir + self._save_tensorboard = save_tensorboard + + def before_step(self): + if self._enable_predicate(self.trainer): + if self._save_tensorboard: + on_trace_ready = torch.profiler.tensorboard_trace_handler( + os.path.join( + self._output_dir, + "log", + "profiler-tensorboard-iter{}".format(self.trainer.iter), + ), + f"worker{comm.get_rank()}", + ) + else: + on_trace_ready = None + self._profiler = torch.profiler.profile( + activities=self._activities, + on_trace_ready=on_trace_ready, + record_shapes=True, + profile_memory=True, + with_stack=True, + with_flops=True, + ) + self._profiler.__enter__() + else: + self._profiler = None + + def after_step(self): + if self._profiler is None: + return + self._profiler.__exit__(None, None, None) + if not self._save_tensorboard: + PathManager.mkdirs(self._output_dir) + out_file = os.path.join(self._output_dir, "profiler-trace-iter{}.json".format(self.trainer.iter)) + if "://" not in out_file: + self._profiler.export_chrome_trace(out_file) + else: + # Support non-posix filesystems + with tempfile.TemporaryDirectory(prefix="detectron2_profiler") as d: + tmp_file = os.path.join(d, "tmp.json") + self._profiler.export_chrome_trace(tmp_file) + with open(tmp_file) as f: + content = f.read() + with PathManager.open(out_file, "w") as f: + f.write(content) + + +class AutogradProfiler(TorchProfiler): + """ + A hook which runs `torch.autograd.profiler.profile`. + + Examples: + :: + hooks.AutogradProfiler( + lambda trainer: 10 < trainer.iter < 20, self.cfg.OUTPUT_DIR + ) + + The above example will run the profiler for iteration 10~20 and dump + results to ``OUTPUT_DIR``. We did not profile the first few iterations + because they are typically slower than the rest. + The result files can be loaded in the ``chrome://tracing`` page in chrome browser. + + Note: + When used together with NCCL on older version of GPUs, + autograd profiler may cause deadlock because it unnecessarily allocates + memory on every device it sees. The memory management calls, if + interleaved with NCCL calls, lead to deadlock on GPUs that do not + support ``cudaLaunchCooperativeKernelMultiDevice``. + """ + + def __init__(self, enable_predicate, output_dir, *, use_cuda=True): + """ + Args: + enable_predicate (callable[trainer -> bool]): a function which takes a trainer, + and returns whether to enable the profiler. + It will be called once every step, and can be used to select which steps to profile. + output_dir (str): the output directory to dump tracing files. + use_cuda (bool): same as in `torch.autograd.profiler.profile`. + """ + warnings.warn("AutogradProfiler has been deprecated in favor of TorchProfiler.") + self._enable_predicate = enable_predicate + self._use_cuda = use_cuda + self._output_dir = output_dir + + def before_step(self): + if self._enable_predicate(self.trainer): + self._profiler = torch.autograd.profiler.profile(use_cuda=self._use_cuda) + self._profiler.__enter__() + else: + self._profiler = None + + +class EvalHook(HookBase): + """ + Run an evaluation function periodically, and at the end of training. + + It is executed every ``eval_period`` iterations and after the last iteration. + """ + + def __init__(self, eval_period, eval_function, eval_after_train=True): + """ + Args: + eval_period (int): the period to run `eval_function`. Set to 0 to + not evaluate periodically (but still evaluate after the last iteration + if `eval_after_train` is True). + eval_function (callable): a function which takes no arguments, and + returns a nested dict of evaluation metrics. + eval_after_train (bool): whether to evaluate after the last iteration + + Note: + This hook must be enabled in all or none workers. + If you would like only certain workers to perform evaluation, + give other workers a no-op function (`eval_function=lambda: None`). + """ + self._period = eval_period + self._func = eval_function + self._eval_after_train = eval_after_train + + def _do_eval(self): + results = self._func() + + if results: + assert isinstance(results, dict), "Eval function must return a dict. Got {} instead.".format(results) + + flattened_results = flatten_results_dict(results) + for k, v in flattened_results.items(): + try: + v = float(v) + except Exception as e: + raise ValueError( + "[EvalHook] eval_function should return a nested dict of float. " + "Got '{}: {}' instead.".format(k, v) + ) from e + self.trainer.storage.put_scalars(**flattened_results, smoothing_hint=False) + + # Evaluation may take different time among workers. + # A barrier make them start the next iteration together. + comm.synchronize() + + def after_step(self): + next_iter = self.trainer.iter + 1 + if self._period > 0 and next_iter % self._period == 0: + # do the last eval in after_train + if next_iter != self.trainer.max_iter: + self._do_eval() + + def after_train(self): + # This condition is to prevent the eval from running after a failed training + if self._eval_after_train and self.trainer.iter + 1 >= self.trainer.max_iter: + self._do_eval() + # func is likely a closure that holds reference to the trainer + # therefore we clean it to avoid circular reference in the end + del self._func + + +class PreciseBN(HookBase): + """ + The standard implementation of BatchNorm uses EMA in inference, which is + sometimes suboptimal. + This class computes the true average of statistics rather than the moving average, + and put true averages to every BN layer in the given model. + + It is executed every ``period`` iterations and after the last iteration. + """ + + def __init__(self, period, model, data_loader, num_iter): + """ + Args: + period (int): the period this hook is run, or 0 to not run during training. + The hook will always run in the end of training. + model (nn.Module): a module whose all BN layers in training mode will be + updated by precise BN. + Note that user is responsible for ensuring the BN layers to be + updated are in training mode when this hook is triggered. + data_loader (iterable): it will produce data to be run by `model(data)`. + num_iter (int): number of iterations used to compute the precise + statistics. + """ + self._logger = logging.getLogger(__name__) + if len(get_bn_modules(model)) == 0: + self._logger.info("PreciseBN is disabled because model does not contain BN layers in training mode.") + self._disabled = True + return + + self._model = model + self._data_loader = data_loader + self._num_iter = num_iter + self._period = period + self._disabled = False + + self._data_iter = None + + def after_step(self): + next_iter = self.trainer.iter + 1 + is_final = next_iter == self.trainer.max_iter + if is_final or (self._period > 0 and next_iter % self._period == 0): + self.update_stats() + + def update_stats(self): + """ + Update the model with precise statistics. Users can manually call this method. + """ + if self._disabled: + return + + if self._data_iter is None: + self._data_iter = iter(self._data_loader) + + def data_loader(): + for num_iter in itertools.count(1): + if num_iter % 100 == 0: + self._logger.info("Running precise-BN ... {}/{} iterations.".format(num_iter, self._num_iter)) + # This way we can reuse the same iterator + yield next(self._data_iter) + + with EventStorage(): # capture events in a new storage to discard them + self._logger.info( + "Running precise-BN for {} iterations... ".format(self._num_iter) + + "Note that this could produce different statistics every time." + ) + update_bn_stats(self._model, data_loader(), self._num_iter) + + +class TorchMemoryStats(HookBase): + """ + Writes pytorch's cuda memory statistics periodically. + """ + + def __init__(self, period=20, max_runs=10): + """ + Args: + period (int): Output stats each 'period' iterations + max_runs (int): Stop the logging after 'max_runs' + """ + + self._logger = logging.getLogger(__name__) + self._period = period + self._max_runs = max_runs + self._runs = 0 + + def after_step(self): + if self._runs > self._max_runs: + return + + if (self.trainer.iter + 1) % self._period == 0 or (self.trainer.iter == self.trainer.max_iter - 1): + if torch.cuda.is_available(): + max_reserved_mb = torch.cuda.max_memory_reserved() / 1024.0 / 1024.0 + reserved_mb = torch.cuda.memory_reserved() / 1024.0 / 1024.0 + max_allocated_mb = torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 + allocated_mb = torch.cuda.memory_allocated() / 1024.0 / 1024.0 + + self._logger.info( + ( + " iter: {} " + " max_reserved_mem: {:.0f}MB " + " reserved_mem: {:.0f}MB " + " max_allocated_mem: {:.0f}MB " + " allocated_mem: {:.0f}MB " + ).format( + self.trainer.iter, + max_reserved_mb, + reserved_mb, + max_allocated_mb, + allocated_mb, + ) + ) + + self._runs += 1 + if self._runs == self._max_runs: + mem_summary = torch.cuda.memory_summary() + self._logger.info("\n" + mem_summary) + + torch.cuda.reset_peak_memory_stats() diff --git a/detectron2/engine/launch.py b/detectron2/engine/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..d61274bd3cfef1a7046506cdfb1eb93ad69fb415 --- /dev/null +++ b/detectron2/engine/launch.py @@ -0,0 +1,125 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +from datetime import timedelta + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from detectron2.utils import comm + +__all__ = ["DEFAULT_TIMEOUT", "launch"] + +DEFAULT_TIMEOUT = timedelta(minutes=30) + + +def _find_free_port(): + import socket + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Binding to port 0 will cause the OS to find an available port for us + sock.bind(("", 0)) + port = sock.getsockname()[1] + sock.close() + # NOTE: there is still a chance the port could be taken by other processes. + return port + + +def launch( + main_func, + num_gpus_per_machine, + num_machines=1, + machine_rank=0, + dist_url=None, + args=(), + timeout=DEFAULT_TIMEOUT, +): + """ + Launch multi-gpu or distributed training. + This function must be called on all machines involved in the training. + It will spawn child processes (defined by ``num_gpus_per_machine``) on each machine. + + Args: + main_func: a function that will be called by `main_func(*args)` + num_gpus_per_machine (int): number of GPUs per machine + num_machines (int): the total number of machines + machine_rank (int): the rank of this machine + dist_url (str): url to connect to for distributed jobs, including protocol + e.g. "tcp://127.0.0.1:8686". + Can be set to "auto" to automatically select a free port on localhost + timeout (timedelta): timeout of the distributed workers + args (tuple): arguments passed to main_func + """ + world_size = num_machines * num_gpus_per_machine + if world_size > 1: + # https://github.com/pytorch/pytorch/pull/14391 + # TODO prctl in spawned processes + + if dist_url == "auto": + assert num_machines == 1, "dist_url=auto not supported in multi-machine jobs." + port = _find_free_port() + dist_url = f"tcp://127.0.0.1:{port}" + if num_machines > 1 and dist_url.startswith("file://"): + logger = logging.getLogger(__name__) + logger.warning("file:// is not a reliable init_method in multi-machine jobs. Prefer tcp://") + + mp.spawn( + _distributed_worker, + nprocs=num_gpus_per_machine, + args=( + main_func, + world_size, + num_gpus_per_machine, + machine_rank, + dist_url, + args, + timeout, + ), + daemon=False, + ) + else: + main_func(*args) + + +def _distributed_worker( + local_rank, + main_func, + world_size, + num_gpus_per_machine, + machine_rank, + dist_url, + args, + timeout=DEFAULT_TIMEOUT, +): + assert torch.cuda.is_available(), "cuda is not available. Please check your installation." + global_rank = machine_rank * num_gpus_per_machine + local_rank + try: + dist.init_process_group( + backend="NCCL", + init_method=dist_url, + world_size=world_size, + rank=global_rank, + timeout=timeout, + ) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error("Process group URL: {}".format(dist_url)) + raise e + + # Setup the local process group (which contains ranks within the same machine) + assert comm._LOCAL_PROCESS_GROUP is None + num_machines = world_size // num_gpus_per_machine + for i in range(num_machines): + ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)) + pg = dist.new_group(ranks_on_i) + if i == machine_rank: + comm._LOCAL_PROCESS_GROUP = pg + + assert num_gpus_per_machine <= torch.cuda.device_count() + torch.cuda.set_device(local_rank) + + # synchronize is needed here to prevent a possible timeout after calling init_process_group + # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172 + comm.synchronize() + + main_func(*args) diff --git a/detectron2/engine/train_loop.py b/detectron2/engine/train_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..58aabb44c3f7b8016c4b6e3c2e10ec479864eb84 --- /dev/null +++ b/detectron2/engine/train_loop.py @@ -0,0 +1,428 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +import time +import weakref +from typing import List, Mapping, Optional + +import numpy as np +import torch +from torch.nn.parallel import DataParallel, DistributedDataParallel + +import detectron2.utils.comm as comm +from detectron2.utils.events import EventStorage, get_event_storage +from detectron2.utils.logger import _log_api_usage + +__all__ = ["HookBase", "TrainerBase", "SimpleTrainer", "AMPTrainer"] + + +class HookBase: + """ + Base class for hooks that can be registered with :class:`TrainerBase`. + + Each hook can implement 4 methods. The way they are called is demonstrated + in the following snippet: + :: + hook.before_train() + for iter in range(start_iter, max_iter): + hook.before_step() + trainer.run_step() + hook.after_step() + iter += 1 + hook.after_train() + + Notes: + 1. In the hook method, users can access ``self.trainer`` to access more + properties about the context (e.g., model, current iteration, or config + if using :class:`DefaultTrainer`). + + 2. A hook that does something in :meth:`before_step` can often be + implemented equivalently in :meth:`after_step`. + If the hook takes non-trivial time, it is strongly recommended to + implement the hook in :meth:`after_step` instead of :meth:`before_step`. + The convention is that :meth:`before_step` should only take negligible time. + + Following this convention will allow hooks that do care about the difference + between :meth:`before_step` and :meth:`after_step` (e.g., timer) to + function properly. + + """ + + trainer: "TrainerBase" = None + """ + A weak reference to the trainer object. Set by the trainer when the hook is registered. + """ + + def before_train(self): + """ + Called before the first iteration. + """ + pass + + def after_train(self): + """ + Called after the last iteration. + """ + pass + + def before_step(self): + """ + Called before each iteration. + """ + pass + + def after_step(self): + """ + Called after each iteration. + """ + pass + + def state_dict(self): + """ + Hooks are stateless by default, but can be made checkpointable by + implementing `state_dict` and `load_state_dict`. + """ + return {} + + +class TrainerBase: + """ + Base class for iterative trainer with hooks. + + The only assumption we made here is: the training runs in a loop. + A subclass can implement what the loop is. + We made no assumptions about the existence of dataloader, optimizer, model, etc. + + Attributes: + iter(int): the current iteration. + + start_iter(int): The iteration to start with. + By convention the minimum possible value is 0. + + max_iter(int): The iteration to end training. + + storage(EventStorage): An EventStorage that's opened during the course of training. + """ + + def __init__(self) -> None: + self._hooks: List[HookBase] = [] + self.iter: int = 0 + self.start_iter: int = 0 + self.max_iter: int + self.storage: EventStorage + _log_api_usage("trainer." + self.__class__.__name__) + + def register_hooks(self, hooks: List[Optional[HookBase]]) -> None: + """ + Register hooks to the trainer. The hooks are executed in the order + they are registered. + + Args: + hooks (list[Optional[HookBase]]): list of hooks + """ + hooks = [h for h in hooks if h is not None] + for h in hooks: + assert isinstance(h, HookBase) + # To avoid circular reference, hooks and trainer cannot own each other. + # This normally does not matter, but will cause memory leak if the + # involved objects contain __del__: + # See http://engineering.hearsaysocial.com/2013/06/16/circular-references-in-python/ + h.trainer = weakref.proxy(self) + self._hooks.extend(hooks) + + def train(self, start_iter: int, max_iter: int): + """ + Args: + start_iter, max_iter (int): See docs above + """ + logger = logging.getLogger(__name__) + logger.info("Starting training from iteration {}".format(start_iter)) + + self.iter = self.start_iter = start_iter + self.max_iter = max_iter + + with EventStorage(start_iter) as self.storage: + try: + self.before_train() + for self.iter in range(start_iter, max_iter): + self.before_step() + self.run_step() + self.after_step() + # self.iter == max_iter can be used by `after_train` to + # tell whether the training successfully finished or failed + # due to exceptions. + self.iter += 1 + except Exception: + logger.exception("Exception during training:") + raise + finally: + self.after_train() + + def before_train(self): + for h in self._hooks: + h.before_train() + + def after_train(self): + self.storage.iter = self.iter + for h in self._hooks: + h.after_train() + + def before_step(self): + # Maintain the invariant that storage.iter == trainer.iter + # for the entire execution of each step + self.storage.iter = self.iter + + for h in self._hooks: + h.before_step() + + def after_step(self): + for h in self._hooks: + h.after_step() + + def run_step(self): + raise NotImplementedError + + def state_dict(self): + ret = {"iteration": self.iter} + hooks_state = {} + for h in self._hooks: + sd = h.state_dict() + if sd: + name = type(h).__qualname__ + if name in hooks_state: + # TODO handle repetitive stateful hooks + continue + hooks_state[name] = sd + if hooks_state: + ret["hooks"] = hooks_state + return ret + + def load_state_dict(self, state_dict): + logger = logging.getLogger(__name__) + self.iter = state_dict["iteration"] + for key, value in state_dict.get("hooks", {}).items(): + for h in self._hooks: + try: + name = type(h).__qualname__ + except AttributeError: + continue + if name == key: + h.load_state_dict(value) + break + else: + logger.warning(f"Cannot find the hook '{key}', its state_dict is ignored.") + + +class SimpleTrainer(TrainerBase): + """ + A simple trainer for the most common type of task: + single-cost single-optimizer single-data-source iterative optimization, + optionally using data-parallelism. + It assumes that every step, you: + + 1. Compute the loss with a data from the data_loader. + 2. Compute the gradients with the above loss. + 3. Update the model with the optimizer. + + All other tasks during training (checkpointing, logging, evaluation, LR schedule) + are maintained by hooks, which can be registered by :meth:`TrainerBase.register_hooks`. + + If you want to do anything fancier than this, + either subclass TrainerBase and implement your own `run_step`, + or write your own training loop. + """ + + def __init__(self, model, data_loader, optimizer): + """ + Args: + model: a torch Module. Takes a data from data_loader and returns a + dict of losses. + data_loader: an iterable. Contains data to be used to call model. + optimizer: a torch optimizer. + """ + super().__init__() + + """ + We set the model to training mode in the trainer. + However it's valid to train a model that's in eval mode. + If you want your model (or a submodule of it) to behave + like evaluation during training, you can overwrite its train() method. + """ + model.train() + + self.model = model + self.data_loader = data_loader + # to access the data loader iterator, call `self._data_loader_iter` + self._data_loader_iter_obj = None + self.optimizer = optimizer + + def run_step(self): + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + data = next(self._data_loader_iter) + data_time = time.perf_counter() - start + + """ + If you want to do something with the losses, you can wrap the model. + """ + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + self.optimizer.zero_grad() + losses.backward() + + self._write_metrics(loss_dict, data_time) + + """ + If you need gradient clipping/scaling or other processing, you can + wrap the optimizer with your custom `step()` method. But it is + suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 + """ + self.optimizer.step() + + @property + def _data_loader_iter(self): + # only create the data loader iterator when it is used + if self._data_loader_iter_obj is None: + self._data_loader_iter_obj = iter(self.data_loader) + return self._data_loader_iter_obj + + def reset_data_loader(self, data_loader): + del self.data_loader + self.data_loader = data_loader + self._data_loader_iter_obj = None + + def _write_metrics( + self, + loss_dict: Mapping[str, torch.Tensor], + data_time: float, + prefix: str = "", + ) -> None: + SimpleTrainer.write_metrics(loss_dict, data_time, prefix) + + @staticmethod + def write_metrics( + loss_dict: Mapping[str, torch.Tensor], + data_time: float, + prefix: str = "", + ) -> None: + """ + Args: + loss_dict (dict): dict of scalar losses + data_time (float): time taken by the dataloader iteration + prefix (str): prefix for logging keys + """ + metrics_dict = {k: v.detach().cpu().item() for k, v in loss_dict.items()} + metrics_dict["data_time"] = data_time + + # Gather metrics among all workers for logging + # This assumes we do DDP-style training, which is currently the only + # supported method in detectron2. + all_metrics_dict = comm.gather(metrics_dict) + + if comm.is_main_process(): + storage = get_event_storage() + + # data_time among workers can have high variance. The actual latency + # caused by data_time is the maximum among workers. + data_time = np.max([x.pop("data_time") for x in all_metrics_dict]) + storage.put_scalar("data_time", data_time) + + # average the rest metrics + metrics_dict = {k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys()} + total_losses_reduced = sum(metrics_dict.values()) + if not np.isfinite(total_losses_reduced): + raise FloatingPointError( + f"Loss became infinite or NaN at iteration={storage.iter}!\n" f"loss_dict = {metrics_dict}" + ) + + storage.put_scalar("{}total_loss".format(prefix), total_losses_reduced) + if len(metrics_dict) > 1: + storage.put_scalars(**metrics_dict) + + def state_dict(self): + ret = super().state_dict() + ret["optimizer"] = self.optimizer.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self.optimizer.load_state_dict(state_dict["optimizer"]) + + +class AMPTrainer(SimpleTrainer): + """ + Like :class:`SimpleTrainer`, but uses PyTorch's native automatic mixed precision + in the training loop. + """ + + def __init__(self, model, data_loader, optimizer, grad_scaler=None): + """ + Args: + model, data_loader, optimizer: same as in :class:`SimpleTrainer`. + grad_scaler: torch GradScaler to automatically scale gradients. + """ + unsupported = "AMPTrainer does not support single-process multi-device training!" + if isinstance(model, DistributedDataParallel): + assert not (model.device_ids and len(model.device_ids) > 1), unsupported + assert not isinstance(model, DataParallel), unsupported + + super().__init__(model, data_loader, optimizer) + + if grad_scaler is None: + from torch.cuda.amp import GradScaler + + grad_scaler = GradScaler() + self.grad_scaler = grad_scaler + + def run_step(self): + """ + Implement the AMP training logic. + """ + assert self.model.training, "[AMPTrainer] model was changed to eval mode!" + assert torch.cuda.is_available(), "[AMPTrainer] CUDA is required for AMP training!" + from torch.cuda.amp import autocast + + start = time.perf_counter() + data = next(self._data_loader_iter) + data_time = time.perf_counter() - start + + with autocast(): + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + self.optimizer.zero_grad() + self.grad_scaler.scale(losses).backward() + + self._write_metrics(loss_dict, data_time) + + self.grad_scaler.step(self.optimizer) + self.grad_scaler.update() + + def state_dict(self): + ret = super().state_dict() + ret["grad_scaler"] = self.grad_scaler.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self.grad_scaler.load_state_dict(state_dict["grad_scaler"]) diff --git a/detectron2/evaluation/__init__.py b/detectron2/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..89354fa5cf8c3fb676b2ab03af19c9d42dd41bbb --- /dev/null +++ b/detectron2/evaluation/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .cityscapes_evaluation import CityscapesInstanceEvaluator, CityscapesSemSegEvaluator +from .coco_evaluation import COCOEvaluator +from .evaluator import DatasetEvaluator, DatasetEvaluators, inference_context, inference_on_dataset +from .lvis_evaluation import LVISEvaluator +from .panoptic_evaluation import COCOPanopticEvaluator +from .pascal_voc_evaluation import PascalVOCDetectionEvaluator +from .rotated_coco_evaluation import RotatedCOCOEvaluator +from .sem_seg_evaluation import SemSegEvaluator +from .testing import print_csv_format, verify_results + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/detectron2/evaluation/cityscapes_evaluation.py b/detectron2/evaluation/cityscapes_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..c6bf68f536327c641e100030080d9b4a3d18ae87 --- /dev/null +++ b/detectron2/evaluation/cityscapes_evaluation.py @@ -0,0 +1,190 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import glob +import logging +import os +import tempfile +from collections import OrderedDict + +import numpy as np +import torch +from PIL import Image + +from detectron2.data import MetadataCatalog +from detectron2.utils import comm +from detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + + +class CityscapesEvaluator(DatasetEvaluator): + """ + Base class for evaluation using cityscapes API. + """ + + def __init__(self, dataset_name): + """ + Args: + dataset_name (str): the name of the dataset. + It must have the following metadata associated with it: + "thing_classes", "gt_dir". + """ + self._metadata = MetadataCatalog.get(dataset_name) + self._cpu_device = torch.device("cpu") + self._logger = logging.getLogger(__name__) + + def reset(self): + self._working_dir = tempfile.TemporaryDirectory(prefix="cityscapes_eval_") + self._temp_dir = self._working_dir.name + # All workers will write to the same results directory + # TODO this does not work in distributed training + assert ( + comm.get_local_size() == comm.get_world_size() + ), "CityscapesEvaluator currently do not work with multiple machines." + self._temp_dir = comm.all_gather(self._temp_dir)[0] + if self._temp_dir != self._working_dir.name: + self._working_dir.cleanup() + self._logger.info("Writing cityscapes results to temporary directory {} ...".format(self._temp_dir)) + + +class CityscapesInstanceEvaluator(CityscapesEvaluator): + """ + Evaluate instance segmentation results on cityscapes dataset using cityscapes API. + + Note: + * It does not work in multi-machine distributed training. + * It contains a synchronization, therefore has to be used on all ranks. + * Only the main process runs evaluation. + """ + + def process(self, inputs, outputs): + from cityscapesscripts.helpers.labels import name2label + + for input, output in zip(inputs, outputs): + file_name = input["file_name"] + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_txt = os.path.join(self._temp_dir, basename + "_pred.txt") + + if "instances" in output: + output = output["instances"].to(self._cpu_device) + num_instances = len(output) + with open(pred_txt, "w") as fout: + for i in range(num_instances): + pred_class = output.pred_classes[i] + classes = self._metadata.thing_classes[pred_class] + class_id = name2label[classes].id + score = output.scores[i] + mask = output.pred_masks[i].numpy().astype("uint8") + png_filename = os.path.join(self._temp_dir, basename + "_{}_{}.png".format(i, classes)) + + Image.fromarray(mask * 255).save(png_filename) + fout.write("{} {} {}\n".format(os.path.basename(png_filename), class_id, score)) + else: + # Cityscapes requires a prediction file for every ground truth image. + with open(pred_txt, "w") as fout: + pass + + def evaluate(self): + """ + Returns: + dict: has a key "segm", whose value is a dict of "AP" and "AP50". + """ + comm.synchronize() + if comm.get_rank() > 0: + return + import cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + cityscapes_eval.args.gtInstancesFile = os.path.join(self._temp_dir, "gtInstances.json") + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py # noqa + gt_dir = PathManager.get_local_path(self._metadata.gt_dir) + groundTruthImgList = glob.glob(os.path.join(gt_dir, "*", "*_gtFine_instanceIds.png")) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(gt, cityscapes_eval.args)) + results = cityscapes_eval.evaluateImgLists(predictionImgList, groundTruthImgList, cityscapes_eval.args)[ + "averages" + ] + + ret = OrderedDict() + ret["segm"] = {"AP": results["allAp"] * 100, "AP50": results["allAp50%"] * 100} + self._working_dir.cleanup() + return ret + + +class CityscapesSemSegEvaluator(CityscapesEvaluator): + """ + Evaluate semantic segmentation results on cityscapes dataset using cityscapes API. + + Note: + * It does not work in multi-machine distributed training. + * It contains a synchronization, therefore has to be used on all ranks. + * Only the main process runs evaluation. + """ + + def process(self, inputs, outputs): + from cityscapesscripts.helpers.labels import trainId2label + + for input, output in zip(inputs, outputs): + file_name = input["file_name"] + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_filename = os.path.join(self._temp_dir, basename + "_pred.png") + + output = output["sem_seg"].argmax(dim=0).to(self._cpu_device).numpy() + pred = 255 * np.ones(output.shape, dtype=np.uint8) + for train_id, label in trainId2label.items(): + if label.ignoreInEval: + continue + pred[output == train_id] = label.id + Image.fromarray(pred).save(pred_filename) + + def evaluate(self): + comm.synchronize() + if comm.get_rank() > 0: + return + # Load the Cityscapes eval script *after* setting the required env var, + # since the script reads CITYSCAPES_DATASET into global variables at load time. + import cityscapesscripts.evaluation.evalPixelLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalPixelLevelSemanticLabeling.py # noqa + gt_dir = PathManager.get_local_path(self._metadata.gt_dir) + groundTruthImgList = glob.glob(os.path.join(gt_dir, "*", "*_gtFine_labelIds.png")) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(cityscapes_eval.args, gt)) + results = cityscapes_eval.evaluateImgLists(predictionImgList, groundTruthImgList, cityscapes_eval.args) + ret = OrderedDict() + ret["sem_seg"] = { + "IoU": 100.0 * results["averageScoreClasses"], + "iIoU": 100.0 * results["averageScoreInstClasses"], + "IoU_sup": 100.0 * results["averageScoreCategories"], + "iIoU_sup": 100.0 * results["averageScoreInstCategories"], + } + self._working_dir.cleanup() + return ret diff --git a/detectron2/evaluation/coco_evaluation.py b/detectron2/evaluation/coco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..e39dd6d1b839d45201cd4582a24a565732f8bbde --- /dev/null +++ b/detectron2/evaluation/coco_evaluation.py @@ -0,0 +1,705 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import copy +import io +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict + +import numpy as np +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tabulate import tabulate + +import detectron2.utils.comm as comm +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.data.datasets.coco import convert_to_coco_json +from detectron2.structures import Boxes, BoxMode, pairwise_iou +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table + +from .evaluator import DatasetEvaluator + +try: + from detectron2.evaluation.fast_eval_api import COCOeval_opt +except ImportError: + COCOeval_opt = COCOeval + + +class COCOEvaluator(DatasetEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + use_fast_impl=True, + kpt_oks_sigmas=(), + allow_cached_coco=True, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have either the following corresponding metadata: + + "json_file": the path to the COCO format annotation + + Or it must be in detectron2's standard dataset format + so it can be converted to COCO format automatically. + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm", "keypoints". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks and run evaluation + in the main process. + Otherwise, will only evaluate the results in the current process. + output_dir (str): optional, an output directory to dump all + results predicted on the dataset. The dump contains two files: + + 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and + contains all the results in the format they are produced by the model. + 2. "coco_instances_results.json" a json file in COCO's result format. + max_dets_per_image (int): limit on the maximum number of detections per image. + By default in COCO, this limit is to 100, but this can be customized + to be greater, as is needed in evaluation metrics AP fixed and AP pool + (see https://arxiv.org/pdf/2102.01066.pdf) + This doesn't affect keypoint evaluation. + use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. + Although the results should be very close to the official implementation in COCO + API, it is still recommended to compute results with the official API for use in + papers. The faster implementation also uses more RAM. + kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. + See http://cocodataset.org/#keypoints-eval + When empty, it will use the defaults in COCO. + Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. + allow_cached_coco (bool): Whether to use cached coco json from previous validation + runs. You should set this to False if you need to use different validation data. + Defaults to True. + """ + self._logger = logging.getLogger(__name__) + self._distributed = distributed + self._output_dir = output_dir + + if use_fast_impl and (COCOeval_opt is COCOeval): + self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.") + use_fast_impl = False + self._use_fast_impl = use_fast_impl + + # COCOeval requires the limit on the number of detections per image (maxDets) to be a list + # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the + # 3rd element (100) is used as the limit on the number of detections per image when + # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, + # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] + else: + max_dets_per_image = [1, 10, max_dets_per_image] + self._max_dets_per_image = max_dets_per_image + + if tasks is not None and isinstance(tasks, CfgNode): + kpt_oks_sigmas = tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, "json_file"): + if output_dir is None: + raise ValueError("output_dir must be provided to COCOEvaluator " "for datasets not in COCO format.") + self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") + + cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") + self._metadata.json_file = cache_path + convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco) + + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + + # Test set json files do not contain annotations (evaluation must be + # performed using the COCO evaluation server). + self._do_evaluation = "annotations" in self._coco_api.dataset + if self._do_evaluation: + self._kpt_oks_sigmas = kpt_oks_sigmas + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + if len(prediction) > 1: + self._predictions.append(prediction) + + def evaluate(self, img_ids=None): + """ + Args: + img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset + """ + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return {} + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions, img_ids=img_ids) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + """ + Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions. + """ + tasks = {"bbox"} + for pred in predictions: + if "segmentation" in pred: + tasks.add("segm") + if "keypoints" in pred: + tasks.add("keypoints") + return sorted(tasks) + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format("unofficial" if self._use_fast_impl else "official") + ) + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + use_fast_impl=self._use_fast_impl, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results(coco_eval, task, class_names=self._metadata.get("thing_classes")) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], + }[iou_type] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") + for idx, metric in enumerate(metrics) + } + self._logger.info("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + has_mask = instances.has("pred_masks") + if has_mask: + # use RLE to encode the masks, because they are too large and takes memory + # since this evaluator stores outputs of the entire dataset + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] for mask in instances.pred_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + has_keypoints = instances.has("pred_keypoints") + if has_keypoints: + keypoints = instances.pred_keypoints + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + if has_mask: + result["segmentation"] = rles[k] + if has_keypoints: + # In COCO annotations, + # keypoints coordinates are pixel indices. + # However our predictions are floating point coordinates. + # Therefore we subtract 0.5 to be consistent with the annotation format. + # This is the inverse of data loading logic in `datasets/coco.py`. + keypoints[k][:, :2] -= 0.5 + result["keypoints"] = keypoints[k].flatten().tolist() + results.append(result) + return results + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official COCO API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"]) + anno = coco_api.loadAnns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) for obj in anno if obj["iscrowd"] == 0 + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + iou_type, + kpt_oks_sigmas=None, + use_fast_impl=True, + img_ids=None, + max_dets_per_image=None, +): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + if iou_type == "segm": + coco_results = copy.deepcopy(coco_results) + # When evaluating mask AP, if the results contain bbox, cocoapi will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in coco_results: + c.pop("bbox", None) + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type) + # For COCO, the default max_dets_per_image is [1, 10, 100]. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] # Default from COCOEval + else: + assert ( + len(max_dets_per_image) >= 3 + ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3" + # In the case that user supplies a custom input for max_dets_per_image, + # apply COCOevalMaxDets to evaluate AP with the custom input. + if max_dets_per_image[2] != 100: + coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type) + if iou_type != "keypoints": + coco_eval.params.maxDets = max_dets_per_image + + if img_ids is not None: + coco_eval.params.imgIds = img_ids + + if iou_type == "keypoints": + # Use the COCO default keypoint OKS sigmas unless overrides are specified + if kpt_oks_sigmas: + assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!" + coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas) + # COCOAPI requires every detection and every gt to have keypoints, so + # we just take the first entry from both + num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3 + num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3 + num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas) + assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, ( + f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. " + f"Ground truth contains {num_keypoints_gt} keypoints. " + f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. " + "They have to agree with each other. For meaning of OKS, please refer to " + "http://cocodataset.org/#keypoints-eval." + ) + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval + + +class COCOevalMaxDets(COCOeval): + """ + Modified version of COCOeval for evaluating AP with a custom + maxDets (by default for COCO, maxDets is 100) + """ + + def summarize(self): + """ + Compute and display summary metrics for evaluation results given + a custom value for max_dets_per_image + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) if iouThr is None else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + # Evaluate AP using the custom limit on maximum detections per image + stats[0] = _summarize(1, maxDets=self.params.maxDets[2]) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm" or iouType == "bbox": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() diff --git a/detectron2/evaluation/evaluator.py b/detectron2/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..43202361ee30ff4dbb24944473f4c56255f58624 --- /dev/null +++ b/detectron2/evaluation/evaluator.py @@ -0,0 +1,221 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import datetime +import logging +import time +from collections import OrderedDict, abc +from contextlib import ExitStack, contextmanager +from typing import List, Union + +import torch +from torch import nn + +from detectron2.utils.comm import get_world_size, is_main_process +from detectron2.utils.logger import log_every_n_seconds + + +class DatasetEvaluator: + """ + Base class for a dataset evaluator. + + The function :func:`inference_on_dataset` runs the model over + all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs. + + This class will accumulate information of the inputs/outputs (by :meth:`process`), + and produce evaluation results in the end (by :meth:`evaluate`). + """ + + def reset(self): + """ + Preparation for a new round of evaluation. + Should be called before starting a round of evaluation. + """ + pass + + def process(self, inputs, outputs): + """ + Process the pair of inputs and outputs. + If they contain batches, the pairs can be consumed one-by-one using `zip`: + + .. code-block:: python + + for input_, output in zip(inputs, outputs): + # do evaluation on single input/output pair + ... + + Args: + inputs (list): the inputs that's used to call the model. + outputs (list): the return value of `model(inputs)` + """ + pass + + def evaluate(self): + """ + Evaluate/summarize the performance, after processing all input/output pairs. + + Returns: + dict: + A new evaluator class can return a dict of arbitrary format + as long as the user can process the results. + In our train_net.py, we expect the following format: + + * key: the name of the task (e.g., bbox) + * value: a dict of {metric name: score}, e.g.: {"AP50": 80} + """ + pass + + +class DatasetEvaluators(DatasetEvaluator): + """ + Wrapper class to combine multiple :class:`DatasetEvaluator` instances. + + This class dispatches every evaluation call to + all of its :class:`DatasetEvaluator`. + """ + + def __init__(self, evaluators): + """ + Args: + evaluators (list): the evaluators to combine. + """ + super().__init__() + self._evaluators = evaluators + + def reset(self): + for evaluator in self._evaluators: + evaluator.reset() + + def process(self, inputs, outputs): + for evaluator in self._evaluators: + evaluator.process(inputs, outputs) + + def evaluate(self): + results = OrderedDict() + for evaluator in self._evaluators: + result = evaluator.evaluate() + if is_main_process() and result is not None: + for k, v in result.items(): + assert k not in results, "Different evaluators produce results with the same key {}".format(k) + results[k] = v + return results + + +def inference_on_dataset(model, data_loader, evaluator: Union[DatasetEvaluator, List[DatasetEvaluator], None]): + """ + Run model on the data_loader and evaluate the metrics with evaluator. + Also benchmark the inference speed of `model.__call__` accurately. + The model will be used in eval mode. + + Args: + model (callable): a callable which takes an object from + `data_loader` and returns some outputs. + + If it's an nn.Module, it will be temporarily set to `eval` mode. + If you wish to evaluate a model in `training` mode instead, you can + wrap the given model and override its behavior of `.eval()` and `.train()`. + data_loader: an iterable object with a length. + The elements it generates will be the inputs to the model. + evaluator: the evaluator(s) to run. Use `None` if you only want to benchmark, + but don't want to do any evaluation. + + Returns: + The return value of `evaluator.evaluate()` + """ + num_devices = get_world_size() + logger = logging.getLogger(__name__) + logger.info("Start inference on {} batches".format(len(data_loader))) + + total = len(data_loader) # inference data loader must have a fixed length + if evaluator is None: + # create a no-op evaluator + evaluator = DatasetEvaluators([]) + if isinstance(evaluator, abc.MutableSequence): + evaluator = DatasetEvaluators(evaluator) + evaluator.reset() + + num_warmup = min(5, total - 1) + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + with ExitStack() as stack: + if isinstance(model, nn.Module): + stack.enter_context(inference_context(model)) + stack.enter_context(torch.no_grad()) + + start_data_time = time.perf_counter() + for idx, inputs in enumerate(data_loader): + total_data_time += time.perf_counter() - start_data_time + if idx == num_warmup: + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + + start_compute_time = time.perf_counter() + outputs = model(inputs) + if torch.cuda.is_available(): + torch.cuda.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + + start_eval_time = time.perf_counter() + evaluator.process(inputs, outputs) + total_eval_time += time.perf_counter() - start_eval_time + + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + data_seconds_per_iter = total_data_time / iters_after_start + compute_seconds_per_iter = total_compute_time / iters_after_start + eval_seconds_per_iter = total_eval_time / iters_after_start + total_seconds_per_iter = (time.perf_counter() - start_time) / iters_after_start + if idx >= num_warmup * 2 or compute_seconds_per_iter > 5: + eta = datetime.timedelta(seconds=int(total_seconds_per_iter * (total - idx - 1))) + log_every_n_seconds( + logging.INFO, + ( + f"Inference done {idx + 1}/{total}. " + f"Dataloading: {data_seconds_per_iter:.4f} s/iter. " + f"Inference: {compute_seconds_per_iter:.4f} s/iter. " + f"Eval: {eval_seconds_per_iter:.4f} s/iter. " + f"Total: {total_seconds_per_iter:.4f} s/iter. " + f"ETA={eta}" + ), + n=5, + ) + start_data_time = time.perf_counter() + + # Measure the time only for this worker (before the synchronization barrier) + total_time = time.perf_counter() - start_time + total_time_str = str(datetime.timedelta(seconds=total_time)) + # NOTE this format is parsed by grep + logger.info( + "Total inference time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_time_str, total_time / (total - num_warmup), num_devices + ) + ) + total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time))) + logger.info( + "Total inference pure compute time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_compute_time_str, total_compute_time / (total - num_warmup), num_devices + ) + ) + + results = evaluator.evaluate() + # An evaluator may return None when not in main process. + # Replace it by an empty dict instead to make it easier for downstream code to handle + if results is None: + results = {} + return results + + +@contextmanager +def inference_context(model): + """ + A context where the model is temporarily changed to eval mode, + and restored to previous mode afterwards. + + Args: + model: a torch Module + """ + training_mode = model.training + model.eval() + yield + model.train(training_mode) diff --git a/detectron2/evaluation/fast_eval_api.py b/detectron2/evaluation/fast_eval_api.py new file mode 100644 index 0000000000000000000000000000000000000000..02f49ab099751cac8febb4ff6bfa9bb2970558f3 --- /dev/null +++ b/detectron2/evaluation/fast_eval_api.py @@ -0,0 +1,115 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import time + +import numpy as np +from pycocotools.cocoeval import COCOeval + +from detectron2 import _C + +logger = logging.getLogger(__name__) + + +class COCOeval_opt(COCOeval): + """ + This is a slightly modified version of the original COCO API, where the functions evaluateImg() + and accumulate() are implemented in C++ to speedup evaluation + """ + + def evaluate(self): + """ + Run per image evaluation on given images and store results in self.evalImgs_cpp, a + datastructure that isn't readable from Python but is used by a c++ implementation of + accumulate(). Unlike the original COCO PythonAPI, we don't populate the datastructure + self.evalImgs because this datastructure is a computational bottleneck. + :return: None + """ + tic = time.time() + + p = self.params + # add backward compatibility if useSegm is specified in params + if p.useSegm is not None: + p.iouType = "segm" if p.useSegm == 1 else "bbox" + logger.info("Evaluate annotation type *{}*".format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() # bottleneck + + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == "segm" or p.iouType == "bbox": + computeIoU = self.computeIoU + elif p.iouType == "keypoints": + computeIoU = self.computeOks + self.ious = {(imgId, catId): computeIoU(imgId, catId) for imgId in p.imgIds for catId in catIds} # bottleneck + + maxDet = p.maxDets[-1] + + # <<<< Beginning of code differences with original COCO API + def convert_instances_to_cpp(instances, is_det=False): + # Convert annotations for a list of instances in an image to a format that's fast + # to access in C++ + instances_cpp = [] + for instance in instances: + instance_cpp = _C.InstanceAnnotation( + int(instance["id"]), + instance["score"] if is_det else instance.get("score", 0.0), + instance["area"], + bool(instance.get("iscrowd", 0)), + bool(instance.get("ignore", 0)), + ) + instances_cpp.append(instance_cpp) + return instances_cpp + + # Convert GT annotations, detections, and IOUs to a format that's fast to access in C++ + ground_truth_instances = [ + [convert_instances_to_cpp(self._gts[imgId, catId]) for catId in p.catIds] for imgId in p.imgIds + ] + detected_instances = [ + [convert_instances_to_cpp(self._dts[imgId, catId], is_det=True) for catId in p.catIds] + for imgId in p.imgIds + ] + ious = [[self.ious[imgId, catId] for catId in catIds] for imgId in p.imgIds] + + if not p.useCats: + # For each image, flatten per-category lists into a single list + ground_truth_instances = [[[o for c in i for o in c]] for i in ground_truth_instances] + detected_instances = [[[o for c in i for o in c]] for i in detected_instances] + + # Call C++ implementation of self.evaluateImgs() + self._evalImgs_cpp = _C.COCOevalEvaluateImages( + p.areaRng, maxDet, p.iouThrs, ious, ground_truth_instances, detected_instances + ) + self._evalImgs = None + + self._paramsEval = copy.deepcopy(self.params) + toc = time.time() + logger.info("COCOeval_opt.evaluate() finished in {:0.2f} seconds.".format(toc - tic)) + # >>>> End of code differences with original COCO API + + def accumulate(self): + """ + Accumulate per image evaluation results and store the result in self.eval. Does not + support changing parameter settings from those used by self.evaluate() + """ + logger.info("Accumulating evaluation results...") + tic = time.time() + assert hasattr(self, "_evalImgs_cpp"), "evaluate() must be called before accmulate() is called." + + self.eval = _C.COCOevalAccumulate(self._paramsEval, self._evalImgs_cpp) + + # recall is num_iou_thresholds X num_categories X num_area_ranges X num_max_detections + self.eval["recall"] = np.array(self.eval["recall"]).reshape(self.eval["counts"][:1] + self.eval["counts"][2:]) + + # precision and scores are num_iou_thresholds X num_recall_thresholds X num_categories X + # num_area_ranges X num_max_detections + self.eval["precision"] = np.array(self.eval["precision"]).reshape(self.eval["counts"]) + self.eval["scores"] = np.array(self.eval["scores"]).reshape(self.eval["counts"]) + toc = time.time() + logger.info("COCOeval_opt.accumulate() finished in {:0.2f} seconds.".format(toc - tic)) diff --git a/detectron2/evaluation/lvis_evaluation.py b/detectron2/evaluation/lvis_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..71e23c6c895f1c2be4f4d494a30175f7e60f8b29 --- /dev/null +++ b/detectron2/evaluation/lvis_evaluation.py @@ -0,0 +1,373 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict + +import torch + +import detectron2.utils.comm as comm +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.structures import Boxes, BoxMode, pairwise_iou +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table + +from .coco_evaluation import instances_to_coco_json +from .evaluator import DatasetEvaluator + + +class LVISEvaluator(DatasetEvaluator): + """ + Evaluate object proposal and instance detection/segmentation outputs using + LVIS's metrics and evaluation API. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have the following corresponding metadata: + "json_file": the path to the LVIS format annotation + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks for evaluation. + Otherwise, will evaluate the results in the current process. + output_dir (str): optional, an output directory to dump results. + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + """ + from lvis import LVIS + + self._logger = logging.getLogger(__name__) + + if tasks is not None and isinstance(tasks, CfgNode): + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._distributed = distributed + self._output_dir = output_dir + self._max_dets_per_image = max_dets_per_image + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + json_file = PathManager.get_local_path(self._metadata.json_file) + self._lvis_api = LVIS(json_file) + # Test set json files do not contain annotations (evaluation must be + # performed using the LVIS evaluation server). + self._do_evaluation = len(self._lvis_api.get_ann_ids()) > 0 + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a LVIS model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a LVIS model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + self._predictions.append(prediction) + + def evaluate(self): + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[LVISEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + for pred in predictions: + if "segmentation" in pred: + return ("bbox", "segm") + return ("bbox",) + + def _eval_predictions(self, predictions): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + + Args: + predictions (list[dict]): list of outputs from the model + """ + self._logger.info("Preparing results in the LVIS format ...") + lvis_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(lvis_results) + + # LVIS evaluator can be used to evaluate results for COCO dataset categories. + # In this case `_metadata` variable will have a field with COCO-specific category mapping. + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = {v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items()} + for result in lvis_results: + result["category_id"] = reverse_id_mapping[result["category_id"]] + else: + # unmap the category ids for LVIS (from 0-indexed to 1-indexed) + for result in lvis_results: + result["category_id"] += 1 + + if self._output_dir: + file_path = os.path.join(self._output_dir, "lvis_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(lvis_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + for task in sorted(tasks): + res = _evaluate_predictions_on_lvis( + self._lvis_api, + lvis_results, + task, + max_dets_per_image=self._max_dets_per_image, + class_names=self._metadata.get("thing_classes"), + ) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._lvis_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, lvis_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official LVIS API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = lvis_api.get_ann_ids(img_ids=[prediction_dict["image_id"]]) + anno = lvis_api.load_anns(ann_ids) + gt_boxes = [BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) for obj in anno] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_lvis(lvis_gt, lvis_results, iou_type, max_dets_per_image=None, class_names=None): + """ + Args: + iou_type (str): + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + }[iou_type] + + logger = logging.getLogger(__name__) + + if len(lvis_results) == 0: # TODO: check if needed + logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + if iou_type == "segm": + lvis_results = copy.deepcopy(lvis_results) + # When evaluating mask AP, if the results contain bbox, LVIS API will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in lvis_results: + c.pop("bbox", None) + + if max_dets_per_image is None: + max_dets_per_image = 300 # Default for LVIS dataset + + from lvis import LVISEval, LVISResults + + logger.info(f"Evaluating with max detections per image = {max_dets_per_image}") + lvis_results = LVISResults(lvis_gt, lvis_results, max_dets=max_dets_per_image) + lvis_eval = LVISEval(lvis_gt, lvis_results, iou_type) + lvis_eval.run() + lvis_eval.print_results() + + # Pull the standard metrics from the LVIS results + results = lvis_eval.get_results() + results = {metric: float(results[metric] * 100) for metric in metrics} + logger.info("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) + return results diff --git a/detectron2/evaluation/panoptic_evaluation.py b/detectron2/evaluation/panoptic_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..55e7a82cbe8f3f8d94da8f1647cefcb19170b0ef --- /dev/null +++ b/detectron2/evaluation/panoptic_evaluation.py @@ -0,0 +1,190 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import io +import itertools +import json +import logging +import os +import tempfile +from collections import OrderedDict +from typing import Optional + +import numpy as np +from PIL import Image +from tabulate import tabulate + +from detectron2.data import MetadataCatalog +from detectron2.utils import comm +from detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + +logger = logging.getLogger(__name__) + + +class COCOPanopticEvaluator(DatasetEvaluator): + """ + Evaluate Panoptic Quality metrics on COCO using PanopticAPI. + It saves panoptic segmentation prediction in `output_dir` + + It contains a synchronize call and has to be called from all workers. + """ + + def __init__(self, dataset_name: str, output_dir: Optional[str] = None): + """ + Args: + dataset_name: name of the dataset + output_dir: output directory to save results for evaluation. + """ + self._metadata = MetadataCatalog.get(dataset_name) + self._thing_contiguous_id_to_dataset_id = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + self._stuff_contiguous_id_to_dataset_id = { + v: k for k, v in self._metadata.stuff_dataset_id_to_contiguous_id.items() + } + + self._output_dir = output_dir + if self._output_dir is not None: + PathManager.mkdirs(self._output_dir) + + def reset(self): + self._predictions = [] + + def _convert_category_id(self, segment_info): + isthing = segment_info.pop("isthing", None) + if isthing is None: + # the model produces panoptic category id directly. No more conversion needed + return segment_info + if isthing is True: + segment_info["category_id"] = self._thing_contiguous_id_to_dataset_id[segment_info["category_id"]] + else: + segment_info["category_id"] = self._stuff_contiguous_id_to_dataset_id[segment_info["category_id"]] + return segment_info + + def process(self, inputs, outputs): + from panopticapi.utils import id2rgb + + for input, output in zip(inputs, outputs): + panoptic_img, segments_info = output["panoptic_seg"] + panoptic_img = panoptic_img.cpu().numpy() + if segments_info is None: + # If "segments_info" is None, we assume "panoptic_img" is a + # H*W int32 image storing the panoptic_id in the format of + # category_id * label_divisor + instance_id. We reserve -1 for + # VOID label, and add 1 to panoptic_img since the official + # evaluation script uses 0 for VOID label. + label_divisor = self._metadata.label_divisor + segments_info = [] + for panoptic_label in np.unique(panoptic_img): + if panoptic_label == -1: + # VOID region. + continue + pred_class = panoptic_label // label_divisor + isthing = pred_class in self._metadata.thing_dataset_id_to_contiguous_id.values() + segments_info.append( + { + "id": int(panoptic_label) + 1, + "category_id": int(pred_class), + "isthing": bool(isthing), + } + ) + # Official evaluation script uses 0 for VOID label. + panoptic_img += 1 + + file_name = os.path.basename(input["file_name"]) + file_name_png = os.path.splitext(file_name)[0] + ".png" + with io.BytesIO() as out: + Image.fromarray(id2rgb(panoptic_img)).save(out, format="PNG") + segments_info = [self._convert_category_id(x) for x in segments_info] + self._predictions.append( + { + "image_id": input["image_id"], + "file_name": file_name_png, + "png_string": out.getvalue(), + "segments_info": segments_info, + } + ) + + def evaluate(self): + comm.synchronize() + + self._predictions = comm.gather(self._predictions) + self._predictions = list(itertools.chain(*self._predictions)) + if not comm.is_main_process(): + return + + # PanopticApi requires local files + gt_json = PathManager.get_local_path(self._metadata.panoptic_json) + gt_folder = PathManager.get_local_path(self._metadata.panoptic_root) + + with tempfile.TemporaryDirectory(prefix="panoptic_eval") as pred_dir: + logger.info("Writing all panoptic predictions to {} ...".format(pred_dir)) + for p in self._predictions: + with open(os.path.join(pred_dir, p["file_name"]), "wb") as f: + f.write(p.pop("png_string")) + + with open(gt_json, "r") as f: + json_data = json.load(f) + json_data["annotations"] = self._predictions + + output_dir = self._output_dir or pred_dir + predictions_json = os.path.join(output_dir, "predictions.json") + with PathManager.open(predictions_json, "w") as f: + f.write(json.dumps(json_data)) + + from panopticapi.evaluation import pq_compute + + with contextlib.redirect_stdout(io.StringIO()): + pq_res = pq_compute( + gt_json, + PathManager.get_local_path(predictions_json), + gt_folder=gt_folder, + pred_folder=pred_dir, + ) + + res = {} + res["PQ"] = 100 * pq_res["All"]["pq"] + res["SQ"] = 100 * pq_res["All"]["sq"] + res["RQ"] = 100 * pq_res["All"]["rq"] + res["PQ_th"] = 100 * pq_res["Things"]["pq"] + res["SQ_th"] = 100 * pq_res["Things"]["sq"] + res["RQ_th"] = 100 * pq_res["Things"]["rq"] + res["PQ_st"] = 100 * pq_res["Stuff"]["pq"] + res["SQ_st"] = 100 * pq_res["Stuff"]["sq"] + res["RQ_st"] = 100 * pq_res["Stuff"]["rq"] + + results = OrderedDict({"panoptic_seg": res}) + _print_panoptic_results(pq_res) + + return results + + +def _print_panoptic_results(pq_res): + headers = ["", "PQ", "SQ", "RQ", "#categories"] + data = [] + for name in ["All", "Things", "Stuff"]: + row = [name] + [pq_res[name][k] * 100 for k in ["pq", "sq", "rq"]] + [pq_res[name]["n"]] + data.append(row) + table = tabulate(data, headers=headers, tablefmt="pipe", floatfmt=".3f", stralign="center", numalign="center") + logger.info("Panoptic Evaluation Results:\n" + table) + + +if __name__ == "__main__": + from detectron2.utils.logger import setup_logger + + logger = setup_logger() + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--gt-json") + parser.add_argument("--gt-dir") + parser.add_argument("--pred-json") + parser.add_argument("--pred-dir") + args = parser.parse_args() + + from panopticapi.evaluation import pq_compute + + with contextlib.redirect_stdout(io.StringIO()): + pq_res = pq_compute(args.gt_json, args.pred_json, gt_folder=args.gt_dir, pred_folder=args.pred_dir) + _print_panoptic_results(pq_res) diff --git a/detectron2/evaluation/pascal_voc_evaluation.py b/detectron2/evaluation/pascal_voc_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..0570a4c7230f512d5f9bd8a30877d5a020b7a5c2 --- /dev/null +++ b/detectron2/evaluation/pascal_voc_evaluation.py @@ -0,0 +1,297 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +import os +import tempfile +import xml.etree.ElementTree as ET +from collections import OrderedDict, defaultdict +from functools import lru_cache + +import numpy as np +import torch + +from detectron2.data import MetadataCatalog +from detectron2.utils import comm +from detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + + +class PascalVOCDetectionEvaluator(DatasetEvaluator): + """ + Evaluate Pascal VOC style AP for Pascal VOC dataset. + It contains a synchronization, therefore has to be called from all ranks. + + Note that the concept of AP can be implemented in different ways and may not + produce identical results. This class mimics the implementation of the official + Pascal VOC Matlab API, and should produce similar but not identical results to the + official API. + """ + + def __init__(self, dataset_name): + """ + Args: + dataset_name (str): name of the dataset, e.g., "voc_2007_test" + """ + self._dataset_name = dataset_name + meta = MetadataCatalog.get(dataset_name) + + # Too many tiny files, download all to local for speed. + annotation_dir_local = PathManager.get_local_path(os.path.join(meta.dirname, "Annotations/")) + self._anno_file_template = os.path.join(annotation_dir_local, "{}.xml") + self._image_set_path = os.path.join(meta.dirname, "ImageSets", "Main", meta.split + ".txt") + self._class_names = meta.thing_classes + assert meta.year in [2007, 2012], meta.year + self._is_2007 = meta.year == 2007 + self._cpu_device = torch.device("cpu") + self._logger = logging.getLogger(__name__) + + def reset(self): + self._predictions = defaultdict(list) # class name -> list of prediction strings + + def process(self, inputs, outputs): + for input, output in zip(inputs, outputs): + image_id = input["image_id"] + instances = output["instances"].to(self._cpu_device) + boxes = instances.pred_boxes.tensor.numpy() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + for box, score, cls in zip(boxes, scores, classes): + xmin, ymin, xmax, ymax = box + # The inverse of data loading logic in `datasets/pascal_voc.py` + xmin += 1 + ymin += 1 + self._predictions[cls].append(f"{image_id} {score:.3f} {xmin:.1f} {ymin:.1f} {xmax:.1f} {ymax:.1f}") + + def evaluate(self): + """ + Returns: + dict: has a key "segm", whose value is a dict of "AP", "AP50", and "AP75". + """ + all_predictions = comm.gather(self._predictions, dst=0) + if not comm.is_main_process(): + return + predictions = defaultdict(list) + for predictions_per_rank in all_predictions: + for clsid, lines in predictions_per_rank.items(): + predictions[clsid].extend(lines) + del all_predictions + + self._logger.info( + "Evaluating {} using {} metric. " + "Note that results do not use the official Matlab API.".format( + self._dataset_name, 2007 if self._is_2007 else 2012 + ) + ) + + with tempfile.TemporaryDirectory(prefix="pascal_voc_eval_") as dirname: + res_file_template = os.path.join(dirname, "{}.txt") + + aps = defaultdict(list) # iou -> ap per class + for cls_id, cls_name in enumerate(self._class_names): + lines = predictions.get(cls_id, [""]) + + with open(res_file_template.format(cls_name), "w") as f: + f.write("\n".join(lines)) + + for thresh in range(50, 100, 5): + rec, prec, ap = voc_eval( + res_file_template, + self._anno_file_template, + self._image_set_path, + cls_name, + ovthresh=thresh / 100.0, + use_07_metric=self._is_2007, + ) + aps[thresh].append(ap * 100) + + ret = OrderedDict() + mAP = {iou: np.mean(x) for iou, x in aps.items()} + ret["bbox"] = {"AP": np.mean(list(mAP.values())), "AP50": mAP[50], "AP75": mAP[75]} + return ret + + +############################################################################## +# +# Below code is modified from +# https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/datasets/voc_eval.py +# -------------------------------------------------------- +# Fast/er R-CNN +# Licensed under The MIT License [see LICENSE for details] +# Written by Bharath Hariharan +# -------------------------------------------------------- + +"""Python implementation of the PASCAL VOC devkit's AP evaluation code.""" + + +@lru_cache(maxsize=None) +def parse_rec(filename): + """Parse a PASCAL VOC xml file.""" + with PathManager.open(filename) as f: + tree = ET.parse(f) + objects = [] + for obj in tree.findall("object"): + obj_struct = {} + obj_struct["name"] = obj.find("name").text + obj_struct["pose"] = obj.find("pose").text + obj_struct["truncated"] = int(obj.find("truncated").text) + obj_struct["difficult"] = int(obj.find("difficult").text) + bbox = obj.find("bndbox") + obj_struct["bbox"] = [ + int(bbox.find("xmin").text), + int(bbox.find("ymin").text), + int(bbox.find("xmax").text), + int(bbox.find("ymax").text), + ] + objects.append(obj_struct) + + return objects + + +def voc_ap(rec, prec, use_07_metric=False): + """Compute VOC AP given precision and recall. If use_07_metric is true, uses + the VOC 07 11-point method (default:False). + """ + if use_07_metric: + # 11 point metric + ap = 0.0 + for t in np.arange(0.0, 1.1, 0.1): + if np.sum(rec >= t) == 0: + p = 0 + else: + p = np.max(prec[rec >= t]) + ap = ap + p / 11.0 + else: + # correct AP calculation + # first append sentinel values at the end + mrec = np.concatenate(([0.0], rec, [1.0])) + mpre = np.concatenate(([0.0], prec, [0.0])) + + # compute the precision envelope + for i in range(mpre.size - 1, 0, -1): + mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) + + # to calculate area under PR curve, look for points + # where X axis (recall) changes value + i = np.where(mrec[1:] != mrec[:-1])[0] + + # and sum (\Delta recall) * prec + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) + return ap + + +def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_metric=False): + """rec, prec, ap = voc_eval(detpath, + annopath, + imagesetfile, + classname, + [ovthresh], + [use_07_metric]) + + Top level function that does the PASCAL VOC evaluation. + + detpath: Path to detections + detpath.format(classname) should produce the detection results file. + annopath: Path to annotations + annopath.format(imagename) should be the xml annotations file. + imagesetfile: Text file containing the list of images, one image per line. + classname: Category name (duh) + [ovthresh]: Overlap threshold (default = 0.5) + [use_07_metric]: Whether to use VOC07's 11 point AP computation + (default False) + """ + # assumes detections are in detpath.format(classname) + # assumes annotations are in annopath.format(imagename) + # assumes imagesetfile is a text file with each line an image name + + # first load gt + # read list of images + with PathManager.open(imagesetfile, "r") as f: + lines = f.readlines() + imagenames = [x.strip() for x in lines] + + # load annots + recs = {} + for imagename in imagenames: + recs[imagename] = parse_rec(annopath.format(imagename)) + + # extract gt objects for this class + class_recs = {} + npos = 0 + for imagename in imagenames: + R = [obj for obj in recs[imagename] if obj["name"] == classname] + bbox = np.array([x["bbox"] for x in R]) + difficult = np.array([x["difficult"] for x in R]).astype(np.bool) + # difficult = np.array([False for x in R]).astype(np.bool) # treat all "difficult" as GT + det = [False] * len(R) + npos = npos + sum(~difficult) + class_recs[imagename] = {"bbox": bbox, "difficult": difficult, "det": det} + + # read dets + detfile = detpath.format(classname) + with open(detfile, "r") as f: + lines = f.readlines() + + splitlines = [x.strip().split(" ") for x in lines] + image_ids = [x[0] for x in splitlines] + confidence = np.array([float(x[1]) for x in splitlines]) + BB = np.array([[float(z) for z in x[2:]] for x in splitlines]).reshape(-1, 4) + + # sort by confidence + sorted_ind = np.argsort(-confidence) + BB = BB[sorted_ind, :] + image_ids = [image_ids[x] for x in sorted_ind] + + # go down dets and mark TPs and FPs + nd = len(image_ids) + tp = np.zeros(nd) + fp = np.zeros(nd) + for d in range(nd): + R = class_recs[image_ids[d]] + bb = BB[d, :].astype(float) + ovmax = -np.inf + BBGT = R["bbox"].astype(float) + + if BBGT.size > 0: + # compute overlaps + # intersection + ixmin = np.maximum(BBGT[:, 0], bb[0]) + iymin = np.maximum(BBGT[:, 1], bb[1]) + ixmax = np.minimum(BBGT[:, 2], bb[2]) + iymax = np.minimum(BBGT[:, 3], bb[3]) + iw = np.maximum(ixmax - ixmin + 1.0, 0.0) + ih = np.maximum(iymax - iymin + 1.0, 0.0) + inters = iw * ih + + # union + uni = ( + (bb[2] - bb[0] + 1.0) * (bb[3] - bb[1] + 1.0) + + (BBGT[:, 2] - BBGT[:, 0] + 1.0) * (BBGT[:, 3] - BBGT[:, 1] + 1.0) + - inters + ) + + overlaps = inters / uni + ovmax = np.max(overlaps) + jmax = np.argmax(overlaps) + + if ovmax > ovthresh: + if not R["difficult"][jmax]: + if not R["det"][jmax]: + tp[d] = 1.0 + R["det"][jmax] = 1 + else: + fp[d] = 1.0 + else: + fp[d] = 1.0 + + # compute precision recall + fp = np.cumsum(fp) + tp = np.cumsum(tp) + rec = tp / float(npos) + # avoid divide by zero in case the first detection matches a difficult + # ground truth + prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) + ap = voc_ap(rec, prec, use_07_metric) + + return rec, prec, ap diff --git a/detectron2/evaluation/rotated_coco_evaluation.py b/detectron2/evaluation/rotated_coco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..8c6b72481b8e09947b3390699a1ac0dee773bfd1 --- /dev/null +++ b/detectron2/evaluation/rotated_coco_evaluation.py @@ -0,0 +1,197 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import json +import os + +import numpy as np +import torch +from pycocotools.cocoeval import COCOeval, maskUtils + +from detectron2.structures import BoxMode, RotatedBoxes, pairwise_iou_rotated +from detectron2.utils.file_io import PathManager + +from .coco_evaluation import COCOEvaluator + + +class RotatedCOCOeval(COCOeval): + @staticmethod + def is_rotated(box_list): + if type(box_list) == np.ndarray: + return box_list.shape[1] == 5 + elif type(box_list) == list: + if box_list == []: # cannot decide the box_dim + return False + return np.all( + np.array([(len(obj) == 5) and ((type(obj) == list) or (type(obj) == np.ndarray)) for obj in box_list]) + ) + return False + + @staticmethod + def boxlist_to_tensor(boxlist, output_box_dim): + if type(boxlist) == np.ndarray: + box_tensor = torch.from_numpy(boxlist) + elif type(boxlist) == list: + if boxlist == []: + return torch.zeros((0, output_box_dim), dtype=torch.float32) + else: + box_tensor = torch.FloatTensor(boxlist) + else: + raise Exception("Unrecognized boxlist type") + + input_box_dim = box_tensor.shape[1] + if input_box_dim != output_box_dim: + if input_box_dim == 4 and output_box_dim == 5: + box_tensor = BoxMode.convert(box_tensor, BoxMode.XYWH_ABS, BoxMode.XYWHA_ABS) + else: + raise Exception( + "Unable to convert from {}-dim box to {}-dim box".format(input_box_dim, output_box_dim) + ) + return box_tensor + + def compute_iou_dt_gt(self, dt, gt, is_crowd): + if self.is_rotated(dt) or self.is_rotated(gt): + # TODO: take is_crowd into consideration + assert all(c == 0 for c in is_crowd) + dt = RotatedBoxes(self.boxlist_to_tensor(dt, output_box_dim=5)) + gt = RotatedBoxes(self.boxlist_to_tensor(gt, output_box_dim=5)) + return pairwise_iou_rotated(dt, gt) + else: + # This is the same as the classical COCO evaluation + return maskUtils.iou(dt, gt, is_crowd) + + def computeIoU(self, imgId, catId): + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return [] + inds = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in inds] + if len(dt) > p.maxDets[-1]: + dt = dt[0 : p.maxDets[-1]] + + assert p.iouType == "bbox", "unsupported iouType for iou computation" + + g = [g["bbox"] for g in gt] + d = [d["bbox"] for d in dt] + + # compute iou between each dt and gt region + iscrowd = [int(o["iscrowd"]) for o in gt] + + # Note: this function is copied from cocoeval.py in cocoapi + # and the major difference is here. + ious = self.compute_iou_dt_gt(d, g, iscrowd) + return ious + + +class RotatedCOCOEvaluator(COCOEvaluator): + """ + Evaluate object proposal/instance detection outputs using COCO-like metrics and APIs, + with rotated boxes support. + Note: this uses IOU only and does not consider angle differences. + """ + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + + prediction["instances"] = self.instances_to_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + self._predictions.append(prediction) + + def instances_to_json(self, instances, img_id): + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + if boxes.shape[1] == 4: + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + + results.append(result) + return results + + def _eval_predictions(self, predictions, img_ids=None): # img_ids: unused + """ + Evaluate predictions on the given tasks. + Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = {v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items()} + for result in coco_results: + result["category_id"] = reverse_id_mapping[result["category_id"]] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + + assert self._tasks is None or set(self._tasks) == { + "bbox" + }, "[RotatedCOCOEvaluator] Only bbox evaluation is supported" + coco_eval = ( + self._evaluate_predictions_on_coco(self._coco_api, coco_results) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + task = "bbox" + res = self._derive_coco_results(coco_eval, task, class_names=self._metadata.get("thing_classes")) + self._results[task] = res + + def _evaluate_predictions_on_coco(self, coco_gt, coco_results): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + coco_dt = coco_gt.loadRes(coco_results) + + # Only bbox is supported for now + coco_eval = RotatedCOCOeval(coco_gt, coco_dt, iouType="bbox") + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval diff --git a/detectron2/evaluation/sem_seg_evaluation.py b/detectron2/evaluation/sem_seg_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..0cb4d42496e711e7cf8fc72eabfb7d923cf0815a --- /dev/null +++ b/detectron2/evaluation/sem_seg_evaluation.py @@ -0,0 +1,194 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import json +import logging +import os +from collections import OrderedDict +from typing import Optional, Union + +import numpy as np +import pycocotools.mask as mask_util +import torch +from PIL import Image + +from detectron2.data import DatasetCatalog, MetadataCatalog +from detectron2.utils.comm import all_gather, is_main_process, synchronize +from detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + + +def load_image_into_numpy_array( + filename: str, + copy: bool = False, + dtype: Optional[Union[np.dtype, str]] = None, +) -> np.ndarray: + with PathManager.open(filename, "rb") as f: + array = np.array(Image.open(f), copy=copy, dtype=dtype) + return array + + +class SemSegEvaluator(DatasetEvaluator): + """ + Evaluate semantic segmentation metrics. + """ + + def __init__( + self, + dataset_name, + distributed=True, + output_dir=None, + *, + sem_seg_loading_fn=load_image_into_numpy_array, + num_classes=None, + ignore_label=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + distributed (bool): if True, will collect results from all ranks for evaluation. + Otherwise, will evaluate the results in the current process. + output_dir (str): an output directory to dump results. + sem_seg_loading_fn: function to read sem seg file and load into numpy array. + Default provided, but projects can customize. + num_classes, ignore_label: deprecated argument + """ + self._logger = logging.getLogger(__name__) + if num_classes is not None: + self._logger.warn("SemSegEvaluator(num_classes) is deprecated! It should be obtained from metadata.") + if ignore_label is not None: + self._logger.warn("SemSegEvaluator(ignore_label) is deprecated! It should be obtained from metadata.") + self._dataset_name = dataset_name + self._distributed = distributed + self._output_dir = output_dir + + self._cpu_device = torch.device("cpu") + + self.input_file_to_gt_file = { + dataset_record["file_name"]: dataset_record["sem_seg_file_name"] + for dataset_record in DatasetCatalog.get(dataset_name) + } + + meta = MetadataCatalog.get(dataset_name) + # Dict that maps contiguous training ids to COCO category ids + try: + c2d = meta.stuff_dataset_id_to_contiguous_id + self._contiguous_id_to_dataset_id = {v: k for k, v in c2d.items()} + except AttributeError: + self._contiguous_id_to_dataset_id = None + self._class_names = meta.stuff_classes + self.sem_seg_loading_fn = sem_seg_loading_fn + self._num_classes = len(meta.stuff_classes) + if num_classes is not None: + assert self._num_classes == num_classes, f"{self._num_classes} != {num_classes}" + self._ignore_label = ignore_label if ignore_label is not None else meta.ignore_label + + def reset(self): + self._conf_matrix = np.zeros((self._num_classes + 1, self._num_classes + 1), dtype=np.int64) + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a model. + It is a list of dicts. Each dict corresponds to an image and + contains keys like "height", "width", "file_name". + outputs: the outputs of a model. It is either list of semantic segmentation predictions + (Tensor [H, W]) or list of dicts with key "sem_seg" that contains semantic + segmentation prediction in the same format. + """ + for input, output in zip(inputs, outputs): + output = output["sem_seg"].argmax(dim=0).to(self._cpu_device) + pred = np.array(output, dtype=np.int) + gt_filename = self.input_file_to_gt_file[input["file_name"]] + gt = self.sem_seg_loading_fn(gt_filename, dtype=np.int) + + gt[gt == self._ignore_label] = self._num_classes + + self._conf_matrix += np.bincount( + (self._num_classes + 1) * pred.reshape(-1) + gt.reshape(-1), + minlength=self._conf_matrix.size, + ).reshape(self._conf_matrix.shape) + + self._predictions.extend(self.encode_json_sem_seg(pred, input["file_name"])) + + def evaluate(self): + """ + Evaluates standard semantic segmentation metrics (http://cocodataset.org/#stuff-eval): + + * Mean intersection-over-union averaged across classes (mIoU) + * Frequency Weighted IoU (fwIoU) + * Mean pixel accuracy averaged across classes (mACC) + * Pixel Accuracy (pACC) + """ + if self._distributed: + synchronize() + conf_matrix_list = all_gather(self._conf_matrix) + self._predictions = all_gather(self._predictions) + self._predictions = list(itertools.chain(*self._predictions)) + if not is_main_process(): + return + + self._conf_matrix = np.zeros_like(self._conf_matrix) + for conf_matrix in conf_matrix_list: + self._conf_matrix += conf_matrix + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "sem_seg_predictions.json") + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(self._predictions)) + + acc = np.full(self._num_classes, np.nan, dtype=np.float) + iou = np.full(self._num_classes, np.nan, dtype=np.float) + tp = self._conf_matrix.diagonal()[:-1].astype(np.float) + pos_gt = np.sum(self._conf_matrix[:-1, :-1], axis=0).astype(np.float) + class_weights = pos_gt / np.sum(pos_gt) + pos_pred = np.sum(self._conf_matrix[:-1, :-1], axis=1).astype(np.float) + acc_valid = pos_gt > 0 + acc[acc_valid] = tp[acc_valid] / pos_gt[acc_valid] + iou_valid = (pos_gt + pos_pred) > 0 + union = pos_gt + pos_pred - tp + iou[acc_valid] = tp[acc_valid] / union[acc_valid] + macc = np.sum(acc[acc_valid]) / np.sum(acc_valid) + miou = np.sum(iou[acc_valid]) / np.sum(iou_valid) + fiou = np.sum(iou[acc_valid] * class_weights[acc_valid]) + pacc = np.sum(tp) / np.sum(pos_gt) + + res = {} + res["mIoU"] = 100 * miou + res["fwIoU"] = 100 * fiou + for i, name in enumerate(self._class_names): + res["IoU-{}".format(name)] = 100 * iou[i] + res["mACC"] = 100 * macc + res["pACC"] = 100 * pacc + for i, name in enumerate(self._class_names): + res["ACC-{}".format(name)] = 100 * acc[i] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "sem_seg_evaluation.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(res, f) + results = OrderedDict({"sem_seg": res}) + self._logger.info(results) + return results + + def encode_json_sem_seg(self, sem_seg, input_file_name): + """ + Convert semantic segmentation to COCO stuff format with segments encoded as RLEs. + See http://cocodataset.org/#format-results + """ + json_list = [] + for label in np.unique(sem_seg): + if self._contiguous_id_to_dataset_id is not None: + assert ( + label in self._contiguous_id_to_dataset_id + ), "Label {} is not in the metadata info for {}".format(label, self._dataset_name) + dataset_id = self._contiguous_id_to_dataset_id[label] + else: + dataset_id = int(label) + mask = (sem_seg == label).astype(np.uint8) + mask_rle = mask_util.encode(np.array(mask[:, :, None], order="F"))[0] + mask_rle["counts"] = mask_rle["counts"].decode("utf-8") + json_list.append({"file_name": input_file_name, "category_id": dataset_id, "segmentation": mask_rle}) + return json_list diff --git a/detectron2/evaluation/testing.py b/detectron2/evaluation/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..d38542f5ea64ecd82636f9c5277652d0e815b756 --- /dev/null +++ b/detectron2/evaluation/testing.py @@ -0,0 +1,86 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import pprint +import sys +from collections.abc import Mapping + +import numpy as np + + +def print_csv_format(results): + """ + Print main metrics in a format similar to Detectron, + so that they are easy to copypaste into a spreadsheet. + + Args: + results (OrderedDict[dict]): task_name -> {metric -> score} + unordered dict can also be printed, but in arbitrary order + """ + assert isinstance(results, Mapping) or not len(results), results + logger = logging.getLogger(__name__) + for task, res in results.items(): + if isinstance(res, Mapping): + # Don't print "AP-category" metrics since they are usually not tracked. + important_res = [(k, v) for k, v in res.items() if "-" not in k] + logger.info("copypaste: Task: {}".format(task)) + logger.info("copypaste: " + ",".join([k[0] for k in important_res])) + logger.info("copypaste: " + ",".join(["{0:.4f}".format(k[1]) for k in important_res])) + else: + logger.info(f"copypaste: {task}={res}") + + +def verify_results(cfg, results): + """ + Args: + results (OrderedDict[dict]): task_name -> {metric -> score} + + Returns: + bool: whether the verification succeeds or not + """ + expected_results = cfg.TEST.EXPECTED_RESULTS + if not len(expected_results): + return True + + ok = True + for task, metric, expected, tolerance in expected_results: + actual = results[task].get(metric, None) + if actual is None: + ok = False + continue + if not np.isfinite(actual): + ok = False + continue + diff = abs(actual - expected) + if diff > tolerance: + ok = False + + logger = logging.getLogger(__name__) + if not ok: + logger.error("Result verification failed!") + logger.error("Expected Results: " + str(expected_results)) + logger.error("Actual Results: " + pprint.pformat(results)) + + sys.exit(1) + else: + logger.info("Results verification passed.") + return ok + + +def flatten_results_dict(results): + """ + Expand a hierarchical dict of scalars into a flat dict of scalars. + If results[k1][k2][k3] = v, the returned dict will have the entry + {"k1/k2/k3": v}. + + Args: + results (dict): + """ + r = {} + for k, v in results.items(): + if isinstance(v, Mapping): + v = flatten_results_dict(v) + for kk, vv in v.items(): + r[k + "/" + kk] = vv + else: + r[k] = v + return r diff --git a/detectron2/export/README.md b/detectron2/export/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9fcd33513fb81ef3aeb4d3c8d9732324dffa2646 --- /dev/null +++ b/detectron2/export/README.md @@ -0,0 +1,13 @@ + +This directory contains code to prepare a detectron2 model for deployment. +Currently it supports exporting a detectron2 model to Caffe2 format through ONNX. + +Please see [documentation](https://detectron2.readthedocs.io/tutorials/deployment.html) for its usage. + + +### Acknowledgements + +Thanks to Mobile Vision team at Facebook for developing the Caffe2 conversion tools. + +Thanks to Computing Platform Department - PAI team at Alibaba Group (@bddpqq, @chenbohua3) who +help export Detectron2 models to TorchScript. diff --git a/detectron2/export/__init__.py b/detectron2/export/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aee7c8d4e4330bdcce3420342f9f7248a9aefceb --- /dev/null +++ b/detectron2/export/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +try: + from caffe2.proto import caffe2_pb2 as _tmp + + # caffe2 is optional +except ImportError: + pass +else: + from .api import * + +from .flatten import TracingAdapter +from .torchscript import dump_torchscript_IR, scripting_with_instances + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/detectron2/export/api.py b/detectron2/export/api.py new file mode 100644 index 0000000000000000000000000000000000000000..427130fdabbdfec911e476e8a416b132ef6b756d --- /dev/null +++ b/detectron2/export/api.py @@ -0,0 +1,234 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import os + +import torch +from caffe2.proto import caffe2_pb2 +from torch import nn + +from detectron2.config import CfgNode +from detectron2.utils.file_io import PathManager + +from .caffe2_inference import ProtobufDetectionModel +from .caffe2_modeling import META_ARCH_CAFFE2_EXPORT_TYPE_MAP, convert_batched_inputs_to_c2_format +from .shared import get_pb_arg_vali, get_pb_arg_vals, save_graph + +__all__ = [ + "add_export_config", + "Caffe2Model", + "Caffe2Tracer", +] + + +def add_export_config(cfg): + return cfg + + +class Caffe2Tracer: + """ + Make a detectron2 model traceable with Caffe2 operators. + This class creates a traceable version of a detectron2 model which: + + 1. Rewrite parts of the model using ops in Caffe2. Note that some ops do + not have GPU implementation in Caffe2. + 2. Remove post-processing and only produce raw layer outputs + + After making a traceable model, the class provide methods to export such a + model to different deployment formats. + Exported graph produced by this class take two input tensors: + + 1. (1, C, H, W) float "data" which is an image (usually in [0, 255]). + (H, W) often has to be padded to multiple of 32 (depend on the model + architecture). + 2. 1x3 float "im_info", each row of which is (height, width, 1.0). + Height and width are true image shapes before padding. + + The class currently only supports models using builtin meta architectures. + Batch inference is not supported, and contributions are welcome. + """ + + def __init__(self, cfg: CfgNode, model: nn.Module, inputs): + """ + Args: + cfg (CfgNode): a detectron2 config used to construct caffe2-compatible model. + model (nn.Module): An original pytorch model. Must be among a few official models + in detectron2 that can be converted to become caffe2-compatible automatically. + Weights have to be already loaded to this model. + inputs: sample inputs that the given model takes for inference. + Will be used to trace the model. For most models, random inputs with + no detected objects will not work as they lead to wrong traces. + """ + assert isinstance(cfg, CfgNode), cfg + assert isinstance(model, torch.nn.Module), type(model) + + # TODO make it support custom models, by passing in c2 model directly + C2MetaArch = META_ARCH_CAFFE2_EXPORT_TYPE_MAP[cfg.MODEL.META_ARCHITECTURE] + self.traceable_model = C2MetaArch(cfg, copy.deepcopy(model)) + self.inputs = inputs + self.traceable_inputs = self.traceable_model.get_caffe2_inputs(inputs) + + def export_caffe2(self): + """ + Export the model to Caffe2's protobuf format. + The returned object can be saved with its :meth:`.save_protobuf()` method. + The result can be loaded and executed using Caffe2 runtime. + + Returns: + :class:`Caffe2Model` + """ + from .caffe2_export import export_caffe2_detection_model + + predict_net, init_net = export_caffe2_detection_model(self.traceable_model, self.traceable_inputs) + return Caffe2Model(predict_net, init_net) + + def export_onnx(self): + """ + Export the model to ONNX format. + Note that the exported model contains custom ops only available in caffe2, therefore it + cannot be directly executed by other runtime (such as onnxruntime or TensorRT). + Post-processing or transformation passes may be applied on the model to accommodate + different runtimes, but we currently do not provide support for them. + + Returns: + onnx.ModelProto: an onnx model. + """ + from .caffe2_export import export_onnx_model as export_onnx_model_impl + + return export_onnx_model_impl(self.traceable_model, (self.traceable_inputs,)) + + def export_torchscript(self): + """ + Export the model to a ``torch.jit.TracedModule`` by tracing. + The returned object can be saved to a file by ``.save()``. + + Returns: + torch.jit.TracedModule: a torch TracedModule + """ + logger = logging.getLogger(__name__) + logger.info("Tracing the model with torch.jit.trace ...") + with torch.no_grad(): + return torch.jit.trace(self.traceable_model, (self.traceable_inputs,)) + + +class Caffe2Model(nn.Module): + """ + A wrapper around the traced model in Caffe2's protobuf format. + The exported graph has different inputs/outputs from the original Pytorch + model, as explained in :class:`Caffe2Tracer`. This class wraps around the + exported graph to simulate the same interface as the original Pytorch model. + It also provides functions to save/load models in Caffe2's format.' + + Examples: + :: + c2_model = Caffe2Tracer(cfg, torch_model, inputs).export_caffe2() + inputs = [{"image": img_tensor_CHW}] + outputs = c2_model(inputs) + orig_outputs = torch_model(inputs) + """ + + def __init__(self, predict_net, init_net): + super().__init__() + self.eval() # always in eval mode + self._predict_net = predict_net + self._init_net = init_net + self._predictor = None + + __init__.__HIDE_SPHINX_DOC__ = True + + @property + def predict_net(self): + """ + caffe2.core.Net: the underlying caffe2 predict net + """ + return self._predict_net + + @property + def init_net(self): + """ + caffe2.core.Net: the underlying caffe2 init net + """ + return self._init_net + + def save_protobuf(self, output_dir): + """ + Save the model as caffe2's protobuf format. + It saves the following files: + + * "model.pb": definition of the graph. Can be visualized with + tools like `netron `_. + * "model_init.pb": model parameters + * "model.pbtxt": human-readable definition of the graph. Not + needed for deployment. + + Args: + output_dir (str): the output directory to save protobuf files. + """ + logger = logging.getLogger(__name__) + logger.info("Saving model to {} ...".format(output_dir)) + if not PathManager.exists(output_dir): + PathManager.mkdirs(output_dir) + + with PathManager.open(os.path.join(output_dir, "model.pb"), "wb") as f: + f.write(self._predict_net.SerializeToString()) + with PathManager.open(os.path.join(output_dir, "model.pbtxt"), "w") as f: + f.write(str(self._predict_net)) + with PathManager.open(os.path.join(output_dir, "model_init.pb"), "wb") as f: + f.write(self._init_net.SerializeToString()) + + def save_graph(self, output_file, inputs=None): + """ + Save the graph as SVG format. + + Args: + output_file (str): a SVG file + inputs: optional inputs given to the model. + If given, the inputs will be used to run the graph to record + shape of every tensor. The shape information will be + saved together with the graph. + """ + from .caffe2_export import run_and_save_graph + + if inputs is None: + save_graph(self._predict_net, output_file, op_only=False) + else: + size_divisibility = get_pb_arg_vali(self._predict_net, "size_divisibility", 0) + device = get_pb_arg_vals(self._predict_net, "device", b"cpu").decode("ascii") + inputs = convert_batched_inputs_to_c2_format(inputs, size_divisibility, device) + inputs = [x.cpu().numpy() for x in inputs] + run_and_save_graph(self._predict_net, self._init_net, inputs, output_file) + + @staticmethod + def load_protobuf(dir): + """ + Args: + dir (str): a directory used to save Caffe2Model with + :meth:`save_protobuf`. + The files "model.pb" and "model_init.pb" are needed. + + Returns: + Caffe2Model: the caffe2 model loaded from this directory. + """ + predict_net = caffe2_pb2.NetDef() + with PathManager.open(os.path.join(dir, "model.pb"), "rb") as f: + predict_net.ParseFromString(f.read()) + + init_net = caffe2_pb2.NetDef() + with PathManager.open(os.path.join(dir, "model_init.pb"), "rb") as f: + init_net.ParseFromString(f.read()) + + return Caffe2Model(predict_net, init_net) + + def __call__(self, inputs): + """ + An interface that wraps around a Caffe2 model and mimics detectron2's models' + input/output format. See details about the format at :doc:`/tutorials/models`. + This is used to compare the outputs of caffe2 model with its original torch model. + + Due to the extra conversion between Pytorch/Caffe2, this method is not meant for + benchmark. Because of the conversion, this method also has dependency + on detectron2 in order to convert to detectron2's output format. + """ + if self._predictor is None: + self._predictor = ProtobufDetectionModel(self._predict_net, self._init_net) + return self._predictor(inputs) diff --git a/detectron2/export/c10.py b/detectron2/export/c10.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9d1bdc3022c78a59442eb4e786d382a361478e --- /dev/null +++ b/detectron2/export/c10.py @@ -0,0 +1,534 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import math +from typing import Dict + +import torch +import torch.nn.functional as F + +from detectron2.layers import ShapeSpec, cat +from detectron2.layers.roi_align_rotated import ROIAlignRotated +from detectron2.modeling import poolers +from detectron2.modeling.proposal_generator import rpn +from detectron2.modeling.roi_heads.mask_head import mask_rcnn_inference +from detectron2.structures import Boxes, ImageList, Instances, Keypoints + +from .shared import alias, to_device + +""" +This file contains caffe2-compatible implementation of several detectron2 components. +""" + + +class Caffe2Boxes(Boxes): + """ + Representing a list of detectron2.structures.Boxes from minibatch, each box + is represented by a 5d vector (batch index + 4 coordinates), or a 6d vector + (batch index + 5 coordinates) for RotatedBoxes. + """ + + def __init__(self, tensor): + assert isinstance(tensor, torch.Tensor) + assert tensor.dim() == 2 and tensor.size(-1) in [4, 5, 6], tensor.size() + # TODO: make tensor immutable when dim is Nx5 for Boxes, + # and Nx6 for RotatedBoxes? + self.tensor = tensor + + +# TODO clean up this class, maybe just extend Instances +class InstancesList(object): + """ + Tensor representation of a list of Instances object for a batch of images. + + When dealing with a batch of images with Caffe2 ops, a list of bboxes + (instances) are usually represented by single Tensor with size + (sigma(Ni), 5) or (sigma(Ni), 4) plus a batch split Tensor. This class is + for providing common functions to convert between these two representations. + """ + + def __init__(self, im_info, indices, extra_fields=None): + # [N, 3] -> (H, W, Scale) + self.im_info = im_info + # [N,] -> indice of batch to which the instance belongs + self.indices = indices + # [N, ...] + self.batch_extra_fields = extra_fields or {} + + self.image_size = self.im_info + + def get_fields(self): + """like `get_fields` in the Instances object, + but return each field in tensor representations""" + ret = {} + for k, v in self.batch_extra_fields.items(): + # if isinstance(v, torch.Tensor): + # tensor_rep = v + # elif isinstance(v, (Boxes, Keypoints)): + # tensor_rep = v.tensor + # else: + # raise ValueError("Can't find tensor representation for: {}".format()) + ret[k] = v + return ret + + def has(self, name): + return name in self.batch_extra_fields + + def set(self, name, value): + data_len = len(value) + if len(self.batch_extra_fields): + assert len(self) == data_len, "Adding a field of length {} to a Instances of length {}".format( + data_len, len(self) + ) + self.batch_extra_fields[name] = value + + def __setattr__(self, name, val): + if name in ["im_info", "indices", "batch_extra_fields", "image_size"]: + super().__setattr__(name, val) + else: + self.set(name, val) + + def __getattr__(self, name): + if name not in self.batch_extra_fields: + raise AttributeError("Cannot find field '{}' in the given Instances!".format(name)) + return self.batch_extra_fields[name] + + def __len__(self): + return len(self.indices) + + def flatten(self): + ret = [] + for _, v in self.batch_extra_fields.items(): + if isinstance(v, (Boxes, Keypoints)): + ret.append(v.tensor) + else: + ret.append(v) + return ret + + @staticmethod + def to_d2_instances_list(instances_list): + """ + Convert InstancesList to List[Instances]. The input `instances_list` can + also be a List[Instances], in this case this method is a non-op. + """ + if not isinstance(instances_list, InstancesList): + assert all(isinstance(x, Instances) for x in instances_list) + return instances_list + + ret = [] + for i, info in enumerate(instances_list.im_info): + instances = Instances(torch.Size([int(info[0].item()), int(info[1].item())])) + + ids = instances_list.indices == i + for k, v in instances_list.batch_extra_fields.items(): + if isinstance(v, torch.Tensor): + instances.set(k, v[ids]) + continue + elif isinstance(v, Boxes): + instances.set(k, v[ids, -4:]) + continue + + target_type, tensor_source = v + assert isinstance(tensor_source, torch.Tensor) + assert tensor_source.shape[0] == instances_list.indices.shape[0] + tensor_source = tensor_source[ids] + + if issubclass(target_type, Boxes): + instances.set(k, Boxes(tensor_source[:, -4:])) + elif issubclass(target_type, Keypoints): + instances.set(k, Keypoints(tensor_source)) + elif issubclass(target_type, torch.Tensor): + instances.set(k, tensor_source) + else: + raise ValueError("Can't handle targe type: {}".format(target_type)) + + ret.append(instances) + return ret + + +class Caffe2Compatible(object): + """ + A model can inherit this class to indicate that it can be traced and deployed with caffe2. + """ + + def _get_tensor_mode(self): + return self._tensor_mode + + def _set_tensor_mode(self, v): + self._tensor_mode = v + + tensor_mode = property(_get_tensor_mode, _set_tensor_mode) + """ + If true, the model expects C2-style tensor only inputs/outputs format. + """ + + +class Caffe2RPN(Caffe2Compatible, rpn.RPN): + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = super(Caffe2Compatible, cls).from_config(cfg, input_shape) + assert tuple(cfg.MODEL.RPN.BBOX_REG_WEIGHTS) == (1.0, 1.0, 1.0, 1.0) or tuple( + cfg.MODEL.RPN.BBOX_REG_WEIGHTS + ) == (1.0, 1.0, 1.0, 1.0, 1.0) + return ret + + def _generate_proposals(self, images, objectness_logits_pred, anchor_deltas_pred, gt_instances=None): + assert isinstance(images, ImageList) + if self.tensor_mode: + im_info = images.image_sizes + else: + im_info = torch.tensor([[im_sz[0], im_sz[1], 1.0] for im_sz in images.image_sizes]).to( + images.tensor.device + ) + assert isinstance(im_info, torch.Tensor) + + rpn_rois_list = [] + rpn_roi_probs_list = [] + for scores, bbox_deltas, cell_anchors_tensor, feat_stride in zip( + objectness_logits_pred, + anchor_deltas_pred, + iter(self.anchor_generator.cell_anchors), + self.anchor_generator.strides, + ): + scores = scores.detach() + bbox_deltas = bbox_deltas.detach() + + rpn_rois, rpn_roi_probs = torch.ops._caffe2.GenerateProposals( + scores, + bbox_deltas, + im_info, + cell_anchors_tensor, + spatial_scale=1.0 / feat_stride, + pre_nms_topN=self.pre_nms_topk[self.training], + post_nms_topN=self.post_nms_topk[self.training], + nms_thresh=self.nms_thresh, + min_size=self.min_box_size, + # correct_transform_coords=True, # deprecated argument + angle_bound_on=True, # Default + angle_bound_lo=-180, + angle_bound_hi=180, + clip_angle_thresh=1.0, # Default + legacy_plus_one=False, + ) + rpn_rois_list.append(rpn_rois) + rpn_roi_probs_list.append(rpn_roi_probs) + + # For FPN in D2, in RPN all proposals from different levels are concated + # together, ranked and picked by top post_nms_topk. Then in ROIPooler + # it calculates level_assignments and calls the RoIAlign from + # the corresponding level. + + if len(objectness_logits_pred) == 1: + rpn_rois = rpn_rois_list[0] + rpn_roi_probs = rpn_roi_probs_list[0] + else: + assert len(rpn_rois_list) == len(rpn_roi_probs_list) + rpn_post_nms_topN = self.post_nms_topk[self.training] + + device = rpn_rois_list[0].device + input_list = [to_device(x, "cpu") for x in (rpn_rois_list + rpn_roi_probs_list)] + + # TODO remove this after confirming rpn_max_level/rpn_min_level + # is not needed in CollectRpnProposals. + feature_strides = list(self.anchor_generator.strides) + rpn_min_level = int(math.log2(feature_strides[0])) + rpn_max_level = int(math.log2(feature_strides[-1])) + assert (rpn_max_level - rpn_min_level + 1) == len( + rpn_rois_list + ), "CollectRpnProposals requires continuous levels" + + rpn_rois = torch.ops._caffe2.CollectRpnProposals( + input_list, + # NOTE: in current implementation, rpn_max_level and rpn_min_level + # are not needed, only the subtraction of two matters and it + # can be infer from the number of inputs. Keep them now for + # consistency. + rpn_max_level=2 + len(rpn_rois_list) - 1, + rpn_min_level=2, + rpn_post_nms_topN=rpn_post_nms_topN, + ) + rpn_rois = to_device(rpn_rois, device) + rpn_roi_probs = [] + + proposals = self.c2_postprocess(im_info, rpn_rois, rpn_roi_probs, self.tensor_mode) + return proposals, {} + + def forward(self, images, features, gt_instances=None): + assert not self.training + features = [features[f] for f in self.in_features] + objectness_logits_pred, anchor_deltas_pred = self.rpn_head(features) + return self._generate_proposals( + images, + objectness_logits_pred, + anchor_deltas_pred, + gt_instances, + ) + + @staticmethod + def c2_postprocess(im_info, rpn_rois, rpn_roi_probs, tensor_mode): + proposals = InstancesList( + im_info=im_info, + indices=rpn_rois[:, 0], + extra_fields={ + "proposal_boxes": Caffe2Boxes(rpn_rois), + "objectness_logits": (torch.Tensor, rpn_roi_probs), + }, + ) + if not tensor_mode: + proposals = InstancesList.to_d2_instances_list(proposals) + else: + proposals = [proposals] + return proposals + + +class Caffe2ROIPooler(Caffe2Compatible, poolers.ROIPooler): + @staticmethod + def c2_preprocess(box_lists): + assert all(isinstance(x, Boxes) for x in box_lists) + if all(isinstance(x, Caffe2Boxes) for x in box_lists): + # input is pure-tensor based + assert len(box_lists) == 1 + pooler_fmt_boxes = box_lists[0].tensor + else: + pooler_fmt_boxes = poolers.convert_boxes_to_pooler_format(box_lists) + return pooler_fmt_boxes + + def forward(self, x, box_lists): + assert not self.training + + pooler_fmt_boxes = self.c2_preprocess(box_lists) + num_level_assignments = len(self.level_poolers) + + if num_level_assignments == 1: + if isinstance(self.level_poolers[0], ROIAlignRotated): + c2_roi_align = torch.ops._caffe2.RoIAlignRotated + aligned = True + else: + c2_roi_align = torch.ops._caffe2.RoIAlign + aligned = self.level_poolers[0].aligned + + x0 = x[0] + if x0.is_quantized: + x0 = x0.dequantize() + + out = c2_roi_align( + x0, + pooler_fmt_boxes, + order="NCHW", + spatial_scale=float(self.level_poolers[0].spatial_scale), + pooled_h=int(self.output_size[0]), + pooled_w=int(self.output_size[1]), + sampling_ratio=int(self.level_poolers[0].sampling_ratio), + aligned=aligned, + ) + return out + + device = pooler_fmt_boxes.device + assert self.max_level - self.min_level + 1 == 4, "Currently DistributeFpnProposals only support 4 levels" + fpn_outputs = torch.ops._caffe2.DistributeFpnProposals( + to_device(pooler_fmt_boxes, "cpu"), + roi_canonical_scale=self.canonical_box_size, + roi_canonical_level=self.canonical_level, + roi_max_level=self.max_level, + roi_min_level=self.min_level, + legacy_plus_one=False, + ) + fpn_outputs = [to_device(x, device) for x in fpn_outputs] + + rois_fpn_list = fpn_outputs[:-1] + rois_idx_restore_int32 = fpn_outputs[-1] + + roi_feat_fpn_list = [] + for roi_fpn, x_level, pooler in zip(rois_fpn_list, x, self.level_poolers): + if isinstance(pooler, ROIAlignRotated): + c2_roi_align = torch.ops._caffe2.RoIAlignRotated + aligned = True + else: + c2_roi_align = torch.ops._caffe2.RoIAlign + aligned = bool(pooler.aligned) + + if x_level.is_quantized: + x_level = x_level.dequantize() + + roi_feat_fpn = c2_roi_align( + x_level, + roi_fpn, + order="NCHW", + spatial_scale=float(pooler.spatial_scale), + pooled_h=int(self.output_size[0]), + pooled_w=int(self.output_size[1]), + sampling_ratio=int(pooler.sampling_ratio), + aligned=aligned, + ) + roi_feat_fpn_list.append(roi_feat_fpn) + + roi_feat_shuffled = cat(roi_feat_fpn_list, dim=0) + assert roi_feat_shuffled.numel() > 0 and rois_idx_restore_int32.numel() > 0, ( + "Caffe2 export requires tracing with a model checkpoint + input that can produce valid" + " detections. But no detections were obtained with the given checkpoint and input!" + ) + roi_feat = torch.ops._caffe2.BatchPermutation(roi_feat_shuffled, rois_idx_restore_int32) + return roi_feat + + +class Caffe2FastRCNNOutputsInference: + def __init__(self, tensor_mode): + self.tensor_mode = tensor_mode # whether the output is caffe2 tensor mode + + def __call__(self, box_predictor, predictions, proposals): + """equivalent to FastRCNNOutputLayers.inference""" + num_classes = box_predictor.num_classes + score_thresh = box_predictor.test_score_thresh + nms_thresh = box_predictor.test_nms_thresh + topk_per_image = box_predictor.test_topk_per_image + is_rotated = len(box_predictor.box2box_transform.weights) == 5 + + if is_rotated: + box_dim = 5 + assert box_predictor.box2box_transform.weights[4] == 1, ( + "The weights for Rotated BBoxTransform in C2 have only 4 dimensions," + + " thus enforcing the angle weight to be 1 for now" + ) + box2box_transform_weights = box_predictor.box2box_transform.weights[:4] + else: + box_dim = 4 + box2box_transform_weights = box_predictor.box2box_transform.weights + + class_logits, box_regression = predictions + if num_classes + 1 == class_logits.shape[1]: + class_prob = F.softmax(class_logits, -1) + else: + assert num_classes == class_logits.shape[1] + class_prob = F.sigmoid(class_logits) + # BoxWithNMSLimit will infer num_classes from the shape of the class_prob + # So append a zero column as placeholder for the background class + class_prob = torch.cat((class_prob, torch.zeros(class_prob.shape[0], 1)), dim=1) + + assert box_regression.shape[1] % box_dim == 0 + cls_agnostic_bbox_reg = box_regression.shape[1] // box_dim == 1 + + input_tensor_mode = proposals[0].proposal_boxes.tensor.shape[1] == box_dim + 1 + + rois = type(proposals[0].proposal_boxes).cat([p.proposal_boxes for p in proposals]) + device, dtype = rois.tensor.device, rois.tensor.dtype + if input_tensor_mode: + im_info = proposals[0].image_size + rois = rois.tensor + else: + im_info = torch.tensor([[sz[0], sz[1], 1.0] for sz in [x.image_size for x in proposals]]) + batch_ids = cat( + [torch.full((b, 1), i, dtype=dtype, device=device) for i, b in enumerate(len(p) for p in proposals)], + dim=0, + ) + rois = torch.cat([batch_ids, rois.tensor], dim=1) + + roi_pred_bbox, roi_batch_splits = torch.ops._caffe2.BBoxTransform( + to_device(rois, "cpu"), + to_device(box_regression, "cpu"), + to_device(im_info, "cpu"), + weights=box2box_transform_weights, + apply_scale=True, + rotated=is_rotated, + angle_bound_on=True, + angle_bound_lo=-180, + angle_bound_hi=180, + clip_angle_thresh=1.0, + legacy_plus_one=False, + ) + roi_pred_bbox = to_device(roi_pred_bbox, device) + roi_batch_splits = to_device(roi_batch_splits, device) + + nms_outputs = torch.ops._caffe2.BoxWithNMSLimit( + to_device(class_prob, "cpu"), + to_device(roi_pred_bbox, "cpu"), + to_device(roi_batch_splits, "cpu"), + score_thresh=float(score_thresh), + nms=float(nms_thresh), + detections_per_im=int(topk_per_image), + soft_nms_enabled=False, + soft_nms_method="linear", + soft_nms_sigma=0.5, + soft_nms_min_score_thres=0.001, + rotated=is_rotated, + cls_agnostic_bbox_reg=cls_agnostic_bbox_reg, + input_boxes_include_bg_cls=False, + output_classes_include_bg_cls=False, + legacy_plus_one=False, + ) + roi_score_nms = to_device(nms_outputs[0], device) + roi_bbox_nms = to_device(nms_outputs[1], device) + roi_class_nms = to_device(nms_outputs[2], device) + roi_batch_splits_nms = to_device(nms_outputs[3], device) + roi_keeps_nms = to_device(nms_outputs[4], device) + roi_keeps_size_nms = to_device(nms_outputs[5], device) + if not self.tensor_mode: + roi_class_nms = roi_class_nms.to(torch.int64) + + roi_batch_ids = cat( + [ + torch.full((b, 1), i, dtype=dtype, device=device) + for i, b in enumerate(int(x.item()) for x in roi_batch_splits_nms) + ], + dim=0, + ) + + roi_class_nms = alias(roi_class_nms, "class_nms") + roi_score_nms = alias(roi_score_nms, "score_nms") + roi_bbox_nms = alias(roi_bbox_nms, "bbox_nms") + roi_batch_splits_nms = alias(roi_batch_splits_nms, "batch_splits_nms") + roi_keeps_nms = alias(roi_keeps_nms, "keeps_nms") + roi_keeps_size_nms = alias(roi_keeps_size_nms, "keeps_size_nms") + + results = InstancesList( + im_info=im_info, + indices=roi_batch_ids[:, 0], + extra_fields={ + "pred_boxes": Caffe2Boxes(roi_bbox_nms), + "scores": roi_score_nms, + "pred_classes": roi_class_nms, + }, + ) + + if not self.tensor_mode: + results = InstancesList.to_d2_instances_list(results) + batch_splits = roi_batch_splits_nms.int().tolist() + kept_indices = list(roi_keeps_nms.to(torch.int64).split(batch_splits)) + else: + results = [results] + kept_indices = [roi_keeps_nms] + + return results, kept_indices + + +class Caffe2MaskRCNNInference: + def __call__(self, pred_mask_logits, pred_instances): + """equivalent to mask_head.mask_rcnn_inference""" + if all(isinstance(x, InstancesList) for x in pred_instances): + assert len(pred_instances) == 1 + mask_probs_pred = pred_mask_logits.sigmoid() + mask_probs_pred = alias(mask_probs_pred, "mask_fcn_probs") + pred_instances[0].pred_masks = mask_probs_pred + else: + mask_rcnn_inference(pred_mask_logits, pred_instances) + + +class Caffe2KeypointRCNNInference: + def __init__(self, use_heatmap_max_keypoint): + self.use_heatmap_max_keypoint = use_heatmap_max_keypoint + + def __call__(self, pred_keypoint_logits, pred_instances): + # just return the keypoint heatmap for now, + # there will be option to call HeatmapMaxKeypointOp + output = alias(pred_keypoint_logits, "kps_score") + if all(isinstance(x, InstancesList) for x in pred_instances): + assert len(pred_instances) == 1 + if self.use_heatmap_max_keypoint: + device = output.device + output = torch.ops._caffe2.HeatmapMaxKeypoint( + to_device(output, "cpu"), + pred_instances[0].pred_boxes.tensor, + should_output_softmax=True, # worth make it configerable? + ) + output = to_device(output, device) + output = alias(output, "keypoints_out") + pred_instances[0].pred_keypoints = output + return pred_keypoint_logits diff --git a/detectron2/export/caffe2_export.py b/detectron2/export/caffe2_export.py new file mode 100644 index 0000000000000000000000000000000000000000..44eef0a732523f600c8018772d6d24cfa6a0b934 --- /dev/null +++ b/detectron2/export/caffe2_export.py @@ -0,0 +1,204 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import copy +import io +import logging +from typing import List + +import numpy as np +import onnx +import torch +from caffe2.proto import caffe2_pb2 +from caffe2.python import core +from caffe2.python.onnx.backend import Caffe2Backend +from tabulate import tabulate +from termcolor import colored +from torch.onnx import OperatorExportTypes + +from .shared import ( + ScopedWS, + construct_init_net_from_params, + fuse_alias_placeholder, + fuse_copy_between_cpu_and_gpu, + get_params_from_init_net, + group_norm_replace_aten_with_caffe2, + infer_device_type, + remove_dead_end_ops, + remove_reshape_for_fc, + save_graph, +) + +logger = logging.getLogger(__name__) + + +def export_onnx_model(model, inputs): + """ + Trace and export a model to onnx format. + + Args: + model (nn.Module): + inputs (tuple[args]): the model will be called by `model(*inputs)` + + Returns: + an onnx model + """ + assert isinstance(model, torch.nn.Module) + + # make sure all modules are in eval mode, onnx may change the training state + # of the module if the states are not consistent + def _check_eval(module): + assert not module.training + + model.apply(_check_eval) + + # Export the model to ONNX + with torch.no_grad(): + with io.BytesIO() as f: + torch.onnx.export( + model, + inputs, + f, + operator_export_type=OperatorExportTypes.ONNX_ATEN_FALLBACK, + # verbose=True, # NOTE: uncomment this for debugging + # export_params=True, + ) + onnx_model = onnx.load_from_string(f.getvalue()) + + # Apply ONNX's Optimization + all_passes = onnx.optimizer.get_available_passes() + passes = ["fuse_bn_into_conv"] + assert all(p in all_passes for p in passes) + onnx_model = onnx.optimizer.optimize(onnx_model, passes) + return onnx_model + + +def _op_stats(net_def): + type_count = {} + for t in [op.type for op in net_def.op]: + type_count[t] = type_count.get(t, 0) + 1 + type_count_list = sorted(type_count.items(), key=lambda kv: kv[0]) # alphabet + type_count_list = sorted(type_count_list, key=lambda kv: -kv[1]) # count + return "\n".join("{:>4}x {}".format(count, name) for name, count in type_count_list) + + +def _assign_device_option( + predict_net: caffe2_pb2.NetDef, init_net: caffe2_pb2.NetDef, tensor_inputs: List[torch.Tensor] +): + """ + ONNX exported network doesn't have concept of device, assign necessary + device option for each op in order to make it runable on GPU runtime. + """ + + def _get_device_type(torch_tensor): + assert torch_tensor.device.type in ["cpu", "cuda"] + assert torch_tensor.device.index == 0 + return torch_tensor.device.type + + def _assign_op_device_option(net_proto, net_ssa, blob_device_types): + for op, ssa_i in zip(net_proto.op, net_ssa): + if op.type in ["CopyCPUToGPU", "CopyGPUToCPU"]: + op.device_option.CopyFrom(core.DeviceOption(caffe2_pb2.CUDA, 0)) + else: + devices = [blob_device_types[b] for b in ssa_i[0] + ssa_i[1]] + assert all(d == devices[0] for d in devices) + if devices[0] == "cuda": + op.device_option.CopyFrom(core.DeviceOption(caffe2_pb2.CUDA, 0)) + + # update ops in predict_net + predict_net_input_device_types = { + (name, 0): _get_device_type(tensor) for name, tensor in zip(predict_net.external_input, tensor_inputs) + } + predict_net_device_types = infer_device_type( + predict_net, known_status=predict_net_input_device_types, device_name_style="pytorch" + ) + predict_net_ssa, _ = core.get_ssa(predict_net) + _assign_op_device_option(predict_net, predict_net_ssa, predict_net_device_types) + + # update ops in init_net + init_net_ssa, versions = core.get_ssa(init_net) + init_net_output_device_types = { + (name, versions[name]): predict_net_device_types[(name, 0)] for name in init_net.external_output + } + init_net_device_types = infer_device_type( + init_net, known_status=init_net_output_device_types, device_name_style="pytorch" + ) + _assign_op_device_option(init_net, init_net_ssa, init_net_device_types) + + +def export_caffe2_detection_model(model: torch.nn.Module, tensor_inputs: List[torch.Tensor]): + """ + Export a caffe2-compatible Detectron2 model to caffe2 format via ONNX. + + Arg: + model: a caffe2-compatible version of detectron2 model, defined in caffe2_modeling.py + tensor_inputs: a list of tensors that caffe2 model takes as input. + """ + model = copy.deepcopy(model) + assert isinstance(model, torch.nn.Module) + assert hasattr(model, "encode_additional_info") + + # Export via ONNX + logger.info( + "Exporting a {} model via ONNX ...".format(type(model).__name__) + + " Some warnings from ONNX are expected and are usually not to worry about." + ) + onnx_model = export_onnx_model(model, (tensor_inputs,)) + # Convert ONNX model to Caffe2 protobuf + init_net, predict_net = Caffe2Backend.onnx_graph_to_caffe2_net(onnx_model) + ops_table = [[op.type, op.input, op.output] for op in predict_net.op] + table = tabulate(ops_table, headers=["type", "input", "output"], tablefmt="pipe") + logger.info("ONNX export Done. Exported predict_net (before optimizations):\n" + colored(table, "cyan")) + + # Apply protobuf optimization + fuse_alias_placeholder(predict_net, init_net) + if any(t.device.type != "cpu" for t in tensor_inputs): + fuse_copy_between_cpu_and_gpu(predict_net) + remove_dead_end_ops(init_net) + _assign_device_option(predict_net, init_net, tensor_inputs) + params, device_options = get_params_from_init_net(init_net) + predict_net, params = remove_reshape_for_fc(predict_net, params) + init_net = construct_init_net_from_params(params, device_options) + group_norm_replace_aten_with_caffe2(predict_net) + + # Record necessary information for running the pb model in Detectron2 system. + model.encode_additional_info(predict_net, init_net) + + logger.info("Operators used in predict_net: \n{}".format(_op_stats(predict_net))) + logger.info("Operators used in init_net: \n{}".format(_op_stats(init_net))) + + return predict_net, init_net + + +def run_and_save_graph(predict_net, init_net, tensor_inputs, graph_save_path): + """ + Run the caffe2 model on given inputs, recording the shape and draw the graph. + + predict_net/init_net: caffe2 model. + tensor_inputs: a list of tensors that caffe2 model takes as input. + graph_save_path: path for saving graph of exported model. + """ + + logger.info("Saving graph of ONNX exported model to {} ...".format(graph_save_path)) + save_graph(predict_net, graph_save_path, op_only=False) + + # Run the exported Caffe2 net + logger.info("Running ONNX exported model ...") + with ScopedWS("__ws_tmp__", True) as ws: + ws.RunNetOnce(init_net) + initialized_blobs = set(ws.Blobs()) + uninitialized = [inp for inp in predict_net.external_input if inp not in initialized_blobs] + for name, blob in zip(uninitialized, tensor_inputs): + ws.FeedBlob(name, blob) + + try: + ws.RunNetOnce(predict_net) + except RuntimeError as e: + logger.warning("Encountered RuntimeError: \n{}".format(str(e))) + + ws_blobs = {b: ws.FetchBlob(b) for b in ws.Blobs()} + blob_sizes = {b: ws_blobs[b].shape for b in ws_blobs if isinstance(ws_blobs[b], np.ndarray)} + + logger.info("Saving graph with blob shapes to {} ...".format(graph_save_path)) + save_graph(predict_net, graph_save_path, op_only=False, blob_sizes=blob_sizes) + + return ws_blobs diff --git a/detectron2/export/caffe2_inference.py b/detectron2/export/caffe2_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..cebd8b56fea9800eb17d8a62729c4d9769516136 --- /dev/null +++ b/detectron2/export/caffe2_inference.py @@ -0,0 +1,151 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from itertools import count + +import numpy as np +import torch +from caffe2.proto import caffe2_pb2 +from caffe2.python import core + +from .caffe2_modeling import META_ARCH_CAFFE2_EXPORT_TYPE_MAP, convert_batched_inputs_to_c2_format +from .shared import ScopedWS, get_pb_arg_vali, get_pb_arg_vals, infer_device_type + +logger = logging.getLogger(__name__) + + +# ===== ref: mobile-vision predictor's 'Caffe2Wrapper' class ====== +class ProtobufModel(torch.nn.Module): + """ + Wrapper of a caffe2's protobuf model. + It works just like nn.Module, but running caffe2 under the hood. + Input/Output are tuple[tensor] that match the caffe2 net's external_input/output. + """ + + _ids = count(0) + + def __init__(self, predict_net, init_net): + logger.info(f"Initializing ProtobufModel for: {predict_net.name} ...") + super().__init__() + assert isinstance(predict_net, caffe2_pb2.NetDef) + assert isinstance(init_net, caffe2_pb2.NetDef) + # create unique temporary workspace for each instance + self.ws_name = "__tmp_ProtobufModel_{}__".format(next(self._ids)) + self.net = core.Net(predict_net) + + logger.info("Running init_net once to fill the parameters ...") + with ScopedWS(self.ws_name, is_reset=True, is_cleanup=False) as ws: + ws.RunNetOnce(init_net) + uninitialized_external_input = [] + for blob in self.net.Proto().external_input: + if blob not in ws.Blobs(): + uninitialized_external_input.append(blob) + ws.CreateBlob(blob) + ws.CreateNet(self.net) + + self._error_msgs = set() + self._input_blobs = uninitialized_external_input + + def _infer_output_devices(self, inputs): + """ + Returns: + list[str]: list of device for each external output + """ + + def _get_device_type(torch_tensor): + assert torch_tensor.device.type in ["cpu", "cuda"] + assert torch_tensor.device.index == 0 + return torch_tensor.device.type + + predict_net = self.net.Proto() + input_device_types = {(name, 0): _get_device_type(tensor) for name, tensor in zip(self._input_blobs, inputs)} + device_type_map = infer_device_type(predict_net, known_status=input_device_types, device_name_style="pytorch") + ssa, versions = core.get_ssa(predict_net) + versioned_outputs = [(name, versions[name]) for name in predict_net.external_output] + output_devices = [device_type_map[outp] for outp in versioned_outputs] + return output_devices + + def forward(self, inputs): + """ + Args: + inputs (tuple[torch.Tensor]) + + Returns: + tuple[torch.Tensor] + """ + assert len(inputs) == len(self._input_blobs), ( + f"Length of inputs ({len(inputs)}) " f"doesn't match the required input blobs: {self._input_blobs}" + ) + + with ScopedWS(self.ws_name, is_reset=False, is_cleanup=False) as ws: + for b, tensor in zip(self._input_blobs, inputs): + ws.FeedBlob(b, tensor) + + try: + ws.RunNet(self.net.Proto().name) + except RuntimeError as e: + if str(e) not in self._error_msgs: + self._error_msgs.add(str(e)) + logger.warning("Encountered new RuntimeError: \n{}".format(str(e))) + logger.warning("Catch the error and use partial results.") + + c2_outputs = [ws.FetchBlob(b) for b in self.net.Proto().external_output] + # Remove outputs of current run, this is necessary in order to + # prevent fetching the result from previous run if the model fails + # in the middle. + for b in self.net.Proto().external_output: + # Needs to create uninitialized blob to make the net runable. + # This is "equivalent" to: ws.RemoveBlob(b) then ws.CreateBlob(b), + # but there'no such API. + ws.FeedBlob(b, f"{b}, a C++ native class of type nullptr (uninitialized).") + + # Cast output to torch.Tensor on the desired device + output_devices = ( + self._infer_output_devices(inputs) + if any(t.device.type != "cpu" for t in inputs) + else ["cpu" for _ in self.net.Proto().external_output] + ) + + outputs = [] + for name, c2_output, device in zip(self.net.Proto().external_output, c2_outputs, output_devices): + if not isinstance(c2_output, np.ndarray): + raise RuntimeError("Invalid output for blob {}, received: {}".format(name, c2_output)) + outputs.append(torch.tensor(c2_output).to(device=device)) + return tuple(outputs) + + +class ProtobufDetectionModel(torch.nn.Module): + """ + A class works just like a pytorch meta arch in terms of inference, but running + caffe2 model under the hood. + """ + + def __init__(self, predict_net, init_net, *, convert_outputs=None): + """ + Args: + predict_net, init_net (core.Net): caffe2 nets + convert_outptus (callable): a function that converts caffe2 + outputs to the same format of the original pytorch model. + By default, use the one defined in the caffe2 meta_arch. + """ + super().__init__() + self.protobuf_model = ProtobufModel(predict_net, init_net) + self.size_divisibility = get_pb_arg_vali(predict_net, "size_divisibility", 0) + self.device = get_pb_arg_vals(predict_net, "device", b"cpu").decode("ascii") + + if convert_outputs is None: + meta_arch = get_pb_arg_vals(predict_net, "meta_architecture", b"GeneralizedRCNN") + meta_arch = META_ARCH_CAFFE2_EXPORT_TYPE_MAP[meta_arch.decode("ascii")] + self._convert_outputs = meta_arch.get_outputs_converter(predict_net, init_net) + else: + self._convert_outputs = convert_outputs + + def _convert_inputs(self, batched_inputs): + # currently all models convert inputs in the same way + return convert_batched_inputs_to_c2_format(batched_inputs, self.size_divisibility, self.device) + + def forward(self, batched_inputs): + c2_inputs = self._convert_inputs(batched_inputs) + c2_results = self.protobuf_model(c2_inputs) + c2_results = dict(zip(self.protobuf_model.net.Proto().external_output, c2_results)) + return self._convert_outputs(batched_inputs, c2_inputs, c2_results) diff --git a/detectron2/export/caffe2_modeling.py b/detectron2/export/caffe2_modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..5ca183d7f6aaa530626ec18e69f3211447f9a80c --- /dev/null +++ b/detectron2/export/caffe2_modeling.py @@ -0,0 +1,402 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import functools +import io +import struct +import types + +import torch + +from detectron2.modeling import meta_arch +from detectron2.modeling.box_regression import Box2BoxTransform +from detectron2.modeling.roi_heads import keypoint_head +from detectron2.structures import Boxes, ImageList, Instances, RotatedBoxes + +from .c10 import Caffe2Compatible +from .caffe2_patch import ROIHeadsPatcher, patch_generalized_rcnn +from .shared import ( + alias, + check_set_pb_arg, + get_pb_arg_floats, + get_pb_arg_valf, + get_pb_arg_vali, + get_pb_arg_vals, + mock_torch_nn_functional_interpolate, +) + + +def assemble_rcnn_outputs_by_name(image_sizes, tensor_outputs, force_mask_on=False): + """ + A function to assemble caffe2 model's outputs (i.e. Dict[str, Tensor]) + to detectron2's format (i.e. list of Instances instance). + This only works when the model follows the Caffe2 detectron's naming convention. + + Args: + image_sizes (List[List[int, int]]): [H, W] of every image. + tensor_outputs (Dict[str, Tensor]): external_output to its tensor. + + force_mask_on (Bool): if true, the it make sure there'll be pred_masks even + if the mask is not found from tensor_outputs (usually due to model crash) + """ + + results = [Instances(image_size) for image_size in image_sizes] + + batch_splits = tensor_outputs.get("batch_splits", None) + if batch_splits: + raise NotImplementedError() + assert len(image_sizes) == 1 + result = results[0] + + bbox_nms = tensor_outputs["bbox_nms"] + score_nms = tensor_outputs["score_nms"] + class_nms = tensor_outputs["class_nms"] + # Detection will always success because Conv support 0-batch + assert bbox_nms is not None + assert score_nms is not None + assert class_nms is not None + if bbox_nms.shape[1] == 5: + result.pred_boxes = RotatedBoxes(bbox_nms) + else: + result.pred_boxes = Boxes(bbox_nms) + result.scores = score_nms + result.pred_classes = class_nms.to(torch.int64) + + mask_fcn_probs = tensor_outputs.get("mask_fcn_probs", None) + if mask_fcn_probs is not None: + # finish the mask pred + mask_probs_pred = mask_fcn_probs + num_masks = mask_probs_pred.shape[0] + class_pred = result.pred_classes + indices = torch.arange(num_masks, device=class_pred.device) + mask_probs_pred = mask_probs_pred[indices, class_pred][:, None] + result.pred_masks = mask_probs_pred + elif force_mask_on: + # NOTE: there's no way to know the height/width of mask here, it won't be + # used anyway when batch size is 0, so just set them to 0. + result.pred_masks = torch.zeros([0, 1, 0, 0], dtype=torch.uint8) + + keypoints_out = tensor_outputs.get("keypoints_out", None) + kps_score = tensor_outputs.get("kps_score", None) + if keypoints_out is not None: + # keypoints_out: [N, 4, #kypoints], where 4 is in order of (x, y, score, prob) + keypoints_tensor = keypoints_out + # NOTE: it's possible that prob is not calculated if "should_output_softmax" + # is set to False in HeatmapMaxKeypoint, so just using raw score, seems + # it doesn't affect mAP. TODO: check more carefully. + keypoint_xyp = keypoints_tensor.transpose(1, 2)[:, :, [0, 1, 2]] + result.pred_keypoints = keypoint_xyp + elif kps_score is not None: + # keypoint heatmap to sparse data structure + pred_keypoint_logits = kps_score + keypoint_head.keypoint_rcnn_inference(pred_keypoint_logits, [result]) + + return results + + +def _cast_to_f32(f64): + return struct.unpack("f", struct.pack("f", f64))[0] + + +def set_caffe2_compatible_tensor_mode(model, enable=True): + def _fn(m): + if isinstance(m, Caffe2Compatible): + m.tensor_mode = enable + + model.apply(_fn) + + +def convert_batched_inputs_to_c2_format(batched_inputs, size_divisibility, device): + """ + See get_caffe2_inputs() below. + """ + assert all(isinstance(x, dict) for x in batched_inputs) + assert all(x["image"].dim() == 3 for x in batched_inputs) + + images = [x["image"] for x in batched_inputs] + images = ImageList.from_tensors(images, size_divisibility) + + im_info = [] + for input_per_image, image_size in zip(batched_inputs, images.image_sizes): + target_height = input_per_image.get("height", image_size[0]) + target_width = input_per_image.get("width", image_size[1]) # noqa + # NOTE: The scale inside im_info is kept as convention and for providing + # post-processing information if further processing is needed. For + # current Caffe2 model definitions that don't include post-processing inside + # the model, this number is not used. + # NOTE: There can be a slight difference between width and height + # scales, using a single number can results in numerical difference + # compared with D2's post-processing. + scale = target_height / image_size[0] + im_info.append([image_size[0], image_size[1], scale]) + im_info = torch.Tensor(im_info) + + return images.tensor.to(device), im_info.to(device) + + +class Caffe2MetaArch(Caffe2Compatible, torch.nn.Module): + """ + Base class for caffe2-compatible implementation of a meta architecture. + The forward is traceable and its traced graph can be converted to caffe2 + graph through ONNX. + """ + + def __init__(self, cfg, torch_model): + """ + Args: + cfg (CfgNode): + torch_model (nn.Module): the detectron2 model (meta_arch) to be + converted. + """ + super().__init__() + self._wrapped_model = torch_model + self.eval() + set_caffe2_compatible_tensor_mode(self, True) + + def get_caffe2_inputs(self, batched_inputs): + """ + Convert pytorch-style structured inputs to caffe2-style inputs that + are tuples of tensors. + + Args: + batched_inputs (list[dict]): inputs to a detectron2 model + in its standard format. Each dict has "image" (CHW tensor), and optionally + "height" and "width". + + Returns: + tuple[Tensor]: + tuple of tensors that will be the inputs to the + :meth:`forward` method. For existing models, the first + is an NCHW tensor (padded and batched); the second is + a im_info Nx3 tensor, where the rows are + (height, width, unused legacy parameter) + """ + return convert_batched_inputs_to_c2_format( + batched_inputs, + self._wrapped_model.backbone.size_divisibility, + self._wrapped_model.device, + ) + + def encode_additional_info(self, predict_net, init_net): + """ + Save extra metadata that will be used by inference in the output protobuf. + """ + pass + + def forward(self, inputs): + """ + Run the forward in caffe2-style. It has to use caffe2-compatible ops + and the method will be used for tracing. + + Args: + inputs (tuple[Tensor]): inputs defined by :meth:`get_caffe2_input`. + They will be the inputs of the converted caffe2 graph. + + Returns: + tuple[Tensor]: output tensors. They will be the outputs of the + converted caffe2 graph. + """ + raise NotImplementedError + + def _caffe2_preprocess_image(self, inputs): + """ + Caffe2 implementation of preprocess_image, which is called inside each MetaArch's forward. + It normalizes the input images, and the final caffe2 graph assumes the + inputs have been batched already. + """ + data, im_info = inputs + data = alias(data, "data") + im_info = alias(im_info, "im_info") + mean, std = self._wrapped_model.pixel_mean, self._wrapped_model.pixel_std + normalized_data = (data - mean) / std + normalized_data = alias(normalized_data, "normalized_data") + + # Pack (data, im_info) into ImageList which is recognized by self.inference. + images = ImageList(tensor=normalized_data, image_sizes=im_info) + return images + + @staticmethod + def get_outputs_converter(predict_net, init_net): + """ + Creates a function that converts outputs of the caffe2 model to + detectron2's standard format. + The function uses information in `predict_net` and `init_net` that are + available at inferene time. Therefore the function logic can be used in inference. + + The returned function has the following signature: + + def convert(batched_inputs, c2_inputs, c2_results) -> detectron2_outputs + + Where + + * batched_inputs (list[dict]): the original input format of the meta arch + * c2_inputs (tuple[Tensor]): the caffe2 inputs. + * c2_results (dict[str, Tensor]): the caffe2 output format, + corresponding to the outputs of the :meth:`forward` function. + * detectron2_outputs: the original output format of the meta arch. + + This function can be used to compare the outputs of the original meta arch and + the converted caffe2 graph. + + Returns: + callable: a callable of the above signature. + """ + raise NotImplementedError + + +class Caffe2GeneralizedRCNN(Caffe2MetaArch): + def __init__(self, cfg, torch_model): + assert isinstance(torch_model, meta_arch.GeneralizedRCNN) + torch_model = patch_generalized_rcnn(torch_model) + super().__init__(cfg, torch_model) + + try: + use_heatmap_max_keypoint = cfg.EXPORT_CAFFE2.USE_HEATMAP_MAX_KEYPOINT + except AttributeError: + use_heatmap_max_keypoint = False + self.roi_heads_patcher = ROIHeadsPatcher(self._wrapped_model.roi_heads, use_heatmap_max_keypoint) + + def encode_additional_info(self, predict_net, init_net): + size_divisibility = self._wrapped_model.backbone.size_divisibility + check_set_pb_arg(predict_net, "size_divisibility", "i", size_divisibility) + check_set_pb_arg(predict_net, "device", "s", str.encode(str(self._wrapped_model.device), "ascii")) + check_set_pb_arg(predict_net, "meta_architecture", "s", b"GeneralizedRCNN") + + @mock_torch_nn_functional_interpolate() + def forward(self, inputs): + if not self.tensor_mode: + return self._wrapped_model.inference(inputs) + images = self._caffe2_preprocess_image(inputs) + features = self._wrapped_model.backbone(images.tensor) + proposals, _ = self._wrapped_model.proposal_generator(images, features) + with self.roi_heads_patcher.mock_roi_heads(): + detector_results, _ = self._wrapped_model.roi_heads(images, features, proposals) + return tuple(detector_results[0].flatten()) + + @staticmethod + def get_outputs_converter(predict_net, init_net): + def f(batched_inputs, c2_inputs, c2_results): + _, im_info = c2_inputs + image_sizes = [[int(im[0]), int(im[1])] for im in im_info] + results = assemble_rcnn_outputs_by_name(image_sizes, c2_results) + return meta_arch.GeneralizedRCNN._postprocess(results, batched_inputs, image_sizes) + + return f + + +class Caffe2RetinaNet(Caffe2MetaArch): + def __init__(self, cfg, torch_model): + assert isinstance(torch_model, meta_arch.RetinaNet) + super().__init__(cfg, torch_model) + + @mock_torch_nn_functional_interpolate() + def forward(self, inputs): + assert self.tensor_mode + images = self._caffe2_preprocess_image(inputs) + + # explicitly return the images sizes to avoid removing "im_info" by ONNX + # since it's not used in the forward path + return_tensors = [images.image_sizes] + + features = self._wrapped_model.backbone(images.tensor) + features = [features[f] for f in self._wrapped_model.head_in_features] + for i, feature_i in enumerate(features): + features[i] = alias(feature_i, "feature_{}".format(i), is_backward=True) + return_tensors.append(features[i]) + + pred_logits, pred_anchor_deltas = self._wrapped_model.head(features) + for i, (box_cls_i, box_delta_i) in enumerate(zip(pred_logits, pred_anchor_deltas)): + return_tensors.append(alias(box_cls_i, "box_cls_{}".format(i))) + return_tensors.append(alias(box_delta_i, "box_delta_{}".format(i))) + + return tuple(return_tensors) + + def encode_additional_info(self, predict_net, init_net): + size_divisibility = self._wrapped_model.backbone.size_divisibility + check_set_pb_arg(predict_net, "size_divisibility", "i", size_divisibility) + check_set_pb_arg(predict_net, "device", "s", str.encode(str(self._wrapped_model.device), "ascii")) + check_set_pb_arg(predict_net, "meta_architecture", "s", b"RetinaNet") + + # Inference parameters: + check_set_pb_arg(predict_net, "score_threshold", "f", _cast_to_f32(self._wrapped_model.test_score_thresh)) + check_set_pb_arg(predict_net, "topk_candidates", "i", self._wrapped_model.test_topk_candidates) + check_set_pb_arg(predict_net, "nms_threshold", "f", _cast_to_f32(self._wrapped_model.test_nms_thresh)) + check_set_pb_arg( + predict_net, + "max_detections_per_image", + "i", + self._wrapped_model.max_detections_per_image, + ) + + check_set_pb_arg( + predict_net, + "bbox_reg_weights", + "floats", + [_cast_to_f32(w) for w in self._wrapped_model.box2box_transform.weights], + ) + self._encode_anchor_generator_cfg(predict_net) + + def _encode_anchor_generator_cfg(self, predict_net): + # serialize anchor_generator for future use + serialized_anchor_generator = io.BytesIO() + torch.save(self._wrapped_model.anchor_generator, serialized_anchor_generator) + # Ideally we can put anchor generating inside the model, then we don't + # need to store this information. + bytes = serialized_anchor_generator.getvalue() + check_set_pb_arg(predict_net, "serialized_anchor_generator", "s", bytes) + + @staticmethod + def get_outputs_converter(predict_net, init_net): + self = types.SimpleNamespace() + serialized_anchor_generator = io.BytesIO(get_pb_arg_vals(predict_net, "serialized_anchor_generator", None)) + self.anchor_generator = torch.load(serialized_anchor_generator) + bbox_reg_weights = get_pb_arg_floats(predict_net, "bbox_reg_weights", None) + self.box2box_transform = Box2BoxTransform(weights=tuple(bbox_reg_weights)) + self.test_score_thresh = get_pb_arg_valf(predict_net, "score_threshold", None) + self.test_topk_candidates = get_pb_arg_vali(predict_net, "topk_candidates", None) + self.test_nms_thresh = get_pb_arg_valf(predict_net, "nms_threshold", None) + self.max_detections_per_image = get_pb_arg_vali(predict_net, "max_detections_per_image", None) + + # hack to reuse inference code from RetinaNet + for meth in [ + "forward_inference", + "inference_single_image", + "_transpose_dense_predictions", + "_decode_multi_level_predictions", + "_decode_per_level_predictions", + ]: + setattr(self, meth, functools.partial(getattr(meta_arch.RetinaNet, meth), self)) + + def f(batched_inputs, c2_inputs, c2_results): + _, im_info = c2_inputs + image_sizes = [[int(im[0]), int(im[1])] for im in im_info] + dummy_images = ImageList( + torch.randn( + ( + len(im_info), + 3, + ) + + tuple(image_sizes[0]) + ), + image_sizes, + ) + + num_features = len([x for x in c2_results.keys() if x.startswith("box_cls_")]) + pred_logits = [c2_results["box_cls_{}".format(i)] for i in range(num_features)] + pred_anchor_deltas = [c2_results["box_delta_{}".format(i)] for i in range(num_features)] + + # For each feature level, feature should have the same batch size and + # spatial dimension as the box_cls and box_delta. + dummy_features = [x.clone()[:, 0:0, :, :] for x in pred_logits] + # self.num_classess can be inferred + self.num_classes = pred_logits[0].shape[1] // (pred_anchor_deltas[0].shape[1] // 4) + + results = self.forward_inference(dummy_images, dummy_features, [pred_logits, pred_anchor_deltas]) + return meta_arch.GeneralizedRCNN._postprocess(results, batched_inputs, image_sizes) + + return f + + +META_ARCH_CAFFE2_EXPORT_TYPE_MAP = { + "GeneralizedRCNN": Caffe2GeneralizedRCNN, + "RetinaNet": Caffe2RetinaNet, +} diff --git a/detectron2/export/caffe2_patch.py b/detectron2/export/caffe2_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..d344ba18c52f09e5ba1ef415dd357ccfe7da0247 --- /dev/null +++ b/detectron2/export/caffe2_patch.py @@ -0,0 +1,149 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import contextlib +from unittest import mock + +import torch + +from detectron2.modeling import poolers +from detectron2.modeling.proposal_generator import rpn +from detectron2.modeling.roi_heads import keypoint_head, mask_head +from detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers + +from .c10 import ( + Caffe2Compatible, + Caffe2FastRCNNOutputsInference, + Caffe2KeypointRCNNInference, + Caffe2MaskRCNNInference, + Caffe2ROIPooler, + Caffe2RPN, +) + + +class GenericMixin(object): + pass + + +class Caffe2CompatibleConverter(object): + """ + A GenericUpdater which implements the `create_from` interface, by modifying + module object and assign it with another class replaceCls. + """ + + def __init__(self, replaceCls): + self.replaceCls = replaceCls + + def create_from(self, module): + # update module's class to the new class + assert isinstance(module, torch.nn.Module) + if issubclass(self.replaceCls, GenericMixin): + # replaceCls should act as mixin, create a new class on-the-fly + new_class = type( + "{}MixedWith{}".format(self.replaceCls.__name__, module.__class__.__name__), + (self.replaceCls, module.__class__), + {}, # {"new_method": lambda self: ...}, + ) + module.__class__ = new_class + else: + # replaceCls is complete class, this allow arbitrary class swap + module.__class__ = self.replaceCls + + # initialize Caffe2Compatible + if isinstance(module, Caffe2Compatible): + module.tensor_mode = False + + return module + + +def patch(model, target, updater, *args, **kwargs): + """ + recursively (post-order) update all modules with the target type and its + subclasses, make a initialization/composition/inheritance/... via the + updater.create_from. + """ + for name, module in model.named_children(): + model._modules[name] = patch(module, target, updater, *args, **kwargs) + if isinstance(model, target): + return updater.create_from(model, *args, **kwargs) + return model + + +def patch_generalized_rcnn(model): + ccc = Caffe2CompatibleConverter + model = patch(model, rpn.RPN, ccc(Caffe2RPN)) + model = patch(model, poolers.ROIPooler, ccc(Caffe2ROIPooler)) + + return model + + +@contextlib.contextmanager +def mock_fastrcnn_outputs_inference(tensor_mode, check=True, box_predictor_type=FastRCNNOutputLayers): + with mock.patch.object( + box_predictor_type, + "inference", + autospec=True, + side_effect=Caffe2FastRCNNOutputsInference(tensor_mode), + ) as mocked_func: + yield + if check: + assert mocked_func.call_count > 0 + + +@contextlib.contextmanager +def mock_mask_rcnn_inference(tensor_mode, patched_module, check=True): + with mock.patch( + "{}.mask_rcnn_inference".format(patched_module), side_effect=Caffe2MaskRCNNInference() + ) as mocked_func: + yield + if check: + assert mocked_func.call_count > 0 + + +@contextlib.contextmanager +def mock_keypoint_rcnn_inference(tensor_mode, patched_module, use_heatmap_max_keypoint, check=True): + with mock.patch( + "{}.keypoint_rcnn_inference".format(patched_module), + side_effect=Caffe2KeypointRCNNInference(use_heatmap_max_keypoint), + ) as mocked_func: + yield + if check: + assert mocked_func.call_count > 0 + + +class ROIHeadsPatcher: + def __init__(self, heads, use_heatmap_max_keypoint): + self.heads = heads + self.use_heatmap_max_keypoint = use_heatmap_max_keypoint + + @contextlib.contextmanager + def mock_roi_heads(self, tensor_mode=True): + """ + Patching several inference functions inside ROIHeads and its subclasses + + Args: + tensor_mode (bool): whether the inputs/outputs are caffe2's tensor + format or not. Default to True. + """ + # NOTE: this requries the `keypoint_rcnn_inference` and `mask_rcnn_inference` + # are called inside the same file as BaseXxxHead due to using mock.patch. + kpt_heads_mod = keypoint_head.BaseKeypointRCNNHead.__module__ + mask_head_mod = mask_head.BaseMaskRCNNHead.__module__ + + mock_ctx_managers = [ + mock_fastrcnn_outputs_inference( + tensor_mode=tensor_mode, + check=True, + box_predictor_type=type(self.heads.box_predictor), + ) + ] + if getattr(self.heads, "keypoint_on", False): + mock_ctx_managers += [ + mock_keypoint_rcnn_inference(tensor_mode, kpt_heads_mod, self.use_heatmap_max_keypoint) + ] + if getattr(self.heads, "mask_on", False): + mock_ctx_managers += [mock_mask_rcnn_inference(tensor_mode, mask_head_mod)] + + with contextlib.ExitStack() as stack: # python 3.3+ + for mgr in mock_ctx_managers: + stack.enter_context(mgr) + yield diff --git a/detectron2/export/flatten.py b/detectron2/export/flatten.py new file mode 100644 index 0000000000000000000000000000000000000000..6a715fa997d5425077cca23d17aa69fbbb3f5f9e --- /dev/null +++ b/detectron2/export/flatten.py @@ -0,0 +1,316 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import collections +from dataclasses import dataclass +from typing import Callable, List, Optional, Tuple + +import torch +from torch import nn + +from detectron2.structures import Boxes, Instances, ROIMasks +from detectron2.utils.registry import _convert_target_to_string, locate + +from .torchscript_patch import patch_builtin_len + + +@dataclass +class Schema: + """ + A Schema defines how to flatten a possibly hierarchical object into tuple of + primitive objects, so it can be used as inputs/outputs of PyTorch's tracing. + + PyTorch does not support tracing a function that produces rich output + structures (e.g. dict, Instances, Boxes). To trace such a function, we + flatten the rich object into tuple of tensors, and return this tuple of tensors + instead. Meanwhile, we also need to know how to "rebuild" the original object + from the flattened results, so we can evaluate the flattened results. + A Schema defines how to flatten an object, and while flattening it, it records + necessary schemas so that the object can be rebuilt using the flattened outputs. + + The flattened object and the schema object is returned by ``.flatten`` classmethod. + Then the original object can be rebuilt with the ``__call__`` method of schema. + + A Schema is a dataclass that can be serialized easily. + """ + + # inspired by FetchMapper in tensorflow/python/client/session.py + + @classmethod + def flatten(cls, obj): + raise NotImplementedError + + def __call__(self, values): + raise NotImplementedError + + @staticmethod + def _concat(values): + ret = () + sizes = [] + for v in values: + assert isinstance(v, tuple), "Flattened results must be a tuple" + ret = ret + v + sizes.append(len(v)) + return ret, sizes + + @staticmethod + def _split(values, sizes): + if len(sizes): + expected_len = sum(sizes) + assert len(values) == expected_len, f"Values has length {len(values)} but expect length {expected_len}." + ret = [] + for k in range(len(sizes)): + begin, end = sum(sizes[:k]), sum(sizes[: k + 1]) + ret.append(values[begin:end]) + return ret + + +@dataclass +class ListSchema(Schema): + schemas: List[Schema] # the schemas that define how to flatten each element in the list + sizes: List[int] # the flattened length of each element + + def __call__(self, values): + values = self._split(values, self.sizes) + if len(values) != len(self.schemas): + raise ValueError(f"Values has length {len(values)} but schemas " f"has length {len(self.schemas)}!") + values = [m(v) for m, v in zip(self.schemas, values)] + return list(values) + + @classmethod + def flatten(cls, obj): + res = [flatten_to_tuple(k) for k in obj] + values, sizes = cls._concat([k[0] for k in res]) + return values, cls([k[1] for k in res], sizes) + + +@dataclass +class TupleSchema(ListSchema): + def __call__(self, values): + return tuple(super().__call__(values)) + + +@dataclass +class IdentitySchema(Schema): + def __call__(self, values): + return values[0] + + @classmethod + def flatten(cls, obj): + return (obj,), cls() + + +@dataclass +class DictSchema(ListSchema): + keys: List[str] + + def __call__(self, values): + values = super().__call__(values) + return dict(zip(self.keys, values)) + + @classmethod + def flatten(cls, obj): + for k in obj.keys(): + if not isinstance(k, str): + raise KeyError("Only support flattening dictionaries if keys are str.") + keys = sorted(obj.keys()) + values = [obj[k] for k in keys] + ret, schema = ListSchema.flatten(values) + return ret, cls(schema.schemas, schema.sizes, keys) + + +@dataclass +class InstancesSchema(DictSchema): + def __call__(self, values): + image_size, fields = values[-1], values[:-1] + fields = super().__call__(fields) + return Instances(image_size, **fields) + + @classmethod + def flatten(cls, obj): + ret, schema = super().flatten(obj.get_fields()) + size = obj.image_size + if not isinstance(size, torch.Tensor): + size = torch.tensor(size) + return ret + (size,), schema + + +@dataclass +class TensorWrapSchema(Schema): + """ + For classes that are simple wrapper of tensors, e.g. + Boxes, RotatedBoxes, BitMasks + """ + + class_name: str + + def __call__(self, values): + return locate(self.class_name)(values[0]) + + @classmethod + def flatten(cls, obj): + return (obj.tensor,), cls(_convert_target_to_string(type(obj))) + + +# if more custom structures needed in the future, can allow +# passing in extra schemas for custom types +def flatten_to_tuple(obj): + """ + Flatten an object so it can be used for PyTorch tracing. + Also returns how to rebuild the original object from the flattened outputs. + + Returns: + res (tuple): the flattened results that can be used as tracing outputs + schema: an object with a ``__call__`` method such that ``schema(res) == obj``. + It is a pure dataclass that can be serialized. + """ + schemas = [ + ((str, bytes), IdentitySchema), + (list, ListSchema), + (tuple, TupleSchema), + (collections.abc.Mapping, DictSchema), + (Instances, InstancesSchema), + ((Boxes, ROIMasks), TensorWrapSchema), + ] + for klass, schema in schemas: + if isinstance(obj, klass): + F = schema + break + else: + F = IdentitySchema + + return F.flatten(obj) + + +class TracingAdapter(nn.Module): + """ + A model may take rich input/output format (e.g. dict or custom classes), + but `torch.jit.trace` requires tuple of tensors as input/output. + This adapter flattens input/output format of a model so it becomes traceable. + + It also records the necessary schema to rebuild model's inputs/outputs from flattened + inputs/outputs. + + Example: + :: + outputs = model(inputs) # inputs/outputs may be rich structure + adapter = TracingAdapter(model, inputs) + + # can now trace the model, with adapter.flattened_inputs, or another + # tuple of tensors with the same length and meaning + traced = torch.jit.trace(adapter, adapter.flattened_inputs) + + # traced model can only produce flattened outputs (tuple of tensors) + flattened_outputs = traced(*adapter.flattened_inputs) + # adapter knows the schema to convert it back (new_outputs == outputs) + new_outputs = adapter.outputs_schema(flattened_outputs) + """ + + flattened_inputs: Tuple[torch.Tensor] = None + """ + Flattened version of inputs given to this class's constructor. + """ + + inputs_schema: Schema = None + """ + Schema of the inputs given to this class's constructor. + """ + + outputs_schema: Schema = None + """ + Schema of the output produced by calling the given model with inputs. + """ + + def __init__( + self, + model: nn.Module, + inputs, + inference_func: Optional[Callable] = None, + allow_non_tensor: bool = False, + ): + """ + Args: + model: an nn.Module + inputs: An input argument or a tuple of input arguments used to call model. + After flattening, it has to only consist of tensors. + inference_func: a callable that takes (model, *inputs), calls the + model with inputs, and return outputs. By default it + is ``lambda model, *inputs: model(*inputs)``. Can be override + if you need to call the model differently. + allow_non_tensor: allow inputs/outputs to contain non-tensor objects. + This option will filter out non-tensor objects to make the + model traceable, but ``inputs_schema``/``outputs_schema`` cannot be + used anymore because inputs/outputs cannot be rebuilt from pure tensors. + This is useful when you're only interested in the single trace of + execution (e.g. for flop count), but not interested in + generalizing the traced graph to new inputs. + """ + super().__init__() + if isinstance(model, (nn.parallel.distributed.DistributedDataParallel, nn.DataParallel)): + model = model.module + self.model = model + if not isinstance(inputs, tuple): + inputs = (inputs,) + self.inputs = inputs + self.allow_non_tensor = allow_non_tensor + + if inference_func is None: + inference_func = lambda model, *inputs: model(*inputs) # noqa + self.inference_func = inference_func + + self.flattened_inputs, self.inputs_schema = flatten_to_tuple(inputs) + + if all(isinstance(x, torch.Tensor) for x in self.flattened_inputs): + return + if self.allow_non_tensor: + self.flattened_inputs = tuple([x for x in self.flattened_inputs if isinstance(x, torch.Tensor)]) + self.inputs_schema = None + else: + for input in self.flattened_inputs: + if not isinstance(input, torch.Tensor): + raise ValueError("Inputs for tracing must only contain tensors. " f"Got a {type(input)} instead.") + + def forward(self, *args: torch.Tensor): + with torch.no_grad(), patch_builtin_len(): + if self.inputs_schema is not None: + inputs_orig_format = self.inputs_schema(args) + else: + if len(args) != len(self.flattened_inputs) or any( + x is not y for x, y in zip(args, self.flattened_inputs) + ): + raise ValueError( + "TracingAdapter does not contain valid inputs_schema." + " So it cannot generalize to other inputs and must be" + " traced with `.flattened_inputs`." + ) + inputs_orig_format = self.inputs + + outputs = self.inference_func(self.model, *inputs_orig_format) + flattened_outputs, schema = flatten_to_tuple(outputs) + + flattened_output_tensors = tuple([x for x in flattened_outputs if isinstance(x, torch.Tensor)]) + if len(flattened_output_tensors) < len(flattened_outputs): + if self.allow_non_tensor: + flattened_outputs = flattened_output_tensors + self.outputs_schema = None + else: + raise ValueError("Model cannot be traced because some model outputs " "cannot flatten to tensors.") + else: # schema is valid + if self.outputs_schema is None: + self.outputs_schema = schema + else: + assert self.outputs_schema == schema, ( + "Model should always return outputs with the same " "structure so it can be traced!" + ) + return flattened_outputs + + def _create_wrapper(self, traced_model): + """ + Return a function that has an input/output interface the same as the + original model, but it calls the given traced model under the hood. + """ + + def forward(*args): + flattened_inputs, _ = flatten_to_tuple(args) + flattened_outputs = traced_model(*flattened_inputs) + return self.outputs_schema(flattened_outputs) + + return forward diff --git a/detectron2/export/shared.py b/detectron2/export/shared.py new file mode 100644 index 0000000000000000000000000000000000000000..140ebec4311bbf0ce6695a5fab46e2f103d73d6e --- /dev/null +++ b/detectron2/export/shared.py @@ -0,0 +1,1004 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import collections +import contextlib +import copy +import functools +import logging +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from unittest import mock + +import caffe2.python.utils as putils +import numpy as np +import torch +import torch.nn.functional as F +from caffe2.proto import caffe2_pb2 +from caffe2.python import core, net_drawer, workspace +from torch.nn.functional import interpolate as interp + +logger = logging.getLogger(__name__) + + +# ==== torch/utils_toffee/cast.py ======================================= + + +def to_device(t, device_str): + """ + This function is a replacement of .to(another_device) such that it allows the + casting to be traced properly by explicitly calling the underlying copy ops. + It also avoids introducing unncessary op when casting to the same device. + """ + src = t.device + dst = torch.device(device_str) + + if src == dst: + return t + elif src.type == "cuda" and dst.type == "cpu": + return torch.ops._caffe2.CopyGPUToCPU(t) + elif src.type == "cpu" and dst.type == "cuda": + return torch.ops._caffe2.CopyCPUToGPU(t) + else: + raise RuntimeError("Can't cast tensor from device {} to device {}".format(src, dst)) + + +# ==== torch/utils_toffee/interpolate.py ======================================= + + +# Note: borrowed from vision/detection/fair/detectron/detectron/modeling/detector.py +def BilinearInterpolation(tensor_in, up_scale): + assert up_scale % 2 == 0, "Scale should be even" + + def upsample_filt(size): + factor = (size + 1) // 2 + if size % 2 == 1: + center = factor - 1 + else: + center = factor - 0.5 + + og = np.ogrid[:size, :size] + return (1 - abs(og[0] - center) / factor) * (1 - abs(og[1] - center) / factor) + + kernel_size = int(up_scale) * 2 + bil_filt = upsample_filt(kernel_size) + + dim = int(tensor_in.shape[1]) + kernel = np.zeros((dim, dim, kernel_size, kernel_size), dtype=np.float32) + kernel[range(dim), range(dim), :, :] = bil_filt + + tensor_out = F.conv_transpose2d( + tensor_in, + weight=to_device(torch.Tensor(kernel), tensor_in.device), + bias=None, + stride=int(up_scale), + padding=int(up_scale / 2), + ) + + return tensor_out + + +# NOTE: ONNX is incompatible with traced torch.nn.functional.interpolate if +# using dynamic `scale_factor` rather than static `size`. (T43166860) +# NOTE: Caffe2 Int8 conversion might not be able to quantize `size` properly. +def onnx_compatibale_interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # NOTE: The input dimensions are interpreted in the form: + # `mini-batch x channels x [optional depth] x [optional height] x width`. + if size is None and scale_factor is not None: + if input.dim() == 4: + if isinstance(scale_factor, (int, float)): + height_scale, width_scale = (scale_factor, scale_factor) + else: + assert isinstance(scale_factor, (tuple, list)) + assert len(scale_factor) == 2 + height_scale, width_scale = scale_factor + + assert not align_corners, "No matching C2 op for align_corners == True" + if mode == "nearest": + return torch.ops._caffe2.ResizeNearest( + input, order="NCHW", width_scale=width_scale, height_scale=height_scale + ) + elif mode == "bilinear": + logger.warning( + "Use F.conv_transpose2d for bilinear interpolate" + " because there's no such C2 op, this may cause significant" + " slowdown and the boundary pixels won't be as same as" + " using F.interpolate due to padding." + ) + assert height_scale == width_scale + return BilinearInterpolation(input, up_scale=height_scale) + logger.warning("Output size is not static, it might cause ONNX conversion issue") + + return interp(input, size, scale_factor, mode, align_corners) + + +@contextlib.contextmanager +def mock_torch_nn_functional_interpolate(): + if torch.onnx.is_in_onnx_export(): + with mock.patch("torch.nn.functional.interpolate", side_effect=onnx_compatibale_interpolate): + yield + else: + yield + + +# ==== torch/utils_caffe2/ws_utils.py ========================================== + + +class ScopedWS(object): + def __init__(self, ws_name, is_reset, is_cleanup=False): + self.ws_name = ws_name + self.is_reset = is_reset + self.is_cleanup = is_cleanup + self.org_ws = "" + + def __enter__(self): + self.org_ws = workspace.CurrentWorkspace() + if self.ws_name is not None: + workspace.SwitchWorkspace(self.ws_name, True) + if self.is_reset: + workspace.ResetWorkspace() + + return workspace + + def __exit__(self, *args): + if self.is_cleanup: + workspace.ResetWorkspace() + if self.ws_name is not None: + workspace.SwitchWorkspace(self.org_ws) + + +def fetch_any_blob(name): + bb = None + try: + bb = workspace.FetchBlob(name) + except TypeError: + bb = workspace.FetchInt8Blob(name) + except Exception as e: + logger.error("Get blob {} error: {}".format(name, e)) + + return bb + + +# ==== torch/utils_caffe2/protobuf.py ========================================== + + +def get_pb_arg(pb, arg_name): + for x in pb.arg: + if x.name == arg_name: + return x + return None + + +def get_pb_arg_valf(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return arg.f if arg is not None else default_val + + +def get_pb_arg_floats(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return list(map(float, arg.floats)) if arg is not None else default_val + + +def get_pb_arg_ints(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return list(map(int, arg.ints)) if arg is not None else default_val + + +def get_pb_arg_vali(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return arg.i if arg is not None else default_val + + +def get_pb_arg_vals(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return arg.s if arg is not None else default_val + + +def get_pb_arg_valstrings(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return list(arg.strings) if arg is not None else default_val + + +def check_set_pb_arg(pb, arg_name, arg_attr, arg_value, allow_override=False): + arg = get_pb_arg(pb, arg_name) + if arg is None: + arg = putils.MakeArgument(arg_name, arg_value) + assert hasattr(arg, arg_attr) + pb.arg.extend([arg]) + if allow_override and getattr(arg, arg_attr) != arg_value: + logger.warning("Override argument {}: {} -> {}".format(arg_name, getattr(arg, arg_attr), arg_value)) + setattr(arg, arg_attr, arg_value) + else: + assert arg is not None + assert getattr(arg, arg_attr) == arg_value, "Existing value {}, new value {}".format( + getattr(arg, arg_attr), arg_value + ) + + +def _create_const_fill_op_from_numpy(name, tensor, device_option=None): + assert type(tensor) == np.ndarray + kTypeNameMapper = { + np.dtype("float32"): "GivenTensorFill", + np.dtype("int32"): "GivenTensorIntFill", + np.dtype("int64"): "GivenTensorInt64Fill", + np.dtype("uint8"): "GivenTensorStringFill", + } + + args_dict = {} + if tensor.dtype == np.dtype("uint8"): + args_dict.update({"values": [str(tensor.data)], "shape": [1]}) + else: + args_dict.update({"values": tensor, "shape": tensor.shape}) + + if device_option is not None: + args_dict["device_option"] = device_option + + return core.CreateOperator(kTypeNameMapper[tensor.dtype], [], [name], **args_dict) + + +def _create_const_fill_op_from_c2_int8_tensor(name, int8_tensor): + assert type(int8_tensor) == workspace.Int8Tensor + kTypeNameMapper = { + np.dtype("int32"): "Int8GivenIntTensorFill", + np.dtype("uint8"): "Int8GivenTensorFill", + } + + tensor = int8_tensor.data + assert tensor.dtype in [np.dtype("uint8"), np.dtype("int32")] + values = tensor.tobytes() if tensor.dtype == np.dtype("uint8") else tensor + + return core.CreateOperator( + kTypeNameMapper[tensor.dtype], + [], + [name], + values=values, + shape=tensor.shape, + Y_scale=int8_tensor.scale, + Y_zero_point=int8_tensor.zero_point, + ) + + +def create_const_fill_op( + name: str, + blob: Union[np.ndarray, workspace.Int8Tensor], + device_option: Optional[caffe2_pb2.DeviceOption] = None, +) -> caffe2_pb2.OperatorDef: + """ + Given a blob object, return the Caffe2 operator that creates this blob + as constant. Currently support NumPy tensor and Caffe2 Int8Tensor. + """ + + tensor_type = type(blob) + assert tensor_type in [ + np.ndarray, + workspace.Int8Tensor, + ], 'Error when creating const fill op for "{}", unsupported blob type: {}'.format(name, type(blob)) + + if tensor_type == np.ndarray: + return _create_const_fill_op_from_numpy(name, blob, device_option) + elif tensor_type == workspace.Int8Tensor: + assert device_option is None + return _create_const_fill_op_from_c2_int8_tensor(name, blob) + + +def construct_init_net_from_params( + params: Dict[str, Any], device_options: Optional[Dict[str, caffe2_pb2.DeviceOption]] = None +) -> caffe2_pb2.NetDef: + """ + Construct the init_net from params dictionary + """ + init_net = caffe2_pb2.NetDef() + device_options = device_options or {} + for name, blob in params.items(): + if isinstance(blob, str): + logger.warning( + ("Blob {} with type {} is not supported in generating init net," " skipped.".format(name, type(blob))) + ) + continue + init_net.op.extend([create_const_fill_op(name, blob, device_option=device_options.get(name, None))]) + init_net.external_output.append(name) + return init_net + + +def get_producer_map(ssa): + """ + Return dict from versioned blob to (i, j), + where i is index of producer op, j is the index of output of that op. + """ + producer_map = {} + for i in range(len(ssa)): + outputs = ssa[i][1] + for j, outp in enumerate(outputs): + producer_map[outp] = (i, j) + return producer_map + + +def get_consumer_map(ssa): + """ + Return dict from versioned blob to list of (i, j), + where i is index of consumer op, j is the index of input of that op. + """ + consumer_map = collections.defaultdict(list) + for i in range(len(ssa)): + inputs = ssa[i][0] + for j, inp in enumerate(inputs): + consumer_map[inp].append((i, j)) + return consumer_map + + +def get_params_from_init_net( + init_net: caffe2_pb2.NetDef, +) -> [Dict[str, Any], Dict[str, caffe2_pb2.DeviceOption]]: + """ + Take the output blobs from init_net by running it. + Outputs: + params: dict from blob name to numpy array + device_options: dict from blob name to the device option of its creating op + """ + + # NOTE: this assumes that the params is determined by producer op with the + # only exception be CopyGPUToCPU which is CUDA op but returns CPU tensor. + def _get_device_option(producer_op): + if producer_op.type == "CopyGPUToCPU": + return caffe2_pb2.DeviceOption() + else: + return producer_op.device_option + + with ScopedWS("__get_params_from_init_net__", is_reset=True, is_cleanup=True) as ws: + ws.RunNetOnce(init_net) + params = {b: fetch_any_blob(b) for b in init_net.external_output} + ssa, versions = core.get_ssa(init_net) + producer_map = get_producer_map(ssa) + device_options = { + b: _get_device_option(init_net.op[producer_map[(b, versions[b])][0]]) for b in init_net.external_output + } + return params, device_options + + +def _updater_raise(op, input_types, output_types): + raise RuntimeError( + "Failed to apply updater for op {} given input_types {} and" + " output_types {}".format(op, input_types, output_types) + ) + + +def _generic_status_identifier( + predict_net: caffe2_pb2.NetDef, + status_updater: Callable, + known_status: Dict[Tuple[str, int], Any], +) -> Dict[Tuple[str, int], Any]: + """ + Statically infer the status of each blob, the status can be such as device type + (CPU/GPU), layout (NCHW/NHWC), data type (float32/int8), etc. "Blob" here + is versioned blob (Tuple[str, int]) in the format compatible with ssa. + Inputs: + predict_net: the caffe2 network + status_updater: a callable, given an op and the status of its input/output, + it returns the updated status of input/output. `None` is used for + representing unknown status. + known_status: a dict containing known status, used as initialization. + Outputs: + A dict mapping from versioned blob to its status + """ + ssa, versions = core.get_ssa(predict_net) + versioned_ext_input = [(b, 0) for b in predict_net.external_input] + versioned_ext_output = [(b, versions[b]) for b in predict_net.external_output] + all_versioned_blobs = set().union(*[set(x[0] + x[1]) for x in ssa]) + + allowed_vbs = all_versioned_blobs.union(versioned_ext_input).union(versioned_ext_output) + assert all(k in allowed_vbs for k in known_status) + assert all(v is not None for v in known_status.values()) + _known_status = copy.deepcopy(known_status) + + def _check_and_update(key, value): + assert value is not None + if key in _known_status: + if not _known_status[key] == value: + raise RuntimeError( + "Confilict status for {}, existing status {}, new status {}".format(key, _known_status[key], value) + ) + _known_status[key] = value + + def _update_i(op, ssa_i): + versioned_inputs = ssa_i[0] + versioned_outputs = ssa_i[1] + + inputs_status = [_known_status.get(b, None) for b in versioned_inputs] + outputs_status = [_known_status.get(b, None) for b in versioned_outputs] + + new_inputs_status, new_outputs_status = status_updater(op, inputs_status, outputs_status) + + for versioned_blob, status in zip( + versioned_inputs + versioned_outputs, new_inputs_status + new_outputs_status + ): + if status is not None: + _check_and_update(versioned_blob, status) + + for op, ssa_i in zip(predict_net.op, ssa): + _update_i(op, ssa_i) + for op, ssa_i in zip(reversed(predict_net.op), reversed(ssa)): + _update_i(op, ssa_i) + + # NOTE: This strictly checks all the blob from predict_net must be assgined + # a known status. However sometimes it's impossible (eg. having deadend op), + # we may relax this constraint if + for k in all_versioned_blobs: + if k not in _known_status: + raise NotImplementedError( + "Can not infer the status for {}. Currently only support the case where" + " a single forward and backward pass can identify status for all blobs.".format(k) + ) + + return _known_status + + +def infer_device_type( + predict_net: caffe2_pb2.NetDef, + known_status: Dict[Tuple[str, int], Any], + device_name_style: str = "caffe2", +) -> Dict[Tuple[str, int], str]: + """Return the device type ("cpu" or "gpu"/"cuda") of each (versioned) blob""" + + assert device_name_style in ["caffe2", "pytorch"] + _CPU_STR = "cpu" + _GPU_STR = "gpu" if device_name_style == "caffe2" else "cuda" + + def _copy_cpu_to_gpu_updater(op, input_types, output_types): + if input_types[0] == _GPU_STR or output_types[0] == _CPU_STR: + _updater_raise(op, input_types, output_types) + return ([_CPU_STR], [_GPU_STR]) + + def _copy_gpu_to_cpu_updater(op, input_types, output_types): + if input_types[0] == _CPU_STR or output_types[0] == _GPU_STR: + _updater_raise(op, input_types, output_types) + return ([_GPU_STR], [_CPU_STR]) + + def _other_ops_updater(op, input_types, output_types): + non_none_types = [x for x in input_types + output_types if x is not None] + if len(non_none_types) > 0: + the_type = non_none_types[0] + if not all(x == the_type for x in non_none_types): + _updater_raise(op, input_types, output_types) + else: + the_type = None + return ([the_type for _ in op.input], [the_type for _ in op.output]) + + def _device_updater(op, *args, **kwargs): + return { + "CopyCPUToGPU": _copy_cpu_to_gpu_updater, + "CopyGPUToCPU": _copy_gpu_to_cpu_updater, + }.get( + op.type, _other_ops_updater + )(op, *args, **kwargs) + + return _generic_status_identifier(predict_net, _device_updater, known_status) + + +# ==== torch/utils_caffe2/vis.py =============================================== + + +def _modify_blob_names(ops, blob_rename_f): + ret = [] + + def _replace_list(blob_list, replaced_list): + del blob_list[:] + blob_list.extend(replaced_list) + + for x in ops: + cur = copy.deepcopy(x) + _replace_list(cur.input, list(map(blob_rename_f, cur.input))) + _replace_list(cur.output, list(map(blob_rename_f, cur.output))) + ret.append(cur) + + return ret + + +def _rename_blob(name, blob_sizes, blob_ranges): + def _list_to_str(bsize): + ret = ", ".join([str(x) for x in bsize]) + ret = "[" + ret + "]" + return ret + + ret = name + if blob_sizes is not None and name in blob_sizes: + ret += "\n" + _list_to_str(blob_sizes[name]) + if blob_ranges is not None and name in blob_ranges: + ret += "\n" + _list_to_str(blob_ranges[name]) + + return ret + + +# graph_name could not contain word 'graph' +def save_graph(net, file_name, graph_name="net", op_only=True, blob_sizes=None, blob_ranges=None): + blob_rename_f = functools.partial(_rename_blob, blob_sizes=blob_sizes, blob_ranges=blob_ranges) + return save_graph_base(net, file_name, graph_name, op_only, blob_rename_f) + + +def save_graph_base(net, file_name, graph_name="net", op_only=True, blob_rename_func=None): + graph = None + ops = net.op + if blob_rename_func is not None: + ops = _modify_blob_names(ops, blob_rename_func) + if not op_only: + graph = net_drawer.GetPydotGraph(ops, graph_name, rankdir="TB") + else: + graph = net_drawer.GetPydotGraphMinimal(ops, graph_name, rankdir="TB", minimal_dependency=True) + + try: + par_dir = os.path.dirname(file_name) + if not os.path.exists(par_dir): + os.makedirs(par_dir) + + format = os.path.splitext(os.path.basename(file_name))[-1] + if format == ".png": + graph.write_png(file_name) + elif format == ".pdf": + graph.write_pdf(file_name) + elif format == ".svg": + graph.write_svg(file_name) + else: + print("Incorrect format {}".format(format)) + except Exception as e: + print("Error when writing graph to image {}".format(e)) + + return graph + + +# ==== torch/utils_toffee/aten_to_caffe2.py ==================================== + + +def group_norm_replace_aten_with_caffe2(predict_net: caffe2_pb2.NetDef): + """ + For ONNX exported model, GroupNorm will be represented as ATen op, + this can be a drop in replacement from ATen to GroupNorm + """ + count = 0 + for op in predict_net.op: + if op.type == "ATen": + op_name = get_pb_arg_vals(op, "operator", None) # return byte in py3 + if op_name and op_name.decode() == "group_norm": + op.arg.remove(get_pb_arg(op, "operator")) + + if get_pb_arg_vali(op, "cudnn_enabled", None): + op.arg.remove(get_pb_arg(op, "cudnn_enabled")) + + num_groups = get_pb_arg_vali(op, "num_groups", None) + if num_groups is not None: + op.arg.remove(get_pb_arg(op, "num_groups")) + check_set_pb_arg(op, "group", "i", num_groups) + + op.type = "GroupNorm" + count += 1 + if count > 1: + logger.info("Replaced {} ATen operator to GroupNormOp".format(count)) + + +# ==== torch/utils_toffee/alias.py ============================================= + + +def alias(x, name, is_backward=False): + if not torch.onnx.is_in_onnx_export(): + return x + assert isinstance(x, torch.Tensor) + return torch.ops._caffe2.AliasWithName(x, name, is_backward=is_backward) + + +def fuse_alias_placeholder(predict_net, init_net): + """Remove AliasWithName placeholder and rename the input/output of it""" + # First we finish all the re-naming + for i, op in enumerate(predict_net.op): + if op.type == "AliasWithName": + assert len(op.input) == 1 + assert len(op.output) == 1 + name = get_pb_arg_vals(op, "name", None).decode() + is_backward = bool(get_pb_arg_vali(op, "is_backward", 0)) + rename_op_input(predict_net, init_net, i, 0, name, from_producer=is_backward) + rename_op_output(predict_net, i, 0, name) + + # Remove AliasWithName, should be very safe since it's a non-op + new_ops = [] + for op in predict_net.op: + if op.type != "AliasWithName": + new_ops.append(op) + else: + # safety check + assert op.input == op.output + assert op.input[0] == op.arg[0].s.decode() + del predict_net.op[:] + predict_net.op.extend(new_ops) + + +# ==== torch/utils_caffe2/graph_transform.py =================================== + + +class IllegalGraphTransformError(ValueError): + """When a graph transform function call can't be executed.""" + + +def _rename_versioned_blob_in_proto( + proto: caffe2_pb2.NetDef, + old_name: str, + new_name: str, + version: int, + ssa: List[Tuple[List[Tuple[str, int]], List[Tuple[str, int]]]], + start_versions: Dict[str, int], + end_versions: Dict[str, int], +): + """In given proto, rename all blobs with matched version""" + # Operater list + for op, i_th_ssa in zip(proto.op, ssa): + versioned_inputs, versioned_outputs = i_th_ssa + for i in range(len(op.input)): + if versioned_inputs[i] == (old_name, version): + op.input[i] = new_name + for i in range(len(op.output)): + if versioned_outputs[i] == (old_name, version): + op.output[i] = new_name + # external_input + if start_versions.get(old_name, 0) == version: + for i in range(len(proto.external_input)): + if proto.external_input[i] == old_name: + proto.external_input[i] = new_name + # external_output + if end_versions.get(old_name, 0) == version: + for i in range(len(proto.external_output)): + if proto.external_output[i] == old_name: + proto.external_output[i] = new_name + + +def rename_op_input( + predict_net: caffe2_pb2.NetDef, + init_net: caffe2_pb2.NetDef, + op_id: int, + input_id: int, + new_name: str, + from_producer: bool = False, +): + """ + Rename the op_id-th operator in predict_net, change it's input_id-th input's + name to the new_name. It also does automatic re-route and change + external_input and init_net if necessary. + - It requires the input is only consumed by this op. + - This function modifies predict_net and init_net in-place. + - When from_producer is enable, this also updates other operators that consumes + the same input. Be cautious because may trigger unintended behavior. + """ + assert isinstance(predict_net, caffe2_pb2.NetDef) + assert isinstance(init_net, caffe2_pb2.NetDef) + + init_net_ssa, init_net_versions = core.get_ssa(init_net) + predict_net_ssa, predict_net_versions = core.get_ssa(predict_net, copy.deepcopy(init_net_versions)) + + versioned_inputs, versioned_outputs = predict_net_ssa[op_id] + old_name, version = versioned_inputs[input_id] + + if from_producer: + producer_map = get_producer_map(predict_net_ssa) + if (old_name, version) not in producer_map: + raise NotImplementedError( + "Can't find producer, the input {} is probably from" + " init_net, this is not supported yet.".format(old_name) + ) + producer = producer_map[(old_name, version)] + rename_op_output(predict_net, producer[0], producer[1], new_name) + return + + def contain_targets(op_ssa): + return (old_name, version) in op_ssa[0] + + is_consumer = [contain_targets(op_ssa) for op_ssa in predict_net_ssa] + if sum(is_consumer) > 1: + raise IllegalGraphTransformError( + ( + "Input '{}' of operator(#{}) are consumed by other ops, please use" + + " rename_op_output on the producer instead. Offending op: \n{}" + ).format(old_name, op_id, predict_net.op[op_id]) + ) + + # update init_net + _rename_versioned_blob_in_proto(init_net, old_name, new_name, version, init_net_ssa, {}, init_net_versions) + # update predict_net + _rename_versioned_blob_in_proto( + predict_net, + old_name, + new_name, + version, + predict_net_ssa, + init_net_versions, + predict_net_versions, + ) + + +def rename_op_output(predict_net: caffe2_pb2.NetDef, op_id: int, output_id: int, new_name: str): + """ + Rename the op_id-th operator in predict_net, change it's output_id-th input's + name to the new_name. It also does automatic re-route and change + external_output and if necessary. + - It allows multiple consumers of its output. + - This function modifies predict_net in-place, doesn't need init_net. + """ + assert isinstance(predict_net, caffe2_pb2.NetDef) + + ssa, blob_versions = core.get_ssa(predict_net) + + versioned_inputs, versioned_outputs = ssa[op_id] + old_name, version = versioned_outputs[output_id] + + # update predict_net + _rename_versioned_blob_in_proto(predict_net, old_name, new_name, version, ssa, {}, blob_versions) + + +def get_sub_graph_external_input_output( + predict_net: caffe2_pb2.NetDef, sub_graph_op_indices: List[int] +) -> Tuple[List[Tuple[str, int]], List[Tuple[str, int]]]: + """ + Return the list of external input/output of sub-graph, + each element is tuple of the name and corresponding version in predict_net. + + external input/output is defined the same way as caffe2 NetDef. + """ + ssa, versions = core.get_ssa(predict_net) + + all_inputs = [] + all_outputs = [] + for op_id in sub_graph_op_indices: + all_inputs += [inp for inp in ssa[op_id][0] if inp not in all_inputs] + all_outputs += list(ssa[op_id][1]) # ssa output won't repeat + + # for versioned blobs, external inputs are just those blob in all_inputs + # but not in all_outputs + ext_inputs = [inp for inp in all_inputs if inp not in all_outputs] + + # external outputs are essentially outputs of this subgraph that are used + # outside of this sub-graph (including predict_net.external_output) + all_other_inputs = sum( + (ssa[i][0] for i in range(len(ssa)) if i not in sub_graph_op_indices), + [(outp, versions[outp]) for outp in predict_net.external_output], + ) + ext_outputs = [outp for outp in all_outputs if outp in set(all_other_inputs)] + + return ext_inputs, ext_outputs + + +class DiGraph: + """A DAG representation of caffe2 graph, each vertice is a versioned blob.""" + + def __init__(self): + self.vertices = set() + self.graph = collections.defaultdict(list) + + def add_edge(self, u, v): + self.graph[u].append(v) + self.vertices.add(u) + self.vertices.add(v) + + # grab from https://www.geeksforgeeks.org/find-paths-given-source-destination/ + def get_all_paths(self, s, d): + visited = {k: False for k in self.vertices} + path = [] + all_paths = [] + + def _get_all_paths_util(graph, u, d, visited, path): + visited[u] = True + path.append(u) + if u == d: + all_paths.append(copy.deepcopy(path)) + else: + for i in graph[u]: + if not visited[i]: + _get_all_paths_util(graph, i, d, visited, path) + path.pop() + visited[u] = False + + _get_all_paths_util(self.graph, s, d, visited, path) + return all_paths + + @staticmethod + def from_ssa(ssa): + graph = DiGraph() + for op_id in range(len(ssa)): + for inp in ssa[op_id][0]: + for outp in ssa[op_id][1]: + graph.add_edge(inp, outp) + return graph + + +def _get_dependency_chain(ssa, versioned_target, versioned_source): + """ + Return the index list of relevant operator to produce target blob from source blob, + if there's no dependency, return empty list. + """ + + # finding all paths between nodes can be O(N!), thus we can only search + # in the subgraph using the op starting from the first consumer of source blob + # to the producer of the target blob. + consumer_map = get_consumer_map(ssa) + producer_map = get_producer_map(ssa) + start_op = min(x[0] for x in consumer_map[versioned_source]) - 15 + end_op = producer_map[versioned_target][0] + 15 if versioned_target in producer_map else start_op + sub_graph_ssa = ssa[start_op : end_op + 1] + if len(sub_graph_ssa) > 30: + logger.warning( + "Subgraph bebetween {} and {} is large (from op#{} to op#{}), it" + " might take non-trival time to find all paths between them.".format( + versioned_source, versioned_target, start_op, end_op + ) + ) + + dag = DiGraph.from_ssa(sub_graph_ssa) + paths = dag.get_all_paths(versioned_source, versioned_target) # include two ends + ops_in_paths = [[producer_map[blob][0] for blob in path[1:]] for path in paths] + return sorted(set().union(*[set(ops) for ops in ops_in_paths])) + + +def identify_reshape_sub_graph(predict_net: caffe2_pb2.NetDef) -> List[List[int]]: + """ + Idenfity the reshape sub-graph in a protobuf. + The reshape sub-graph is defined as matching the following pattern: + + (input_blob) -> Op_1 -> ... -> Op_N -> (new_shape) -─┐ + └-------------------------------------------> Reshape -> (output_blob) + + Return: + List of sub-graphs, each sub-graph is represented as a list of indices + of the relavent ops, [Op_1, Op_2, ..., Op_N, Reshape] + """ + + ssa, _ = core.get_ssa(predict_net) + + ret = [] + for i, op in enumerate(predict_net.op): + if op.type == "Reshape": + assert len(op.input) == 2 + input_ssa = ssa[i][0] + data_source = input_ssa[0] + shape_source = input_ssa[1] + op_indices = _get_dependency_chain(ssa, shape_source, data_source) + ret.append(op_indices + [i]) + return ret + + +def remove_reshape_for_fc(predict_net, params): + """ + In PyTorch nn.Linear has to take 2D tensor, this often leads to reshape + a 4D tensor to 2D by calling .view(). However this (dynamic) reshaping + doesn't work well with ONNX and Int8 tools, and cause using extra + ops (eg. ExpandDims) that might not be available on mobile. + Luckily Caffe2 supports 4D tensor for FC, so we can remove those reshape + after exporting ONNX model. + """ + from caffe2.python import core + + # find all reshape sub-graph that can be removed, which is now all Reshape + # sub-graph whose output is only consumed by FC. + # TODO: to make it safer, we may need the actually value to better determine + # if a Reshape before FC is removable. + reshape_sub_graphs = identify_reshape_sub_graph(predict_net) + sub_graphs_to_remove = [] + for reshape_sub_graph in reshape_sub_graphs: + reshape_op_id = reshape_sub_graph[-1] + assert predict_net.op[reshape_op_id].type == "Reshape" + ssa, _ = core.get_ssa(predict_net) + reshape_output = ssa[reshape_op_id][1][0] + consumers = [i for i in range(len(ssa)) if reshape_output in ssa[i][0]] + if all(predict_net.op[consumer].type == "FC" for consumer in consumers): + # safety check if the sub-graph is isolated, for this reshape sub-graph, + # it means it has one non-param external input and one external output. + ext_inputs, ext_outputs = get_sub_graph_external_input_output(predict_net, reshape_sub_graph) + non_params_ext_inputs = [inp for inp in ext_inputs if inp[1] != 0] + if len(non_params_ext_inputs) == 1 and len(ext_outputs) == 1: + sub_graphs_to_remove.append(reshape_sub_graph) + + # perform removing subgraph by: + # 1: rename the Reshape's output to its input, then the graph can be + # seen as in-place itentify, meaning whose external input/output are the same. + # 2: simply remove those ops. + remove_op_ids = [] + params_to_remove = [] + for sub_graph in sub_graphs_to_remove: + logger.info( + "Remove Reshape sub-graph:\n{}".format( + "".join(["(#{:>4})\n{}".format(i, predict_net.op[i]) for i in sub_graph]) + ) + ) + reshape_op_id = sub_graph[-1] + new_reshap_output = predict_net.op[reshape_op_id].input[0] + rename_op_output(predict_net, reshape_op_id, 0, new_reshap_output) + ext_inputs, ext_outputs = get_sub_graph_external_input_output(predict_net, sub_graph) + non_params_ext_inputs = [inp for inp in ext_inputs if inp[1] != 0] + params_ext_inputs = [inp for inp in ext_inputs if inp[1] == 0] + assert len(non_params_ext_inputs) == 1 and len(ext_outputs) == 1 + assert ext_outputs[0][0] == non_params_ext_inputs[0][0] + assert ext_outputs[0][1] == non_params_ext_inputs[0][1] + 1 + remove_op_ids.extend(sub_graph) + params_to_remove.extend(params_ext_inputs) + + predict_net = copy.deepcopy(predict_net) + new_ops = [op for i, op in enumerate(predict_net.op) if i not in remove_op_ids] + del predict_net.op[:] + predict_net.op.extend(new_ops) + for versioned_params in params_to_remove: + name = versioned_params[0] + logger.info("Remove params: {} from init_net and predict_net.external_input".format(name)) + del params[name] + predict_net.external_input.remove(name) + + return predict_net, params + + +def fuse_copy_between_cpu_and_gpu(predict_net: caffe2_pb2.NetDef): + """ + In-place fuse extra copy ops between cpu/gpu for the following case: + a -CopyAToB-> b -CopyBToA> c1 -NextOp1-> d1 + -CopyBToA> c2 -NextOp2-> d2 + The fused network will look like: + a -NextOp1-> d1 + -NextOp2-> d2 + """ + + _COPY_OPS = ["CopyCPUToGPU", "CopyGPUToCPU"] + + def _fuse_once(predict_net): + ssa, blob_versions = core.get_ssa(predict_net) + consumer_map = get_consumer_map(ssa) + versioned_external_output = [(name, blob_versions[name]) for name in predict_net.external_output] + + for op_id, op in enumerate(predict_net.op): + if op.type in _COPY_OPS: + fw_copy_versioned_output = ssa[op_id][1][0] + consumer_ids = [x[0] for x in consumer_map[fw_copy_versioned_output]] + reverse_op_type = _COPY_OPS[1 - _COPY_OPS.index(op.type)] + + is_fusable = ( + len(consumer_ids) > 0 + and fw_copy_versioned_output not in versioned_external_output + and all( + predict_net.op[_op_id].type == reverse_op_type + and ssa[_op_id][1][0] not in versioned_external_output + for _op_id in consumer_ids + ) + ) + + if is_fusable: + for rv_copy_op_id in consumer_ids: + # making each NextOp uses "a" directly and removing Copy ops + rs_copy_versioned_output = ssa[rv_copy_op_id][1][0] + next_op_id, inp_id = consumer_map[rs_copy_versioned_output][0] + predict_net.op[next_op_id].input[inp_id] = op.input[0] + # remove CopyOps + new_ops = [op for i, op in enumerate(predict_net.op) if i != op_id and i not in consumer_ids] + del predict_net.op[:] + predict_net.op.extend(new_ops) + return True + + return False + + # _fuse_once returns False is nothing can be fused + while _fuse_once(predict_net): + pass + + +def remove_dead_end_ops(net_def: caffe2_pb2.NetDef): + """remove ops if its output is not used or not in external_output""" + ssa, versions = core.get_ssa(net_def) + versioned_external_output = [(name, versions[name]) for name in net_def.external_output] + consumer_map = get_consumer_map(ssa) + removed_op_ids = set() + + def _is_dead_end(versioned_blob): + return not ( + versioned_blob in versioned_external_output + or ( + len(consumer_map[versioned_blob]) > 0 + and all(x[0] not in removed_op_ids for x in consumer_map[versioned_blob]) + ) + ) + + for i, ssa_i in reversed(list(enumerate(ssa))): + versioned_outputs = ssa_i[1] + if all(_is_dead_end(outp) for outp in versioned_outputs): + removed_op_ids.add(i) + + # simply removing those deadend ops should have no effect to external_output + new_ops = [op for i, op in enumerate(net_def.op) if i not in removed_op_ids] + del net_def.op[:] + net_def.op.extend(new_ops) diff --git a/detectron2/export/torchscript.py b/detectron2/export/torchscript.py new file mode 100644 index 0000000000000000000000000000000000000000..106d6130a987af9c1fe98b8ee118aaafa59c4199 --- /dev/null +++ b/detectron2/export/torchscript.py @@ -0,0 +1,131 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import os + +import torch + +from detectron2.utils.file_io import PathManager + +from .torchscript_patch import freeze_training_mode, patch_instances + +__all__ = ["scripting_with_instances", "dump_torchscript_IR"] + + +def scripting_with_instances(model, fields): + """ + Run :func:`torch.jit.script` on a model that uses the :class:`Instances` class. Since + attributes of :class:`Instances` are "dynamically" added in eager mode，it is difficult + for scripting to support it out of the box. This function is made to support scripting + a model that uses :class:`Instances`. It does the following: + + 1. Create a scriptable ``new_Instances`` class which behaves similarly to ``Instances``, + but with all attributes been "static". + The attributes need to be statically declared in the ``fields`` argument. + 2. Register ``new_Instances``, and force scripting compiler to + use it when trying to compile ``Instances``. + + After this function, the process will be reverted. User should be able to script another model + using different fields. + + Example: + Assume that ``Instances`` in the model consist of two attributes named + ``proposal_boxes`` and ``objectness_logits`` with type :class:`Boxes` and + :class:`Tensor` respectively during inference. You can call this function like: + :: + fields = {"proposal_boxes": Boxes, "objectness_logits": torch.Tensor} + torchscipt_model = scripting_with_instances(model, fields) + + Note: + It only support models in evaluation mode. + + Args: + model (nn.Module): The input model to be exported by scripting. + fields (Dict[str, type]): Attribute names and corresponding type that + ``Instances`` will use in the model. Note that all attributes used in ``Instances`` + need to be added, regardless of whether they are inputs/outputs of the model. + Data type not defined in detectron2 is not supported for now. + + Returns: + torch.jit.ScriptModule: the model in torchscript format + """ + assert not model.training, "Currently we only support exporting models in evaluation mode to torchscript" + + with freeze_training_mode(model), patch_instances(fields): + scripted_model = torch.jit.script(model) + return scripted_model + + +# alias for old name +export_torchscript_with_instances = scripting_with_instances + + +def dump_torchscript_IR(model, dir): + """ + Dump IR of a TracedModule/ScriptModule/Function in various format (code, graph, + inlined graph). Useful for debugging. + + Args: + model (TracedModule/ScriptModule/ScriptFUnction): traced or scripted module + dir (str): output directory to dump files. + """ + dir = os.path.expanduser(dir) + PathManager.mkdirs(dir) + + def _get_script_mod(mod): + if isinstance(mod, torch.jit.TracedModule): + return mod._actual_script_module + return mod + + # Dump pretty-printed code: https://pytorch.org/docs/stable/jit.html#inspecting-code + with PathManager.open(os.path.join(dir, "model_ts_code.txt"), "w") as f: + + def get_code(mod): + # Try a few ways to get code using private attributes. + try: + # This contains more information than just `mod.code` + return _get_script_mod(mod)._c.code + except AttributeError: + pass + try: + return mod.code + except AttributeError: + return None + + def dump_code(prefix, mod): + code = get_code(mod) + name = prefix or "root model" + if code is None: + f.write(f"Could not found code for {name} (type={mod.original_name})\n") + f.write("\n") + else: + f.write(f"\nCode for {name}, type={mod.original_name}:\n") + f.write(code) + f.write("\n") + f.write("-" * 80) + + for name, m in mod.named_children(): + dump_code(prefix + "." + name, m) + + if isinstance(model, torch.jit.ScriptFunction): + f.write(get_code(model)) + else: + dump_code("", model) + + def _get_graph(model): + try: + # Recursively dump IR of all modules + return _get_script_mod(model)._c.dump_to_str(True, False, False) + except AttributeError: + return model.graph.str() + + with PathManager.open(os.path.join(dir, "model_ts_IR.txt"), "w") as f: + f.write(_get_graph(model)) + + # Dump IR of the entire graph (all submodules inlined) + with PathManager.open(os.path.join(dir, "model_ts_IR_inlined.txt"), "w") as f: + f.write(str(model.inlined_graph)) + + if not isinstance(model, torch.jit.ScriptFunction): + # Dump the model structure in pytorch style + with PathManager.open(os.path.join(dir, "model.txt"), "w") as f: + f.write(str(model)) diff --git a/detectron2/export/torchscript_patch.py b/detectron2/export/torchscript_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..237114027a1efdf5eb78956fa144d03e332774ff --- /dev/null +++ b/detectron2/export/torchscript_patch.py @@ -0,0 +1,363 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import os +import sys +import tempfile +from contextlib import ExitStack, contextmanager +from copy import deepcopy +from unittest import mock + +import torch +from torch import nn + +# need some explicit imports due to https://github.com/pytorch/pytorch/issues/38964 +import detectron2 # noqa F401 +from detectron2.structures import Boxes, Instances +from detectron2.utils.env import _import_file + +_counter = 0 + + +def _clear_jit_cache(): + from torch.jit._recursive import concrete_type_store + from torch.jit._state import _jit_caching_layer + + concrete_type_store.type_store.clear() # for modules + _jit_caching_layer.clear() # for free functions + + +def _add_instances_conversion_methods(newInstances): + """ + Add from_instances methods to the scripted Instances class. + """ + cls_name = newInstances.__name__ + + @torch.jit.unused + def from_instances(instances: Instances): + """ + Create scripted Instances from original Instances + """ + fields = instances.get_fields() + image_size = instances.image_size + ret = newInstances(image_size) + for name, val in fields.items(): + assert hasattr(ret, f"_{name}"), f"No attribute named {name} in {cls_name}" + setattr(ret, name, deepcopy(val)) + return ret + + newInstances.from_instances = from_instances + + +@contextmanager +def patch_instances(fields): + """ + A contextmanager, under which the Instances class in detectron2 is replaced + by a statically-typed scriptable class, defined by `fields`. + See more in `scripting_with_instances`. + """ + + with tempfile.TemporaryDirectory(prefix="detectron2") as dir, tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", suffix=".py", dir=dir, delete=False + ) as f: + try: + # Objects that use Instances should not reuse previously-compiled + # results in cache, because `Instances` could be a new class each time. + _clear_jit_cache() + + cls_name, s = _gen_instance_module(fields) + f.write(s) + f.flush() + f.close() + + module = _import(f.name) + new_instances = getattr(module, cls_name) + _ = torch.jit.script(new_instances) + # let torchscript think Instances was scripted already + Instances.__torch_script_class__ = True + # let torchscript find new_instances when looking for the jit type of Instances + Instances._jit_override_qualname = torch._jit_internal._qualified_name(new_instances) + + _add_instances_conversion_methods(new_instances) + yield new_instances + finally: + try: + del Instances.__torch_script_class__ + del Instances._jit_override_qualname + except AttributeError: + pass + sys.modules.pop(module.__name__) + + +def _gen_instance_class(fields): + """ + Args: + fields (dict[name: type]) + """ + + class _FieldType: + def __init__(self, name, type_): + assert isinstance(name, str), f"Field name must be str, got {name}" + self.name = name + self.type_ = type_ + self.annotation = f"{type_.__module__}.{type_.__name__}" + + fields = [_FieldType(k, v) for k, v in fields.items()] + + def indent(level, s): + return " " * 4 * level + s + + lines = [] + + global _counter + _counter += 1 + + cls_name = "ScriptedInstances{}".format(_counter) + + field_names = tuple(x.name for x in fields) + extra_args = ", ".join([f"{f.name}: Optional[{f.annotation}] = None" for f in fields]) + lines.append(f""" +class {cls_name}: + def __init__(self, image_size: Tuple[int, int], {extra_args}): + self.image_size = image_size + self._field_names = {field_names} +""") + + for f in fields: + lines.append(indent(2, f"self._{f.name} = torch.jit.annotate(Optional[{f.annotation}], {f.name})")) + + for f in fields: + lines.append(f""" + @property + def {f.name}(self) -> {f.annotation}: + # has to use a local for type refinement + # https://pytorch.org/docs/stable/jit_language_reference.html#optional-type-refinement + t = self._{f.name} + assert t is not None, "{f.name} is None and cannot be accessed!" + return t + + @{f.name}.setter + def {f.name}(self, value: {f.annotation}) -> None: + self._{f.name} = value +""") + + # support method `__len__` + lines.append(""" + def __len__(self) -> int: +""") + for f in fields: + lines.append(f""" + t = self._{f.name} + if t is not None: + return len(t) +""") + lines.append(""" + raise NotImplementedError("Empty Instances does not support __len__!") +""") + + # support method `has` + lines.append(""" + def has(self, name: str) -> bool: +""") + for f in fields: + lines.append(f""" + if name == "{f.name}": + return self._{f.name} is not None +""") + lines.append(""" + return False +""") + + # support method `to` + none_args = ", None" * len(fields) + lines.append(f""" + def to(self, device: torch.device) -> "{cls_name}": + ret = {cls_name}(self.image_size{none_args}) +""") + for f in fields: + if hasattr(f.type_, "to"): + lines.append(f""" + t = self._{f.name} + if t is not None: + ret._{f.name} = t.to(device) +""") + else: + # For now, ignore fields that cannot be moved to devices. + # Maybe can support other tensor-like classes (e.g. __torch_function__) + pass + lines.append(""" + return ret +""") + + # support method `getitem` + none_args = ", None" * len(fields) + lines.append(f""" + def __getitem__(self, item) -> "{cls_name}": + ret = {cls_name}(self.image_size{none_args}) +""") + for f in fields: + lines.append(f""" + t = self._{f.name} + if t is not None: + ret._{f.name} = t[item] +""") + lines.append(""" + return ret +""") + + # support method `cat` + # this version does not contain checks that all instances have same size and fields + none_args = ", None" * len(fields) + lines.append(f""" + def cat(self, instances: List["{cls_name}"]) -> "{cls_name}": + ret = {cls_name}(self.image_size{none_args}) +""") + for f in fields: + lines.append(f""" + t = self._{f.name} + if t is not None: + values: List[{f.annotation}] = [x.{f.name} for x in instances] + if torch.jit.isinstance(t, torch.Tensor): + ret._{f.name} = torch.cat(values, dim=0) + else: + ret._{f.name} = t.cat(values) +""") + lines.append(""" + return ret""") + + # support method `get_fields()` + lines.append(""" + def get_fields(self) -> Dict[str, Tensor]: + ret = {} + """) + for f in fields: + if f.type_ == Boxes: + stmt = "t.tensor" + elif f.type_ == torch.Tensor: + stmt = "t" + else: + stmt = f'assert False, "unsupported type {str(f.type_)}"' + lines.append(f""" + t = self._{f.name} + if t is not None: + ret["{f.name}"] = {stmt} + """) + lines.append(""" + return ret""") + return cls_name, os.linesep.join(lines) + + +def _gen_instance_module(fields): + # TODO: find a more automatic way to enable import of other classes + s = """ +from copy import deepcopy +import torch +from torch import Tensor +import typing +from typing import * + +import detectron2 +from detectron2.structures import Boxes, Instances + +""" + + cls_name, cls_def = _gen_instance_class(fields) + s += cls_def + return cls_name, s + + +def _import(path): + return _import_file("{}{}".format(sys.modules[__name__].__name__, _counter), path, make_importable=True) + + +@contextmanager +def patch_builtin_len(modules=()): + """ + Patch the builtin len() function of a few detectron2 modules + to use __len__ instead, because __len__ does not convert values to + integers and therefore is friendly to tracing. + + Args: + modules (list[stsr]): names of extra modules to patch len(), in + addition to those in detectron2. + """ + + def _new_len(obj): + return obj.__len__() + + with ExitStack() as stack: + MODULES = [ + "detectron2.modeling.roi_heads.fast_rcnn", + "detectron2.modeling.roi_heads.mask_head", + "detectron2.modeling.roi_heads.keypoint_head", + ] + list(modules) + ctxs = [stack.enter_context(mock.patch(mod + ".len")) for mod in MODULES] + for m in ctxs: + m.side_effect = _new_len + yield + + +def patch_nonscriptable_classes(): + """ + Apply patches on a few nonscriptable detectron2 classes. + Should not have side-effects on eager usage. + """ + # __prepare_scriptable__ can also be added to models for easier maintenance. + # But it complicates the clean model code. + + from detectron2.modeling.backbone import FPN, ResNet + + # Due to https://github.com/pytorch/pytorch/issues/36061, + # we change backbone to use ModuleList for scripting. + # (note: this changes param names in state_dict) + + def prepare_resnet(self): + ret = deepcopy(self) + ret.stages = nn.ModuleList(ret.stages) + for k in self.stage_names: + delattr(ret, k) + return ret + + ResNet.__prepare_scriptable__ = prepare_resnet + + def prepare_fpn(self): + ret = deepcopy(self) + ret.lateral_convs = nn.ModuleList(ret.lateral_convs) + ret.output_convs = nn.ModuleList(ret.output_convs) + for name, _ in self.named_children(): + if name.startswith("fpn_"): + delattr(ret, name) + return ret + + FPN.__prepare_scriptable__ = prepare_fpn + + # Annotate some attributes to be constants for the purpose of scripting, + # even though they are not constants in eager mode. + from detectron2.modeling.roi_heads import StandardROIHeads + + if hasattr(StandardROIHeads, "__annotations__"): + # copy first to avoid editing annotations of base class + StandardROIHeads.__annotations__ = deepcopy(StandardROIHeads.__annotations__) + StandardROIHeads.__annotations__["mask_on"] = torch.jit.Final[bool] + StandardROIHeads.__annotations__["keypoint_on"] = torch.jit.Final[bool] + + +# These patches are not supposed to have side-effects. +patch_nonscriptable_classes() + + +@contextmanager +def freeze_training_mode(model): + """ + A context manager that annotates the "training" attribute of every submodule + to constant, so that the training codepath in these modules can be + meta-compiled away. Upon exiting, the annotations are reverted. + """ + classes = {type(x) for x in model.modules()} + # __constants__ is the old way to annotate constants and not compatible + # with __annotations__ . + classes = {x for x in classes if not hasattr(x, "__constants__")} + for cls in classes: + cls.__annotations__["training"] = torch.jit.Final[bool] + yield + for cls in classes: + cls.__annotations__["training"] = bool diff --git a/detectron2/layers/__init__.py b/detectron2/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b793778ec4c08ed1b762afca376acc07a2cb963b --- /dev/null +++ b/detectron2/layers/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .aspp import ASPP +from .batch_norm import CycleBatchNormList, FrozenBatchNorm2d, NaiveSyncBatchNorm, get_norm +from .blocks import CNNBlockBase, DepthwiseSeparableConv2d +from .deform_conv import DeformConv, ModulatedDeformConv +from .losses import ciou_loss, diou_loss +from .mask_ops import paste_masks_in_image +from .nms import batched_nms, batched_nms_rotated, nms, nms_rotated +from .roi_align import ROIAlign, roi_align +from .roi_align_rotated import ROIAlignRotated, roi_align_rotated +from .shape_spec import ShapeSpec +from .wrappers import ( + BatchNorm2d, + Conv2d, + ConvTranspose2d, + Linear, + cat, + cross_entropy, + empty_input_loss_func_wrapper, + interpolate, + move_device_like, + nonzero_tuple, + shapes_to_tensor, +) + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/detectron2/layers/aspp.py b/detectron2/layers/aspp.py new file mode 100644 index 0000000000000000000000000000000000000000..79e249cf38ecbabee7c20db2752ad420ded984fc --- /dev/null +++ b/detectron2/layers/aspp.py @@ -0,0 +1,145 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from copy import deepcopy + +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from .batch_norm import get_norm +from .blocks import DepthwiseSeparableConv2d +from .wrappers import Conv2d + + +class ASPP(nn.Module): + """ + Atrous Spatial Pyramid Pooling (ASPP). + """ + + def __init__( + self, + in_channels, + out_channels, + dilations, + *, + norm, + activation, + pool_kernel_size=None, + dropout: float = 0.0, + use_depthwise_separable_conv=False, + ): + """ + Args: + in_channels (int): number of input channels for ASPP. + out_channels (int): number of output channels. + dilations (list): a list of 3 dilations in ASPP. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. norm is + applied to all conv layers except the conv following + global average pooling. + activation (callable): activation function. + pool_kernel_size (tuple, list): the average pooling size (kh, kw) + for image pooling layer in ASPP. If set to None, it always + performs global average pooling. If not None, it must be + divisible by the shape of inputs in forward(). It is recommended + to use a fixed input feature size in training, and set this + option to match this size, so that it performs global average + pooling in training, and the size of the pooling window stays + consistent in inference. + dropout (float): apply dropout on the output of ASPP. It is used in + the official DeepLab implementation with a rate of 0.1: + https://github.com/tensorflow/models/blob/21b73d22f3ed05b650e85ac50849408dd36de32e/research/deeplab/model.py#L532 # noqa + use_depthwise_separable_conv (bool): use DepthwiseSeparableConv2d + for 3x3 convs in ASPP, proposed in :paper:`DeepLabV3+`. + """ + super(ASPP, self).__init__() + assert len(dilations) == 3, "ASPP expects 3 dilations, got {}".format(len(dilations)) + self.pool_kernel_size = pool_kernel_size + self.dropout = dropout + use_bias = norm == "" + self.convs = nn.ModuleList() + # conv 1x1 + self.convs.append( + Conv2d( + in_channels, + out_channels, + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + activation=deepcopy(activation), + ) + ) + weight_init.c2_xavier_fill(self.convs[-1]) + # atrous convs + for dilation in dilations: + if use_depthwise_separable_conv: + self.convs.append( + DepthwiseSeparableConv2d( + in_channels, + out_channels, + kernel_size=3, + padding=dilation, + dilation=dilation, + norm1=norm, + activation1=deepcopy(activation), + norm2=norm, + activation2=deepcopy(activation), + ) + ) + else: + self.convs.append( + Conv2d( + in_channels, + out_channels, + kernel_size=3, + padding=dilation, + dilation=dilation, + bias=use_bias, + norm=get_norm(norm, out_channels), + activation=deepcopy(activation), + ) + ) + weight_init.c2_xavier_fill(self.convs[-1]) + # image pooling + # We do not add BatchNorm because the spatial resolution is 1x1, + # the original TF implementation has BatchNorm. + if pool_kernel_size is None: + image_pooling = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + Conv2d(in_channels, out_channels, 1, bias=True, activation=deepcopy(activation)), + ) + else: + image_pooling = nn.Sequential( + nn.AvgPool2d(kernel_size=pool_kernel_size, stride=1), + Conv2d(in_channels, out_channels, 1, bias=True, activation=deepcopy(activation)), + ) + weight_init.c2_xavier_fill(image_pooling[1]) + self.convs.append(image_pooling) + + self.project = Conv2d( + 5 * out_channels, + out_channels, + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + activation=deepcopy(activation), + ) + weight_init.c2_xavier_fill(self.project) + + def forward(self, x): + size = x.shape[-2:] + if self.pool_kernel_size is not None: + if size[0] % self.pool_kernel_size[0] or size[1] % self.pool_kernel_size[1]: + raise ValueError( + "`pool_kernel_size` must be divisible by the shape of inputs. " + "Input size: {} `pool_kernel_size`: {}".format(size, self.pool_kernel_size) + ) + res = [] + for conv in self.convs: + res.append(conv(x)) + res[-1] = F.interpolate(res[-1], size=size, mode="bilinear", align_corners=False) + res = torch.cat(res, dim=1) + res = self.project(res) + res = F.dropout(res, self.dropout, training=self.training) if self.dropout > 0 else res + return res diff --git a/detectron2/layers/batch_norm.py b/detectron2/layers/batch_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..915fd4efbb720c6cff768fa2893dee195ac5a7be --- /dev/null +++ b/detectron2/layers/batch_norm.py @@ -0,0 +1,298 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +import torch.distributed as dist +from fvcore.nn.distributed import differentiable_all_reduce +from torch import nn +from torch.nn import functional as F + +from detectron2.utils import comm, env + +from .wrappers import BatchNorm2d + + +class FrozenBatchNorm2d(nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + It contains non-trainable buffers called + "weight" and "bias", "running_mean", "running_var", + initialized to perform identity transformation. + + The pre-trained backbone models from Caffe2 only contain "weight" and "bias", + which are computed from the original four parameters of BN. + The affine transform `x * weight + bias` will perform the equivalent + computation of `(x - running_mean) / sqrt(running_var) * weight + bias`. + When loading a backbone model from Caffe2, "running_mean" and "running_var" + will be left unchanged as identity transformation. + + Other pre-trained backbone models may contain all 4 parameters. + + The forward is implemented by `F.batch_norm(..., training=False)`. + """ + + _version = 3 + + def __init__(self, num_features, eps=1e-5): + super().__init__() + self.num_features = num_features + self.eps = eps + self.register_buffer("weight", torch.ones(num_features)) + self.register_buffer("bias", torch.zeros(num_features)) + self.register_buffer("running_mean", torch.zeros(num_features)) + self.register_buffer("running_var", torch.ones(num_features) - eps) + + def forward(self, x): + if x.requires_grad: + # When gradients are needed, F.batch_norm will use extra memory + # because its backward op computes gradients for weight/bias as well. + scale = self.weight * (self.running_var + self.eps).rsqrt() + bias = self.bias - self.running_mean * scale + scale = scale.reshape(1, -1, 1, 1) + bias = bias.reshape(1, -1, 1, 1) + out_dtype = x.dtype # may be half + return x * scale.to(out_dtype) + bias.to(out_dtype) + else: + # When gradients are not needed, F.batch_norm is a single fused op + # and provide more optimization opportunities. + return F.batch_norm( + x, + self.running_mean, + self.running_var, + self.weight, + self.bias, + training=False, + eps=self.eps, + ) + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + version = local_metadata.get("version", None) + + if version is None or version < 2: + # No running_mean/var in early versions + # This will silent the warnings + if prefix + "running_mean" not in state_dict: + state_dict[prefix + "running_mean"] = torch.zeros_like(self.running_mean) + if prefix + "running_var" not in state_dict: + state_dict[prefix + "running_var"] = torch.ones_like(self.running_var) + + super()._load_from_state_dict( + state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ) + + def __repr__(self): + return "FrozenBatchNorm2d(num_features={}, eps={})".format(self.num_features, self.eps) + + @classmethod + def convert_frozen_batchnorm(cls, module): + """ + Convert all BatchNorm/SyncBatchNorm in module into FrozenBatchNorm. + + Args: + module (torch.nn.Module): + + Returns: + If module is BatchNorm/SyncBatchNorm, returns a new module. + Otherwise, in-place convert module and return it. + + Similar to convert_sync_batchnorm in + https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/batchnorm.py + """ + bn_module = nn.modules.batchnorm + bn_module = (bn_module.BatchNorm2d, bn_module.SyncBatchNorm) + res = module + if isinstance(module, bn_module): + res = cls(module.num_features) + if module.affine: + res.weight.data = module.weight.data.clone().detach() + res.bias.data = module.bias.data.clone().detach() + res.running_mean.data = module.running_mean.data + res.running_var.data = module.running_var.data + res.eps = module.eps + else: + for name, child in module.named_children(): + new_child = cls.convert_frozen_batchnorm(child) + if new_child is not child: + res.add_module(name, new_child) + return res + + +def get_norm(norm, out_channels): + """ + Args: + norm (str or callable): either one of BN, SyncBN, FrozenBN, GN; + or a callable that takes a channel number and returns + the normalization layer as a nn.Module. + + Returns: + nn.Module or None: the normalization layer + """ + if norm is None: + return None + if isinstance(norm, str): + if len(norm) == 0: + return None + norm = { + "BN": BatchNorm2d, + # Fixed in https://github.com/pytorch/pytorch/pull/36382 + "SyncBN": NaiveSyncBatchNorm if env.TORCH_VERSION <= (1, 5) else nn.SyncBatchNorm, + "FrozenBN": FrozenBatchNorm2d, + "GN": lambda channels: nn.GroupNorm(32, channels), + # for debugging: + "nnSyncBN": nn.SyncBatchNorm, + "naiveSyncBN": NaiveSyncBatchNorm, + # expose stats_mode N as an option to caller, required for zero-len inputs + "naiveSyncBN_N": lambda channels: NaiveSyncBatchNorm(channels, stats_mode="N"), + "LN": lambda channels: LayerNorm(channels), + }[norm] + return norm(out_channels) + + +class NaiveSyncBatchNorm(BatchNorm2d): + """ + In PyTorch<=1.5, ``nn.SyncBatchNorm`` has incorrect gradient + when the batch size on each worker is different. + (e.g., when scale augmentation is used, or when it is applied to mask head). + + This is a slower but correct alternative to `nn.SyncBatchNorm`. + + Note: + There isn't a single definition of Sync BatchNorm. + + When ``stats_mode==""``, this module computes overall statistics by using + statistics of each worker with equal weight. The result is true statistics + of all samples (as if they are all on one worker) only when all workers + have the same (N, H, W). This mode does not support inputs with zero batch size. + + When ``stats_mode=="N"``, this module computes overall statistics by weighting + the statistics of each worker by their ``N``. The result is true statistics + of all samples (as if they are all on one worker) only when all workers + have the same (H, W). It is slower than ``stats_mode==""``. + + Even though the result of this module may not be the true statistics of all samples, + it may still be reasonable because it might be preferrable to assign equal weights + to all workers, regardless of their (H, W) dimension, instead of putting larger weight + on larger images. From preliminary experiments, little difference is found between such + a simplified implementation and an accurate computation of overall mean & variance. + """ + + def __init__(self, *args, stats_mode="", **kwargs): + super().__init__(*args, **kwargs) + assert stats_mode in ["", "N"] + self._stats_mode = stats_mode + + def forward(self, input): + if comm.get_world_size() == 1 or not self.training: + return super().forward(input) + + B, C = input.shape[0], input.shape[1] + + half_input = input.dtype == torch.float16 + if half_input: + # fp16 does not have good enough numerics for the reduction here + input = input.float() + mean = torch.mean(input, dim=[0, 2, 3]) + meansqr = torch.mean(input * input, dim=[0, 2, 3]) + + if self._stats_mode == "": + assert B > 0, 'SyncBatchNorm(stats_mode="") does not support zero batch size.' + vec = torch.cat([mean, meansqr], dim=0) + vec = differentiable_all_reduce(vec) * (1.0 / dist.get_world_size()) + mean, meansqr = torch.split(vec, C) + momentum = self.momentum + else: + if B == 0: + vec = torch.zeros([2 * C + 1], device=mean.device, dtype=mean.dtype) + vec = vec + input.sum() # make sure there is gradient w.r.t input + else: + vec = torch.cat([mean, meansqr, torch.ones([1], device=mean.device, dtype=mean.dtype)], dim=0) + vec = differentiable_all_reduce(vec * B) + + total_batch = vec[-1].detach() + momentum = total_batch.clamp(max=1) * self.momentum # no update if total_batch is 0 + mean, meansqr, _ = torch.split(vec / total_batch.clamp(min=1), C) # avoid div-by-zero + + var = meansqr - mean * mean + invstd = torch.rsqrt(var + self.eps) + scale = self.weight * invstd + bias = self.bias - mean * scale + scale = scale.reshape(1, -1, 1, 1) + bias = bias.reshape(1, -1, 1, 1) + + self.running_mean += momentum * (mean.detach() - self.running_mean) + self.running_var += momentum * (var.detach() - self.running_var) + ret = input * scale + bias + if half_input: + ret = ret.half() + return ret + + +class CycleBatchNormList(nn.ModuleList): + """ + Implement domain-specific BatchNorm by cycling. + + When a BatchNorm layer is used for multiple input domains or input + features, it might need to maintain a separate test-time statistics + for each domain. See Sec 5.2 in :paper:`rethinking-batchnorm`. + + This module implements it by using N separate BN layers + and it cycles through them every time a forward() is called. + + NOTE: The caller of this module MUST guarantee to always call + this module by multiple of N times. Otherwise its test-time statistics + will be incorrect. + """ + + def __init__(self, length: int, bn_class=nn.BatchNorm2d, **kwargs): + """ + Args: + length: number of BatchNorm layers to cycle. + bn_class: the BatchNorm class to use + kwargs: arguments of the BatchNorm class, such as num_features. + """ + self._affine = kwargs.pop("affine", True) + super().__init__([bn_class(**kwargs, affine=False) for k in range(length)]) + if self._affine: + # shared affine, domain-specific BN + channels = self[0].num_features + self.weight = nn.Parameter(torch.ones(channels)) + self.bias = nn.Parameter(torch.zeros(channels)) + self._pos = 0 + + def forward(self, x): + ret = self[self._pos](x) + self._pos = (self._pos + 1) % len(self) + + if self._affine: + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + return ret * w + b + else: + return ret + + def extra_repr(self): + return f"affine={self._affine}" + + +class LayerNorm(nn.Module): + """ + A LayerNorm variant, popularized by Transformers, that performs point-wise mean and + variance normalization over the channel dimension for inputs that have shape + (batch_size, channels, height, width). + https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa B950 + """ + + def __init__(self, normalized_shape, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.normalized_shape = (normalized_shape,) + + def forward(self, x): + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x diff --git a/detectron2/layers/blocks.py b/detectron2/layers/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..ba0371f29fc55740fbad2d3c1bf78ca35fcab944 --- /dev/null +++ b/detectron2/layers/blocks.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import fvcore.nn.weight_init as weight_init +from torch import nn + +from .batch_norm import FrozenBatchNorm2d, get_norm +from .wrappers import Conv2d + +""" +CNN building blocks. +""" + + +class CNNBlockBase(nn.Module): + """ + A CNN block is assumed to have input channels, output channels and a stride. + The input and output of `forward()` method must be NCHW tensors. + The method can perform arbitrary computation but must match the given + channels and stride specification. + + Attribute: + in_channels (int): + out_channels (int): + stride (int): + """ + + def __init__(self, in_channels, out_channels, stride): + """ + The `__init__` method of any subclass should also contain these arguments. + + Args: + in_channels (int): + out_channels (int): + stride (int): + """ + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + + def freeze(self): + """ + Make this block not trainable. + This method sets all parameters to `requires_grad=False`, + and convert all BatchNorm layers to FrozenBatchNorm + + Returns: + the block itself + """ + for p in self.parameters(): + p.requires_grad = False + FrozenBatchNorm2d.convert_frozen_batchnorm(self) + return self + + +class DepthwiseSeparableConv2d(nn.Module): + """ + A kxk depthwise convolution + a 1x1 convolution. + + In :paper:`xception`, norm & activation are applied on the second conv. + :paper:`mobilenet` uses norm & activation on both convs. + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size=3, + padding=1, + dilation=1, + *, + norm1=None, + activation1=None, + norm2=None, + activation2=None, + ): + """ + Args: + norm1, norm2 (str or callable): normalization for the two conv layers. + activation1, activation2 (callable(Tensor) -> Tensor): activation + function for the two conv layers. + """ + super().__init__() + self.depthwise = Conv2d( + in_channels, + in_channels, + kernel_size=kernel_size, + padding=padding, + dilation=dilation, + groups=in_channels, + bias=not norm1, + norm=get_norm(norm1, in_channels), + activation=activation1, + ) + self.pointwise = Conv2d( + in_channels, + out_channels, + kernel_size=1, + bias=not norm2, + norm=get_norm(norm2, out_channels), + activation=activation2, + ) + + # default initialization + weight_init.c2_msra_fill(self.depthwise) + weight_init.c2_msra_fill(self.pointwise) + + def forward(self, x): + return self.pointwise(self.depthwise(x)) diff --git a/detectron2/layers/csrc/README.md b/detectron2/layers/csrc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..778ed3da0bae89820831bcd8a72ff7b9cad8d4dd --- /dev/null +++ b/detectron2/layers/csrc/README.md @@ -0,0 +1,7 @@ + + +To add a new Op: + +1. Create a new directory +2. Implement new ops there +3. Delcare its Python interface in `vision.cpp`. diff --git a/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated.h b/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated.h new file mode 100644 index 0000000000000000000000000000000000000000..03f4211003f42f601f0cfcf4a690f5da4a0a1f67 --- /dev/null +++ b/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated.h @@ -0,0 +1,115 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +at::Tensor ROIAlignRotated_forward_cpu( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio); + +at::Tensor ROIAlignRotated_backward_cpu( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio); + +#if defined(WITH_CUDA) || defined(WITH_HIP) +at::Tensor ROIAlignRotated_forward_cuda( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio); + +at::Tensor ROIAlignRotated_backward_cuda( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio); +#endif + +// Interface for Python +inline at::Tensor ROIAlignRotated_forward( + const at::Tensor& input, + const at::Tensor& rois, + const double spatial_scale, + const int64_t pooled_height, + const int64_t pooled_width, + const int64_t sampling_ratio) { + if (input.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return ROIAlignRotated_forward_cuda( + input, + rois, + spatial_scale, + pooled_height, + pooled_width, + sampling_ratio); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + return ROIAlignRotated_forward_cpu( + input, rois, spatial_scale, pooled_height, pooled_width, sampling_ratio); +} + +inline at::Tensor ROIAlignRotated_backward( + const at::Tensor& grad, + const at::Tensor& rois, + const double spatial_scale, + const int64_t pooled_height, + const int64_t pooled_width, + const int64_t batch_size, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t sampling_ratio) { + if (grad.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return ROIAlignRotated_backward_cuda( + grad, + rois, + spatial_scale, + pooled_height, + pooled_width, + batch_size, + channels, + height, + width, + sampling_ratio); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + return ROIAlignRotated_backward_cpu( + grad, + rois, + spatial_scale, + pooled_height, + pooled_width, + batch_size, + channels, + height, + width, + sampling_ratio); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.cpp b/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2a3d3056cc71a4acaafb570739a9dd247a7eb1ed --- /dev/null +++ b/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.cpp @@ -0,0 +1,522 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include "ROIAlignRotated.h" + +// Note: this implementation originates from the Caffe2 ROIAlignRotated Op +// and PyTorch ROIAlign (non-rotated) Op implementations. +// The key difference between this implementation and those ones is +// we don't do "legacy offset" in this version, as there aren't many previous +// works, if any, using the "legacy" ROIAlignRotated Op. +// This would make the interface a bit cleaner. + +namespace detectron2 { + +namespace { +template +struct PreCalc { + int pos1; + int pos2; + int pos3; + int pos4; + T w1; + T w2; + T w3; + T w4; +}; + +template +void pre_calc_for_bilinear_interpolate( + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int iy_upper, + const int ix_upper, + T roi_start_h, + T roi_start_w, + T bin_size_h, + T bin_size_w, + int roi_bin_grid_h, + int roi_bin_grid_w, + T roi_center_h, + T roi_center_w, + T cos_theta, + T sin_theta, + std::vector>& pre_calc) { + int pre_calc_index = 0; + for (int ph = 0; ph < pooled_height; ph++) { + for (int pw = 0; pw < pooled_width; pw++) { + for (int iy = 0; iy < iy_upper; iy++) { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < ix_upper; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + // In image space, (y, x) is the order for Right Handed System, + // and this is essentially multiplying the point by a rotation matrix + // to rotate it counterclockwise through angle theta. + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + // deal with: inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + PreCalc pc; + pc.pos1 = 0; + pc.pos2 = 0; + pc.pos3 = 0; + pc.pos4 = 0; + pc.w1 = 0; + pc.w2 = 0; + pc.w3 = 0; + pc.w4 = 0; + pre_calc[pre_calc_index] = pc; + pre_calc_index += 1; + continue; + } + + if (y < 0) { + y = 0; + } + if (x < 0) { + x = 0; + } + + int y_low = (int)y; + int x_low = (int)x; + int y_high; + int x_high; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + T w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + // save weights and indices + PreCalc pc; + pc.pos1 = y_low * width + x_low; + pc.pos2 = y_low * width + x_high; + pc.pos3 = y_high * width + x_low; + pc.pos4 = y_high * width + x_high; + pc.w1 = w1; + pc.w2 = w2; + pc.w3 = w3; + pc.w4 = w4; + pre_calc[pre_calc_index] = pc; + + pre_calc_index += 1; + } + } + } + } +} + +template +void bilinear_interpolate_gradient( + const int height, + const int width, + T y, + T x, + T& w1, + T& w2, + T& w3, + T& w4, + int& x_low, + int& x_high, + int& y_low, + int& y_high) { + // deal with cases that inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + w1 = w2 = w3 = w4 = 0.; + x_low = x_high = y_low = y_high = -1; + return; + } + + if (y < 0) { + y = 0; + } + + if (x < 0) { + x = 0; + } + + y_low = (int)y; + x_low = (int)x; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + + // reference in forward + // T v1 = input[y_low * width + x_low]; + // T v2 = input[y_low * width + x_high]; + // T v3 = input[y_high * width + x_low]; + // T v4 = input[y_high * width + x_high]; + // T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + + w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + return; +} + +template +inline void add(T* address, const T& val) { + *address += val; +} + +} // namespace + +template +void ROIAlignRotatedForward( + const int nthreads, + const T* input, + const T& spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + const T* rois, + T* output) { + int n_rois = nthreads / channels / pooled_width / pooled_height; + // (n, c, ph, pw) is an element in the pooled output + // can be parallelized using omp + // #pragma omp parallel for num_threads(32) + for (int n = 0; n < n_rois; n++) { + int index_n = n * channels * pooled_width * pooled_height; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + AT_ASSERTM( + roi_width >= 0 && roi_height >= 0, + "ROIs in ROIAlignRotated do not have non-negative size!"); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // We do average (integral) pooling inside a bin + const T count = std::max(roi_bin_grid_h * roi_bin_grid_w, 1); // e.g. = 4 + + // we want to precalculate indices and weights shared by all channels, + // this is the key point of optimization + std::vector> pre_calc( + roi_bin_grid_h * roi_bin_grid_w * pooled_width * pooled_height); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + pre_calc_for_bilinear_interpolate( + height, + width, + pooled_height, + pooled_width, + roi_bin_grid_h, + roi_bin_grid_w, + roi_start_h, + roi_start_w, + bin_size_h, + bin_size_w, + roi_bin_grid_h, + roi_bin_grid_w, + roi_center_h, + roi_center_w, + cos_theta, + sin_theta, + pre_calc); + + for (int c = 0; c < channels; c++) { + int index_n_c = index_n + c * pooled_width * pooled_height; + const T* offset_input = + input + (roi_batch_ind * channels + c) * height * width; + int pre_calc_index = 0; + + for (int ph = 0; ph < pooled_height; ph++) { + for (int pw = 0; pw < pooled_width; pw++) { + int index = index_n_c + ph * pooled_width + pw; + + T output_val = 0.; + for (int iy = 0; iy < roi_bin_grid_h; iy++) { + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + PreCalc pc = pre_calc[pre_calc_index]; + output_val += pc.w1 * offset_input[pc.pos1] + + pc.w2 * offset_input[pc.pos2] + + pc.w3 * offset_input[pc.pos3] + pc.w4 * offset_input[pc.pos4]; + + pre_calc_index += 1; + } + } + output_val /= count; + + output[index] = output_val; + } // for pw + } // for ph + } // for c + } // for n +} + +template +void ROIAlignRotatedBackward( + const int nthreads, + // may not be contiguous. should index using n_stride, etc + const T* grad_output, + const T& spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + T* grad_input, + const T* rois, + const int n_stride, + const int c_stride, + const int h_stride, + const int w_stride) { + for (int index = 0; index < nthreads; index++) { + // (n, c, ph, pw) is an element in the pooled output + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + AT_ASSERTM( + roi_width >= 0 && roi_height >= 0, + "ROIs in ROIAlignRotated do not have non-negative size!"); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + T* offset_grad_input = + grad_input + ((roi_batch_ind * channels + c) * height * width); + + int output_offset = n * n_stride + c * c_stride; + const T* offset_grad_output = grad_output + output_offset; + const T grad_output_this_bin = + offset_grad_output[ph * h_stride + pw * w_stride]; + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + // We do average (integral) pooling inside a bin + const T count = roi_bin_grid_h * roi_bin_grid_w; // e.g. = 4 + + for (int iy = 0; iy < roi_bin_grid_h; iy++) { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + + T w1, w2, w3, w4; + int x_low, x_high, y_low, y_high; + + bilinear_interpolate_gradient( + height, width, y, x, w1, w2, w3, w4, x_low, x_high, y_low, y_high); + + T g1 = grad_output_this_bin * w1 / count; + T g2 = grad_output_this_bin * w2 / count; + T g3 = grad_output_this_bin * w3 / count; + T g4 = grad_output_this_bin * w4 / count; + + if (x_low >= 0 && x_high >= 0 && y_low >= 0 && y_high >= 0) { + // atomic add is not needed for now since it is single threaded + add(offset_grad_input + y_low * width + x_low, static_cast(g1)); + add(offset_grad_input + y_low * width + x_high, static_cast(g2)); + add(offset_grad_input + y_high * width + x_low, static_cast(g3)); + add(offset_grad_input + y_high * width + x_high, static_cast(g4)); + } // if + } // ix + } // iy + } // for +} // ROIAlignRotatedBackward + +at::Tensor ROIAlignRotated_forward_cpu( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio) { + AT_ASSERTM(input.device().is_cpu(), "input must be a CPU tensor"); + AT_ASSERTM(rois.device().is_cpu(), "rois must be a CPU tensor"); + + at::TensorArg input_t{input, "input", 1}, rois_t{rois, "rois", 2}; + + at::CheckedFrom c = "ROIAlign_forward_cpu"; + at::checkAllSameType(c, {input_t, rois_t}); + + auto num_rois = rois.size(0); + auto channels = input.size(1); + auto height = input.size(2); + auto width = input.size(3); + + at::Tensor output = at::zeros( + {num_rois, channels, pooled_height, pooled_width}, input.options()); + + auto output_size = num_rois * pooled_height * pooled_width * channels; + + if (output.numel() == 0) { + return output; + } + + auto input_ = input.contiguous(), rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.scalar_type(), "ROIAlignRotated_forward", [&] { + ROIAlignRotatedForward( + output_size, + input_.data_ptr(), + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + rois_.data_ptr(), + output.data_ptr()); + }); + return output; +} + +at::Tensor ROIAlignRotated_backward_cpu( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio) { + AT_ASSERTM(grad.device().is_cpu(), "grad must be a CPU tensor"); + AT_ASSERTM(rois.device().is_cpu(), "rois must be a CPU tensor"); + + at::TensorArg grad_t{grad, "grad", 1}, rois_t{rois, "rois", 2}; + + at::CheckedFrom c = "ROIAlignRotated_backward_cpu"; + at::checkAllSameType(c, {grad_t, rois_t}); + + at::Tensor grad_input = + at::zeros({batch_size, channels, height, width}, grad.options()); + + // handle possibly empty gradients + if (grad.numel() == 0) { + return grad_input; + } + + // get stride values to ensure indexing into gradients is correct. + int n_stride = grad.stride(0); + int c_stride = grad.stride(1); + int h_stride = grad.stride(2); + int w_stride = grad.stride(3); + + auto rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + grad.scalar_type(), "ROIAlignRotated_forward", [&] { + ROIAlignRotatedBackward( + grad.numel(), + grad.data_ptr(), + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + grad_input.data_ptr(), + rois_.data_ptr(), + n_stride, + c_stride, + h_stride, + w_stride); + }); + return grad_input; +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cuda.cu b/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..fca186519143b168a912c880a4cf495a0a5a9322 --- /dev/null +++ b/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cuda.cu @@ -0,0 +1,443 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include +#include +#include + +// TODO make it in a common file +#define CUDA_1D_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; \ + i += blockDim.x * gridDim.x) + +// Note: this implementation originates from the Caffe2 ROIAlignRotated Op +// and PyTorch ROIAlign (non-rotated) Op implementations. +// The key difference between this implementation and those ones is +// we don't do "legacy offset" in this version, as there aren't many previous +// works, if any, using the "legacy" ROIAlignRotated Op. +// This would make the interface a bit cleaner. + +namespace detectron2 { + +namespace { + +template +__device__ T bilinear_interpolate( + const T* input, + const int height, + const int width, + T y, + T x) { + // deal with cases that inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + return 0; + } + + if (y < 0) { + y = 0; + } + + if (x < 0) { + x = 0; + } + + int y_low = (int)y; + int x_low = (int)x; + int y_high; + int x_high; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + // do bilinear interpolation + T v1 = input[y_low * width + x_low]; + T v2 = input[y_low * width + x_high]; + T v3 = input[y_high * width + x_low]; + T v4 = input[y_high * width + x_high]; + T w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + + return val; +} + +template +__device__ void bilinear_interpolate_gradient( + const int height, + const int width, + T y, + T x, + T& w1, + T& w2, + T& w3, + T& w4, + int& x_low, + int& x_high, + int& y_low, + int& y_high) { + // deal with cases that inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + w1 = w2 = w3 = w4 = 0.; + x_low = x_high = y_low = y_high = -1; + return; + } + + if (y < 0) { + y = 0; + } + + if (x < 0) { + x = 0; + } + + y_low = (int)y; + x_low = (int)x; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + + // reference in forward + // T v1 = input[y_low * width + x_low]; + // T v2 = input[y_low * width + x_high]; + // T v3 = input[y_high * width + x_low]; + // T v4 = input[y_high * width + x_high]; + // T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + + w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + return; +} + +} // namespace + +template +__global__ void RoIAlignRotatedForward( + const int nthreads, + const T* input, + const T spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + const T* rois, + T* top_data) { + CUDA_1D_KERNEL_LOOP(index, nthreads) { + // (n, c, ph, pw) is an element in the pooled output + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + const T* offset_input = + input + (roi_batch_ind * channels + c) * height * width; + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + // We do average (inte gral) pooling inside a bin + const T count = max(roi_bin_grid_h * roi_bin_grid_w, 1); // e.g. = 4 + + T output_val = 0.; + for (int iy = 0; iy < roi_bin_grid_h; iy++) // e.g., iy = 0, 1 + { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + + T val = bilinear_interpolate(offset_input, height, width, y, x); + output_val += val; + } + } + output_val /= count; + + top_data[index] = output_val; + } +} + +template +__global__ void RoIAlignRotatedBackwardFeature( + const int nthreads, + const T* top_diff, + const int num_rois, + const T spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + T* bottom_diff, + const T* rois) { + CUDA_1D_KERNEL_LOOP(index, nthreads) { + // (n, c, ph, pw) is an element in the pooled output + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + T* offset_bottom_diff = + bottom_diff + (roi_batch_ind * channels + c) * height * width; + + int top_offset = (n * channels + c) * pooled_height * pooled_width; + const T* offset_top_diff = top_diff + top_offset; + const T top_diff_this_bin = offset_top_diff[ph * pooled_width + pw]; + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + // We do average (integral) pooling inside a bin + const T count = roi_bin_grid_h * roi_bin_grid_w; // e.g. = 4 + + for (int iy = 0; iy < roi_bin_grid_h; iy++) // e.g., iy = 0, 1 + { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + + T w1, w2, w3, w4; + int x_low, x_high, y_low, y_high; + + bilinear_interpolate_gradient( + height, width, y, x, w1, w2, w3, w4, x_low, x_high, y_low, y_high); + + T g1 = top_diff_this_bin * w1 / count; + T g2 = top_diff_this_bin * w2 / count; + T g3 = top_diff_this_bin * w3 / count; + T g4 = top_diff_this_bin * w4 / count; + + if (x_low >= 0 && x_high >= 0 && y_low >= 0 && y_high >= 0) { + atomicAdd( + offset_bottom_diff + y_low * width + x_low, static_cast(g1)); + atomicAdd( + offset_bottom_diff + y_low * width + x_high, static_cast(g2)); + atomicAdd( + offset_bottom_diff + y_high * width + x_low, static_cast(g3)); + atomicAdd( + offset_bottom_diff + y_high * width + x_high, static_cast(g4)); + } // if + } // ix + } // iy + } // CUDA_1D_KERNEL_LOOP +} // RoIAlignRotatedBackward + +at::Tensor ROIAlignRotated_forward_cuda( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio) { + AT_ASSERTM(input.device().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(rois.device().is_cuda(), "rois must be a CUDA tensor"); + at::TensorArg input_t{input, "input", 1}, rois_t{rois, "rois", 2}; + + at::CheckedFrom c = "ROIAlignRotated_forward_cuda"; + at::checkAllSameGPU(c, {input_t, rois_t}); + at::checkAllSameType(c, {input_t, rois_t}); + at::cuda::CUDAGuard device_guard(input.device()); + + auto num_rois = rois.size(0); + auto channels = input.size(1); + auto height = input.size(2); + auto width = input.size(3); + + auto output = at::empty( + {num_rois, channels, pooled_height, pooled_width}, input.options()); + auto output_size = num_rois * pooled_height * pooled_width * channels; + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + dim3 grid(std::min( + at::cuda::ATenCeilDiv( + static_cast(output_size), static_cast(512)), + static_cast(4096))); + dim3 block(512); + + if (output.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return output; + } + + auto input_ = input.contiguous(), rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "ROIAlignRotated_forward", [&] { + RoIAlignRotatedForward<<>>( + output_size, + input_.data_ptr(), + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + rois_.data_ptr(), + output.data_ptr()); + }); + cudaDeviceSynchronize(); + AT_CUDA_CHECK(cudaGetLastError()); + return output; +} + +// TODO remove the dependency on input and use instead its sizes -> save memory +at::Tensor ROIAlignRotated_backward_cuda( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio) { + AT_ASSERTM(grad.device().is_cuda(), "grad must be a CUDA tensor"); + AT_ASSERTM(rois.device().is_cuda(), "rois must be a CUDA tensor"); + + at::TensorArg grad_t{grad, "grad", 1}, rois_t{rois, "rois", 2}; + at::CheckedFrom c = "ROIAlign_backward_cuda"; + at::checkAllSameGPU(c, {grad_t, rois_t}); + at::checkAllSameType(c, {grad_t, rois_t}); + at::cuda::CUDAGuard device_guard(grad.device()); + + auto num_rois = rois.size(0); + auto grad_input = + at::zeros({batch_size, channels, height, width}, grad.options()); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + dim3 grid(std::min( + at::cuda::ATenCeilDiv( + static_cast(grad.numel()), static_cast(512)), + static_cast(4096))); + dim3 block(512); + + // handle possibly empty gradients + if (grad.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return grad_input; + } + + auto grad_ = grad.contiguous(), rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES( + grad.scalar_type(), "ROIAlignRotated_backward", [&] { + RoIAlignRotatedBackwardFeature<<>>( + grad.numel(), + grad_.data_ptr(), + num_rois, + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + grad_input.data_ptr(), + rois_.data_ptr()); + }); + AT_CUDA_CHECK(cudaGetLastError()); + return grad_input; +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated.h b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated.h new file mode 100644 index 0000000000000000000000000000000000000000..3bf383b8ed9b358b5313d433a9682c294dfb77e4 --- /dev/null +++ b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated.h @@ -0,0 +1,35 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +at::Tensor box_iou_rotated_cpu( + const at::Tensor& boxes1, + const at::Tensor& boxes2); + +#if defined(WITH_CUDA) || defined(WITH_HIP) +at::Tensor box_iou_rotated_cuda( + const at::Tensor& boxes1, + const at::Tensor& boxes2); +#endif + +// Interface for Python +// inline is needed to prevent multiple function definitions when this header is +// included by different cpps +inline at::Tensor box_iou_rotated( + const at::Tensor& boxes1, + const at::Tensor& boxes2) { + assert(boxes1.device().is_cuda() == boxes2.device().is_cuda()); + if (boxes1.device().is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return box_iou_rotated_cuda(boxes1.contiguous(), boxes2.contiguous()); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + + return box_iou_rotated_cpu(boxes1.contiguous(), boxes2.contiguous()); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.cpp b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c843487b5fa4e8077dd27402ec99009266ddda8d --- /dev/null +++ b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.cpp @@ -0,0 +1,39 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include "box_iou_rotated.h" +#include "box_iou_rotated_utils.h" + +namespace detectron2 { + +template +void box_iou_rotated_cpu_kernel( + const at::Tensor& boxes1, + const at::Tensor& boxes2, + at::Tensor& ious) { + auto num_boxes1 = boxes1.size(0); + auto num_boxes2 = boxes2.size(0); + + for (int i = 0; i < num_boxes1; i++) { + for (int j = 0; j < num_boxes2; j++) { + ious[i * num_boxes2 + j] = single_box_iou_rotated( + boxes1[i].data_ptr(), boxes2[j].data_ptr()); + } + } +} + +at::Tensor box_iou_rotated_cpu( + // input must be contiguous: + const at::Tensor& boxes1, + const at::Tensor& boxes2) { + auto num_boxes1 = boxes1.size(0); + auto num_boxes2 = boxes2.size(0); + at::Tensor ious = + at::empty({num_boxes1 * num_boxes2}, boxes1.options().dtype(at::kFloat)); + + box_iou_rotated_cpu_kernel(boxes1, boxes2, ious); + + // reshape from 1d array to 2d array + auto shape = std::vector{num_boxes1, num_boxes2}; + return ious.reshape(shape); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cuda.cu b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..952710e53041187907fbd113f8d0d0fa24134a86 --- /dev/null +++ b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cuda.cu @@ -0,0 +1,130 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include +#include +#include +#include "box_iou_rotated_utils.h" + +namespace detectron2 { + +// 2D block with 32 * 16 = 512 threads per block +const int BLOCK_DIM_X = 32; +const int BLOCK_DIM_Y = 16; + +template +__global__ void box_iou_rotated_cuda_kernel( + const int n_boxes1, + const int n_boxes2, + const T* dev_boxes1, + const T* dev_boxes2, + T* dev_ious) { + const int row_start = blockIdx.x * blockDim.x; + const int col_start = blockIdx.y * blockDim.y; + + const int row_size = min(n_boxes1 - row_start, blockDim.x); + const int col_size = min(n_boxes2 - col_start, blockDim.y); + + __shared__ float block_boxes1[BLOCK_DIM_X * 5]; + __shared__ float block_boxes2[BLOCK_DIM_Y * 5]; + + // It's safe to copy using threadIdx.x since BLOCK_DIM_X >= BLOCK_DIM_Y + if (threadIdx.x < row_size && threadIdx.y == 0) { + block_boxes1[threadIdx.x * 5 + 0] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 0]; + block_boxes1[threadIdx.x * 5 + 1] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 1]; + block_boxes1[threadIdx.x * 5 + 2] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 2]; + block_boxes1[threadIdx.x * 5 + 3] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 3]; + block_boxes1[threadIdx.x * 5 + 4] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 4]; + } + + if (threadIdx.x < col_size && threadIdx.y == 0) { + block_boxes2[threadIdx.x * 5 + 0] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 0]; + block_boxes2[threadIdx.x * 5 + 1] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 1]; + block_boxes2[threadIdx.x * 5 + 2] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 2]; + block_boxes2[threadIdx.x * 5 + 3] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 3]; + block_boxes2[threadIdx.x * 5 + 4] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 4]; + } + __syncthreads(); + + if (threadIdx.x < row_size && threadIdx.y < col_size) { + int offset = (row_start + threadIdx.x) * n_boxes2 + col_start + threadIdx.y; + dev_ious[offset] = single_box_iou_rotated( + block_boxes1 + threadIdx.x * 5, block_boxes2 + threadIdx.y * 5); + } +} + +at::Tensor box_iou_rotated_cuda( + // input must be contiguous + const at::Tensor& boxes1, + const at::Tensor& boxes2) { + using scalar_t = float; + AT_ASSERTM( + boxes1.scalar_type() == at::kFloat, "boxes1 must be a float tensor"); + AT_ASSERTM( + boxes2.scalar_type() == at::kFloat, "boxes2 must be a float tensor"); + AT_ASSERTM(boxes1.is_cuda(), "boxes1 must be a CUDA tensor"); + AT_ASSERTM(boxes2.is_cuda(), "boxes2 must be a CUDA tensor"); + at::cuda::CUDAGuard device_guard(boxes1.device()); + + auto num_boxes1 = boxes1.size(0); + auto num_boxes2 = boxes2.size(0); + + at::Tensor ious = + at::empty({num_boxes1 * num_boxes2}, boxes1.options().dtype(at::kFloat)); + + bool transpose = false; + if (num_boxes1 > 0 && num_boxes2 > 0) { + scalar_t *data1 = boxes1.data_ptr(), + *data2 = boxes2.data_ptr(); + + if (num_boxes2 > 65535 * BLOCK_DIM_Y) { + AT_ASSERTM( + num_boxes1 <= 65535 * BLOCK_DIM_Y, + "Too many boxes for box_iou_rotated_cuda!"); + // x dim is allowed to be large, but y dim cannot, + // so we transpose the two to avoid "invalid configuration argument" + // error. We assume one of them is small. Otherwise the result is hard to + // fit in memory anyway. + std::swap(num_boxes1, num_boxes2); + std::swap(data1, data2); + transpose = true; + } + + const int blocks_x = + at::cuda::ATenCeilDiv(static_cast(num_boxes1), BLOCK_DIM_X); + const int blocks_y = + at::cuda::ATenCeilDiv(static_cast(num_boxes2), BLOCK_DIM_Y); + + dim3 blocks(blocks_x, blocks_y); + dim3 threads(BLOCK_DIM_X, BLOCK_DIM_Y); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + box_iou_rotated_cuda_kernel<<>>( + num_boxes1, + num_boxes2, + data1, + data2, + (scalar_t*)ious.data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + } + + // reshape from 1d array to 2d array + auto shape = std::vector{num_boxes1, num_boxes2}; + if (transpose) { + return ious.view(shape).t(); + } else { + return ious.view(shape); + } +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..b54a5dde2ca11a74d29c4d8adb7fe1634f5baf9c --- /dev/null +++ b/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h @@ -0,0 +1,370 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once + +#include +#include + +#if defined(__CUDACC__) || __HCC__ == 1 || __HIP__ == 1 +// Designates functions callable from the host (CPU) and the device (GPU) +#define HOST_DEVICE __host__ __device__ +#define HOST_DEVICE_INLINE HOST_DEVICE __forceinline__ +#else +#include +#define HOST_DEVICE +#define HOST_DEVICE_INLINE HOST_DEVICE inline +#endif + +namespace detectron2 { + +namespace { + +template +struct RotatedBox { + T x_ctr, y_ctr, w, h, a; +}; + +template +struct Point { + T x, y; + HOST_DEVICE_INLINE Point(const T& px = 0, const T& py = 0) : x(px), y(py) {} + HOST_DEVICE_INLINE Point operator+(const Point& p) const { + return Point(x + p.x, y + p.y); + } + HOST_DEVICE_INLINE Point& operator+=(const Point& p) { + x += p.x; + y += p.y; + return *this; + } + HOST_DEVICE_INLINE Point operator-(const Point& p) const { + return Point(x - p.x, y - p.y); + } + HOST_DEVICE_INLINE Point operator*(const T coeff) const { + return Point(x * coeff, y * coeff); + } +}; + +template +HOST_DEVICE_INLINE T dot_2d(const Point& A, const Point& B) { + return A.x * B.x + A.y * B.y; +} + +// R: result type. can be different from input type +template +HOST_DEVICE_INLINE R cross_2d(const Point& A, const Point& B) { + return static_cast(A.x) * static_cast(B.y) - + static_cast(B.x) * static_cast(A.y); +} + +template +HOST_DEVICE_INLINE void get_rotated_vertices( + const RotatedBox& box, + Point (&pts)[4]) { + // M_PI / 180. == 0.01745329251 + double theta = box.a * 0.01745329251; + T cosTheta2 = (T)cos(theta) * 0.5f; + T sinTheta2 = (T)sin(theta) * 0.5f; + + // y: top --> down; x: left --> right + pts[0].x = box.x_ctr + sinTheta2 * box.h + cosTheta2 * box.w; + pts[0].y = box.y_ctr + cosTheta2 * box.h - sinTheta2 * box.w; + pts[1].x = box.x_ctr - sinTheta2 * box.h + cosTheta2 * box.w; + pts[1].y = box.y_ctr - cosTheta2 * box.h - sinTheta2 * box.w; + pts[2].x = 2 * box.x_ctr - pts[0].x; + pts[2].y = 2 * box.y_ctr - pts[0].y; + pts[3].x = 2 * box.x_ctr - pts[1].x; + pts[3].y = 2 * box.y_ctr - pts[1].y; +} + +template +HOST_DEVICE_INLINE int get_intersection_points( + const Point (&pts1)[4], + const Point (&pts2)[4], + Point (&intersections)[24]) { + // Line vector + // A line from p1 to p2 is: p1 + (p2-p1)*t, t=[0,1] + Point vec1[4], vec2[4]; + for (int i = 0; i < 4; i++) { + vec1[i] = pts1[(i + 1) % 4] - pts1[i]; + vec2[i] = pts2[(i + 1) % 4] - pts2[i]; + } + + // When computing the intersection area, it doesn't hurt if we have + // more (duplicated/approximate) intersections/vertices than needed, + // while it can cause drastic difference if we miss an intersection/vertex. + // Therefore, we add an epsilon to relax the comparisons between + // the float point numbers that decide the intersection points. + double EPS = 1e-5; + + // Line test - test all line combos for intersection + int num = 0; // number of intersections + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + // Solve for 2x2 Ax=b + T det = cross_2d(vec2[j], vec1[i]); + + // This takes care of parallel lines + if (fabs(det) <= 1e-14) { + continue; + } + + auto vec12 = pts2[j] - pts1[i]; + + T t1 = cross_2d(vec2[j], vec12) / det; + T t2 = cross_2d(vec1[i], vec12) / det; + + if (t1 > -EPS && t1 < 1.0f + EPS && t2 > -EPS && t2 < 1.0f + EPS) { + intersections[num++] = pts1[i] + vec1[i] * t1; + } + } + } + + // Check for vertices of rect1 inside rect2 + { + const auto& AB = vec2[0]; + const auto& DA = vec2[3]; + auto ABdotAB = dot_2d(AB, AB); + auto ADdotAD = dot_2d(DA, DA); + for (int i = 0; i < 4; i++) { + // assume ABCD is the rectangle, and P is the point to be judged + // P is inside ABCD iff. P's projection on AB lies within AB + // and P's projection on AD lies within AD + + auto AP = pts1[i] - pts2[0]; + + auto APdotAB = dot_2d(AP, AB); + auto APdotAD = -dot_2d(AP, DA); + + if ((APdotAB > -EPS) && (APdotAD > -EPS) && (APdotAB < ABdotAB + EPS) && + (APdotAD < ADdotAD + EPS)) { + intersections[num++] = pts1[i]; + } + } + } + + // Reverse the check - check for vertices of rect2 inside rect1 + { + const auto& AB = vec1[0]; + const auto& DA = vec1[3]; + auto ABdotAB = dot_2d(AB, AB); + auto ADdotAD = dot_2d(DA, DA); + for (int i = 0; i < 4; i++) { + auto AP = pts2[i] - pts1[0]; + + auto APdotAB = dot_2d(AP, AB); + auto APdotAD = -dot_2d(AP, DA); + + if ((APdotAB > -EPS) && (APdotAD > -EPS) && (APdotAB < ABdotAB + EPS) && + (APdotAD < ADdotAD + EPS)) { + intersections[num++] = pts2[i]; + } + } + } + + return num; +} + +template +HOST_DEVICE_INLINE int convex_hull_graham( + const Point (&p)[24], + const int& num_in, + Point (&q)[24], + bool shift_to_zero = false) { + assert(num_in >= 2); + + // Step 1: + // Find point with minimum y + // if more than 1 points have the same minimum y, + // pick the one with the minimum x. + int t = 0; + for (int i = 1; i < num_in; i++) { + if (p[i].y < p[t].y || (p[i].y == p[t].y && p[i].x < p[t].x)) { + t = i; + } + } + auto& start = p[t]; // starting point + + // Step 2: + // Subtract starting point from every points (for sorting in the next step) + for (int i = 0; i < num_in; i++) { + q[i] = p[i] - start; + } + + // Swap the starting point to position 0 + auto tmp = q[0]; + q[0] = q[t]; + q[t] = tmp; + + // Step 3: + // Sort point 1 ~ num_in according to their relative cross-product values + // (essentially sorting according to angles) + // If the angles are the same, sort according to their distance to origin + T dist[24]; +#if defined(__CUDACC__) || __HCC__ == 1 || __HIP__ == 1 + // compute distance to origin before sort, and sort them together with the + // points + for (int i = 0; i < num_in; i++) { + dist[i] = dot_2d(q[i], q[i]); + } + + // CUDA version + // In the future, we can potentially use thrust + // for sorting here to improve speed (though not guaranteed) + for (int i = 1; i < num_in - 1; i++) { + for (int j = i + 1; j < num_in; j++) { + T crossProduct = cross_2d(q[i], q[j]); + if ((crossProduct < -1e-6) || + (fabs(crossProduct) < 1e-6 && dist[i] > dist[j])) { + auto q_tmp = q[i]; + q[i] = q[j]; + q[j] = q_tmp; + auto dist_tmp = dist[i]; + dist[i] = dist[j]; + dist[j] = dist_tmp; + } + } + } +#else + // CPU version + std::sort( + q + 1, q + num_in, [](const Point& A, const Point& B) -> bool { + T temp = cross_2d(A, B); + if (fabs(temp) < 1e-6) { + return dot_2d(A, A) < dot_2d(B, B); + } else { + return temp > 0; + } + }); + // compute distance to origin after sort, since the points are now different. + for (int i = 0; i < num_in; i++) { + dist[i] = dot_2d(q[i], q[i]); + } +#endif + + // Step 4: + // Make sure there are at least 2 points (that don't overlap with each other) + // in the stack + int k; // index of the non-overlapped second point + for (k = 1; k < num_in; k++) { + if (dist[k] > 1e-8) { + break; + } + } + if (k == num_in) { + // We reach the end, which means the convex hull is just one point + q[0] = p[t]; + return 1; + } + q[1] = q[k]; + int m = 2; // 2 points in the stack + // Step 5: + // Finally we can start the scanning process. + // When a non-convex relationship between the 3 points is found + // (either concave shape or duplicated points), + // we pop the previous point from the stack + // until the 3-point relationship is convex again, or + // until the stack only contains two points + for (int i = k + 1; i < num_in; i++) { + while (m > 1) { + auto q1 = q[i] - q[m - 2], q2 = q[m - 1] - q[m - 2]; + // cross_2d() uses FMA and therefore computes round(round(q1.x*q2.y) - + // q2.x*q1.y) So it may not return 0 even when q1==q2. Therefore we + // compare round(q1.x*q2.y) and round(q2.x*q1.y) directly. (round means + // round to nearest floating point). + if (q1.x * q2.y >= q2.x * q1.y) + m--; + else + break; + } + // Using double also helps, but float can solve the issue for now. + // while (m > 1 && cross_2d(q[i] - q[m - 2], q[m - 1] - q[m - 2]) + // >= 0) { + // m--; + // } + q[m++] = q[i]; + } + + // Step 6 (Optional): + // In general sense we need the original coordinates, so we + // need to shift the points back (reverting Step 2) + // But if we're only interested in getting the area/perimeter of the shape + // We can simply return. + if (!shift_to_zero) { + for (int i = 0; i < m; i++) { + q[i] += start; + } + } + + return m; +} + +template +HOST_DEVICE_INLINE T polygon_area(const Point (&q)[24], const int& m) { + if (m <= 2) { + return 0; + } + + T area = 0; + for (int i = 1; i < m - 1; i++) { + area += fabs(cross_2d(q[i] - q[0], q[i + 1] - q[0])); + } + + return area / 2.0; +} + +template +HOST_DEVICE_INLINE T rotated_boxes_intersection( + const RotatedBox& box1, + const RotatedBox& box2) { + // There are up to 4 x 4 + 4 + 4 = 24 intersections (including dups) returned + // from rotated_rect_intersection_pts + Point intersectPts[24], orderedPts[24]; + + Point pts1[4]; + Point pts2[4]; + get_rotated_vertices(box1, pts1); + get_rotated_vertices(box2, pts2); + + int num = get_intersection_points(pts1, pts2, intersectPts); + + if (num <= 2) { + return 0.0; + } + + // Convex Hull to order the intersection points in clockwise order and find + // the contour area. + int num_convex = convex_hull_graham(intersectPts, num, orderedPts, true); + return polygon_area(orderedPts, num_convex); +} + +} // namespace + +template +HOST_DEVICE_INLINE T +single_box_iou_rotated(T const* const box1_raw, T const* const box2_raw) { + // shift center to the middle point to achieve higher precision in result + RotatedBox box1, box2; + auto center_shift_x = (box1_raw[0] + box2_raw[0]) / 2.0; + auto center_shift_y = (box1_raw[1] + box2_raw[1]) / 2.0; + box1.x_ctr = box1_raw[0] - center_shift_x; + box1.y_ctr = box1_raw[1] - center_shift_y; + box1.w = box1_raw[2]; + box1.h = box1_raw[3]; + box1.a = box1_raw[4]; + box2.x_ctr = box2_raw[0] - center_shift_x; + box2.y_ctr = box2_raw[1] - center_shift_y; + box2.w = box2_raw[2]; + box2.h = box2_raw[3]; + box2.a = box2_raw[4]; + + T area1 = box1.w * box1.h; + T area2 = box2.w * box2.h; + if (area1 < 1e-14 || area2 < 1e-14) { + return 0.f; + } + + T intersection = rotated_boxes_intersection(box1, box2); + T iou = intersection / (area1 + area2 - intersection); + return iou; +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/cocoeval/cocoeval.cpp b/detectron2/layers/csrc/cocoeval/cocoeval.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0a5b7b907c06720fefc77b0dfd921b8ec3ecf2be --- /dev/null +++ b/detectron2/layers/csrc/cocoeval/cocoeval.cpp @@ -0,0 +1,507 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include "cocoeval.h" +#include +#include +#include +#include + +using namespace pybind11::literals; + +namespace detectron2 { + +namespace COCOeval { + +// Sort detections from highest score to lowest, such that +// detection_instances[detection_sorted_indices[t]] >= +// detection_instances[detection_sorted_indices[t+1]]. Use stable_sort to match +// original COCO API +void SortInstancesByDetectionScore( + const std::vector& detection_instances, + std::vector* detection_sorted_indices) { + detection_sorted_indices->resize(detection_instances.size()); + std::iota( + detection_sorted_indices->begin(), detection_sorted_indices->end(), 0); + std::stable_sort( + detection_sorted_indices->begin(), + detection_sorted_indices->end(), + [&detection_instances](size_t j1, size_t j2) { + return detection_instances[j1].score > detection_instances[j2].score; + }); +} + +// Partition the ground truth objects based on whether or not to ignore them +// based on area +void SortInstancesByIgnore( + const std::array& area_range, + const std::vector& ground_truth_instances, + std::vector* ground_truth_sorted_indices, + std::vector* ignores) { + ignores->clear(); + ignores->reserve(ground_truth_instances.size()); + for (auto o : ground_truth_instances) { + ignores->push_back( + o.ignore || o.area < area_range[0] || o.area > area_range[1]); + } + + ground_truth_sorted_indices->resize(ground_truth_instances.size()); + std::iota( + ground_truth_sorted_indices->begin(), + ground_truth_sorted_indices->end(), + 0); + std::stable_sort( + ground_truth_sorted_indices->begin(), + ground_truth_sorted_indices->end(), + [&ignores](size_t j1, size_t j2) { + return (int)(*ignores)[j1] < (int)(*ignores)[j2]; + }); +} + +// For each IOU threshold, greedily match each detected instance to a ground +// truth instance (if possible) and store the results +void MatchDetectionsToGroundTruth( + const std::vector& detection_instances, + const std::vector& detection_sorted_indices, + const std::vector& ground_truth_instances, + const std::vector& ground_truth_sorted_indices, + const std::vector& ignores, + const std::vector>& ious, + const std::vector& iou_thresholds, + const std::array& area_range, + ImageEvaluation* results) { + // Initialize memory to store return data matches and ignore + const int num_iou_thresholds = iou_thresholds.size(); + const int num_ground_truth = ground_truth_sorted_indices.size(); + const int num_detections = detection_sorted_indices.size(); + std::vector ground_truth_matches( + num_iou_thresholds * num_ground_truth, 0); + std::vector& detection_matches = results->detection_matches; + std::vector& detection_ignores = results->detection_ignores; + std::vector& ground_truth_ignores = results->ground_truth_ignores; + detection_matches.resize(num_iou_thresholds * num_detections, 0); + detection_ignores.resize(num_iou_thresholds * num_detections, false); + ground_truth_ignores.resize(num_ground_truth); + for (auto g = 0; g < num_ground_truth; ++g) { + ground_truth_ignores[g] = ignores[ground_truth_sorted_indices[g]]; + } + + for (auto t = 0; t < num_iou_thresholds; ++t) { + for (auto d = 0; d < num_detections; ++d) { + // information about best match so far (match=-1 -> unmatched) + double best_iou = std::min(iou_thresholds[t], 1 - 1e-10); + int match = -1; + for (auto g = 0; g < num_ground_truth; ++g) { + // if this ground truth instance is already matched and not a + // crowd, it cannot be matched to another detection + if (ground_truth_matches[t * num_ground_truth + g] > 0 && + !ground_truth_instances[ground_truth_sorted_indices[g]].is_crowd) { + continue; + } + + // if detected instance matched to a regular ground truth + // instance, we can break on the first ground truth instance + // tagged as ignore (because they are sorted by the ignore tag) + if (match >= 0 && !ground_truth_ignores[match] && + ground_truth_ignores[g]) { + break; + } + + // if IOU overlap is the best so far, store the match appropriately + if (ious[d][ground_truth_sorted_indices[g]] >= best_iou) { + best_iou = ious[d][ground_truth_sorted_indices[g]]; + match = g; + } + } + // if match was made, store id of match for both detection and + // ground truth + if (match >= 0) { + detection_ignores[t * num_detections + d] = ground_truth_ignores[match]; + detection_matches[t * num_detections + d] = + ground_truth_instances[ground_truth_sorted_indices[match]].id; + ground_truth_matches[t * num_ground_truth + match] = + detection_instances[detection_sorted_indices[d]].id; + } + + // set unmatched detections outside of area range to ignore + const InstanceAnnotation& detection = + detection_instances[detection_sorted_indices[d]]; + detection_ignores[t * num_detections + d] = + detection_ignores[t * num_detections + d] || + (detection_matches[t * num_detections + d] == 0 && + (detection.area < area_range[0] || detection.area > area_range[1])); + } + } + + // store detection score results + results->detection_scores.resize(detection_sorted_indices.size()); + for (size_t d = 0; d < detection_sorted_indices.size(); ++d) { + results->detection_scores[d] = + detection_instances[detection_sorted_indices[d]].score; + } +} + +std::vector EvaluateImages( + const std::vector>& area_ranges, + int max_detections, + const std::vector& iou_thresholds, + const ImageCategoryInstances>& image_category_ious, + const ImageCategoryInstances& + image_category_ground_truth_instances, + const ImageCategoryInstances& + image_category_detection_instances) { + const int num_area_ranges = area_ranges.size(); + const int num_images = image_category_ground_truth_instances.size(); + const int num_categories = + image_category_ious.size() > 0 ? image_category_ious[0].size() : 0; + std::vector detection_sorted_indices; + std::vector ground_truth_sorted_indices; + std::vector ignores; + std::vector results_all( + num_images * num_area_ranges * num_categories); + + // Store results for each image, category, and area range combination. Results + // for each IOU threshold are packed into the same ImageEvaluation object + for (auto i = 0; i < num_images; ++i) { + for (auto c = 0; c < num_categories; ++c) { + const std::vector& ground_truth_instances = + image_category_ground_truth_instances[i][c]; + const std::vector& detection_instances = + image_category_detection_instances[i][c]; + + SortInstancesByDetectionScore( + detection_instances, &detection_sorted_indices); + if ((int)detection_sorted_indices.size() > max_detections) { + detection_sorted_indices.resize(max_detections); + } + + for (size_t a = 0; a < area_ranges.size(); ++a) { + SortInstancesByIgnore( + area_ranges[a], + ground_truth_instances, + &ground_truth_sorted_indices, + &ignores); + + MatchDetectionsToGroundTruth( + detection_instances, + detection_sorted_indices, + ground_truth_instances, + ground_truth_sorted_indices, + ignores, + image_category_ious[i][c], + iou_thresholds, + area_ranges[a], + &results_all + [c * num_area_ranges * num_images + a * num_images + i]); + } + } + } + + return results_all; +} + +// Convert a python list to a vector +template +std::vector list_to_vec(const py::list& l) { + std::vector v(py::len(l)); + for (int i = 0; i < (int)py::len(l); ++i) { + v[i] = l[i].cast(); + } + return v; +} + +// Helper function to Accumulate() +// Considers the evaluation results applicable to a particular category, area +// range, and max_detections parameter setting, which begin at +// evaluations[evaluation_index]. Extracts a sorted list of length n of all +// applicable detection instances concatenated across all images in the dataset, +// which are represented by the outputs evaluation_indices, detection_scores, +// image_detection_indices, and detection_sorted_indices--all of which are +// length n. evaluation_indices[i] stores the applicable index into +// evaluations[] for instance i, which has detection score detection_score[i], +// and is the image_detection_indices[i]'th of the list of detections +// for the image containing i. detection_sorted_indices[] defines a sorted +// permutation of the 3 other outputs +int BuildSortedDetectionList( + const std::vector& evaluations, + const int64_t evaluation_index, + const int64_t num_images, + const int max_detections, + std::vector* evaluation_indices, + std::vector* detection_scores, + std::vector* detection_sorted_indices, + std::vector* image_detection_indices) { + assert(evaluations.size() >= evaluation_index + num_images); + + // Extract a list of object instances of the applicable category, area + // range, and max detections requirements such that they can be sorted + image_detection_indices->clear(); + evaluation_indices->clear(); + detection_scores->clear(); + image_detection_indices->reserve(num_images * max_detections); + evaluation_indices->reserve(num_images * max_detections); + detection_scores->reserve(num_images * max_detections); + int num_valid_ground_truth = 0; + for (auto i = 0; i < num_images; ++i) { + const ImageEvaluation& evaluation = evaluations[evaluation_index + i]; + + for (int d = 0; + d < (int)evaluation.detection_scores.size() && d < max_detections; + ++d) { // detected instances + evaluation_indices->push_back(evaluation_index + i); + image_detection_indices->push_back(d); + detection_scores->push_back(evaluation.detection_scores[d]); + } + for (auto ground_truth_ignore : evaluation.ground_truth_ignores) { + if (!ground_truth_ignore) { + ++num_valid_ground_truth; + } + } + } + + // Sort detections by decreasing score, using stable sort to match + // python implementation + detection_sorted_indices->resize(detection_scores->size()); + std::iota( + detection_sorted_indices->begin(), detection_sorted_indices->end(), 0); + std::stable_sort( + detection_sorted_indices->begin(), + detection_sorted_indices->end(), + [&detection_scores](size_t j1, size_t j2) { + return (*detection_scores)[j1] > (*detection_scores)[j2]; + }); + + return num_valid_ground_truth; +} + +// Helper function to Accumulate() +// Compute a precision recall curve given a sorted list of detected instances +// encoded in evaluations, evaluation_indices, detection_scores, +// detection_sorted_indices, image_detection_indices (see +// BuildSortedDetectionList()). Using vectors precisions and recalls +// and temporary storage, output the results into precisions_out, recalls_out, +// and scores_out, which are large buffers containing many precion/recall curves +// for all possible parameter settings, with precisions_out_index and +// recalls_out_index defining the applicable indices to store results. +void ComputePrecisionRecallCurve( + const int64_t precisions_out_index, + const int64_t precisions_out_stride, + const int64_t recalls_out_index, + const std::vector& recall_thresholds, + const int iou_threshold_index, + const int num_iou_thresholds, + const int num_valid_ground_truth, + const std::vector& evaluations, + const std::vector& evaluation_indices, + const std::vector& detection_scores, + const std::vector& detection_sorted_indices, + const std::vector& image_detection_indices, + std::vector* precisions, + std::vector* recalls, + std::vector* precisions_out, + std::vector* scores_out, + std::vector* recalls_out) { + assert(recalls_out->size() > recalls_out_index); + + // Compute precision/recall for each instance in the sorted list of detections + int64_t true_positives_sum = 0, false_positives_sum = 0; + precisions->clear(); + recalls->clear(); + precisions->reserve(detection_sorted_indices.size()); + recalls->reserve(detection_sorted_indices.size()); + assert(!evaluations.empty() || detection_sorted_indices.empty()); + for (auto detection_sorted_index : detection_sorted_indices) { + const ImageEvaluation& evaluation = + evaluations[evaluation_indices[detection_sorted_index]]; + const auto num_detections = + evaluation.detection_matches.size() / num_iou_thresholds; + const auto detection_index = iou_threshold_index * num_detections + + image_detection_indices[detection_sorted_index]; + assert(evaluation.detection_matches.size() > detection_index); + assert(evaluation.detection_ignores.size() > detection_index); + const int64_t detection_match = + evaluation.detection_matches[detection_index]; + const bool detection_ignores = + evaluation.detection_ignores[detection_index]; + const auto true_positive = detection_match > 0 && !detection_ignores; + const auto false_positive = detection_match == 0 && !detection_ignores; + if (true_positive) { + ++true_positives_sum; + } + if (false_positive) { + ++false_positives_sum; + } + + const double recall = + static_cast(true_positives_sum) / num_valid_ground_truth; + recalls->push_back(recall); + const int64_t num_valid_detections = + true_positives_sum + false_positives_sum; + const double precision = num_valid_detections > 0 + ? static_cast(true_positives_sum) / num_valid_detections + : 0.0; + precisions->push_back(precision); + } + + (*recalls_out)[recalls_out_index] = !recalls->empty() ? recalls->back() : 0; + + for (int64_t i = static_cast(precisions->size()) - 1; i > 0; --i) { + if ((*precisions)[i] > (*precisions)[i - 1]) { + (*precisions)[i - 1] = (*precisions)[i]; + } + } + + // Sample the per instance precision/recall list at each recall threshold + for (size_t r = 0; r < recall_thresholds.size(); ++r) { + // first index in recalls >= recall_thresholds[r] + std::vector::iterator low = std::lower_bound( + recalls->begin(), recalls->end(), recall_thresholds[r]); + size_t precisions_index = low - recalls->begin(); + + const auto results_ind = precisions_out_index + r * precisions_out_stride; + assert(results_ind < precisions_out->size()); + assert(results_ind < scores_out->size()); + if (precisions_index < precisions->size()) { + (*precisions_out)[results_ind] = (*precisions)[precisions_index]; + (*scores_out)[results_ind] = + detection_scores[detection_sorted_indices[precisions_index]]; + } else { + (*precisions_out)[results_ind] = 0; + (*scores_out)[results_ind] = 0; + } + } +} +py::dict Accumulate( + const py::object& params, + const std::vector& evaluations) { + const std::vector recall_thresholds = + list_to_vec(params.attr("recThrs")); + const std::vector max_detections = + list_to_vec(params.attr("maxDets")); + const int num_iou_thresholds = py::len(params.attr("iouThrs")); + const int num_recall_thresholds = py::len(params.attr("recThrs")); + const int num_categories = params.attr("useCats").cast() == 1 + ? py::len(params.attr("catIds")) + : 1; + const int num_area_ranges = py::len(params.attr("areaRng")); + const int num_max_detections = py::len(params.attr("maxDets")); + const int num_images = py::len(params.attr("imgIds")); + + std::vector precisions_out( + num_iou_thresholds * num_recall_thresholds * num_categories * + num_area_ranges * num_max_detections, + -1); + std::vector recalls_out( + num_iou_thresholds * num_categories * num_area_ranges * + num_max_detections, + -1); + std::vector scores_out( + num_iou_thresholds * num_recall_thresholds * num_categories * + num_area_ranges * num_max_detections, + -1); + + // Consider the list of all detected instances in the entire dataset in one + // large list. evaluation_indices, detection_scores, + // image_detection_indices, and detection_sorted_indices all have the same + // length as this list, such that each entry corresponds to one detected + // instance + std::vector evaluation_indices; // indices into evaluations[] + std::vector detection_scores; // detection scores of each instance + std::vector detection_sorted_indices; // sorted indices of all + // instances in the dataset + std::vector + image_detection_indices; // indices into the list of detected instances in + // the same image as each instance + std::vector precisions, recalls; + + for (auto c = 0; c < num_categories; ++c) { + for (auto a = 0; a < num_area_ranges; ++a) { + for (auto m = 0; m < num_max_detections; ++m) { + // The COCO PythonAPI assumes evaluations[] (the return value of + // COCOeval::EvaluateImages() is one long list storing results for each + // combination of category, area range, and image id, with categories in + // the outermost loop and images in the innermost loop. + const int64_t evaluations_index = + c * num_area_ranges * num_images + a * num_images; + int num_valid_ground_truth = BuildSortedDetectionList( + evaluations, + evaluations_index, + num_images, + max_detections[m], + &evaluation_indices, + &detection_scores, + &detection_sorted_indices, + &image_detection_indices); + + if (num_valid_ground_truth == 0) { + continue; + } + + for (auto t = 0; t < num_iou_thresholds; ++t) { + // recalls_out is a flattened vectors representing a + // num_iou_thresholds X num_categories X num_area_ranges X + // num_max_detections matrix + const int64_t recalls_out_index = + t * num_categories * num_area_ranges * num_max_detections + + c * num_area_ranges * num_max_detections + + a * num_max_detections + m; + + // precisions_out and scores_out are flattened vectors + // representing a num_iou_thresholds X num_recall_thresholds X + // num_categories X num_area_ranges X num_max_detections matrix + const int64_t precisions_out_stride = + num_categories * num_area_ranges * num_max_detections; + const int64_t precisions_out_index = t * num_recall_thresholds * + num_categories * num_area_ranges * num_max_detections + + c * num_area_ranges * num_max_detections + + a * num_max_detections + m; + + ComputePrecisionRecallCurve( + precisions_out_index, + precisions_out_stride, + recalls_out_index, + recall_thresholds, + t, + num_iou_thresholds, + num_valid_ground_truth, + evaluations, + evaluation_indices, + detection_scores, + detection_sorted_indices, + image_detection_indices, + &precisions, + &recalls, + &precisions_out, + &scores_out, + &recalls_out); + } + } + } + } + + time_t rawtime; + struct tm local_time; + std::array buffer; + time(&rawtime); +#ifdef _WIN32 + localtime_s(&local_time, &rawtime); +#else + localtime_r(&rawtime, &local_time); +#endif + strftime( + buffer.data(), 200, "%Y-%m-%d %H:%num_max_detections:%S", &local_time); + return py::dict( + "params"_a = params, + "counts"_a = std::vector( + {num_iou_thresholds, + num_recall_thresholds, + num_categories, + num_area_ranges, + num_max_detections}), + "date"_a = buffer, + "precision"_a = precisions_out, + "recall"_a = recalls_out, + "scores"_a = scores_out); +} + +} // namespace COCOeval + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/cocoeval/cocoeval.h b/detectron2/layers/csrc/cocoeval/cocoeval.h new file mode 100644 index 0000000000000000000000000000000000000000..db246e49a026b7cd989b305f4d3d98100be3c912 --- /dev/null +++ b/detectron2/layers/csrc/cocoeval/cocoeval.h @@ -0,0 +1,88 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once + +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace detectron2 { + +namespace COCOeval { + +// Annotation data for a single object instance in an image +struct InstanceAnnotation { + InstanceAnnotation( + uint64_t id, + double score, + double area, + bool is_crowd, + bool ignore) + : id{id}, score{score}, area{area}, is_crowd{is_crowd}, ignore{ignore} {} + uint64_t id; + double score = 0.; + double area = 0.; + bool is_crowd = false; + bool ignore = false; +}; + +// Stores intermediate results for evaluating detection results for a single +// image that has D detected instances and G ground truth instances. This stores +// matches between detected and ground truth instances +struct ImageEvaluation { + // For each of the D detected instances, the id of the matched ground truth + // instance, or 0 if unmatched + std::vector detection_matches; + + // The detection score of each of the D detected instances + std::vector detection_scores; + + // Marks whether or not each of G instances was ignored from evaluation (e.g., + // because it's outside area_range) + std::vector ground_truth_ignores; + + // Marks whether or not each of D instances was ignored from evaluation (e.g., + // because it's outside aRng) + std::vector detection_ignores; +}; + +template +using ImageCategoryInstances = std::vector>>; + +// C++ implementation of COCO API cocoeval.py::COCOeval.evaluateImg(). For each +// combination of image, category, area range settings, and IOU thresholds to +// evaluate, it matches detected instances to ground truth instances and stores +// the results into a vector of ImageEvaluation results, which will be +// interpreted by the COCOeval::Accumulate() function to produce precion-recall +// curves. The parameters of nested vectors have the following semantics: +// image_category_ious[i][c][d][g] is the intersection over union of the d'th +// detected instance and g'th ground truth instance of +// category category_ids[c] in image image_ids[i] +// image_category_ground_truth_instances[i][c] is a vector of ground truth +// instances in image image_ids[i] of category category_ids[c] +// image_category_detection_instances[i][c] is a vector of detected +// instances in image image_ids[i] of category category_ids[c] +std::vector EvaluateImages( + const std::vector>& area_ranges, // vector of 2-tuples + int max_detections, + const std::vector& iou_thresholds, + const ImageCategoryInstances>& image_category_ious, + const ImageCategoryInstances& + image_category_ground_truth_instances, + const ImageCategoryInstances& + image_category_detection_instances); + +// C++ implementation of COCOeval.accumulate(), which generates precision +// recall curves for each set of category, IOU threshold, detection area range, +// and max number of detections parameters. It is assumed that the parameter +// evaluations is the return value of the functon COCOeval::EvaluateImages(), +// which was called with the same parameter settings params +py::dict Accumulate( + const py::object& params, + const std::vector& evalutations); + +} // namespace COCOeval +} // namespace detectron2 diff --git a/detectron2/layers/csrc/cuda_version.cu b/detectron2/layers/csrc/cuda_version.cu new file mode 100644 index 0000000000000000000000000000000000000000..6dfe1b90c1f65c443681813fd3e3386c9faa3360 --- /dev/null +++ b/detectron2/layers/csrc/cuda_version.cu @@ -0,0 +1,26 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +#include + +namespace detectron2 { +int get_cudart_version() { +// Not a ROCM platform: Either HIP is not used, or +// it is used, but platform is not ROCM (i.e. it is CUDA) +#if !defined(__HIP_PLATFORM_HCC__) + return CUDART_VERSION; +#else + int version = 0; + +#if HIP_VERSION_MAJOR != 0 + // Create a convention similar to that of CUDA, as assumed by other + // parts of the code. + + version = HIP_VERSION_MINOR; + version += (HIP_VERSION_MAJOR * 100); +#else + hipRuntimeGetVersion(&version); +#endif + return version; +#endif +} +} // namespace detectron2 diff --git a/detectron2/layers/csrc/deformable/deform_conv.h b/detectron2/layers/csrc/deformable/deform_conv.h new file mode 100644 index 0000000000000000000000000000000000000000..965c1bfd47b58f9802d1c3fd69a5962517b2da61 --- /dev/null +++ b/detectron2/layers/csrc/deformable/deform_conv.h @@ -0,0 +1,377 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +#if defined(WITH_CUDA) || defined(WITH_HIP) +int deform_conv_forward_cuda( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step); + +int deform_conv_backward_input_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step); + +int deform_conv_backward_parameters_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step); + +void modulated_deform_conv_cuda_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias); + +void modulated_deform_conv_cuda_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias); + +#endif + +inline int deform_conv_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + if (input.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return deform_conv_forward_cuda( + input, + weight, + offset, + output, + columns, + ones, + kW, + kH, + dW, + dH, + padW, + padH, + dilationW, + dilationH, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline int deform_conv_backward_input( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + if (gradOutput.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(input.is_cuda(), "input tensor is not on GPU!"); + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return deform_conv_backward_input_cuda( + input, + offset, + gradOutput, + gradInput, + gradOffset, + weight, + columns, + kW, + kH, + dW, + dH, + padW, + padH, + dilationW, + dilationH, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline int deform_conv_backward_filter( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step) { + if (gradOutput.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(input.is_cuda(), "input tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return deform_conv_backward_parameters_cuda( + input, + offset, + gradOutput, + gradWeight, + columns, + ones, + kW, + kH, + dW, + dH, + padW, + padH, + dilationW, + dilationH, + group, + deformable_group, + scale, + im2col_step); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline void modulated_deform_conv_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias) { + if (input.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(bias.is_cuda(), "bias tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return modulated_deform_conv_cuda_forward( + input, + weight, + bias, + ones, + offset, + mask, + output, + columns, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group, + with_bias); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline void modulated_deform_conv_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias) { + if (grad_output.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(input.is_cuda(), "input tensor is not on GPU!"); + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(bias.is_cuda(), "bias tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return modulated_deform_conv_cuda_backward( + input, + weight, + bias, + ones, + offset, + mask, + columns, + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group, + with_bias); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/deformable/deform_conv_cuda.cu b/detectron2/layers/csrc/deformable/deform_conv_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..2072bb856ec40b61c3826cead2fb7bb7c971a089 --- /dev/null +++ b/detectron2/layers/csrc/deformable/deform_conv_cuda.cu @@ -0,0 +1,1223 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +// modified from +// https://github.com/open-mmlab/mmdetection/blob/master/mmdet/ops/dcn/src/deform_conv_cuda.cpp +// Original license: Apache 2.0 + +// modify from +// https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/deform_conv_cuda.c +// Original license: Apache 2.0 + +#include + +#include "deform_conv.h" + +#include +#include + +namespace detectron2 { + +void deformable_im2col( + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor data_col); + +void deformable_col2im( + const at::Tensor data_col, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_im); + +void deformable_col2im_coord( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_offset); + +void modulated_deformable_im2col_cuda( + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor data_col); + +void modulated_deformable_col2im_cuda( + const at::Tensor data_col, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_im); + +void modulated_deformable_col2im_coord_cuda( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_offset, + at::Tensor grad_mask); + +void shape_check( + at::Tensor input, + at::Tensor offset, + at::Tensor* gradOutput, + at::Tensor weight, + int kH, + int kW, + int dH, + int dW, + int padH, + int padW, + int dilationH, + int dilationW, + int group, + int deformable_group) { + TORCH_CHECK( + weight.ndimension() == 4, + "4D weight tensor (nOutputPlane,nInputPlane,kH,kW) expected, " + "but got: %s", + weight.ndimension()); + + TORCH_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + TORCH_CHECK( + kW > 0 && kH > 0, + "kernel size should be greater than zero, but got kH: %d kW: %d", + kH, + kW); + + TORCH_CHECK( + (weight.size(2) == kH && weight.size(3) == kW), + "kernel size should be consistent with weight, ", + "but got kH: %d kW: %d weight.size(2): %d, weight.size(3): %d", + kH, + kW, + weight.size(2), + weight.size(3)); + + TORCH_CHECK( + dW > 0 && dH > 0, + "stride should be greater than zero, but got dH: %d dW: %d", + dH, + dW); + + TORCH_CHECK( + dilationW > 0 && dilationH > 0, + "dilation should be greater than 0, but got dilationH: %d dilationW: %d", + dilationH, + dilationW); + + int ndim = input.ndimension(); + int dimf = 0; + int dimh = 1; + int dimw = 2; + + if (ndim == 4) { + dimf++; + dimh++; + dimw++; + } + + TORCH_CHECK( + ndim == 3 || ndim == 4, + "3D or 4D input tensor expected but got: %s", + ndim); + + long nInputPlane = weight.size(1) * group; + long inputHeight = input.size(dimh); + long inputWidth = input.size(dimw); + long nOutputPlane = weight.size(0); + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + + TORCH_CHECK( + nInputPlane % deformable_group == 0, + "input channels must divide deformable group size"); + + if (outputWidth < 1 || outputHeight < 1) + AT_ERROR( + "Given input size: (%ld x %ld x %ld). " + "Calculated output size: (%ld x %ld x %ld). Output size is too small", + nInputPlane, + inputHeight, + inputWidth, + nOutputPlane, + outputHeight, + outputWidth); + + TORCH_CHECK( + input.size(1) == nInputPlane, + "invalid number of input planes, expected: %d, but got: %d", + nInputPlane, + input.size(1)); + + TORCH_CHECK( + (inputHeight + 2 * padH >= kH && inputWidth + 2 * padW >= kW), + "input image is smaller than kernel"); + + TORCH_CHECK( + (offset.size(2) == outputHeight && offset.size(3) == outputWidth), + "invalid spatial size of offset, expected height: %d width: %d, but " + "got height: %d width: %d", + outputHeight, + outputWidth, + offset.size(2), + offset.size(3)); + + TORCH_CHECK( + (offset.size(1) == deformable_group * 2 * kH * kW), + "invalid number of channels of offset"); + + if (gradOutput != NULL) { + TORCH_CHECK( + gradOutput->size(dimf) == nOutputPlane, + "invalid number of gradOutput planes, expected: %d, but got: %d", + nOutputPlane, + gradOutput->size(dimf)); + + TORCH_CHECK( + (gradOutput->size(dimh) == outputHeight && + gradOutput->size(dimw) == outputWidth), + "invalid size of gradOutput, expected height: %d width: %d , but " + "got height: %d width: %d", + outputHeight, + outputWidth, + gradOutput->size(dimh), + gradOutput->size(dimw)); + } +} + +int deform_conv_forward_cuda( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + // todo: resize columns to include im2col: done + // todo: add im2col_step as input + // todo: add new output buffer and transpose it to output (or directly + // transpose output) todo: possibly change data indexing because of + // parallel_imgs + + shape_check( + input, + offset, + NULL, + weight, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + group, + deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + weight = weight.contiguous(); + + int batch = 1; + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input.unsqueeze_(0); + offset.unsqueeze_(0); + } + + // todo: assert batchsize dividable by im2col_step + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = weight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + TORCH_CHECK((offset.size(0) == batchSize), "invalid batch size of offset"); + + output = output.view( + {batchSize / im2col_step, + im2col_step, + nOutputPlane, + outputHeight, + outputWidth}); + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < outputHeight * outputWidth) { + ones = at::ones({outputHeight, outputWidth}, input.options()); + } + + input = input.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + offset = offset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + + at::Tensor output_buffer = at::zeros( + {batchSize / im2col_step, + nOutputPlane, + im2col_step * outputHeight, + outputWidth}, + output.options()); + + output_buffer = output_buffer.view( + {output_buffer.size(0), + group, + output_buffer.size(1) / group, + output_buffer.size(2), + output_buffer.size(3)}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + deformable_im2col( + input[elt], + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + columns); + + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + + for (int g = 0; g < group; g++) { + output_buffer[elt][g] = output_buffer[elt][g] + .flatten(1) + .addmm_(weight[g].flatten(1), columns[g]) + .view_as(output_buffer[elt][g]); + } + } + + output_buffer = output_buffer.view( + {output_buffer.size(0), + output_buffer.size(1) * output_buffer.size(2), + output_buffer.size(3), + output_buffer.size(4)}); + + output_buffer = output_buffer.view( + {batchSize / im2col_step, + nOutputPlane, + im2col_step, + outputHeight, + outputWidth}); + output_buffer.transpose_(1, 2); + output.copy_(output_buffer); + output = output.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + output = output.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + offset = offset.view({offset.size(1), offset.size(2), offset.size(3)}); + } + + return 1; +} + +int deform_conv_backward_input_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + shape_check( + input, + offset, + &gradOutput, + weight, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + group, + deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + gradOutput = gradOutput.contiguous(); + weight = weight.contiguous(); + + int batch = 1; + + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input = input.view({1, input.size(0), input.size(1), input.size(2)}); + offset = offset.view({1, offset.size(0), offset.size(1), offset.size(2)}); + gradOutput = gradOutput.view( + {1, gradOutput.size(0), gradOutput.size(1), gradOutput.size(2)}); + } + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = weight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + TORCH_CHECK((offset.size(0) == batchSize), 3, "invalid batch size of offset"); + gradInput = gradInput.view({batchSize, nInputPlane, inputHeight, inputWidth}); + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + // change order of grad output + gradOutput = gradOutput.view( + {batchSize / im2col_step, + im2col_step, + nOutputPlane, + outputHeight, + outputWidth}); + gradOutput.transpose_(1, 2); + + gradInput = gradInput.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + input = input.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + gradOffset = gradOffset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + offset = offset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + // divide into groups + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + gradOutput = gradOutput.view( + {gradOutput.size(0), + group, + gradOutput.size(1) / group, + gradOutput.size(2), + gradOutput.size(3), + gradOutput.size(4)}); + + for (int g = 0; g < group; g++) { + columns[g] = columns[g].addmm_( + weight[g].flatten(1).transpose(0, 1), + gradOutput[elt][g].flatten(1), + 0.0f, + 1.0f); + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + gradOutput = gradOutput.view( + {gradOutput.size(0), + gradOutput.size(1) * gradOutput.size(2), + gradOutput.size(3), + gradOutput.size(4), + gradOutput.size(5)}); + + deformable_col2im_coord( + columns, + input[elt], + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + gradOffset[elt]); + + deformable_col2im( + columns, + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + gradInput[elt]); + } + + gradOutput.transpose_(1, 2); + gradOutput = + gradOutput.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + gradInput = gradInput.view({batchSize, nInputPlane, inputHeight, inputWidth}); + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + gradOffset = gradOffset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + gradOutput = gradOutput.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + gradInput = gradInput.view({nInputPlane, inputHeight, inputWidth}); + offset = offset.view({offset.size(1), offset.size(2), offset.size(3)}); + gradOffset = + gradOffset.view({offset.size(1), offset.size(2), offset.size(3)}); + } + + return 1; +} + +int deform_conv_backward_parameters_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step) { + // todo: transpose and reshape outGrad + // todo: reshape columns + // todo: add im2col_step as input + + shape_check( + input, + offset, + &gradOutput, + gradWeight, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + group, + deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + gradOutput = gradOutput.contiguous(); + + int batch = 1; + + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input = input.view( + at::IntList({1, input.size(0), input.size(1), input.size(2)})); + gradOutput = gradOutput.view( + {1, gradOutput.size(0), gradOutput.size(1), gradOutput.size(2)}); + } + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = gradWeight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + TORCH_CHECK((offset.size(0) == batchSize), "invalid batch size of offset"); + + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + gradOutput = gradOutput.view( + {batchSize / im2col_step, + im2col_step, + nOutputPlane, + outputHeight, + outputWidth}); + gradOutput.transpose_(1, 2); + + at::Tensor gradOutputBuffer = at::zeros_like(gradOutput); + gradOutputBuffer = gradOutputBuffer.view( + {batchSize / im2col_step, + nOutputPlane, + im2col_step, + outputHeight, + outputWidth}); + gradOutputBuffer.copy_(gradOutput); + // gradOutput is not contiguous, so we do reshape (instead of view) next + gradOutputBuffer = gradOutputBuffer.reshape( + {batchSize / im2col_step, + nOutputPlane, + im2col_step * outputHeight, + outputWidth}); + + gradOutput.transpose_(1, 2); + gradOutput = + gradOutput.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + input = input.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + offset = offset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + deformable_im2col( + input[elt], + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + columns); + + // divide into group + gradOutputBuffer = gradOutputBuffer.view( + {gradOutputBuffer.size(0), + group, + gradOutputBuffer.size(1) / group, + gradOutputBuffer.size(2), + gradOutputBuffer.size(3)}); + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + gradWeight = gradWeight.view( + {group, + gradWeight.size(0) / group, + gradWeight.size(1), + gradWeight.size(2), + gradWeight.size(3)}); + + for (int g = 0; g < group; g++) { + gradWeight[g] = gradWeight[g] + .flatten(1) + .addmm_( + gradOutputBuffer[elt][g].flatten(1), + columns[g].transpose(1, 0), + 1.0, + scale) + .view_as(gradWeight[g]); + } + gradOutputBuffer = gradOutputBuffer.view( + {gradOutputBuffer.size(0), + gradOutputBuffer.size(1) * gradOutputBuffer.size(2), + gradOutputBuffer.size(3), + gradOutputBuffer.size(4)}); + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + gradWeight = gradWeight.view( + {gradWeight.size(0) * gradWeight.size(1), + gradWeight.size(2), + gradWeight.size(3), + gradWeight.size(4)}); + } + + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + gradOutput = gradOutput.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + } + + return 1; +} + +void modulated_deform_conv_cuda_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias) { + shape_check( + input, + offset, + NULL, + weight, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group); + + TORCH_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + TORCH_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + if (kernel_h_ != kernel_h || kernel_w_ != kernel_w) + AT_ERROR( + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", + kernel_h_, + kernel_w, + kernel_h_, + kernel_w_); + if (channels != channels_kernel * group) + AT_ERROR( + "Input shape and kernel channels wont match: (%d vs %d).", + channels, + channels_kernel * group); + + const int height_out = + (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = + (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + // mask shape check + TORCH_CHECK( + (mask.size(2) == height_out && mask.size(3) == width_out), + "invalid spatial size of mask, expected height: %d width: %d, but " + "got height: %d width: %d", + height_out, + width_out, + mask.size(2), + mask.size(3)); + + TORCH_CHECK( + (mask.size(1) == deformable_group * kernel_h * kernel_w), + "invalid number of channels of mask"); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < height_out * width_out) { + // Resize plane and fill with ones... + ones = at::ones({height_out, width_out}, input.options()); + } + + // resize output + output = output.view({batch, channels_out, height_out, width_out}).zero_(); + // resize temporary columns + columns = at::zeros( + {channels * kernel_h * kernel_w, 1 * height_out * width_out}, + input.options()); + + output = output.view( + {output.size(0), + group, + output.size(1) / group, + output.size(2), + output.size(3)}); + + for (int b = 0; b < batch; b++) { + modulated_deformable_im2col_cuda( + input[b], + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + columns); + + // divide into group + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + + for (int g = 0; g < group; g++) { + output[b][g] = output[b][g] + .flatten(1) + .addmm_(weight[g].flatten(1), columns[g]) + .view_as(output[b][g]); + } + + weight = weight.view( + {weight.size(0) * weight.size(1), + weight.size(2), + weight.size(3), + weight.size(4)}); + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + } + + output = output.view( + {output.size(0), + output.size(1) * output.size(2), + output.size(3), + output.size(4)}); + + if (with_bias) { + output += bias.view({1, bias.size(0), 1, 1}); + } +} + +void modulated_deform_conv_cuda_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias) { + shape_check( + input, + offset, + &grad_output, + weight, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group); + + TORCH_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + TORCH_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + if (kernel_h_ != kernel_h || kernel_w_ != kernel_w) + AT_ERROR( + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", + kernel_h_, + kernel_w, + kernel_h_, + kernel_w_); + if (channels != channels_kernel * group) + AT_ERROR( + "Input shape and kernel channels wont match: (%d vs %d).", + channels, + channels_kernel * group); + + const int height_out = + (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = + (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + // mask shape check + TORCH_CHECK( + (mask.size(2) == height_out && mask.size(3) == width_out), + "invalid spatial size of mask, expected height: %d width: %d, but " + "got height: %d width: %d", + height_out, + width_out, + mask.size(2), + mask.size(3)); + + TORCH_CHECK( + (mask.size(1) == deformable_group * kernel_h * kernel_w), + "invalid number of channels of mask"); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < height_out * width_out) { + // Resize plane and fill with ones... + ones = at::ones({height_out, width_out}, input.options()); + } + + grad_input = grad_input.view({batch, channels, height, width}); + columns = at::zeros( + {channels * kernel_h * kernel_w, height_out * width_out}, + input.options()); + + grad_output = grad_output.view( + {grad_output.size(0), + group, + grad_output.size(1) / group, + grad_output.size(2), + grad_output.size(3)}); + + for (int b = 0; b < batch; b++) { + // divide int group + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + + for (int g = 0; g < group; g++) { + columns[g].addmm_( + weight[g].flatten(1).transpose(0, 1), + grad_output[b][g].flatten(1), + 0.0f, + 1.0f); + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + weight = weight.view( + {weight.size(0) * weight.size(1), + weight.size(2), + weight.size(3), + weight.size(4)}); + + // gradient w.r.t. input coordinate data + modulated_deformable_col2im_coord_cuda( + columns, + input[b], + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + grad_offset[b], + grad_mask[b]); + // gradient w.r.t. input data + modulated_deformable_col2im_cuda( + columns, + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + grad_input[b]); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and + // group + modulated_deformable_im2col_cuda( + input[b], + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + columns); + + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + grad_weight = grad_weight.view( + {group, + grad_weight.size(0) / group, + grad_weight.size(1), + grad_weight.size(2), + grad_weight.size(3)}); + if (with_bias) + grad_bias = grad_bias.view({group, grad_bias.size(0) / group}); + + for (int g = 0; g < group; g++) { + grad_weight[g] = + grad_weight[g] + .flatten(1) + .addmm_(grad_output[b][g].flatten(1), columns[g].transpose(0, 1)) + .view_as(grad_weight[g]); + if (with_bias) { + grad_bias[g] = + grad_bias[g] + .view({-1, 1}) + .addmm_(grad_output[b][g].flatten(1), ones.view({-1, 1})) + .view(-1); + } + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + grad_weight = grad_weight.view( + {grad_weight.size(0) * grad_weight.size(1), + grad_weight.size(2), + grad_weight.size(3), + grad_weight.size(4)}); + if (with_bias) + grad_bias = grad_bias.view({grad_bias.size(0) * grad_bias.size(1)}); + } + grad_output = grad_output.view( + {grad_output.size(0) * grad_output.size(1), + grad_output.size(2), + grad_output.size(3), + grad_output.size(4)}); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/deformable/deform_conv_cuda_kernel.cu b/detectron2/layers/csrc/deformable/deform_conv_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..f299c7add116685e9c87a187a85ea63f9f808867 --- /dev/null +++ b/detectron2/layers/csrc/deformable/deform_conv_cuda_kernel.cu @@ -0,0 +1,1288 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +// modified from +// https://github.com/open-mmlab/mmdetection/blob/master/mmdet/ops/dcn/src/deform_conv_cuda_kernel.cu +// Original license: Apache 2.0 +// clang-format off + +// modify from +// https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/deform_conv_cuda_kernel.cu + +/*! + ******************* BEGIN Caffe Copyright Notice and Disclaimer ***************** + * + * COPYRIGHT + * + * All contributions by the University of California: + * Copyright (c) 2014-2017 The Regents of the University of California (Regents) + * All rights reserved. + * + * All other contributions: + * Copyright (c) 2014-2017, the respective contributors + * All rights reserved. + * + * Caffe uses a shared copyright model: each contributor holds copyright over + * their contributions to Caffe. The project versioning records all such + * contribution and copyright details. If a contributor wants to further mark + * their specific copyright on a particular contribution, they should indicate + * their copyright solely in the commit message of the change when it is + * committed. + * + * LICENSE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + *AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + *IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + *FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + *DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + *SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + *CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + *OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + *OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * CONTRIBUTION AGREEMENT + * + * By contributing to the BVLC/caffe repository through pull-request, comment, + * or otherwise, the contributor releases their content to the + * license and copyright terms herein. + * + ***************** END Caffe Copyright Notice and Disclaimer ********************* + * + * Copyright (c) 2018 Microsoft + * Licensed under The MIT License [see LICENSE for details] + * \file modulated_deformable_im2col.cuh + * \brief Function definitions of converting an image to + * column matrix based on kernel, padding, dilation, and offset. + * These functions are mainly used in deformable convolution operators. + * \ref: https://arxiv.org/abs/1703.06211 + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai, Xizhou Zhu, Han Hu, Dazhi Cheng + */ + +#include +#include +#include +#include +#include +#include + +using namespace at; + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + + +namespace { + +const int CUDA_NUM_THREADS = 1024; +const int kMaxGridNum = 65535; + +inline int GET_BLOCKS(const int N) { + return std::min(kMaxGridNum, (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS); +} + +} + +template +__device__ scalar_t deformable_im2col_bilinear( + const scalar_t* bottom_data, + const int data_width, + const int height, + const int width, + scalar_t h, + scalar_t w) { + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t get_gradient_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int h, + const int w, + const int height, + const int width) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t get_coordinate_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int height, + const int width, + const scalar_t* im_data, + const int data_width, + const int bp_dir) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } else if (bp_dir == 1) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void deformable_im2col_gpu_kernel( + const int n, + const scalar_t* data_im, + const scalar_t* data_offset, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int num_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* data_col) { + CUDA_KERNEL_LOOP(index, n) { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + scalar_t* data_col_ptr = data_col + + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * + // height + h_in) * width + w_in; + const scalar_t* data_im_ptr = + data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) { + for (int j = 0; j < kernel_w; ++j) { + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) { + // const scalar_t map_h = i * dilation_h + offset_h; + // const scalar_t map_w = j * dilation_w + offset_w; + // const int cur_height = height - h_in; + // const int cur_width = width - w_in; + // val = deformable_im2col_bilinear(data_im_ptr, width, cur_height, + // cur_width, map_h, map_w); + val = deformable_im2col_bilinear( + data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + + +template +__global__ void deformable_col2im_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_offset, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_im) { + CUDA_KERNEL_LOOP(index, n) { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = + (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = + index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) { + for (int dx = -2; dx <= 2; dx++) { + if (cur_h + dy >= 0 && cur_h + dy < height && cur_w + dx >= 0 && + cur_w + dx < width && abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) { + int cur_bottom_grad_pos = + ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = get_gradient_weight( + cur_inv_h_data, + cur_inv_w_data, + cur_h + dy, + cur_w + dx, + height, + width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + + +template +__global__ void deformable_col2im_coord_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_im, + const scalar_t* data_offset, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int offset_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_offset) { + CUDA_KERNEL_LOOP(index, n) { + scalar_t val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t* data_col_ptr = data_col + + deformable_group_index * channel_per_deformable_group * batch_size * + width_col * height_col; + const scalar_t* data_im_ptr = data_im + + (b * deformable_group + deformable_group_index) * + channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; + col_c += col_step) { + const int col_pos = + (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = + (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = + (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = + (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) { + inv_h = inv_w = -2; + } + const scalar_t weight = get_coordinate_weight( + inv_h, + inv_w, + height, + width, + data_im_ptr + cnt * height * width, + width, + bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + + grad_offset[index] = val; + } +} + + +namespace detectron2 { + +void deformable_im2col( + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor data_col) { + // num_axes should be smaller than block size + // todo: check parallel_imgs is correctly passed in + int height_col = + (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = + (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = channels * height_col * width_col * parallel_imgs; + int channel_per_deformable_group = channels / deformable_group; + + at::cuda::CUDAGuard device_guard(data_im.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_im.scalar_type(), "deformable_im2col_gpu", ([&] { + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + scalar_t* data_col_ = data_col.data_ptr(); + + deformable_im2col_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_im_, + data_offset_, + height, + width, + ksize_h, + ksize_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + parallel_imgs, + channels, + deformable_group, + height_col, + width_col, + data_col_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in deformable_im2col: %s\n", cudaGetErrorString(err)); + } +} + + +void deformable_col2im( + const at::Tensor data_col, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_im) { + // todo: make sure parallel_imgs is passed in correctly + int height_col = + (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = + (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = + channels * ksize_h * ksize_w * height_col * width_col * parallel_imgs; + int channel_per_deformable_group = channels / deformable_group; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "deformable_col2im_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + scalar_t* grad_im_ = grad_im.data_ptr(); + + deformable_col2im_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_offset_, + channels, + height, + width, + ksize_h, + ksize_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + parallel_imgs, + deformable_group, + height_col, + width_col, + grad_im_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in deformable_col2im: %s\n", cudaGetErrorString(err)); + } +} + + +void deformable_col2im_coord( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_offset) { + int height_col = + (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = + (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = height_col * width_col * 2 * ksize_h * ksize_w * + deformable_group * parallel_imgs; + int channel_per_deformable_group = + channels * ksize_h * ksize_w / deformable_group; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "deformable_col2im_coord_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + scalar_t* grad_offset_ = grad_offset.data_ptr(); + + deformable_col2im_coord_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_im_, + data_offset_, + channels, + height, + width, + ksize_h, + ksize_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + parallel_imgs, + 2 * ksize_h * ksize_w * deformable_group, + deformable_group, + height_col, + width_col, + grad_offset_); + })); +} + +} // namespace detectron2 + + +template +__device__ scalar_t dmcn_im2col_bilinear( + const scalar_t* bottom_data, + const int data_width, + const int height, + const int width, + scalar_t h, + scalar_t w) { + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t dmcn_get_gradient_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int h, + const int w, + const int height, + const int width) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t dmcn_get_coordinate_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int height, + const int width, + const scalar_t* im_data, + const int data_width, + const int bp_dir) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } else if (bp_dir == 1) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void modulated_deformable_im2col_gpu_kernel( + const int n, + const scalar_t* data_im, + const scalar_t* data_offset, + const scalar_t* data_mask, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int num_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* data_col) { + CUDA_KERNEL_LOOP(index, n) { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t* data_col_ptr = data_col + + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * + // height + h_in) * width + w_in; + const scalar_t* data_im_ptr = + data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + + const scalar_t* data_mask_ptr = data_mask + + (b_col * deformable_group + deformable_group_index) * kernel_h * + kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) { + for (int j = 0; j < kernel_w; ++j) { + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + + w_col; + const int data_mask_hw_ptr = + ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + // if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) { + // const float map_h = i * dilation_h + offset_h; + // const float map_w = j * dilation_w + offset_w; + // const int cur_height = height - h_in; + // const int cur_width = width - w_in; + // val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, + // cur_width, map_h, map_w); + val = dmcn_im2col_bilinear( + data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * height_col * width_col; + // data_col_ptr += height_col * width_col; + } + } + } +} + +template +__global__ void modulated_deformable_col2im_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_offset, + const scalar_t* data_mask, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_im) { + CUDA_KERNEL_LOOP(index, n) { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = + (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = + index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + const scalar_t* data_mask_ptr = data_mask + + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * + height_col * width_col; + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const int data_mask_hw_ptr = + ((i * kernel_w + j) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index] * mask; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) { + for (int dx = -2; dx <= 2; dx++) { + if (cur_h + dy >= 0 && cur_h + dy < height && cur_w + dx >= 0 && + cur_w + dx < width && abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) { + int cur_bottom_grad_pos = + ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = dmcn_get_gradient_weight( + cur_inv_h_data, + cur_inv_w_data, + cur_h + dy, + cur_w + dx, + height, + width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void modulated_deformable_col2im_coord_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_im, + const scalar_t* data_offset, + const scalar_t* data_mask, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int offset_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_offset, + scalar_t* grad_mask) { + CUDA_KERNEL_LOOP(index, n) { + scalar_t val = 0, mval = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t* data_col_ptr = data_col + + deformable_group_index * channel_per_deformable_group * batch_size * + width_col * height_col; + const scalar_t* data_im_ptr = data_im + + (b * deformable_group + deformable_group_index) * + channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + const scalar_t* data_mask_ptr = data_mask + + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * + height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; + col_c += col_step) { + const int col_pos = + (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = + (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = + (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = + (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + + w_out); + const int data_mask_hw_ptr = + (((i * kernel_w + j) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) { + inv_h = inv_w = -2; + } else { + mval += data_col_ptr[col_pos] * + dmcn_im2col_bilinear( + data_im_ptr + cnt * height * width, + width, + height, + width, + inv_h, + inv_w); + } + const scalar_t weight = dmcn_get_coordinate_weight( + inv_h, + inv_w, + height, + width, + data_im_ptr + cnt * height * width, + width, + bp_dir); + val += weight * data_col_ptr[col_pos] * mask; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + if (offset_c % 2 == 0) + // KERNEL_ASSIGN(grad_mask[(((b * deformable_group + + // deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * + // height_col + h) * width_col + w], mask_req, mval); + grad_mask + [(((b * deformable_group + deformable_group_index) * kernel_h * + kernel_w + + offset_c / 2) * + height_col + + h) * + width_col + + w] = mval; + } +} + + +namespace detectron2 { + +void modulated_deformable_im2col_cuda( + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + + at::cuda::CUDAGuard device_guard(data_im.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_im.scalar_type(), "modulated_deformable_im2col_gpu", ([&] { + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + const scalar_t* data_mask_ = data_mask.data_ptr(); + scalar_t* data_col_ = data_col.data_ptr(); + + modulated_deformable_im2col_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_im_, + data_offset_, + data_mask_, + height_im, + width_im, + kernel_h, + kenerl_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + batch_size, + channels, + deformable_group, + height_col, + width_col, + data_col_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf( + "error in modulated_deformable_im2col_cuda: %s\n", + cudaGetErrorString(err)); + } +} + +void modulated_deformable_col2im_cuda( + const at::Tensor data_col, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_im) { + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = + channels * kernel_h * kernel_w * batch_size * height_col * width_col; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "modulated_deformable_col2im_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + const scalar_t* data_mask_ = data_mask.data_ptr(); + scalar_t* grad_im_ = grad_im.data_ptr(); + + modulated_deformable_col2im_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_offset_, + data_mask_, + channels, + height_im, + width_im, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + batch_size, + deformable_group, + height_col, + width_col, + grad_im_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf( + "error in modulated_deformable_col2im_cuda: %s\n", + cudaGetErrorString(err)); + } +} + +void modulated_deformable_col2im_coord_cuda( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_offset, + at::Tensor grad_mask) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * + kernel_w * deformable_group; + const int channel_per_deformable_group = + channels * kernel_h * kernel_w / deformable_group; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "modulated_deformable_col2im_coord_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + const scalar_t* data_mask_ = data_mask.data_ptr(); + scalar_t* grad_offset_ = grad_offset.data_ptr(); + scalar_t* grad_mask_ = grad_mask.data_ptr(); + + modulated_deformable_col2im_coord_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_im_, + data_offset_, + data_mask_, + channels, + height_im, + width_im, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + batch_size, + 2 * kernel_h * kernel_w * deformable_group, + deformable_group, + height_col, + width_col, + grad_offset_, + grad_mask_); + })); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf( + "error in modulated_deformable_col2im_coord_cuda: %s\n", + cudaGetErrorString(err)); + } +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/nms_rotated/nms_rotated.h b/detectron2/layers/csrc/nms_rotated/nms_rotated.h new file mode 100644 index 0000000000000000000000000000000000000000..12aca388e47b12dafd20999f2991a9d42f4b904b --- /dev/null +++ b/detectron2/layers/csrc/nms_rotated/nms_rotated.h @@ -0,0 +1,39 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +at::Tensor nms_rotated_cpu( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold); + +#if defined(WITH_CUDA) || defined(WITH_HIP) +at::Tensor nms_rotated_cuda( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold); +#endif + +// Interface for Python +// inline is needed to prevent multiple function definitions when this header is +// included by different cpps +inline at::Tensor nms_rotated( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold) { + assert(dets.device().is_cuda() == scores.device().is_cuda()); + if (dets.device().is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return nms_rotated_cuda( + dets.contiguous(), scores.contiguous(), iou_threshold); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + + return nms_rotated_cpu(dets.contiguous(), scores.contiguous(), iou_threshold); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.cpp b/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d7556e645b604aa83d86cc702b783fd8ecedffcc --- /dev/null +++ b/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.cpp @@ -0,0 +1,75 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include "../box_iou_rotated/box_iou_rotated_utils.h" +#include "nms_rotated.h" + +namespace detectron2 { + +template +at::Tensor nms_rotated_cpu_kernel( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold) { + // nms_rotated_cpu_kernel is modified from torchvision's nms_cpu_kernel, + // however, the code in this function is much shorter because + // we delegate the IoU computation for rotated boxes to + // the single_box_iou_rotated function in box_iou_rotated_utils.h + AT_ASSERTM(dets.device().is_cpu(), "dets must be a CPU tensor"); + AT_ASSERTM(scores.device().is_cpu(), "scores must be a CPU tensor"); + AT_ASSERTM( + dets.scalar_type() == scores.scalar_type(), + "dets should have the same type as scores"); + + if (dets.numel() == 0) { + return at::empty({0}, dets.options().dtype(at::kLong)); + } + + auto order_t = std::get<1>(scores.sort(0, /* descending=*/true)); + + auto ndets = dets.size(0); + at::Tensor suppressed_t = at::zeros({ndets}, dets.options().dtype(at::kByte)); + at::Tensor keep_t = at::zeros({ndets}, dets.options().dtype(at::kLong)); + + auto suppressed = suppressed_t.data_ptr(); + auto keep = keep_t.data_ptr(); + auto order = order_t.data_ptr(); + + int64_t num_to_keep = 0; + + for (int64_t _i = 0; _i < ndets; _i++) { + auto i = order[_i]; + if (suppressed[i] == 1) { + continue; + } + + keep[num_to_keep++] = i; + + for (int64_t _j = _i + 1; _j < ndets; _j++) { + auto j = order[_j]; + if (suppressed[j] == 1) { + continue; + } + + auto ovr = single_box_iou_rotated( + dets[i].data_ptr(), dets[j].data_ptr()); + if (ovr >= iou_threshold) { + suppressed[j] = 1; + } + } + } + return keep_t.narrow(/*dim=*/0, /*start=*/0, /*length=*/num_to_keep); +} + +at::Tensor nms_rotated_cpu( + // input must be contiguous + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold) { + auto result = at::empty({0}, dets.options()); + + AT_DISPATCH_FLOATING_TYPES(dets.scalar_type(), "nms_rotated", [&] { + result = nms_rotated_cpu_kernel(dets, scores, iou_threshold); + }); + return result; +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/nms_rotated/nms_rotated_cuda.cu b/detectron2/layers/csrc/nms_rotated/nms_rotated_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..2a3db5c62e7a2da52ccf5bac980653c943d630fd --- /dev/null +++ b/detectron2/layers/csrc/nms_rotated/nms_rotated_cuda.cu @@ -0,0 +1,145 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include +#include +#include +#ifdef WITH_CUDA +#include "../box_iou_rotated/box_iou_rotated_utils.h" +#endif +// TODO avoid this when pytorch supports "same directory" hipification +#ifdef WITH_HIP +#include "box_iou_rotated/box_iou_rotated_utils.h" +#endif + +using namespace detectron2; + +namespace { +int const threadsPerBlock = sizeof(unsigned long long) * 8; +} + +template +__global__ void nms_rotated_cuda_kernel( + const int n_boxes, + const double iou_threshold, + const T* dev_boxes, + unsigned long long* dev_mask) { + // nms_rotated_cuda_kernel is modified from torchvision's nms_cuda_kernel + + const int row_start = blockIdx.y; + const int col_start = blockIdx.x; + + // if (row_start > col_start) return; + + const int row_size = + min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); + const int col_size = + min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); + + // Compared to nms_cuda_kernel, where each box is represented with 4 values + // (x1, y1, x2, y2), each rotated box is represented with 5 values + // (x_center, y_center, width, height, angle_degrees) here. + __shared__ T block_boxes[threadsPerBlock * 5]; + if (threadIdx.x < col_size) { + block_boxes[threadIdx.x * 5 + 0] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; + block_boxes[threadIdx.x * 5 + 1] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; + block_boxes[threadIdx.x * 5 + 2] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; + block_boxes[threadIdx.x * 5 + 3] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; + block_boxes[threadIdx.x * 5 + 4] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; + } + __syncthreads(); + + if (threadIdx.x < row_size) { + const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; + const T* cur_box = dev_boxes + cur_box_idx * 5; + int i = 0; + unsigned long long t = 0; + int start = 0; + if (row_start == col_start) { + start = threadIdx.x + 1; + } + for (i = start; i < col_size; i++) { + // Instead of devIoU used by original horizontal nms, here + // we use the single_box_iou_rotated function from box_iou_rotated_utils.h + if (single_box_iou_rotated(cur_box, block_boxes + i * 5) > + iou_threshold) { + t |= 1ULL << i; + } + } + const int col_blocks = at::cuda::ATenCeilDiv(n_boxes, threadsPerBlock); + dev_mask[cur_box_idx * col_blocks + col_start] = t; + } +} + +namespace detectron2 { + +at::Tensor nms_rotated_cuda( + // input must be contiguous + const at::Tensor& dets, + const at::Tensor& scores, + double iou_threshold) { + // using scalar_t = float; + AT_ASSERTM(dets.is_cuda(), "dets must be a CUDA tensor"); + AT_ASSERTM(scores.is_cuda(), "scores must be a CUDA tensor"); + at::cuda::CUDAGuard device_guard(dets.device()); + + auto order_t = std::get<1>(scores.sort(0, /* descending=*/true)); + auto dets_sorted = dets.index_select(0, order_t); + + auto dets_num = dets.size(0); + + const int col_blocks = + at::cuda::ATenCeilDiv(static_cast(dets_num), threadsPerBlock); + + at::Tensor mask = + at::empty({dets_num * col_blocks}, dets.options().dtype(at::kLong)); + + dim3 blocks(col_blocks, col_blocks); + dim3 threads(threadsPerBlock); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES( + dets_sorted.scalar_type(), "nms_rotated_kernel_cuda", [&] { + nms_rotated_cuda_kernel<<>>( + dets_num, + iou_threshold, + dets_sorted.data_ptr(), + (unsigned long long*)mask.data_ptr()); + }); + + at::Tensor mask_cpu = mask.to(at::kCPU); + unsigned long long* mask_host = + (unsigned long long*)mask_cpu.data_ptr(); + + std::vector remv(col_blocks); + memset(&remv[0], 0, sizeof(unsigned long long) * col_blocks); + + at::Tensor keep = + at::empty({dets_num}, dets.options().dtype(at::kLong).device(at::kCPU)); + int64_t* keep_out = keep.data_ptr(); + + int num_to_keep = 0; + for (int i = 0; i < dets_num; i++) { + int nblock = i / threadsPerBlock; + int inblock = i % threadsPerBlock; + + if (!(remv[nblock] & (1ULL << inblock))) { + keep_out[num_to_keep++] = i; + unsigned long long* p = mask_host + i * col_blocks; + for (int j = nblock; j < col_blocks; j++) { + remv[j] |= p[j]; + } + } + } + + AT_CUDA_CHECK(cudaGetLastError()); + return order_t.index( + {keep.narrow(/*dim=*/0, /*start=*/0, /*length=*/num_to_keep) + .to(order_t.device(), keep.scalar_type())}); +} + +} // namespace detectron2 diff --git a/detectron2/layers/csrc/vision.cpp b/detectron2/layers/csrc/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c9a2cd4f20e6f58be1c5783d67c64232dd59b560 --- /dev/null +++ b/detectron2/layers/csrc/vision.cpp @@ -0,0 +1,117 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +#include +#include "ROIAlignRotated/ROIAlignRotated.h" +#include "box_iou_rotated/box_iou_rotated.h" +#include "cocoeval/cocoeval.h" +#include "deformable/deform_conv.h" +#include "nms_rotated/nms_rotated.h" + +namespace detectron2 { + +#if defined(WITH_CUDA) || defined(WITH_HIP) +extern int get_cudart_version(); +#endif + +std::string get_cuda_version() { +#if defined(WITH_CUDA) || defined(WITH_HIP) + std::ostringstream oss; + +#if defined(WITH_CUDA) + oss << "CUDA "; +#else + oss << "HIP "; +#endif + + // copied from + // https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/cuda/detail/CUDAHooks.cpp#L231 + auto printCudaStyleVersion = [&](int v) { + oss << (v / 1000) << "." << (v / 10 % 100); + if (v % 10 != 0) { + oss << "." << (v % 10); + } + }; + printCudaStyleVersion(get_cudart_version()); + return oss.str(); +#else // neither CUDA nor HIP + return std::string("not available"); +#endif +} + +bool has_cuda() { +#if defined(WITH_CUDA) + return true; +#else + return false; +#endif +} + +// similar to +// https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Version.cpp +std::string get_compiler_version() { + std::ostringstream ss; +#if defined(__GNUC__) +#ifndef __clang__ + +#if ((__GNUC__ <= 4) && (__GNUC_MINOR__ <= 8)) +#error "GCC >= 4.9 is required!" +#endif + + { ss << "GCC " << __GNUC__ << "." << __GNUC_MINOR__; } +#endif +#endif + +#if defined(__clang_major__) + { + ss << "clang " << __clang_major__ << "." << __clang_minor__ << "." + << __clang_patchlevel__; + } +#endif + +#if defined(_MSC_VER) + { ss << "MSVC " << _MSC_FULL_VER; } +#endif + return ss.str(); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("get_compiler_version", &get_compiler_version, "get_compiler_version"); + m.def("get_cuda_version", &get_cuda_version, "get_cuda_version"); + m.def("has_cuda", &has_cuda, "has_cuda"); + + m.def("deform_conv_forward", &deform_conv_forward, "deform_conv_forward"); + m.def( + "deform_conv_backward_input", + &deform_conv_backward_input, + "deform_conv_backward_input"); + m.def( + "deform_conv_backward_filter", + &deform_conv_backward_filter, + "deform_conv_backward_filter"); + m.def( + "modulated_deform_conv_forward", + &modulated_deform_conv_forward, + "modulated_deform_conv_forward"); + m.def( + "modulated_deform_conv_backward", + &modulated_deform_conv_backward, + "modulated_deform_conv_backward"); + + m.def("COCOevalAccumulate", &COCOeval::Accumulate, "COCOeval::Accumulate"); + m.def( + "COCOevalEvaluateImages", + &COCOeval::EvaluateImages, + "COCOeval::EvaluateImages"); + pybind11::class_(m, "InstanceAnnotation") + .def(pybind11::init()); + pybind11::class_(m, "ImageEvaluation") + .def(pybind11::init<>()); +} + +TORCH_LIBRARY(detectron2, m) { + m.def("nms_rotated", &nms_rotated); + m.def("box_iou_rotated", &box_iou_rotated); + m.def("roi_align_rotated_forward", &ROIAlignRotated_forward); + m.def("roi_align_rotated_backward", &ROIAlignRotated_backward); +} +} // namespace detectron2 diff --git a/detectron2/layers/deform_conv.py b/detectron2/layers/deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..a37be39e8b3a869906a937ebca4a30a6087a455a --- /dev/null +++ b/detectron2/layers/deform_conv.py @@ -0,0 +1,486 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from functools import lru_cache + +import torch +from torch import nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair +from torchvision.ops import deform_conv2d + +from detectron2.utils.develop import create_dummy_class, create_dummy_func + +from .wrappers import _NewEmptyTensorOp + + +class _DeformConv(Function): + @staticmethod + def forward( + ctx, + input, + offset, + weight, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + im2col_step=64, + ): + if input is not None and input.dim() != 4: + raise ValueError("Expected 4D tensor as input, got {}D tensor instead.".format(input.dim())) + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + + ctx.save_for_backward(input, offset, weight) + + output = input.new_empty(_DeformConv._output_size(input, weight, ctx.padding, ctx.dilation, ctx.stride)) + + ctx.bufs_ = [input.new_empty(0), input.new_empty(0)] # columns, ones + + if not input.is_cuda: + # TODO: let torchvision support full features of our deformconv. + if deformable_groups != 1: + raise NotImplementedError("Deformable Conv with deformable_groups != 1 is not supported on CPUs!") + return deform_conv2d(input, offset, weight, stride=stride, padding=padding, dilation=dilation) + else: + cur_im2col_step = _DeformConv._cal_im2col_step(input.shape[0], ctx.im2col_step) + assert (input.shape[0] % cur_im2col_step) == 0, "im2col step must divide batchsize" + + _C.deform_conv_forward( + input, + weight, + offset, + output, + ctx.bufs_[0], + ctx.bufs_[1], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + cur_im2col_step, + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, offset, weight = ctx.saved_tensors + + grad_input = grad_offset = grad_weight = None + + if not grad_output.is_cuda: + raise NotImplementedError("Deformable Conv is not supported on CPUs!") + else: + cur_im2col_step = _DeformConv._cal_im2col_step(input.shape[0], ctx.im2col_step) + assert (input.shape[0] % cur_im2col_step) == 0, "im2col step must divide batchsize" + + if ctx.needs_input_grad[0] or ctx.needs_input_grad[1]: + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + _C.deform_conv_backward_input( + input, + offset, + grad_output, + grad_input, + grad_offset, + weight, + ctx.bufs_[0], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + cur_im2col_step, + ) + + if ctx.needs_input_grad[2]: + grad_weight = torch.zeros_like(weight) + _C.deform_conv_backward_filter( + input, + offset, + grad_output, + grad_weight, + ctx.bufs_[0], + ctx.bufs_[1], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + 1, + cur_im2col_step, + ) + + return grad_input, grad_offset, grad_weight, None, None, None, None, None, None + + @staticmethod + def _output_size(input, weight, padding, dilation, stride): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = padding[d] + kernel = dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1,) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + "convolution input is too small (output would be {})".format("x".join(map(str, output_size))) + ) + return output_size + + @staticmethod + @lru_cache(maxsize=128) + def _cal_im2col_step(input_size, default_size): + """ + Calculate proper im2col step size, which should be divisible by input_size and not larger + than prefer_size. Meanwhile the step size should be as large as possible to be more + efficient. So we choose the largest one among all divisors of input_size which are smaller + than prefer_size. + :param input_size: input batch size . + :param default_size: default preferred im2col step size. + :return: the largest proper step size. + """ + if input_size <= default_size: + return input_size + best_step = 1 + for step in range(2, min(int(math.sqrt(input_size)) + 1, default_size)): + if input_size % step == 0: + if input_size // step <= default_size: + return input_size // step + best_step = step + + return best_step + + +class _ModulatedDeformConv(Function): + @staticmethod + def forward( + ctx, + input, + offset, + mask, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + ): + ctx.stride = stride + ctx.padding = padding + ctx.dilation = dilation + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.with_bias = bias is not None + if not ctx.with_bias: + bias = input.new_empty(1) # fake tensor + if not input.is_cuda: + raise NotImplementedError("Deformable Conv is not supported on CPUs!") + if weight.requires_grad or mask.requires_grad or offset.requires_grad or input.requires_grad: + ctx.save_for_backward(input, offset, mask, weight, bias) + output = input.new_empty(_ModulatedDeformConv._infer_shape(ctx, input, weight)) + ctx._bufs = [input.new_empty(0), input.new_empty(0)] + _C.modulated_deform_conv_forward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + output, + ctx._bufs[1], + weight.shape[2], + weight.shape[3], + ctx.stride, + ctx.stride, + ctx.padding, + ctx.padding, + ctx.dilation, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, + ctx.with_bias, + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + if not grad_output.is_cuda: + raise NotImplementedError("Deformable Conv is not supported on CPUs!") + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + grad_mask = torch.zeros_like(mask) + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(bias) + _C.modulated_deform_conv_backward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + ctx._bufs[1], + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + weight.shape[2], + weight.shape[3], + ctx.stride, + ctx.stride, + ctx.padding, + ctx.padding, + ctx.dilation, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, + ctx.with_bias, + ) + if not ctx.with_bias: + grad_bias = None + + return ( + grad_input, + grad_offset, + grad_mask, + grad_weight, + grad_bias, + None, + None, + None, + None, + None, + ) + + @staticmethod + def _infer_shape(ctx, input, weight): + n = input.size(0) + channels_out = weight.size(0) + height, width = input.shape[2:4] + kernel_h, kernel_w = weight.shape[2:4] + height_out = (height + 2 * ctx.padding - (ctx.dilation * (kernel_h - 1) + 1)) // ctx.stride + 1 + width_out = (width + 2 * ctx.padding - (ctx.dilation * (kernel_w - 1) + 1)) // ctx.stride + 1 + return n, channels_out, height_out, width_out + + +deform_conv = _DeformConv.apply +modulated_deform_conv = _ModulatedDeformConv.apply + + +class DeformConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=False, + norm=None, + activation=None, + ): + """ + Deformable convolution from :paper:`deformconv`. + + Arguments are similar to :class:`Conv2D`. Extra arguments: + + Args: + deformable_groups (int): number of groups used in deformable convolution. + norm (nn.Module, optional): a normalization layer + activation (callable(Tensor) -> Tensor): a callable activation function + """ + super(DeformConv, self).__init__() + + assert not bias + assert in_channels % groups == 0, "in_channels {} cannot be divisible by groups {}".format(in_channels, groups) + assert out_channels % groups == 0, "out_channels {} cannot be divisible by groups {}".format( + out_channels, groups + ) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.norm = norm + self.activation = activation + + self.weight = nn.Parameter(torch.Tensor(out_channels, in_channels // self.groups, *self.kernel_size)) + self.bias = None + + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + + def forward(self, x, offset): + if x.numel() == 0: + # When input is empty, we want to return a empty tensor with "correct" shape, + # So that the following operations will not panic + # if they check for the shape of the tensor. + # This computes the height and width of the output tensor + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // s + 1 + for i, p, di, k, s in zip(x.shape[-2:], self.padding, self.dilation, self.kernel_size, self.stride) + ] + output_shape = [x.shape[0], self.weight.shape[0]] + output_shape + return _NewEmptyTensorOp.apply(x, output_shape) + + x = deform_conv( + x, + offset, + self.weight, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + ) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + def extra_repr(self): + tmpstr = "in_channels=" + str(self.in_channels) + tmpstr += ", out_channels=" + str(self.out_channels) + tmpstr += ", kernel_size=" + str(self.kernel_size) + tmpstr += ", stride=" + str(self.stride) + tmpstr += ", padding=" + str(self.padding) + tmpstr += ", dilation=" + str(self.dilation) + tmpstr += ", groups=" + str(self.groups) + tmpstr += ", deformable_groups=" + str(self.deformable_groups) + tmpstr += ", bias=False" + return tmpstr + + +class ModulatedDeformConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=True, + norm=None, + activation=None, + ): + """ + Modulated deformable convolution from :paper:`deformconv2`. + + Arguments are similar to :class:`Conv2D`. Extra arguments: + + Args: + deformable_groups (int): number of groups used in deformable convolution. + norm (nn.Module, optional): a normalization layer + activation (callable(Tensor) -> Tensor): a callable activation function + """ + super(ModulatedDeformConv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + self.deformable_groups = deformable_groups + self.with_bias = bias + self.norm = norm + self.activation = activation + + self.weight = nn.Parameter(torch.Tensor(out_channels, in_channels // groups, *self.kernel_size)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.bias = None + + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + if self.bias is not None: + nn.init.constant_(self.bias, 0) + + def forward(self, x, offset, mask): + if x.numel() == 0: + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // s + 1 + for i, p, di, k, s in zip(x.shape[-2:], self.padding, self.dilation, self.kernel_size, self.stride) + ] + output_shape = [x.shape[0], self.weight.shape[0]] + output_shape + return _NewEmptyTensorOp.apply(x, output_shape) + + x = modulated_deform_conv( + x, + offset, + mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + ) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + def extra_repr(self): + tmpstr = "in_channels=" + str(self.in_channels) + tmpstr += ", out_channels=" + str(self.out_channels) + tmpstr += ", kernel_size=" + str(self.kernel_size) + tmpstr += ", stride=" + str(self.stride) + tmpstr += ", padding=" + str(self.padding) + tmpstr += ", dilation=" + str(self.dilation) + tmpstr += ", groups=" + str(self.groups) + tmpstr += ", deformable_groups=" + str(self.deformable_groups) + tmpstr += ", bias=" + str(self.with_bias) + return tmpstr + + +try: + from detectron2 import _C +except ImportError: + # TODO: register ops natively so there is no need to import _C. + _msg = "detectron2 is not compiled successfully, please build following the instructions!" + _args = ("detectron2._C", _msg) + DeformConv = create_dummy_class("DeformConv", *_args) + ModulatedDeformConv = create_dummy_class("ModulatedDeformConv", *_args) + deform_conv = create_dummy_func("deform_conv", *_args) + modulated_deform_conv = create_dummy_func("modulated_deform_conv", *_args) diff --git a/detectron2/layers/losses.py b/detectron2/layers/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..7bd389ccbd766e89292743825dde55989f398421 --- /dev/null +++ b/detectron2/layers/losses.py @@ -0,0 +1,134 @@ +import math + +import torch + + +def diou_loss( + boxes1: torch.Tensor, + boxes2: torch.Tensor, + reduction: str = "none", + eps: float = 1e-7, +) -> torch.Tensor: + """ + Distance Intersection over Union Loss (Zhaohui Zheng et. al) + https://arxiv.org/abs/1911.08287 + Args: + boxes1, boxes2 (Tensor): box locations in XYXY format, shape (N, 4) or (4,). + reduction: 'none' | 'mean' | 'sum' + 'none': No reduction will be applied to the output. + 'mean': The output will be averaged. + 'sum': The output will be summed. + eps (float): small number to prevent division by zero + """ + + x1, y1, x2, y2 = boxes1.unbind(dim=-1) + x1g, y1g, x2g, y2g = boxes2.unbind(dim=-1) + + # TODO: use torch._assert_async() when pytorch 1.8 support is dropped + assert (x2 >= x1).all(), "bad box: x1 larger than x2" + assert (y2 >= y1).all(), "bad box: y1 larger than y2" + + # Intersection keypoints + xkis1 = torch.max(x1, x1g) + ykis1 = torch.max(y1, y1g) + xkis2 = torch.min(x2, x2g) + ykis2 = torch.min(y2, y2g) + + intsct = torch.zeros_like(x1) + mask = (ykis2 > ykis1) & (xkis2 > xkis1) + intsct[mask] = (xkis2[mask] - xkis1[mask]) * (ykis2[mask] - ykis1[mask]) + union = (x2 - x1) * (y2 - y1) + (x2g - x1g) * (y2g - y1g) - intsct + eps + iou = intsct / union + + # smallest enclosing box + xc1 = torch.min(x1, x1g) + yc1 = torch.min(y1, y1g) + xc2 = torch.max(x2, x2g) + yc2 = torch.max(y2, y2g) + diag_len = ((xc2 - xc1) ** 2) + ((yc2 - yc1) ** 2) + eps + + # centers of boxes + x_p = (x2 + x1) / 2 + y_p = (y2 + y1) / 2 + x_g = (x1g + x2g) / 2 + y_g = (y1g + y2g) / 2 + distance = ((x_p - x_g) ** 2) + ((y_p - y_g) ** 2) + + # Eqn. (7) + loss = 1 - iou + (distance / diag_len) + if reduction == "mean": + loss = loss.mean() if loss.numel() > 0 else 0.0 * loss.sum() + elif reduction == "sum": + loss = loss.sum() + + return loss + + +def ciou_loss( + boxes1: torch.Tensor, + boxes2: torch.Tensor, + reduction: str = "none", + eps: float = 1e-7, +) -> torch.Tensor: + """ + Complete Intersection over Union Loss (Zhaohui Zheng et. al) + https://arxiv.org/abs/1911.08287 + Args: + boxes1, boxes2 (Tensor): box locations in XYXY format, shape (N, 4) or (4,). + reduction: 'none' | 'mean' | 'sum' + 'none': No reduction will be applied to the output. + 'mean': The output will be averaged. + 'sum': The output will be summed. + eps (float): small number to prevent division by zero + """ + + x1, y1, x2, y2 = boxes1.unbind(dim=-1) + x1g, y1g, x2g, y2g = boxes2.unbind(dim=-1) + + # TODO: use torch._assert_async() when pytorch 1.8 support is dropped + assert (x2 >= x1).all(), "bad box: x1 larger than x2" + assert (y2 >= y1).all(), "bad box: y1 larger than y2" + + # Intersection keypoints + xkis1 = torch.max(x1, x1g) + ykis1 = torch.max(y1, y1g) + xkis2 = torch.min(x2, x2g) + ykis2 = torch.min(y2, y2g) + + intsct = torch.zeros_like(x1) + mask = (ykis2 > ykis1) & (xkis2 > xkis1) + intsct[mask] = (xkis2[mask] - xkis1[mask]) * (ykis2[mask] - ykis1[mask]) + union = (x2 - x1) * (y2 - y1) + (x2g - x1g) * (y2g - y1g) - intsct + eps + iou = intsct / union + + # smallest enclosing box + xc1 = torch.min(x1, x1g) + yc1 = torch.min(y1, y1g) + xc2 = torch.max(x2, x2g) + yc2 = torch.max(y2, y2g) + diag_len = ((xc2 - xc1) ** 2) + ((yc2 - yc1) ** 2) + eps + + # centers of boxes + x_p = (x2 + x1) / 2 + y_p = (y2 + y1) / 2 + x_g = (x1g + x2g) / 2 + y_g = (y1g + y2g) / 2 + distance = ((x_p - x_g) ** 2) + ((y_p - y_g) ** 2) + + # width and height of boxes + w_pred = x2 - x1 + h_pred = y2 - y1 + w_gt = x2g - x1g + h_gt = y2g - y1g + v = (4 / (math.pi**2)) * torch.pow((torch.atan(w_gt / h_gt) - torch.atan(w_pred / h_pred)), 2) + with torch.no_grad(): + alpha = v / (1 - iou + v + eps) + + # Eqn. (10) + loss = 1 - iou + (distance / diag_len) + alpha * v + if reduction == "mean": + loss = loss.mean() if loss.numel() > 0 else 0.0 * loss.sum() + elif reduction == "sum": + loss = loss.sum() + + return loss diff --git a/detectron2/layers/mask_ops.py b/detectron2/layers/mask_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..f100c0e76849e91a2a3dc3a6ef8a624774db34fb --- /dev/null +++ b/detectron2/layers/mask_ops.py @@ -0,0 +1,268 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Tuple + +import numpy as np +import torch +from PIL import Image +from torch.nn import functional as F + +__all__ = ["paste_masks_in_image"] + + +BYTES_PER_FLOAT = 4 +# TODO: This memory limit may be too much or too little. It would be better to +# determine it based on available resources. +GPU_MEM_LIMIT = 1024**3 # 1 GB memory limit + + +def _do_paste_mask(masks, boxes, img_h: int, img_w: int, skip_empty: bool = True): + """ + Args: + masks: N, 1, H, W + boxes: N, 4 + img_h, img_w (int): + skip_empty (bool): only paste masks within the region that + tightly bound all boxes, and returns the results this region only. + An important optimization for CPU. + + Returns: + if skip_empty == False, a mask of shape (N, img_h, img_w) + if skip_empty == True, a mask of shape (N, h', w'), and the slice + object for the corresponding region. + """ + # On GPU, paste all masks together (up to chunk size) + # by using the entire image to sample the masks + # Compared to pasting them one by one, + # this has more operations but is faster on COCO-scale dataset. + device = masks.device + + if skip_empty and not torch.jit.is_scripting(): + x0_int, y0_int = torch.clamp(boxes.min(dim=0).values.floor()[:2] - 1, min=0).to(dtype=torch.int32) + x1_int = torch.clamp(boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32) + y1_int = torch.clamp(boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32) + else: + x0_int, y0_int = 0, 0 + x1_int, y1_int = img_w, img_h + x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1 + + N = masks.shape[0] + + img_y = torch.arange(y0_int, y1_int, device=device, dtype=torch.float32) + 0.5 + img_x = torch.arange(x0_int, x1_int, device=device, dtype=torch.float32) + 0.5 + img_y = (img_y - y0) / (y1 - y0) * 2 - 1 + img_x = (img_x - x0) / (x1 - x0) * 2 - 1 + # img_x, img_y have shapes (N, w), (N, h) + + gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1)) + gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1)) + grid = torch.stack([gx, gy], dim=3) + + if not torch.jit.is_scripting(): + if not masks.dtype.is_floating_point: + masks = masks.float() + img_masks = F.grid_sample(masks, grid.to(masks.dtype), align_corners=False) + + if skip_empty and not torch.jit.is_scripting(): + return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) + else: + return img_masks[:, 0], () + + +# Annotate boxes as Tensor (but not Boxes) in order to use scripting +@torch.jit.script_if_tracing +def paste_masks_in_image( + masks: torch.Tensor, boxes: torch.Tensor, image_shape: Tuple[int, int], threshold: float = 0.5 +): + """ + Paste a set of masks that are of a fixed resolution (e.g., 28 x 28) into an image. + The location, height, and width for pasting each mask is determined by their + corresponding bounding boxes in boxes. + + Note: + This is a complicated but more accurate implementation. In actual deployment, it is + often enough to use a faster but less accurate implementation. + See :func:`paste_mask_in_image_old` in this file for an alternative implementation. + + Args: + masks (tensor): Tensor of shape (Bimg, Hmask, Wmask), where Bimg is the number of + detected object instances in the image and Hmask, Wmask are the mask width and mask + height of the predicted mask (e.g., Hmask = Wmask = 28). Values are in [0, 1]. + boxes (Boxes or Tensor): A Boxes of length Bimg or Tensor of shape (Bimg, 4). + boxes[i] and masks[i] correspond to the same object instance. + image_shape (tuple): height, width + threshold (float): A threshold in [0, 1] for converting the (soft) masks to + binary masks. + + Returns: + img_masks (Tensor): A tensor of shape (Bimg, Himage, Wimage), where Bimg is the + number of detected object instances and Himage, Wimage are the image width + and height. img_masks[i] is a binary mask for object instance i. + """ + + assert masks.shape[-1] == masks.shape[-2], "Only square mask predictions are supported" + N = len(masks) + if N == 0: + return masks.new_empty((0,) + image_shape, dtype=torch.uint8) + if not isinstance(boxes, torch.Tensor): + boxes = boxes.tensor + device = boxes.device + assert len(boxes) == N, boxes.shape + + img_h, img_w = image_shape + + # The actual implementation split the input into chunks, + # and paste them chunk by chunk. + if device.type == "cpu" or torch.jit.is_scripting(): + # CPU is most efficient when they are pasted one by one with skip_empty=True + # so that it performs minimal number of operations. + num_chunks = N + else: + # GPU benefits from parallelism for larger chunks, but may have memory issue + # int(img_h) because shape may be tensors in tracing + num_chunks = int(np.ceil(N * int(img_h) * int(img_w) * BYTES_PER_FLOAT / GPU_MEM_LIMIT)) + assert num_chunks <= N, "Default GPU_MEM_LIMIT in mask_ops.py is too small; try increasing it" + chunks = torch.chunk(torch.arange(N, device=device), num_chunks) + + img_masks = torch.zeros(N, img_h, img_w, device=device, dtype=torch.bool if threshold >= 0 else torch.uint8) + for inds in chunks: + masks_chunk, spatial_inds = _do_paste_mask( + masks[inds, None, :, :], boxes[inds], img_h, img_w, skip_empty=device.type == "cpu" + ) + + if threshold >= 0: + masks_chunk = (masks_chunk >= threshold).to(dtype=torch.bool) + else: + # for visualization and debugging + masks_chunk = (masks_chunk * 255).to(dtype=torch.uint8) + + if torch.jit.is_scripting(): # Scripting does not use the optimized codepath + img_masks[inds] = masks_chunk + else: + img_masks[(inds,) + spatial_inds] = masks_chunk + return img_masks + + +# The below are the original paste function (from Detectron1) which has +# larger quantization error. +# It is faster on CPU, while the aligned one is faster on GPU thanks to grid_sample. + + +def paste_mask_in_image_old(mask, box, img_h, img_w, threshold): + """ + Paste a single mask in an image. + This is a per-box implementation of :func:`paste_masks_in_image`. + This function has larger quantization error due to incorrect pixel + modeling and is not used any more. + + Args: + mask (Tensor): A tensor of shape (Hmask, Wmask) storing the mask of a single + object instance. Values are in [0, 1]. + box (Tensor): A tensor of shape (4, ) storing the x0, y0, x1, y1 box corners + of the object instance. + img_h, img_w (int): Image height and width. + threshold (float): Mask binarization threshold in [0, 1]. + + Returns: + im_mask (Tensor): + The resized and binarized object mask pasted into the original + image plane (a tensor of shape (img_h, img_w)). + """ + # Conversion from continuous box coordinates to discrete pixel coordinates + # via truncation (cast to int32). This determines which pixels to paste the + # mask onto. + box = box.to(dtype=torch.int32) # Continuous to discrete coordinate conversion + # An example (1D) box with continuous coordinates (x0=0.7, x1=4.3) will map to + # a discrete coordinates (x0=0, x1=4). Note that box is mapped to 5 = x1 - x0 + 1 + # pixels (not x1 - x0 pixels). + samples_w = box[2] - box[0] + 1 # Number of pixel samples, *not* geometric width + samples_h = box[3] - box[1] + 1 # Number of pixel samples, *not* geometric height + + # Resample the mask from it's original grid to the new samples_w x samples_h grid + mask = Image.fromarray(mask.cpu().numpy()) + mask = mask.resize((samples_w, samples_h), resample=Image.BILINEAR) + mask = np.array(mask, copy=False) + + if threshold >= 0: + mask = np.array(mask > threshold, dtype=np.uint8) + mask = torch.from_numpy(mask) + else: + # for visualization and debugging, we also + # allow it to return an unmodified mask + mask = torch.from_numpy(mask * 255).to(torch.uint8) + + im_mask = torch.zeros((img_h, img_w), dtype=torch.uint8) + x_0 = max(box[0], 0) + x_1 = min(box[2] + 1, img_w) + y_0 = max(box[1], 0) + y_1 = min(box[3] + 1, img_h) + + im_mask[y_0:y_1, x_0:x_1] = mask[(y_0 - box[1]) : (y_1 - box[1]), (x_0 - box[0]) : (x_1 - box[0])] + return im_mask + + +# Our pixel modeling requires extrapolation for any continuous +# coordinate < 0.5 or > length - 0.5. When sampling pixels on the masks, +# we would like this extrapolation to be an interpolation between boundary values and zero, +# instead of using absolute zero or boundary values. +# Therefore `paste_mask_in_image_old` is often used with zero padding around the masks like this: +# masks, scale = pad_masks(masks[:, 0, :, :], 1) +# boxes = scale_boxes(boxes.tensor, scale) + + +def pad_masks(masks, padding): + """ + Args: + masks (tensor): A tensor of shape (B, M, M) representing B masks. + padding (int): Number of cells to pad on all sides. + + Returns: + The padded masks and the scale factor of the padding size / original size. + """ + B = masks.shape[0] + M = masks.shape[-1] + pad2 = 2 * padding + scale = float(M + pad2) / M + padded_masks = masks.new_zeros((B, M + pad2, M + pad2)) + padded_masks[:, padding:-padding, padding:-padding] = masks + return padded_masks, scale + + +def scale_boxes(boxes, scale): + """ + Args: + boxes (tensor): A tensor of shape (B, 4) representing B boxes with 4 + coords representing the corners x0, y0, x1, y1, + scale (float): The box scaling factor. + + Returns: + Scaled boxes. + """ + w_half = (boxes[:, 2] - boxes[:, 0]) * 0.5 + h_half = (boxes[:, 3] - boxes[:, 1]) * 0.5 + x_c = (boxes[:, 2] + boxes[:, 0]) * 0.5 + y_c = (boxes[:, 3] + boxes[:, 1]) * 0.5 + + w_half *= scale + h_half *= scale + + scaled_boxes = torch.zeros_like(boxes) + scaled_boxes[:, 0] = x_c - w_half + scaled_boxes[:, 2] = x_c + w_half + scaled_boxes[:, 1] = y_c - h_half + scaled_boxes[:, 3] = y_c + h_half + return scaled_boxes + + +@torch.jit.script_if_tracing +def _paste_masks_tensor_shape( + masks: torch.Tensor, + boxes: torch.Tensor, + image_shape: Tuple[torch.Tensor, torch.Tensor], + threshold: float = 0.5, +): + """ + A wrapper of paste_masks_in_image where image_shape is Tensor. + During tracing, shapes might be tensors instead of ints. The Tensor->int + conversion should be scripted rather than traced. + """ + return paste_masks_in_image(masks, boxes, (int(image_shape[0]), int(image_shape[1])), threshold) diff --git a/detectron2/layers/nms.py b/detectron2/layers/nms.py new file mode 100644 index 0000000000000000000000000000000000000000..46c754f79c2ff00e7829443870470ac6892d0633 --- /dev/null +++ b/detectron2/layers/nms.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import torch +from torchvision.ops import boxes as box_ops +from torchvision.ops import nms # noqa . for compatibility + + +def batched_nms(boxes: torch.Tensor, scores: torch.Tensor, idxs: torch.Tensor, iou_threshold: float): + """ + Same as torchvision.ops.boxes.batched_nms, but with float(). + """ + assert boxes.shape[-1] == 4 + # Note: Torchvision already has a strategy (https://github.com/pytorch/vision/issues/1311) + # to decide whether to use coordinate trick or for loop to implement batched_nms. So we + # just call it directly. + # Fp16 does not have enough range for batched NMS, so adding float(). + return box_ops.batched_nms(boxes.float(), scores, idxs, iou_threshold) + + +# Note: this function (nms_rotated) might be moved into +# torchvision/ops/boxes.py in the future +def nms_rotated(boxes, scores, iou_threshold): + """ + Performs non-maximum suppression (NMS) on the rotated boxes according + to their intersection-over-union (IoU). + + Rotated NMS iteratively removes lower scoring rotated boxes which have an + IoU greater than iou_threshold with another (higher scoring) rotated box. + + Note that RotatedBox (5, 3, 4, 2, -90) covers exactly the same region as + RotatedBox (5, 3, 4, 2, 90) does, and their IoU will be 1. However, they + can be representing completely different objects in certain tasks, e.g., OCR. + + As for the question of whether rotated-NMS should treat them as faraway boxes + even though their IOU is 1, it depends on the application and/or ground truth annotation. + + As an extreme example, consider a single character v and the square box around it. + + If the angle is 0 degree, the object (text) would be read as 'v'; + + If the angle is 90 degrees, the object (text) would become '>'; + + If the angle is 180 degrees, the object (text) would become '^'; + + If the angle is 270/-90 degrees, the object (text) would become '<' + + All of these cases have IoU of 1 to each other, and rotated NMS that only + uses IoU as criterion would only keep one of them with the highest score - + which, practically, still makes sense in most cases because typically + only one of theses orientations is the correct one. Also, it does not matter + as much if the box is only used to classify the object (instead of transcribing + them with a sequential OCR recognition model) later. + + On the other hand, when we use IoU to filter proposals that are close to the + ground truth during training, we should definitely take the angle into account if + we know the ground truth is labeled with the strictly correct orientation (as in, + upside-down words are annotated with -180 degrees even though they can be covered + with a 0/90/-90 degree box, etc.) + + The way the original dataset is annotated also matters. For example, if the dataset + is a 4-point polygon dataset that does not enforce ordering of vertices/orientation, + we can estimate a minimum rotated bounding box to this polygon, but there's no way + we can tell the correct angle with 100% confidence (as shown above, there could be 4 different + rotated boxes, with angles differed by 90 degrees to each other, covering the exactly + same region). In that case we have to just use IoU to determine the box + proximity (as many detection benchmarks (even for text) do) unless there're other + assumptions we can make (like width is always larger than height, or the object is not + rotated by more than 90 degrees CCW/CW, etc.) + + In summary, not considering angles in rotated NMS seems to be a good option for now, + but we should be aware of its implications. + + Args: + boxes (Tensor[N, 5]): Rotated boxes to perform NMS on. They are expected to be in + (x_center, y_center, width, height, angle_degrees) format. + scores (Tensor[N]): Scores for each one of the rotated boxes + iou_threshold (float): Discards all overlapping rotated boxes with IoU < iou_threshold + + Returns: + keep (Tensor): int64 tensor with the indices of the elements that have been kept + by Rotated NMS, sorted in decreasing order of scores + """ + return torch.ops.detectron2.nms_rotated(boxes, scores, iou_threshold) + + +# Note: this function (batched_nms_rotated) might be moved into +# torchvision/ops/boxes.py in the future +def batched_nms_rotated(boxes, scores, idxs, iou_threshold): + """ + Performs non-maximum suppression in a batched fashion. + + Each index value correspond to a category, and NMS + will not be applied between elements of different categories. + + Args: + boxes (Tensor[N, 5]): + boxes where NMS will be performed. They + are expected to be in (x_ctr, y_ctr, width, height, angle_degrees) format + scores (Tensor[N]): + scores for each one of the boxes + idxs (Tensor[N]): + indices of the categories for each one of the boxes. + iou_threshold (float): + discards all overlapping boxes + with IoU < iou_threshold + + Returns: + Tensor: + int64 tensor with the indices of the elements that have been kept + by NMS, sorted in decreasing order of scores + """ + assert boxes.shape[-1] == 5 + + if boxes.numel() == 0: + return torch.empty((0,), dtype=torch.int64, device=boxes.device) + boxes = boxes.float() # fp16 does not have enough range for batched NMS + # Strategy: in order to perform NMS independently per class, + # we add an offset to all the boxes. The offset is dependent + # only on the class idx, and is large enough so that boxes + # from different classes do not overlap + + # Note that batched_nms in torchvision/ops/boxes.py only uses max_coordinate, + # which won't handle negative coordinates correctly. + # Here by using min_coordinate we can make sure the negative coordinates are + # correctly handled. + max_coordinate = (torch.max(boxes[:, 0], boxes[:, 1]) + torch.max(boxes[:, 2], boxes[:, 3]) / 2).max() + min_coordinate = (torch.min(boxes[:, 0], boxes[:, 1]) - torch.max(boxes[:, 2], boxes[:, 3]) / 2).min() + offsets = idxs.to(boxes) * (max_coordinate - min_coordinate + 1) + boxes_for_nms = boxes.clone() # avoid modifying the original values in boxes + boxes_for_nms[:, :2] += offsets[:, None] + keep = nms_rotated(boxes_for_nms, scores, iou_threshold) + return keep diff --git a/detectron2/layers/roi_align.py b/detectron2/layers/roi_align.py new file mode 100644 index 0000000000000000000000000000000000000000..163462e1f194e1e4100da92d76d9516f7cc22e35 --- /dev/null +++ b/detectron2/layers/roi_align.py @@ -0,0 +1,74 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from torch import nn +from torchvision.ops import roi_align + + +# NOTE: torchvision's RoIAlign has a different default aligned=False +class ROIAlign(nn.Module): + def __init__(self, output_size, spatial_scale, sampling_ratio, aligned=True): + """ + Args: + output_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sampling_ratio (int): number of inputs samples to take for each output + sample. 0 to take samples densely. + aligned (bool): if False, use the legacy implementation in + Detectron. If True, align the results more perfectly. + + Note: + The meaning of aligned=True: + + Given a continuous coordinate c, its two neighboring pixel indices (in our + pixel model) are computed by floor(c - 0.5) and ceil(c - 0.5). For example, + c=1.3 has pixel neighbors with discrete indices [0] and [1] (which are sampled + from the underlying signal at continuous coordinates 0.5 and 1.5). But the original + roi_align (aligned=False) does not subtract the 0.5 when computing neighboring + pixel indices and therefore it uses pixels with a slightly incorrect alignment + (relative to our pixel model) when performing bilinear interpolation. + + With `aligned=True`, + we first appropriately scale the ROI and then shift it by -0.5 + prior to calling roi_align. This produces the correct neighbors; see + detectron2/tests/test_roi_align.py for verification. + + The difference does not make a difference to the model's performance if + ROIAlign is used together with conv layers. + """ + super().__init__() + self.output_size = output_size + self.spatial_scale = spatial_scale + self.sampling_ratio = sampling_ratio + self.aligned = aligned + + from torchvision import __version__ + + version = tuple(int(x) for x in __version__.split(".")[:2]) + # https://github.com/pytorch/vision/pull/2438 + assert version >= (0, 7), "Require torchvision >= 0.7" + + def forward(self, input, rois): + """ + Args: + input: NCHW images + rois: Bx5 boxes. First column is the index into N. The other 4 columns are xyxy. + """ + assert rois.dim() == 2 and rois.size(1) == 5 + if input.is_quantized: + input = input.dequantize() + return roi_align( + input, + rois.to(dtype=input.dtype), + self.output_size, + self.spatial_scale, + self.sampling_ratio, + self.aligned, + ) + + def __repr__(self): + tmpstr = self.__class__.__name__ + "(" + tmpstr += "output_size=" + str(self.output_size) + tmpstr += ", spatial_scale=" + str(self.spatial_scale) + tmpstr += ", sampling_ratio=" + str(self.sampling_ratio) + tmpstr += ", aligned=" + str(self.aligned) + tmpstr += ")" + return tmpstr diff --git a/detectron2/layers/roi_align_rotated.py b/detectron2/layers/roi_align_rotated.py new file mode 100644 index 0000000000000000000000000000000000000000..73e77760112e3ec9629adca2797982e2ec78304d --- /dev/null +++ b/detectron2/layers/roi_align_rotated.py @@ -0,0 +1,91 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +from torch import nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + + +class _ROIAlignRotated(Function): + @staticmethod + def forward(ctx, input, roi, output_size, spatial_scale, sampling_ratio): + ctx.save_for_backward(roi) + ctx.output_size = _pair(output_size) + ctx.spatial_scale = spatial_scale + ctx.sampling_ratio = sampling_ratio + ctx.input_shape = input.size() + output = torch.ops.detectron2.roi_align_rotated_forward( + input, roi, spatial_scale, output_size[0], output_size[1], sampling_ratio + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + (rois,) = ctx.saved_tensors + output_size = ctx.output_size + spatial_scale = ctx.spatial_scale + sampling_ratio = ctx.sampling_ratio + bs, ch, h, w = ctx.input_shape + grad_input = torch.ops.detectron2.roi_align_rotated_backward( + grad_output, + rois, + spatial_scale, + output_size[0], + output_size[1], + bs, + ch, + h, + w, + sampling_ratio, + ) + return grad_input, None, None, None, None, None + + +roi_align_rotated = _ROIAlignRotated.apply + + +class ROIAlignRotated(nn.Module): + def __init__(self, output_size, spatial_scale, sampling_ratio): + """ + Args: + output_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sampling_ratio (int): number of inputs samples to take for each output + sample. 0 to take samples densely. + + Note: + ROIAlignRotated supports continuous coordinate by default: + Given a continuous coordinate c, its two neighboring pixel indices (in our + pixel model) are computed by floor(c - 0.5) and ceil(c - 0.5). For example, + c=1.3 has pixel neighbors with discrete indices [0] and [1] (which are sampled + from the underlying signal at continuous coordinates 0.5 and 1.5). + """ + super(ROIAlignRotated, self).__init__() + self.output_size = output_size + self.spatial_scale = spatial_scale + self.sampling_ratio = sampling_ratio + + def forward(self, input, rois): + """ + Args: + input: NCHW images + rois: Bx6 boxes. First column is the index into N. + The other 5 columns are (x_ctr, y_ctr, width, height, angle_degrees). + """ + assert rois.dim() == 2 and rois.size(1) == 6 + orig_dtype = input.dtype + if orig_dtype == torch.float16: + input = input.float() + rois = rois.float() + return roi_align_rotated(input, rois, self.output_size, self.spatial_scale, self.sampling_ratio).to( + dtype=orig_dtype + ) + + def __repr__(self): + tmpstr = self.__class__.__name__ + "(" + tmpstr += "output_size=" + str(self.output_size) + tmpstr += ", spatial_scale=" + str(self.spatial_scale) + tmpstr += ", sampling_ratio=" + str(self.sampling_ratio) + tmpstr += ")" + return tmpstr diff --git a/detectron2/layers/rotated_boxes.py b/detectron2/layers/rotated_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..1f25c277b4c6868a832b3e469b05f1eb946c6276 --- /dev/null +++ b/detectron2/layers/rotated_boxes.py @@ -0,0 +1,22 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import, division, print_function, unicode_literals + +import torch + + +def pairwise_iou_rotated(boxes1, boxes2): + """ + Return intersection-over-union (Jaccard index) of boxes. + + Both sets of boxes are expected to be in + (x_center, y_center, width, height, angle) format. + + Arguments: + boxes1 (Tensor[N, 5]) + boxes2 (Tensor[M, 5]) + + Returns: + iou (Tensor[N, M]): the NxM matrix containing the pairwise + IoU values for every element in boxes1 and boxes2 + """ + return torch.ops.detectron2.box_iou_rotated(boxes1, boxes2) diff --git a/detectron2/layers/shape_spec.py b/detectron2/layers/shape_spec.py new file mode 100644 index 0000000000000000000000000000000000000000..8dac3c59b96576710656abebe9b5eac25868abbb --- /dev/null +++ b/detectron2/layers/shape_spec.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class ShapeSpec: + """ + A simple structure that contains basic shape specification about a tensor. + It is often used as the auxiliary inputs/outputs of models, + to complement the lack of shape inference ability among pytorch modules. + """ + + channels: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + stride: Optional[int] = None diff --git a/detectron2/layers/wrappers.py b/detectron2/layers/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..cb85d18aed009a4ba4a5b4139c1714a074c0c1d6 --- /dev/null +++ b/detectron2/layers/wrappers.py @@ -0,0 +1,143 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Wrappers around on some nn functions, mainly to support empty tensors. + +Ideally, add support directly in PyTorch to empty tensors in those functions. + +These can be removed once https://github.com/pytorch/pytorch/issues/12013 +is implemented +""" + +from typing import List, Optional + +import torch +from torch.nn import functional as F + + +def shapes_to_tensor(x: List[int], device: Optional[torch.device] = None) -> torch.Tensor: + """ + Turn a list of integer scalars or integer Tensor scalars into a vector, + in a way that's both traceable and scriptable. + + In tracing, `x` should be a list of scalar Tensor, so the output can trace to the inputs. + In scripting or eager, `x` should be a list of int. + """ + if torch.jit.is_scripting(): + return torch.as_tensor(x, device=device) + if torch.jit.is_tracing(): + assert all([isinstance(t, torch.Tensor) for t in x]), "Shape should be tensor during tracing!" + # as_tensor should not be used in tracing because it records a constant + ret = torch.stack(x) + if ret.device != device: # avoid recording a hard-coded device if not necessary + ret = ret.to(device=device) + return ret + return torch.as_tensor(x, device=device) + + +def cat(tensors: List[torch.Tensor], dim: int = 0): + """ + Efficient version of torch.cat that avoids a copy if there is only a single element in a list + """ + assert isinstance(tensors, (list, tuple)) + if len(tensors) == 1: + return tensors[0] + return torch.cat(tensors, dim) + + +def empty_input_loss_func_wrapper(loss_func): + def wrapped_loss_func(input, target, *, reduction="mean", **kwargs): + """ + Same as `loss_func`, but returns 0 (instead of nan) for empty inputs. + """ + if target.numel() == 0 and reduction == "mean": + return input.sum() * 0.0 # connect the gradient + return loss_func(input, target, reduction=reduction, **kwargs) + + return wrapped_loss_func + + +cross_entropy = empty_input_loss_func_wrapper(F.cross_entropy) + + +class _NewEmptyTensorOp(torch.autograd.Function): + @staticmethod + def forward(ctx, x, new_shape): + ctx.shape = x.shape + return x.new_empty(new_shape) + + @staticmethod + def backward(ctx, grad): + shape = ctx.shape + return _NewEmptyTensorOp.apply(grad, shape), None + + +class Conv2d(torch.nn.Conv2d): + """ + A wrapper around :class:`torch.nn.Conv2d` to support empty inputs and more features. + """ + + def __init__(self, *args, **kwargs): + """ + Extra keyword arguments supported in addition to those in `torch.nn.Conv2d`: + + Args: + norm (nn.Module, optional): a normalization layer + activation (callable(Tensor) -> Tensor): a callable activation function + + It assumes that norm layer is used before activation. + """ + norm = kwargs.pop("norm", None) + activation = kwargs.pop("activation", None) + super().__init__(*args, **kwargs) + + self.norm = norm + self.activation = activation + + def forward(self, x): + # torchscript does not support SyncBatchNorm yet + # https://github.com/pytorch/pytorch/issues/40507 + # and we skip these codes in torchscript since: + # 1. currently we only support torchscript in evaluation mode + # 2. features needed by exporting module to torchscript are added in PyTorch 1.6 or + # later version, `Conv2d` in these PyTorch versions has already supported empty inputs. + if not torch.jit.is_scripting(): + if x.numel() == 0 and self.training: + # https://github.com/pytorch/pytorch/issues/12013 + assert not isinstance( + self.norm, torch.nn.SyncBatchNorm + ), "SyncBatchNorm does not support empty inputs!" + + x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + +ConvTranspose2d = torch.nn.ConvTranspose2d +BatchNorm2d = torch.nn.BatchNorm2d +interpolate = F.interpolate +Linear = torch.nn.Linear + + +def nonzero_tuple(x): + """ + A 'as_tuple=True' version of torch.nonzero to support torchscript. + because of https://github.com/pytorch/pytorch/issues/38718 + """ + if torch.jit.is_scripting(): + if x.dim() == 0: + return x.unsqueeze(0).nonzero().unbind(1) + return x.nonzero().unbind(1) + else: + return x.nonzero(as_tuple=True) + + +@torch.jit.script_if_tracing +def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + """ + Tracing friendly way to cast tensor to another tensor's device. Device will be treated + as constant during tracing, scripting the casting process as whole can workaround this issue. + """ + return src.to(dst.device) diff --git a/detectron2/model_zoo/__init__.py b/detectron2/model_zoo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1144331fee61ad472dd46fb785a275e51f9c99f7 --- /dev/null +++ b/detectron2/model_zoo/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Model Zoo API for Detectron2: a collection of functions to create common model architectures +listed in `MODEL_ZOO.md `_, +and optionally load their pre-trained weights. +""" + +from .model_zoo import get, get_checkpoint_url, get_config, get_config_file + +__all__ = ["get_checkpoint_url", "get", "get_config_file", "get_config"] diff --git a/detectron2/model_zoo/model_zoo.py b/detectron2/model_zoo/model_zoo.py new file mode 100644 index 0000000000000000000000000000000000000000..3af56b51bd603a3111a423f47da5d83794acd5e3 --- /dev/null +++ b/detectron2/model_zoo/model_zoo.py @@ -0,0 +1,212 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import os +from typing import Optional + +import pkg_resources +import torch + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import CfgNode, LazyConfig, get_cfg, instantiate +from detectron2.modeling import build_model + + +class _ModelZooUrls(object): + """ + Mapping from names to officially released Detectron2 pre-trained models. + """ + + S3_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" + + # format: {config_path.yaml} -> model_id/model_final_{commit}.pkl + CONFIG_PATH_TO_URL_SUFFIX = { + # COCO Detection with Faster R-CNN + "COCO-Detection/faster_rcnn_R_50_C4_1x": "137257644/model_final_721ade.pkl", + "COCO-Detection/faster_rcnn_R_50_DC5_1x": "137847829/model_final_51d356.pkl", + "COCO-Detection/faster_rcnn_R_50_FPN_1x": "137257794/model_final_b275ba.pkl", + "COCO-Detection/faster_rcnn_R_50_C4_3x": "137849393/model_final_f97cb7.pkl", + "COCO-Detection/faster_rcnn_R_50_DC5_3x": "137849425/model_final_68d202.pkl", + "COCO-Detection/faster_rcnn_R_50_FPN_3x": "137849458/model_final_280758.pkl", + "COCO-Detection/faster_rcnn_R_101_C4_3x": "138204752/model_final_298dad.pkl", + "COCO-Detection/faster_rcnn_R_101_DC5_3x": "138204841/model_final_3e0943.pkl", + "COCO-Detection/faster_rcnn_R_101_FPN_3x": "137851257/model_final_f6e8b1.pkl", + "COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x": "139173657/model_final_68b088.pkl", + # COCO Detection with RetinaNet + "COCO-Detection/retinanet_R_50_FPN_1x": "190397773/model_final_bfca0b.pkl", + "COCO-Detection/retinanet_R_50_FPN_3x": "190397829/model_final_5bd44e.pkl", + "COCO-Detection/retinanet_R_101_FPN_3x": "190397697/model_final_971ab9.pkl", + # COCO Detection with RPN and Fast R-CNN + "COCO-Detection/rpn_R_50_C4_1x": "137258005/model_final_450694.pkl", + "COCO-Detection/rpn_R_50_FPN_1x": "137258492/model_final_02ce48.pkl", + "COCO-Detection/fast_rcnn_R_50_FPN_1x": "137635226/model_final_e5f7ce.pkl", + # COCO Instance Segmentation Baselines with Mask R-CNN + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x": "137259246/model_final_9243eb.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x": "137260150/model_final_4f86c3.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x": "137260431/model_final_a54504.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x": "137849525/model_final_4ce675.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x": "137849551/model_final_84107b.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x": "137849600/model_final_f10217.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x": "138363239/model_final_a2914c.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x": "138363294/model_final_0464b7.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x": "138205316/model_final_a3ec72.pkl", + "COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x": "139653917/model_final_2d9806.pkl", # noqa + # New baselines using Large-Scale Jitter and Longer Training Schedule + "new_baselines/mask_rcnn_R_50_FPN_100ep_LSJ": "42047764/model_final_bb69de.pkl", + "new_baselines/mask_rcnn_R_50_FPN_200ep_LSJ": "42047638/model_final_89a8d3.pkl", + "new_baselines/mask_rcnn_R_50_FPN_400ep_LSJ": "42019571/model_final_14d201.pkl", + "new_baselines/mask_rcnn_R_101_FPN_100ep_LSJ": "42025812/model_final_4f7b58.pkl", + "new_baselines/mask_rcnn_R_101_FPN_200ep_LSJ": "42131867/model_final_0bb7ae.pkl", + "new_baselines/mask_rcnn_R_101_FPN_400ep_LSJ": "42073830/model_final_f96b26.pkl", + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ": "42047771/model_final_b7fbab.pkl", # noqa + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ": "42132721/model_final_5d87c1.pkl", # noqa + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ": "42025447/model_final_f1362d.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ": "42047784/model_final_6ba57e.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ": "42047642/model_final_27b9c1.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ": "42045954/model_final_ef3a80.pkl", # noqa + # COCO Person Keypoint Detection Baselines with Keypoint R-CNN + "COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x": "137261548/model_final_04e291.pkl", + "COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x": "137849621/model_final_a6e10b.pkl", + "COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x": "138363331/model_final_997cc7.pkl", + "COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x": "139686956/model_final_5ad38f.pkl", + # COCO Panoptic Segmentation Baselines with Panoptic FPN + "COCO-PanopticSegmentation/panoptic_fpn_R_50_1x": "139514544/model_final_dbfeb4.pkl", + "COCO-PanopticSegmentation/panoptic_fpn_R_50_3x": "139514569/model_final_c10459.pkl", + "COCO-PanopticSegmentation/panoptic_fpn_R_101_3x": "139514519/model_final_cafdb1.pkl", + # LVIS Instance Segmentation Baselines with Mask R-CNN + "LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x": "144219072/model_final_571f7c.pkl", # noqa + "LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x": "144219035/model_final_824ab5.pkl", # noqa + "LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x": "144219108/model_final_5e3439.pkl", # noqa + # Cityscapes & Pascal VOC Baselines + "Cityscapes/mask_rcnn_R_50_FPN": "142423278/model_final_af9cf5.pkl", + "PascalVOC-Detection/faster_rcnn_R_50_C4": "142202221/model_final_b1acc2.pkl", + # Other Settings + "Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5": "138602867/model_final_65c703.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5": "144998336/model_final_821d0b.pkl", + "Misc/cascade_mask_rcnn_R_50_FPN_1x": "138602847/model_final_e9d89b.pkl", + "Misc/cascade_mask_rcnn_R_50_FPN_3x": "144998488/model_final_480dd8.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_syncbn": "169527823/model_final_3b3c51.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_gn": "138602888/model_final_dc5d9e.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_3x_gn": "138602908/model_final_01ca85.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_9x_gn": "183808979/model_final_da7b4c.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn": "184226666/model_final_5ce33e.pkl", + "Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x": "139797668/model_final_be35db.pkl", + "Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv": "18131413/model_0039999_e76410.pkl", # noqa + # D1 Comparisons + "Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x": "137781054/model_final_7ab50c.pkl", # noqa + "Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x": "137781281/model_final_62ca52.pkl", # noqa + "Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x": "137781195/model_final_cce136.pkl", + } + + @staticmethod + def query(config_path: str) -> Optional[str]: + """ + Args: + config_path: relative config filename + """ + name = config_path.replace(".yaml", "").replace(".py", "") + if name in _ModelZooUrls.CONFIG_PATH_TO_URL_SUFFIX: + suffix = _ModelZooUrls.CONFIG_PATH_TO_URL_SUFFIX[name] + return _ModelZooUrls.S3_PREFIX + name + "/" + suffix + return None + + +def get_checkpoint_url(config_path): + """ + Returns the URL to the model trained using the given config + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + + Returns: + str: a URL to the model + """ + url = _ModelZooUrls.query(config_path) + if url is None: + raise RuntimeError("Pretrained model for {} is not available!".format(config_path)) + return url + + +def get_config_file(config_path): + """ + Returns path to a builtin config file. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + + Returns: + str: the real path to the config file. + """ + cfg_file = pkg_resources.resource_filename("detectron2.model_zoo", os.path.join("configs", config_path)) + if not os.path.exists(cfg_file): + raise RuntimeError("{} not available in Model Zoo!".format(config_path)) + return cfg_file + + +def get_config(config_path, trained: bool = False): + """ + Returns a config object for a model in model zoo. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + trained (bool): If True, will set ``MODEL.WEIGHTS`` to trained model zoo weights. + If False, the checkpoint specified in the config file's ``MODEL.WEIGHTS`` is used + instead; this will typically (though not always) initialize a subset of weights using + an ImageNet pre-trained model, while randomly initializing the other weights. + + Returns: + CfgNode or omegaconf.DictConfig: a config object + """ + cfg_file = get_config_file(config_path) + if cfg_file.endswith(".yaml"): + cfg = get_cfg() + cfg.merge_from_file(cfg_file) + if trained: + cfg.MODEL.WEIGHTS = get_checkpoint_url(config_path) + return cfg + elif cfg_file.endswith(".py"): + cfg = LazyConfig.load(cfg_file) + if trained: + url = get_checkpoint_url(config_path) + if "train" in cfg and "init_checkpoint" in cfg.train: + cfg.train.init_checkpoint = url + else: + raise NotImplementedError + return cfg + + +def get(config_path, trained: bool = False, device: Optional[str] = None): + """ + Get a model specified by relative path under Detectron2's official ``configs/`` directory. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + trained (bool): see :func:`get_config`. + device (str or None): overwrite the device in config, if given. + + Returns: + nn.Module: a detectron2 model. Will be in training mode. + + Example: + :: + from detectron2 import model_zoo + model = model_zoo.get("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml", trained=True) + """ + cfg = get_config(config_path, trained) + if device is None and not torch.cuda.is_available(): + device = "cpu" + if device is not None and isinstance(cfg, CfgNode): + cfg.MODEL.DEVICE = device + + if isinstance(cfg, CfgNode): + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + model = instantiate(cfg.model) + if device is not None: + model = model.to(device) + if "train" in cfg and "init_checkpoint" in cfg.train: + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + return model diff --git a/detectron2/modeling/__init__.py b/detectron2/modeling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b8c4afc9e5ecc5c4c806ea9a12327c513f07bbb1 --- /dev/null +++ b/detectron2/modeling/__init__.py @@ -0,0 +1,64 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from detectron2.layers import ShapeSpec + +from .anchor_generator import ANCHOR_GENERATOR_REGISTRY, build_anchor_generator +from .backbone import ( + BACKBONE_REGISTRY, + FPN, + Backbone, + MViT, + ResNet, + ResNetBlockBase, + SimpleFeaturePyramid, + SwinTransformer, + ViT, + build_backbone, + build_resnet_backbone, + get_vit_lr_decay_rate, + make_stage, +) +from .meta_arch import ( + FCOS, + META_ARCH_REGISTRY, + SEM_SEG_HEADS_REGISTRY, + GeneralizedRCNN, + PanopticFPN, + ProposalNetwork, + RetinaNet, + SemanticSegmentor, + build_model, + build_sem_seg_head, +) +from .mmdet_wrapper import MMDetBackbone, MMDetDetector +from .postprocessing import detector_postprocess +from .proposal_generator import ( + PROPOSAL_GENERATOR_REGISTRY, + RPN_HEAD_REGISTRY, + build_proposal_generator, + build_rpn_head, +) +from .roi_heads import ( + ROI_BOX_HEAD_REGISTRY, + ROI_HEADS_REGISTRY, + ROI_KEYPOINT_HEAD_REGISTRY, + ROI_MASK_HEAD_REGISTRY, + BaseKeypointRCNNHead, + BaseMaskRCNNHead, + FastRCNNOutputLayers, + ROIHeads, + StandardROIHeads, + build_box_head, + build_keypoint_head, + build_mask_head, + build_roi_heads, +) +from .test_time_augmentation import DatasetMapperTTA, GeneralizedRCNNWithTTA + +_EXCLUDE = {"ShapeSpec"} +__all__ = [k for k in globals().keys() if k not in _EXCLUDE and not k.startswith("_")] + + +from detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/detectron2/modeling/anchor_generator.py b/detectron2/modeling/anchor_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..ef261614f31a2d85cd0825b5ca2cc2b012f676ec --- /dev/null +++ b/detectron2/modeling/anchor_generator.py @@ -0,0 +1,381 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import collections +import math +from typing import List + +import torch +from torch import nn + +from detectron2.config import configurable +from detectron2.layers import ShapeSpec, move_device_like +from detectron2.structures import Boxes, RotatedBoxes +from detectron2.utils.registry import Registry + +ANCHOR_GENERATOR_REGISTRY = Registry("ANCHOR_GENERATOR") +ANCHOR_GENERATOR_REGISTRY.__doc__ = """ +Registry for modules that creates object detection anchors for feature maps. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +class BufferList(nn.Module): + """ + Similar to nn.ParameterList, but for buffers + """ + + def __init__(self, buffers): + super().__init__() + for i, buffer in enumerate(buffers): + # Use non-persistent buffer so the values are not saved in checkpoint + self.register_buffer(str(i), buffer, persistent=False) + + def __len__(self): + return len(self._buffers) + + def __iter__(self): + return iter(self._buffers.values()) + + +def _create_grid_offsets(size: List[int], stride: int, offset: float, target_device_tensor: torch.Tensor): + grid_height, grid_width = size + shifts_x = move_device_like( + torch.arange(offset * stride, grid_width * stride, step=stride, dtype=torch.float32), + target_device_tensor, + ) + shifts_y = move_device_like( + torch.arange(offset * stride, grid_height * stride, step=stride, dtype=torch.float32), + target_device_tensor, + ) + + shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x) + shift_x = shift_x.reshape(-1) + shift_y = shift_y.reshape(-1) + return shift_x, shift_y + + +def _broadcast_params(params, num_features, name): + """ + If one size (or aspect ratio) is specified and there are multiple feature + maps, we "broadcast" anchors of that single size (or aspect ratio) + over all feature maps. + + If params is list[float], or list[list[float]] with len(params) == 1, repeat + it num_features time. + + Returns: + list[list[float]]: param for each feature + """ + assert isinstance(params, collections.abc.Sequence), f"{name} in anchor generator has to be a list! Got {params}." + assert len(params), f"{name} in anchor generator cannot be empty!" + if not isinstance(params[0], collections.abc.Sequence): # params is list[float] + return [params] * num_features + if len(params) == 1: + return list(params) * num_features + assert len(params) == num_features, ( + f"Got {name} of length {len(params)} in anchor generator, " + f"but the number of input features is {num_features}!" + ) + return params + + +@ANCHOR_GENERATOR_REGISTRY.register() +class DefaultAnchorGenerator(nn.Module): + """ + Compute anchors in the standard ways described in + "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks". + """ + + box_dim: torch.jit.Final[int] = 4 + """ + the dimension of each anchor box. + """ + + @configurable + def __init__(self, *, sizes, aspect_ratios, strides, offset=0.5): + """ + This interface is experimental. + + Args: + sizes (list[list[float]] or list[float]): + If ``sizes`` is list[list[float]], ``sizes[i]`` is the list of anchor sizes + (i.e. sqrt of anchor area) to use for the i-th feature map. + If ``sizes`` is list[float], ``sizes`` is used for all feature maps. + Anchor sizes are given in absolute lengths in units of + the input image; they do not dynamically scale if the input image size changes. + aspect_ratios (list[list[float]] or list[float]): list of aspect ratios + (i.e. height / width) to use for anchors. Same "broadcast" rule for `sizes` applies. + strides (list[int]): stride of each input feature. + offset (float): Relative offset between the center of the first anchor and the top-left + corner of the image. Value has to be in [0, 1). + Recommend to use 0.5, which means half stride. + """ + super().__init__() + + self.strides = strides + self.num_features = len(self.strides) + sizes = _broadcast_params(sizes, self.num_features, "sizes") + aspect_ratios = _broadcast_params(aspect_ratios, self.num_features, "aspect_ratios") + self.cell_anchors = self._calculate_anchors(sizes, aspect_ratios) + + self.offset = offset + assert 0.0 <= self.offset < 1.0, self.offset + + @classmethod + def from_config(cls, cfg, input_shape: List[ShapeSpec]): + return { + "sizes": cfg.MODEL.ANCHOR_GENERATOR.SIZES, + "aspect_ratios": cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS, + "strides": [x.stride for x in input_shape], + "offset": cfg.MODEL.ANCHOR_GENERATOR.OFFSET, + } + + def _calculate_anchors(self, sizes, aspect_ratios): + cell_anchors = [self.generate_cell_anchors(s, a).float() for s, a in zip(sizes, aspect_ratios)] + return BufferList(cell_anchors) + + @property + @torch.jit.unused + def num_cell_anchors(self): + """ + Alias of `num_anchors`. + """ + return self.num_anchors + + @property + @torch.jit.unused + def num_anchors(self): + """ + Returns: + list[int]: Each int is the number of anchors at every pixel + location, on that feature map. + For example, if at every pixel we use anchors of 3 aspect + ratios and 5 sizes, the number of anchors is 15. + (See also ANCHOR_GENERATOR.SIZES and ANCHOR_GENERATOR.ASPECT_RATIOS in config) + + In standard RPN models, `num_anchors` on every feature map is the same. + """ + return [len(cell_anchors) for cell_anchors in self.cell_anchors] + + def _grid_anchors(self, grid_sizes: List[List[int]]): + """ + Returns: + list[Tensor]: #featuremap tensors, each is (#locations x #cell_anchors) x 4 + """ + anchors = [] + # buffers() not supported by torchscript. use named_buffers() instead + buffers: List[torch.Tensor] = [x[1] for x in self.cell_anchors.named_buffers()] + for size, stride, base_anchors in zip(grid_sizes, self.strides, buffers): + shift_x, shift_y = _create_grid_offsets(size, stride, self.offset, base_anchors) + shifts = torch.stack((shift_x, shift_y, shift_x, shift_y), dim=1) + + anchors.append((shifts.view(-1, 1, 4) + base_anchors.view(1, -1, 4)).reshape(-1, 4)) + + return anchors + + def generate_cell_anchors(self, sizes=(32, 64, 128, 256, 512), aspect_ratios=(0.5, 1, 2)): + """ + Generate a tensor storing canonical anchor boxes, which are all anchor + boxes of different sizes and aspect_ratios centered at (0, 0). + We can later build the set of anchors for a full feature map by + shifting and tiling these tensors (see `meth:_grid_anchors`). + + Args: + sizes (tuple[float]): + aspect_ratios (tuple[float]]): + + Returns: + Tensor of shape (len(sizes) * len(aspect_ratios), 4) storing anchor boxes + in XYXY format. + """ + + # This is different from the anchor generator defined in the original Faster R-CNN + # code or Detectron. They yield the same AP, however the old version defines cell + # anchors in a less natural way with a shift relative to the feature grid and + # quantization that results in slightly different sizes for different aspect ratios. + # See also https://github.com/facebookresearch/Detectron/issues/227 + + anchors = [] + for size in sizes: + area = size**2.0 + for aspect_ratio in aspect_ratios: + # s * s = w * h + # a = h / w + # ... some algebra ... + # w = sqrt(s * s / a) + # h = a * w + w = math.sqrt(area / aspect_ratio) + h = aspect_ratio * w + x0, y0, x1, y1 = -w / 2.0, -h / 2.0, w / 2.0, h / 2.0 + anchors.append([x0, y0, x1, y1]) + return torch.tensor(anchors) + + def forward(self, features: List[torch.Tensor]): + """ + Args: + features (list[Tensor]): list of backbone feature maps on which to generate anchors. + + Returns: + list[Boxes]: a list of Boxes containing all the anchors for each feature map + (i.e. the cell anchors repeated over all locations in the feature map). + The number of anchors of each feature map is Hi x Wi x num_cell_anchors, + where Hi, Wi are resolution of the feature map divided by anchor stride. + """ + grid_sizes = [feature_map.shape[-2:] for feature_map in features] + anchors_over_all_feature_maps = self._grid_anchors(grid_sizes) + return [Boxes(x) for x in anchors_over_all_feature_maps] + + +@ANCHOR_GENERATOR_REGISTRY.register() +class RotatedAnchorGenerator(nn.Module): + """ + Compute rotated anchors used by Rotated RPN (RRPN), described in + "Arbitrary-Oriented Scene Text Detection via Rotation Proposals". + """ + + box_dim: int = 5 + """ + the dimension of each anchor box. + """ + + @configurable + def __init__(self, *, sizes, aspect_ratios, strides, angles, offset=0.5): + """ + This interface is experimental. + + Args: + sizes (list[list[float]] or list[float]): + If sizes is list[list[float]], sizes[i] is the list of anchor sizes + (i.e. sqrt of anchor area) to use for the i-th feature map. + If sizes is list[float], the sizes are used for all feature maps. + Anchor sizes are given in absolute lengths in units of + the input image; they do not dynamically scale if the input image size changes. + aspect_ratios (list[list[float]] or list[float]): list of aspect ratios + (i.e. height / width) to use for anchors. Same "broadcast" rule for `sizes` applies. + strides (list[int]): stride of each input feature. + angles (list[list[float]] or list[float]): list of angles (in degrees CCW) + to use for anchors. Same "broadcast" rule for `sizes` applies. + offset (float): Relative offset between the center of the first anchor and the top-left + corner of the image. Value has to be in [0, 1). + Recommend to use 0.5, which means half stride. + """ + super().__init__() + + self.strides = strides + self.num_features = len(self.strides) + sizes = _broadcast_params(sizes, self.num_features, "sizes") + aspect_ratios = _broadcast_params(aspect_ratios, self.num_features, "aspect_ratios") + angles = _broadcast_params(angles, self.num_features, "angles") + self.cell_anchors = self._calculate_anchors(sizes, aspect_ratios, angles) + + self.offset = offset + assert 0.0 <= self.offset < 1.0, self.offset + + @classmethod + def from_config(cls, cfg, input_shape: List[ShapeSpec]): + return { + "sizes": cfg.MODEL.ANCHOR_GENERATOR.SIZES, + "aspect_ratios": cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS, + "strides": [x.stride for x in input_shape], + "offset": cfg.MODEL.ANCHOR_GENERATOR.OFFSET, + "angles": cfg.MODEL.ANCHOR_GENERATOR.ANGLES, + } + + def _calculate_anchors(self, sizes, aspect_ratios, angles): + cell_anchors = [ + self.generate_cell_anchors(size, aspect_ratio, angle).float() + for size, aspect_ratio, angle in zip(sizes, aspect_ratios, angles) + ] + return BufferList(cell_anchors) + + @property + def num_cell_anchors(self): + """ + Alias of `num_anchors`. + """ + return self.num_anchors + + @property + def num_anchors(self): + """ + Returns: + list[int]: Each int is the number of anchors at every pixel + location, on that feature map. + For example, if at every pixel we use anchors of 3 aspect + ratios, 2 sizes and 5 angles, the number of anchors is 30. + (See also ANCHOR_GENERATOR.SIZES, ANCHOR_GENERATOR.ASPECT_RATIOS + and ANCHOR_GENERATOR.ANGLES in config) + + In standard RRPN models, `num_anchors` on every feature map is the same. + """ + return [len(cell_anchors) for cell_anchors in self.cell_anchors] + + def _grid_anchors(self, grid_sizes): + anchors = [] + for size, stride, base_anchors in zip(grid_sizes, self.strides, self.cell_anchors): + shift_x, shift_y = _create_grid_offsets(size, stride, self.offset, base_anchors) + zeros = torch.zeros_like(shift_x) + shifts = torch.stack((shift_x, shift_y, zeros, zeros, zeros), dim=1) + + anchors.append((shifts.view(-1, 1, 5) + base_anchors.view(1, -1, 5)).reshape(-1, 5)) + + return anchors + + def generate_cell_anchors( + self, + sizes=(32, 64, 128, 256, 512), + aspect_ratios=(0.5, 1, 2), + angles=(-90, -60, -30, 0, 30, 60, 90), + ): + """ + Generate a tensor storing canonical anchor boxes, which are all anchor + boxes of different sizes, aspect_ratios, angles centered at (0, 0). + We can later build the set of anchors for a full feature map by + shifting and tiling these tensors (see `meth:_grid_anchors`). + + Args: + sizes (tuple[float]): + aspect_ratios (tuple[float]]): + angles (tuple[float]]): + + Returns: + Tensor of shape (len(sizes) * len(aspect_ratios) * len(angles), 5) + storing anchor boxes in (x_ctr, y_ctr, w, h, angle) format. + """ + anchors = [] + for size in sizes: + area = size**2.0 + for aspect_ratio in aspect_ratios: + # s * s = w * h + # a = h / w + # ... some algebra ... + # w = sqrt(s * s / a) + # h = a * w + w = math.sqrt(area / aspect_ratio) + h = aspect_ratio * w + anchors.extend([0, 0, w, h, a] for a in angles) + + return torch.tensor(anchors) + + def forward(self, features): + """ + Args: + features (list[Tensor]): list of backbone feature maps on which to generate anchors. + + Returns: + list[RotatedBoxes]: a list of Boxes containing all the anchors for each feature map + (i.e. the cell anchors repeated over all locations in the feature map). + The number of anchors of each feature map is Hi x Wi x num_cell_anchors, + where Hi, Wi are resolution of the feature map divided by anchor stride. + """ + grid_sizes = [feature_map.shape[-2:] for feature_map in features] + anchors_over_all_feature_maps = self._grid_anchors(grid_sizes) + return [RotatedBoxes(x) for x in anchors_over_all_feature_maps] + + +def build_anchor_generator(cfg, input_shape): + """ + Built an anchor generator from `cfg.MODEL.ANCHOR_GENERATOR.NAME`. + """ + anchor_generator = cfg.MODEL.ANCHOR_GENERATOR.NAME + return ANCHOR_GENERATOR_REGISTRY.get(anchor_generator)(cfg, input_shape) diff --git a/detectron2/modeling/backbone/__init__.py b/detectron2/modeling/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..32359f7011a05680fa77324ce13b87196157a610 --- /dev/null +++ b/detectron2/modeling/backbone/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build import build_backbone, BACKBONE_REGISTRY # noqa F401 isort:skip + +from .backbone import Backbone +from .fpn import FPN +from .mvit import MViT +from .regnet import RegNet +from .resnet import ( + BasicStem, + BottleneckBlock, + ResNet, + ResNetBlockBase, + build_resnet_backbone, + make_stage, +) +from .swin import SwinTransformer +from .vit import SimpleFeaturePyramid, ViT, get_vit_lr_decay_rate + +__all__ = [k for k in globals().keys() if not k.startswith("_")] +# TODO can expose more resnet blocks after careful consideration diff --git a/detectron2/modeling/backbone/backbone.py b/detectron2/modeling/backbone/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..533f2d1eef323c9538f54d82b99f93ce534e628b --- /dev/null +++ b/detectron2/modeling/backbone/backbone.py @@ -0,0 +1,73 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from abc import ABCMeta, abstractmethod +from typing import Dict + +import torch.nn as nn + +from detectron2.layers import ShapeSpec + +__all__ = ["Backbone"] + + +class Backbone(nn.Module, metaclass=ABCMeta): + """ + Abstract base class for network backbones. + """ + + def __init__(self): + """ + The `__init__` method of any subclass can specify its own set of arguments. + """ + super().__init__() + + @abstractmethod + def forward(self): + """ + Subclasses must override this method, but adhere to the same return type. + + Returns: + dict[str->Tensor]: mapping from feature name (e.g., "res2") to tensor + """ + pass + + @property + def size_divisibility(self) -> int: + """ + Some backbones require the input height and width to be divisible by a + specific integer. This is typically true for encoder / decoder type networks + with lateral connection (e.g., FPN) for which feature maps need to match + dimension in the "bottom up" and "top down" paths. Set to 0 if no specific + input size divisibility is required. + """ + return 0 + + @property + def padding_constraints(self) -> Dict[str, int]: + """ + This property is a generalization of size_divisibility. Some backbones and training + recipes require specific padding constraints, such as enforcing divisibility by a specific + integer (e.g., FPN) or padding to a square (e.g., ViTDet with large-scale jitter + in :paper:vitdet). `padding_constraints` contains these optional items like: + { + "size_divisibility": int, + "square_size": int, + # Future options are possible + } + `size_divisibility` will read from here if presented and `square_size` indicates the + square padding size if `square_size` > 0. + + TODO: use type of Dict[str, int] to avoid torchscipt issues. The type of padding_constraints + could be generalized as TypedDict (Python 3.8+) to support more types in the future. + """ + return {} + + def output_shape(self): + """ + Returns: + dict[str->ShapeSpec] + """ + # this is a backward-compatible default + return { + name: ShapeSpec(channels=self._out_feature_channels[name], stride=self._out_feature_strides[name]) + for name in self._out_features + } diff --git a/detectron2/modeling/backbone/build.py b/detectron2/modeling/backbone/build.py new file mode 100644 index 0000000000000000000000000000000000000000..af02141172bebe9a2a27a88c81673c2710b4d73f --- /dev/null +++ b/detectron2/modeling/backbone/build.py @@ -0,0 +1,33 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from detectron2.layers import ShapeSpec +from detectron2.utils.registry import Registry + +from .backbone import Backbone + +BACKBONE_REGISTRY = Registry("BACKBONE") +BACKBONE_REGISTRY.__doc__ = """ +Registry for backbones, which extract feature maps from images + +The registered object must be a callable that accepts two arguments: + +1. A :class:`detectron2.config.CfgNode` +2. A :class:`detectron2.layers.ShapeSpec`, which contains the input shape specification. + +Registered object must return instance of :class:`Backbone`. +""" + + +def build_backbone(cfg, input_shape=None): + """ + Build a backbone from `cfg.MODEL.BACKBONE.NAME`. + + Returns: + an instance of :class:`Backbone` + """ + if input_shape is None: + input_shape = ShapeSpec(channels=len(cfg.MODEL.PIXEL_MEAN)) + + backbone_name = cfg.MODEL.BACKBONE.NAME + backbone = BACKBONE_REGISTRY.get(backbone_name)(cfg, input_shape) + assert isinstance(backbone, Backbone) + return backbone diff --git a/detectron2/modeling/backbone/fpn.py b/detectron2/modeling/backbone/fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..9bd44ecd82b7669bb38d35aaf0d16b5bfd7d6d18 --- /dev/null +++ b/detectron2/modeling/backbone/fpn.py @@ -0,0 +1,261 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math + +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn.functional as F +from torch import nn + +from detectron2.layers import Conv2d, ShapeSpec, get_norm + +from .backbone import Backbone +from .build import BACKBONE_REGISTRY +from .resnet import build_resnet_backbone + +__all__ = ["build_resnet_fpn_backbone", "build_retinanet_resnet_fpn_backbone", "FPN"] + + +class FPN(Backbone): + """ + This module implements :paper:`FPN`. + It creates pyramid features built on top of some input feature maps. + """ + + _fuse_type: torch.jit.Final[str] + + def __init__( + self, + bottom_up, + in_features, + out_channels, + norm="", + top_block=None, + fuse_type="sum", + square_pad=0, + ): + """ + Args: + bottom_up (Backbone): module representing the bottom up subnetwork. + Must be a subclass of :class:`Backbone`. The multi-scale feature + maps generated by the bottom up network, and listed in `in_features`, + are used to generate FPN levels. + in_features (list[str]): names of the input feature maps coming + from the backbone to which FPN is attached. For example, if the + backbone produces ["res2", "res3", "res4"], any *contiguous* sublist + of these may be used; order must be from high to low resolution. + out_channels (int): number of channels in the output feature maps. + norm (str): the normalization to use. + top_block (nn.Module or None): if provided, an extra operation will + be performed on the output of the last (smallest resolution) + FPN output, and the result will extend the result list. The top_block + further downsamples the feature map. It must have an attribute + "num_levels", meaning the number of extra FPN levels added by + this block, and "in_feature", which is a string representing + its input feature (e.g., p5). + fuse_type (str): types for fusing the top down features and the lateral + ones. It can be "sum" (default), which sums up element-wise; or "avg", + which takes the element-wise mean of the two. + square_pad (int): If > 0, require input images to be padded to specific square size. + """ + super(FPN, self).__init__() + assert isinstance(bottom_up, Backbone) + assert in_features, in_features + + # Feature map strides and channels from the bottom up network (e.g. ResNet) + input_shapes = bottom_up.output_shape() + strides = [input_shapes[f].stride for f in in_features] + in_channels_per_feature = [input_shapes[f].channels for f in in_features] + + _assert_strides_are_log2_contiguous(strides) + lateral_convs = [] + output_convs = [] + + use_bias = norm == "" + for idx, in_channels in enumerate(in_channels_per_feature): + lateral_norm = get_norm(norm, out_channels) + output_norm = get_norm(norm, out_channels) + + lateral_conv = Conv2d(in_channels, out_channels, kernel_size=1, bias=use_bias, norm=lateral_norm) + output_conv = Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + ) + weight_init.c2_xavier_fill(lateral_conv) + weight_init.c2_xavier_fill(output_conv) + stage = int(math.log2(strides[idx])) + self.add_module("fpn_lateral{}".format(stage), lateral_conv) + self.add_module("fpn_output{}".format(stage), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + self.top_block = top_block + self.in_features = tuple(in_features) + self.bottom_up = bottom_up + # Return feature names are "p", like ["p2", "p3", ..., "p6"] + self._out_feature_strides = {"p{}".format(int(math.log2(s))): s for s in strides} + # top block output feature maps. + if self.top_block is not None: + for s in range(stage, stage + self.top_block.num_levels): + self._out_feature_strides["p{}".format(s + 1)] = 2 ** (s + 1) + + self._out_features = list(self._out_feature_strides.keys()) + self._out_feature_channels = {k: out_channels for k in self._out_features} + self._size_divisibility = strides[-1] + self._square_pad = square_pad + assert fuse_type in {"avg", "sum"} + self._fuse_type = fuse_type + + @property + def size_divisibility(self): + return self._size_divisibility + + @property + def padding_constraints(self): + return {"square_size": self._square_pad} + + def forward(self, x): + """ + Args: + input (dict[str->Tensor]): mapping feature map name (e.g., "res5") to + feature map tensor for each feature level in high to low resolution order. + + Returns: + dict[str->Tensor]: + mapping from feature map name to FPN feature map tensor + in high to low resolution order. Returned feature names follow the FPN + paper convention: "p", where stage has stride = 2 ** stage e.g., + ["p2", "p3", ..., "p6"]. + """ + bottom_up_features = self.bottom_up(x) + results = [] + prev_features = self.lateral_convs[0](bottom_up_features[self.in_features[-1]]) + results.append(self.output_convs[0](prev_features)) + + # Reverse feature maps into top-down order (from low to high resolution) + for idx, (lateral_conv, output_conv) in enumerate(zip(self.lateral_convs, self.output_convs)): + # Slicing of ModuleList is not supported https://github.com/pytorch/pytorch/issues/47336 + # Therefore we loop over all modules but skip the first one + if idx > 0: + features = self.in_features[-idx - 1] + features = bottom_up_features[features] + top_down_features = F.interpolate(prev_features, scale_factor=2.0, mode="nearest") + lateral_features = lateral_conv(features) + prev_features = lateral_features + top_down_features + if self._fuse_type == "avg": + prev_features /= 2 + results.insert(0, output_conv(prev_features)) + + if self.top_block is not None: + if self.top_block.in_feature in bottom_up_features: + top_block_in_feature = bottom_up_features[self.top_block.in_feature] + else: + top_block_in_feature = results[self._out_features.index(self.top_block.in_feature)] + results.extend(self.top_block(top_block_in_feature)) + assert len(self._out_features) == len(results) + return {f: res for f, res in zip(self._out_features, results)} + + def output_shape(self): + return { + name: ShapeSpec(channels=self._out_feature_channels[name], stride=self._out_feature_strides[name]) + for name in self._out_features + } + + +def _assert_strides_are_log2_contiguous(strides): + """ + Assert that each stride is 2x times its preceding stride, i.e. "contiguous in log2". + """ + for i, stride in enumerate(strides[1:], 1): + assert stride == 2 * strides[i - 1], "Strides {} {} are not log2 contiguous".format(stride, strides[i - 1]) + + +class LastLevelMaxPool(nn.Module): + """ + This module is used in the original FPN to generate a downsampled + P6 feature from P5. + """ + + def __init__(self): + super().__init__() + self.num_levels = 1 + self.in_feature = "p5" + + def forward(self, x): + return [F.max_pool2d(x, kernel_size=1, stride=2, padding=0)] + + +class LastLevelP6P7(nn.Module): + """ + This module is used in RetinaNet to generate extra layers, P6 and P7 from + C5 feature. + """ + + def __init__(self, in_channels, out_channels, in_feature="res5"): + super().__init__() + self.num_levels = 2 + self.in_feature = in_feature + self.p6 = nn.Conv2d(in_channels, out_channels, 3, 2, 1) + self.p7 = nn.Conv2d(out_channels, out_channels, 3, 2, 1) + for module in [self.p6, self.p7]: + weight_init.c2_xavier_fill(module) + + def forward(self, c5): + p6 = self.p6(c5) + p7 = self.p7(F.relu(p6)) + return [p6, p7] + + +@BACKBONE_REGISTRY.register() +def build_resnet_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelMaxPool(), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone + + +@BACKBONE_REGISTRY.register() +def build_retinanet_resnet_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + in_channels_p6p7 = bottom_up.output_shape()["res5"].channels + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelP6P7(in_channels_p6p7, out_channels), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone diff --git a/detectron2/modeling/backbone/mvit.py b/detectron2/modeling/backbone/mvit.py new file mode 100644 index 0000000000000000000000000000000000000000..5f6de8c9e74a32ecb56ebf05da238a4d06291fc1 --- /dev/null +++ b/detectron2/modeling/backbone/mvit.py @@ -0,0 +1,444 @@ +import logging + +import numpy as np +import torch +import torch.nn as nn +from fairscale.nn.checkpoint import checkpoint_wrapper +from timm.models.layers import DropPath, Mlp, trunc_normal_ + +from .backbone import Backbone +from .utils import ( + PatchEmbed, + add_decomposed_rel_pos, + get_abs_pos, + window_partition, + window_unpartition, +) + +logger = logging.getLogger(__name__) + + +__all__ = ["MViT"] + + +def attention_pool(x, pool, norm=None): + # (B, H, W, C) -> (B, C, H, W) + x = x.permute(0, 3, 1, 2) + x = pool(x) + # (B, C, H1, W1) -> (B, H1, W1, C) + x = x.permute(0, 2, 3, 1) + if norm: + x = norm(x) + + return x + + +class MultiScaleAttention(nn.Module): + """Multiscale Multi-head Attention block.""" + + def __init__( + self, + dim, + dim_out, + num_heads, + qkv_bias=True, + norm_layer=nn.LayerNorm, + pool_kernel=(3, 3), + stride_q=1, + stride_kv=1, + residual_pooling=True, + window_size=0, + use_rel_pos=False, + rel_pos_zero_init=True, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + dim_out (int): Number of output channels. + num_heads (int): Number of attention heads. + qkv_bias (bool: If True, add a learnable bias to query, key, value. + norm_layer (nn.Module): Normalization layer. + pool_kernel (tuple): kernel size for qkv pooling layers. + stride_q (int): stride size for q pooling layer. + stride_kv (int): stride size for kv pooling layer. + residual_pooling (bool): If true, enable residual pooling. + use_rel_pos (bool): If True, add relative postional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (int or None): Input resolution. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim_out // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim_out * 3, bias=qkv_bias) + self.proj = nn.Linear(dim_out, dim_out) + + # qkv pooling + pool_padding = [k // 2 for k in pool_kernel] + dim_conv = dim_out // num_heads + self.pool_q = nn.Conv2d( + dim_conv, + dim_conv, + pool_kernel, + stride=stride_q, + padding=pool_padding, + groups=dim_conv, + bias=False, + ) + self.norm_q = norm_layer(dim_conv) + self.pool_k = nn.Conv2d( + dim_conv, + dim_conv, + pool_kernel, + stride=stride_kv, + padding=pool_padding, + groups=dim_conv, + bias=False, + ) + self.norm_k = norm_layer(dim_conv) + self.pool_v = nn.Conv2d( + dim_conv, + dim_conv, + pool_kernel, + stride=stride_kv, + padding=pool_padding, + groups=dim_conv, + bias=False, + ) + self.norm_v = norm_layer(dim_conv) + + self.window_size = window_size + if window_size: + self.q_win_size = window_size // stride_q + self.kv_win_size = window_size // stride_kv + self.residual_pooling = residual_pooling + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + # initialize relative positional embeddings + assert input_size[0] == input_size[1] + size = input_size[0] + rel_dim = 2 * max(size // stride_q, size // stride_kv) - 1 + self.rel_pos_h = nn.Parameter(torch.zeros(rel_dim, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(rel_dim, head_dim)) + + if not rel_pos_zero_init: + trunc_normal_(self.rel_pos_h, std=0.02) + trunc_normal_(self.rel_pos_w, std=0.02) + + def forward(self, x): + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H, W, C) + qkv = self.qkv(x).reshape(B, H, W, 3, self.num_heads, -1).permute(3, 0, 4, 1, 2, 5) + # q, k, v with shape (B * nHead, H, W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H, W, -1).unbind(0) + + q = attention_pool(q, self.pool_q, self.norm_q) + k = attention_pool(k, self.pool_k, self.norm_k) + v = attention_pool(v, self.pool_v, self.norm_v) + + ori_q = q + if self.window_size: + q, q_hw_pad = window_partition(q, self.q_win_size) + k, kv_hw_pad = window_partition(k, self.kv_win_size) + v, _ = window_partition(v, self.kv_win_size) + q_hw = (self.q_win_size, self.q_win_size) + kv_hw = (self.kv_win_size, self.kv_win_size) + else: + q_hw = q.shape[1:3] + kv_hw = k.shape[1:3] + + q = q.view(q.shape[0], np.prod(q_hw), -1) + k = k.view(k.shape[0], np.prod(kv_hw), -1) + v = v.view(v.shape[0], np.prod(kv_hw), -1) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, q_hw, kv_hw) + + attn = attn.softmax(dim=-1) + x = attn @ v + + x = x.view(x.shape[0], q_hw[0], q_hw[1], -1) + + if self.window_size: + x = window_unpartition(x, self.q_win_size, q_hw_pad, ori_q.shape[1:3]) + + if self.residual_pooling: + x += ori_q + + H, W = x.shape[1], x.shape[2] + x = x.view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +class MultiScaleBlock(nn.Module): + """Multiscale Transformer blocks""" + + def __init__( + self, + dim, + dim_out, + num_heads, + mlp_ratio=4.0, + qkv_bias=True, + drop_path=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + qkv_pool_kernel=(3, 3), + stride_q=1, + stride_kv=1, + residual_pooling=True, + window_size=0, + use_rel_pos=False, + rel_pos_zero_init=True, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + dim_out (int): Number of output channels. + num_heads (int): Number of attention heads in the MViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + qkv_pool_kernel (tuple): kernel size for qkv pooling layers. + stride_q (int): stride size for q pooling layer. + stride_kv (int): stride size for kv pooling layer. + residual_pooling (bool): If true, enable residual pooling. + window_size (int): Window size for window attention blocks. If it equals 0, then not + use window attention. + use_rel_pos (bool): If True, add relative postional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (int or None): Input resolution. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = MultiScaleAttention( + dim, + dim_out, + num_heads=num_heads, + qkv_bias=qkv_bias, + norm_layer=norm_layer, + pool_kernel=qkv_pool_kernel, + stride_q=stride_q, + stride_kv=stride_kv, + residual_pooling=residual_pooling, + window_size=window_size, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim_out) + self.mlp = Mlp( + in_features=dim_out, + hidden_features=int(dim_out * mlp_ratio), + out_features=dim_out, + act_layer=act_layer, + ) + + if dim != dim_out: + self.proj = nn.Linear(dim, dim_out) + + if stride_q > 1: + kernel_skip = stride_q + 1 + padding_skip = int(kernel_skip // 2) + self.pool_skip = nn.MaxPool2d(kernel_skip, stride_q, padding_skip, ceil_mode=False) + + def forward(self, x): + x_norm = self.norm1(x) + x_block = self.attn(x_norm) + + if hasattr(self, "proj"): + x = self.proj(x_norm) + if hasattr(self, "pool_skip"): + x = attention_pool(x, self.pool_skip) + + x = x + self.drop_path(x_block) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class MViT(Backbone): + """ + This module implements Multiscale Vision Transformer (MViT) backbone in :paper:'mvitv2'. + """ + + def __init__( + self, + img_size=224, + patch_kernel=(7, 7), + patch_stride=(4, 4), + patch_padding=(3, 3), + in_chans=3, + embed_dim=96, + depth=16, + num_heads=1, + last_block_indexes=(0, 2, 11, 15), + qkv_pool_kernel=(3, 3), + adaptive_kv_stride=4, + adaptive_window_size=56, + residual_pooling=True, + mlp_ratio=4.0, + qkv_bias=True, + drop_path_rate=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + use_abs_pos=False, + use_rel_pos=True, + rel_pos_zero_init=True, + use_act_checkpoint=False, + pretrain_img_size=224, + pretrain_use_cls_token=True, + out_features=("scale2", "scale3", "scale4", "scale5"), + ): + """ + Args: + img_size (int): Input image size. + patch_kernel (tuple): kernel size for patch embedding. + patch_stride (tuple): stride size for patch embedding. + patch_padding (tuple): padding size for patch embedding. + in_chans (int): Number of input image channels. + embed_dim (int): Patch embedding dimension. + depth (int): Depth of MViT. + num_heads (int): Number of base attention heads in each MViT block. + last_block_indexes (tuple): Block indexes for last blocks in each stage. + qkv_pool_kernel (tuple): kernel size for qkv pooling layers. + adaptive_kv_stride (int): adaptive stride size for kv pooling. + adaptive_window_size (int): adaptive window size for window attention blocks. + residual_pooling (bool): If true, enable residual pooling. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path_rate (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_abs_pos (bool): If True, use absolute positional embeddings. + use_rel_pos (bool): If True, add relative postional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. + use_act_checkpoint (bool): If True, use activation checkpointing. + pretrain_img_size (int): input image size for pretraining models. + pretrain_use_cls_token (bool): If True, pretrainig models use class token. + out_features (tuple): name of the feature maps from each stage. + """ + super().__init__() + self.pretrain_use_cls_token = pretrain_use_cls_token + + self.patch_embed = PatchEmbed( + kernel_size=patch_kernel, + stride=patch_stride, + padding=patch_padding, + in_chans=in_chans, + embed_dim=embed_dim, + ) + + if use_abs_pos: + # Initialize absoluate positional embedding with pretrain image size. + num_patches = (pretrain_img_size // patch_stride[0]) * (pretrain_img_size // patch_stride[1]) + num_positions = (num_patches + 1) if pretrain_use_cls_token else num_patches + self.pos_embed = nn.Parameter(torch.zeros(1, num_positions, embed_dim)) + else: + self.pos_embed = None + + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + dim_out = embed_dim + stride_kv = adaptive_kv_stride + window_size = adaptive_window_size + input_size = (img_size // patch_stride[0], img_size // patch_stride[1]) + stage = 2 + stride = patch_stride[0] + self._out_feature_strides = {} + self._out_feature_channels = {} + self.blocks = nn.ModuleList() + for i in range(depth): + # Multiply stride_kv by 2 if it's the last block of stage2 and stage3. + if i == last_block_indexes[1] or i == last_block_indexes[2]: + stride_kv_ = stride_kv * 2 + else: + stride_kv_ = stride_kv + # hybrid window attention: global attention in last three stages. + window_size_ = 0 if i in last_block_indexes[1:] else window_size + block = MultiScaleBlock( + dim=embed_dim, + dim_out=dim_out, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + qkv_pool_kernel=qkv_pool_kernel, + stride_q=2 if i - 1 in last_block_indexes else 1, + stride_kv=stride_kv_, + residual_pooling=residual_pooling, + window_size=window_size_, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size, + ) + if use_act_checkpoint: + block = checkpoint_wrapper(block) + self.blocks.append(block) + + embed_dim = dim_out + if i in last_block_indexes: + name = f"scale{stage}" + if name in out_features: + self._out_feature_channels[name] = dim_out + self._out_feature_strides[name] = stride + self.add_module(f"{name}_norm", norm_layer(dim_out)) + + dim_out *= 2 + num_heads *= 2 + stride_kv = max(stride_kv // 2, 1) + stride *= 2 + stage += 1 + if i - 1 in last_block_indexes: + window_size = window_size // 2 + input_size = [s // 2 for s in input_size] + + self._out_features = out_features + self._last_block_indexes = last_block_indexes + + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=0.02) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, x): + x = self.patch_embed(x) + + if self.pos_embed is not None: + x = x + get_abs_pos(self.pos_embed, self.pretrain_use_cls_token, x.shape[1:3]) + + outputs = {} + stage = 2 + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in self._last_block_indexes: + name = f"scale{stage}" + if name in self._out_features: + x_out = getattr(self, f"{name}_norm")(x) + outputs[name] = x_out.permute(0, 3, 1, 2) + stage += 1 + + return outputs diff --git a/detectron2/modeling/backbone/regnet.py b/detectron2/modeling/backbone/regnet.py new file mode 100644 index 0000000000000000000000000000000000000000..4e81043176990e0013223a55bb9d120d15ea4c02 --- /dev/null +++ b/detectron2/modeling/backbone/regnet.py @@ -0,0 +1,446 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Implementation of RegNet models from :paper:`dds` and :paper:`scaling`. + +This code is adapted from https://github.com/facebookresearch/pycls with minimal modifications. +Some code duplication exists between RegNet and ResNets (e.g., ResStem) in order to simplify +model loading. +""" + +import numpy as np +from torch import nn + +from detectron2.layers import CNNBlockBase, ShapeSpec, get_norm + +from .backbone import Backbone + +__all__ = [ + "AnyNet", + "RegNet", + "ResStem", + "SimpleStem", + "VanillaBlock", + "ResBasicBlock", + "ResBottleneckBlock", +] + + +def conv2d(w_in, w_out, k, *, stride=1, groups=1, bias=False): + """Helper for building a conv2d layer.""" + assert k % 2 == 1, "Only odd size kernels supported to avoid padding issues." + s, p, g, b = stride, (k - 1) // 2, groups, bias + return nn.Conv2d(w_in, w_out, k, stride=s, padding=p, groups=g, bias=b) + + +def gap2d(): + """Helper for building a global average pooling layer.""" + return nn.AdaptiveAvgPool2d((1, 1)) + + +def pool2d(k, *, stride=1): + """Helper for building a pool2d layer.""" + assert k % 2 == 1, "Only odd size kernels supported to avoid padding issues." + return nn.MaxPool2d(k, stride=stride, padding=(k - 1) // 2) + + +def init_weights(m): + """Performs ResNet-style weight initialization.""" + if isinstance(m, nn.Conv2d): + # Note that there is no bias due to BN + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(mean=0.0, std=np.sqrt(2.0 / fan_out)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.weight.data.normal_(mean=0.0, std=0.01) + m.bias.data.zero_() + + +class ResStem(CNNBlockBase): + """ResNet stem for ImageNet: 7x7, BN, AF, MaxPool.""" + + def __init__(self, w_in, w_out, norm, activation_class): + super().__init__(w_in, w_out, 4) + self.conv = conv2d(w_in, w_out, 7, stride=2) + self.bn = get_norm(norm, w_out) + self.af = activation_class() + self.pool = pool2d(3, stride=2) + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class SimpleStem(CNNBlockBase): + """Simple stem for ImageNet: 3x3, BN, AF.""" + + def __init__(self, w_in, w_out, norm, activation_class): + super().__init__(w_in, w_out, 2) + self.conv = conv2d(w_in, w_out, 3, stride=2) + self.bn = get_norm(norm, w_out) + self.af = activation_class() + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class SE(nn.Module): + """Squeeze-and-Excitation (SE) block: AvgPool, FC, Act, FC, Sigmoid.""" + + def __init__(self, w_in, w_se, activation_class): + super().__init__() + self.avg_pool = gap2d() + self.f_ex = nn.Sequential( + conv2d(w_in, w_se, 1, bias=True), + activation_class(), + conv2d(w_se, w_in, 1, bias=True), + nn.Sigmoid(), + ) + + def forward(self, x): + return x * self.f_ex(self.avg_pool(x)) + + +class VanillaBlock(CNNBlockBase): + """Vanilla block: [3x3 conv, BN, Relu] x2.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, _params): + super().__init__(w_in, w_out, stride) + self.a = conv2d(w_in, w_out, 3, stride=stride) + self.a_bn = get_norm(norm, w_out) + self.a_af = activation_class() + self.b = conv2d(w_out, w_out, 3) + self.b_bn = get_norm(norm, w_out) + self.b_af = activation_class() + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class BasicTransform(nn.Module): + """Basic transformation: [3x3 conv, BN, Relu] x2.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, _params): + super().__init__() + self.a = conv2d(w_in, w_out, 3, stride=stride) + self.a_bn = get_norm(norm, w_out) + self.a_af = activation_class() + self.b = conv2d(w_out, w_out, 3) + self.b_bn = get_norm(norm, w_out) + self.b_bn.final_bn = True + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class ResBasicBlock(CNNBlockBase): + """Residual basic block: x + f(x), f = basic transform.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, params): + super().__init__(w_in, w_out, stride) + self.proj, self.bn = None, None + if (w_in != w_out) or (stride != 1): + self.proj = conv2d(w_in, w_out, 1, stride=stride) + self.bn = get_norm(norm, w_out) + self.f = BasicTransform(w_in, w_out, stride, norm, activation_class, params) + self.af = activation_class() + + def forward(self, x): + x_p = self.bn(self.proj(x)) if self.proj else x + return self.af(x_p + self.f(x)) + + +class BottleneckTransform(nn.Module): + """Bottleneck transformation: 1x1, 3x3 [+SE], 1x1.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, params): + super().__init__() + w_b = int(round(w_out * params["bot_mul"])) + w_se = int(round(w_in * params["se_r"])) + groups = w_b // params["group_w"] + self.a = conv2d(w_in, w_b, 1) + self.a_bn = get_norm(norm, w_b) + self.a_af = activation_class() + self.b = conv2d(w_b, w_b, 3, stride=stride, groups=groups) + self.b_bn = get_norm(norm, w_b) + self.b_af = activation_class() + self.se = SE(w_b, w_se, activation_class) if w_se else None + self.c = conv2d(w_b, w_out, 1) + self.c_bn = get_norm(norm, w_out) + self.c_bn.final_bn = True + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class ResBottleneckBlock(CNNBlockBase): + """Residual bottleneck block: x + f(x), f = bottleneck transform.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, params): + super().__init__(w_in, w_out, stride) + self.proj, self.bn = None, None + if (w_in != w_out) or (stride != 1): + self.proj = conv2d(w_in, w_out, 1, stride=stride) + self.bn = get_norm(norm, w_out) + self.f = BottleneckTransform(w_in, w_out, stride, norm, activation_class, params) + self.af = activation_class() + + def forward(self, x): + x_p = self.bn(self.proj(x)) if self.proj else x + return self.af(x_p + self.f(x)) + + +class AnyStage(nn.Module): + """AnyNet stage (sequence of blocks w/ the same output shape).""" + + def __init__(self, w_in, w_out, stride, d, block_class, norm, activation_class, params): + super().__init__() + for i in range(d): + block = block_class(w_in, w_out, stride, norm, activation_class, params) + self.add_module("b{}".format(i + 1), block) + stride, w_in = 1, w_out + + def forward(self, x): + for block in self.children(): + x = block(x) + return x + + +class AnyNet(Backbone): + """AnyNet model. See :paper:`dds`.""" + + def __init__( + self, + *, + stem_class, + stem_width, + block_class, + depths, + widths, + group_widths, + strides, + bottleneck_ratios, + se_ratio, + activation_class, + freeze_at=0, + norm="BN", + out_features=None, + ): + """ + Args: + stem_class (callable): A callable taking 4 arguments (channels in, channels out, + normalization, callable returning an activation function) that returns another + callable implementing the stem module. + stem_width (int): The number of output channels that the stem produces. + block_class (callable): A callable taking 6 arguments (channels in, channels out, + stride, normalization, callable returning an activation function, a dict of + block-specific parameters) that returns another callable implementing the repeated + block module. + depths (list[int]): Number of blocks in each stage. + widths (list[int]): For each stage, the number of output channels of each block. + group_widths (list[int]): For each stage, the number of channels per group in group + convolution, if the block uses group convolution. + strides (list[int]): The stride that each network stage applies to its input. + bottleneck_ratios (list[float]): For each stage, the ratio of the number of bottleneck + channels to the number of block input channels (or, equivalently, output channels), + if the block uses a bottleneck. + se_ratio (float): The ratio of the number of channels used inside the squeeze-excitation + (SE) module to it number of input channels, if SE the block uses SE. + activation_class (callable): A callable taking no arguments that returns another + callable implementing an activation function. + freeze_at (int): The number of stages at the beginning to freeze. + see :meth:`freeze` for detailed explanation. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + out_features (list[str]): name of the layers whose outputs should + be returned in forward. RegNet's use "stem" and "s1", "s2", etc for the stages after + the stem. If None, will return the output of the last layer. + """ + super().__init__() + self.stem = stem_class(3, stem_width, norm, activation_class) + + current_stride = self.stem.stride + self._out_feature_strides = {"stem": current_stride} + self._out_feature_channels = {"stem": self.stem.out_channels} + self.stages_and_names = [] + prev_w = stem_width + + for i, (d, w, s, b, g) in enumerate(zip(depths, widths, strides, bottleneck_ratios, group_widths)): + params = {"bot_mul": b, "group_w": g, "se_r": se_ratio} + stage = AnyStage(prev_w, w, s, d, block_class, norm, activation_class, params) + name = "s{}".format(i + 1) + self.add_module(name, stage) + self.stages_and_names.append((stage, name)) + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in stage.children()]) + ) + self._out_feature_channels[name] = list(stage.children())[-1].out_channels + prev_w = w + + self.apply(init_weights) + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, "Available children: {} does not include {}".format( + ", ".join(children), out_feature + ) + self.freeze(freeze_at) + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert x.dim() == 4, f"Model takes an input of shape (N, C, H, W). Got {x.shape} instead!" + outputs = {} + x = self.stem(x) + if "stem" in self._out_features: + outputs["stem"] = x + for stage, name in self.stages_and_names: + x = stage(x) + if name in self._out_features: + outputs[name] = x + return outputs + + def output_shape(self): + return { + name: ShapeSpec(channels=self._out_feature_channels[name], stride=self._out_feature_strides[name]) + for name in self._out_features + } + + def freeze(self, freeze_at=0): + """ + Freeze the first several stages of the model. Commonly used in fine-tuning. + + Layers that produce the same feature map spatial size are defined as one + "stage" by :paper:`FPN`. + + Args: + freeze_at (int): number of stages to freeze. + `1` means freezing the stem. `2` means freezing the stem and + one residual stage, etc. + + Returns: + nn.Module: this model itself + """ + if freeze_at >= 1: + self.stem.freeze() + for idx, (stage, _) in enumerate(self.stages_and_names, start=2): + if freeze_at >= idx: + for block in stage.children(): + block.freeze() + return self + + +def adjust_block_compatibility(ws, bs, gs): + """Adjusts the compatibility of widths, bottlenecks, and groups.""" + assert len(ws) == len(bs) == len(gs) + assert all(w > 0 and b > 0 and g > 0 for w, b, g in zip(ws, bs, gs)) + vs = [int(max(1, w * b)) for w, b in zip(ws, bs)] + gs = [int(min(g, v)) for g, v in zip(gs, vs)] + ms = [np.lcm(g, b) if b > 1 else g for g, b in zip(gs, bs)] + vs = [max(m, int(round(v / m) * m)) for v, m in zip(vs, ms)] + ws = [int(v / b) for v, b in zip(vs, bs)] + assert all(w * b % g == 0 for w, b, g in zip(ws, bs, gs)) + return ws, bs, gs + + +def generate_regnet_parameters(w_a, w_0, w_m, d, q=8): + """Generates per stage widths and depths from RegNet parameters.""" + assert w_a >= 0 and w_0 > 0 and w_m > 1 and w_0 % q == 0 + # Generate continuous per-block ws + ws_cont = np.arange(d) * w_a + w_0 + # Generate quantized per-block ws + ks = np.round(np.log(ws_cont / w_0) / np.log(w_m)) + ws_all = w_0 * np.power(w_m, ks) + ws_all = np.round(np.divide(ws_all, q)).astype(int) * q + # Generate per stage ws and ds (assumes ws_all are sorted) + ws, ds = np.unique(ws_all, return_counts=True) + # Compute number of actual stages and total possible stages + num_stages, total_stages = len(ws), ks.max() + 1 + # Convert numpy arrays to lists and return + ws, ds, ws_all, ws_cont = (x.tolist() for x in (ws, ds, ws_all, ws_cont)) + return ws, ds, num_stages, total_stages, ws_all, ws_cont + + +class RegNet(AnyNet): + """RegNet model. See :paper:`dds`.""" + + def __init__( + self, + *, + stem_class, + stem_width, + block_class, + depth, + w_a, + w_0, + w_m, + group_width, + stride=2, + bottleneck_ratio=1.0, + se_ratio=0.0, + activation_class=None, + freeze_at=0, + norm="BN", + out_features=None, + ): + """ + Build a RegNet from the parameterization described in :paper:`dds` Section 3.3. + + Args: + See :class:`AnyNet` for arguments that are not listed here. + depth (int): Total number of blocks in the RegNet. + w_a (float): Factor by which block width would increase prior to quantizing block widths + by stage. See :paper:`dds` Section 3.3. + w_0 (int): Initial block width. See :paper:`dds` Section 3.3. + w_m (float): Parameter controlling block width quantization. + See :paper:`dds` Section 3.3. + group_width (int): Number of channels per group in group convolution, if the block uses + group convolution. + bottleneck_ratio (float): The ratio of the number of bottleneck channels to the number + of block input channels (or, equivalently, output channels), if the block uses a + bottleneck. + stride (int): The stride that each network stage applies to its input. + """ + ws, ds = generate_regnet_parameters(w_a, w_0, w_m, depth)[0:2] + ss = [stride for _ in ws] + bs = [bottleneck_ratio for _ in ws] + gs = [group_width for _ in ws] + ws, bs, gs = adjust_block_compatibility(ws, bs, gs) + + def default_activation_class(): + return nn.ReLU(inplace=True) + + super().__init__( + stem_class=stem_class, + stem_width=stem_width, + block_class=block_class, + depths=ds, + widths=ws, + strides=ss, + group_widths=gs, + bottleneck_ratios=bs, + se_ratio=se_ratio, + activation_class=default_activation_class if activation_class is None else activation_class, + freeze_at=freeze_at, + norm=norm, + out_features=out_features, + ) diff --git a/detectron2/modeling/backbone/resnet.py b/detectron2/modeling/backbone/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..decc9b57b0dbbfdb73e6081adb1b27bc99b38866 --- /dev/null +++ b/detectron2/modeling/backbone/resnet.py @@ -0,0 +1,685 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import fvcore.nn.weight_init as weight_init +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from detectron2.layers import ( + CNNBlockBase, + Conv2d, + DeformConv, + ModulatedDeformConv, + ShapeSpec, + get_norm, +) + +from .backbone import Backbone +from .build import BACKBONE_REGISTRY + +__all__ = [ + "ResNetBlockBase", + "BasicBlock", + "BottleneckBlock", + "DeformBottleneckBlock", + "BasicStem", + "ResNet", + "make_stage", + "build_resnet_backbone", +] + + +class BasicBlock(CNNBlockBase): + """ + The basic residual block for ResNet-18 and ResNet-34 defined in :paper:`ResNet`, + with two 3x3 conv layers and a projection shortcut if needed. + """ + + def __init__(self, in_channels, out_channels, *, stride=1, norm="BN"): + """ + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + stride (int): Stride for the first conv. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + self.conv2 = Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + out = self.conv2(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class BottleneckBlock(CNNBlockBase): + """ + The standard bottleneck residual block used by ResNet-50, 101 and 152 + defined in :paper:`ResNet`. It contains 3 conv layers with kernels + 1x1, 3x3, 1x1, and a projection shortcut if needed. + """ + + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + ): + """ + Args: + bottleneck_channels (int): number of output channels for the 3x3 + "bottleneck" conv layers. + num_groups (int): number of groups for the 3x3 conv layer. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + stride_in_1x1 (bool): when stride>1, whether to put stride in the + first 1x1 convolution or the bottleneck 3x3 convolution. + dilation (int): the dilation rate of the 3x3 conv layer. + """ + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + # The original MSRA ResNet models have stride in the first 1x1 conv + # The subsequent fb.torch.resnet and Caffe2 ResNe[X]t implementations have + # stride in the 3x3 conv + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.conv3, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + # Zero-initialize the last normalization in each residual branch, + # so that at the beginning, the residual branch starts with zeros, + # and each residual block behaves like an identity. + # See Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "For BN layers, the learnable scaling coefficient γ is initialized + # to be 1, except for each residual block's last BN + # where γ is initialized to be 0." + + # nn.init.constant_(self.conv3.norm.weight, 0) + # TODO this somehow hurts performance when training GN models from scratch. + # Add it as an option when we need to use this code to train a backbone. + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + out = self.conv2(out) + out = F.relu_(out) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class DeformBottleneckBlock(CNNBlockBase): + """ + Similar to :class:`BottleneckBlock`, but with :paper:`deformable conv ` + in the 3x3 convolution. + """ + + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + deform_modulated=False, + deform_num_groups=1, + ): + super().__init__(in_channels, out_channels, stride) + self.deform_modulated = deform_modulated + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + if deform_modulated: + deform_conv_op = ModulatedDeformConv + # offset channels are 2 or 3 (if with modulated) * kernel_size * kernel_size + offset_channels = 27 + else: + deform_conv_op = DeformConv + offset_channels = 18 + + self.conv2_offset = Conv2d( + bottleneck_channels, + offset_channels * deform_num_groups, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + dilation=dilation, + ) + self.conv2 = deform_conv_op( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + deformable_groups=deform_num_groups, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.conv3, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + nn.init.constant_(self.conv2_offset.weight, 0) + nn.init.constant_(self.conv2_offset.bias, 0) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + if self.deform_modulated: + offset_mask = self.conv2_offset(out) + offset_x, offset_y, mask = torch.chunk(offset_mask, 3, dim=1) + offset = torch.cat((offset_x, offset_y), dim=1) + mask = mask.sigmoid() + out = self.conv2(out, offset, mask) + else: + offset = self.conv2_offset(out) + out = self.conv2(out, offset) + out = F.relu_(out) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class BasicStem(CNNBlockBase): + """ + The standard ResNet stem (layers before the first residual block), + with a conv, relu and max_pool. + """ + + def __init__(self, in_channels=3, out_channels=64, norm="BN"): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, 4) + self.in_channels = in_channels + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False, + norm=get_norm(norm, out_channels), + ) + weight_init.c2_msra_fill(self.conv1) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +class ResNet(Backbone): + """ + Implement :paper:`ResNet`. + """ + + def __init__(self, stem, stages, num_classes=None, out_features=None, freeze_at=0): + """ + Args: + stem (nn.Module): a stem module + stages (list[list[CNNBlockBase]]): several (typically 4) stages, + each contains multiple :class:`CNNBlockBase`. + num_classes (None or int): if None, will not perform classification. + Otherwise, will create a linear layer. + out_features (list[str]): name of the layers whose outputs should + be returned in forward. Can be anything in "stem", "linear", or "res2" ... + If None, will return the output of the last layer. + freeze_at (int): The number of stages at the beginning to freeze. + see :meth:`freeze` for detailed explanation. + """ + super().__init__() + self.stem = stem + self.num_classes = num_classes + + current_stride = self.stem.stride + self._out_feature_strides = {"stem": current_stride} + self._out_feature_channels = {"stem": self.stem.out_channels} + + self.stage_names, self.stages = [], [] + + if out_features is not None: + # Avoid keeping unused layers in this module. They consume extra memory + # and may cause allreduce to fail + num_stages = max([{"res2": 1, "res3": 2, "res4": 3, "res5": 4}.get(f, 0) for f in out_features]) + stages = stages[:num_stages] + for i, blocks in enumerate(stages): + assert len(blocks) > 0, len(blocks) + for block in blocks: + assert isinstance(block, CNNBlockBase), block + + name = "res" + str(i + 2) + stage = nn.Sequential(*blocks) + + self.add_module(name, stage) + self.stage_names.append(name) + self.stages.append(stage) + + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in blocks]) + ) + self._out_feature_channels[name] = curr_channels = blocks[-1].out_channels + self.stage_names = tuple(self.stage_names) # Make it static for scripting + + if num_classes is not None: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.linear = nn.Linear(curr_channels, num_classes) + + # Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "The 1000-way fully-connected layer is initialized by + # drawing weights from a zero-mean Gaussian with standard deviation of 0.01." + nn.init.normal_(self.linear.weight, std=0.01) + name = "linear" + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, "Available children: {}".format(", ".join(children)) + self.freeze(freeze_at) + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert x.dim() == 4, f"ResNet takes an input of shape (N, C, H, W). Got {x.shape} instead!" + outputs = {} + x = self.stem(x) + if "stem" in self._out_features: + outputs["stem"] = x + for name, stage in zip(self.stage_names, self.stages): + x = stage(x) + if name in self._out_features: + outputs[name] = x + if self.num_classes is not None: + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.linear(x) + if "linear" in self._out_features: + outputs["linear"] = x + return outputs + + def output_shape(self): + return { + name: ShapeSpec(channels=self._out_feature_channels[name], stride=self._out_feature_strides[name]) + for name in self._out_features + } + + def freeze(self, freeze_at=0): + """ + Freeze the first several stages of the ResNet. Commonly used in + fine-tuning. + + Layers that produce the same feature map spatial size are defined as one + "stage" by :paper:`FPN`. + + Args: + freeze_at (int): number of stages to freeze. + `1` means freezing the stem. `2` means freezing the stem and + one residual stage, etc. + + Returns: + nn.Module: this ResNet itself + """ + if freeze_at >= 1: + self.stem.freeze() + for idx, stage in enumerate(self.stages, start=2): + if freeze_at >= idx: + for block in stage.children(): + block.freeze() + return self + + @staticmethod + def make_stage(block_class, num_blocks, *, in_channels, out_channels, **kwargs): + """ + Create a list of blocks of the same type that forms one ResNet stage. + + Args: + block_class (type): a subclass of CNNBlockBase that's used to create all blocks in this + stage. A module of this type must not change spatial resolution of inputs unless its + stride != 1. + num_blocks (int): number of blocks in this stage + in_channels (int): input channels of the entire stage. + out_channels (int): output channels of **every block** in the stage. + kwargs: other arguments passed to the constructor of + `block_class`. If the argument name is "xx_per_block", the + argument is a list of values to be passed to each block in the + stage. Otherwise, the same argument is passed to every block + in the stage. + + Returns: + list[CNNBlockBase]: a list of block module. + + Examples: + :: + stage = ResNet.make_stage( + BottleneckBlock, 3, in_channels=16, out_channels=64, + bottleneck_channels=16, num_groups=1, + stride_per_block=[2, 1, 1], + dilations_per_block=[1, 1, 2] + ) + + Usually, layers that produce the same feature map spatial size are defined as one + "stage" (in :paper:`FPN`). Under such definition, ``stride_per_block[1:]`` should + all be 1. + """ + blocks = [] + for i in range(num_blocks): + curr_kwargs = {} + for k, v in kwargs.items(): + if k.endswith("_per_block"): + assert len(v) == num_blocks, ( + f"Argument '{k}' of make_stage should have the " f"same length as num_blocks={num_blocks}." + ) + newk = k[: -len("_per_block")] + assert newk not in kwargs, f"Cannot call make_stage with both {k} and {newk}!" + curr_kwargs[newk] = v[i] + else: + curr_kwargs[k] = v + + blocks.append(block_class(in_channels=in_channels, out_channels=out_channels, **curr_kwargs)) + in_channels = out_channels + return blocks + + @staticmethod + def make_default_stages(depth, block_class=None, **kwargs): + """ + Created list of ResNet stages from pre-defined depth (one of 18, 34, 50, 101, 152). + If it doesn't create the ResNet variant you need, please use :meth:`make_stage` + instead for fine-grained customization. + + Args: + depth (int): depth of ResNet + block_class (type): the CNN block class. Has to accept + `bottleneck_channels` argument for depth > 50. + By default it is BasicBlock or BottleneckBlock, based on the + depth. + kwargs: + other arguments to pass to `make_stage`. Should not contain + stride and channels, as they are predefined for each depth. + + Returns: + list[list[CNNBlockBase]]: modules in all stages; see arguments of + :class:`ResNet.__init__`. + """ + num_blocks_per_stage = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + }[depth] + if block_class is None: + block_class = BasicBlock if depth < 50 else BottleneckBlock + if depth < 50: + in_channels = [64, 64, 128, 256] + out_channels = [64, 128, 256, 512] + else: + in_channels = [64, 256, 512, 1024] + out_channels = [256, 512, 1024, 2048] + ret = [] + for n, s, i, o in zip(num_blocks_per_stage, [1, 2, 2, 2], in_channels, out_channels): + if depth >= 50: + kwargs["bottleneck_channels"] = o // 4 + ret.append( + ResNet.make_stage( + block_class=block_class, + num_blocks=n, + stride_per_block=[s] + [1] * (n - 1), + in_channels=i, + out_channels=o, + **kwargs, + ) + ) + return ret + + +ResNetBlockBase = CNNBlockBase +""" +Alias for backward compatibiltiy. +""" + + +def make_stage(*args, **kwargs): + """ + Deprecated alias for backward compatibiltiy. + """ + return ResNet.make_stage(*args, **kwargs) + + +@BACKBONE_REGISTRY.register() +def build_resnet_backbone(cfg, input_shape): + """ + Create a ResNet instance from config. + + Returns: + ResNet: a :class:`ResNet` instance. + """ + # need registration of new blocks/stems? + norm = cfg.MODEL.RESNETS.NORM + stem = BasicStem( + in_channels=input_shape.channels, + out_channels=cfg.MODEL.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + ) + + # fmt: off + freeze_at = cfg.MODEL.BACKBONE.FREEZE_AT + out_features = cfg.MODEL.RESNETS.OUT_FEATURES + depth = cfg.MODEL.RESNETS.DEPTH + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group + in_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + res5_dilation = cfg.MODEL.RESNETS.RES5_DILATION + deform_on_per_stage = cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE + deform_modulated = cfg.MODEL.RESNETS.DEFORM_MODULATED + deform_num_groups = cfg.MODEL.RESNETS.DEFORM_NUM_GROUPS + # fmt: on + assert res5_dilation in {1, 2}, "res5_dilation cannot be {}.".format(res5_dilation) + + num_blocks_per_stage = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + }[depth] + + if depth in [18, 34]: + assert out_channels == 64, "Must set MODEL.RESNETS.RES2_OUT_CHANNELS = 64 for R18/R34" + assert not any(deform_on_per_stage), "MODEL.RESNETS.DEFORM_ON_PER_STAGE unsupported for R18/R34" + assert res5_dilation == 1, "Must set MODEL.RESNETS.RES5_DILATION = 1 for R18/R34" + assert num_groups == 1, "Must set MODEL.RESNETS.NUM_GROUPS = 1 for R18/R34" + + stages = [] + + for idx, stage_idx in enumerate(range(2, 6)): + # res5_dilation is used this way as a convention in R-FCN & Deformable Conv paper + dilation = res5_dilation if stage_idx == 5 else 1 + first_stride = 1 if idx == 0 or (stage_idx == 5 and dilation == 2) else 2 + stage_kargs = { + "num_blocks": num_blocks_per_stage[idx], + "stride_per_block": [first_stride] + [1] * (num_blocks_per_stage[idx] - 1), + "in_channels": in_channels, + "out_channels": out_channels, + "norm": norm, + } + # Use BasicBlock for R18 and R34. + if depth in [18, 34]: + stage_kargs["block_class"] = BasicBlock + else: + stage_kargs["bottleneck_channels"] = bottleneck_channels + stage_kargs["stride_in_1x1"] = stride_in_1x1 + stage_kargs["dilation"] = dilation + stage_kargs["num_groups"] = num_groups + if deform_on_per_stage[idx]: + stage_kargs["block_class"] = DeformBottleneckBlock + stage_kargs["deform_modulated"] = deform_modulated + stage_kargs["deform_num_groups"] = deform_num_groups + else: + stage_kargs["block_class"] = BottleneckBlock + blocks = ResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return ResNet(stem, stages, out_features=out_features, freeze_at=freeze_at) diff --git a/detectron2/modeling/backbone/swin.py b/detectron2/modeling/backbone/swin.py new file mode 100644 index 0000000000000000000000000000000000000000..0b5606dcc3d8b4b98ee0f6d28aff2ee94f76f213 --- /dev/null +++ b/detectron2/modeling/backbone/swin.py @@ -0,0 +1,663 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Implementation of Swin models from :paper:`swin`. + +This code is adapted from https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/master/mmdet/models/backbones/swin_transformer.py with minimal modifications. # noqa +-------------------------------------------------------- +Swin Transformer +Copyright (c) 2021 Microsoft +Licensed under The MIT License [see LICENSE for details] +Written by Ze Liu, Yutong Lin, Yixuan Wei +-------------------------------------------------------- +LICENSE: https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/461e003166a8083d0b620beacd4662a2df306bd6/LICENSE +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from detectron2.modeling.backbone.backbone import Backbone + + +class Mlp(nn.Module): + """Multilayer perceptron.""" + + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. + Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) + ) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=0.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """Swin Transformer Block. + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__( + self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """Patch Merging Layer + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. + Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + ): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0)) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(Backbone): + """Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted + Windows` - https://arxiv.org/pdf/2103.14030 + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=(2, 2, 6, 2), + num_heads=(3, 6, 12, 24), + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False, + ): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + ) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1], + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]) + ) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + ) + self.layers.append(layer) + + num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f"norm{i_layer}" + self.add_module(layer_name, layer) + + self._freeze_stages() + self._out_features = ["p{}".format(i) for i in self.out_indices] + self._out_feature_channels = {"p{}".format(i): self.embed_dim * 2**i for i in self.out_indices} + self._out_feature_strides = {"p{}".format(i): 2 ** (i + 2) for i in self.out_indices} + self._size_devisibility = 32 + + self.apply(self._init_weights) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @property + def size_divisibility(self): + return self._size_divisibility + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic") + x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = {} + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f"norm{i}") + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs["p{}".format(i)] = out + + return outs diff --git a/detectron2/modeling/backbone/utils.py b/detectron2/modeling/backbone/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9ee6db9ff68fdd8a210236661e4040c2e38e78ca --- /dev/null +++ b/detectron2/modeling/backbone/utils.py @@ -0,0 +1,183 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = [ + "window_partition", + "window_unpartition", + "add_decomposed_rel_pos", + "get_abs_pos", + "PatchEmbed", +] + + +def window_partition(x, window_size): + """ + Partition into non-overlapping windows with padding if needed. + Args: + x (tensor): input tokens with [B, H, W, C]. + window_size (int): window size. + + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition(windows, window_size, pad_hw, hw): + """ + Window unpartition into original sequences and removing padding. + Args: + x (tensor): input tokens with [B * num_windows, window_size, window_size, C]. + window_size (int): window size. + pad_hw (Tuple): padded height and width (Hp, Wp). + hw (Tuple): original height and width (H, W) before padding. + + Returns: + x: unpartitioned sequences with [B, H, W, C]. + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :].contiguous() + return x + + +def get_rel_pos(q_size, k_size, rel_pos): + """ + Get relative positional embeddings according to the relative positions of + query and key sizes. + Args: + q_size (int): size of query q. + k_size (int): size of key k. + rel_pos (Tensor): relative position embeddings (L, C). + + Returns: + Extracted positional embeddings according to relative positions. + """ + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos if needed. + if rel_pos.shape[0] != max_rel_dist: + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), + size=max_rel_dist, + mode="linear", + ) + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + else: + rel_pos_resized = rel_pos + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + +def add_decomposed_rel_pos(attn, q, rel_pos_h, rel_pos_w, q_size, k_size): + """ + Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. + https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950 + Args: + attn (Tensor): attention map. + q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). + rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. + rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. + q_size (Tuple): spatial sequence size of query q with (q_h, q_w). + k_size (Tuple): spatial sequence size of key k with (k_h, k_w). + + Returns: + attn (Tensor): attention map with added relative positional embeddings. + """ + q_h, q_w = q_size + k_h, k_w = k_size + Rh = get_rel_pos(q_h, k_h, rel_pos_h) + Rw = get_rel_pos(q_w, k_w, rel_pos_w) + + B, _, dim = q.shape + r_q = q.reshape(B, q_h, q_w, dim) + rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) + rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) + + attn = (attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :]).view( + B, q_h * q_w, k_h * k_w + ) + + return attn + + +def get_abs_pos(abs_pos, has_cls_token, hw): + """ + Calculate absolute positional embeddings. If needed, resize embeddings and remove cls_token + dimension for the original embeddings. + Args: + abs_pos (Tensor): absolute positional embeddings with (1, num_position, C). + has_cls_token (bool): If true, has 1 embedding in abs_pos for cls token. + hw (Tuple): size of input image tokens. + + Returns: + Absolute positional embeddings after processing with shape (1, H, W, C) + """ + h, w = hw + if has_cls_token: + abs_pos = abs_pos[:, 1:] + xy_num = abs_pos.shape[1] + size = int(math.sqrt(xy_num)) + assert size * size == xy_num + + if size != h or size != w: + new_abs_pos = F.interpolate( + abs_pos.reshape(1, size, size, -1).permute(0, 3, 1, 2), + size=(h, w), + mode="bicubic", + align_corners=False, + ) + + return new_abs_pos.permute(0, 2, 3, 1) + else: + return abs_pos.reshape(1, h, w, -1) + + +class PatchEmbed(nn.Module): + """ + Image to Patch Embedding. + """ + + def __init__(self, kernel_size=(16, 16), stride=(16, 16), padding=(0, 0), in_chans=3, embed_dim=768): + """ + Args: + kernel_size (Tuple): kernel size of the projection layer. + stride (Tuple): stride of the projection layer. + padding (Tuple): padding size of the projection layer. + in_chans (int): Number of input image channels. + embed_dim (int): embed_dim (int): Patch embedding dimension. + """ + super().__init__() + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding) + + def forward(self, x): + x = self.proj(x) + # B C H W -> B H W C + x = x.permute(0, 2, 3, 1) + return x diff --git a/detectron2/modeling/backbone/vit.py b/detectron2/modeling/backbone/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..4908717eabd5b400c149cd953eb80bef07f6cbb5 --- /dev/null +++ b/detectron2/modeling/backbone/vit.py @@ -0,0 +1,520 @@ +import logging +import math + +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn as nn +from fairscale.nn.checkpoint import checkpoint_wrapper +from timm.models.layers import DropPath, Mlp, trunc_normal_ + +from detectron2.layers import CNNBlockBase, Conv2d, get_norm +from detectron2.modeling.backbone.fpn import _assert_strides_are_log2_contiguous + +from .backbone import Backbone +from .utils import ( + PatchEmbed, + add_decomposed_rel_pos, + get_abs_pos, + window_partition, + window_unpartition, +) + +logger = logging.getLogger(__name__) + + +__all__ = ["ViT", "SimpleFeaturePyramid", "get_vit_lr_decay_rate"] + + +class Attention(nn.Module): + """Multi-head Attention block with relative position embeddings.""" + + def __init__( + self, + dim, + num_heads=8, + qkv_bias=True, + use_rel_pos=False, + rel_pos_zero_init=True, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + qkv_bias (bool: If True, add a learnable bias to query, key, value. + rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (int or None): Input resolution for calculating the relative positional + parameter size. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.proj = nn.Linear(dim, dim) + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + # initialize relative positional embeddings + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) + + if not rel_pos_zero_init: + trunc_normal_(self.rel_pos_h, std=0.02) + trunc_normal_(self.rel_pos_w, std=0.02) + + def forward(self, x): + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H * W, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + # q, k, v with shape (B * nHead, H * W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) + + attn = attn.softmax(dim=-1) + x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +class ResBottleneckBlock(CNNBlockBase): + """ + The standard bottleneck residual block without the last activation layer. + It contains 3 conv layers with kernels 1x1, 3x3, 1x1. + """ + + def __init__( + self, + in_channels, + out_channels, + bottleneck_channels, + norm="LN", + act_layer=nn.GELU, + ): + """ + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + bottleneck_channels (int): number of output channels for the 3x3 + "bottleneck" conv layers. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + act_layer (callable): activation for all conv layers. + """ + super().__init__(in_channels, out_channels, 1) + + self.conv1 = Conv2d(in_channels, bottleneck_channels, 1, bias=False) + self.norm1 = get_norm(norm, bottleneck_channels) + self.act1 = act_layer() + + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + 3, + padding=1, + bias=False, + ) + self.norm2 = get_norm(norm, bottleneck_channels) + self.act2 = act_layer() + + self.conv3 = Conv2d(bottleneck_channels, out_channels, 1, bias=False) + self.norm3 = get_norm(norm, out_channels) + + for layer in [self.conv1, self.conv2, self.conv3]: + weight_init.c2_msra_fill(layer) + for layer in [self.norm1, self.norm2]: + layer.weight.data.fill_(1.0) + layer.bias.data.zero_() + # zero init last norm layer. + self.norm3.weight.data.zero_() + self.norm3.bias.data.zero_() + + def forward(self, x): + out = x + for layer in self.children(): + out = layer(out) + + out = x + out + return out + + +class Block(nn.Module): + """Transformer blocks with support of window attention and residual propagation blocks""" + + def __init__( + self, + dim, + num_heads, + mlp_ratio=4.0, + qkv_bias=True, + drop_path=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + use_rel_pos=False, + rel_pos_zero_init=True, + window_size=0, + use_residual_block=False, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. If it equals 0, then not + use window attention. + use_residual_block (bool): If True, use a residual block after the MLP block. + input_size (int or None): Input resolution for calculating the relative positional + parameter size. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size if window_size == 0 else (window_size, window_size), + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), act_layer=act_layer) + + self.window_size = window_size + + self.use_residual_block = use_residual_block + if use_residual_block: + # Use a residual block with bottleneck channel as dim // 2 + self.residual = ResBottleneckBlock( + in_channels=dim, + out_channels=dim, + bottleneck_channels=dim // 2, + norm="LN", + act_layer=act_layer, + ) + + def forward(self, x): + shortcut = x + x = self.norm1(x) + # Window partition + if self.window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, self.window_size) + + x = self.attn(x) + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + if self.use_residual_block: + x = self.residual(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) + + return x + + +class ViT(Backbone): + """ + This module implements Vision Transformer (ViT) backbone in :paper:`vitdet`. + "Exploring Plain Vision Transformer Backbones for Object Detection", + https://arxiv.org/abs/2203.16527 + """ + + def __init__( + self, + img_size=1024, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + drop_path_rate=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + use_abs_pos=True, + use_rel_pos=False, + rel_pos_zero_init=True, + window_size=0, + window_block_indexes=(), + residual_block_indexes=(), + use_act_checkpoint=False, + pretrain_img_size=224, + pretrain_use_cls_token=True, + out_feature="last_feat", + ): + """ + Args: + img_size (int): Input image size. + patch_size (int): Patch size. + in_chans (int): Number of input image channels. + embed_dim (int): Patch embedding dimension. + depth (int): Depth of ViT. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path_rate (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_abs_pos (bool): If True, use absolute positional embeddings. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. + window_block_indexes (list): Indexes for blocks using window attention. + residual_block_indexes (list): Indexes for blocks using conv propagation. + use_act_checkpoint (bool): If True, use activation checkpointing. + pretrain_img_size (int): input image size for pretraining models. + pretrain_use_cls_token (bool): If True, pretrainig models use class token. + out_feature (str): name of the feature from the last block. + """ + super().__init__() + self.pretrain_use_cls_token = pretrain_use_cls_token + + self.patch_embed = PatchEmbed( + kernel_size=(patch_size, patch_size), + stride=(patch_size, patch_size), + in_chans=in_chans, + embed_dim=embed_dim, + ) + + if use_abs_pos: + # Initialize absolute positional embedding with pretrain image size. + num_patches = (pretrain_img_size // patch_size) * (pretrain_img_size // patch_size) + num_positions = (num_patches + 1) if pretrain_use_cls_token else num_patches + self.pos_embed = nn.Parameter(torch.zeros(1, num_positions, embed_dim)) + else: + self.pos_embed = None + + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + window_size=window_size if i in window_block_indexes else 0, + use_residual_block=i in residual_block_indexes, + input_size=(img_size // patch_size, img_size // patch_size), + ) + if use_act_checkpoint: + block = checkpoint_wrapper(block) + self.blocks.append(block) + + self._out_feature_channels = {out_feature: embed_dim} + self._out_feature_strides = {out_feature: patch_size} + self._out_features = [out_feature] + + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=0.02) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, x): + x = self.patch_embed(x) + if self.pos_embed is not None: + x = x + get_abs_pos(self.pos_embed, self.pretrain_use_cls_token, (x.shape[1], x.shape[2])) + + for blk in self.blocks: + x = blk(x) + + outputs = {self._out_features[0]: x.permute(0, 3, 1, 2)} + return outputs + + +class SimpleFeaturePyramid(Backbone): + """ + This module implements SimpleFeaturePyramid in :paper:`vitdet`. + It creates pyramid features built on top of the input feature map. + """ + + def __init__( + self, + net, + in_feature, + out_channels, + scale_factors, + top_block=None, + norm="LN", + square_pad=0, + ): + """ + Args: + net (Backbone): module representing the subnetwork backbone. + Must be a subclass of :class:`Backbone`. + in_feature (str): names of the input feature maps coming + from the net. + out_channels (int): number of channels in the output feature maps. + scale_factors (list[float]): list of scaling factors to upsample or downsample + the input features for creating pyramid features. + top_block (nn.Module or None): if provided, an extra operation will + be performed on the output of the last (smallest resolution) + pyramid output, and the result will extend the result list. The top_block + further downsamples the feature map. It must have an attribute + "num_levels", meaning the number of extra pyramid levels added by + this block, and "in_feature", which is a string representing + its input feature (e.g., p5). + norm (str): the normalization to use. + square_pad (int): If > 0, require input images to be padded to specific square size. + """ + super(SimpleFeaturePyramid, self).__init__() + assert isinstance(net, Backbone) + + self.scale_factors = scale_factors + + input_shapes = net.output_shape() + strides = [int(input_shapes[in_feature].stride / scale) for scale in scale_factors] + _assert_strides_are_log2_contiguous(strides) + + dim = input_shapes[in_feature].channels + self.stages = [] + use_bias = norm == "" + for idx, scale in enumerate(scale_factors): + out_dim = dim + if scale == 4.0: + layers = [ + nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2), + get_norm(norm, dim // 2), + nn.GELU(), + nn.ConvTranspose2d(dim // 2, dim // 4, kernel_size=2, stride=2), + ] + out_dim = dim // 4 + elif scale == 2.0: + layers = [nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2)] + out_dim = dim // 2 + elif scale == 1.0: + layers = [] + elif scale == 0.5: + layers = [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + raise NotImplementedError(f"scale_factor={scale} is not supported yet.") + + layers.extend( + [ + Conv2d( + out_dim, + out_channels, + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + ), + Conv2d( + out_channels, + out_channels, + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + ), + ] + ) + layers = nn.Sequential(*layers) + + stage = int(math.log2(strides[idx])) + self.add_module(f"simfp_{stage}", layers) + self.stages.append(layers) + + self.net = net + self.in_feature = in_feature + self.top_block = top_block + # Return feature names are "p", like ["p2", "p3", ..., "p6"] + self._out_feature_strides = {"p{}".format(int(math.log2(s))): s for s in strides} + # top block output feature maps. + if self.top_block is not None: + for s in range(stage, stage + self.top_block.num_levels): + self._out_feature_strides["p{}".format(s + 1)] = 2 ** (s + 1) + + self._out_features = list(self._out_feature_strides.keys()) + self._out_feature_channels = {k: out_channels for k in self._out_features} + self._size_divisibility = strides[-1] + self._square_pad = square_pad + + @property + def padding_constraints(self): + return { + "size_divisiblity": self._size_divisibility, + "square_size": self._square_pad, + } + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + + Returns: + dict[str->Tensor]: + mapping from feature map name to pyramid feature map tensor + in high to low resolution order. Returned feature names follow the FPN + convention: "p", where stage has stride = 2 ** stage e.g., + ["p2", "p3", ..., "p6"]. + """ + bottom_up_features = self.net(x) + features = bottom_up_features[self.in_feature] + results = [] + + for stage in self.stages: + results.append(stage(features)) + + if self.top_block is not None: + if self.top_block.in_feature in bottom_up_features: + top_block_in_feature = bottom_up_features[self.top_block.in_feature] + else: + top_block_in_feature = results[self._out_features.index(self.top_block.in_feature)] + results.extend(self.top_block(top_block_in_feature)) + assert len(self._out_features) == len(results) + return {f: res for f, res in zip(self._out_features, results)} + + +def get_vit_lr_decay_rate(name, lr_decay_rate=1.0, num_layers=12): + """ + Calculate lr decay rate for different ViT blocks. + Args: + name (string): parameter name. + lr_decay_rate (float): base lr decay rate. + num_layers (int): number of ViT blocks. + + Returns: + lr decay rate for the given parameter. + """ + layer_id = num_layers + 1 + if name.startswith("backbone"): + if ".pos_embed" in name or ".patch_embed" in name: + layer_id = 0 + elif ".blocks." in name and ".residual." not in name: + layer_id = int(name[name.find(".blocks.") :].split(".")[2]) + 1 + + return lr_decay_rate ** (num_layers + 1 - layer_id) diff --git a/detectron2/modeling/box_regression.py b/detectron2/modeling/box_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..0b592158fbdc294008155c915278356515d2b799 --- /dev/null +++ b/detectron2/modeling/box_regression.py @@ -0,0 +1,352 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List, Tuple, Union + +import torch +from fvcore.nn import giou_loss, smooth_l1_loss +from torch.nn import functional as F + +from detectron2.layers import cat, ciou_loss, diou_loss +from detectron2.structures import Boxes + +# Value for clamping large dw and dh predictions. The heuristic is that we clamp +# such that dw and dh are no larger than what would transform a 16px box into a +# 1000px box (based on a small anchor, 16px, and a typical image size, 1000px). +_DEFAULT_SCALE_CLAMP = math.log(1000.0 / 16) + + +__all__ = ["Box2BoxTransform", "Box2BoxTransformRotated", "Box2BoxTransformLinear"] + + +@torch.jit.script +class Box2BoxTransform(object): + """ + The box-to-box transform defined in R-CNN. The transformation is parameterized + by 4 deltas: (dx, dy, dw, dh). The transformation scales the box's width and height + by exp(dw), exp(dh) and shifts a box's center by the offset (dx * width, dy * height). + """ + + def __init__(self, weights: Tuple[float, float, float, float], scale_clamp: float = _DEFAULT_SCALE_CLAMP): + """ + Args: + weights (4-element tuple): Scaling factors that are applied to the + (dx, dy, dw, dh) deltas. In Fast R-CNN, these were originally set + such that the deltas have unit variance; now they are treated as + hyperparameters of the system. + scale_clamp (float): When predicting deltas, the predicted box scaling + factors (dw and dh) are clamped such that they are <= scale_clamp. + """ + self.weights = weights + self.scale_clamp = scale_clamp + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx, dy, dw, dh) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true (unless + any delta is too large and is clamped). + + Args: + src_boxes (Tensor): source boxes, e.g., object proposals + target_boxes (Tensor): target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_widths = src_boxes[:, 2] - src_boxes[:, 0] + src_heights = src_boxes[:, 3] - src_boxes[:, 1] + src_ctr_x = src_boxes[:, 0] + 0.5 * src_widths + src_ctr_y = src_boxes[:, 1] + 0.5 * src_heights + + target_widths = target_boxes[:, 2] - target_boxes[:, 0] + target_heights = target_boxes[:, 3] - target_boxes[:, 1] + target_ctr_x = target_boxes[:, 0] + 0.5 * target_widths + target_ctr_y = target_boxes[:, 1] + 0.5 * target_heights + + wx, wy, ww, wh = self.weights + dx = wx * (target_ctr_x - src_ctr_x) / src_widths + dy = wy * (target_ctr_y - src_ctr_y) / src_heights + dw = ww * torch.log(target_widths / src_widths) + dh = wh * torch.log(target_heights / src_heights) + + deltas = torch.stack((dx, dy, dw, dh), dim=1) + assert (src_widths > 0).all().item(), "Input boxes to Box2BoxTransform are not valid!" + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx, dy, dw, dh) to `boxes`. + + Args: + deltas (Tensor): transformation deltas of shape (N, k*4), where k >= 1. + deltas[i] represents k potentially different class-specific + box transformations for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 4) + """ + deltas = deltas.float() # ensure fp32 for decoding precision + boxes = boxes.to(deltas.dtype) + + widths = boxes[:, 2] - boxes[:, 0] + heights = boxes[:, 3] - boxes[:, 1] + ctr_x = boxes[:, 0] + 0.5 * widths + ctr_y = boxes[:, 1] + 0.5 * heights + + wx, wy, ww, wh = self.weights + dx = deltas[:, 0::4] / wx + dy = deltas[:, 1::4] / wy + dw = deltas[:, 2::4] / ww + dh = deltas[:, 3::4] / wh + + # Prevent sending too large values into torch.exp() + dw = torch.clamp(dw, max=self.scale_clamp) + dh = torch.clamp(dh, max=self.scale_clamp) + + pred_ctr_x = dx * widths[:, None] + ctr_x[:, None] + pred_ctr_y = dy * heights[:, None] + ctr_y[:, None] + pred_w = torch.exp(dw) * widths[:, None] + pred_h = torch.exp(dh) * heights[:, None] + + x1 = pred_ctr_x - 0.5 * pred_w + y1 = pred_ctr_y - 0.5 * pred_h + x2 = pred_ctr_x + 0.5 * pred_w + y2 = pred_ctr_y + 0.5 * pred_h + pred_boxes = torch.stack((x1, y1, x2, y2), dim=-1) + return pred_boxes.reshape(deltas.shape) + + +@torch.jit.script +class Box2BoxTransformRotated(object): + """ + The box-to-box transform defined in Rotated R-CNN. The transformation is parameterized + by 5 deltas: (dx, dy, dw, dh, da). The transformation scales the box's width and height + by exp(dw), exp(dh), shifts a box's center by the offset (dx * width, dy * height), + and rotate a box's angle by da (radians). + Note: angles of deltas are in radians while angles of boxes are in degrees. + """ + + def __init__( + self, + weights: Tuple[float, float, float, float, float], + scale_clamp: float = _DEFAULT_SCALE_CLAMP, + ): + """ + Args: + weights (5-element tuple): Scaling factors that are applied to the + (dx, dy, dw, dh, da) deltas. These are treated as + hyperparameters of the system. + scale_clamp (float): When predicting deltas, the predicted box scaling + factors (dw and dh) are clamped such that they are <= scale_clamp. + """ + self.weights = weights + self.scale_clamp = scale_clamp + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx, dy, dw, dh, da) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true (unless + any delta is too large and is clamped). + + Args: + src_boxes (Tensor): Nx5 source boxes, e.g., object proposals + target_boxes (Tensor): Nx5 target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_ctr_x, src_ctr_y, src_widths, src_heights, src_angles = torch.unbind(src_boxes, dim=1) + + target_ctr_x, target_ctr_y, target_widths, target_heights, target_angles = torch.unbind(target_boxes, dim=1) + + wx, wy, ww, wh, wa = self.weights + dx = wx * (target_ctr_x - src_ctr_x) / src_widths + dy = wy * (target_ctr_y - src_ctr_y) / src_heights + dw = ww * torch.log(target_widths / src_widths) + dh = wh * torch.log(target_heights / src_heights) + # Angles of deltas are in radians while angles of boxes are in degrees. + # the conversion to radians serve as a way to normalize the values + da = target_angles - src_angles + da = (da + 180.0) % 360.0 - 180.0 # make it in [-180, 180) + da *= wa * math.pi / 180.0 + + deltas = torch.stack((dx, dy, dw, dh, da), dim=1) + assert (src_widths > 0).all().item(), "Input boxes to Box2BoxTransformRotated are not valid!" + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx, dy, dw, dh, da) to `boxes`. + + Args: + deltas (Tensor): transformation deltas of shape (N, k*5). + deltas[i] represents box transformation for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 5) + """ + assert deltas.shape[1] % 5 == 0 and boxes.shape[1] == 5 + + boxes = boxes.to(deltas.dtype).unsqueeze(2) + + ctr_x = boxes[:, 0] + ctr_y = boxes[:, 1] + widths = boxes[:, 2] + heights = boxes[:, 3] + angles = boxes[:, 4] + + wx, wy, ww, wh, wa = self.weights + + dx = deltas[:, 0::5] / wx + dy = deltas[:, 1::5] / wy + dw = deltas[:, 2::5] / ww + dh = deltas[:, 3::5] / wh + da = deltas[:, 4::5] / wa + + # Prevent sending too large values into torch.exp() + dw = torch.clamp(dw, max=self.scale_clamp) + dh = torch.clamp(dh, max=self.scale_clamp) + + pred_boxes = torch.zeros_like(deltas) + pred_boxes[:, 0::5] = dx * widths + ctr_x # x_ctr + pred_boxes[:, 1::5] = dy * heights + ctr_y # y_ctr + pred_boxes[:, 2::5] = torch.exp(dw) * widths # width + pred_boxes[:, 3::5] = torch.exp(dh) * heights # height + + # Following original RRPN implementation, + # angles of deltas are in radians while angles of boxes are in degrees. + pred_angle = da * 180.0 / math.pi + angles + pred_angle = (pred_angle + 180.0) % 360.0 - 180.0 # make it in [-180, 180) + + pred_boxes[:, 4::5] = pred_angle + + return pred_boxes + + +class Box2BoxTransformLinear(object): + """ + The linear box-to-box transform defined in FCOS. The transformation is parameterized + by the distance from the center of (square) src box to 4 edges of the target box. + """ + + def __init__(self, normalize_by_size=True): + """ + Args: + normalize_by_size: normalize deltas by the size of src (anchor) boxes. + """ + self.normalize_by_size = normalize_by_size + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx1, dy1, dx2, dy2) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true. + The center of src must be inside target boxes. + + Args: + src_boxes (Tensor): square source boxes, e.g., anchors + target_boxes (Tensor): target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_ctr_x = 0.5 * (src_boxes[:, 0] + src_boxes[:, 2]) + src_ctr_y = 0.5 * (src_boxes[:, 1] + src_boxes[:, 3]) + + target_l = src_ctr_x - target_boxes[:, 0] + target_t = src_ctr_y - target_boxes[:, 1] + target_r = target_boxes[:, 2] - src_ctr_x + target_b = target_boxes[:, 3] - src_ctr_y + + deltas = torch.stack((target_l, target_t, target_r, target_b), dim=1) + if self.normalize_by_size: + stride_w = src_boxes[:, 2] - src_boxes[:, 0] + stride_h = src_boxes[:, 3] - src_boxes[:, 1] + strides = torch.stack([stride_w, stride_h, stride_w, stride_h], axis=1) + deltas = deltas / strides + + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx1, dy1, dx2, dy2) to `boxes`. + + Args: + deltas (Tensor): transformation deltas of shape (N, k*4), where k >= 1. + deltas[i] represents k potentially different class-specific + box transformations for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 4) + """ + # Ensure the output is a valid box. See Sec 2.1 of https://arxiv.org/abs/2006.09214 + deltas = F.relu(deltas) + boxes = boxes.to(deltas.dtype) + + ctr_x = 0.5 * (boxes[:, 0] + boxes[:, 2]) + ctr_y = 0.5 * (boxes[:, 1] + boxes[:, 3]) + if self.normalize_by_size: + stride_w = boxes[:, 2] - boxes[:, 0] + stride_h = boxes[:, 3] - boxes[:, 1] + strides = torch.stack([stride_w, stride_h, stride_w, stride_h], axis=1) + deltas = deltas * strides + + l = deltas[:, 0::4] + t = deltas[:, 1::4] + r = deltas[:, 2::4] + b = deltas[:, 3::4] + + pred_boxes = torch.zeros_like(deltas) + pred_boxes[:, 0::4] = ctr_x[:, None] - l # x1 + pred_boxes[:, 1::4] = ctr_y[:, None] - t # y1 + pred_boxes[:, 2::4] = ctr_x[:, None] + r # x2 + pred_boxes[:, 3::4] = ctr_y[:, None] + b # y2 + return pred_boxes + + +def _dense_box_regression_loss( + anchors: List[Union[Boxes, torch.Tensor]], + box2box_transform: Box2BoxTransform, + pred_anchor_deltas: List[torch.Tensor], + gt_boxes: List[torch.Tensor], + fg_mask: torch.Tensor, + box_reg_loss_type="smooth_l1", + smooth_l1_beta=0.0, +): + """ + Compute loss for dense multi-level box regression. + Loss is accumulated over ``fg_mask``. + + Args: + anchors: #lvl anchor boxes, each is (HixWixA, 4) + pred_anchor_deltas: #lvl predictions, each is (N, HixWixA, 4) + gt_boxes: N ground truth boxes, each has shape (R, 4) (R = sum(Hi * Wi * A)) + fg_mask: the foreground boolean mask of shape (N, R) to compute loss on + box_reg_loss_type (str): Loss type to use. Supported losses: "smooth_l1", "giou", + "diou", "ciou". + smooth_l1_beta (float): beta parameter for the smooth L1 regression loss. Default to + use L1 loss. Only used when `box_reg_loss_type` is "smooth_l1" + """ + if isinstance(anchors[0], Boxes): + anchors = type(anchors[0]).cat(anchors).tensor # (R, 4) + else: + anchors = cat(anchors) + if box_reg_loss_type == "smooth_l1": + gt_anchor_deltas = [box2box_transform.get_deltas(anchors, k) for k in gt_boxes] + gt_anchor_deltas = torch.stack(gt_anchor_deltas) # (N, R, 4) + loss_box_reg = smooth_l1_loss( + cat(pred_anchor_deltas, dim=1)[fg_mask], + gt_anchor_deltas[fg_mask], + beta=smooth_l1_beta, + reduction="sum", + ) + elif box_reg_loss_type == "giou": + pred_boxes = [box2box_transform.apply_deltas(k, anchors) for k in cat(pred_anchor_deltas, dim=1)] + loss_box_reg = giou_loss(torch.stack(pred_boxes)[fg_mask], torch.stack(gt_boxes)[fg_mask], reduction="sum") + elif box_reg_loss_type == "diou": + pred_boxes = [box2box_transform.apply_deltas(k, anchors) for k in cat(pred_anchor_deltas, dim=1)] + loss_box_reg = diou_loss(torch.stack(pred_boxes)[fg_mask], torch.stack(gt_boxes)[fg_mask], reduction="sum") + elif box_reg_loss_type == "ciou": + pred_boxes = [box2box_transform.apply_deltas(k, anchors) for k in cat(pred_anchor_deltas, dim=1)] + loss_box_reg = ciou_loss(torch.stack(pred_boxes)[fg_mask], torch.stack(gt_boxes)[fg_mask], reduction="sum") + else: + raise ValueError(f"Invalid dense box regression loss type '{box_reg_loss_type}'") + return loss_box_reg diff --git a/detectron2/modeling/matcher.py b/detectron2/modeling/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..e3cf46f334d507da66104992f82b4d5abd822b7f --- /dev/null +++ b/detectron2/modeling/matcher.py @@ -0,0 +1,122 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List + +import torch + +from detectron2.layers import nonzero_tuple + + +# TODO: the name is too general +class Matcher(object): + """ + This class assigns to each predicted "element" (e.g., a box) a ground-truth + element. Each predicted element will have exactly zero or one matches; each + ground-truth element may be matched to zero or more predicted elements. + + The matching is determined by the MxN match_quality_matrix, that characterizes + how well each (ground-truth, prediction)-pair match each other. For example, + if the elements are boxes, this matrix may contain box intersection-over-union + overlap values. + + The matcher returns (a) a vector of length N containing the index of the + ground-truth element m in [0, M) that matches to prediction n in [0, N). + (b) a vector of length N containing the labels for each prediction. + """ + + def __init__(self, thresholds: List[float], labels: List[int], allow_low_quality_matches: bool = False): + """ + Args: + thresholds (list): a list of thresholds used to stratify predictions + into levels. + labels (list): a list of values to label predictions belonging at + each level. A label can be one of {-1, 0, 1} signifying + {ignore, negative class, positive class}, respectively. + allow_low_quality_matches (bool): if True, produce additional matches + for predictions with maximum match quality lower than high_threshold. + See set_low_quality_matches_ for more details. + + For example, + thresholds = [0.3, 0.5] + labels = [0, -1, 1] + All predictions with iou < 0.3 will be marked with 0 and + thus will be considered as false positives while training. + All predictions with 0.3 <= iou < 0.5 will be marked with -1 and + thus will be ignored. + All predictions with 0.5 <= iou will be marked with 1 and + thus will be considered as true positives. + """ + # Add -inf and +inf to first and last position in thresholds + thresholds = thresholds[:] + assert thresholds[0] > 0 + thresholds.insert(0, -float("inf")) + thresholds.append(float("inf")) + # Currently torchscript does not support all + generator + assert all([low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])]) + assert all([l in [-1, 0, 1] for l in labels]) + assert len(labels) == len(thresholds) - 1 + self.thresholds = thresholds + self.labels = labels + self.allow_low_quality_matches = allow_low_quality_matches + + def __call__(self, match_quality_matrix): + """ + Args: + match_quality_matrix (Tensor[float]): an MxN tensor, containing the + pairwise quality between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` + for selecting indices in :meth:`set_low_quality_matches_`). + + Returns: + matches (Tensor[int64]): a vector of length N, where matches[i] is a matched + ground-truth index in [0, M) + match_labels (Tensor[int8]): a vector of length N, where pred_labels[i] indicates + whether a prediction is a true or false positive or ignored + """ + assert match_quality_matrix.dim() == 2 + if match_quality_matrix.numel() == 0: + default_matches = match_quality_matrix.new_full((match_quality_matrix.size(1),), 0, dtype=torch.int64) + # When no gt boxes exist, we define IOU = 0 and therefore set labels + # to `self.labels[0]`, which usually defaults to background class 0 + # To choose to ignore instead, can make labels=[-1,0,-1,1] + set appropriate thresholds + default_match_labels = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), self.labels[0], dtype=torch.int8 + ) + return default_matches, default_match_labels + + assert torch.all(match_quality_matrix >= 0) + + # match_quality_matrix is M (gt) x N (predicted) + # Max over gt elements (dim 0) to find best gt candidate for each prediction + matched_vals, matches = match_quality_matrix.max(dim=0) + + match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + + for l, low, high in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): + low_high = (matched_vals >= low) & (matched_vals < high) + match_labels[low_high] = l + + if self.allow_low_quality_matches: + self.set_low_quality_matches_(match_labels, match_quality_matrix) + + return matches, match_labels + + def set_low_quality_matches_(self, match_labels, match_quality_matrix): + """ + Produce additional matches for predictions that have only low-quality matches. + Specifically, for each ground-truth G find the set of predictions that have + maximum overlap with it (including ties); for each prediction in that set, if + it is unmatched, then match it to the ground-truth G. + + This function implements the RPN assignment case (i) in Sec. 3.1.2 of + :paper:`Faster R-CNN`. + """ + # For each gt, find the prediction with which it has highest quality + highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) + # Find the highest quality match available, even if it is low, including ties. + # Note that the matches qualities must be positive due to the use of + # `torch.nonzero`. + _, pred_inds_with_highest_quality = nonzero_tuple(match_quality_matrix == highest_quality_foreach_gt[:, None]) + # If an anchor was labeled positive only due to a low-quality match + # with gt_A, but it has larger overlap with gt_B, it's matched index will still be gt_B. + # This follows the implementation in Detectron, and is found to have no significant impact. + match_labels[pred_inds_with_highest_quality] = 1 diff --git a/detectron2/modeling/meta_arch/__init__.py b/detectron2/modeling/meta_arch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ff811ae6e2dfcfeb5eb9e54df91131128609b488 --- /dev/null +++ b/detectron2/modeling/meta_arch/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +from .build import META_ARCH_REGISTRY, build_model # isort:skip + +from .dense_detector import DenseDetector +from .fcos import FCOS +from .panoptic_fpn import PanopticFPN + +# import all the meta_arch, so they will be registered +from .rcnn import GeneralizedRCNN, ProposalNetwork +from .retinanet import RetinaNet +from .semantic_seg import SEM_SEG_HEADS_REGISTRY, SemanticSegmentor, build_sem_seg_head + +__all__ = list(globals().keys()) diff --git a/detectron2/modeling/meta_arch/build.py b/detectron2/modeling/meta_arch/build.py new file mode 100644 index 0000000000000000000000000000000000000000..3427215746c9a146bd902f22ea9b26d121c36b27 --- /dev/null +++ b/detectron2/modeling/meta_arch/build.py @@ -0,0 +1,25 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from detectron2.utils.logger import _log_api_usage +from detectron2.utils.registry import Registry + +META_ARCH_REGISTRY = Registry("META_ARCH") # noqa F401 isort:skip +META_ARCH_REGISTRY.__doc__ = """ +Registry for meta-architectures, i.e. the whole model. + +The registered object will be called with `obj(cfg)` +and expected to return a `nn.Module` object. +""" + + +def build_model(cfg): + """ + Build the whole model architecture, defined by ``cfg.MODEL.META_ARCHITECTURE``. + Note that it does not load any weights from ``cfg``. + """ + meta_arch = cfg.MODEL.META_ARCHITECTURE + model = META_ARCH_REGISTRY.get(meta_arch)(cfg) + model.to(torch.device(cfg.MODEL.DEVICE)) + _log_api_usage("modeling.meta_arch." + meta_arch) + return model diff --git a/detectron2/modeling/meta_arch/dense_detector.py b/detectron2/modeling/meta_arch/dense_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..5d66cf428c386f3f54776184f56b6adf827f45a1 --- /dev/null +++ b/detectron2/modeling/meta_arch/dense_detector.py @@ -0,0 +1,282 @@ +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch +from torch import Tensor, nn + +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.layers import move_device_like +from detectron2.modeling import Backbone +from detectron2.structures import Boxes, ImageList, Instances +from detectron2.utils.events import get_event_storage + +from ..postprocessing import detector_postprocess + + +def permute_to_N_HWA_K(tensor, K: int): + """ + Transpose/reshape a tensor from (N, (Ai x K), H, W) to (N, (HxWxAi), K) + """ + assert tensor.dim() == 4, tensor.shape + N, _, H, W = tensor.shape + tensor = tensor.view(N, -1, K, H, W) + tensor = tensor.permute(0, 3, 4, 1, 2) + tensor = tensor.reshape(N, -1, K) # Size=(N,HWA,K) + return tensor + + +class DenseDetector(nn.Module): + """ + Base class for dense detector. We define a dense detector as a fully-convolutional model that + makes per-pixel (i.e. dense) predictions. + """ + + def __init__( + self, + backbone: Backbone, + head: nn.Module, + head_in_features: Optional[List[str]] = None, + *, + pixel_mean, + pixel_std, + ): + """ + Args: + backbone: backbone module + head: head module + head_in_features: backbone features to use in head. Default to all backbone features. + pixel_mean (Tuple[float]): + Values to be used for image normalization (BGR order). + To train on images of different number of channels, set different mean & std. + Default values are the mean pixel value from ImageNet: [103.53, 116.28, 123.675] + pixel_std (Tuple[float]): + When using pre-trained models in Detectron1 or any MSRA models, + std has been absorbed into its conv1 weights, so the std needs to be set 1. + Otherwise, you can use [57.375, 57.120, 58.395] (ImageNet std) + """ + super().__init__() + + self.backbone = backbone + self.head = head + if head_in_features is None: + shapes = self.backbone.output_shape() + self.head_in_features = sorted(shapes.keys(), key=lambda x: shapes[x].stride) + else: + self.head_in_features = head_in_features + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def forward(self, batched_inputs: List[Dict[str, Tensor]]): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper` . + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + + * image: Tensor, image in (C, H, W) format. + * instances: Instances + + Other information that's included in the original dicts, such as: + + * "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + + Returns: + In training, dict[str, Tensor]: mapping from a named loss to a tensor storing the + loss. Used during training only. In inference, the standard output format, described + in :doc:`/tutorials/models`. + """ + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + features = [features[f] for f in self.head_in_features] + predictions = self.head(features) + + if self.training: + assert not torch.jit.is_scripting(), "Not supported" + assert "instances" in batched_inputs[0], "Instance annotations are missing in training!" + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + return self.forward_training(images, features, predictions, gt_instances) + else: + results = self.forward_inference(images, features, predictions) + if torch.jit.is_scripting(): + return results + + processed_results = [] + for results_per_image, input_per_image, image_size in zip(results, batched_inputs, images.image_sizes): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + def forward_training(self, images, features, predictions, gt_instances): + raise NotImplementedError() + + def preprocess_image(self, batched_inputs: List[Dict[str, Tensor]]): + """ + Normalize, pad and batch the input images. + """ + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + def _transpose_dense_predictions( + self, predictions: List[List[Tensor]], dims_per_anchor: List[int] + ) -> List[List[Tensor]]: + """ + Transpose the dense per-level predictions. + + Args: + predictions: a list of outputs, each is a list of per-level + predictions with shape (N, Ai x K, Hi, Wi), where N is the + number of images, Ai is the number of anchors per location on + level i, K is the dimension of predictions per anchor. + dims_per_anchor: the value of K for each predictions. e.g. 4 for + box prediction, #classes for classification prediction. + + Returns: + List[List[Tensor]]: each prediction is transposed to (N, Hi x Wi x Ai, K). + """ + assert len(predictions) == len(dims_per_anchor) + res: List[List[Tensor]] = [] + for pred, dim_per_anchor in zip(predictions, dims_per_anchor): + pred = [permute_to_N_HWA_K(x, dim_per_anchor) for x in pred] + res.append(pred) + return res + + def _ema_update(self, name: str, value: float, initial_value: float, momentum: float = 0.9): + """ + Apply EMA update to `self.name` using `value`. + + This is mainly used for loss normalizer. In Detectron1, loss is normalized by number + of foreground samples in the batch. When batch size is 1 per GPU, #foreground has a + large variance and using it lead to lower performance. Therefore we maintain an EMA of + #foreground to stabilize the normalizer. + + Args: + name: name of the normalizer + value: the new value to update + initial_value: the initial value to start with + momentum: momentum of EMA + + Returns: + float: the updated EMA value + """ + if hasattr(self, name): + old = getattr(self, name) + else: + old = initial_value + new = old * momentum + value * (1 - momentum) + setattr(self, name, new) + return new + + def _decode_per_level_predictions( + self, + anchors: Boxes, + pred_scores: Tensor, + pred_deltas: Tensor, + score_thresh: float, + topk_candidates: int, + image_size: Tuple[int, int], + ) -> Instances: + """ + Decode boxes and classification predictions of one featuer level, by + the following steps: + 1. filter the predictions based on score threshold and top K scores. + 2. transform the box regression outputs + 3. return the predicted scores, classes and boxes + + Args: + anchors: Boxes, anchor for this feature level + pred_scores: HxWxA,K + pred_deltas: HxWxA,4 + + Returns: + Instances: with field "scores", "pred_boxes", "pred_classes". + """ + # Apply two filtering to make NMS faster. + # 1. Keep boxes with confidence score higher than threshold + keep_idxs = pred_scores > score_thresh + pred_scores = pred_scores[keep_idxs] + topk_idxs = torch.nonzero(keep_idxs) # Kx2 + + # 2. Keep top k top scoring boxes only + num_topk = min(topk_candidates, topk_idxs.size(0)) + pred_scores, idxs = pred_scores.topk(num_topk) + topk_idxs = topk_idxs[idxs] + + anchor_idxs, classes_idxs = topk_idxs.unbind(dim=1) + + pred_boxes = self.box2box_transform.apply_deltas(pred_deltas[anchor_idxs], anchors.tensor[anchor_idxs]) + return Instances(image_size, pred_boxes=Boxes(pred_boxes), scores=pred_scores, pred_classes=classes_idxs) + + def _decode_multi_level_predictions( + self, + anchors: List[Boxes], + pred_scores: List[Tensor], + pred_deltas: List[Tensor], + score_thresh: float, + topk_candidates: int, + image_size: Tuple[int, int], + ) -> Instances: + """ + Run `_decode_per_level_predictions` for all feature levels and concat the results. + """ + predictions = [ + self._decode_per_level_predictions( + anchors_i, + box_cls_i, + box_reg_i, + self.test_score_thresh, + self.test_topk_candidates, + image_size, + ) + # Iterate over every feature level + for box_cls_i, box_reg_i, anchors_i in zip(pred_scores, pred_deltas, anchors) + ] + return predictions[0].cat(predictions) # 'Instances.cat' is not scriptale but this is + + def visualize_training(self, batched_inputs, results): + """ + A function used to visualize ground truth images and final network predictions. + It shows ground truth bounding boxes on the original image and up to 20 + predicted object bounding boxes on the original image. + + Args: + batched_inputs (list): a list that contains input to the model. + results (List[Instances]): a list of #images elements returned by forward_inference(). + """ + from detectron2.utils.visualizer import Visualizer + + assert len(batched_inputs) == len(results), "Cannot visualize inputs and results of different sizes" + storage = get_event_storage() + max_boxes = 20 + + image_index = 0 # only visualize a single image + img = batched_inputs[image_index]["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + v_gt = v_gt.overlay_instances(boxes=batched_inputs[image_index]["instances"].gt_boxes) + anno_img = v_gt.get_image() + processed_results = detector_postprocess(results[image_index], img.shape[0], img.shape[1]) + predicted_boxes = processed_results.pred_boxes.tensor.detach().cpu().numpy() + + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances(boxes=predicted_boxes[0:max_boxes]) + prop_img = v_pred.get_image() + vis_img = np.vstack((anno_img, prop_img)) + vis_img = vis_img.transpose(2, 0, 1) + vis_name = f"Top: GT bounding boxes; Bottom: {max_boxes} Highest Scoring Results" + storage.put_image(vis_name, vis_img) diff --git a/detectron2/modeling/meta_arch/fcos.py b/detectron2/modeling/meta_arch/fcos.py new file mode 100644 index 0000000000000000000000000000000000000000..807a4772430e2f9234022029a4b394e7b6ece5d9 --- /dev/null +++ b/detectron2/modeling/meta_arch/fcos.py @@ -0,0 +1,317 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from typing import List, Optional, Tuple + +import torch +from fvcore.nn import sigmoid_focal_loss_jit +from torch import nn +from torch.nn import functional as F + +from detectron2.layers import ShapeSpec, batched_nms +from detectron2.structures import Boxes, ImageList, Instances, pairwise_point_box_distance +from detectron2.utils.events import get_event_storage + +from ..anchor_generator import DefaultAnchorGenerator +from ..backbone import Backbone +from ..box_regression import Box2BoxTransformLinear, _dense_box_regression_loss +from .dense_detector import DenseDetector +from .retinanet import RetinaNetHead + +__all__ = ["FCOS"] + +logger = logging.getLogger(__name__) + + +class FCOS(DenseDetector): + """ + Implement FCOS in :paper:`fcos`. + """ + + def __init__( + self, + *, + backbone: Backbone, + head: nn.Module, + head_in_features: Optional[List[str]] = None, + box2box_transform=None, + num_classes, + center_sampling_radius: float = 1.5, + focal_loss_alpha=0.25, + focal_loss_gamma=2.0, + test_score_thresh=0.2, + test_topk_candidates=1000, + test_nms_thresh=0.6, + max_detections_per_image=100, + pixel_mean, + pixel_std, + ): + """ + Args: + center_sampling_radius: radius of the "center" of a groundtruth box, + within which all anchor points are labeled positive. + Other arguments mean the same as in :class:`RetinaNet`. + """ + super().__init__(backbone, head, head_in_features, pixel_mean=pixel_mean, pixel_std=pixel_std) + + self.num_classes = num_classes + + # FCOS uses one anchor point per location. + # We represent the anchor point by a box whose size equals the anchor stride. + feature_shapes = backbone.output_shape() + fpn_strides = [feature_shapes[k].stride for k in self.head_in_features] + self.anchor_generator = DefaultAnchorGenerator( + sizes=[[k] for k in fpn_strides], aspect_ratios=[1.0], strides=fpn_strides + ) + + # FCOS parameterizes box regression by a linear transform, + # where predictions are normalized by anchor stride (equal to anchor size). + if box2box_transform is None: + box2box_transform = Box2BoxTransformLinear(normalize_by_size=True) + self.box2box_transform = box2box_transform + + self.center_sampling_radius = float(center_sampling_radius) + + # Loss parameters: + self.focal_loss_alpha = focal_loss_alpha + self.focal_loss_gamma = focal_loss_gamma + + # Inference parameters: + self.test_score_thresh = test_score_thresh + self.test_topk_candidates = test_topk_candidates + self.test_nms_thresh = test_nms_thresh + self.max_detections_per_image = max_detections_per_image + + def forward_training(self, images, features, predictions, gt_instances): + # Transpose the Hi*Wi*A dimension to the middle: + pred_logits, pred_anchor_deltas, pred_centerness = self._transpose_dense_predictions( + predictions, [self.num_classes, 4, 1] + ) + anchors = self.anchor_generator(features) + gt_labels, gt_boxes = self.label_anchors(anchors, gt_instances) + return self.losses(anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes, pred_centerness) + + @torch.no_grad() + def _match_anchors(self, gt_boxes: Boxes, anchors: List[Boxes]): + """ + Match ground-truth boxes to a set of multi-level anchors. + + Args: + gt_boxes: Ground-truth boxes from instances of an image. + anchors: List of anchors for each feature map (of different scales). + + Returns: + torch.Tensor + A tensor of shape `(M, R)`, given `M` ground-truth boxes and total + `R` anchor points from all feature levels, indicating the quality + of match between m-th box and r-th anchor. Higher value indicates + better match. + """ + # Naming convention: (M = ground-truth boxes, R = anchor points) + # Anchor points are represented as square boxes of size = stride. + num_anchors_per_level = [len(x) for x in anchors] + anchors = Boxes.cat(anchors) # (R, 4) + anchor_centers = anchors.get_centers() # (R, 2) + anchor_sizes = anchors.tensor[:, 2] - anchors.tensor[:, 0] # (R, ) + + lower_bound = anchor_sizes * 4 + lower_bound[: num_anchors_per_level[0]] = 0 + upper_bound = anchor_sizes * 8 + upper_bound[-num_anchors_per_level[-1] :] = float("inf") + + gt_centers = gt_boxes.get_centers() + + # FCOS with center sampling: anchor point must be close enough to + # ground-truth box center. + center_dists = (anchor_centers[None, :, :] - gt_centers[:, None, :]).abs_() + sampling_regions = self.center_sampling_radius * anchor_sizes[None, :] + + match_quality_matrix = center_dists.max(dim=2).values < sampling_regions + + pairwise_dist = pairwise_point_box_distance(anchor_centers, gt_boxes) + pairwise_dist = pairwise_dist.permute(1, 0, 2) # (M, R, 4) + + # The original FCOS anchor matching rule: anchor point must be inside GT. + match_quality_matrix &= pairwise_dist.min(dim=2).values > 0 + + # Multilevel anchor matching in FCOS: each anchor is only responsible + # for certain scale range. + pairwise_dist = pairwise_dist.max(dim=2).values + match_quality_matrix &= (pairwise_dist > lower_bound[None, :]) & (pairwise_dist < upper_bound[None, :]) + # Match the GT box with minimum area, if there are multiple GT matches. + gt_areas = gt_boxes.area() # (M, ) + + match_quality_matrix = match_quality_matrix.to(torch.float32) + match_quality_matrix *= 1e8 - gt_areas[:, None] + return match_quality_matrix # (M, R) + + @torch.no_grad() + def label_anchors(self, anchors: List[Boxes], gt_instances: List[Instances]): + """ + Same interface as :meth:`RetinaNet.label_anchors`, but implemented with FCOS + anchor matching rule. + + Unlike RetinaNet, there are no ignored anchors. + """ + + gt_labels, matched_gt_boxes = [], [] + + for inst in gt_instances: + if len(inst) > 0: + match_quality_matrix = self._match_anchors(inst.gt_boxes, anchors) + + # Find matched ground-truth box per anchor. Un-matched anchors are + # assigned -1. This is equivalent to using an anchor matcher as used + # in R-CNN/RetinaNet: `Matcher(thresholds=[1e-5], labels=[0, 1])` + match_quality, matched_idxs = match_quality_matrix.max(dim=0) + matched_idxs[match_quality < 1e-5] = -1 + + matched_gt_boxes_i = inst.gt_boxes.tensor[matched_idxs.clip(min=0)] + gt_labels_i = inst.gt_classes[matched_idxs.clip(min=0)] + + # Anchors with matched_idxs = -1 are labeled background. + gt_labels_i[matched_idxs < 0] = self.num_classes + else: + matched_gt_boxes_i = torch.zeros_like(Boxes.cat(anchors).tensor) + gt_labels_i = torch.full( + (len(matched_gt_boxes_i),), + fill_value=self.num_classes, + dtype=torch.long, + device=matched_gt_boxes_i.device, + ) + + gt_labels.append(gt_labels_i) + matched_gt_boxes.append(matched_gt_boxes_i) + + return gt_labels, matched_gt_boxes + + def losses(self, anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes, pred_centerness): + """ + This method is almost identical to :meth:`RetinaNet.losses`, with an extra + "loss_centerness" in the returned dict. + """ + num_images = len(gt_labels) + gt_labels = torch.stack(gt_labels) # (M, R) + + pos_mask = (gt_labels >= 0) & (gt_labels != self.num_classes) + num_pos_anchors = pos_mask.sum().item() + get_event_storage().put_scalar("num_pos_anchors", num_pos_anchors / num_images) + normalizer = self._ema_update("loss_normalizer", max(num_pos_anchors, 1), 300) + + # classification and regression loss + gt_labels_target = F.one_hot(gt_labels, num_classes=self.num_classes + 1)[ + :, :, :-1 + ] # no loss for the last (background) class + loss_cls = sigmoid_focal_loss_jit( + torch.cat(pred_logits, dim=1), + gt_labels_target.to(pred_logits[0].dtype), + alpha=self.focal_loss_alpha, + gamma=self.focal_loss_gamma, + reduction="sum", + ) + + loss_box_reg = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + gt_boxes, + pos_mask, + box_reg_loss_type="giou", + ) + + ctrness_targets = self.compute_ctrness_targets(anchors, gt_boxes) # (M, R) + pred_centerness = torch.cat(pred_centerness, dim=1).squeeze(dim=2) # (M, R) + ctrness_loss = F.binary_cross_entropy_with_logits( + pred_centerness[pos_mask], ctrness_targets[pos_mask], reduction="sum" + ) + return { + "loss_fcos_cls": loss_cls / normalizer, + "loss_fcos_loc": loss_box_reg / normalizer, + "loss_fcos_ctr": ctrness_loss / normalizer, + } + + def compute_ctrness_targets(self, anchors: List[Boxes], gt_boxes: List[torch.Tensor]): + anchors = Boxes.cat(anchors).tensor # Rx4 + reg_targets = [self.box2box_transform.get_deltas(anchors, m) for m in gt_boxes] + reg_targets = torch.stack(reg_targets, dim=0) # NxRx4 + if len(reg_targets) == 0: + return reg_targets.new_zeros(len(reg_targets)) + left_right = reg_targets[:, :, [0, 2]] + top_bottom = reg_targets[:, :, [1, 3]] + ctrness = (left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * ( + top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0] + ) + return torch.sqrt(ctrness) + + def forward_inference( + self, + images: ImageList, + features: List[torch.Tensor], + predictions: List[List[torch.Tensor]], + ): + pred_logits, pred_anchor_deltas, pred_centerness = self._transpose_dense_predictions( + predictions, [self.num_classes, 4, 1] + ) + anchors = self.anchor_generator(features) + + results: List[Instances] = [] + for img_idx, image_size in enumerate(images.image_sizes): + scores_per_image = [ + # Multiply and sqrt centerness & classification scores + # (See eqn. 4 in https://arxiv.org/abs/2006.09214) + torch.sqrt(x[img_idx].sigmoid_() * y[img_idx].sigmoid_()) + for x, y in zip(pred_logits, pred_centerness) + ] + deltas_per_image = [x[img_idx] for x in pred_anchor_deltas] + results_per_image = self.inference_single_image(anchors, scores_per_image, deltas_per_image, image_size) + results.append(results_per_image) + return results + + def inference_single_image( + self, + anchors: List[Boxes], + box_cls: List[torch.Tensor], + box_delta: List[torch.Tensor], + image_size: Tuple[int, int], + ): + """ + Identical to :meth:`RetinaNet.inference_single_image. + """ + pred = self._decode_multi_level_predictions( + anchors, + box_cls, + box_delta, + self.test_score_thresh, + self.test_topk_candidates, + image_size, + ) + keep = batched_nms(pred.pred_boxes.tensor, pred.scores, pred.pred_classes, self.test_nms_thresh) + return pred[keep[: self.max_detections_per_image]] + + +class FCOSHead(RetinaNetHead): + """ + The head used in :paper:`fcos`. It adds an additional centerness + prediction branch on top of :class:`RetinaNetHead`. + """ + + def __init__(self, *, input_shape: List[ShapeSpec], conv_dims: List[int], **kwargs): + super().__init__(input_shape=input_shape, conv_dims=conv_dims, num_anchors=1, **kwargs) + # Unlike original FCOS, we do not add an additional learnable scale layer + # because it's found to have no benefits after normalizing regression targets by stride. + self._num_features = len(input_shape) + self.ctrness = nn.Conv2d(conv_dims[-1], 1, kernel_size=3, stride=1, padding=1) + torch.nn.init.normal_(self.ctrness.weight, std=0.01) + torch.nn.init.constant_(self.ctrness.bias, 0) + + def forward(self, features): + assert len(features) == self._num_features + logits = [] + bbox_reg = [] + ctrness = [] + for feature in features: + logits.append(self.cls_score(self.cls_subnet(feature))) + bbox_feature = self.bbox_subnet(feature) + bbox_reg.append(self.bbox_pred(bbox_feature)) + ctrness.append(self.ctrness(bbox_feature)) + return logits, bbox_reg, ctrness diff --git a/detectron2/modeling/meta_arch/panoptic_fpn.py b/detectron2/modeling/meta_arch/panoptic_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..f2453f14224d97b8b2f44cfe6250610eb921a657 --- /dev/null +++ b/detectron2/modeling/meta_arch/panoptic_fpn.py @@ -0,0 +1,266 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from typing import Dict, List + +import torch +from torch import nn + +from detectron2.config import configurable +from detectron2.structures import ImageList + +from ..postprocessing import detector_postprocess, sem_seg_postprocess +from .build import META_ARCH_REGISTRY +from .rcnn import GeneralizedRCNN +from .semantic_seg import build_sem_seg_head + +__all__ = ["PanopticFPN"] + + +@META_ARCH_REGISTRY.register() +class PanopticFPN(GeneralizedRCNN): + """ + Implement the paper :paper:`PanopticFPN`. + """ + + @configurable + def __init__( + self, + *, + sem_seg_head: nn.Module, + combine_overlap_thresh: float = 0.5, + combine_stuff_area_thresh: float = 4096, + combine_instances_score_thresh: float = 0.5, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + sem_seg_head: a module for the semantic segmentation head. + combine_overlap_thresh: combine masks into one instances if + they have enough overlap + combine_stuff_area_thresh: ignore stuff areas smaller than this threshold + combine_instances_score_thresh: ignore instances whose score is + smaller than this threshold + + Other arguments are the same as :class:`GeneralizedRCNN`. + """ + super().__init__(**kwargs) + self.sem_seg_head = sem_seg_head + # options when combining instance & semantic outputs + self.combine_overlap_thresh = combine_overlap_thresh + self.combine_stuff_area_thresh = combine_stuff_area_thresh + self.combine_instances_score_thresh = combine_instances_score_thresh + + @classmethod + def from_config(cls, cfg): + ret = super().from_config(cfg) + ret.update( + { + "combine_overlap_thresh": cfg.MODEL.PANOPTIC_FPN.COMBINE.OVERLAP_THRESH, + "combine_stuff_area_thresh": cfg.MODEL.PANOPTIC_FPN.COMBINE.STUFF_AREA_LIMIT, + "combine_instances_score_thresh": cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH, # noqa + } + ) + ret["sem_seg_head"] = build_sem_seg_head(cfg, ret["backbone"].output_shape()) + logger = logging.getLogger(__name__) + if not cfg.MODEL.PANOPTIC_FPN.COMBINE.ENABLED: + logger.warning( + "PANOPTIC_FPN.COMBINED.ENABLED is no longer used. " + " model.inference(do_postprocess=) should be used to toggle postprocessing." + ) + if cfg.MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT != 1.0: + w = cfg.MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT + logger.warning("PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT should be replaced by weights on each ROI head.") + + def update_weight(x): + if isinstance(x, dict): + return {k: v * w for k, v in x.items()} + else: + return x * w + + roi_heads = ret["roi_heads"] + roi_heads.box_predictor.loss_weight = update_weight(roi_heads.box_predictor.loss_weight) + roi_heads.mask_head.loss_weight = update_weight(roi_heads.mask_head.loss_weight) + return ret + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper`. + Each item in the list contains the inputs for one image. + + For now, each item in the list is a dict that contains: + + * "image": Tensor, image in (C, H, W) format. + * "instances": Instances + * "sem_seg": semantic segmentation ground truth. + * Other information that's included in the original dicts, such as: + "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + + Returns: + list[dict]: + each dict has the results for one image. The dict contains the following keys: + + * "instances": see :meth:`GeneralizedRCNN.forward` for its format. + * "sem_seg": see :meth:`SemanticSegmentor.forward` for its format. + * "panoptic_seg": See the return value of + :func:`combine_semantic_and_instance_outputs` for its format. + """ + if not self.training: + return self.inference(batched_inputs) + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + + assert "sem_seg" in batched_inputs[0] + gt_sem_seg = [x["sem_seg"].to(self.device) for x in batched_inputs] + gt_sem_seg = ImageList.from_tensors( + gt_sem_seg, + self.backbone.size_divisibility, + self.sem_seg_head.ignore_value, + self.backbone.padding_constraints, + ).tensor + sem_seg_results, sem_seg_losses = self.sem_seg_head(features, gt_sem_seg) + + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + proposals, proposal_losses = self.proposal_generator(images, features, gt_instances) + detector_results, detector_losses = self.roi_heads(images, features, proposals, gt_instances) + + losses = sem_seg_losses + losses.update(proposal_losses) + losses.update(detector_losses) + return losses + + def inference(self, batched_inputs: List[Dict[str, torch.Tensor]], do_postprocess: bool = True): + """ + Run inference on the given inputs. + + Args: + batched_inputs (list[dict]): same as in :meth:`forward` + do_postprocess (bool): whether to apply post-processing on the outputs. + + Returns: + When do_postprocess=True, see docs in :meth:`forward`. + Otherwise, returns a (list[Instances], list[Tensor]) that contains + the raw detector outputs, and raw semantic segmentation outputs. + """ + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + sem_seg_results, sem_seg_losses = self.sem_seg_head(features, None) + proposals, _ = self.proposal_generator(images, features, None) + detector_results, _ = self.roi_heads(images, features, proposals, None) + + if do_postprocess: + processed_results = [] + for sem_seg_result, detector_result, input_per_image, image_size in zip( + sem_seg_results, detector_results, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + sem_seg_r = sem_seg_postprocess(sem_seg_result, image_size, height, width) + detector_r = detector_postprocess(detector_result, height, width) + + processed_results.append({"sem_seg": sem_seg_r, "instances": detector_r}) + + panoptic_r = combine_semantic_and_instance_outputs( + detector_r, + sem_seg_r.argmax(dim=0), + self.combine_overlap_thresh, + self.combine_stuff_area_thresh, + self.combine_instances_score_thresh, + ) + processed_results[-1]["panoptic_seg"] = panoptic_r + return processed_results + else: + return detector_results, sem_seg_results + + +def combine_semantic_and_instance_outputs( + instance_results, + semantic_results, + overlap_threshold, + stuff_area_thresh, + instances_score_thresh, +): + """ + Implement a simple combining logic following + "combine_semantic_and_instance_predictions.py" in panopticapi + to produce panoptic segmentation outputs. + + Args: + instance_results: output of :func:`detector_postprocess`. + semantic_results: an (H, W) tensor, each element is the contiguous semantic + category id + + Returns: + panoptic_seg (Tensor): of shape (height, width) where the values are ids for each segment. + segments_info (list[dict]): Describe each segment in `panoptic_seg`. + Each dict contains keys "id", "category_id", "isthing". + """ + panoptic_seg = torch.zeros_like(semantic_results, dtype=torch.int32) + + # sort instance outputs by scores + sorted_inds = torch.argsort(-instance_results.scores) + + current_segment_id = 0 + segments_info = [] + + instance_masks = instance_results.pred_masks.to(dtype=torch.bool, device=panoptic_seg.device) + + # Add instances one-by-one, check for overlaps with existing ones + for inst_id in sorted_inds: + score = instance_results.scores[inst_id].item() + if score < instances_score_thresh: + break + mask = instance_masks[inst_id] # H,W + mask_area = mask.sum().item() + + if mask_area == 0: + continue + + intersect = (mask > 0) & (panoptic_seg > 0) + intersect_area = intersect.sum().item() + + if intersect_area * 1.0 / mask_area > overlap_threshold: + continue + + if intersect_area > 0: + mask = mask & (panoptic_seg == 0) + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + segments_info.append( + { + "id": current_segment_id, + "isthing": True, + "score": score, + "category_id": instance_results.pred_classes[inst_id].item(), + "instance_id": inst_id.item(), + } + ) + + # Add semantic results to remaining empty areas + semantic_labels = torch.unique(semantic_results).cpu().tolist() + for semantic_label in semantic_labels: + if semantic_label == 0: # 0 is a special "thing" class + continue + mask = (semantic_results == semantic_label) & (panoptic_seg == 0) + mask_area = mask.sum().item() + if mask_area < stuff_area_thresh: + continue + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + segments_info.append( + { + "id": current_segment_id, + "isthing": False, + "category_id": semantic_label, + "area": mask_area, + } + ) + + return panoptic_seg, segments_info diff --git a/detectron2/modeling/meta_arch/rcnn.py b/detectron2/modeling/meta_arch/rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..06862c020d86a8a171ba800f3c2d55dd4d459b72 --- /dev/null +++ b/detectron2/modeling/meta_arch/rcnn.py @@ -0,0 +1,335 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch +from torch import nn + +from detectron2.config import configurable +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.layers import move_device_like +from detectron2.structures import ImageList, Instances +from detectron2.utils.events import get_event_storage +from detectron2.utils.logger import log_first_n + +from ..backbone import Backbone, build_backbone +from ..postprocessing import detector_postprocess +from ..proposal_generator import build_proposal_generator +from ..roi_heads import build_roi_heads +from .build import META_ARCH_REGISTRY + +__all__ = ["GeneralizedRCNN", "ProposalNetwork"] + + +@META_ARCH_REGISTRY.register() +class GeneralizedRCNN(nn.Module): + """ + Generalized R-CNN. Any models that contains the following three components: + 1. Per-image feature extraction (aka backbone) + 2. Region proposal generation + 3. Per-region feature extraction and prediction + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + proposal_generator: nn.Module, + roi_heads: nn.Module, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + input_format: Optional[str] = None, + vis_period: int = 0, + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + proposal_generator: a module that generates proposals using backbone features + roi_heads: a ROI head that performs per-region computation + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + input_format: describe the meaning of channels of input. Needed by visualization + vis_period: the period to run visualization. Set to 0 to disable. + """ + super().__init__() + self.backbone = backbone + self.proposal_generator = proposal_generator + self.roi_heads = roi_heads + + self.input_format = input_format + self.vis_period = vis_period + if vis_period > 0: + assert input_format is not None, "input_format is required for visualization!" + + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + assert ( + self.pixel_mean.shape == self.pixel_std.shape + ), f"{self.pixel_mean} and {self.pixel_std} have different shapes!" + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + return { + "backbone": backbone, + "proposal_generator": build_proposal_generator(cfg, backbone.output_shape()), + "roi_heads": build_roi_heads(cfg, backbone.output_shape()), + "input_format": cfg.INPUT.FORMAT, + "vis_period": cfg.VIS_PERIOD, + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + } + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def visualize_training(self, batched_inputs, proposals): + """ + A function used to visualize images and proposals. It shows ground truth + bounding boxes on the original image and up to 20 top-scoring predicted + object proposals on the original image. Users can implement different + visualization functions for different models. + + Args: + batched_inputs (list): a list that contains input to the model. + proposals (list): a list that contains predicted proposals. Both + batched_inputs and proposals should have the same length. + """ + from detectron2.utils.visualizer import Visualizer + + storage = get_event_storage() + max_vis_prop = 20 + + for input, prop in zip(batched_inputs, proposals): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + v_gt = v_gt.overlay_instances(boxes=input["instances"].gt_boxes) + anno_img = v_gt.get_image() + box_size = min(len(prop.proposal_boxes), max_vis_prop) + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances(boxes=prop.proposal_boxes[0:box_size].tensor.cpu().numpy()) + prop_img = v_pred.get_image() + vis_img = np.concatenate((anno_img, prop_img), axis=1) + vis_img = vis_img.transpose(2, 0, 1) + vis_name = "Left: GT bounding boxes; Right: Predicted proposals" + storage.put_image(vis_name, vis_img) + break # only visualize one image in a batch + + def forward(self, batched_inputs: List[Dict[str, torch.Tensor]]): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper` . + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + + * image: Tensor, image in (C, H, W) format. + * instances (optional): groundtruth :class:`Instances` + * proposals (optional): :class:`Instances`, precomputed proposals. + + Other information that's included in the original dicts, such as: + + * "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + + Returns: + list[dict]: + Each dict is the output for one input image. + The dict contains one key "instances" whose value is a :class:`Instances`. + The :class:`Instances` object has the following keys: + "pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints" + """ + if not self.training: + return self.inference(batched_inputs) + + images = self.preprocess_image(batched_inputs) + if "instances" in batched_inputs[0]: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + else: + gt_instances = None + + features = self.backbone(images.tensor) + + if self.proposal_generator is not None: + proposals, proposal_losses = self.proposal_generator(images, features, gt_instances) + else: + assert "proposals" in batched_inputs[0] + proposals = [x["proposals"].to(self.device) for x in batched_inputs] + proposal_losses = {} + + _, detector_losses = self.roi_heads(images, features, proposals, gt_instances) + if self.vis_period > 0: + storage = get_event_storage() + if storage.iter % self.vis_period == 0: + self.visualize_training(batched_inputs, proposals) + + losses = {} + losses.update(detector_losses) + losses.update(proposal_losses) + return losses + + def inference( + self, + batched_inputs: List[Dict[str, torch.Tensor]], + detected_instances: Optional[List[Instances]] = None, + do_postprocess: bool = True, + ): + """ + Run inference on the given inputs. + + Args: + batched_inputs (list[dict]): same as in :meth:`forward` + detected_instances (None or list[Instances]): if not None, it + contains an `Instances` object per image. The `Instances` + object contains "pred_boxes" and "pred_classes" which are + known boxes in the image. + The inference will then skip the detection of bounding boxes, + and only predict other per-ROI outputs. + do_postprocess (bool): whether to apply post-processing on the outputs. + + Returns: + When do_postprocess=True, same as in :meth:`forward`. + Otherwise, a list[Instances] containing raw network outputs. + """ + assert not self.training + + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + + if detected_instances is None: + if self.proposal_generator is not None: + proposals, _ = self.proposal_generator(images, features, None) + else: + assert "proposals" in batched_inputs[0] + proposals = [x["proposals"].to(self.device) for x in batched_inputs] + + results, _ = self.roi_heads(images, features, proposals, None) + else: + detected_instances = [x.to(self.device) for x in detected_instances] + results = self.roi_heads.forward_with_given_boxes(features, detected_instances) + + if do_postprocess: + assert not torch.jit.is_scripting(), "Scripting is not supported for postprocess." + return GeneralizedRCNN._postprocess(results, batched_inputs, images.image_sizes) + else: + return results + + def preprocess_image(self, batched_inputs: List[Dict[str, torch.Tensor]]): + """ + Normalize, pad and batch the input images. + """ + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess(instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes): + """ + Rescale the output instances to the target size. + """ + # note: private function; subject to changes + processed_results = [] + for results_per_image, input_per_image, image_size in zip(instances, batched_inputs, image_sizes): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + +@META_ARCH_REGISTRY.register() +class ProposalNetwork(nn.Module): + """ + A meta architecture that only predicts object proposals. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + proposal_generator: nn.Module, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + proposal_generator: a module that generates proposals using backbone features + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + """ + super().__init__() + self.backbone = backbone + self.proposal_generator = proposal_generator + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + return { + "backbone": backbone, + "proposal_generator": build_proposal_generator(cfg, backbone.output_shape()), + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + } + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def forward(self, batched_inputs): + """ + Args: + Same as in :class:`GeneralizedRCNN.forward` + + Returns: + list[dict]: + Each dict is the output for one input image. + The dict contains one key "proposals" whose value is a + :class:`Instances` with keys "proposal_boxes" and "objectness_logits". + """ + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + features = self.backbone(images.tensor) + + if "instances" in batched_inputs[0]: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + elif "targets" in batched_inputs[0]: + log_first_n(logging.WARN, "'targets' in the model inputs is now renamed to 'instances'!", n=10) + gt_instances = [x["targets"].to(self.device) for x in batched_inputs] + else: + gt_instances = None + proposals, proposal_losses = self.proposal_generator(images, features, gt_instances) + # In training, the proposals are not useful at all but we generate them anyway. + # This makes RPN-only models about 5% slower. + if self.training: + return proposal_losses + + processed_results = [] + for results_per_image, input_per_image, image_size in zip(proposals, batched_inputs, images.image_sizes): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"proposals": r}) + return processed_results diff --git a/detectron2/modeling/meta_arch/retinanet.py b/detectron2/modeling/meta_arch/retinanet.py new file mode 100644 index 0000000000000000000000000000000000000000..7dcefc45e0d5add62ca71f11271f6556a2e9a835 --- /dev/null +++ b/detectron2/modeling/meta_arch/retinanet.py @@ -0,0 +1,414 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import math +from typing import List, Tuple + +import torch +from fvcore.nn import sigmoid_focal_loss_jit +from torch import Tensor, nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.layers import CycleBatchNormList, ShapeSpec, batched_nms, cat, get_norm +from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage + +from ..anchor_generator import build_anchor_generator +from ..backbone import Backbone, build_backbone +from ..box_regression import Box2BoxTransform, _dense_box_regression_loss +from ..matcher import Matcher +from .build import META_ARCH_REGISTRY +from .dense_detector import DenseDetector, permute_to_N_HWA_K # noqa + +__all__ = ["RetinaNet"] + + +logger = logging.getLogger(__name__) + + +@META_ARCH_REGISTRY.register() +class RetinaNet(DenseDetector): + """ + Implement RetinaNet in :paper:`RetinaNet`. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + head: nn.Module, + head_in_features, + anchor_generator, + box2box_transform, + anchor_matcher, + num_classes, + focal_loss_alpha=0.25, + focal_loss_gamma=2.0, + smooth_l1_beta=0.0, + box_reg_loss_type="smooth_l1", + test_score_thresh=0.05, + test_topk_candidates=1000, + test_nms_thresh=0.5, + max_detections_per_image=100, + pixel_mean, + pixel_std, + vis_period=0, + input_format="BGR", + ): + """ + NOTE: this interface is experimental. + + Args: + backbone: a backbone module, must follow detectron2's backbone interface + head (nn.Module): a module that predicts logits and regression deltas + for each level from a list of per-level features + head_in_features (Tuple[str]): Names of the input feature maps to be used in head + anchor_generator (nn.Module): a module that creates anchors from a + list of features. Usually an instance of :class:`AnchorGenerator` + box2box_transform (Box2BoxTransform): defines the transform from anchors boxes to + instance boxes + anchor_matcher (Matcher): label the anchors by matching them with ground truth. + num_classes (int): number of classes. Used to label background proposals. + + # Loss parameters: + focal_loss_alpha (float): focal_loss_alpha + focal_loss_gamma (float): focal_loss_gamma + smooth_l1_beta (float): smooth_l1_beta + box_reg_loss_type (str): Options are "smooth_l1", "giou", "diou", "ciou" + + # Inference parameters: + test_score_thresh (float): Inference cls score threshold, only anchors with + score > INFERENCE_TH are considered for inference (to improve speed) + test_topk_candidates (int): Select topk candidates before NMS + test_nms_thresh (float): Overlap threshold used for non-maximum suppression + (suppress boxes with IoU >= this threshold) + max_detections_per_image (int): + Maximum number of detections to return per image during inference + (100 is based on the limit established for the COCO dataset). + + pixel_mean, pixel_std: see :class:`DenseDetector`. + """ + super().__init__(backbone, head, head_in_features, pixel_mean=pixel_mean, pixel_std=pixel_std) + self.num_classes = num_classes + + # Anchors + self.anchor_generator = anchor_generator + self.box2box_transform = box2box_transform + self.anchor_matcher = anchor_matcher + + # Loss parameters: + self.focal_loss_alpha = focal_loss_alpha + self.focal_loss_gamma = focal_loss_gamma + self.smooth_l1_beta = smooth_l1_beta + self.box_reg_loss_type = box_reg_loss_type + # Inference parameters: + self.test_score_thresh = test_score_thresh + self.test_topk_candidates = test_topk_candidates + self.test_nms_thresh = test_nms_thresh + self.max_detections_per_image = max_detections_per_image + # Vis parameters + self.vis_period = vis_period + self.input_format = input_format + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + backbone_shape = backbone.output_shape() + feature_shapes = [backbone_shape[f] for f in cfg.MODEL.RETINANET.IN_FEATURES] + head = RetinaNetHead(cfg, feature_shapes) + anchor_generator = build_anchor_generator(cfg, feature_shapes) + return { + "backbone": backbone, + "head": head, + "anchor_generator": anchor_generator, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RETINANET.BBOX_REG_WEIGHTS), + "anchor_matcher": Matcher( + cfg.MODEL.RETINANET.IOU_THRESHOLDS, + cfg.MODEL.RETINANET.IOU_LABELS, + allow_low_quality_matches=True, + ), + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + "num_classes": cfg.MODEL.RETINANET.NUM_CLASSES, + "head_in_features": cfg.MODEL.RETINANET.IN_FEATURES, + # Loss parameters: + "focal_loss_alpha": cfg.MODEL.RETINANET.FOCAL_LOSS_ALPHA, + "focal_loss_gamma": cfg.MODEL.RETINANET.FOCAL_LOSS_GAMMA, + "smooth_l1_beta": cfg.MODEL.RETINANET.SMOOTH_L1_LOSS_BETA, + "box_reg_loss_type": cfg.MODEL.RETINANET.BBOX_REG_LOSS_TYPE, + # Inference parameters: + "test_score_thresh": cfg.MODEL.RETINANET.SCORE_THRESH_TEST, + "test_topk_candidates": cfg.MODEL.RETINANET.TOPK_CANDIDATES_TEST, + "test_nms_thresh": cfg.MODEL.RETINANET.NMS_THRESH_TEST, + "max_detections_per_image": cfg.TEST.DETECTIONS_PER_IMAGE, + # Vis parameters + "vis_period": cfg.VIS_PERIOD, + "input_format": cfg.INPUT.FORMAT, + } + + def forward_training(self, images, features, predictions, gt_instances): + # Transpose the Hi*Wi*A dimension to the middle: + pred_logits, pred_anchor_deltas = self._transpose_dense_predictions(predictions, [self.num_classes, 4]) + anchors = self.anchor_generator(features) + gt_labels, gt_boxes = self.label_anchors(anchors, gt_instances) + return self.losses(anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes) + + def losses(self, anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes): + """ + Args: + anchors (list[Boxes]): a list of #feature level Boxes + gt_labels, gt_boxes: see output of :meth:`RetinaNet.label_anchors`. + Their shapes are (N, R) and (N, R, 4), respectively, where R is + the total number of anchors across levels, i.e. sum(Hi x Wi x Ai) + pred_logits, pred_anchor_deltas: both are list[Tensor]. Each element in the + list corresponds to one level and has shape (N, Hi * Wi * Ai, K or 4). + Where K is the number of classes used in `pred_logits`. + + Returns: + dict[str, Tensor]: + mapping from a named loss to a scalar tensor storing the loss. + Used during training only. The dict keys are: "loss_cls" and "loss_box_reg" + """ + num_images = len(gt_labels) + gt_labels = torch.stack(gt_labels) # (N, R) + + valid_mask = gt_labels >= 0 + pos_mask = (gt_labels >= 0) & (gt_labels != self.num_classes) + num_pos_anchors = pos_mask.sum().item() + get_event_storage().put_scalar("num_pos_anchors", num_pos_anchors / num_images) + normalizer = self._ema_update("loss_normalizer", max(num_pos_anchors, 1), 100) + + # classification and regression loss + gt_labels_target = F.one_hot(gt_labels[valid_mask], num_classes=self.num_classes + 1)[ + :, :-1 + ] # no loss for the last (background) class + loss_cls = sigmoid_focal_loss_jit( + cat(pred_logits, dim=1)[valid_mask], + gt_labels_target.to(pred_logits[0].dtype), + alpha=self.focal_loss_alpha, + gamma=self.focal_loss_gamma, + reduction="sum", + ) + + loss_box_reg = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + gt_boxes, + pos_mask, + box_reg_loss_type=self.box_reg_loss_type, + smooth_l1_beta=self.smooth_l1_beta, + ) + + return { + "loss_cls": loss_cls / normalizer, + "loss_box_reg": loss_box_reg / normalizer, + } + + @torch.no_grad() + def label_anchors(self, anchors, gt_instances): + """ + Args: + anchors (list[Boxes]): A list of #feature level Boxes. + The Boxes contains anchors of this image on the specific feature level. + gt_instances (list[Instances]): a list of N `Instances`s. The i-th + `Instances` contains the ground-truth per-instance annotations + for the i-th input image. + + Returns: + list[Tensor]: List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across all feature maps (sum(Hi * Wi * A)). + Label values are in {-1, 0, ..., K}, with -1 means ignore, and K means background. + + list[Tensor]: i-th element is a Rx4 tensor, where R is the total number of anchors + across feature maps. The values are the matched gt boxes for each anchor. + Values are undefined for those anchors not labeled as foreground. + """ + anchors = Boxes.cat(anchors) # Rx4 + + gt_labels = [] + matched_gt_boxes = [] + for gt_per_image in gt_instances: + match_quality_matrix = pairwise_iou(gt_per_image.gt_boxes, anchors) + matched_idxs, anchor_labels = self.anchor_matcher(match_quality_matrix) + del match_quality_matrix + + if len(gt_per_image) > 0: + matched_gt_boxes_i = gt_per_image.gt_boxes.tensor[matched_idxs] + + gt_labels_i = gt_per_image.gt_classes[matched_idxs] + # Anchors with label 0 are treated as background. + gt_labels_i[anchor_labels == 0] = self.num_classes + # Anchors with label -1 are ignored. + gt_labels_i[anchor_labels == -1] = -1 + else: + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + gt_labels_i = torch.zeros_like(matched_idxs) + self.num_classes + + gt_labels.append(gt_labels_i) + matched_gt_boxes.append(matched_gt_boxes_i) + + return gt_labels, matched_gt_boxes + + def forward_inference(self, images: ImageList, features: List[Tensor], predictions: List[List[Tensor]]): + pred_logits, pred_anchor_deltas = self._transpose_dense_predictions(predictions, [self.num_classes, 4]) + anchors = self.anchor_generator(features) + + results: List[Instances] = [] + for img_idx, image_size in enumerate(images.image_sizes): + scores_per_image = [x[img_idx].sigmoid_() for x in pred_logits] + deltas_per_image = [x[img_idx] for x in pred_anchor_deltas] + results_per_image = self.inference_single_image(anchors, scores_per_image, deltas_per_image, image_size) + results.append(results_per_image) + return results + + def inference_single_image( + self, + anchors: List[Boxes], + box_cls: List[Tensor], + box_delta: List[Tensor], + image_size: Tuple[int, int], + ): + """ + Single-image inference. Return bounding-box detection results by thresholding + on scores and applying non-maximum suppression (NMS). + + Arguments: + anchors (list[Boxes]): list of #feature levels. Each entry contains + a Boxes object, which contains all the anchors in that feature level. + box_cls (list[Tensor]): list of #feature levels. Each entry contains + tensor of size (H x W x A, K) + box_delta (list[Tensor]): Same shape as 'box_cls' except that K becomes 4. + image_size (tuple(H, W)): a tuple of the image height and width. + + Returns: + Same as `inference`, but for only one image. + """ + pred = self._decode_multi_level_predictions( + anchors, + box_cls, + box_delta, + self.test_score_thresh, + self.test_topk_candidates, + image_size, + ) + keep = batched_nms( # per-class NMS + pred.pred_boxes.tensor, pred.scores, pred.pred_classes, self.test_nms_thresh + ) + return pred[keep[: self.max_detections_per_image]] + + +class RetinaNetHead(nn.Module): + """ + The head used in RetinaNet for object classification and box regression. + It has two subnets for the two tasks, with a common structure but separate parameters. + """ + + @configurable + def __init__( + self, + *, + input_shape: List[ShapeSpec], + num_classes, + num_anchors, + conv_dims: List[int], + norm="", + prior_prob=0.01, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (List[ShapeSpec]): input shape + num_classes (int): number of classes. Used to label background proposals. + num_anchors (int): number of generated anchors + conv_dims (List[int]): dimensions for each convolution layer + norm (str or callable): + Normalization for conv layers except for the two output layers. + See :func:`detectron2.layers.get_norm` for supported types. + prior_prob (float): Prior weight for computing bias + """ + super().__init__() + + self._num_features = len(input_shape) + if norm == "BN" or norm == "SyncBN": + logger.info(f"Using domain-specific {norm} in RetinaNetHead with len={self._num_features}.") + bn_class = nn.BatchNorm2d if norm == "BN" else nn.SyncBatchNorm + + def norm(c): + return CycleBatchNormList(length=self._num_features, bn_class=bn_class, num_features=c) + + else: + norm_name = str(type(get_norm(norm, 1))) + if "BN" in norm_name: + logger.warning(f"Shared BatchNorm (type={norm_name}) may not work well in RetinaNetHead.") + + cls_subnet = [] + bbox_subnet = [] + for in_channels, out_channels in zip([input_shape[0].channels] + list(conv_dims), conv_dims): + cls_subnet.append(nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)) + if norm: + cls_subnet.append(get_norm(norm, out_channels)) + cls_subnet.append(nn.ReLU()) + bbox_subnet.append(nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)) + if norm: + bbox_subnet.append(get_norm(norm, out_channels)) + bbox_subnet.append(nn.ReLU()) + + self.cls_subnet = nn.Sequential(*cls_subnet) + self.bbox_subnet = nn.Sequential(*bbox_subnet) + self.cls_score = nn.Conv2d(conv_dims[-1], num_anchors * num_classes, kernel_size=3, stride=1, padding=1) + self.bbox_pred = nn.Conv2d(conv_dims[-1], num_anchors * 4, kernel_size=3, stride=1, padding=1) + + # Initialization + for modules in [self.cls_subnet, self.bbox_subnet, self.cls_score, self.bbox_pred]: + for layer in modules.modules(): + if isinstance(layer, nn.Conv2d): + torch.nn.init.normal_(layer.weight, mean=0, std=0.01) + torch.nn.init.constant_(layer.bias, 0) + + # Use prior in model initialization to improve stability + bias_value = -(math.log((1 - prior_prob) / prior_prob)) + torch.nn.init.constant_(self.cls_score.bias, bias_value) + + @classmethod + def from_config(cls, cfg, input_shape: List[ShapeSpec]): + num_anchors = build_anchor_generator(cfg, input_shape).num_cell_anchors + assert ( + len(set(num_anchors)) == 1 + ), "Using different number of anchors between levels is not currently supported!" + num_anchors = num_anchors[0] + + return { + "input_shape": input_shape, + "num_classes": cfg.MODEL.RETINANET.NUM_CLASSES, + "conv_dims": [input_shape[0].channels] * cfg.MODEL.RETINANET.NUM_CONVS, + "prior_prob": cfg.MODEL.RETINANET.PRIOR_PROB, + "norm": cfg.MODEL.RETINANET.NORM, + "num_anchors": num_anchors, + } + + def forward(self, features: List[Tensor]): + """ + Arguments: + features (list[Tensor]): FPN feature map tensors in high to low resolution. + Each tensor in the list correspond to different feature levels. + + Returns: + logits (list[Tensor]): #lvl tensors, each has shape (N, AxK, Hi, Wi). + The tensor predicts the classification probability + at each spatial position for each of the A anchors and K object + classes. + bbox_reg (list[Tensor]): #lvl tensors, each has shape (N, Ax4, Hi, Wi). + The tensor predicts 4-vector (dx,dy,dw,dh) box + regression values for every anchor. These values are the + relative offset between the anchor and the ground truth box. + """ + assert len(features) == self._num_features + logits = [] + bbox_reg = [] + for feature in features: + logits.append(self.cls_score(self.cls_subnet(feature))) + bbox_reg.append(self.bbox_pred(self.bbox_subnet(feature))) + return logits, bbox_reg diff --git a/detectron2/modeling/meta_arch/semantic_seg.py b/detectron2/modeling/meta_arch/semantic_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..f554babd956df3bbbd917c9851a4c428a2ab9607 --- /dev/null +++ b/detectron2/modeling/meta_arch/semantic_seg.py @@ -0,0 +1,258 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Callable, Dict, Optional, Tuple, Union + +import fvcore.nn.weight_init as weight_init +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ShapeSpec, get_norm +from detectron2.structures import ImageList +from detectron2.utils.registry import Registry + +from ..backbone import Backbone, build_backbone +from ..postprocessing import sem_seg_postprocess +from .build import META_ARCH_REGISTRY + +__all__ = [ + "SemanticSegmentor", + "SEM_SEG_HEADS_REGISTRY", + "SemSegFPNHead", + "build_sem_seg_head", +] + + +SEM_SEG_HEADS_REGISTRY = Registry("SEM_SEG_HEADS") +SEM_SEG_HEADS_REGISTRY.__doc__ = """ +Registry for semantic segmentation heads, which make semantic segmentation predictions +from feature maps. +""" + + +@META_ARCH_REGISTRY.register() +class SemanticSegmentor(nn.Module): + """ + Main class for semantic segmentation architectures. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + sem_seg_head: nn.Module, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + sem_seg_head: a module that predicts semantic segmentation from backbone features + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + """ + super().__init__() + self.backbone = backbone + self.sem_seg_head = sem_seg_head + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + sem_seg_head = build_sem_seg_head(cfg, backbone.output_shape()) + return { + "backbone": backbone, + "sem_seg_head": sem_seg_head, + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + } + + @property + def device(self): + return self.pixel_mean.device + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper`. + Each item in the list contains the inputs for one image. + + For now, each item in the list is a dict that contains: + + * "image": Tensor, image in (C, H, W) format. + * "sem_seg": semantic segmentation ground truth + * Other information that's included in the original dicts, such as: + "height", "width" (int): the output resolution of the model (may be different + from input resolution), used in inference. + + + Returns: + list[dict]: + Each dict is the output for one input image. + The dict contains one key "sem_seg" whose value is a + Tensor that represents the + per-pixel segmentation prediced by the head. + The prediction has shape KxHxW that represents the logits of + each class for each pixel. + """ + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + + features = self.backbone(images.tensor) + + if "sem_seg" in batched_inputs[0]: + targets = [x["sem_seg"].to(self.device) for x in batched_inputs] + targets = ImageList.from_tensors( + targets, + self.backbone.size_divisibility, + self.sem_seg_head.ignore_value, + self.backbone.padding_constraints, + ).tensor + else: + targets = None + results, losses = self.sem_seg_head(features, targets) + + if self.training: + return losses + + processed_results = [] + for result, input_per_image, image_size in zip(results, batched_inputs, images.image_sizes): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = sem_seg_postprocess(result, image_size, height, width) + processed_results.append({"sem_seg": r}) + return processed_results + + +def build_sem_seg_head(cfg, input_shape): + """ + Build a semantic segmentation head from `cfg.MODEL.SEM_SEG_HEAD.NAME`. + """ + name = cfg.MODEL.SEM_SEG_HEAD.NAME + return SEM_SEG_HEADS_REGISTRY.get(name)(cfg, input_shape) + + +@SEM_SEG_HEADS_REGISTRY.register() +class SemSegFPNHead(nn.Module): + """ + A semantic segmentation head described in :paper:`PanopticFPN`. + It takes a list of FPN features as input, and applies a sequence of + 3x3 convs and upsampling to scale all of them to the stride defined by + ``common_stride``. Then these features are added and used to make final + predictions by another 1x1 conv layer. + """ + + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + num_classes: int, + conv_dims: int, + common_stride: int, + loss_weight: float = 1.0, + norm: Optional[Union[str, Callable]] = None, + ignore_value: int = -1, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape: shapes (channels and stride) of the input features + num_classes: number of classes to predict + conv_dims: number of output channels for the intermediate conv layers. + common_stride: the common stride that all features will be upscaled to + loss_weight: loss weight + norm (str or callable): normalization for all conv layers + ignore_value: category id to be ignored during training. + """ + super().__init__() + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + if not len(input_shape): + raise ValueError("SemSegFPNHead(input_shape=) cannot be empty!") + self.in_features = [k for k, v in input_shape] + feature_strides = [v.stride for k, v in input_shape] + feature_channels = [v.channels for k, v in input_shape] + + self.ignore_value = ignore_value + self.common_stride = common_stride + self.loss_weight = loss_weight + + self.scale_heads = [] + for in_feature, stride, channels in zip(self.in_features, feature_strides, feature_channels): + head_ops = [] + head_length = max(1, int(np.log2(stride) - np.log2(self.common_stride))) + for k in range(head_length): + norm_module = get_norm(norm, conv_dims) + conv = Conv2d( + channels if k == 0 else conv_dims, + conv_dims, + kernel_size=3, + stride=1, + padding=1, + bias=not norm, + norm=norm_module, + activation=F.relu, + ) + weight_init.c2_msra_fill(conv) + head_ops.append(conv) + if stride != self.common_stride: + head_ops.append(nn.Upsample(scale_factor=2, mode="bilinear", align_corners=False)) + self.scale_heads.append(nn.Sequential(*head_ops)) + self.add_module(in_feature, self.scale_heads[-1]) + self.predictor = Conv2d(conv_dims, num_classes, kernel_size=1, stride=1, padding=0) + weight_init.c2_msra_fill(self.predictor) + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + return { + "input_shape": {k: v for k, v in input_shape.items() if k in cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES}, + "ignore_value": cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + "num_classes": cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, + "conv_dims": cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM, + "common_stride": cfg.MODEL.SEM_SEG_HEAD.COMMON_STRIDE, + "norm": cfg.MODEL.SEM_SEG_HEAD.NORM, + "loss_weight": cfg.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT, + } + + def forward(self, features, targets=None): + """ + Returns: + In training, returns (None, dict of losses) + In inference, returns (CxHxW logits, {}) + """ + x = self.layers(features) + if self.training: + return None, self.losses(x, targets) + else: + x = F.interpolate(x, scale_factor=self.common_stride, mode="bilinear", align_corners=False) + return x, {} + + def layers(self, features): + for i, f in enumerate(self.in_features): + if i == 0: + x = self.scale_heads[i](features[f]) + else: + x = x + self.scale_heads[i](features[f]) + x = self.predictor(x) + return x + + def losses(self, predictions, targets): + predictions = predictions.float() # https://github.com/pytorch/pytorch/issues/48163 + predictions = F.interpolate( + predictions, + scale_factor=self.common_stride, + mode="bilinear", + align_corners=False, + ) + loss = F.cross_entropy(predictions, targets, reduction="mean", ignore_index=self.ignore_value) + losses = {"loss_sem_seg": loss * self.loss_weight} + return losses diff --git a/detectron2/modeling/mmdet_wrapper.py b/detectron2/modeling/mmdet_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..ebe1a16d273f52e514236f188859420c47d15652 --- /dev/null +++ b/detectron2/modeling/mmdet_wrapper.py @@ -0,0 +1,265 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +from collections import OrderedDict +from collections.abc import Mapping +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from omegaconf import DictConfig, OmegaConf +from torch import Tensor, nn + +from detectron2.layers import ShapeSpec +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detectron2.utils.events import get_event_storage + +from .backbone import Backbone + +logger = logging.getLogger(__name__) + + +def _to_container(cfg): + """ + mmdet will assert the type of dict/list. + So convert omegaconf objects to dict/list. + """ + if isinstance(cfg, DictConfig): + cfg = OmegaConf.to_container(cfg, resolve=True) + from mmcv.utils import ConfigDict + + return ConfigDict(cfg) + + +class MMDetBackbone(Backbone): + """ + Wrapper of mmdetection backbones to use in detectron2. + + mmdet backbones produce list/tuple of tensors, while detectron2 backbones + produce a dict of tensors. This class wraps the given backbone to produce + output in detectron2's convention, so it can be used in place of detectron2 + backbones. + """ + + def __init__( + self, + backbone: Union[nn.Module, Mapping], + neck: Union[nn.Module, Mapping, None] = None, + *, + output_shapes: List[ShapeSpec], + output_names: Optional[List[str]] = None, + ): + """ + Args: + backbone: either a backbone module or a mmdet config dict that defines a + backbone. The backbone takes a 4D image tensor and returns a + sequence of tensors. + neck: either a backbone module or a mmdet config dict that defines a + neck. The neck takes outputs of backbone and returns a + sequence of tensors. If None, no neck is used. + output_shapes: shape for every output of the backbone (or neck, if given). + stride and channels are often needed. + output_names: names for every output of the backbone (or neck, if given). + By default, will use "out0", "out1", ... + """ + super().__init__() + if isinstance(backbone, Mapping): + from mmdet.models import build_backbone + + backbone = build_backbone(_to_container(backbone)) + self.backbone = backbone + + if isinstance(neck, Mapping): + from mmdet.models import build_neck + + neck = build_neck(_to_container(neck)) + self.neck = neck + + # "Neck" weights, if any, are part of neck itself. This is the interface + # of mmdet so we follow it. Reference: + # https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/two_stage.py + logger.info("Initializing mmdet backbone weights...") + self.backbone.init_weights() + # train() in mmdet modules is non-trivial, and has to be explicitly + # called. Reference: + # https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/backbones/resnet.py + self.backbone.train() + if self.neck is not None: + logger.info("Initializing mmdet neck weights ...") + if isinstance(self.neck, nn.Sequential): + for m in self.neck: + m.init_weights() + else: + self.neck.init_weights() + self.neck.train() + + self._output_shapes = output_shapes + if not output_names: + output_names = [f"out{i}" for i in range(len(output_shapes))] + self._output_names = output_names + + def forward(self, x) -> Dict[str, Tensor]: + outs = self.backbone(x) + if self.neck is not None: + outs = self.neck(outs) + assert isinstance(outs, (list, tuple)), "mmdet backbone should return a list/tuple of tensors!" + if len(outs) != len(self._output_shapes): + raise ValueError( + "Length of output_shapes does not match outputs from the mmdet backbone: " + f"{len(outs)} != {len(self._output_shapes)}" + ) + return {k: v for k, v in zip(self._output_names, outs)} + + def output_shape(self) -> Dict[str, ShapeSpec]: + return {k: v for k, v in zip(self._output_names, self._output_shapes)} + + +class MMDetDetector(nn.Module): + """ + Wrapper of a mmdetection detector model, for detection and instance segmentation. + Input/output formats of this class follow detectron2's convention, so a + mmdetection model can be trained and evaluated in detectron2. + """ + + def __init__( + self, + detector: Union[nn.Module, Mapping], + *, + # Default is 32 regardless of model: + # https://github.com/open-mmlab/mmdetection/tree/master/configs/_base_/datasets + size_divisibility=32, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + ): + """ + Args: + detector: a mmdet detector, or a mmdet config dict that defines a detector. + size_divisibility: pad input images to multiple of this number + pixel_mean: per-channel mean to normalize input image + pixel_std: per-channel stddev to normalize input image + """ + super().__init__() + if isinstance(detector, Mapping): + from mmdet.models import build_detector + + detector = build_detector(_to_container(detector)) + self.detector = detector + self.size_divisibility = size_divisibility + + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + assert ( + self.pixel_mean.shape == self.pixel_std.shape + ), f"{self.pixel_mean} and {self.pixel_std} have different shapes!" + + def forward(self, batched_inputs: List[Dict[str, torch.Tensor]]): + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, size_divisibility=self.size_divisibility).tensor + metas = [] + rescale = {"height" in x for x in batched_inputs} + if len(rescale) != 1: + raise ValueError("Some inputs have original height/width, but some don't!") + rescale = list(rescale)[0] + output_shapes = [] + for input in batched_inputs: + meta = {} + c, h, w = input["image"].shape + meta["img_shape"] = meta["ori_shape"] = (h, w, c) + if rescale: + scale_factor = np.array([w / input["width"], h / input["height"]] * 2, dtype="float32") + ori_shape = (input["height"], input["width"]) + output_shapes.append(ori_shape) + meta["ori_shape"] = ori_shape + (c,) + else: + scale_factor = 1.0 + output_shapes.append((h, w)) + meta["scale_factor"] = scale_factor + meta["flip"] = False + padh, padw = images.shape[-2:] + meta["pad_shape"] = (padh, padw, c) + metas.append(meta) + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + if gt_instances[0].has("gt_masks"): + from mmdet.core import BitmapMasks as mm_BitMasks + from mmdet.core import PolygonMasks as mm_PolygonMasks + + def convert_mask(m, shape): + # mmdet mask format + if isinstance(m, BitMasks): + return mm_BitMasks(m.tensor.cpu().numpy(), shape[0], shape[1]) + else: + return mm_PolygonMasks(m.polygons, shape[0], shape[1]) + + gt_masks = [convert_mask(x.gt_masks, x.image_size) for x in gt_instances] + losses_and_metrics = self.detector.forward_train( + images, + metas, + [x.gt_boxes.tensor for x in gt_instances], + [x.gt_classes for x in gt_instances], + gt_masks=gt_masks, + ) + else: + losses_and_metrics = self.detector.forward_train( + images, + metas, + [x.gt_boxes.tensor for x in gt_instances], + [x.gt_classes for x in gt_instances], + ) + return _parse_losses(losses_and_metrics) + else: + results = self.detector.simple_test(images, metas, rescale=rescale) + results = [{"instances": _convert_mmdet_result(r, shape)} for r, shape in zip(results, output_shapes)] + return results + + @property + def device(self): + return self.pixel_mean.device + + +# Reference: show_result() in +# https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/base.py +def _convert_mmdet_result(result, shape: Tuple[int, int]) -> Instances: + if isinstance(result, tuple): + bbox_result, segm_result = result + if isinstance(segm_result, tuple): + segm_result = segm_result[0] + else: + bbox_result, segm_result = result, None + + bboxes = torch.from_numpy(np.vstack(bbox_result)) # Nx5 + bboxes, scores = bboxes[:, :4], bboxes[:, -1] + labels = [torch.full((bbox.shape[0],), i, dtype=torch.int32) for i, bbox in enumerate(bbox_result)] + labels = torch.cat(labels) + inst = Instances(shape) + inst.pred_boxes = Boxes(bboxes) + inst.scores = scores + inst.pred_classes = labels + + if segm_result is not None and len(labels) > 0: + segm_result = list(itertools.chain(*segm_result)) + segm_result = [torch.from_numpy(x) if isinstance(x, np.ndarray) else x for x in segm_result] + segm_result = torch.stack(segm_result, dim=0) + inst.pred_masks = segm_result + return inst + + +# reference: https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/base.py +def _parse_losses(losses: Dict[str, Tensor]) -> Dict[str, Tensor]: + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError(f"{loss_name} is not a tensor or list of tensors") + + if "loss" not in loss_name: + # put metrics to storage; don't return them + storage = get_event_storage() + value = log_vars.pop(loss_name).cpu().item() + storage.put_scalar(loss_name, value) + return log_vars diff --git a/detectron2/modeling/poolers.py b/detectron2/modeling/poolers.py new file mode 100644 index 0000000000000000000000000000000000000000..07d48075fcb0db0239d9b67bed864da1b4344aeb --- /dev/null +++ b/detectron2/modeling/poolers.py @@ -0,0 +1,243 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List, Optional + +import torch +from torch import nn +from torchvision.ops import RoIPool + +from detectron2.layers import ROIAlign, ROIAlignRotated, cat, nonzero_tuple, shapes_to_tensor +from detectron2.structures import Boxes + +""" +To export ROIPooler to torchscript, in this file, variables that should be annotated with +`Union[List[Boxes], List[RotatedBoxes]]` are only annotated with `List[Boxes]`. + +TODO: Correct these annotations when torchscript support `Union`. +https://github.com/pytorch/pytorch/issues/41412 +""" + +__all__ = ["ROIPooler"] + + +def assign_boxes_to_levels( + box_lists: List[Boxes], + min_level: int, + max_level: int, + canonical_box_size: int, + canonical_level: int, +): + """ + Map each box in `box_lists` to a feature map level index and return the assignment + vector. + + Args: + box_lists (list[Boxes] | list[RotatedBoxes]): A list of N Boxes or N RotatedBoxes, + where N is the number of images in the batch. + min_level (int): Smallest feature map level index. The input is considered index 0, + the output of stage 1 is index 1, and so. + max_level (int): Largest feature map level index. + canonical_box_size (int): A canonical box size in pixels (sqrt(box area)). + canonical_level (int): The feature map level index on which a canonically-sized box + should be placed. + + Returns: + A tensor of length M, where M is the total number of boxes aggregated over all + N batch images. The memory layout corresponds to the concatenation of boxes + from all images. Each element is the feature map index, as an offset from + `self.min_level`, for the corresponding box (so value i means the box is at + `self.min_level + i`). + """ + box_sizes = torch.sqrt(cat([boxes.area() for boxes in box_lists])) + # Eqn.(1) in FPN paper + level_assignments = torch.floor(canonical_level + torch.log2(box_sizes / canonical_box_size + 1e-8)) + # clamp level to (min, max), in case the box size is too large or too small + # for the available feature maps + level_assignments = torch.clamp(level_assignments, min=min_level, max=max_level) + return level_assignments.to(torch.int64) - min_level + + +# script the module to avoid hardcoded device type +@torch.jit.script_if_tracing +def _convert_boxes_to_pooler_format(boxes: torch.Tensor, sizes: torch.Tensor) -> torch.Tensor: + sizes = sizes.to(device=boxes.device) + indices = torch.repeat_interleave(torch.arange(len(sizes), dtype=boxes.dtype, device=boxes.device), sizes) + return cat([indices[:, None], boxes], dim=1) + + +def convert_boxes_to_pooler_format(box_lists: List[Boxes]): + """ + Convert all boxes in `box_lists` to the low-level format used by ROI pooling ops + (see description under Returns). + + Args: + box_lists (list[Boxes] | list[RotatedBoxes]): + A list of N Boxes or N RotatedBoxes, where N is the number of images in the batch. + + Returns: + When input is list[Boxes]: + A tensor of shape (M, 5), where M is the total number of boxes aggregated over all + N batch images. + The 5 columns are (batch index, x0, y0, x1, y1), where batch index + is the index in [0, N) identifying which batch image the box with corners at + (x0, y0, x1, y1) comes from. + When input is list[RotatedBoxes]: + A tensor of shape (M, 6), where M is the total number of boxes aggregated over all + N batch images. + The 6 columns are (batch index, x_ctr, y_ctr, width, height, angle_degrees), + where batch index is the index in [0, N) identifying which batch image the + rotated box (x_ctr, y_ctr, width, height, angle_degrees) comes from. + """ + boxes = torch.cat([x.tensor for x in box_lists], dim=0) + # __len__ returns Tensor in tracing. + sizes = shapes_to_tensor([x.__len__() for x in box_lists]) + return _convert_boxes_to_pooler_format(boxes, sizes) + + +@torch.jit.script_if_tracing +def _create_zeros( + batch_target: Optional[torch.Tensor], + channels: int, + height: int, + width: int, + like_tensor: torch.Tensor, +) -> torch.Tensor: + batches = batch_target.shape[0] if batch_target is not None else 0 + sizes = (batches, channels, height, width) + return torch.zeros(sizes, dtype=like_tensor.dtype, device=like_tensor.device) + + +class ROIPooler(nn.Module): + """ + Region of interest feature map pooler that supports pooling from one or more + feature maps. + """ + + def __init__( + self, + output_size, + scales, + sampling_ratio, + pooler_type, + canonical_box_size=224, + canonical_level=4, + ): + """ + Args: + output_size (int, tuple[int] or list[int]): output size of the pooled region, + e.g., 14 x 14. If tuple or list is given, the length must be 2. + scales (list[float]): The scale for each low-level pooling op relative to + the input image. For a feature map with stride s relative to the input + image, scale is defined as 1/s. The stride must be power of 2. + When there are multiple scales, they must form a pyramid, i.e. they must be + a monotically decreasing geometric sequence with a factor of 1/2. + sampling_ratio (int): The `sampling_ratio` parameter for the ROIAlign op. + pooler_type (string): Name of the type of pooling operation that should be applied. + For instance, "ROIPool" or "ROIAlignV2". + canonical_box_size (int): A canonical box size in pixels (sqrt(box area)). The default + is heuristically defined as 224 pixels in the FPN paper (based on ImageNet + pre-training). + canonical_level (int): The feature map level index from which a canonically-sized box + should be placed. The default is defined as level 4 (stride=16) in the FPN paper, + i.e., a box of size 224x224 will be placed on the feature with stride=16. + The box placement for all boxes will be determined from their sizes w.r.t + canonical_box_size. For example, a box whose area is 4x that of a canonical box + should be used to pool features from feature level ``canonical_level+1``. + + Note that the actual input feature maps given to this module may not have + sufficiently many levels for the input boxes. If the boxes are too large or too + small for the input feature maps, the closest level will be used. + """ + super().__init__() + + if isinstance(output_size, int): + output_size = (output_size, output_size) + assert len(output_size) == 2 + assert isinstance(output_size[0], int) and isinstance(output_size[1], int) + self.output_size = output_size + + if pooler_type == "ROIAlign": + self.level_poolers = nn.ModuleList( + ROIAlign(output_size, spatial_scale=scale, sampling_ratio=sampling_ratio, aligned=False) + for scale in scales + ) + elif pooler_type == "ROIAlignV2": + self.level_poolers = nn.ModuleList( + ROIAlign(output_size, spatial_scale=scale, sampling_ratio=sampling_ratio, aligned=True) + for scale in scales + ) + elif pooler_type == "ROIPool": + self.level_poolers = nn.ModuleList(RoIPool(output_size, spatial_scale=scale) for scale in scales) + elif pooler_type == "ROIAlignRotated": + self.level_poolers = nn.ModuleList( + ROIAlignRotated(output_size, spatial_scale=scale, sampling_ratio=sampling_ratio) for scale in scales + ) + else: + raise ValueError("Unknown pooler type: {}".format(pooler_type)) + + # Map scale (defined as 1 / stride) to its feature map level under the + # assumption that stride is a power of 2. + min_level = -(math.log2(scales[0])) + max_level = -(math.log2(scales[-1])) + assert math.isclose(min_level, int(min_level)) and math.isclose( + max_level, int(max_level) + ), "Featuremap stride is not power of 2!" + self.min_level = int(min_level) + self.max_level = int(max_level) + assert ( + len(scales) == self.max_level - self.min_level + 1 + ), "[ROIPooler] Sizes of input featuremaps do not form a pyramid!" + assert 0 <= self.min_level and self.min_level <= self.max_level + self.canonical_level = canonical_level + assert canonical_box_size > 0 + self.canonical_box_size = canonical_box_size + + def forward(self, x: List[torch.Tensor], box_lists: List[Boxes]): + """ + Args: + x (list[Tensor]): A list of feature maps of NCHW shape, with scales matching those + used to construct this module. + box_lists (list[Boxes] | list[RotatedBoxes]): + A list of N Boxes or N RotatedBoxes, where N is the number of images in the batch. + The box coordinates are defined on the original image and + will be scaled by the `scales` argument of :class:`ROIPooler`. + + Returns: + Tensor: + A tensor of shape (M, C, output_size, output_size) where M is the total number of + boxes aggregated over all N batch images and C is the number of channels in `x`. + """ + num_level_assignments = len(self.level_poolers) + + assert isinstance(x, list) and isinstance(box_lists, list), "Arguments to pooler must be lists" + assert ( + len(x) == num_level_assignments + ), "unequal value, num_level_assignments={}, but x is list of {} Tensors".format(num_level_assignments, len(x)) + + assert len(box_lists) == x[0].size( + 0 + ), "unequal value, x[0] batch dim 0 is {}, but box_list has length {}".format(x[0].size(0), len(box_lists)) + if len(box_lists) == 0: + return _create_zeros(None, x[0].shape[1], *self.output_size, x[0]) + + pooler_fmt_boxes = convert_boxes_to_pooler_format(box_lists) + + if num_level_assignments == 1: + return self.level_poolers[0](x[0], pooler_fmt_boxes) + + level_assignments = assign_boxes_to_levels( + box_lists, self.min_level, self.max_level, self.canonical_box_size, self.canonical_level + ) + + num_channels = x[0].shape[1] + output_size = self.output_size[0] + + output = _create_zeros(pooler_fmt_boxes, num_channels, output_size, output_size, x[0]) + + for level, pooler in enumerate(self.level_poolers): + inds = nonzero_tuple(level_assignments == level)[0] + pooler_fmt_boxes_level = pooler_fmt_boxes[inds] + # Use index_put_ instead of advance indexing, to avoid pytorch/issues/49852 + output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level)) + + return output diff --git a/detectron2/modeling/postprocessing.py b/detectron2/modeling/postprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b42a2d119f6143b0f92ef30156ed39a01b03eb --- /dev/null +++ b/detectron2/modeling/postprocessing.py @@ -0,0 +1,96 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +from torch.nn import functional as F + +from detectron2.structures import Instances, ROIMasks + + +# perhaps should rename to "resize_instance" +def detector_postprocess(results: Instances, output_height: int, output_width: int, mask_threshold: float = 0.5): + """ + Resize the output instances. + The input images are often resized when entering an object detector. + As a result, we often need the outputs of the detector in a different + resolution from its inputs. + + This function will resize the raw outputs of an R-CNN detector + to produce outputs according to the desired output resolution. + + Args: + results (Instances): the raw outputs from the detector. + `results.image_size` contains the input image resolution the detector sees. + This object might be modified in-place. + output_height, output_width: the desired output resolution. + Returns: + Instances: the resized output from the model, based on the output resolution + """ + if isinstance(output_width, torch.Tensor): + # This shape might (but not necessarily) be tensors during tracing. + # Converts integer tensors to float temporaries to ensure true + # division is performed when computing scale_x and scale_y. + output_width_tmp = output_width.float() + output_height_tmp = output_height.float() + new_size = torch.stack([output_height, output_width]) + else: + new_size = (output_height, output_width) + output_width_tmp = output_width + output_height_tmp = output_height + + scale_x, scale_y = ( + output_width_tmp / results.image_size[1], + output_height_tmp / results.image_size[0], + ) + results = Instances(new_size, **results.get_fields()) + + if results.has("pred_boxes"): + output_boxes = results.pred_boxes + elif results.has("proposal_boxes"): + output_boxes = results.proposal_boxes + else: + output_boxes = None + assert output_boxes is not None, "Predictions must contain boxes!" + + output_boxes.scale(scale_x, scale_y) + output_boxes.clip(results.image_size) + + results = results[output_boxes.nonempty()] + + if results.has("pred_masks"): + if isinstance(results.pred_masks, ROIMasks): + roi_masks = results.pred_masks + else: + # pred_masks is a tensor of shape (N, 1, M, M) + roi_masks = ROIMasks(results.pred_masks[:, 0, :, :]) + results.pred_masks = roi_masks.to_bitmasks( + results.pred_boxes, output_height, output_width, mask_threshold + ).tensor # TODO return ROIMasks/BitMask object in the future + + if results.has("pred_keypoints"): + results.pred_keypoints[:, :, 0] *= scale_x + results.pred_keypoints[:, :, 1] *= scale_y + + return results + + +def sem_seg_postprocess(result, img_size, output_height, output_width): + """ + Return semantic segmentation predictions in the original resolution. + + The input images are often resized when entering semantic segmentor. Moreover, in same + cases, they also padded inside segmentor to be divisible by maximum network stride. + As a result, we often need the predictions of the segmentor in a different + resolution from its inputs. + + Args: + result (Tensor): semantic segmentation prediction logits. A tensor of shape (C, H, W), + where C is the number of classes, and H, W are the height and width of the prediction. + img_size (tuple): image size that segmentor is taking as input. + output_height, output_width: the desired output resolution. + + Returns: + semantic segmentation prediction (Tensor): A tensor of the shape + (C, output_height, output_width) that contains per-pixel soft predictions. + """ + result = result[:, : img_size[0], : img_size[1]].expand(1, -1, -1, -1) + result = F.interpolate(result, size=(output_height, output_width), mode="bilinear", align_corners=False)[0] + return result diff --git a/detectron2/modeling/proposal_generator/__init__.py b/detectron2/modeling/proposal_generator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..638cfa844542d3ca253978606ac17190192e0c9b --- /dev/null +++ b/detectron2/modeling/proposal_generator/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build import PROPOSAL_GENERATOR_REGISTRY, build_proposal_generator +from .rpn import RPN, RPN_HEAD_REGISTRY, StandardRPNHead, build_rpn_head + +__all__ = list(globals().keys()) diff --git a/detectron2/modeling/proposal_generator/build.py b/detectron2/modeling/proposal_generator/build.py new file mode 100644 index 0000000000000000000000000000000000000000..34eb12d00d94ff905b796e75e2c4c5845257c8e9 --- /dev/null +++ b/detectron2/modeling/proposal_generator/build.py @@ -0,0 +1,24 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from detectron2.utils.registry import Registry + +PROPOSAL_GENERATOR_REGISTRY = Registry("PROPOSAL_GENERATOR") +PROPOSAL_GENERATOR_REGISTRY.__doc__ = """ +Registry for proposal generator, which produces object proposals from feature maps. + +The registered object will be called with `obj(cfg, input_shape)`. +The call should return a `nn.Module` object. +""" + +from . import rpn, rrpn # noqa F401 isort:skip + + +def build_proposal_generator(cfg, input_shape): + """ + Build a proposal generator from `cfg.MODEL.PROPOSAL_GENERATOR.NAME`. + The name can be "PrecomputedProposals" to use no proposal generator. + """ + name = cfg.MODEL.PROPOSAL_GENERATOR.NAME + if name == "PrecomputedProposals": + return None + + return PROPOSAL_GENERATOR_REGISTRY.get(name)(cfg, input_shape) diff --git a/detectron2/modeling/proposal_generator/proposal_utils.py b/detectron2/modeling/proposal_generator/proposal_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..11ea84b600df06219eb7bc7a55ee23d7894bd0dd --- /dev/null +++ b/detectron2/modeling/proposal_generator/proposal_utils.py @@ -0,0 +1,195 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import math +from typing import List, Tuple, Union + +import torch + +from detectron2.layers import batched_nms, cat, move_device_like +from detectron2.structures import Boxes, Instances + +logger = logging.getLogger(__name__) + + +def _is_tracing(): + # (fixed in TORCH_VERSION >= 1.9) + if torch.jit.is_scripting(): + # https://github.com/pytorch/pytorch/issues/47379 + return False + else: + return torch.jit.is_tracing() + + +def find_top_rpn_proposals( + proposals: List[torch.Tensor], + pred_objectness_logits: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + nms_thresh: float, + pre_nms_topk: int, + post_nms_topk: int, + min_box_size: float, + training: bool, +): + """ + For each feature map, select the `pre_nms_topk` highest scoring proposals, + apply NMS, clip proposals, and remove small boxes. Return the `post_nms_topk` + highest scoring proposals among all the feature maps for each image. + + Args: + proposals (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A, 4). + All proposal predictions on the feature maps. + pred_objectness_logits (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A). + image_sizes (list[tuple]): sizes (h, w) for each image + nms_thresh (float): IoU threshold to use for NMS + pre_nms_topk (int): number of top k scoring proposals to keep before applying NMS. + When RPN is run on multiple feature maps (as in FPN) this number is per + feature map. + post_nms_topk (int): number of top k scoring proposals to keep after applying NMS. + When RPN is run on multiple feature maps (as in FPN) this number is total, + over all feature maps. + min_box_size (float): minimum proposal box side length in pixels (absolute units + wrt input images). + training (bool): True if proposals are to be used in training, otherwise False. + This arg exists only to support a legacy bug; look for the "NB: Legacy bug ..." + comment. + + Returns: + list[Instances]: list of N Instances. The i-th Instances + stores post_nms_topk object proposals for image i, sorted by their + objectness score in descending order. + """ + num_images = len(image_sizes) + device = ( + proposals[0].device if torch.jit.is_scripting() else ("cpu" if torch.jit.is_tracing() else proposals[0].device) + ) + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + level_ids = [] # #lvl Tensor, each of shape (topk,) + batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) + for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + level_ids.append( + move_device_like( + torch.full((num_proposals_i,), level_id, dtype=torch.int64, device=device), + proposals[0], + ) + ) + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + level_ids = cat(level_ids, dim=0) + + # 3. For each image, run a per-level NMS, and choose topk results. + results: List[Instances] = [] + for n, image_size in enumerate(image_sizes): + boxes = Boxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + lvl = level_ids + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError("Predicted boxes or scores contain Inf/NaN. Training has diverged.") + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + lvl = lvl[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img, lvl = boxes[keep], scores_per_img[keep], lvl[keep] + + keep = batched_nms(boxes.tensor, scores_per_img, lvl, nms_thresh) + # In Detectron1, there was different behavior during training vs. testing. + # (https://github.com/facebookresearch/Detectron/issues/459) + # During training, topk is over the proposals from *all* images in the training batch. + # During testing, it is over the proposals for each image separately. + # As a result, the training behavior becomes batch-dependent, + # and the configuration "POST_NMS_TOPK_TRAIN" end up relying on the batch size. + # This bug is addressed in Detectron2 to make the behavior independent of batch size. + keep = keep[:post_nms_topk] # keep is already sorted + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results + + +def add_ground_truth_to_proposals( + gt: Union[List[Instances], List[Boxes]], proposals: List[Instances] +) -> List[Instances]: + """ + Call `add_ground_truth_to_proposals_single_image` for all images. + + Args: + gt(Union[List[Instances], List[Boxes]): list of N elements. Element i is a Instances + representing the ground-truth for image i. + proposals (list[Instances]): list of N elements. Element i is a Instances + representing the proposals for image i. + + Returns: + list[Instances]: list of N Instances. Each is the proposals for the image, + with field "proposal_boxes" and "objectness_logits". + """ + assert gt is not None + + if len(proposals) != len(gt): + raise ValueError("proposals and gt should have the same length as the number of images!") + if len(proposals) == 0: + return proposals + + return [add_ground_truth_to_proposals_single_image(gt_i, proposals_i) for gt_i, proposals_i in zip(gt, proposals)] + + +def add_ground_truth_to_proposals_single_image(gt: Union[Instances, Boxes], proposals: Instances) -> Instances: + """ + Augment `proposals` with `gt`. + + Args: + Same as `add_ground_truth_to_proposals`, but with gt and proposals + per image. + + Returns: + Same as `add_ground_truth_to_proposals`, but for only one image. + """ + if isinstance(gt, Boxes): + # convert Boxes to Instances + gt = Instances(proposals.image_size, gt_boxes=gt) + + gt_boxes = gt.gt_boxes + device = proposals.objectness_logits.device + # Assign all ground-truth boxes an objectness logit corresponding to + # P(object) = sigmoid(logit) =~ 1. + gt_logit_value = math.log((1.0 - 1e-10) / (1 - (1.0 - 1e-10))) + gt_logits = gt_logit_value * torch.ones(len(gt_boxes), device=device) + + # Concatenating gt_boxes with proposals requires them to have the same fields + gt_proposal = Instances(proposals.image_size, **gt.get_fields()) + gt_proposal.proposal_boxes = gt_boxes + gt_proposal.objectness_logits = gt_logits + + for key in proposals.get_fields().keys(): + assert gt_proposal.has(key), "The attribute '{}' in `proposals` does not exist in `gt`".format(key) + + # NOTE: Instances.cat only use fields from the first item. Extra fields in latter items + # will be thrown away. + new_proposals = Instances.cat([proposals, gt_proposal]) + + return new_proposals diff --git a/detectron2/modeling/proposal_generator/rpn.py b/detectron2/modeling/proposal_generator/rpn.py new file mode 100644 index 0000000000000000000000000000000000000000..e326d20ccadf16751d0f17c8a1cd29412367444d --- /dev/null +++ b/detectron2/modeling/proposal_generator/rpn.py @@ -0,0 +1,522 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ShapeSpec, cat +from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage +from detectron2.utils.memory import retry_if_cuda_oom +from detectron2.utils.registry import Registry + +from ..anchor_generator import build_anchor_generator +from ..box_regression import Box2BoxTransform, _dense_box_regression_loss +from ..matcher import Matcher +from ..sampling import subsample_labels +from .build import PROPOSAL_GENERATOR_REGISTRY +from .proposal_utils import find_top_rpn_proposals + +RPN_HEAD_REGISTRY = Registry("RPN_HEAD") +RPN_HEAD_REGISTRY.__doc__ = """ +Registry for RPN heads, which take feature maps and perform +objectness classification and bounding box regression for anchors. + +The registered object will be called with `obj(cfg, input_shape)`. +The call should return a `nn.Module` object. +""" + + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + L: number of feature maps per image on which RPN is run + A: number of cell anchors (must be the same for all feature maps) + Hi, Wi: height and width of the i-th feature map + B: size of the box parameterization + +Naming convention: + + objectness: refers to the binary classification of an anchor as object vs. not object. + + deltas: refers to the 4-d (dx, dy, dw, dh) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransform`), or 5d for rotated boxes. + + pred_objectness_logits: predicted objectness scores in [-inf, +inf]; use + sigmoid(pred_objectness_logits) to estimate P(object). + + gt_labels: ground-truth binary classification labels for objectness + + pred_anchor_deltas: predicted box2box transform deltas + + gt_anchor_deltas: ground-truth box2box transform deltas +""" + + +def build_rpn_head(cfg, input_shape): + """ + Build an RPN head defined by `cfg.MODEL.RPN.HEAD_NAME`. + """ + name = cfg.MODEL.RPN.HEAD_NAME + return RPN_HEAD_REGISTRY.get(name)(cfg, input_shape) + + +@RPN_HEAD_REGISTRY.register() +class StandardRPNHead(nn.Module): + """ + Standard RPN classification and regression heads described in :paper:`Faster R-CNN`. + Uses a 3x3 conv to produce a shared hidden state from which one 1x1 conv predicts + objectness logits for each anchor and a second 1x1 conv predicts bounding-box deltas + specifying how to deform each anchor into an object proposal. + """ + + @configurable + def __init__(self, *, in_channels: int, num_anchors: int, box_dim: int = 4, conv_dims: List[int] = (-1,)): + """ + NOTE: this interface is experimental. + + Args: + in_channels (int): number of input feature channels. When using multiple + input features, they must have the same number of channels. + num_anchors (int): number of anchors to predict for *each spatial position* + on the feature map. The total number of anchors for each + feature map will be `num_anchors * H * W`. + box_dim (int): dimension of a box, which is also the number of box regression + predictions to make for each anchor. An axis aligned box has + box_dim=4, while a rotated box has box_dim=5. + conv_dims (list[int]): a list of integers representing the output channels + of N conv layers. Set it to -1 to use the same number of output channels + as input channels. + """ + super().__init__() + cur_channels = in_channels + # Keeping the old variable names and structure for backwards compatiblity. + # Otherwise the old checkpoints will fail to load. + if len(conv_dims) == 1: + out_channels = cur_channels if conv_dims[0] == -1 else conv_dims[0] + # 3x3 conv for the hidden representation + self.conv = self._get_rpn_conv(cur_channels, out_channels) + cur_channels = out_channels + else: + self.conv = nn.Sequential() + for k, conv_dim in enumerate(conv_dims): + out_channels = cur_channels if conv_dim == -1 else conv_dim + if out_channels <= 0: + raise ValueError(f"Conv output channels should be greater than 0. Got {out_channels}") + conv = self._get_rpn_conv(cur_channels, out_channels) + self.conv.add_module(f"conv{k}", conv) + cur_channels = out_channels + # 1x1 conv for predicting objectness logits + self.objectness_logits = nn.Conv2d(cur_channels, num_anchors, kernel_size=1, stride=1) + # 1x1 conv for predicting box2box transform deltas + self.anchor_deltas = nn.Conv2d(cur_channels, num_anchors * box_dim, kernel_size=1, stride=1) + + # Keeping the order of weights initialization same for backwards compatiblility. + for layer in self.modules(): + if isinstance(layer, nn.Conv2d): + nn.init.normal_(layer.weight, std=0.01) + nn.init.constant_(layer.bias, 0) + + def _get_rpn_conv(self, in_channels, out_channels): + return Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + activation=nn.ReLU(), + ) + + @classmethod + def from_config(cls, cfg, input_shape): + # Standard RPN is shared across levels: + in_channels = [s.channels for s in input_shape] + assert len(set(in_channels)) == 1, "Each level must have the same channel!" + in_channels = in_channels[0] + + # RPNHead should take the same input as anchor generator + # NOTE: it assumes that creating an anchor generator does not have unwanted side effect. + anchor_generator = build_anchor_generator(cfg, input_shape) + num_anchors = anchor_generator.num_anchors + box_dim = anchor_generator.box_dim + assert len(set(num_anchors)) == 1, "Each level must have the same number of anchors per spatial position" + return { + "in_channels": in_channels, + "num_anchors": num_anchors[0], + "box_dim": box_dim, + "conv_dims": cfg.MODEL.RPN.CONV_DIMS, + } + + def forward(self, features: List[torch.Tensor]): + """ + Args: + features (list[Tensor]): list of feature maps + + Returns: + list[Tensor]: A list of L elements. + Element i is a tensor of shape (N, A, Hi, Wi) representing + the predicted objectness logits for all anchors. A is the number of cell anchors. + list[Tensor]: A list of L elements. Element i is a tensor of shape + (N, A*box_dim, Hi, Wi) representing the predicted "deltas" used to transform anchors + to proposals. + """ + pred_objectness_logits = [] + pred_anchor_deltas = [] + for x in features: + t = self.conv(x) + pred_objectness_logits.append(self.objectness_logits(t)) + pred_anchor_deltas.append(self.anchor_deltas(t)) + return pred_objectness_logits, pred_anchor_deltas + + +@PROPOSAL_GENERATOR_REGISTRY.register() +class RPN(nn.Module): + """ + Region Proposal Network, introduced by :paper:`Faster R-CNN`. + """ + + @configurable + def __init__( + self, + *, + in_features: List[str], + head: nn.Module, + anchor_generator: nn.Module, + anchor_matcher: Matcher, + box2box_transform: Box2BoxTransform, + batch_size_per_image: int, + positive_fraction: float, + pre_nms_topk: Tuple[float, float], + post_nms_topk: Tuple[float, float], + nms_thresh: float = 0.7, + min_box_size: float = 0.0, + anchor_boundary_thresh: float = -1.0, + loss_weight: Union[float, Dict[str, float]] = 1.0, + box_reg_loss_type: str = "smooth_l1", + smooth_l1_beta: float = 0.0, + ): + """ + NOTE: this interface is experimental. + + Args: + in_features (list[str]): list of names of input features to use + head (nn.Module): a module that predicts logits and regression deltas + for each level from a list of per-level features + anchor_generator (nn.Module): a module that creates anchors from a + list of features. Usually an instance of :class:`AnchorGenerator` + anchor_matcher (Matcher): label the anchors by matching them with ground truth. + box2box_transform (Box2BoxTransform): defines the transform from anchors boxes to + instance boxes + batch_size_per_image (int): number of anchors per image to sample for training + positive_fraction (float): fraction of foreground anchors to sample for training + pre_nms_topk (tuple[float]): (train, test) that represents the + number of top k proposals to select before NMS, in + training and testing. + post_nms_topk (tuple[float]): (train, test) that represents the + number of top k proposals to select after NMS, in + training and testing. + nms_thresh (float): NMS threshold used to de-duplicate the predicted proposals + min_box_size (float): remove proposal boxes with any side smaller than this threshold, + in the unit of input image pixels + anchor_boundary_thresh (float): legacy option + loss_weight (float|dict): weights to use for losses. Can be single float for weighting + all rpn losses together, or a dict of individual weightings. Valid dict keys are: + "loss_rpn_cls" - applied to classification loss + "loss_rpn_loc" - applied to box regression loss + box_reg_loss_type (str): Loss type to use. Supported losses: "smooth_l1", "giou". + smooth_l1_beta (float): beta parameter for the smooth L1 regression loss. Default to + use L1 loss. Only used when `box_reg_loss_type` is "smooth_l1" + """ + super().__init__() + self.in_features = in_features + self.rpn_head = head + self.anchor_generator = anchor_generator + self.anchor_matcher = anchor_matcher + self.box2box_transform = box2box_transform + self.batch_size_per_image = batch_size_per_image + self.positive_fraction = positive_fraction + # Map from self.training state to train/test settings + self.pre_nms_topk = {True: pre_nms_topk[0], False: pre_nms_topk[1]} + self.post_nms_topk = {True: post_nms_topk[0], False: post_nms_topk[1]} + self.nms_thresh = nms_thresh + self.min_box_size = float(min_box_size) + self.anchor_boundary_thresh = anchor_boundary_thresh + if isinstance(loss_weight, float): + loss_weight = {"loss_rpn_cls": loss_weight, "loss_rpn_loc": loss_weight} + self.loss_weight = loss_weight + self.box_reg_loss_type = box_reg_loss_type + self.smooth_l1_beta = smooth_l1_beta + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + in_features = cfg.MODEL.RPN.IN_FEATURES + ret = { + "in_features": in_features, + "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, + "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, + "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, + "loss_weight": { + "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, + "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, + }, + "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), + "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, + "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, + } + + ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) + ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) + + ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) + ret["anchor_matcher"] = Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True + ) + ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) + return ret + + def _subsample_labels(self, label): + """ + Randomly sample a subset of positive and negative examples, and overwrite + the label vector to the ignore value (-1) for all elements that are not + included in the sample. + + Args: + labels (Tensor): a vector of -1, 0, 1. Will be modified in-place and returned. + """ + pos_idx, neg_idx = subsample_labels(label, self.batch_size_per_image, self.positive_fraction, 0) + # Fill with the ignore label (-1), then set positive and negative labels + label.fill_(-1) + label.scatter_(0, pos_idx, 1) + label.scatter_(0, neg_idx, 0) + return label + + @torch.jit.unused + @torch.no_grad() + def label_and_sample_anchors( + self, anchors: List[Boxes], gt_instances: List[Instances] + ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """ + Args: + anchors (list[Boxes]): anchors for each feature map. + gt_instances: the ground-truth instances for each image. + + Returns: + list[Tensor]: + List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across all feature maps R = sum(Hi * Wi * A). + Label values are in {-1, 0, 1}, with meanings: -1 = ignore; 0 = negative + class; 1 = positive class. + list[Tensor]: + i-th element is a Rx4 tensor. The values are the matched gt boxes for each + anchor. Values are undefined for those anchors not labeled as 1. + """ + anchors = Boxes.cat(anchors) + + gt_boxes = [x.gt_boxes for x in gt_instances] + image_sizes = [x.image_size for x in gt_instances] + del gt_instances + + gt_labels = [] + matched_gt_boxes = [] + for image_size_i, gt_boxes_i in zip(image_sizes, gt_boxes): + """ + image_size_i: (h, w) for the i-th image + gt_boxes_i: ground-truth boxes for i-th image + """ + + match_quality_matrix = retry_if_cuda_oom(pairwise_iou)(gt_boxes_i, anchors) + matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix) + # Matching is memory-expensive and may result in CPU tensors. But the result is small + gt_labels_i = gt_labels_i.to(device=gt_boxes_i.device) + del match_quality_matrix + + if self.anchor_boundary_thresh >= 0: + # Discard anchors that go out of the boundaries of the image + # NOTE: This is legacy functionality that is turned off by default in Detectron2 + anchors_inside_image = anchors.inside_box(image_size_i, self.anchor_boundary_thresh) + gt_labels_i[~anchors_inside_image] = -1 + + # A vector of labels (-1, 0, 1) for each anchor + gt_labels_i = self._subsample_labels(gt_labels_i) + + if len(gt_boxes_i) == 0: + # These values won't be used anyway since the anchor is labeled as background + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + else: + # TODO wasted indexing computation for ignored boxes + matched_gt_boxes_i = gt_boxes_i[matched_idxs].tensor + + gt_labels.append(gt_labels_i) # N,AHW + matched_gt_boxes.append(matched_gt_boxes_i) + return gt_labels, matched_gt_boxes + + @torch.jit.unused + def losses( + self, + anchors: List[Boxes], + pred_objectness_logits: List[torch.Tensor], + gt_labels: List[torch.Tensor], + pred_anchor_deltas: List[torch.Tensor], + gt_boxes: List[torch.Tensor], + ) -> Dict[str, torch.Tensor]: + """ + Return the losses from a set of RPN predictions and their associated ground-truth. + + Args: + anchors (list[Boxes or RotatedBoxes]): anchors for each feature map, each + has shape (Hi*Wi*A, B), where B is box dimension (4 or 5). + pred_objectness_logits (list[Tensor]): A list of L elements. + Element i is a tensor of shape (N, Hi*Wi*A) representing + the predicted objectness logits for all anchors. + gt_labels (list[Tensor]): Output of :meth:`label_and_sample_anchors`. + pred_anchor_deltas (list[Tensor]): A list of L elements. Element i is a tensor of shape + (N, Hi*Wi*A, 4 or 5) representing the predicted "deltas" used to transform anchors + to proposals. + gt_boxes (list[Tensor]): Output of :meth:`label_and_sample_anchors`. + + Returns: + dict[loss name -> loss value]: A dict mapping from loss name to loss value. + Loss names are: `loss_rpn_cls` for objectness classification and + `loss_rpn_loc` for proposal localization. + """ + num_images = len(gt_labels) + gt_labels = torch.stack(gt_labels) # (N, sum(Hi*Wi*Ai)) + + # Log the number of positive/negative anchors per-image that's used in training + pos_mask = gt_labels == 1 + num_pos_anchors = pos_mask.sum().item() + num_neg_anchors = (gt_labels == 0).sum().item() + storage = get_event_storage() + storage.put_scalar("rpn/num_pos_anchors", num_pos_anchors / num_images) + storage.put_scalar("rpn/num_neg_anchors", num_neg_anchors / num_images) + + localization_loss = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + gt_boxes, + pos_mask, + box_reg_loss_type=self.box_reg_loss_type, + smooth_l1_beta=self.smooth_l1_beta, + ) + + valid_mask = gt_labels >= 0 + objectness_loss = F.binary_cross_entropy_with_logits( + cat(pred_objectness_logits, dim=1)[valid_mask], + gt_labels[valid_mask].to(torch.float32), + reduction="sum", + ) + normalizer = self.batch_size_per_image * num_images + losses = { + "loss_rpn_cls": objectness_loss / normalizer, + # The original Faster R-CNN paper uses a slightly different normalizer + # for loc loss. But it doesn't matter in practice + "loss_rpn_loc": localization_loss / normalizer, + } + losses = {k: v * self.loss_weight.get(k, 1.0) for k, v in losses.items()} + return losses + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + gt_instances: Optional[List[Instances]] = None, + ): + """ + Args: + images (ImageList): input images of length `N` + features (dict[str, Tensor]): input data as a mapping from feature + map name to tensor. Axis 0 represents the number of images `N` in + the input data; axes 1-3 are channels, height, and width, which may + vary between feature maps (e.g., if a feature pyramid is used). + gt_instances (list[Instances], optional): a length `N` list of `Instances`s. + Each `Instances` stores ground-truth instances for the corresponding image. + + Returns: + proposals: list[Instances]: contains fields "proposal_boxes", "objectness_logits" + loss: dict[Tensor] or None + """ + features = [features[f] for f in self.in_features] + anchors = self.anchor_generator(features) + + pred_objectness_logits, pred_anchor_deltas = self.rpn_head(features) + # Transpose the Hi*Wi*A dimension to the middle: + pred_objectness_logits = [ + # (N, A, Hi, Wi) -> (N, Hi, Wi, A) -> (N, Hi*Wi*A) + score.permute(0, 2, 3, 1).flatten(1) + for score in pred_objectness_logits + ] + pred_anchor_deltas = [ + # (N, A*B, Hi, Wi) -> (N, A, B, Hi, Wi) -> (N, Hi, Wi, A, B) -> (N, Hi*Wi*A, B) + x.view(x.shape[0], -1, self.anchor_generator.box_dim, x.shape[-2], x.shape[-1]) + .permute(0, 3, 4, 1, 2) + .flatten(1, -2) + for x in pred_anchor_deltas + ] + + if self.training: + assert gt_instances is not None, "RPN requires gt_instances in training!" + gt_labels, gt_boxes = self.label_and_sample_anchors(anchors, gt_instances) + losses = self.losses(anchors, pred_objectness_logits, gt_labels, pred_anchor_deltas, gt_boxes) + else: + losses = {} + proposals = self.predict_proposals(anchors, pred_objectness_logits, pred_anchor_deltas, images.image_sizes) + return proposals, losses + + def predict_proposals( + self, + anchors: List[Boxes], + pred_objectness_logits: List[torch.Tensor], + pred_anchor_deltas: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + ): + """ + Decode all the predicted box regression deltas to proposals. Find the top proposals + by applying NMS and removing boxes that are too small. + + Returns: + proposals (list[Instances]): list of N Instances. The i-th Instances + stores post_nms_topk object proposals for image i, sorted by their + objectness score in descending order. + """ + # The proposals are treated as fixed for joint training with roi heads. + # This approach ignores the derivative w.r.t. the proposal boxes’ coordinates that + # are also network responses. + with torch.no_grad(): + pred_proposals = self._decode_proposals(anchors, pred_anchor_deltas) + return find_top_rpn_proposals( + pred_proposals, + pred_objectness_logits, + image_sizes, + self.nms_thresh, + self.pre_nms_topk[self.training], + self.post_nms_topk[self.training], + self.min_box_size, + self.training, + ) + + def _decode_proposals(self, anchors: List[Boxes], pred_anchor_deltas: List[torch.Tensor]): + """ + Transform anchors into proposals by applying the predicted anchor deltas. + + Returns: + proposals (list[Tensor]): A list of L tensors. Tensor i has shape + (N, Hi*Wi*A, B) + """ + N = pred_anchor_deltas[0].shape[0] + proposals = [] + # For each feature map + for anchors_i, pred_anchor_deltas_i in zip(anchors, pred_anchor_deltas): + B = anchors_i.tensor.size(1) + pred_anchor_deltas_i = pred_anchor_deltas_i.reshape(-1, B) + # Expand anchors to shape (N*Hi*Wi*A, B) + anchors_i = anchors_i.tensor.unsqueeze(0).expand(N, -1, -1).reshape(-1, B) + proposals_i = self.box2box_transform.apply_deltas(pred_anchor_deltas_i, anchors_i) + # Append feature map proposals with shape (N, Hi*Wi*A, B) + proposals.append(proposals_i.view(N, -1, B)) + return proposals diff --git a/detectron2/modeling/proposal_generator/rrpn.py b/detectron2/modeling/proposal_generator/rrpn.py new file mode 100644 index 0000000000000000000000000000000000000000..05a390081ca94b3875e56c89902ce6613ca16c29 --- /dev/null +++ b/detectron2/modeling/proposal_generator/rrpn.py @@ -0,0 +1,204 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +from typing import Dict, List + +import torch + +from detectron2.config import configurable +from detectron2.layers import ShapeSpec, batched_nms_rotated, cat +from detectron2.structures import Instances, RotatedBoxes, pairwise_iou_rotated +from detectron2.utils.memory import retry_if_cuda_oom + +from ..box_regression import Box2BoxTransformRotated +from .build import PROPOSAL_GENERATOR_REGISTRY +from .proposal_utils import _is_tracing +from .rpn import RPN + +logger = logging.getLogger(__name__) + + +def find_top_rrpn_proposals( + proposals, + pred_objectness_logits, + image_sizes, + nms_thresh, + pre_nms_topk, + post_nms_topk, + min_box_size, + training, +): + """ + For each feature map, select the `pre_nms_topk` highest scoring proposals, + apply NMS, clip proposals, and remove small boxes. Return the `post_nms_topk` + highest scoring proposals among all the feature maps if `training` is True, + otherwise, returns the highest `post_nms_topk` scoring proposals for each + feature map. + + Args: + proposals (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A, 5). + All proposal predictions on the feature maps. + pred_objectness_logits (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A). + image_sizes (list[tuple]): sizes (h, w) for each image + nms_thresh (float): IoU threshold to use for NMS + pre_nms_topk (int): number of top k scoring proposals to keep before applying NMS. + When RRPN is run on multiple feature maps (as in FPN) this number is per + feature map. + post_nms_topk (int): number of top k scoring proposals to keep after applying NMS. + When RRPN is run on multiple feature maps (as in FPN) this number is total, + over all feature maps. + min_box_size(float): minimum proposal box side length in pixels (absolute units wrt + input images). + training (bool): True if proposals are to be used in training, otherwise False. + This arg exists only to support a legacy bug; look for the "NB: Legacy bug ..." + comment. + + Returns: + proposals (list[Instances]): list of N Instances. The i-th Instances + stores post_nms_topk object proposals for image i. + """ + num_images = len(image_sizes) + device = proposals[0].device + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + level_ids = [] # #lvl Tensor, each of shape (topk,) + batch_idx = torch.arange(num_images, device=device) + for level_id, proposals_i, logits_i in zip(itertools.count(), proposals, pred_objectness_logits): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 5 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + level_ids.append(torch.full((num_proposals_i,), level_id, dtype=torch.int64, device=device)) + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + level_ids = cat(level_ids, dim=0) + + # 3. For each image, run a per-level NMS, and choose topk results. + results = [] + for n, image_size in enumerate(image_sizes): + boxes = RotatedBoxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + lvl = level_ids + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError("Predicted boxes or scores contain Inf/NaN. Training has diverged.") + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + lvl = lvl[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img, lvl = (boxes[keep], scores_per_img[keep], lvl[keep]) + + keep = batched_nms_rotated(boxes.tensor, scores_per_img, lvl, nms_thresh) + # In Detectron1, there was different behavior during training vs. testing. + # (https://github.com/facebookresearch/Detectron/issues/459) + # During training, topk is over the proposals from *all* images in the training batch. + # During testing, it is over the proposals for each image separately. + # As a result, the training behavior becomes batch-dependent, + # and the configuration "POST_NMS_TOPK_TRAIN" end up relying on the batch size. + # This bug is addressed in Detectron2 to make the behavior independent of batch size. + keep = keep[:post_nms_topk] + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results + + +@PROPOSAL_GENERATOR_REGISTRY.register() +class RRPN(RPN): + """ + Rotated Region Proposal Network described in :paper:`RRPN`. + """ + + @configurable + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.anchor_boundary_thresh >= 0: + raise NotImplementedError("anchor_boundary_thresh is a legacy option not implemented for RRPN.") + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = super().from_config(cfg, input_shape) + ret["box2box_transform"] = Box2BoxTransformRotated(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS) + return ret + + @torch.no_grad() + def label_and_sample_anchors(self, anchors: List[RotatedBoxes], gt_instances: List[Instances]): + """ + Args: + anchors (list[RotatedBoxes]): anchors for each feature map. + gt_instances: the ground-truth instances for each image. + + Returns: + list[Tensor]: + List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across feature maps. Label values are in {-1, 0, 1}, + with meanings: -1 = ignore; 0 = negative class; 1 = positive class. + list[Tensor]: + i-th element is a Nx5 tensor, where N is the total number of anchors across + feature maps. The values are the matched gt boxes for each anchor. + Values are undefined for those anchors not labeled as 1. + """ + anchors = RotatedBoxes.cat(anchors) + + gt_boxes = [x.gt_boxes for x in gt_instances] + del gt_instances + + gt_labels = [] + matched_gt_boxes = [] + for gt_boxes_i in gt_boxes: + """ + gt_boxes_i: ground-truth boxes for i-th image + """ + match_quality_matrix = retry_if_cuda_oom(pairwise_iou_rotated)(gt_boxes_i, anchors) + matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix) + # Matching is memory-expensive and may result in CPU tensors. But the result is small + gt_labels_i = gt_labels_i.to(device=gt_boxes_i.device) + + # A vector of labels (-1, 0, 1) for each anchor + gt_labels_i = self._subsample_labels(gt_labels_i) + + if len(gt_boxes_i) == 0: + # These values won't be used anyway since the anchor is labeled as background + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + else: + # TODO wasted indexing computation for ignored boxes + matched_gt_boxes_i = gt_boxes_i[matched_idxs].tensor + + gt_labels.append(gt_labels_i) # N,AHW + matched_gt_boxes.append(matched_gt_boxes_i) + return gt_labels, matched_gt_boxes + + @torch.no_grad() + def predict_proposals(self, anchors, pred_objectness_logits, pred_anchor_deltas, image_sizes): + pred_proposals = self._decode_proposals(anchors, pred_anchor_deltas) + return find_top_rrpn_proposals( + pred_proposals, + pred_objectness_logits, + image_sizes, + self.nms_thresh, + self.pre_nms_topk[self.training], + self.post_nms_topk[self.training], + self.min_box_size, + self.training, + ) diff --git a/detectron2/modeling/roi_heads/__init__.py b/detectron2/modeling/roi_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b4ee229cdff89c47d07cc4ca084959cb28eacd0f --- /dev/null +++ b/detectron2/modeling/roi_heads/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .box_head import ROI_BOX_HEAD_REGISTRY, FastRCNNConvFCHead, build_box_head +from .cascade_rcnn import CascadeROIHeads +from .fast_rcnn import FastRCNNOutputLayers +from .keypoint_head import ( + ROI_KEYPOINT_HEAD_REGISTRY, + BaseKeypointRCNNHead, + KRCNNConvDeconvUpsampleHead, + build_keypoint_head, +) +from .mask_head import ( + ROI_MASK_HEAD_REGISTRY, + BaseMaskRCNNHead, + MaskRCNNConvUpsampleHead, + build_mask_head, +) +from .roi_heads import ( + ROI_HEADS_REGISTRY, + Res5ROIHeads, + ROIHeads, + StandardROIHeads, + build_roi_heads, + select_foreground_proposals, +) +from .rotated_fast_rcnn import RROIHeads + +from . import cascade_rcnn # isort:skip + +__all__ = list(globals().keys()) diff --git a/detectron2/modeling/roi_heads/box_head.py b/detectron2/modeling/roi_heads/box_head.py new file mode 100644 index 0000000000000000000000000000000000000000..53889d66ef5d5d663985f6bae0dda2ff5cb84567 --- /dev/null +++ b/detectron2/modeling/roi_heads/box_head.py @@ -0,0 +1,117 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List + +import fvcore.nn.weight_init as weight_init +import numpy as np +import torch +from torch import nn + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ShapeSpec, get_norm +from detectron2.utils.registry import Registry + +__all__ = ["FastRCNNConvFCHead", "build_box_head", "ROI_BOX_HEAD_REGISTRY"] + +ROI_BOX_HEAD_REGISTRY = Registry("ROI_BOX_HEAD") +ROI_BOX_HEAD_REGISTRY.__doc__ = """ +Registry for box heads, which make box predictions from per-region features. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +# To get torchscript support, we make the head a subclass of `nn.Sequential`. +# Therefore, to add new layers in this head class, please make sure they are +# added in the order they will be used in forward(). +@ROI_BOX_HEAD_REGISTRY.register() +class FastRCNNConvFCHead(nn.Sequential): + """ + A head with several 3x3 conv layers (each followed by norm & relu) and then + several fc layers (each followed by relu). + """ + + @configurable + def __init__(self, input_shape: ShapeSpec, *, conv_dims: List[int], fc_dims: List[int], conv_norm=""): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature. + conv_dims (list[int]): the output dimensions of the conv layers + fc_dims (list[int]): the output dimensions of the fc layers + conv_norm (str or callable): normalization for the conv layers. + See :func:`detectron2.layers.get_norm` for supported types. + """ + super().__init__() + assert len(conv_dims) + len(fc_dims) > 0 + + self._output_size = (input_shape.channels, input_shape.height, input_shape.width) + + self.conv_norm_relus = [] + for k, conv_dim in enumerate(conv_dims): + conv = Conv2d( + self._output_size[0], + conv_dim, + kernel_size=3, + padding=1, + bias=not conv_norm, + norm=get_norm(conv_norm, conv_dim), + activation=nn.ReLU(), + ) + self.add_module("conv{}".format(k + 1), conv) + self.conv_norm_relus.append(conv) + self._output_size = (conv_dim, self._output_size[1], self._output_size[2]) + + self.fcs = [] + for k, fc_dim in enumerate(fc_dims): + if k == 0: + self.add_module("flatten", nn.Flatten()) + fc = nn.Linear(int(np.prod(self._output_size)), fc_dim) + self.add_module("fc{}".format(k + 1), fc) + self.add_module("fc_relu{}".format(k + 1), nn.ReLU()) + self.fcs.append(fc) + self._output_size = fc_dim + + for layer in self.conv_norm_relus: + weight_init.c2_msra_fill(layer) + for layer in self.fcs: + weight_init.c2_xavier_fill(layer) + + @classmethod + def from_config(cls, cfg, input_shape): + num_conv = cfg.MODEL.ROI_BOX_HEAD.NUM_CONV + conv_dim = cfg.MODEL.ROI_BOX_HEAD.CONV_DIM + num_fc = cfg.MODEL.ROI_BOX_HEAD.NUM_FC + fc_dim = cfg.MODEL.ROI_BOX_HEAD.FC_DIM + return { + "input_shape": input_shape, + "conv_dims": [conv_dim] * num_conv, + "fc_dims": [fc_dim] * num_fc, + "conv_norm": cfg.MODEL.ROI_BOX_HEAD.NORM, + } + + def forward(self, x): + for layer in self: + x = layer(x) + return x + + @property + @torch.jit.unused + def output_shape(self): + """ + Returns: + ShapeSpec: the output feature shape + """ + o = self._output_size + if isinstance(o, int): + return ShapeSpec(channels=o) + else: + return ShapeSpec(channels=o[0], height=o[1], width=o[2]) + + +def build_box_head(cfg, input_shape): + """ + Build a box head defined by `cfg.MODEL.ROI_BOX_HEAD.NAME`. + """ + name = cfg.MODEL.ROI_BOX_HEAD.NAME + return ROI_BOX_HEAD_REGISTRY.get(name)(cfg, input_shape) diff --git a/detectron2/modeling/roi_heads/cascade_rcnn.py b/detectron2/modeling/roi_heads/cascade_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..874e2e5237f76aae2bf1bafa4a3fe57e9e64c8f6 --- /dev/null +++ b/detectron2/modeling/roi_heads/cascade_rcnn.py @@ -0,0 +1,293 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List + +import torch +from torch import nn +from torch.autograd.function import Function + +from detectron2.config import configurable +from detectron2.layers import ShapeSpec +from detectron2.structures import Boxes, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage + +from ..box_regression import Box2BoxTransform +from ..matcher import Matcher +from ..poolers import ROIPooler +from .box_head import build_box_head +from .fast_rcnn import FastRCNNOutputLayers, fast_rcnn_inference +from .roi_heads import ROI_HEADS_REGISTRY, StandardROIHeads + + +class _ScaleGradient(Function): + @staticmethod + def forward(ctx, input, scale): + ctx.scale = scale + return input + + @staticmethod + def backward(ctx, grad_output): + return grad_output * ctx.scale, None + + +@ROI_HEADS_REGISTRY.register() +class CascadeROIHeads(StandardROIHeads): + """ + The ROI heads that implement :paper:`Cascade R-CNN`. + """ + + @configurable + def __init__( + self, + *, + box_in_features: List[str], + box_pooler: ROIPooler, + box_heads: List[nn.Module], + box_predictors: List[nn.Module], + proposal_matchers: List[Matcher], + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + box_pooler (ROIPooler): pooler that extracts region features from given boxes + box_heads (list[nn.Module]): box head for each cascade stage + box_predictors (list[nn.Module]): box predictor for each cascade stage + proposal_matchers (list[Matcher]): matcher with different IoU thresholds to + match boxes with ground truth for each stage. The first matcher matches + RPN proposals with ground truth, the other matchers use boxes predicted + by the previous stage as proposals and match them with ground truth. + """ + assert "proposal_matcher" not in kwargs, ( + "CascadeROIHeads takes 'proposal_matchers=' for each stage instead " "of one 'proposal_matcher='." + ) + # The first matcher matches RPN proposals with ground truth, done in the base class + kwargs["proposal_matcher"] = proposal_matchers[0] + num_stages = self.num_cascade_stages = len(box_heads) + box_heads = nn.ModuleList(box_heads) + box_predictors = nn.ModuleList(box_predictors) + assert len(box_predictors) == num_stages, f"{len(box_predictors)} != {num_stages}!" + assert len(proposal_matchers) == num_stages, f"{len(proposal_matchers)} != {num_stages}!" + super().__init__( + box_in_features=box_in_features, + box_pooler=box_pooler, + box_head=box_heads, + box_predictor=box_predictors, + **kwargs, + ) + self.proposal_matchers = proposal_matchers + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + ret.pop("proposal_matcher") + return ret + + @classmethod + def _init_box_head(cls, cfg, input_shape): + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + cascade_bbox_reg_weights = cfg.MODEL.ROI_BOX_CASCADE_HEAD.BBOX_REG_WEIGHTS + cascade_ious = cfg.MODEL.ROI_BOX_CASCADE_HEAD.IOUS + assert len(cascade_bbox_reg_weights) == len(cascade_ious) + assert cfg.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG, \ + "CascadeROIHeads only support class-agnostic regression now!" + assert cascade_ious[0] == cfg.MODEL.ROI_HEADS.IOU_THRESHOLDS[0] + # fmt: on + + in_channels = [input_shape[f].channels for f in in_features] + # Check all channel counts are equal + assert len(set(in_channels)) == 1, in_channels + in_channels = in_channels[0] + + box_pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + pooled_shape = ShapeSpec(channels=in_channels, width=pooler_resolution, height=pooler_resolution) + + box_heads, box_predictors, proposal_matchers = [], [], [] + for match_iou, bbox_reg_weights in zip(cascade_ious, cascade_bbox_reg_weights): + box_head = build_box_head(cfg, pooled_shape) + box_heads.append(box_head) + box_predictors.append( + FastRCNNOutputLayers( + cfg, + box_head.output_shape, + box2box_transform=Box2BoxTransform(weights=bbox_reg_weights), + ) + ) + proposal_matchers.append(Matcher([match_iou], [0, 1], allow_low_quality_matches=False)) + return { + "box_in_features": in_features, + "box_pooler": box_pooler, + "box_heads": box_heads, + "box_predictors": box_predictors, + "proposal_matchers": proposal_matchers, + } + + def forward(self, images, features, proposals, targets=None): + del images + if self.training: + proposals = self.label_and_sample_proposals(proposals, targets) + + if self.training: + # Need targets to box head + losses = self._forward_box(features, proposals, targets) + losses.update(self._forward_mask(features, proposals)) + losses.update(self._forward_keypoint(features, proposals)) + return proposals, losses + else: + pred_instances = self._forward_box(features, proposals) + pred_instances = self.forward_with_given_boxes(features, pred_instances) + return pred_instances, {} + + def _forward_box(self, features, proposals, targets=None): + """ + Args: + features, targets: the same as in + Same as in :meth:`ROIHeads.forward`. + proposals (list[Instances]): the per-image object proposals with + their matching ground truth. + Each has fields "proposal_boxes", and "objectness_logits", + "gt_classes", "gt_boxes". + """ + features = [features[f] for f in self.box_in_features] + head_outputs = [] # (predictor, predictions, proposals) + prev_pred_boxes = None + image_sizes = [x.image_size for x in proposals] + for k in range(self.num_cascade_stages): + if k > 0: + # The output boxes of the previous stage are used to create the input + # proposals of the next stage. + proposals = self._create_proposals_from_boxes(prev_pred_boxes, image_sizes) + if self.training: + proposals = self._match_and_label_boxes(proposals, k, targets) + predictions = self._run_stage(features, proposals, k) + prev_pred_boxes = self.box_predictor[k].predict_boxes(predictions, proposals) + head_outputs.append((self.box_predictor[k], predictions, proposals)) + + if self.training: + losses = {} + storage = get_event_storage() + for stage, (predictor, predictions, proposals) in enumerate(head_outputs): + with storage.name_scope("stage{}".format(stage)): + stage_losses = predictor.losses(predictions, proposals) + losses.update({k + "_stage{}".format(stage): v for k, v in stage_losses.items()}) + return losses + else: + # Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1) + scores_per_stage = [h[0].predict_probs(h[1], h[2]) for h in head_outputs] + + # Average the scores across heads + scores = [ + sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages) + for scores_per_image in zip(*scores_per_stage) + ] + # Use the boxes of the last head + predictor, predictions, proposals = head_outputs[-1] + boxes = predictor.predict_boxes(predictions, proposals) + pred_instances, _ = fast_rcnn_inference( + boxes, + scores, + image_sizes, + predictor.test_score_thresh, + predictor.test_nms_thresh, + predictor.test_topk_per_image, + ) + return pred_instances + + @torch.no_grad() + def _match_and_label_boxes(self, proposals, stage, targets): + """ + Match proposals with groundtruth using the matcher at the given stage. + Label the proposals as foreground or background based on the match. + + Args: + proposals (list[Instances]): One Instances for each image, with + the field "proposal_boxes". + stage (int): the current stage + targets (list[Instances]): the ground truth instances + + Returns: + list[Instances]: the same proposals, but with fields "gt_classes" and "gt_boxes" + """ + num_fg_samples, num_bg_samples = [], [] + for proposals_per_image, targets_per_image in zip(proposals, targets): + match_quality_matrix = pairwise_iou(targets_per_image.gt_boxes, proposals_per_image.proposal_boxes) + # proposal_labels are 0 or 1 + matched_idxs, proposal_labels = self.proposal_matchers[stage](match_quality_matrix) + if len(targets_per_image) > 0: + gt_classes = targets_per_image.gt_classes[matched_idxs] + # Label unmatched proposals (0 label from matcher) as background (label=num_classes) + gt_classes[proposal_labels == 0] = self.num_classes + gt_boxes = targets_per_image.gt_boxes[matched_idxs] + else: + gt_classes = torch.zeros_like(matched_idxs) + self.num_classes + gt_boxes = Boxes(targets_per_image.gt_boxes.tensor.new_zeros((len(proposals_per_image), 4))) + proposals_per_image.gt_classes = gt_classes + proposals_per_image.gt_boxes = gt_boxes + + num_fg_samples.append((proposal_labels == 1).sum().item()) + num_bg_samples.append(proposal_labels.numel() - num_fg_samples[-1]) + + # Log the number of fg/bg samples in each stage + storage = get_event_storage() + storage.put_scalar( + "stage{}/roi_head/num_fg_samples".format(stage), + sum(num_fg_samples) / len(num_fg_samples), + ) + storage.put_scalar( + "stage{}/roi_head/num_bg_samples".format(stage), + sum(num_bg_samples) / len(num_bg_samples), + ) + return proposals + + def _run_stage(self, features, proposals, stage): + """ + Args: + features (list[Tensor]): #lvl input features to ROIHeads + proposals (list[Instances]): #image Instances, with the field "proposal_boxes" + stage (int): the current stage + + Returns: + Same output as `FastRCNNOutputLayers.forward()`. + """ + box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals]) + # The original implementation averages the losses among heads, + # but scale up the parameter gradients of the heads. + # This is equivalent to adding the losses among heads, + # but scale down the gradients on features. + if self.training: + box_features = _ScaleGradient.apply(box_features, 1.0 / self.num_cascade_stages) + box_features = self.box_head[stage](box_features) + return self.box_predictor[stage](box_features) + + def _create_proposals_from_boxes(self, boxes, image_sizes): + """ + Args: + boxes (list[Tensor]): per-image predicted boxes, each of shape Ri x 4 + image_sizes (list[tuple]): list of image shapes in (h, w) + + Returns: + list[Instances]: per-image proposals with the given boxes. + """ + # Just like RPN, the proposals should not have gradients + boxes = [Boxes(b.detach()) for b in boxes] + proposals = [] + for boxes_per_image, image_size in zip(boxes, image_sizes): + boxes_per_image.clip(image_size) + if self.training: + # do not filter empty boxes at inference time, + # because the scores from each stage need to be aligned and added later + boxes_per_image = boxes_per_image[boxes_per_image.nonempty()] + prop = Instances(image_size) + prop.proposal_boxes = boxes_per_image + proposals.append(prop) + return proposals diff --git a/detectron2/modeling/roi_heads/fast_rcnn.py b/detectron2/modeling/roi_heads/fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..ef57ddd17b097cb6d52b099d44a5218235c4ce2a --- /dev/null +++ b/detectron2/modeling/roi_heads/fast_rcnn.py @@ -0,0 +1,556 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +from typing import Callable, Dict, List, Optional, Tuple, Union + +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.data.detection_utils import get_fed_loss_cls_weights +from detectron2.layers import ShapeSpec, batched_nms, cat, cross_entropy, nonzero_tuple +from detectron2.modeling.box_regression import Box2BoxTransform, _dense_box_regression_loss +from detectron2.structures import Boxes, Instances +from detectron2.utils.events import get_event_storage + +__all__ = ["fast_rcnn_inference", "FastRCNNOutputLayers"] + + +logger = logging.getLogger(__name__) + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + R: number of ROIs, combined over all images, in the minibatch + Ri: number of ROIs in image i + K: number of foreground classes. E.g.,there are 80 foreground classes in COCO. + +Naming convention: + + deltas: refers to the 4-d (dx, dy, dw, dh) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransform`). + + pred_class_logits: predicted class scores in [-inf, +inf]; use + softmax(pred_class_logits) to estimate P(class). + + gt_classes: ground-truth classification labels in [0, K], where [0, K) represent + foreground object classes and K represents the background class. + + pred_proposal_deltas: predicted box2box transform deltas for transforming proposals + to detection box predictions. + + gt_proposal_deltas: ground-truth box2box transform deltas +""" + + +def fast_rcnn_inference( + boxes: List[torch.Tensor], + scores: List[torch.Tensor], + image_shapes: List[Tuple[int, int]], + score_thresh: float, + nms_thresh: float, + topk_per_image: int, +): + """ + Call `fast_rcnn_inference_single_image` for all images. + + Args: + boxes (list[Tensor]): A list of Tensors of predicted class-specific or class-agnostic + boxes for each image. Element i has shape (Ri, K * 4) if doing + class-specific regression, or (Ri, 4) if doing class-agnostic + regression, where Ri is the number of predicted objects for image i. + This is compatible with the output of :meth:`FastRCNNOutputLayers.predict_boxes`. + scores (list[Tensor]): A list of Tensors of predicted class scores for each image. + Element i has shape (Ri, K + 1), where Ri is the number of predicted objects + for image i. Compatible with the output of :meth:`FastRCNNOutputLayers.predict_probs`. + image_shapes (list[tuple]): A list of (width, height) tuples for each image in the batch. + score_thresh (float): Only return detections with a confidence score exceeding this + threshold. + nms_thresh (float): The threshold to use for box non-maximum suppression. Value in [0, 1]. + topk_per_image (int): The number of top scoring detections to return. Set < 0 to return + all detections. + + Returns: + instances: (list[Instances]): A list of N instances, one for each image in the batch, + that stores the topk most confidence detections. + kept_indices: (list[Tensor]): A list of 1D tensor of length of N, each element indicates + the corresponding boxes/scores index in [0, Ri) from the input, for image i. + """ + result_per_image = [ + fast_rcnn_inference_single_image( + boxes_per_image, scores_per_image, image_shape, score_thresh, nms_thresh, topk_per_image + ) + for scores_per_image, boxes_per_image, image_shape in zip(scores, boxes, image_shapes) + ] + return [x[0] for x in result_per_image], [x[1] for x in result_per_image] + + +def _log_classification_stats(pred_logits, gt_classes, prefix="fast_rcnn"): + """ + Log the classification metrics to EventStorage. + + Args: + pred_logits: Rx(K+1) logits. The last column is for background class. + gt_classes: R labels + """ + num_instances = gt_classes.numel() + if num_instances == 0: + return + pred_classes = pred_logits.argmax(dim=1) + bg_class_ind = pred_logits.shape[1] - 1 + + fg_inds = (gt_classes >= 0) & (gt_classes < bg_class_ind) + num_fg = fg_inds.nonzero().numel() + fg_gt_classes = gt_classes[fg_inds] + fg_pred_classes = pred_classes[fg_inds] + + num_false_negative = (fg_pred_classes == bg_class_ind).nonzero().numel() + num_accurate = (pred_classes == gt_classes).nonzero().numel() + fg_num_accurate = (fg_pred_classes == fg_gt_classes).nonzero().numel() + + storage = get_event_storage() + storage.put_scalar(f"{prefix}/cls_accuracy", num_accurate / num_instances) + if num_fg > 0: + storage.put_scalar(f"{prefix}/fg_cls_accuracy", fg_num_accurate / num_fg) + storage.put_scalar(f"{prefix}/false_negative", num_false_negative / num_fg) + + +def fast_rcnn_inference_single_image( + boxes, + scores, + image_shape: Tuple[int, int], + score_thresh: float, + nms_thresh: float, + topk_per_image: int, +): + """ + Single-image inference. Return bounding-box detection results by thresholding + on scores and applying non-maximum suppression (NMS). + + Args: + Same as `fast_rcnn_inference`, but with boxes, scores, and image shapes + per image. + + Returns: + Same as `fast_rcnn_inference`, but for only one image. + """ + valid_mask = torch.isfinite(boxes).all(dim=1) & torch.isfinite(scores).all(dim=1) + if not valid_mask.all(): + boxes = boxes[valid_mask] + scores = scores[valid_mask] + + scores = scores[:, :-1] + num_bbox_reg_classes = boxes.shape[1] // 4 + # Convert to Boxes to use the `clip` function ... + boxes = Boxes(boxes.reshape(-1, 4)) + boxes.clip(image_shape) + boxes = boxes.tensor.view(-1, num_bbox_reg_classes, 4) # R x C x 4 + + # 1. Filter results based on detection scores. It can make NMS more efficient + # by filtering out low-confidence detections. + filter_mask = scores > score_thresh # R x K + # R' x 2. First column contains indices of the R predictions; + # Second column contains indices of classes. + filter_inds = filter_mask.nonzero() + if num_bbox_reg_classes == 1: + boxes = boxes[filter_inds[:, 0], 0] + else: + boxes = boxes[filter_mask] + scores = scores[filter_mask] + + # 2. Apply NMS for each class independently. + keep = batched_nms(boxes, scores, filter_inds[:, 1], nms_thresh) + if topk_per_image >= 0: + keep = keep[:topk_per_image] + boxes, scores, filter_inds = boxes[keep], scores[keep], filter_inds[keep] + + result = Instances(image_shape) + result.pred_boxes = Boxes(boxes) + result.scores = scores + result.pred_classes = filter_inds[:, 1] + return result, filter_inds[:, 0] + + +class FastRCNNOutputLayers(nn.Module): + """ + Two linear layers for predicting Fast R-CNN outputs: + + 1. proposal-to-detection box regression deltas + 2. classification scores + """ + + @configurable + def __init__( + self, + input_shape: ShapeSpec, + *, + box2box_transform, + num_classes: int, + test_score_thresh: float = 0.0, + test_nms_thresh: float = 0.5, + test_topk_per_image: int = 100, + cls_agnostic_bbox_reg: bool = False, + smooth_l1_beta: float = 0.0, + box_reg_loss_type: str = "smooth_l1", + loss_weight: Union[float, Dict[str, float]] = 1.0, + use_fed_loss: bool = False, + use_sigmoid_ce: bool = False, + get_fed_loss_cls_weights: Optional[Callable] = None, + fed_loss_num_classes: int = 50, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature to this module + box2box_transform (Box2BoxTransform or Box2BoxTransformRotated): + num_classes (int): number of foreground classes + test_score_thresh (float): threshold to filter predictions results. + test_nms_thresh (float): NMS threshold for prediction results. + test_topk_per_image (int): number of top predictions to produce per image. + cls_agnostic_bbox_reg (bool): whether to use class agnostic for bbox regression + smooth_l1_beta (float): transition point from L1 to L2 loss. Only used if + `box_reg_loss_type` is "smooth_l1" + box_reg_loss_type (str): Box regression loss type. One of: "smooth_l1", "giou", + "diou", "ciou" + loss_weight (float|dict): weights to use for losses. Can be single float for weighting + all losses, or a dict of individual weightings. Valid dict keys are: + * "loss_cls": applied to classification loss + * "loss_box_reg": applied to box regression loss + use_fed_loss (bool): whether to use federated loss which samples additional negative + classes to calculate the loss + use_sigmoid_ce (bool): whether to calculate the loss using weighted average of binary + cross entropy with logits. This could be used together with federated loss + get_fed_loss_cls_weights (Callable): a callable which takes dataset name and frequency + weight power, and returns the probabilities to sample negative classes for + federated loss. The implementation can be found in + detectron2/data/detection_utils.py + fed_loss_num_classes (int): number of federated classes to keep in total + """ + super().__init__() + if isinstance(input_shape, int): # some backward compatibility + input_shape = ShapeSpec(channels=input_shape) + self.num_classes = num_classes + input_size = input_shape.channels * (input_shape.width or 1) * (input_shape.height or 1) + # prediction layer for num_classes foreground classes and one background class (hence + 1) + self.cls_score = nn.Linear(input_size, num_classes + 1) + num_bbox_reg_classes = 1 if cls_agnostic_bbox_reg else num_classes + box_dim = len(box2box_transform.weights) + self.bbox_pred = nn.Linear(input_size, num_bbox_reg_classes * box_dim) + + nn.init.normal_(self.cls_score.weight, std=0.01) + nn.init.normal_(self.bbox_pred.weight, std=0.001) + for l in [self.cls_score, self.bbox_pred]: + nn.init.constant_(l.bias, 0) + + self.box2box_transform = box2box_transform + self.smooth_l1_beta = smooth_l1_beta + self.test_score_thresh = test_score_thresh + self.test_nms_thresh = test_nms_thresh + self.test_topk_per_image = test_topk_per_image + self.box_reg_loss_type = box_reg_loss_type + if isinstance(loss_weight, float): + loss_weight = {"loss_cls": loss_weight, "loss_box_reg": loss_weight} + self.loss_weight = loss_weight + self.use_fed_loss = use_fed_loss + self.use_sigmoid_ce = use_sigmoid_ce + self.fed_loss_num_classes = fed_loss_num_classes + + if self.use_fed_loss: + assert self.use_sigmoid_ce, "Please use sigmoid cross entropy loss with federated loss" + fed_loss_cls_weights = get_fed_loss_cls_weights() + assert ( + len(fed_loss_cls_weights) == self.num_classes + ), "Please check the provided fed_loss_cls_weights. Their size should match num_classes" + self.register_buffer("fed_loss_cls_weights", fed_loss_cls_weights) + + @classmethod + def from_config(cls, cfg, input_shape): + return { + "input_shape": input_shape, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS), + # fmt: off + "num_classes" : cfg.MODEL.ROI_HEADS.NUM_CLASSES, + "cls_agnostic_bbox_reg" : cfg.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG, + "smooth_l1_beta" : cfg.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA, + "test_score_thresh" : cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST, + "test_nms_thresh" : cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST, + "test_topk_per_image" : cfg.TEST.DETECTIONS_PER_IMAGE, + "box_reg_loss_type" : cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE, + "loss_weight" : {"loss_box_reg": cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_WEIGHT}, # noqa + "use_fed_loss" : cfg.MODEL.ROI_BOX_HEAD.USE_FED_LOSS, + "use_sigmoid_ce" : cfg.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE, + "get_fed_loss_cls_weights" : lambda: get_fed_loss_cls_weights(dataset_names=cfg.DATASETS.TRAIN, freq_weight_power=cfg.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT_POWER), # noqa + "fed_loss_num_classes" : cfg.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CLASSES, + # fmt: on + } + + def forward(self, x): + """ + Args: + x: per-region features of shape (N, ...) for N bounding boxes to predict. + + Returns: + (Tensor, Tensor): + First tensor: shape (N,K+1), scores for each of the N box. Each row contains the + scores for K object categories and 1 background class. + + Second tensor: bounding box regression deltas for each box. Shape is shape (N,Kx4), + or (N,4) for class-agnostic regression. + """ + if x.dim() > 2: + x = torch.flatten(x, start_dim=1) + scores = self.cls_score(x) + proposal_deltas = self.bbox_pred(x) + return scores, proposal_deltas + + def losses(self, predictions, proposals): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were used + to compute predictions. The fields ``proposal_boxes``, ``gt_boxes``, + ``gt_classes`` are expected. + + Returns: + Dict[str, Tensor]: dict of losses + """ + scores, proposal_deltas = predictions + + # parse classification outputs + gt_classes = cat([p.gt_classes for p in proposals], dim=0) if len(proposals) else torch.empty(0) + _log_classification_stats(scores, gt_classes) + + # parse box regression outputs + if len(proposals): + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) # Nx4 + assert not proposal_boxes.requires_grad, "Proposals should not require gradients!" + # If "gt_boxes" does not exist, the proposals must be all negative and + # should not be included in regression loss computation. + # Here we just use proposal_boxes as an arbitrary placeholder because its + # value won't be used in self.box_reg_loss(). + gt_boxes = cat( + [(p.gt_boxes if p.has("gt_boxes") else p.proposal_boxes).tensor for p in proposals], + dim=0, + ) + else: + proposal_boxes = gt_boxes = torch.empty((0, 4), device=proposal_deltas.device) + + if self.use_sigmoid_ce: + loss_cls = self.sigmoid_cross_entropy_loss(scores, gt_classes) + else: + loss_cls = cross_entropy(scores, gt_classes, reduction="mean") + + losses = { + "loss_cls": loss_cls, + "loss_box_reg": self.box_reg_loss(proposal_boxes, gt_boxes, proposal_deltas, gt_classes), + } + return {k: v * self.loss_weight.get(k, 1.0) for k, v in losses.items()} + + # Implementation from https://github.com/xingyizhou/CenterNet2/blob/master/projects/CenterNet2/centernet/modeling/roi_heads/fed_loss.py # noqa + # with slight modifications + def get_fed_loss_classes(self, gt_classes, num_fed_loss_classes, num_classes, weight): + """ + Args: + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + num_fed_loss_classes: minimum number of classes to keep when calculating federated loss. + Will sample negative classes if number of unique gt_classes is smaller than this value. + num_classes: number of foreground classes + weight: probabilities used to sample negative classes + + Returns: + Tensor: + classes to keep when calculating the federated loss, including both unique gt + classes and sampled negative classes. + """ + unique_gt_classes = torch.unique(gt_classes) + prob = unique_gt_classes.new_ones(num_classes + 1).float() + prob[-1] = 0 + if len(unique_gt_classes) < num_fed_loss_classes: + prob[:num_classes] = weight.float().clone() + prob[unique_gt_classes] = 0 + sampled_negative_classes = torch.multinomial( + prob, num_fed_loss_classes - len(unique_gt_classes), replacement=False + ) + fed_loss_classes = torch.cat([unique_gt_classes, sampled_negative_classes]) + else: + fed_loss_classes = unique_gt_classes + return fed_loss_classes + + # Implementation from https://github.com/xingyizhou/CenterNet2/blob/master/projects/CenterNet2/centernet/modeling/roi_heads/custom_fast_rcnn.py#L113 # noqa + # with slight modifications + def sigmoid_cross_entropy_loss(self, pred_class_logits, gt_classes): + """ + Args: + pred_class_logits: shape (N, K+1), scores for each of the N box. Each row contains the + scores for K object categories and 1 background class + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + """ + if pred_class_logits.numel() == 0: + return pred_class_logits.new_zeros([1])[0] + + N = pred_class_logits.shape[0] + K = pred_class_logits.shape[1] - 1 + + target = pred_class_logits.new_zeros(N, K + 1) + target[range(len(gt_classes)), gt_classes] = 1 + target = target[:, :K] + + cls_loss = F.binary_cross_entropy_with_logits(pred_class_logits[:, :-1], target, reduction="none") + + if self.use_fed_loss: + fed_loss_classes = self.get_fed_loss_classes( + gt_classes, + num_fed_loss_classes=self.fed_loss_num_classes, + num_classes=K, + weight=self.fed_loss_cls_weights, + ) + fed_loss_classes_mask = fed_loss_classes.new_zeros(K + 1) + fed_loss_classes_mask[fed_loss_classes] = 1 + fed_loss_classes_mask = fed_loss_classes_mask[:K] + weight = fed_loss_classes_mask.view(1, K).expand(N, K).float() + else: + weight = 1 + + loss = torch.sum(cls_loss * weight) / N + return loss + + def box_reg_loss(self, proposal_boxes, gt_boxes, pred_deltas, gt_classes): + """ + Args: + proposal_boxes/gt_boxes are tensors with the same shape (R, 4 or 5). + pred_deltas has shape (R, 4 or 5), or (R, num_classes * (4 or 5)). + gt_classes is a long tensor of shape R, the gt class label of each proposal. + R shall be the number of proposals. + """ + box_dim = proposal_boxes.shape[1] # 4 or 5 + # Regression loss is only computed for foreground proposals (those matched to a GT) + fg_inds = nonzero_tuple((gt_classes >= 0) & (gt_classes < self.num_classes))[0] + if pred_deltas.shape[1] == box_dim: # cls-agnostic regression + fg_pred_deltas = pred_deltas[fg_inds] + else: + fg_pred_deltas = pred_deltas.view(-1, self.num_classes, box_dim)[fg_inds, gt_classes[fg_inds]] + + loss_box_reg = _dense_box_regression_loss( + [proposal_boxes[fg_inds]], + self.box2box_transform, + [fg_pred_deltas.unsqueeze(0)], + [gt_boxes[fg_inds]], + ..., + self.box_reg_loss_type, + self.smooth_l1_beta, + ) + + # The reg loss is normalized using the total number of regions (R), not the number + # of foreground regions even though the box regression loss is only defined on + # foreground regions. Why? Because doing so gives equal training influence to + # each foreground example. To see how, consider two different minibatches: + # (1) Contains a single foreground region + # (2) Contains 100 foreground regions + # If we normalize by the number of foreground regions, the single example in + # minibatch (1) will be given 100 times as much influence as each foreground + # example in minibatch (2). Normalizing by the total number of regions, R, + # means that the single example in minibatch (1) and each of the 100 examples + # in minibatch (2) are given equal influence. + return loss_box_reg / max(gt_classes.numel(), 1.0) # return 0 if empty + + def inference(self, predictions: Tuple[torch.Tensor, torch.Tensor], proposals: List[Instances]): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were + used to compute predictions. The ``proposal_boxes`` field is expected. + + Returns: + list[Instances]: same as `fast_rcnn_inference`. + list[Tensor]: same as `fast_rcnn_inference`. + """ + boxes = self.predict_boxes(predictions, proposals) + scores = self.predict_probs(predictions, proposals) + image_shapes = [x.image_size for x in proposals] + return fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + def predict_boxes_for_gt_classes(self, predictions, proposals): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were used + to compute predictions. The fields ``proposal_boxes``, ``gt_classes`` are expected. + + Returns: + list[Tensor]: + A list of Tensors of predicted boxes for GT classes in case of + class-specific box head. Element i of the list has shape (Ri, B), where Ri is + the number of proposals for image i and B is the box dimension (4 or 5) + """ + if not len(proposals): + return [] + scores, proposal_deltas = predictions + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) + N, B = proposal_boxes.shape + predict_boxes = self.box2box_transform.apply_deltas(proposal_deltas, proposal_boxes) # Nx(KxB) + + K = predict_boxes.shape[1] // B + if K > 1: + gt_classes = torch.cat([p.gt_classes for p in proposals], dim=0) + # Some proposals are ignored or have a background class. Their gt_classes + # cannot be used as index. + gt_classes = gt_classes.clamp_(0, K - 1) + + predict_boxes = predict_boxes.view(N, K, B)[ + torch.arange(N, dtype=torch.long, device=predict_boxes.device), gt_classes + ] + num_prop_per_image = [len(p) for p in proposals] + return predict_boxes.split(num_prop_per_image) + + def predict_boxes(self, predictions: Tuple[torch.Tensor, torch.Tensor], proposals: List[Instances]): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were + used to compute predictions. The ``proposal_boxes`` field is expected. + + Returns: + list[Tensor]: + A list of Tensors of predicted class-specific or class-agnostic boxes + for each image. Element i has shape (Ri, K * B) or (Ri, B), where Ri is + the number of proposals for image i and B is the box dimension (4 or 5) + """ + if not len(proposals): + return [] + _, proposal_deltas = predictions + num_prop_per_image = [len(p) for p in proposals] + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) + predict_boxes = self.box2box_transform.apply_deltas( + proposal_deltas, + proposal_boxes, + ) # Nx(KxB) + return predict_boxes.split(num_prop_per_image) + + def predict_probs(self, predictions: Tuple[torch.Tensor, torch.Tensor], proposals: List[Instances]): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were + used to compute predictions. + + Returns: + list[Tensor]: + A list of Tensors of predicted class probabilities for each image. + Element i has shape (Ri, K + 1), where Ri is the number of proposals for image i. + """ + scores, _ = predictions + num_inst_per_image = [len(p) for p in proposals] + if self.use_sigmoid_ce: + probs = scores.sigmoid() + else: + probs = F.softmax(scores, dim=-1) + return probs.split(num_inst_per_image, dim=0) diff --git a/detectron2/modeling/roi_heads/keypoint_head.py b/detectron2/modeling/roi_heads/keypoint_head.py new file mode 100644 index 0000000000000000000000000000000000000000..ffa1693c63c29b606636112760ed5e8dd34e099c --- /dev/null +++ b/detectron2/modeling/roi_heads/keypoint_head.py @@ -0,0 +1,262 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List + +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ConvTranspose2d, cat, interpolate +from detectron2.structures import Instances, heatmaps_to_keypoints +from detectron2.utils.events import get_event_storage +from detectron2.utils.registry import Registry + +_TOTAL_SKIPPED = 0 + + +__all__ = [ + "ROI_KEYPOINT_HEAD_REGISTRY", + "build_keypoint_head", + "BaseKeypointRCNNHead", + "KRCNNConvDeconvUpsampleHead", +] + + +ROI_KEYPOINT_HEAD_REGISTRY = Registry("ROI_KEYPOINT_HEAD") +ROI_KEYPOINT_HEAD_REGISTRY.__doc__ = """ +Registry for keypoint heads, which make keypoint predictions from per-region features. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +def build_keypoint_head(cfg, input_shape): + """ + Build a keypoint head from `cfg.MODEL.ROI_KEYPOINT_HEAD.NAME`. + """ + name = cfg.MODEL.ROI_KEYPOINT_HEAD.NAME + return ROI_KEYPOINT_HEAD_REGISTRY.get(name)(cfg, input_shape) + + +def keypoint_rcnn_loss(pred_keypoint_logits, instances, normalizer): + """ + Arguments: + pred_keypoint_logits (Tensor): A tensor of shape (N, K, S, S) where N is the total number + of instances in the batch, K is the number of keypoints, and S is the side length + of the keypoint heatmap. The values are spatial logits. + instances (list[Instances]): A list of M Instances, where M is the batch size. + These instances are predictions from the model + that are in 1:1 correspondence with pred_keypoint_logits. + Each Instances should contain a `gt_keypoints` field containing a `structures.Keypoint` + instance. + normalizer (float): Normalize the loss by this amount. + If not specified, we normalize by the number of visible keypoints in the minibatch. + + Returns a scalar tensor containing the loss. + """ + heatmaps = [] + valid = [] + + keypoint_side_len = pred_keypoint_logits.shape[2] + for instances_per_image in instances: + if len(instances_per_image) == 0: + continue + keypoints = instances_per_image.gt_keypoints + heatmaps_per_image, valid_per_image = keypoints.to_heatmap( + instances_per_image.proposal_boxes.tensor, keypoint_side_len + ) + heatmaps.append(heatmaps_per_image.view(-1)) + valid.append(valid_per_image.view(-1)) + + if len(heatmaps): + keypoint_targets = cat(heatmaps, dim=0) + valid = cat(valid, dim=0).to(dtype=torch.uint8) + valid = torch.nonzero(valid).squeeze(1) + + # torch.mean (in binary_cross_entropy_with_logits) doesn't + # accept empty tensors, so handle it separately + if len(heatmaps) == 0 or valid.numel() == 0: + global _TOTAL_SKIPPED + _TOTAL_SKIPPED += 1 + storage = get_event_storage() + storage.put_scalar("kpts_num_skipped_batches", _TOTAL_SKIPPED, smoothing_hint=False) + return pred_keypoint_logits.sum() * 0 + + N, K, H, W = pred_keypoint_logits.shape + pred_keypoint_logits = pred_keypoint_logits.view(N * K, H * W) + + keypoint_loss = F.cross_entropy(pred_keypoint_logits[valid], keypoint_targets[valid], reduction="sum") + + # If a normalizer isn't specified, normalize by the number of visible keypoints in the minibatch + if normalizer is None: + normalizer = valid.numel() + keypoint_loss /= normalizer + + return keypoint_loss + + +def keypoint_rcnn_inference(pred_keypoint_logits: torch.Tensor, pred_instances: List[Instances]): + """ + Post process each predicted keypoint heatmap in `pred_keypoint_logits` into (x, y, score) + and add it to the `pred_instances` as a `pred_keypoints` field. + + Args: + pred_keypoint_logits (Tensor): A tensor of shape (R, K, S, S) where R is the total number + of instances in the batch, K is the number of keypoints, and S is the side length of + the keypoint heatmap. The values are spatial logits. + pred_instances (list[Instances]): A list of N Instances, where N is the number of images. + + Returns: + None. Each element in pred_instances will contain extra "pred_keypoints" and + "pred_keypoint_heatmaps" fields. "pred_keypoints" is a tensor of shape + (#instance, K, 3) where the last dimension corresponds to (x, y, score). + The scores are larger than 0. "pred_keypoint_heatmaps" contains the raw + keypoint logits as passed to this function. + """ + # flatten all bboxes from all images together (list[Boxes] -> Rx4 tensor) + bboxes_flat = cat([b.pred_boxes.tensor for b in pred_instances], dim=0) + + pred_keypoint_logits = pred_keypoint_logits.detach() + keypoint_results = heatmaps_to_keypoints(pred_keypoint_logits, bboxes_flat.detach()) + num_instances_per_image = [len(i) for i in pred_instances] + keypoint_results = keypoint_results[:, :, [0, 1, 3]].split(num_instances_per_image, dim=0) + heatmap_results = pred_keypoint_logits.split(num_instances_per_image, dim=0) + + for keypoint_results_per_image, heatmap_results_per_image, instances_per_image in zip( + keypoint_results, heatmap_results, pred_instances + ): + # keypoint_results_per_image is (num instances)x(num keypoints)x(x, y, score) + # heatmap_results_per_image is (num instances)x(num keypoints)x(side)x(side) + instances_per_image.pred_keypoints = keypoint_results_per_image + instances_per_image.pred_keypoint_heatmaps = heatmap_results_per_image + + +class BaseKeypointRCNNHead(nn.Module): + """ + Implement the basic Keypoint R-CNN losses and inference logic described in + Sec. 5 of :paper:`Mask R-CNN`. + """ + + @configurable + def __init__(self, *, num_keypoints, loss_weight=1.0, loss_normalizer=1.0): + """ + NOTE: this interface is experimental. + + Args: + num_keypoints (int): number of keypoints to predict + loss_weight (float): weight to multiple on the keypoint loss + loss_normalizer (float or str): + If float, divide the loss by `loss_normalizer * #images`. + If 'visible', the loss is normalized by the total number of + visible keypoints across images. + """ + super().__init__() + self.num_keypoints = num_keypoints + self.loss_weight = loss_weight + assert loss_normalizer == "visible" or isinstance(loss_normalizer, float), loss_normalizer + self.loss_normalizer = loss_normalizer + + @classmethod + def from_config(cls, cfg, input_shape): + ret = { + "loss_weight": cfg.MODEL.ROI_KEYPOINT_HEAD.LOSS_WEIGHT, + "num_keypoints": cfg.MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS, + } + normalize_by_visible = cfg.MODEL.ROI_KEYPOINT_HEAD.NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS # noqa + if not normalize_by_visible: + batch_size_per_image = cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE + positive_sample_fraction = cfg.MODEL.ROI_HEADS.POSITIVE_FRACTION + ret["loss_normalizer"] = ret["num_keypoints"] * batch_size_per_image * positive_sample_fraction + else: + ret["loss_normalizer"] = "visible" + return ret + + def forward(self, x, instances: List[Instances]): + """ + Args: + x: input 4D region feature(s) provided by :class:`ROIHeads`. + instances (list[Instances]): contains the boxes & labels corresponding + to the input features. + Exact format is up to its caller to decide. + Typically, this is the foreground instances in training, with + "proposal_boxes" field and other gt annotations. + In inference, it contains boxes that are already predicted. + + Returns: + A dict of losses if in training. The predicted "instances" if in inference. + """ + x = self.layers(x) + if self.training: + num_images = len(instances) + normalizer = None if self.loss_normalizer == "visible" else num_images * self.loss_normalizer + return {"loss_keypoint": keypoint_rcnn_loss(x, instances, normalizer=normalizer) * self.loss_weight} + else: + keypoint_rcnn_inference(x, instances) + return instances + + def layers(self, x): + """ + Neural network layers that makes predictions from regional input features. + """ + raise NotImplementedError + + +# To get torchscript support, we make the head a subclass of `nn.Sequential`. +# Therefore, to add new layers in this head class, please make sure they are +# added in the order they will be used in forward(). +@ROI_KEYPOINT_HEAD_REGISTRY.register() +class KRCNNConvDeconvUpsampleHead(BaseKeypointRCNNHead, nn.Sequential): + """ + A standard keypoint head containing a series of 3x3 convs, followed by + a transpose convolution and bilinear interpolation for upsampling. + It is described in Sec. 5 of :paper:`Mask R-CNN`. + """ + + @configurable + def __init__(self, input_shape, *, num_keypoints, conv_dims, **kwargs): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature + conv_dims: an iterable of output channel counts for each conv in the head + e.g. (512, 512, 512) for three convs outputting 512 channels. + """ + super().__init__(num_keypoints=num_keypoints, **kwargs) + + # default up_scale to 2.0 (this can be made an option) + up_scale = 2.0 + in_channels = input_shape.channels + + for idx, layer_channels in enumerate(conv_dims, 1): + module = Conv2d(in_channels, layer_channels, 3, stride=1, padding=1) + self.add_module("conv_fcn{}".format(idx), module) + self.add_module("conv_fcn_relu{}".format(idx), nn.ReLU()) + in_channels = layer_channels + + deconv_kernel = 4 + self.score_lowres = ConvTranspose2d( + in_channels, num_keypoints, deconv_kernel, stride=2, padding=deconv_kernel // 2 - 1 + ) + self.up_scale = up_scale + + for name, param in self.named_parameters(): + if "bias" in name: + nn.init.constant_(param, 0) + elif "weight" in name: + # Caffe2 implementation uses MSRAFill, which in fact + # corresponds to kaiming_normal_ in PyTorch + nn.init.kaiming_normal_(param, mode="fan_out", nonlinearity="relu") + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + ret["input_shape"] = input_shape + ret["conv_dims"] = cfg.MODEL.ROI_KEYPOINT_HEAD.CONV_DIMS + return ret + + def layers(self, x): + for layer in self: + x = layer(x) + x = interpolate(x, scale_factor=self.up_scale, mode="bilinear", align_corners=False) + return x diff --git a/detectron2/modeling/roi_heads/mask_head.py b/detectron2/modeling/roi_heads/mask_head.py new file mode 100644 index 0000000000000000000000000000000000000000..08a0658fef4b2dac941ac0a8924181b74ef456ae --- /dev/null +++ b/detectron2/modeling/roi_heads/mask_head.py @@ -0,0 +1,293 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List + +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.layers import Conv2d, ConvTranspose2d, ShapeSpec, cat, get_norm +from detectron2.layers.wrappers import move_device_like +from detectron2.structures import Instances +from detectron2.utils.events import get_event_storage +from detectron2.utils.registry import Registry + +__all__ = [ + "BaseMaskRCNNHead", + "MaskRCNNConvUpsampleHead", + "build_mask_head", + "ROI_MASK_HEAD_REGISTRY", +] + + +ROI_MASK_HEAD_REGISTRY = Registry("ROI_MASK_HEAD") +ROI_MASK_HEAD_REGISTRY.__doc__ = """ +Registry for mask heads, which predicts instance masks given +per-region features. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +@torch.jit.unused +def mask_rcnn_loss(pred_mask_logits: torch.Tensor, instances: List[Instances], vis_period: int = 0): + """ + Compute the mask prediction loss defined in the Mask R-CNN paper. + + Args: + pred_mask_logits (Tensor): A tensor of shape (B, C, Hmask, Wmask) or (B, 1, Hmask, Wmask) + for class-specific or class-agnostic, where B is the total number of predicted masks + in all images, C is the number of foreground classes, and Hmask, Wmask are the height + and width of the mask predictions. The values are logits. + instances (list[Instances]): A list of N Instances, where N is the number of images + in the batch. These instances are in 1:1 + correspondence with the pred_mask_logits. The ground-truth labels (class, box, mask, + ...) associated with each instance are stored in fields. + vis_period (int): the period (in steps) to dump visualization. + + Returns: + mask_loss (Tensor): A scalar tensor containing the loss. + """ + cls_agnostic_mask = pred_mask_logits.size(1) == 1 + total_num_masks = pred_mask_logits.size(0) + mask_side_len = pred_mask_logits.size(2) + assert pred_mask_logits.size(2) == pred_mask_logits.size(3), "Mask prediction must be square!" + + gt_classes = [] + gt_masks = [] + for instances_per_image in instances: + if len(instances_per_image) == 0: + continue + if not cls_agnostic_mask: + gt_classes_per_image = instances_per_image.gt_classes.to(dtype=torch.int64) + gt_classes.append(gt_classes_per_image) + + gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize( + instances_per_image.proposal_boxes.tensor, mask_side_len + ).to(device=pred_mask_logits.device) + # A tensor of shape (N, M, M), N=#instances in the image; M=mask_side_len + gt_masks.append(gt_masks_per_image) + + if len(gt_masks) == 0: + return pred_mask_logits.sum() * 0 + + gt_masks = cat(gt_masks, dim=0) + + if cls_agnostic_mask: + pred_mask_logits = pred_mask_logits[:, 0] + else: + indices = torch.arange(total_num_masks) + gt_classes = cat(gt_classes, dim=0) + pred_mask_logits = pred_mask_logits[indices, gt_classes] + + if gt_masks.dtype == torch.bool: + gt_masks_bool = gt_masks + else: + # Here we allow gt_masks to be float as well (depend on the implementation of rasterize()) + gt_masks_bool = gt_masks > 0.5 + gt_masks = gt_masks.to(dtype=torch.float32) + + # Log the training accuracy (using gt classes and 0.5 threshold) + mask_incorrect = (pred_mask_logits > 0.0) != gt_masks_bool + mask_accuracy = 1 - (mask_incorrect.sum().item() / max(mask_incorrect.numel(), 1.0)) + num_positive = gt_masks_bool.sum().item() + false_positive = (mask_incorrect & ~gt_masks_bool).sum().item() / max(gt_masks_bool.numel() - num_positive, 1.0) + false_negative = (mask_incorrect & gt_masks_bool).sum().item() / max(num_positive, 1.0) + + storage = get_event_storage() + storage.put_scalar("mask_rcnn/accuracy", mask_accuracy) + storage.put_scalar("mask_rcnn/false_positive", false_positive) + storage.put_scalar("mask_rcnn/false_negative", false_negative) + if vis_period > 0 and storage.iter % vis_period == 0: + pred_masks = pred_mask_logits.sigmoid() + vis_masks = torch.cat([pred_masks, gt_masks], axis=2) + name = "Left: mask prediction; Right: mask GT" + for idx, vis_mask in enumerate(vis_masks): + vis_mask = torch.stack([vis_mask] * 3, axis=0) + storage.put_image(name + f" ({idx})", vis_mask) + + mask_loss = F.binary_cross_entropy_with_logits(pred_mask_logits, gt_masks, reduction="mean") + return mask_loss + + +def mask_rcnn_inference(pred_mask_logits: torch.Tensor, pred_instances: List[Instances]): + """ + Convert pred_mask_logits to estimated foreground probability masks while also + extracting only the masks for the predicted classes in pred_instances. For each + predicted box, the mask of the same class is attached to the instance by adding a + new "pred_masks" field to pred_instances. + + Args: + pred_mask_logits (Tensor): A tensor of shape (B, C, Hmask, Wmask) or (B, 1, Hmask, Wmask) + for class-specific or class-agnostic, where B is the total number of predicted masks + in all images, C is the number of foreground classes, and Hmask, Wmask are the height + and width of the mask predictions. The values are logits. + pred_instances (list[Instances]): A list of N Instances, where N is the number of images + in the batch. Each Instances must have field "pred_classes". + + Returns: + None. pred_instances will contain an extra "pred_masks" field storing a mask of size (Hmask, + Wmask) for predicted class. Note that the masks are returned as a soft (non-quantized) + masks the resolution predicted by the network; post-processing steps, such as resizing + the predicted masks to the original image resolution and/or binarizing them, is left + to the caller. + """ + cls_agnostic_mask = pred_mask_logits.size(1) == 1 + + if cls_agnostic_mask: + mask_probs_pred = pred_mask_logits.sigmoid() + else: + # Select masks corresponding to the predicted classes + num_masks = pred_mask_logits.shape[0] + class_pred = cat([i.pred_classes for i in pred_instances]) + device = ( + class_pred.device if torch.jit.is_scripting() else ("cpu" if torch.jit.is_tracing() else class_pred.device) + ) + indices = move_device_like(torch.arange(num_masks, device=device), class_pred) + mask_probs_pred = pred_mask_logits[indices, class_pred][:, None].sigmoid() + # mask_probs_pred.shape: (B, 1, Hmask, Wmask) + + num_boxes_per_image = [len(i) for i in pred_instances] + mask_probs_pred = mask_probs_pred.split(num_boxes_per_image, dim=0) + + for prob, instances in zip(mask_probs_pred, pred_instances): + instances.pred_masks = prob # (1, Hmask, Wmask) + + +class BaseMaskRCNNHead(nn.Module): + """ + Implement the basic Mask R-CNN losses and inference logic described in :paper:`Mask R-CNN` + """ + + @configurable + def __init__(self, *, loss_weight: float = 1.0, vis_period: int = 0): + """ + NOTE: this interface is experimental. + + Args: + loss_weight (float): multiplier of the loss + vis_period (int): visualization period + """ + super().__init__() + self.vis_period = vis_period + self.loss_weight = loss_weight + + @classmethod + def from_config(cls, cfg, input_shape): + return {"vis_period": cfg.VIS_PERIOD} + + def forward(self, x, instances: List[Instances]): + """ + Args: + x: input region feature(s) provided by :class:`ROIHeads`. + instances (list[Instances]): contains the boxes & labels corresponding + to the input features. + Exact format is up to its caller to decide. + Typically, this is the foreground instances in training, with + "proposal_boxes" field and other gt annotations. + In inference, it contains boxes that are already predicted. + + Returns: + A dict of losses in training. The predicted "instances" in inference. + """ + x = self.layers(x) + if self.training: + return {"loss_mask": mask_rcnn_loss(x, instances, self.vis_period) * self.loss_weight} + else: + mask_rcnn_inference(x, instances) + return instances + + def layers(self, x): + """ + Neural network layers that makes predictions from input features. + """ + raise NotImplementedError + + +# To get torchscript support, we make the head a subclass of `nn.Sequential`. +# Therefore, to add new layers in this head class, please make sure they are +# added in the order they will be used in forward(). +@ROI_MASK_HEAD_REGISTRY.register() +class MaskRCNNConvUpsampleHead(BaseMaskRCNNHead, nn.Sequential): + """ + A mask head with several conv layers, plus an upsample layer (with `ConvTranspose2d`). + Predictions are made with a final 1x1 conv layer. + """ + + @configurable + def __init__(self, input_shape: ShapeSpec, *, num_classes, conv_dims, conv_norm="", **kwargs): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature + num_classes (int): the number of foreground classes (i.e. background is not + included). 1 if using class agnostic prediction. + conv_dims (list[int]): a list of N>0 integers representing the output dimensions + of N-1 conv layers and the last upsample layer. + conv_norm (str or callable): normalization for the conv layers. + See :func:`detectron2.layers.get_norm` for supported types. + """ + super().__init__(**kwargs) + assert len(conv_dims) >= 1, "conv_dims have to be non-empty!" + + self.conv_norm_relus = [] + + cur_channels = input_shape.channels + for k, conv_dim in enumerate(conv_dims[:-1]): + conv = Conv2d( + cur_channels, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=not conv_norm, + norm=get_norm(conv_norm, conv_dim), + activation=nn.ReLU(), + ) + self.add_module("mask_fcn{}".format(k + 1), conv) + self.conv_norm_relus.append(conv) + cur_channels = conv_dim + + self.deconv = ConvTranspose2d(cur_channels, conv_dims[-1], kernel_size=2, stride=2, padding=0) + self.add_module("deconv_relu", nn.ReLU()) + cur_channels = conv_dims[-1] + + self.predictor = Conv2d(cur_channels, num_classes, kernel_size=1, stride=1, padding=0) + + for layer in self.conv_norm_relus + [self.deconv]: + weight_init.c2_msra_fill(layer) + # use normal distribution initialization for mask prediction layer + nn.init.normal_(self.predictor.weight, std=0.001) + if self.predictor.bias is not None: + nn.init.constant_(self.predictor.bias, 0) + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + conv_dim = cfg.MODEL.ROI_MASK_HEAD.CONV_DIM + num_conv = cfg.MODEL.ROI_MASK_HEAD.NUM_CONV + ret.update( + conv_dims=[conv_dim] * (num_conv + 1), # +1 for ConvTranspose + conv_norm=cfg.MODEL.ROI_MASK_HEAD.NORM, + input_shape=input_shape, + ) + if cfg.MODEL.ROI_MASK_HEAD.CLS_AGNOSTIC_MASK: + ret["num_classes"] = 1 + else: + ret["num_classes"] = cfg.MODEL.ROI_HEADS.NUM_CLASSES + return ret + + def layers(self, x): + for layer in self: + x = layer(x) + return x + + +def build_mask_head(cfg, input_shape): + """ + Build a mask head defined by `cfg.MODEL.ROI_MASK_HEAD.NAME`. + """ + name = cfg.MODEL.ROI_MASK_HEAD.NAME + return ROI_MASK_HEAD_REGISTRY.get(name)(cfg, input_shape) diff --git a/detectron2/modeling/roi_heads/roi_heads.py b/detectron2/modeling/roi_heads/roi_heads.py new file mode 100644 index 0000000000000000000000000000000000000000..f941c2cd94f2dc5f41d8ff0cb955315011cbd4ea --- /dev/null +++ b/detectron2/modeling/roi_heads/roi_heads.py @@ -0,0 +1,859 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import inspect +import logging +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch +from torch import nn + +from detectron2.config import configurable +from detectron2.layers import ShapeSpec, nonzero_tuple +from detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage +from detectron2.utils.registry import Registry + +from ..backbone.resnet import BottleneckBlock, ResNet +from ..matcher import Matcher +from ..poolers import ROIPooler +from ..proposal_generator.proposal_utils import add_ground_truth_to_proposals +from ..sampling import subsample_labels +from .box_head import build_box_head +from .fast_rcnn import FastRCNNOutputLayers +from .keypoint_head import build_keypoint_head +from .mask_head import build_mask_head + +ROI_HEADS_REGISTRY = Registry("ROI_HEADS") +ROI_HEADS_REGISTRY.__doc__ = """ +Registry for ROI heads in a generalized R-CNN model. +ROIHeads take feature maps and region proposals, and +perform per-region computation. + +The registered object will be called with `obj(cfg, input_shape)`. +The call is expected to return an :class:`ROIHeads`. +""" + +logger = logging.getLogger(__name__) + + +def build_roi_heads(cfg, input_shape): + """ + Build ROIHeads defined by `cfg.MODEL.ROI_HEADS.NAME`. + """ + name = cfg.MODEL.ROI_HEADS.NAME + return ROI_HEADS_REGISTRY.get(name)(cfg, input_shape) + + +def select_foreground_proposals( + proposals: List[Instances], bg_label: int +) -> Tuple[List[Instances], List[torch.Tensor]]: + """ + Given a list of N Instances (for N images), each containing a `gt_classes` field, + return a list of Instances that contain only instances with `gt_classes != -1 && + gt_classes != bg_label`. + + Args: + proposals (list[Instances]): A list of N Instances, where N is the number of + images in the batch. + bg_label: label index of background class. + + Returns: + list[Instances]: N Instances, each contains only the selected foreground instances. + list[Tensor]: N boolean vector, correspond to the selection mask of + each Instances object. True for selected instances. + """ + assert isinstance(proposals, (list, tuple)) + assert isinstance(proposals[0], Instances) + assert proposals[0].has("gt_classes") + fg_proposals = [] + fg_selection_masks = [] + for proposals_per_image in proposals: + gt_classes = proposals_per_image.gt_classes + fg_selection_mask = (gt_classes != -1) & (gt_classes != bg_label) + fg_idxs = fg_selection_mask.nonzero().squeeze(1) + fg_proposals.append(proposals_per_image[fg_idxs]) + fg_selection_masks.append(fg_selection_mask) + return fg_proposals, fg_selection_masks + + +def select_proposals_with_visible_keypoints(proposals: List[Instances]) -> List[Instances]: + """ + Args: + proposals (list[Instances]): a list of N Instances, where N is the + number of images. + + Returns: + proposals: only contains proposals with at least one visible keypoint. + + Note that this is still slightly different from Detectron. + In Detectron, proposals for training keypoint head are re-sampled from + all the proposals with IOU>threshold & >=1 visible keypoint. + + Here, the proposals are first sampled from all proposals with + IOU>threshold, then proposals with no visible keypoint are filtered out. + This strategy seems to make no difference on Detectron and is easier to implement. + """ + ret = [] + all_num_fg = [] + for proposals_per_image in proposals: + # If empty/unannotated image (hard negatives), skip filtering for train + if len(proposals_per_image) == 0: + ret.append(proposals_per_image) + continue + gt_keypoints = proposals_per_image.gt_keypoints.tensor + # #fg x K x 3 + vis_mask = gt_keypoints[:, :, 2] >= 1 + xs, ys = gt_keypoints[:, :, 0], gt_keypoints[:, :, 1] + proposal_boxes = proposals_per_image.proposal_boxes.tensor.unsqueeze(dim=1) # #fg x 1 x 4 + kp_in_box = ( + (xs >= proposal_boxes[:, :, 0]) + & (xs <= proposal_boxes[:, :, 2]) + & (ys >= proposal_boxes[:, :, 1]) + & (ys <= proposal_boxes[:, :, 3]) + ) + selection = (kp_in_box & vis_mask).any(dim=1) + selection_idxs = nonzero_tuple(selection)[0] + all_num_fg.append(selection_idxs.numel()) + ret.append(proposals_per_image[selection_idxs]) + + storage = get_event_storage() + storage.put_scalar("keypoint_head/num_fg_samples", np.mean(all_num_fg)) + return ret + + +class ROIHeads(torch.nn.Module): + """ + ROIHeads perform all per-region computation in an R-CNN. + + It typically contains logic to + + 1. (in training only) match proposals with ground truth and sample them + 2. crop the regions and extract per-region features using proposals + 3. make per-region predictions with different heads + + It can have many variants, implemented as subclasses of this class. + This base class contains the logic to match/sample proposals. + But it is not necessary to inherit this class if the sampling logic is not needed. + """ + + @configurable + def __init__( + self, + *, + num_classes, + batch_size_per_image, + positive_fraction, + proposal_matcher, + proposal_append_gt=True, + ): + """ + NOTE: this interface is experimental. + + Args: + num_classes (int): number of foreground classes (i.e. background is not included) + batch_size_per_image (int): number of proposals to sample for training + positive_fraction (float): fraction of positive (foreground) proposals + to sample for training. + proposal_matcher (Matcher): matcher that matches proposals and ground truth + proposal_append_gt (bool): whether to include ground truth as proposals as well + """ + super().__init__() + self.batch_size_per_image = batch_size_per_image + self.positive_fraction = positive_fraction + self.num_classes = num_classes + self.proposal_matcher = proposal_matcher + self.proposal_append_gt = proposal_append_gt + + @classmethod + def from_config(cls, cfg): + return { + "batch_size_per_image": cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.ROI_HEADS.POSITIVE_FRACTION, + "num_classes": cfg.MODEL.ROI_HEADS.NUM_CLASSES, + "proposal_append_gt": cfg.MODEL.ROI_HEADS.PROPOSAL_APPEND_GT, + # Matcher to assign box proposals to gt boxes + "proposal_matcher": Matcher( + cfg.MODEL.ROI_HEADS.IOU_THRESHOLDS, + cfg.MODEL.ROI_HEADS.IOU_LABELS, + allow_low_quality_matches=False, + ), + } + + def _sample_proposals( + self, matched_idxs: torch.Tensor, matched_labels: torch.Tensor, gt_classes: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Based on the matching between N proposals and M groundtruth, + sample the proposals and set their classification labels. + + Args: + matched_idxs (Tensor): a vector of length N, each is the best-matched + gt index in [0, M) for each proposal. + matched_labels (Tensor): a vector of length N, the matcher's label + (one of cfg.MODEL.ROI_HEADS.IOU_LABELS) for each proposal. + gt_classes (Tensor): a vector of length M. + + Returns: + Tensor: a vector of indices of sampled proposals. Each is in [0, N). + Tensor: a vector of the same length, the classification label for + each sampled proposal. Each sample is labeled as either a category in + [0, num_classes) or the background (num_classes). + """ + has_gt = gt_classes.numel() > 0 + # Get the corresponding GT for each proposal + if has_gt: + gt_classes = gt_classes[matched_idxs] + # Label unmatched proposals (0 label from matcher) as background (label=num_classes) + gt_classes[matched_labels == 0] = self.num_classes + # Label ignore proposals (-1 label) + gt_classes[matched_labels == -1] = -1 + else: + gt_classes = torch.zeros_like(matched_idxs) + self.num_classes + + sampled_fg_idxs, sampled_bg_idxs = subsample_labels( + gt_classes, self.batch_size_per_image, self.positive_fraction, self.num_classes + ) + + sampled_idxs = torch.cat([sampled_fg_idxs, sampled_bg_idxs], dim=0) + return sampled_idxs, gt_classes[sampled_idxs] + + @torch.no_grad() + def label_and_sample_proposals(self, proposals: List[Instances], targets: List[Instances]) -> List[Instances]: + """ + Prepare some proposals to be used to train the ROI heads. + It performs box matching between `proposals` and `targets`, and assigns + training labels to the proposals. + It returns ``self.batch_size_per_image`` random samples from proposals and groundtruth + boxes, with a fraction of positives that is no larger than + ``self.positive_fraction``. + + Args: + See :meth:`ROIHeads.forward` + + Returns: + list[Instances]: + length `N` list of `Instances`s containing the proposals + sampled for training. Each `Instances` has the following fields: + + - proposal_boxes: the proposal boxes + - gt_boxes: the ground-truth box that the proposal is assigned to + (this is only meaningful if the proposal has a label > 0; if label = 0 + then the ground-truth box is random) + + Other fields such as "gt_classes", "gt_masks", that's included in `targets`. + """ + # Augment proposals with ground-truth boxes. + # In the case of learned proposals (e.g., RPN), when training starts + # the proposals will be low quality due to random initialization. + # It's possible that none of these initial + # proposals have high enough overlap with the gt objects to be used + # as positive examples for the second stage components (box head, + # cls head, mask head). Adding the gt boxes to the set of proposals + # ensures that the second stage components will have some positive + # examples from the start of training. For RPN, this augmentation improves + # convergence and empirically improves box AP on COCO by about 0.5 + # points (under one tested configuration). + if self.proposal_append_gt: + proposals = add_ground_truth_to_proposals(targets, proposals) + + proposals_with_gt = [] + + num_fg_samples = [] + num_bg_samples = [] + for proposals_per_image, targets_per_image in zip(proposals, targets): + has_gt = len(targets_per_image) > 0 + match_quality_matrix = pairwise_iou(targets_per_image.gt_boxes, proposals_per_image.proposal_boxes) + matched_idxs, matched_labels = self.proposal_matcher(match_quality_matrix) + sampled_idxs, gt_classes = self._sample_proposals( + matched_idxs, matched_labels, targets_per_image.gt_classes + ) + + # Set target attributes of the sampled proposals: + proposals_per_image = proposals_per_image[sampled_idxs] + proposals_per_image.gt_classes = gt_classes + + if has_gt: + sampled_targets = matched_idxs[sampled_idxs] + # We index all the attributes of targets that start with "gt_" + # and have not been added to proposals yet (="gt_classes"). + # NOTE: here the indexing waste some compute, because heads + # like masks, keypoints, etc, will filter the proposals again, + # (by foreground/background, or number of keypoints in the image, etc) + # so we essentially index the data twice. + for trg_name, trg_value in targets_per_image.get_fields().items(): + if trg_name.startswith("gt_") and not proposals_per_image.has(trg_name): + proposals_per_image.set(trg_name, trg_value[sampled_targets]) + # If no GT is given in the image, we don't know what a dummy gt value can be. + # Therefore the returned proposals won't have any gt_* fields, except for a + # gt_classes full of background label. + + num_bg_samples.append((gt_classes == self.num_classes).sum().item()) + num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1]) + proposals_with_gt.append(proposals_per_image) + + # Log the number of fg/bg samples that are selected for training ROI heads + storage = get_event_storage() + storage.put_scalar("roi_head/num_fg_samples", np.mean(num_fg_samples)) + storage.put_scalar("roi_head/num_bg_samples", np.mean(num_bg_samples)) + + return proposals_with_gt + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]: + """ + Args: + images (ImageList): + features (dict[str,Tensor]): input data as a mapping from feature + map name to tensor. Axis 0 represents the number of images `N` in + the input data; axes 1-3 are channels, height, and width, which may + vary between feature maps (e.g., if a feature pyramid is used). + proposals (list[Instances]): length `N` list of `Instances`. The i-th + `Instances` contains object proposals for the i-th input image, + with fields "proposal_boxes" and "objectness_logits". + targets (list[Instances], optional): length `N` list of `Instances`. The i-th + `Instances` contains the ground-truth per-instance annotations + for the i-th input image. Specify `targets` during training only. + It may have the following fields: + + - gt_boxes: the bounding box of each instance. + - gt_classes: the label for each instance with a category ranging in [0, #class]. + - gt_masks: PolygonMasks or BitMasks, the ground-truth masks of each instance. + - gt_keypoints: NxKx3, the groud-truth keypoints for each instance. + + Returns: + list[Instances]: length `N` list of `Instances` containing the + detected instances. Returned during inference only; may be [] during training. + + dict[str->Tensor]: + mapping from a named loss to a tensor storing the loss. Used during training only. + """ + raise NotImplementedError() + + +@ROI_HEADS_REGISTRY.register() +class Res5ROIHeads(ROIHeads): + """ + The ROIHeads in a typical "C4" R-CNN model, where + the box and mask head share the cropping and + the per-region feature computation by a Res5 block. + See :paper:`ResNet` Appendix A. + """ + + @configurable + def __init__( + self, + *, + in_features: List[str], + pooler: ROIPooler, + res5: nn.Module, + box_predictor: nn.Module, + mask_head: Optional[nn.Module] = None, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + in_features (list[str]): list of backbone feature map names to use for + feature extraction + pooler (ROIPooler): pooler to extra region features from backbone + res5 (nn.Sequential): a CNN to compute per-region features, to be used by + ``box_predictor`` and ``mask_head``. Typically this is a "res5" + block from a ResNet. + box_predictor (nn.Module): make box predictions from the feature. + Should have the same interface as :class:`FastRCNNOutputLayers`. + mask_head (nn.Module): transform features to make mask predictions + """ + super().__init__(**kwargs) + self.in_features = in_features + self.pooler = pooler + if isinstance(res5, (list, tuple)): + res5 = nn.Sequential(*res5) + self.res5 = res5 + self.box_predictor = box_predictor + self.mask_on = mask_head is not None + if self.mask_on: + self.mask_head = mask_head + + @classmethod + def from_config(cls, cfg, input_shape): + # fmt: off + ret = super().from_config(cfg) + in_features = ret["in_features"] = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + pooler_scales = (1.0 / input_shape[in_features[0]].stride, ) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + mask_on = cfg.MODEL.MASK_ON + # fmt: on + assert not cfg.MODEL.KEYPOINT_ON + assert len(in_features) == 1 + + ret["pooler"] = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + + # Compatbility with old moco code. Might be useful. + # See notes in StandardROIHeads.from_config + if not inspect.ismethod(cls._build_res5_block): + logger.warning("The behavior of _build_res5_block may change. " "Please do not depend on private methods.") + cls._build_res5_block = classmethod(cls._build_res5_block) + + ret["res5"], out_channels = cls._build_res5_block(cfg) + ret["box_predictor"] = FastRCNNOutputLayers(cfg, ShapeSpec(channels=out_channels, height=1, width=1)) + + if mask_on: + ret["mask_head"] = build_mask_head( + cfg, + ShapeSpec(channels=out_channels, width=pooler_resolution, height=pooler_resolution), + ) + return ret + + @classmethod + def _build_res5_block(cls, cfg): + # fmt: off + stage_channel_factor = 2 ** 3 # res5 is 8x res2 + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group * stage_channel_factor + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS * stage_channel_factor + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + norm = cfg.MODEL.RESNETS.NORM + assert not cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE[-1], \ + "Deformable conv is not yet supported in res5 head." + # fmt: on + + blocks = ResNet.make_stage( + BottleneckBlock, + 3, + stride_per_block=[2, 1, 1], + in_channels=out_channels // 2, + bottleneck_channels=bottleneck_channels, + out_channels=out_channels, + num_groups=num_groups, + norm=norm, + stride_in_1x1=stride_in_1x1, + ) + return nn.Sequential(*blocks), out_channels + + def _shared_roi_transform(self, features: List[torch.Tensor], boxes: List[Boxes]): + x = self.pooler(features, boxes) + return self.res5(x) + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ): + """ + See :meth:`ROIHeads.forward`. + """ + del images + + if self.training: + assert targets + proposals = self.label_and_sample_proposals(proposals, targets) + del targets + + proposal_boxes = [x.proposal_boxes for x in proposals] + box_features = self._shared_roi_transform([features[f] for f in self.in_features], proposal_boxes) + predictions = self.box_predictor(box_features.mean(dim=[2, 3])) + + if self.training: + del features + losses = self.box_predictor.losses(predictions, proposals) + if self.mask_on: + proposals, fg_selection_masks = select_foreground_proposals(proposals, self.num_classes) + # Since the ROI feature transform is shared between boxes and masks, + # we don't need to recompute features. The mask loss is only defined + # on foreground proposals, so we need to select out the foreground + # features. + mask_features = box_features[torch.cat(fg_selection_masks, dim=0)] + del box_features + losses.update(self.mask_head(mask_features, proposals)) + return [], losses + else: + pred_instances, _ = self.box_predictor.inference(predictions, proposals) + pred_instances = self.forward_with_given_boxes(features, pred_instances) + return pred_instances, {} + + def forward_with_given_boxes( + self, features: Dict[str, torch.Tensor], instances: List[Instances] + ) -> List[Instances]: + """ + Use the given boxes in `instances` to produce other (non-box) per-ROI outputs. + + Args: + features: same as in `forward()` + instances (list[Instances]): instances to predict other outputs. Expect the keys + "pred_boxes" and "pred_classes" to exist. + + Returns: + instances (Instances): + the same `Instances` object, with extra + fields such as `pred_masks` or `pred_keypoints`. + """ + assert not self.training + assert instances[0].has("pred_boxes") and instances[0].has("pred_classes") + + if self.mask_on: + feature_list = [features[f] for f in self.in_features] + x = self._shared_roi_transform(feature_list, [x.pred_boxes for x in instances]) + return self.mask_head(x, instances) + else: + return instances + + +@ROI_HEADS_REGISTRY.register() +class StandardROIHeads(ROIHeads): + """ + It's "standard" in a sense that there is no ROI transform sharing + or feature sharing between tasks. + Each head independently processes the input features by each head's + own pooler and head. + + This class is used by most models, such as FPN and C5. + To implement more models, you can subclass it and implement a different + :meth:`forward()` or a head. + """ + + @configurable + def __init__( + self, + *, + box_in_features: List[str], + box_pooler: ROIPooler, + box_head: nn.Module, + box_predictor: nn.Module, + mask_in_features: Optional[List[str]] = None, + mask_pooler: Optional[ROIPooler] = None, + mask_head: Optional[nn.Module] = None, + keypoint_in_features: Optional[List[str]] = None, + keypoint_pooler: Optional[ROIPooler] = None, + keypoint_head: Optional[nn.Module] = None, + train_on_pred_boxes: bool = False, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + box_in_features (list[str]): list of feature names to use for the box head. + box_pooler (ROIPooler): pooler to extra region features for box head + box_head (nn.Module): transform features to make box predictions + box_predictor (nn.Module): make box predictions from the feature. + Should have the same interface as :class:`FastRCNNOutputLayers`. + mask_in_features (list[str]): list of feature names to use for the mask + pooler or mask head. None if not using mask head. + mask_pooler (ROIPooler): pooler to extract region features from image features. + The mask head will then take region features to make predictions. + If None, the mask head will directly take the dict of image features + defined by `mask_in_features` + mask_head (nn.Module): transform features to make mask predictions + keypoint_in_features, keypoint_pooler, keypoint_head: similar to ``mask_*``. + train_on_pred_boxes (bool): whether to use proposal boxes or + predicted boxes from the box head to train other heads. + """ + super().__init__(**kwargs) + # keep self.in_features for backward compatibility + self.in_features = self.box_in_features = box_in_features + self.box_pooler = box_pooler + self.box_head = box_head + self.box_predictor = box_predictor + + self.mask_on = mask_in_features is not None + if self.mask_on: + self.mask_in_features = mask_in_features + self.mask_pooler = mask_pooler + self.mask_head = mask_head + + self.keypoint_on = keypoint_in_features is not None + if self.keypoint_on: + self.keypoint_in_features = keypoint_in_features + self.keypoint_pooler = keypoint_pooler + self.keypoint_head = keypoint_head + + self.train_on_pred_boxes = train_on_pred_boxes + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg) + ret["train_on_pred_boxes"] = cfg.MODEL.ROI_BOX_HEAD.TRAIN_ON_PRED_BOXES + # Subclasses that have not been updated to use from_config style construction + # may have overridden _init_*_head methods. In this case, those overridden methods + # will not be classmethods and we need to avoid trying to call them here. + # We test for this with ismethod which only returns True for bound methods of cls. + # Such subclasses will need to handle calling their overridden _init_*_head methods. + if inspect.ismethod(cls._init_box_head): + ret.update(cls._init_box_head(cfg, input_shape)) + if inspect.ismethod(cls._init_mask_head): + ret.update(cls._init_mask_head(cfg, input_shape)) + if inspect.ismethod(cls._init_keypoint_head): + ret.update(cls._init_keypoint_head(cfg, input_shape)) + return ret + + @classmethod + def _init_box_head(cls, cfg, input_shape): + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + # fmt: on + + # If StandardROIHeads is applied on multiple feature maps (as in FPN), + # then we share the same predictors and therefore the channel counts must be the same + in_channels = [input_shape[f].channels for f in in_features] + # Check all channel counts are equal + assert len(set(in_channels)) == 1, in_channels + in_channels = in_channels[0] + + box_pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + # Here we split "box head" and "box predictor", which is mainly due to historical reasons. + # They are used together so the "box predictor" layers should be part of the "box head". + # New subclasses of ROIHeads do not need "box predictor"s. + box_head = build_box_head( + cfg, ShapeSpec(channels=in_channels, height=pooler_resolution, width=pooler_resolution) + ) + box_predictor = FastRCNNOutputLayers(cfg, box_head.output_shape) + return { + "box_in_features": in_features, + "box_pooler": box_pooler, + "box_head": box_head, + "box_predictor": box_predictor, + } + + @classmethod + def _init_mask_head(cls, cfg, input_shape): + if not cfg.MODEL.MASK_ON: + return {} + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE + # fmt: on + + in_channels = [input_shape[f].channels for f in in_features][0] + + ret = {"mask_in_features": in_features} + ret["mask_pooler"] = ( + ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + if pooler_type + else None + ) + if pooler_type: + shape = ShapeSpec(channels=in_channels, width=pooler_resolution, height=pooler_resolution) + else: + shape = {f: input_shape[f] for f in in_features} + ret["mask_head"] = build_mask_head(cfg, shape) + return ret + + @classmethod + def _init_keypoint_head(cls, cfg, input_shape): + if not cfg.MODEL.KEYPOINT_ON: + return {} + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) # noqa + sampling_ratio = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_TYPE + # fmt: on + + in_channels = [input_shape[f].channels for f in in_features][0] + + ret = {"keypoint_in_features": in_features} + ret["keypoint_pooler"] = ( + ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + if pooler_type + else None + ) + if pooler_type: + shape = ShapeSpec(channels=in_channels, width=pooler_resolution, height=pooler_resolution) + else: + shape = {f: input_shape[f] for f in in_features} + ret["keypoint_head"] = build_keypoint_head(cfg, shape) + return ret + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]: + """ + See :class:`ROIHeads.forward`. + """ + del images + if self.training: + assert targets, "'targets' argument is required during training" + proposals = self.label_and_sample_proposals(proposals, targets) + del targets + + if self.training: + losses = self._forward_box(features, proposals) + # Usually the original proposals used by the box head are used by the mask, keypoint + # heads. But when `self.train_on_pred_boxes is True`, proposals will contain boxes + # predicted by the box head. + losses.update(self._forward_mask(features, proposals)) + losses.update(self._forward_keypoint(features, proposals)) + return proposals, losses + else: + pred_instances = self._forward_box(features, proposals) + # During inference cascaded prediction is used: the mask and keypoints heads are only + # applied to the top scoring box detections. + pred_instances = self.forward_with_given_boxes(features, pred_instances) + return pred_instances, {} + + def forward_with_given_boxes( + self, features: Dict[str, torch.Tensor], instances: List[Instances] + ) -> List[Instances]: + """ + Use the given boxes in `instances` to produce other (non-box) per-ROI outputs. + + This is useful for downstream tasks where a box is known, but need to obtain + other attributes (outputs of other heads). + Test-time augmentation also uses this. + + Args: + features: same as in `forward()` + instances (list[Instances]): instances to predict other outputs. Expect the keys + "pred_boxes" and "pred_classes" to exist. + + Returns: + list[Instances]: + the same `Instances` objects, with extra + fields such as `pred_masks` or `pred_keypoints`. + """ + assert not self.training + assert instances[0].has("pred_boxes") and instances[0].has("pred_classes") + + instances = self._forward_mask(features, instances) + instances = self._forward_keypoint(features, instances) + return instances + + def _forward_box(self, features: Dict[str, torch.Tensor], proposals: List[Instances]): + """ + Forward logic of the box prediction branch. If `self.train_on_pred_boxes is True`, + the function puts predicted boxes in the `proposal_boxes` field of `proposals` argument. + + Args: + features (dict[str, Tensor]): mapping from feature map names to tensor. + Same as in :meth:`ROIHeads.forward`. + proposals (list[Instances]): the per-image object proposals with + their matching ground truth. + Each has fields "proposal_boxes", and "objectness_logits", + "gt_classes", "gt_boxes". + + Returns: + In training, a dict of losses. + In inference, a list of `Instances`, the predicted instances. + """ + features = [features[f] for f in self.box_in_features] + box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals]) + box_features = self.box_head(box_features) + predictions = self.box_predictor(box_features) + del box_features + + if self.training: + losses = self.box_predictor.losses(predictions, proposals) + # proposals is modified in-place below, so losses must be computed first. + if self.train_on_pred_boxes: + with torch.no_grad(): + pred_boxes = self.box_predictor.predict_boxes_for_gt_classes(predictions, proposals) + for proposals_per_image, pred_boxes_per_image in zip(proposals, pred_boxes): + proposals_per_image.proposal_boxes = Boxes(pred_boxes_per_image) + return losses + else: + pred_instances, _ = self.box_predictor.inference(predictions, proposals) + return pred_instances + + def _forward_mask(self, features: Dict[str, torch.Tensor], instances: List[Instances]): + """ + Forward logic of the mask prediction branch. + + Args: + features (dict[str, Tensor]): mapping from feature map names to tensor. + Same as in :meth:`ROIHeads.forward`. + instances (list[Instances]): the per-image instances to train/predict masks. + In training, they can be the proposals. + In inference, they can be the boxes predicted by R-CNN box head. + + Returns: + In training, a dict of losses. + In inference, update `instances` with new fields "pred_masks" and return it. + """ + if not self.mask_on: + return {} if self.training else instances + + if self.training: + # head is only trained on positive proposals. + instances, _ = select_foreground_proposals(instances, self.num_classes) + + if self.mask_pooler is not None: + features = [features[f] for f in self.mask_in_features] + boxes = [x.proposal_boxes if self.training else x.pred_boxes for x in instances] + features = self.mask_pooler(features, boxes) + else: + features = {f: features[f] for f in self.mask_in_features} + return self.mask_head(features, instances) + + def _forward_keypoint(self, features: Dict[str, torch.Tensor], instances: List[Instances]): + """ + Forward logic of the keypoint prediction branch. + + Args: + features (dict[str, Tensor]): mapping from feature map names to tensor. + Same as in :meth:`ROIHeads.forward`. + instances (list[Instances]): the per-image instances to train/predict keypoints. + In training, they can be the proposals. + In inference, they can be the boxes predicted by R-CNN box head. + + Returns: + In training, a dict of losses. + In inference, update `instances` with new fields "pred_keypoints" and return it. + """ + if not self.keypoint_on: + return {} if self.training else instances + + if self.training: + # head is only trained on positive proposals with >=1 visible keypoints. + instances, _ = select_foreground_proposals(instances, self.num_classes) + instances = select_proposals_with_visible_keypoints(instances) + + if self.keypoint_pooler is not None: + features = [features[f] for f in self.keypoint_in_features] + boxes = [x.proposal_boxes if self.training else x.pred_boxes for x in instances] + features = self.keypoint_pooler(features, boxes) + else: + features = {f: features[f] for f in self.keypoint_in_features} + return self.keypoint_head(features, instances) diff --git a/detectron2/modeling/roi_heads/rotated_fast_rcnn.py b/detectron2/modeling/roi_heads/rotated_fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..42e34a4ed4e5650bbc97f31ef2c636aa2e16026b --- /dev/null +++ b/detectron2/modeling/roi_heads/rotated_fast_rcnn.py @@ -0,0 +1,262 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging + +import numpy as np +import torch + +from detectron2.config import configurable +from detectron2.layers import ShapeSpec, batched_nms_rotated +from detectron2.structures import Instances, RotatedBoxes, pairwise_iou_rotated +from detectron2.utils.events import get_event_storage + +from ..box_regression import Box2BoxTransformRotated +from ..poolers import ROIPooler +from ..proposal_generator.proposal_utils import add_ground_truth_to_proposals +from .box_head import build_box_head +from .fast_rcnn import FastRCNNOutputLayers +from .roi_heads import ROI_HEADS_REGISTRY, StandardROIHeads + +logger = logging.getLogger(__name__) + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + R: number of ROIs, combined over all images, in the minibatch + Ri: number of ROIs in image i + K: number of foreground classes. E.g.,there are 80 foreground classes in COCO. + +Naming convention: + + deltas: refers to the 5-d (dx, dy, dw, dh, da) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransformRotated`). + + pred_class_logits: predicted class scores in [-inf, +inf]; use + softmax(pred_class_logits) to estimate P(class). + + gt_classes: ground-truth classification labels in [0, K], where [0, K) represent + foreground object classes and K represents the background class. + + pred_proposal_deltas: predicted rotated box2box transform deltas for transforming proposals + to detection box predictions. + + gt_proposal_deltas: ground-truth rotated box2box transform deltas +""" + + +def fast_rcnn_inference_rotated(boxes, scores, image_shapes, score_thresh, nms_thresh, topk_per_image): + """ + Call `fast_rcnn_inference_single_image_rotated` for all images. + + Args: + boxes (list[Tensor]): A list of Tensors of predicted class-specific or class-agnostic + boxes for each image. Element i has shape (Ri, K * 5) if doing + class-specific regression, or (Ri, 5) if doing class-agnostic + regression, where Ri is the number of predicted objects for image i. + This is compatible with the output of :meth:`FastRCNNOutputLayers.predict_boxes`. + scores (list[Tensor]): A list of Tensors of predicted class scores for each image. + Element i has shape (Ri, K + 1), where Ri is the number of predicted objects + for image i. Compatible with the output of :meth:`FastRCNNOutputLayers.predict_probs`. + image_shapes (list[tuple]): A list of (width, height) tuples for each image in the batch. + score_thresh (float): Only return detections with a confidence score exceeding this + threshold. + nms_thresh (float): The threshold to use for box non-maximum suppression. Value in [0, 1]. + topk_per_image (int): The number of top scoring detections to return. Set < 0 to return + all detections. + + Returns: + instances: (list[Instances]): A list of N instances, one for each image in the batch, + that stores the topk most confidence detections. + kept_indices: (list[Tensor]): A list of 1D tensor of length of N, each element indicates + the corresponding boxes/scores index in [0, Ri) from the input, for image i. + """ + result_per_image = [ + fast_rcnn_inference_single_image_rotated( + boxes_per_image, scores_per_image, image_shape, score_thresh, nms_thresh, topk_per_image + ) + for scores_per_image, boxes_per_image, image_shape in zip(scores, boxes, image_shapes) + ] + return [x[0] for x in result_per_image], [x[1] for x in result_per_image] + + +@torch.no_grad() +def fast_rcnn_inference_single_image_rotated(boxes, scores, image_shape, score_thresh, nms_thresh, topk_per_image): + """ + Single-image inference. Return rotated bounding-box detection results by thresholding + on scores and applying rotated non-maximum suppression (Rotated NMS). + + Args: + Same as `fast_rcnn_inference_rotated`, but with rotated boxes, scores, and image shapes + per image. + + Returns: + Same as `fast_rcnn_inference_rotated`, but for only one image. + """ + valid_mask = torch.isfinite(boxes).all(dim=1) & torch.isfinite(scores).all(dim=1) + if not valid_mask.all(): + boxes = boxes[valid_mask] + scores = scores[valid_mask] + + B = 5 # box dimension + scores = scores[:, :-1] + num_bbox_reg_classes = boxes.shape[1] // B + # Convert to Boxes to use the `clip` function ... + boxes = RotatedBoxes(boxes.reshape(-1, B)) + boxes.clip(image_shape) + boxes = boxes.tensor.view(-1, num_bbox_reg_classes, B) # R x C x B + # Filter results based on detection scores + filter_mask = scores > score_thresh # R x K + # R' x 2. First column contains indices of the R predictions; + # Second column contains indices of classes. + filter_inds = filter_mask.nonzero() + if num_bbox_reg_classes == 1: + boxes = boxes[filter_inds[:, 0], 0] + else: + boxes = boxes[filter_mask] + scores = scores[filter_mask] + + # Apply per-class Rotated NMS + keep = batched_nms_rotated(boxes, scores, filter_inds[:, 1], nms_thresh) + if topk_per_image >= 0: + keep = keep[:topk_per_image] + boxes, scores, filter_inds = boxes[keep], scores[keep], filter_inds[keep] + + result = Instances(image_shape) + result.pred_boxes = RotatedBoxes(boxes) + result.scores = scores + result.pred_classes = filter_inds[:, 1] + + return result, filter_inds[:, 0] + + +class RotatedFastRCNNOutputLayers(FastRCNNOutputLayers): + """ + Two linear layers for predicting Rotated Fast R-CNN outputs. + """ + + @classmethod + def from_config(cls, cfg, input_shape): + args = super().from_config(cfg, input_shape) + args["box2box_transform"] = Box2BoxTransformRotated(weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS) + return args + + def inference(self, predictions, proposals): + """ + Returns: + list[Instances]: same as `fast_rcnn_inference_rotated`. + list[Tensor]: same as `fast_rcnn_inference_rotated`. + """ + boxes = self.predict_boxes(predictions, proposals) + scores = self.predict_probs(predictions, proposals) + image_shapes = [x.image_size for x in proposals] + + return fast_rcnn_inference_rotated( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + +@ROI_HEADS_REGISTRY.register() +class RROIHeads(StandardROIHeads): + """ + This class is used by Rotated Fast R-CNN to detect rotated boxes. + For now, it only supports box predictions but not mask or keypoints. + """ + + @configurable + def __init__(self, **kwargs): + """ + NOTE: this interface is experimental. + """ + super().__init__(**kwargs) + assert not self.mask_on and not self.keypoint_on, "Mask/Keypoints not supported in Rotated ROIHeads." + assert not self.train_on_pred_boxes, "train_on_pred_boxes not implemented for RROIHeads!" + + @classmethod + def _init_box_head(cls, cfg, input_shape): + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + # fmt: on + assert pooler_type in ["ROIAlignRotated"], pooler_type + # assume all channel counts are equal + in_channels = [input_shape[f].channels for f in in_features][0] + + box_pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + box_head = build_box_head( + cfg, ShapeSpec(channels=in_channels, height=pooler_resolution, width=pooler_resolution) + ) + # This line is the only difference v.s. StandardROIHeads + box_predictor = RotatedFastRCNNOutputLayers(cfg, box_head.output_shape) + return { + "box_in_features": in_features, + "box_pooler": box_pooler, + "box_head": box_head, + "box_predictor": box_predictor, + } + + @torch.no_grad() + def label_and_sample_proposals(self, proposals, targets): + """ + Prepare some proposals to be used to train the RROI heads. + It performs box matching between `proposals` and `targets`, and assigns + training labels to the proposals. + It returns `self.batch_size_per_image` random samples from proposals and groundtruth boxes, + with a fraction of positives that is no larger than `self.positive_sample_fraction. + + Args: + See :meth:`StandardROIHeads.forward` + + Returns: + list[Instances]: length `N` list of `Instances`s containing the proposals + sampled for training. Each `Instances` has the following fields: + - proposal_boxes: the rotated proposal boxes + - gt_boxes: the ground-truth rotated boxes that the proposal is assigned to + (this is only meaningful if the proposal has a label > 0; if label = 0 + then the ground-truth box is random) + - gt_classes: the ground-truth classification lable for each proposal + """ + if self.proposal_append_gt: + proposals = add_ground_truth_to_proposals(targets, proposals) + + proposals_with_gt = [] + + num_fg_samples = [] + num_bg_samples = [] + for proposals_per_image, targets_per_image in zip(proposals, targets): + has_gt = len(targets_per_image) > 0 + match_quality_matrix = pairwise_iou_rotated(targets_per_image.gt_boxes, proposals_per_image.proposal_boxes) + matched_idxs, matched_labels = self.proposal_matcher(match_quality_matrix) + sampled_idxs, gt_classes = self._sample_proposals( + matched_idxs, matched_labels, targets_per_image.gt_classes + ) + + proposals_per_image = proposals_per_image[sampled_idxs] + proposals_per_image.gt_classes = gt_classes + + if has_gt: + sampled_targets = matched_idxs[sampled_idxs] + proposals_per_image.gt_boxes = targets_per_image.gt_boxes[sampled_targets] + + num_bg_samples.append((gt_classes == self.num_classes).sum().item()) + num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1]) + proposals_with_gt.append(proposals_per_image) + + # Log the number of fg/bg samples that are selected for training ROI heads + storage = get_event_storage() + storage.put_scalar("roi_head/num_fg_samples", np.mean(num_fg_samples)) + storage.put_scalar("roi_head/num_bg_samples", np.mean(num_bg_samples)) + + return proposals_with_gt diff --git a/detectron2/modeling/sampling.py b/detectron2/modeling/sampling.py new file mode 100644 index 0000000000000000000000000000000000000000..99ffb8d52bd2e275b059472ef5adedf1884aafee --- /dev/null +++ b/detectron2/modeling/sampling.py @@ -0,0 +1,52 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from detectron2.layers import nonzero_tuple + +__all__ = ["subsample_labels"] + + +def subsample_labels(labels: torch.Tensor, num_samples: int, positive_fraction: float, bg_label: int): + """ + Return `num_samples` (or fewer, if not enough found) + random samples from `labels` which is a mixture of positives & negatives. + It will try to return as many positives as possible without + exceeding `positive_fraction * num_samples`, and then try to + fill the remaining slots with negatives. + + Args: + labels (Tensor): (N, ) label vector with values: + * -1: ignore + * bg_label: background ("negative") class + * otherwise: one or more foreground ("positive") classes + num_samples (int): The total number of labels with value >= 0 to return. + Values that are not sampled will be filled with -1 (ignore). + positive_fraction (float): The number of subsampled labels with values > 0 + is `min(num_positives, int(positive_fraction * num_samples))`. The number + of negatives sampled is `min(num_negatives, num_samples - num_positives_sampled)`. + In order words, if there are not enough positives, the sample is filled with + negatives. If there are also not enough negatives, then as many elements are + sampled as is possible. + bg_label (int): label index of background ("negative") class. + + Returns: + pos_idx, neg_idx (Tensor): + 1D vector of indices. The total length of both is `num_samples` or fewer. + """ + positive = nonzero_tuple((labels != -1) & (labels != bg_label))[0] + negative = nonzero_tuple(labels == bg_label)[0] + + num_pos = int(num_samples * positive_fraction) + # protect against not enough positive examples + num_pos = min(positive.numel(), num_pos) + num_neg = num_samples - num_pos + # protect against not enough negative examples + num_neg = min(negative.numel(), num_neg) + + # randomly select positive and negative examples + perm1 = torch.randperm(positive.numel(), device=positive.device)[:num_pos] + perm2 = torch.randperm(negative.numel(), device=negative.device)[:num_neg] + + pos_idx = positive[perm1] + neg_idx = negative[perm2] + return pos_idx, neg_idx diff --git a/detectron2/modeling/test_time_augmentation.py b/detectron2/modeling/test_time_augmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..c6d8885353f20311285a4c145ee149aaaa78ae74 --- /dev/null +++ b/detectron2/modeling/test_time_augmentation.py @@ -0,0 +1,304 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +from contextlib import contextmanager +from itertools import count +from typing import List + +import numpy as np +import torch +from fvcore.transforms import HFlipTransform, NoOpTransform +from torch import nn +from torch.nn.parallel import DistributedDataParallel + +from detectron2.config import configurable +from detectron2.data.detection_utils import read_image +from detectron2.data.transforms import ( + RandomFlip, + ResizeShortestEdge, + ResizeTransform, + apply_augmentations, +) +from detectron2.structures import Boxes, Instances + +from .meta_arch import GeneralizedRCNN +from .postprocessing import detector_postprocess +from .roi_heads.fast_rcnn import fast_rcnn_inference_single_image + +__all__ = ["DatasetMapperTTA", "GeneralizedRCNNWithTTA"] + + +class DatasetMapperTTA: + """ + Implement test-time augmentation for detection data. + It is a callable which takes a dataset dict from a detection dataset, + and returns a list of dataset dicts where the images + are augmented from the input image by the transformations defined in the config. + This is used for test-time augmentation. + """ + + @configurable + def __init__(self, min_sizes: List[int], max_size: int, flip: bool): + """ + Args: + min_sizes: list of short-edge size to resize the image to + max_size: maximum height or width of resized images + flip: whether to apply flipping augmentation + """ + self.min_sizes = min_sizes + self.max_size = max_size + self.flip = flip + + @classmethod + def from_config(cls, cfg): + return { + "min_sizes": cfg.TEST.AUG.MIN_SIZES, + "max_size": cfg.TEST.AUG.MAX_SIZE, + "flip": cfg.TEST.AUG.FLIP, + } + + def __call__(self, dataset_dict): + """ + Args: + dict: a dict in standard model input format. See tutorials for details. + + Returns: + list[dict]: + a list of dicts, which contain augmented version of the input image. + The total number of dicts is ``len(min_sizes) * (2 if flip else 1)``. + Each dict has field "transforms" which is a TransformList, + containing the transforms that are used to generate this image. + """ + numpy_image = dataset_dict["image"].permute(1, 2, 0).numpy() + shape = numpy_image.shape + orig_shape = (dataset_dict["height"], dataset_dict["width"]) + if shape[:2] != orig_shape: + # It transforms the "original" image in the dataset to the input image + pre_tfm = ResizeTransform(orig_shape[0], orig_shape[1], shape[0], shape[1]) + else: + pre_tfm = NoOpTransform() + + # Create all combinations of augmentations to use + aug_candidates = [] # each element is a list[Augmentation] + for min_size in self.min_sizes: + resize = ResizeShortestEdge(min_size, self.max_size) + aug_candidates.append([resize]) # resize only + if self.flip: + flip = RandomFlip(prob=1.0) + aug_candidates.append([resize, flip]) # resize + flip + + # Apply all the augmentations + ret = [] + for aug in aug_candidates: + new_image, tfms = apply_augmentations(aug, np.copy(numpy_image)) + torch_image = torch.from_numpy(np.ascontiguousarray(new_image.transpose(2, 0, 1))) + + dic = copy.deepcopy(dataset_dict) + dic["transforms"] = pre_tfm + tfms + dic["image"] = torch_image + ret.append(dic) + return ret + + +class GeneralizedRCNNWithTTA(nn.Module): + """ + A GeneralizedRCNN with test-time augmentation enabled. + Its :meth:`__call__` method has the same interface as :meth:`GeneralizedRCNN.forward`. + """ + + def __init__(self, cfg, model, tta_mapper=None, batch_size=3): + """ + Args: + cfg (CfgNode): + model (GeneralizedRCNN): a GeneralizedRCNN to apply TTA on. + tta_mapper (callable): takes a dataset dict and returns a list of + augmented versions of the dataset dict. Defaults to + `DatasetMapperTTA(cfg)`. + batch_size (int): batch the augmented images into this batch size for inference. + """ + super().__init__() + if isinstance(model, DistributedDataParallel): + model = model.module + assert isinstance( + model, GeneralizedRCNN + ), "TTA is only supported on GeneralizedRCNN. Got a model of type {}".format(type(model)) + self.cfg = cfg.clone() + assert not self.cfg.MODEL.KEYPOINT_ON, "TTA for keypoint is not supported yet" + assert not self.cfg.MODEL.LOAD_PROPOSALS, "TTA for pre-computed proposals is not supported yet" + + self.model = model + + if tta_mapper is None: + tta_mapper = DatasetMapperTTA(cfg) + self.tta_mapper = tta_mapper + self.batch_size = batch_size + + @contextmanager + def _turn_off_roi_heads(self, attrs): + """ + Open a context where some heads in `model.roi_heads` are temporarily turned off. + Args: + attr (list[str]): the attribute in `model.roi_heads` which can be used + to turn off a specific head, e.g., "mask_on", "keypoint_on". + """ + roi_heads = self.model.roi_heads + old = {} + for attr in attrs: + try: + old[attr] = getattr(roi_heads, attr) + except AttributeError: + # The head may not be implemented in certain ROIHeads + pass + + if len(old.keys()) == 0: + yield + else: + for attr in old.keys(): + setattr(roi_heads, attr, False) + yield + for attr in old.keys(): + setattr(roi_heads, attr, old[attr]) + + def _batch_inference(self, batched_inputs, detected_instances=None): + """ + Execute inference on a list of inputs, + using batch size = self.batch_size, instead of the length of the list. + + Inputs & outputs have the same format as :meth:`GeneralizedRCNN.inference` + """ + if detected_instances is None: + detected_instances = [None] * len(batched_inputs) + + outputs = [] + inputs, instances = [], [] + for idx, input, instance in zip(count(), batched_inputs, detected_instances): + inputs.append(input) + instances.append(instance) + if len(inputs) == self.batch_size or idx == len(batched_inputs) - 1: + outputs.extend( + self.model.inference( + inputs, + instances if instances[0] is not None else None, + do_postprocess=False, + ) + ) + inputs, instances = [], [] + return outputs + + def __call__(self, batched_inputs): + """ + Same input/output format as :meth:`GeneralizedRCNN.forward` + """ + + def _maybe_read_image(dataset_dict): + ret = copy.copy(dataset_dict) + if "image" not in ret: + image = read_image(ret.pop("file_name"), self.model.input_format) + image = torch.from_numpy(np.ascontiguousarray(image.transpose(2, 0, 1))) # CHW + ret["image"] = image + if "height" not in ret and "width" not in ret: + ret["height"] = image.shape[1] + ret["width"] = image.shape[2] + return ret + + return [self._inference_one_image(_maybe_read_image(x)) for x in batched_inputs] + + def _inference_one_image(self, input): + """ + Args: + input (dict): one dataset dict with "image" field being a CHW tensor + + Returns: + dict: one output dict + """ + orig_shape = (input["height"], input["width"]) + augmented_inputs, tfms = self._get_augmented_inputs(input) + # Detect boxes from all augmented versions + with self._turn_off_roi_heads(["mask_on", "keypoint_on"]): + # temporarily disable roi heads + all_boxes, all_scores, all_classes = self._get_augmented_boxes(augmented_inputs, tfms) + # merge all detected boxes to obtain final predictions for boxes + merged_instances = self._merge_detections(all_boxes, all_scores, all_classes, orig_shape) + + if self.cfg.MODEL.MASK_ON: + # Use the detected boxes to obtain masks + augmented_instances = self._rescale_detected_boxes(augmented_inputs, merged_instances, tfms) + # run forward on the detected boxes + outputs = self._batch_inference(augmented_inputs, augmented_instances) + # Delete now useless variables to avoid being out of memory + del augmented_inputs, augmented_instances + # average the predictions + merged_instances.pred_masks = self._reduce_pred_masks(outputs, tfms) + merged_instances = detector_postprocess(merged_instances, *orig_shape) + return {"instances": merged_instances} + else: + return {"instances": merged_instances} + + def _get_augmented_inputs(self, input): + augmented_inputs = self.tta_mapper(input) + tfms = [x.pop("transforms") for x in augmented_inputs] + return augmented_inputs, tfms + + def _get_augmented_boxes(self, augmented_inputs, tfms): + # 1: forward with all augmented images + outputs = self._batch_inference(augmented_inputs) + # 2: union the results + all_boxes = [] + all_scores = [] + all_classes = [] + for output, tfm in zip(outputs, tfms): + # Need to inverse the transforms on boxes, to obtain results on original image + pred_boxes = output.pred_boxes.tensor + original_pred_boxes = tfm.inverse().apply_box(pred_boxes.cpu().numpy()) + all_boxes.append(torch.from_numpy(original_pred_boxes).to(pred_boxes.device)) + + all_scores.extend(output.scores) + all_classes.extend(output.pred_classes) + all_boxes = torch.cat(all_boxes, dim=0) + return all_boxes, all_scores, all_classes + + def _merge_detections(self, all_boxes, all_scores, all_classes, shape_hw): + # select from the union of all results + num_boxes = len(all_boxes) + num_classes = self.cfg.MODEL.ROI_HEADS.NUM_CLASSES + # +1 because fast_rcnn_inference expects background scores as well + all_scores_2d = torch.zeros(num_boxes, num_classes + 1, device=all_boxes.device) + for idx, cls, score in zip(count(), all_classes, all_scores): + all_scores_2d[idx, cls] = score + + merged_instances, _ = fast_rcnn_inference_single_image( + all_boxes, + all_scores_2d, + shape_hw, + 1e-8, + self.cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST, + self.cfg.TEST.DETECTIONS_PER_IMAGE, + ) + + return merged_instances + + def _rescale_detected_boxes(self, augmented_inputs, merged_instances, tfms): + augmented_instances = [] + for input, tfm in zip(augmented_inputs, tfms): + # Transform the target box to the augmented image's coordinate space + pred_boxes = merged_instances.pred_boxes.tensor.cpu().numpy() + pred_boxes = torch.from_numpy(tfm.apply_box(pred_boxes)) + + aug_instances = Instances( + image_size=input["image"].shape[1:3], + pred_boxes=Boxes(pred_boxes), + pred_classes=merged_instances.pred_classes, + scores=merged_instances.scores, + ) + augmented_instances.append(aug_instances) + return augmented_instances + + def _reduce_pred_masks(self, outputs, tfms): + # Should apply inverse transforms on masks. + # We assume only resize & flip are used. pred_masks is a scale-invariant + # representation, so we handle flip specially + for output, tfm in zip(outputs, tfms): + if any(isinstance(t, HFlipTransform) for t in tfm.transforms): + output.pred_masks = output.pred_masks.flip(dims=[3]) + all_pred_masks = torch.stack([o.pred_masks for o in outputs], dim=0) + avg_pred_masks = torch.mean(all_pred_masks, dim=0) + return avg_pred_masks diff --git a/detectron2/projects/README.md b/detectron2/projects/README.md new file mode 100644 index 0000000000000000000000000000000000000000..95afe7ff8c8a9bd2f56621fcc3c1bdac11c256a9 --- /dev/null +++ b/detectron2/projects/README.md @@ -0,0 +1,2 @@ + +Projects live in the [`projects` directory](../../projects) under the root of this repository, but not here. diff --git a/detectron2/projects/__init__.py b/detectron2/projects/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a68207db4ee3c2578e1042b00b3071a946b7adea --- /dev/null +++ b/detectron2/projects/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import importlib +from pathlib import Path + +_PROJECTS = { + "point_rend": "PointRend", + "deeplab": "DeepLab", + "panoptic_deeplab": "Panoptic-DeepLab", +} +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent / "projects" + +if _PROJECT_ROOT.is_dir(): + # This is true only for in-place installation (pip install -e, setup.py develop), + # where setup(package_dir=) does not work: https://github.com/pypa/setuptools/issues/230 + + class _D2ProjectsFinder(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if not name.startswith("detectron2.projects."): + return + project_name = name.split(".")[-1] + project_dir = _PROJECTS.get(project_name) + if not project_dir: + return + target_file = _PROJECT_ROOT / f"{project_dir}/{project_name}/__init__.py" + if not target_file.is_file(): + return + return importlib.util.spec_from_file_location(name, target_file) + + import sys + + sys.meta_path.append(_D2ProjectsFinder()) diff --git a/detectron2/solver/__init__.py b/detectron2/solver/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..42d7dd82b1a2b76de05f6d64c27ad6de7d90464e --- /dev/null +++ b/detectron2/solver/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build import build_lr_scheduler, build_optimizer, get_default_optimizer_params +from .lr_scheduler import LRMultiplier, WarmupCosineLR, WarmupMultiStepLR, WarmupParamScheduler + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/detectron2/solver/build.py b/detectron2/solver/build.py new file mode 100644 index 0000000000000000000000000000000000000000..fd25d933940a941c64cc47b888b7411d8ca1bc02 --- /dev/null +++ b/detectron2/solver/build.py @@ -0,0 +1,292 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +import logging +from collections import defaultdict +from enum import Enum +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Type, Union + +import torch +from fvcore.common.param_scheduler import CosineParamScheduler, MultiStepParamScheduler + +from detectron2.config import CfgNode + +from .lr_scheduler import LRMultiplier, WarmupParamScheduler + +_GradientClipperInput = Union[torch.Tensor, Iterable[torch.Tensor]] +_GradientClipper = Callable[[_GradientClipperInput], None] + + +class GradientClipType(Enum): + VALUE = "value" + NORM = "norm" + + +def _create_gradient_clipper(cfg: CfgNode) -> _GradientClipper: + """ + Creates gradient clipping closure to clip by value or by norm, + according to the provided config. + """ + cfg = copy.deepcopy(cfg) + + def clip_grad_norm(p: _GradientClipperInput): + torch.nn.utils.clip_grad_norm_(p, cfg.CLIP_VALUE, cfg.NORM_TYPE) + + def clip_grad_value(p: _GradientClipperInput): + torch.nn.utils.clip_grad_value_(p, cfg.CLIP_VALUE) + + _GRADIENT_CLIP_TYPE_TO_CLIPPER = { + GradientClipType.VALUE: clip_grad_value, + GradientClipType.NORM: clip_grad_norm, + } + return _GRADIENT_CLIP_TYPE_TO_CLIPPER[GradientClipType(cfg.CLIP_TYPE)] + + +def _generate_optimizer_class_with_gradient_clipping( + optimizer: Type[torch.optim.Optimizer], + *, + per_param_clipper: Optional[_GradientClipper] = None, + global_clipper: Optional[_GradientClipper] = None, +) -> Type[torch.optim.Optimizer]: + """ + Dynamically creates a new type that inherits the type of a given instance + and overrides the `step` method to add gradient clipping + """ + assert ( + per_param_clipper is None or global_clipper is None + ), "Not allowed to use both per-parameter clipping and global clipping" + + def optimizer_wgc_step(self, closure=None): + if per_param_clipper is not None: + for group in self.param_groups: + for p in group["params"]: + per_param_clipper(p) + else: + # global clipper for future use with detr + # (https://github.com/facebookresearch/detr/pull/287) + all_params = itertools.chain(*[g["params"] for g in self.param_groups]) + global_clipper(all_params) + super(type(self), self).step(closure) + + OptimizerWithGradientClip = type( + optimizer.__name__ + "WithGradientClip", + (optimizer,), + {"step": optimizer_wgc_step}, + ) + return OptimizerWithGradientClip + + +def maybe_add_gradient_clipping(cfg: CfgNode, optimizer: Type[torch.optim.Optimizer]) -> Type[torch.optim.Optimizer]: + """ + If gradient clipping is enabled through config options, wraps the existing + optimizer type to become a new dynamically created class OptimizerWithGradientClip + that inherits the given optimizer and overrides the `step` method to + include gradient clipping. + + Args: + cfg: CfgNode, configuration options + optimizer: type. A subclass of torch.optim.Optimizer + + Return: + type: either the input `optimizer` (if gradient clipping is disabled), or + a subclass of it with gradient clipping included in the `step` method. + """ + if not cfg.SOLVER.CLIP_GRADIENTS.ENABLED: + return optimizer + if isinstance(optimizer, torch.optim.Optimizer): + optimizer_type = type(optimizer) + else: + assert issubclass(optimizer, torch.optim.Optimizer), optimizer + optimizer_type = optimizer + + grad_clipper = _create_gradient_clipper(cfg.SOLVER.CLIP_GRADIENTS) + OptimizerWithGradientClip = _generate_optimizer_class_with_gradient_clipping( + optimizer_type, per_param_clipper=grad_clipper + ) + if isinstance(optimizer, torch.optim.Optimizer): + optimizer.__class__ = OptimizerWithGradientClip # a bit hacky, not recommended + return optimizer + else: + return OptimizerWithGradientClip + + +def build_optimizer(cfg: CfgNode, model: torch.nn.Module) -> torch.optim.Optimizer: + """ + Build an optimizer from config. + """ + params = get_default_optimizer_params( + model, + base_lr=cfg.SOLVER.BASE_LR, + weight_decay_norm=cfg.SOLVER.WEIGHT_DECAY_NORM, + bias_lr_factor=cfg.SOLVER.BIAS_LR_FACTOR, + weight_decay_bias=cfg.SOLVER.WEIGHT_DECAY_BIAS, + ) + return maybe_add_gradient_clipping(cfg, torch.optim.SGD)( + params, + lr=cfg.SOLVER.BASE_LR, + momentum=cfg.SOLVER.MOMENTUM, + nesterov=cfg.SOLVER.NESTEROV, + weight_decay=cfg.SOLVER.WEIGHT_DECAY, + ) + + +def get_default_optimizer_params( + model: torch.nn.Module, + base_lr: Optional[float] = None, + weight_decay: Optional[float] = None, + weight_decay_norm: Optional[float] = None, + bias_lr_factor: Optional[float] = 1.0, + weight_decay_bias: Optional[float] = None, + lr_factor_func: Optional[Callable] = None, + overrides: Optional[Dict[str, Dict[str, float]]] = None, +) -> List[Dict[str, Any]]: + """ + Get default param list for optimizer, with support for a few types of + overrides. If no overrides needed, this is equivalent to `model.parameters()`. + + Args: + base_lr: lr for every group by default. Can be omitted to use the one in optimizer. + weight_decay: weight decay for every group by default. Can be omitted to use the one + in optimizer. + weight_decay_norm: override weight decay for params in normalization layers + bias_lr_factor: multiplier of lr for bias parameters. + weight_decay_bias: override weight decay for bias parameters. + lr_factor_func: function to calculate lr decay rate by mapping the parameter names to + corresponding lr decay rate. Note that setting this option requires + also setting ``base_lr``. + overrides: if not `None`, provides values for optimizer hyperparameters + (LR, weight decay) for module parameters with a given name; e.g. + ``{"embedding": {"lr": 0.01, "weight_decay": 0.1}}`` will set the LR and + weight decay values for all module parameters named `embedding`. + + For common detection models, ``weight_decay_norm`` is the only option + needed to be set. ``bias_lr_factor,weight_decay_bias`` are legacy settings + from Detectron1 that are not found useful. + + Example: + :: + torch.optim.SGD(get_default_optimizer_params(model, weight_decay_norm=0), + lr=0.01, weight_decay=1e-4, momentum=0.9) + """ + if overrides is None: + overrides = {} + defaults = {} + if base_lr is not None: + defaults["lr"] = base_lr + if weight_decay is not None: + defaults["weight_decay"] = weight_decay + bias_overrides = {} + if bias_lr_factor is not None and bias_lr_factor != 1.0: + # NOTE: unlike Detectron v1, we now by default make bias hyperparameters + # exactly the same as regular weights. + if base_lr is None: + raise ValueError("bias_lr_factor requires base_lr") + bias_overrides["lr"] = base_lr * bias_lr_factor + if weight_decay_bias is not None: + bias_overrides["weight_decay"] = weight_decay_bias + if len(bias_overrides): + if "bias" in overrides: + raise ValueError("Conflicting overrides for 'bias'") + overrides["bias"] = bias_overrides + if lr_factor_func is not None: + if base_lr is None: + raise ValueError("lr_factor_func requires base_lr") + norm_module_types = ( + torch.nn.BatchNorm1d, + torch.nn.BatchNorm2d, + torch.nn.BatchNorm3d, + torch.nn.SyncBatchNorm, + # NaiveSyncBatchNorm inherits from BatchNorm2d + torch.nn.GroupNorm, + torch.nn.InstanceNorm1d, + torch.nn.InstanceNorm2d, + torch.nn.InstanceNorm3d, + torch.nn.LayerNorm, + torch.nn.LocalResponseNorm, + ) + params: List[Dict[str, Any]] = [] + memo: Set[torch.nn.parameter.Parameter] = set() + for module_name, module in model.named_modules(): + for module_param_name, value in module.named_parameters(recurse=False): + if not value.requires_grad: + continue + # Avoid duplicating parameters + if value in memo: + continue + memo.add(value) + + hyperparams = copy.copy(defaults) + if isinstance(module, norm_module_types) and weight_decay_norm is not None: + hyperparams["weight_decay"] = weight_decay_norm + if lr_factor_func is not None: + hyperparams["lr"] *= lr_factor_func(f"{module_name}.{module_param_name}") + + hyperparams.update(overrides.get(module_param_name, {})) + params.append({"params": [value], **hyperparams}) + return reduce_param_groups(params) + + +def _expand_param_groups(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + # Transform parameter groups into per-parameter structure. + # Later items in `params` can overwrite parameters set in previous items. + ret = defaultdict(dict) + for item in params: + assert "params" in item + cur_params = {x: y for x, y in item.items() if x != "params"} + for param in item["params"]: + ret[param].update({"params": [param], **cur_params}) + return list(ret.values()) + + +def reduce_param_groups(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + # Reorganize the parameter groups and merge duplicated groups. + # The number of parameter groups needs to be as small as possible in order + # to efficiently use the PyTorch multi-tensor optimizer. Therefore instead + # of using a parameter_group per single parameter, we reorganize the + # parameter groups and merge duplicated groups. This approach speeds + # up multi-tensor optimizer significantly. + params = _expand_param_groups(params) + groups = defaultdict(list) # re-group all parameter groups by their hyperparams + for item in params: + cur_params = tuple((x, y) for x, y in item.items() if x != "params") + groups[cur_params].extend(item["params"]) + ret = [] + for param_keys, param_values in groups.items(): + cur = {kv[0]: kv[1] for kv in param_keys} + cur["params"] = param_values + ret.append(cur) + return ret + + +def build_lr_scheduler(cfg: CfgNode, optimizer: torch.optim.Optimizer) -> torch.optim.lr_scheduler._LRScheduler: + """ + Build a LR scheduler from config. + """ + name = cfg.SOLVER.LR_SCHEDULER_NAME + + if name == "WarmupMultiStepLR": + steps = [x for x in cfg.SOLVER.STEPS if x <= cfg.SOLVER.MAX_ITER] + if len(steps) != len(cfg.SOLVER.STEPS): + logger = logging.getLogger(__name__) + logger.warning( + "SOLVER.STEPS contains values larger than SOLVER.MAX_ITER. " "These values will be ignored." + ) + sched = MultiStepParamScheduler( + values=[cfg.SOLVER.GAMMA**k for k in range(len(steps) + 1)], + milestones=steps, + num_updates=cfg.SOLVER.MAX_ITER, + ) + elif name == "WarmupCosineLR": + end_value = cfg.SOLVER.BASE_LR_END / cfg.SOLVER.BASE_LR + assert end_value >= 0.0 and end_value <= 1.0, end_value + sched = CosineParamScheduler(1, end_value) + else: + raise ValueError("Unknown LR scheduler: {}".format(name)) + + sched = WarmupParamScheduler( + sched, + cfg.SOLVER.WARMUP_FACTOR, + min(cfg.SOLVER.WARMUP_ITERS / cfg.SOLVER.MAX_ITER, 1.0), + cfg.SOLVER.WARMUP_METHOD, + ) + return LRMultiplier(optimizer, multiplier=sched, max_iter=cfg.SOLVER.MAX_ITER) diff --git a/detectron2/solver/lr_scheduler.py b/detectron2/solver/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..91f68c5a314992a1f0e8a0b008a6a766993f9df7 --- /dev/null +++ b/detectron2/solver/lr_scheduler.py @@ -0,0 +1,228 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import math +from bisect import bisect_right +from typing import List + +import torch +from fvcore.common.param_scheduler import ( + CompositeParamScheduler, + ConstantParamScheduler, + LinearParamScheduler, + ParamScheduler, +) + +logger = logging.getLogger(__name__) + + +class WarmupParamScheduler(CompositeParamScheduler): + """ + Add an initial warmup stage to another scheduler. + """ + + def __init__( + self, + scheduler: ParamScheduler, + warmup_factor: float, + warmup_length: float, + warmup_method: str = "linear", + ): + """ + Args: + scheduler: warmup will be added at the beginning of this scheduler + warmup_factor: the factor w.r.t the initial value of ``scheduler``, e.g. 0.001 + warmup_length: the relative length (in [0, 1]) of warmup steps w.r.t the entire + training, e.g. 0.01 + warmup_method: one of "linear" or "constant" + """ + end_value = scheduler(warmup_length) # the value to reach when warmup ends + start_value = warmup_factor * scheduler(0.0) + if warmup_method == "constant": + warmup = ConstantParamScheduler(start_value) + elif warmup_method == "linear": + warmup = LinearParamScheduler(start_value, end_value) + else: + raise ValueError("Unknown warmup method: {}".format(warmup_method)) + super().__init__( + [warmup, scheduler], + interval_scaling=["rescaled", "fixed"], + lengths=[warmup_length, 1 - warmup_length], + ) + + +class LRMultiplier(torch.optim.lr_scheduler._LRScheduler): + """ + A LRScheduler which uses fvcore :class:`ParamScheduler` to multiply the + learning rate of each param in the optimizer. + Every step, the learning rate of each parameter becomes its initial value + multiplied by the output of the given :class:`ParamScheduler`. + + The absolute learning rate value of each parameter can be different. + This scheduler can be used as long as the relative scale among them do + not change during training. + + Examples: + :: + LRMultiplier( + opt, + WarmupParamScheduler( + MultiStepParamScheduler( + [1, 0.1, 0.01], + milestones=[60000, 80000], + num_updates=90000, + ), 0.001, 100 / 90000 + ), + max_iter=90000 + ) + """ + + # NOTES: in the most general case, every LR can use its own scheduler. + # Supporting this requires interaction with the optimizer when its parameter + # group is initialized. For example, classyvision implements its own optimizer + # that allows different schedulers for every parameter group. + # To avoid this complexity, we use this class to support the most common cases + # where the relative scale among all LRs stay unchanged during training. In this + # case we only need a total of one scheduler that defines the relative LR multiplier. + + def __init__( + self, + optimizer: torch.optim.Optimizer, + multiplier: ParamScheduler, + max_iter: int, + last_iter: int = -1, + ): + """ + Args: + optimizer, last_iter: See ``torch.optim.lr_scheduler._LRScheduler``. + ``last_iter`` is the same as ``last_epoch``. + multiplier: a fvcore ParamScheduler that defines the multiplier on + every LR of the optimizer + max_iter: the total number of training iterations + """ + if not isinstance(multiplier, ParamScheduler): + raise ValueError( + "_LRMultiplier(multiplier=) must be an instance of fvcore " + f"ParamScheduler. Got {multiplier} instead." + ) + self._multiplier = multiplier + self._max_iter = max_iter + super().__init__(optimizer, last_epoch=last_iter) + + def state_dict(self): + # fvcore schedulers are stateless. Only keep pytorch scheduler states + return {"base_lrs": self.base_lrs, "last_epoch": self.last_epoch} + + def get_lr(self) -> List[float]: + multiplier = self._multiplier(self.last_epoch / self._max_iter) + return [base_lr * multiplier for base_lr in self.base_lrs] + + +""" +Content below is no longer needed! +""" + +# NOTE: PyTorch's LR scheduler interface uses names that assume the LR changes +# only on epoch boundaries. We typically use iteration based schedules instead. +# As a result, "epoch" (e.g., as in self.last_epoch) should be understood to mean +# "iteration" instead. + +# FIXME: ideally this would be achieved with a CombinedLRScheduler, separating +# MultiStepLR with WarmupLR but the current LRScheduler design doesn't allow it. + + +class WarmupMultiStepLR(torch.optim.lr_scheduler._LRScheduler): + def __init__( + self, + optimizer: torch.optim.Optimizer, + milestones: List[int], + gamma: float = 0.1, + warmup_factor: float = 0.001, + warmup_iters: int = 1000, + warmup_method: str = "linear", + last_epoch: int = -1, + ): + logger.warning("WarmupMultiStepLR is deprecated! Use LRMultipilier with fvcore ParamScheduler instead!") + if not list(milestones) == sorted(milestones): + raise ValueError("Milestones should be a list of" " increasing integers. Got {}", milestones) + self.milestones = milestones + self.gamma = gamma + self.warmup_factor = warmup_factor + self.warmup_iters = warmup_iters + self.warmup_method = warmup_method + super().__init__(optimizer, last_epoch) + + def get_lr(self) -> List[float]: + warmup_factor = _get_warmup_factor_at_iter( + self.warmup_method, self.last_epoch, self.warmup_iters, self.warmup_factor + ) + return [ + base_lr * warmup_factor * self.gamma ** bisect_right(self.milestones, self.last_epoch) + for base_lr in self.base_lrs + ] + + def _compute_values(self) -> List[float]: + # The new interface + return self.get_lr() + + +class WarmupCosineLR(torch.optim.lr_scheduler._LRScheduler): + def __init__( + self, + optimizer: torch.optim.Optimizer, + max_iters: int, + warmup_factor: float = 0.001, + warmup_iters: int = 1000, + warmup_method: str = "linear", + last_epoch: int = -1, + ): + logger.warning("WarmupCosineLR is deprecated! Use LRMultipilier with fvcore ParamScheduler instead!") + self.max_iters = max_iters + self.warmup_factor = warmup_factor + self.warmup_iters = warmup_iters + self.warmup_method = warmup_method + super().__init__(optimizer, last_epoch) + + def get_lr(self) -> List[float]: + warmup_factor = _get_warmup_factor_at_iter( + self.warmup_method, self.last_epoch, self.warmup_iters, self.warmup_factor + ) + # Different definitions of half-cosine with warmup are possible. For + # simplicity we multiply the standard half-cosine schedule by the warmup + # factor. An alternative is to start the period of the cosine at warmup_iters + # instead of at 0. In the case that warmup_iters << max_iters the two are + # very close to each other. + return [ + base_lr * warmup_factor * 0.5 * (1.0 + math.cos(math.pi * self.last_epoch / self.max_iters)) + for base_lr in self.base_lrs + ] + + def _compute_values(self) -> List[float]: + # The new interface + return self.get_lr() + + +def _get_warmup_factor_at_iter(method: str, iter: int, warmup_iters: int, warmup_factor: float) -> float: + """ + Return the learning rate warmup factor at a specific iteration. + See :paper:`ImageNet in 1h` for more details. + + Args: + method (str): warmup method; either "constant" or "linear". + iter (int): iteration at which to calculate the warmup factor. + warmup_iters (int): the number of warmup iterations. + warmup_factor (float): the base warmup factor (the meaning changes according + to the method used). + + Returns: + float: the effective warmup factor at the given iteration. + """ + if iter >= warmup_iters: + return 1.0 + + if method == "constant": + return warmup_factor + elif method == "linear": + alpha = iter / warmup_iters + return warmup_factor * (1 - alpha) + alpha + else: + raise ValueError("Unknown warmup method: {}".format(method)) diff --git a/detectron2/structures/__init__.py b/detectron2/structures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4a161bdb4579416a89310ee3744431017f75c2d4 --- /dev/null +++ b/detectron2/structures/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .boxes import Boxes, BoxMode, pairwise_ioa, pairwise_iou, pairwise_point_box_distance +from .image_list import ImageList +from .instances import Instances +from .keypoints import Keypoints, heatmaps_to_keypoints +from .masks import BitMasks, PolygonMasks, ROIMasks, polygons_to_bitmask +from .rotated_boxes import RotatedBoxes +from .rotated_boxes import pairwise_iou as pairwise_iou_rotated + +__all__ = [k for k in globals().keys() if not k.startswith("_")] + + +from detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/detectron2/structures/boxes.py b/detectron2/structures/boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..ff1586676ccdbe398c024ea6674f149b5f80c0ea --- /dev/null +++ b/detectron2/structures/boxes.py @@ -0,0 +1,418 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from enum import IntEnum, unique +from typing import List, Tuple, Union + +import numpy as np +import torch +from torch import device + +_RawBoxType = Union[List[float], Tuple[float, ...], torch.Tensor, np.ndarray] + + +@unique +class BoxMode(IntEnum): + """ + Enum of different ways to represent a box. + """ + + XYXY_ABS = 0 + """ + (x0, y0, x1, y1) in absolute floating points coordinates. + The coordinates in range [0, width or height]. + """ + XYWH_ABS = 1 + """ + (x0, y0, w, h) in absolute floating points coordinates. + """ + XYXY_REL = 2 + """ + Not yet supported! + (x0, y0, x1, y1) in range [0, 1]. They are relative to the size of the image. + """ + XYWH_REL = 3 + """ + Not yet supported! + (x0, y0, w, h) in range [0, 1]. They are relative to the size of the image. + """ + XYWHA_ABS = 4 + """ + (xc, yc, w, h, a) in absolute floating points coordinates. + (xc, yc) is the center of the rotated box, and the angle a is in degrees ccw. + """ + + @staticmethod + def convert(box: _RawBoxType, from_mode: "BoxMode", to_mode: "BoxMode") -> _RawBoxType: + """ + Args: + box: can be a k-tuple, k-list or an Nxk array/tensor, where k = 4 or 5 + from_mode, to_mode (BoxMode) + + Returns: + The converted box of the same type. + """ + if from_mode == to_mode: + return box + + original_type = type(box) + is_numpy = isinstance(box, np.ndarray) + single_box = isinstance(box, (list, tuple)) + if single_box: + assert len(box) == 4 or len(box) == 5, ( + "BoxMode.convert takes either a k-tuple/list or an Nxk array/tensor," " where k == 4 or 5" + ) + arr = torch.tensor(box)[None, :] + else: + # avoid modifying the input box + if is_numpy: + arr = torch.from_numpy(np.asarray(box)).clone() + arr = arr[None, :] + else: + arr = box.clone() + + assert to_mode not in [BoxMode.XYXY_REL, BoxMode.XYWH_REL] and from_mode not in [ + BoxMode.XYXY_REL, + BoxMode.XYWH_REL, + ], "Relative mode not yet supported!" + + if from_mode == BoxMode.XYWHA_ABS and to_mode == BoxMode.XYXY_ABS: + assert arr.shape[-1] == 5, "The last dimension of input shape must be 5 for XYWHA format" + original_dtype = arr.dtype + arr = arr.double() + + w = arr[:, 2] + h = arr[:, 3] + a = arr[:, 4] + c = torch.abs(torch.cos(a * math.pi / 180.0)) + s = torch.abs(torch.sin(a * math.pi / 180.0)) + # This basically computes the horizontal bounding rectangle of the rotated box + new_w = c * w + s * h + new_h = c * h + s * w + + # convert center to top-left corner + arr[:, 0] -= new_w / 2.0 + arr[:, 1] -= new_h / 2.0 + # bottom-right corner + arr[:, 2] = arr[:, 0] + new_w + arr[:, 3] = arr[:, 1] + new_h + + arr = arr[:, :4].to(dtype=original_dtype) + elif from_mode == BoxMode.XYWH_ABS and to_mode == BoxMode.XYWHA_ABS: + original_dtype = arr.dtype + arr = arr.double() + arr[:, 0] += arr[:, 2] / 2.0 + arr[:, 1] += arr[:, 3] / 2.0 + angles = torch.zeros((arr.shape[0], 1), dtype=arr.dtype) + arr = torch.cat((arr, angles), axis=1).to(dtype=original_dtype) + else: + if to_mode == BoxMode.XYXY_ABS and from_mode == BoxMode.XYWH_ABS: + arr[:, 2] += arr[:, 0] + arr[:, 3] += arr[:, 1] + elif from_mode == BoxMode.XYXY_ABS and to_mode == BoxMode.XYWH_ABS: + arr[:, 2] -= arr[:, 0] + arr[:, 3] -= arr[:, 1] + else: + raise NotImplementedError( + "Conversion from BoxMode {} to {} is not supported yet".format(from_mode, to_mode) + ) + + if single_box: + return original_type(arr.flatten().tolist()) + if is_numpy: + return arr.numpy() + else: + return arr + + +class Boxes: + """ + This structure stores a list of boxes as a Nx4 torch.Tensor. + It supports some common methods about boxes + (`area`, `clip`, `nonempty`, etc), + and also behaves like a Tensor + (support indexing, `to(device)`, `.device`, and iteration over all boxes) + + Attributes: + tensor (torch.Tensor): float matrix of Nx4. Each row is (x1, y1, x2, y2). + """ + + def __init__(self, tensor: torch.Tensor): + """ + Args: + tensor (Tensor[float]): a Nx4 matrix. Each row is (x1, y1, x2, y2). + """ + if not isinstance(tensor, torch.Tensor): + tensor = torch.as_tensor(tensor, dtype=torch.float32, device=torch.device("cpu")) + else: + tensor = tensor.to(torch.float32) + if tensor.numel() == 0: + # Use reshape, so we don't end up creating a new tensor that does not depend on + # the inputs (and consequently confuses jit) + tensor = tensor.reshape((-1, 4)).to(dtype=torch.float32) + assert tensor.dim() == 2 and tensor.size(-1) == 4, tensor.size() + + self.tensor = tensor + + def clone(self) -> "Boxes": + """ + Clone the Boxes. + + Returns: + Boxes + """ + return Boxes(self.tensor.clone()) + + def to(self, device: torch.device): + # Boxes are assumed float32 and does not support to(dtype) + return Boxes(self.tensor.to(device=device)) + + def area(self) -> torch.Tensor: + """ + Computes the area of all the boxes. + + Returns: + torch.Tensor: a vector with areas of each box. + """ + box = self.tensor + area = (box[:, 2] - box[:, 0]) * (box[:, 3] - box[:, 1]) + return area + + def clip(self, box_size: Tuple[int, int]) -> None: + """ + Clip (in place) the boxes by limiting x coordinates to the range [0, width] + and y coordinates to the range [0, height]. + + Args: + box_size (height, width): The clipping box's size. + """ + assert torch.isfinite(self.tensor).all(), "Box tensor contains infinite or NaN!" + h, w = box_size + x1 = self.tensor[:, 0].clamp(min=0, max=w) + y1 = self.tensor[:, 1].clamp(min=0, max=h) + x2 = self.tensor[:, 2].clamp(min=0, max=w) + y2 = self.tensor[:, 3].clamp(min=0, max=h) + self.tensor = torch.stack((x1, y1, x2, y2), dim=-1) + + def nonempty(self, threshold: float = 0.0) -> torch.Tensor: + """ + Find boxes that are non-empty. + A box is considered empty, if either of its side is no larger than threshold. + + Returns: + Tensor: + a binary vector which represents whether each box is empty + (False) or non-empty (True). + """ + box = self.tensor + widths = box[:, 2] - box[:, 0] + heights = box[:, 3] - box[:, 1] + keep = (widths > threshold) & (heights > threshold) + return keep + + def __getitem__(self, item) -> "Boxes": + """ + Args: + item: int, slice, or a BoolTensor + + Returns: + Boxes: Create a new :class:`Boxes` by indexing. + + The following usage are allowed: + + 1. `new_boxes = boxes[3]`: return a `Boxes` which contains only one box. + 2. `new_boxes = boxes[2:10]`: return a slice of boxes. + 3. `new_boxes = boxes[vector]`, where vector is a torch.BoolTensor + with `length = len(boxes)`. Nonzero elements in the vector will be selected. + + Note that the returned Boxes might share storage with this Boxes, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return Boxes(self.tensor[item].view(1, -1)) + b = self.tensor[item] + assert b.dim() == 2, "Indexing on Boxes with {} failed to return a matrix!".format(item) + return Boxes(b) + + def __len__(self) -> int: + return self.tensor.shape[0] + + def __repr__(self) -> str: + return "Boxes(" + str(self.tensor) + ")" + + def inside_box(self, box_size: Tuple[int, int], boundary_threshold: int = 0) -> torch.Tensor: + """ + Args: + box_size (height, width): Size of the reference box. + boundary_threshold (int): Boxes that extend beyond the reference box + boundary by more than boundary_threshold are considered "outside". + + Returns: + a binary vector, indicating whether each box is inside the reference box. + """ + height, width = box_size + inds_inside = ( + (self.tensor[..., 0] >= -boundary_threshold) + & (self.tensor[..., 1] >= -boundary_threshold) + & (self.tensor[..., 2] < width + boundary_threshold) + & (self.tensor[..., 3] < height + boundary_threshold) + ) + return inds_inside + + def get_centers(self) -> torch.Tensor: + """ + Returns: + The box centers in a Nx2 array of (x, y). + """ + return (self.tensor[:, :2] + self.tensor[:, 2:]) / 2 + + def scale(self, scale_x: float, scale_y: float) -> None: + """ + Scale the box with horizontal and vertical scaling factors + """ + self.tensor[:, 0::2] *= scale_x + self.tensor[:, 1::2] *= scale_y + + @classmethod + def cat(cls, boxes_list: List["Boxes"]) -> "Boxes": + """ + Concatenates a list of Boxes into a single Boxes + + Arguments: + boxes_list (list[Boxes]) + + Returns: + Boxes: the concatenated Boxes + """ + assert isinstance(boxes_list, (list, tuple)) + if len(boxes_list) == 0: + return cls(torch.empty(0)) + assert all([isinstance(box, Boxes) for box in boxes_list]) + + # use torch.cat (v.s. layers.cat) so the returned boxes never share storage with input + cat_boxes = cls(torch.cat([b.tensor for b in boxes_list], dim=0)) + return cat_boxes + + @property + def device(self) -> device: + return self.tensor.device + + # type "Iterator[torch.Tensor]", yield, and iter() not supported by torchscript + # https://github.com/pytorch/pytorch/issues/18627 + @torch.jit.unused + def __iter__(self): + """ + Yield a box as a Tensor of shape (4,) at a time. + """ + yield from self.tensor + + +def pairwise_intersection(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Given two lists of boxes of size N and M, + compute the intersection area between __all__ N x M pairs of boxes. + The box order must be (xmin, ymin, xmax, ymax) + + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + + Returns: + Tensor: intersection, sized [N,M]. + """ + boxes1, boxes2 = boxes1.tensor, boxes2.tensor + width_height = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) - torch.max( + boxes1[:, None, :2], boxes2[:, :2] + ) # [N,M,2] + + width_height.clamp_(min=0) # [N,M,2] + intersection = width_height.prod(dim=2) # [N,M] + return intersection + + +# implementation from https://github.com/kuangliu/torchcv/blob/master/torchcv/utils/box.py +# with slight modifications +def pairwise_iou(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Given two lists of boxes of size N and M, compute the IoU + (intersection over union) between **all** N x M pairs of boxes. + The box order must be (xmin, ymin, xmax, ymax). + + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + + Returns: + Tensor: IoU, sized [N,M]. + """ + area1 = boxes1.area() # [N] + area2 = boxes2.area() # [M] + inter = pairwise_intersection(boxes1, boxes2) + + # handle empty boxes + iou = torch.where( + inter > 0, + inter / (area1[:, None] + area2 - inter), + torch.zeros(1, dtype=inter.dtype, device=inter.device), + ) + return iou + + +def pairwise_ioa(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Similar to :func:`pariwise_iou` but compute the IoA (intersection over boxes2 area). + + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + + Returns: + Tensor: IoA, sized [N,M]. + """ + area2 = boxes2.area() # [M] + inter = pairwise_intersection(boxes1, boxes2) + + # handle empty boxes + ioa = torch.where(inter > 0, inter / area2, torch.zeros(1, dtype=inter.dtype, device=inter.device)) + return ioa + + +def pairwise_point_box_distance(points: torch.Tensor, boxes: Boxes): + """ + Pairwise distance between N points and M boxes. The distance between a + point and a box is represented by the distance from the point to 4 edges + of the box. Distances are all positive when the point is inside the box. + + Args: + points: Nx2 coordinates. Each row is (x, y) + boxes: M boxes + + Returns: + Tensor: distances of size (N, M, 4). The 4 values are distances from + the point to the left, top, right, bottom of the box. + """ + x, y = points.unsqueeze(dim=2).unbind(dim=1) # (N, 1) + x0, y0, x1, y1 = boxes.tensor.unsqueeze(dim=0).unbind(dim=2) # (1, M) + return torch.stack([x - x0, y - y0, x1 - x, y1 - y], dim=2) + + +def matched_pairwise_iou(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Compute pairwise intersection over union (IOU) of two sets of matched + boxes that have the same number of boxes. + Similar to :func:`pairwise_iou`, but computes only diagonal elements of the matrix. + + Args: + boxes1 (Boxes): bounding boxes, sized [N,4]. + boxes2 (Boxes): same length as boxes1 + Returns: + Tensor: iou, sized [N]. + """ + assert len(boxes1) == len(boxes2), "boxlists should have the same" "number of entries, got {}, {}".format( + len(boxes1), len(boxes2) + ) + area1 = boxes1.area() # [N] + area2 = boxes2.area() # [N] + box1, box2 = boxes1.tensor, boxes2.tensor + lt = torch.max(box1[:, :2], box2[:, :2]) # [N,2] + rb = torch.min(box1[:, 2:], box2[:, 2:]) # [N,2] + wh = (rb - lt).clamp(min=0) # [N,2] + inter = wh[:, 0] * wh[:, 1] # [N] + iou = inter / (area1 + area2 - inter) # [N] + return iou diff --git a/detectron2/structures/image_list.py b/detectron2/structures/image_list.py new file mode 100644 index 0000000000000000000000000000000000000000..a63b0623cb40ff8c8fa8a030f4706c8ec71b6b87 --- /dev/null +++ b/detectron2/structures/image_list.py @@ -0,0 +1,109 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import division + +from typing import Any, List, Tuple + +import torch +from torch import device + +from detectron2.layers.wrappers import shapes_to_tensor + + +class ImageList(object): + """ + Structure that holds a list of images (of possibly + varying sizes) as a single tensor. + This works by padding the images to the same size. + The original sizes of each image is stored in `image_sizes`. + + Attributes: + image_sizes (list[tuple[int, int]]): each tuple is (h, w). + During tracing, it becomes list[Tensor] instead. + """ + + def __init__(self, tensor: torch.Tensor, image_sizes: List[Tuple[int, int]], padding_mask: torch.Tensor = None): + """ + Arguments: + tensor (Tensor): of shape (N, H, W) or (N, C_1, ..., C_K, H, W) where K >= 1 + image_sizes (list[tuple[int, int]]): Each tuple is (h, w). It can + be smaller than (H, W) due to padding. + """ + self.tensor = tensor + self.image_sizes = image_sizes + self.padding_mask = padding_mask + + def __len__(self) -> int: + return len(self.image_sizes) + + def __getitem__(self, idx) -> torch.Tensor: + """ + Access the individual image in its original size. + + Args: + idx: int or slice + + Returns: + Tensor: an image of shape (H, W) or (C_1, ..., C_K, H, W) where K >= 1 + """ + size = self.image_sizes[idx] + return self.tensor[idx, ..., : size[0], : size[1]] + + @torch.jit.unused + def to(self, *args: Any, **kwargs: Any) -> "ImageList": + cast_tensor = self.tensor.to(*args, **kwargs) + return ImageList( + cast_tensor, + self.image_sizes, + padding_mask=(self.padding_mask.to(*args, **kwargs) if self.padding_mask is not None else None), + ) + + @property + def device(self) -> device: + return self.tensor.device + + @staticmethod + def from_tensors(tensors: List[torch.Tensor], size_divisibility: int = 0, pad_value: float = 0.0) -> "ImageList": + """ + Args: + tensors: a tuple or list of `torch.Tensor`, each of shape (Hi, Wi) or + (C_1, ..., C_K, Hi, Wi) where K >= 1. The Tensors will be padded + to the same shape with `pad_value`. + size_divisibility (int): If `size_divisibility > 0`, add padding to ensure + the common height and width is divisible by `size_divisibility`. + This depends on the model and many models need a divisibility of 32. + pad_value (float): value to pad + + Returns: + an `ImageList`. + """ + assert len(tensors) > 0 + assert isinstance(tensors, (tuple, list)) + for t in tensors: + assert isinstance(t, torch.Tensor), type(t) + assert t.shape[:-2] == tensors[0].shape[:-2], t.shape + + image_sizes = [(im.shape[-2], im.shape[-1]) for im in tensors] + image_sizes_tensor = [shapes_to_tensor(x) for x in image_sizes] + max_size = torch.stack(image_sizes_tensor).max(0).values + + if size_divisibility > 1: + stride = size_divisibility + # the last two dims are H,W, both subject to divisibility requirement + max_size = (max_size + (stride - 1)).div(stride, rounding_mode="floor") * stride + + # handle weirdness of scripting and tracing ... + if torch.jit.is_scripting(): + max_size: List[int] = max_size.to(dtype=torch.long).tolist() + else: + if torch.jit.is_tracing(): + image_sizes = image_sizes_tensor + + # max_size can be a tensor in tracing mode, therefore convert to list + batch_shape = [len(tensors)] + list(tensors[0].shape[:-2]) + list(max_size) + batched_imgs = tensors[0].new_full(batch_shape, pad_value) + batched_masks = tensors[0].new_full([len(tensors)] + list(max_size), 1.0).bool() + for img, pad_img, m in zip(tensors, batched_imgs, batched_masks): + pad_img[..., : img.shape[-2], : img.shape[-1]].copy_(img) + m[: img.shape[-2], : img.shape[-1]] = False + + return ImageList(batched_imgs.contiguous(), image_sizes, batched_masks) diff --git a/detectron2/structures/instances.py b/detectron2/structures/instances.py new file mode 100644 index 0000000000000000000000000000000000000000..fb0e99598d8bf4490d9d1072fec3e3b588e43a30 --- /dev/null +++ b/detectron2/structures/instances.py @@ -0,0 +1,193 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +from typing import Any, Dict, List, Tuple, Union + +import torch + + +class Instances: + """ + This class represents a list of instances in an image. + It stores the attributes of instances (e.g., boxes, masks, labels, scores) as "fields". + All fields must have the same ``__len__`` which is the number of instances. + + All other (non-field) attributes of this class are considered private: + they must start with '_' and are not modifiable by a user. + + Some basic usage: + + 1. Set/get/check a field: + + .. code-block:: python + + instances.gt_boxes = Boxes(...) + print(instances.pred_masks) # a tensor of shape (N, H, W) + print('gt_masks' in instances) + + 2. ``len(instances)`` returns the number of instances + 3. Indexing: ``instances[indices]`` will apply the indexing on all the fields + and returns a new :class:`Instances`. + Typically, ``indices`` is a integer vector of indices, + or a binary mask of length ``num_instances`` + + .. code-block:: python + + category_3_detections = instances[instances.pred_classes == 3] + confident_detections = instances[instances.scores > 0.9] + """ + + def __init__(self, image_size: Tuple[int, int], **kwargs: Any): + """ + Args: + image_size (height, width): the spatial size of the image. + kwargs: fields to add to this `Instances`. + """ + self._image_size = image_size + self._fields: Dict[str, Any] = {} + for k, v in kwargs.items(): + self.set(k, v) + + @property + def image_size(self) -> Tuple[int, int]: + """ + Returns: + tuple: height, width + """ + return self._image_size + + def __setattr__(self, name: str, val: Any) -> None: + if name.startswith("_"): + super().__setattr__(name, val) + else: + self.set(name, val) + + def __getattr__(self, name: str) -> Any: + if name == "_fields" or name not in self._fields: + raise AttributeError("Cannot find field '{}' in the given Instances!".format(name)) + return self._fields[name] + + def set(self, name: str, value: Any) -> None: + """ + Set the field named `name` to `value`. + The length of `value` must be the number of instances, + and must agree with other existing fields in this object. + """ + data_len = len(value) + if len(self._fields): + assert len(self) == data_len, "Adding a field of length {} to a Instances of length {}".format( + data_len, len(self) + ) + self._fields[name] = value + + def has(self, name: str) -> bool: + """ + Returns: + bool: whether the field called `name` exists. + """ + return name in self._fields + + def remove(self, name: str) -> None: + """ + Remove the field called `name`. + """ + del self._fields[name] + + def get(self, name: str) -> Any: + """ + Returns the field called `name`. + """ + return self._fields[name] + + def get_fields(self) -> Dict[str, Any]: + """ + Returns: + dict: a dict which maps names (str) to data of the fields + + Modifying the returned dict will modify this instance. + """ + return self._fields + + # Tensor-like methods + def to(self, *args: Any, **kwargs: Any) -> "Instances": + """ + Returns: + Instances: all fields are called with a `to(device)`, if the field has this method. + """ + ret = Instances(self._image_size) + for k, v in self._fields.items(): + if hasattr(v, "to"): + v = v.to(*args, **kwargs) + ret.set(k, v) + return ret + + def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> "Instances": + """ + Args: + item: an index-like object and will be used to index all the fields. + + Returns: + If `item` is a string, return the data in the corresponding field. + Otherwise, returns an `Instances` where all fields are indexed by `item`. + """ + if type(item) == int: + if item >= len(self) or item < -len(self): + raise IndexError("Instances index out of range!") + else: + item = slice(item, None, len(self)) + + ret = Instances(self._image_size) + for k, v in self._fields.items(): + ret.set(k, v[item]) + return ret + + def __len__(self) -> int: + for v in self._fields.values(): + # use __len__ because len() has to be int and is not friendly to tracing + return v.__len__() + raise NotImplementedError("Empty Instances does not support __len__!") + + def __iter__(self): + raise NotImplementedError("`Instances` object is not iterable!") + + @staticmethod + def cat(instance_lists: List["Instances"]) -> "Instances": + """ + Args: + instance_lists (list[Instances]) + + Returns: + Instances + """ + assert all(isinstance(i, Instances) for i in instance_lists) + assert len(instance_lists) > 0 + if len(instance_lists) == 1: + return instance_lists[0] + + image_size = instance_lists[0].image_size + if not isinstance(image_size, torch.Tensor): # could be a tensor in tracing + for i in instance_lists[1:]: + assert i.image_size == image_size + ret = Instances(image_size) + for k in instance_lists[0]._fields.keys(): + values = [i.get(k) for i in instance_lists] + v0 = values[0] + if isinstance(v0, torch.Tensor): + values = torch.cat(values, dim=0) + elif isinstance(v0, list): + values = list(itertools.chain(*values)) + elif hasattr(type(v0), "cat"): + values = type(v0).cat(values) + else: + raise ValueError("Unsupported type {} for concatenation".format(type(v0))) + ret.set(k, values) + return ret + + def __str__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={}, ".format(len(self)) + s += "image_height={}, ".format(self._image_size[0]) + s += "image_width={}, ".format(self._image_size[1]) + s += "fields=[{}])".format(", ".join((f"{k}: {v}" for k, v in self._fields.items()))) + return s + + __repr__ = __str__ diff --git a/detectron2/structures/keypoints.py b/detectron2/structures/keypoints.py new file mode 100644 index 0000000000000000000000000000000000000000..efcc6d5f6f619ff912bfa9775e7de7c58a3293ed --- /dev/null +++ b/detectron2/structures/keypoints.py @@ -0,0 +1,233 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Any, List, Tuple, Union + +import numpy as np +import torch +from torch.nn import functional as F + + +class Keypoints: + """ + Stores keypoint **annotation** data. GT Instances have a `gt_keypoints` property + containing the x,y location and visibility flag of each keypoint. This tensor has shape + (N, K, 3) where N is the number of instances and K is the number of keypoints per instance. + + The visibility flag follows the COCO format and must be one of three integers: + + * v=0: not labeled (in which case x=y=0) + * v=1: labeled but not visible + * v=2: labeled and visible + """ + + def __init__(self, keypoints: Union[torch.Tensor, np.ndarray, List[List[float]]]): + """ + Arguments: + keypoints: A Tensor, numpy array, or list of the x, y, and visibility of each keypoint. + The shape should be (N, K, 3) where N is the number of + instances, and K is the number of keypoints per instance. + """ + device = keypoints.device if isinstance(keypoints, torch.Tensor) else torch.device("cpu") + keypoints = torch.as_tensor(keypoints, dtype=torch.float32, device=device) + assert keypoints.dim() == 3 and keypoints.shape[2] == 3, keypoints.shape + self.tensor = keypoints + + def __len__(self) -> int: + return self.tensor.size(0) + + def to(self, *args: Any, **kwargs: Any) -> "Keypoints": + return type(self)(self.tensor.to(*args, **kwargs)) + + @property + def device(self) -> torch.device: + return self.tensor.device + + def to_heatmap(self, boxes: torch.Tensor, heatmap_size: int) -> torch.Tensor: + """ + Convert keypoint annotations to a heatmap of one-hot labels for training, + as described in :paper:`Mask R-CNN`. + + Arguments: + boxes: Nx4 tensor, the boxes to draw the keypoints to + + Returns: + heatmaps: + A tensor of shape (N, K), each element is integer spatial label + in the range [0, heatmap_size**2 - 1] for each keypoint in the input. + valid: + A tensor of shape (N, K) containing whether each keypoint is in the roi or not. + """ + return _keypoints_to_heatmap(self.tensor, boxes, heatmap_size) + + def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> "Keypoints": + """ + Create a new `Keypoints` by indexing on this `Keypoints`. + + The following usage are allowed: + + 1. `new_kpts = kpts[3]`: return a `Keypoints` which contains only one instance. + 2. `new_kpts = kpts[2:10]`: return a slice of key points. + 3. `new_kpts = kpts[vector]`, where vector is a torch.ByteTensor + with `length = len(kpts)`. Nonzero elements in the vector will be selected. + + Note that the returned Keypoints might share storage with this Keypoints, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return Keypoints([self.tensor[item]]) + return Keypoints(self.tensor[item]) + + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.tensor)) + return s + + @staticmethod + def cat(keypoints_list: List["Keypoints"]) -> "Keypoints": + """ + Concatenates a list of Keypoints into a single Keypoints + + Arguments: + keypoints_list (list[Keypoints]) + + Returns: + Keypoints: the concatenated Keypoints + """ + assert isinstance(keypoints_list, (list, tuple)) + assert len(keypoints_list) > 0 + assert all(isinstance(keypoints, Keypoints) for keypoints in keypoints_list) + + cat_kpts = type(keypoints_list[0])(torch.cat([kpts.tensor for kpts in keypoints_list], dim=0)) + return cat_kpts + + +# TODO make this nicer, this is a direct translation from C2 (but removing the inner loop) +def _keypoints_to_heatmap( + keypoints: torch.Tensor, rois: torch.Tensor, heatmap_size: int +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Encode keypoint locations into a target heatmap for use in SoftmaxWithLoss across space. + + Maps keypoints from the half-open interval [x1, x2) on continuous image coordinates to the + closed interval [0, heatmap_size - 1] on discrete image coordinates. We use the + continuous-discrete conversion from Heckbert 1990 ("What is the coordinate of a pixel?"): + d = floor(c) and c = d + 0.5, where d is a discrete coordinate and c is a continuous coordinate. + + Arguments: + keypoints: tensor of keypoint locations in of shape (N, K, 3). + rois: Nx4 tensor of rois in xyxy format + heatmap_size: integer side length of square heatmap. + + Returns: + heatmaps: A tensor of shape (N, K) containing an integer spatial label + in the range [0, heatmap_size**2 - 1] for each keypoint in the input. + valid: A tensor of shape (N, K) containing whether each keypoint is in + the roi or not. + """ + + if rois.numel() == 0: + return rois.new().long(), rois.new().long() + offset_x = rois[:, 0] + offset_y = rois[:, 1] + scale_x = heatmap_size / (rois[:, 2] - rois[:, 0]) + scale_y = heatmap_size / (rois[:, 3] - rois[:, 1]) + + offset_x = offset_x[:, None] + offset_y = offset_y[:, None] + scale_x = scale_x[:, None] + scale_y = scale_y[:, None] + + x = keypoints[..., 0] + y = keypoints[..., 1] + + x_boundary_inds = x == rois[:, 2][:, None] + y_boundary_inds = y == rois[:, 3][:, None] + + x = (x - offset_x) * scale_x + x = x.floor().long() + y = (y - offset_y) * scale_y + y = y.floor().long() + + x[x_boundary_inds] = heatmap_size - 1 + y[y_boundary_inds] = heatmap_size - 1 + + valid_loc = (x >= 0) & (y >= 0) & (x < heatmap_size) & (y < heatmap_size) + vis = keypoints[..., 2] > 0 + valid = (valid_loc & vis).long() + + lin_ind = y * heatmap_size + x + heatmaps = lin_ind * valid + + return heatmaps, valid + + +@torch.jit.script_if_tracing +def heatmaps_to_keypoints(maps: torch.Tensor, rois: torch.Tensor) -> torch.Tensor: + """ + Extract predicted keypoint locations from heatmaps. + + Args: + maps (Tensor): (#ROIs, #keypoints, POOL_H, POOL_W). The predicted heatmap of logits for + each ROI and each keypoint. + rois (Tensor): (#ROIs, 4). The box of each ROI. + + Returns: + Tensor of shape (#ROIs, #keypoints, 4) with the last dimension corresponding to + (x, y, logit, score) for each keypoint. + + When converting discrete pixel indices in an NxN image to a continuous keypoint coordinate, + we maintain consistency with :meth:`Keypoints.to_heatmap` by using the conversion from + Heckbert 1990: c = d + 0.5, where d is a discrete coordinate and c is a continuous coordinate. + """ + # The decorator use of torch.no_grad() was not supported by torchscript. + # https://github.com/pytorch/pytorch/issues/44768 + maps = maps.detach() + rois = rois.detach() + + offset_x = rois[:, 0] + offset_y = rois[:, 1] + + widths = (rois[:, 2] - rois[:, 0]).clamp(min=1) + heights = (rois[:, 3] - rois[:, 1]).clamp(min=1) + widths_ceil = widths.ceil() + heights_ceil = heights.ceil() + + num_rois, num_keypoints = maps.shape[:2] + xy_preds = maps.new_zeros(rois.shape[0], num_keypoints, 4) + + width_corrections = widths / widths_ceil + height_corrections = heights / heights_ceil + + keypoints_idx = torch.arange(num_keypoints, device=maps.device) + + for i in range(num_rois): + outsize = (int(heights_ceil[i]), int(widths_ceil[i])) + roi_map = F.interpolate(maps[[i]], size=outsize, mode="bicubic", align_corners=False).squeeze( + 0 + ) # #keypoints x H x W + + # softmax over the spatial region + max_score, _ = roi_map.view(num_keypoints, -1).max(1) + max_score = max_score.view(num_keypoints, 1, 1) + tmp_full_resolution = (roi_map - max_score).exp_() + tmp_pool_resolution = (maps[i] - max_score).exp_() + # Produce scores over the region H x W, but normalize with POOL_H x POOL_W, + # so that the scores of objects of different absolute sizes will be more comparable + roi_map_scores = tmp_full_resolution / tmp_pool_resolution.sum((1, 2), keepdim=True) + + w = roi_map.shape[2] + pos = roi_map.view(num_keypoints, -1).argmax(1) + + x_int = pos % w + y_int = (pos - x_int) // w + + assert (roi_map_scores[keypoints_idx, y_int, x_int] == roi_map_scores.view(num_keypoints, -1).max(1)[0]).all() + + x = (x_int.float() + 0.5) * width_corrections[i] + y = (y_int.float() + 0.5) * height_corrections[i] + + xy_preds[i, :, 0] = x + offset_x[i] + xy_preds[i, :, 1] = y + offset_y[i] + xy_preds[i, :, 2] = roi_map[keypoints_idx, y_int, x_int] + xy_preds[i, :, 3] = roi_map_scores[keypoints_idx, y_int, x_int] + + return xy_preds diff --git a/detectron2/structures/masks.py b/detectron2/structures/masks.py new file mode 100644 index 0000000000000000000000000000000000000000..06330eb0980a3aa26d341a81efed017487c011f1 --- /dev/null +++ b/detectron2/structures/masks.py @@ -0,0 +1,522 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +from typing import Any, Iterator, List, Union + +import numpy as np +import pycocotools.mask as mask_util +import torch +from torch import device + +from detectron2.layers.roi_align import ROIAlign +from detectron2.utils.memory import retry_if_cuda_oom + +from .boxes import Boxes + + +def polygon_area(x, y): + # Using the shoelace formula + # https://stackoverflow.com/questions/24467972/calculate-area-of-polygon-given-x-y-coordinates + return 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) + + +def polygons_to_bitmask(polygons: List[np.ndarray], height: int, width: int) -> np.ndarray: + """ + Args: + polygons (list[ndarray]): each array has shape (Nx2,) + height, width (int) + + Returns: + ndarray: a bool mask of shape (height, width) + """ + if len(polygons) == 0: + # COCOAPI does not support empty polygons + return np.zeros((height, width)).astype(np.bool_) + rles = mask_util.frPyObjects(polygons, height, width) + rle = mask_util.merge(rles) + return mask_util.decode(rle).astype(np.bool_) + + +def rasterize_polygons_within_box(polygons: List[np.ndarray], box: np.ndarray, mask_size: int) -> torch.Tensor: + """ + Rasterize the polygons into a mask image and + crop the mask content in the given box. + The cropped mask is resized to (mask_size, mask_size). + + This function is used when generating training targets for mask head in Mask R-CNN. + Given original ground-truth masks for an image, new ground-truth mask + training targets in the size of `mask_size x mask_size` + must be provided for each predicted box. This function will be called to + produce such targets. + + Args: + polygons (list[ndarray[float]]): a list of polygons, which represents an instance. + box: 4-element numpy array + mask_size (int): + + Returns: + Tensor: BoolTensor of shape (mask_size, mask_size) + """ + # 1. Shift the polygons w.r.t the boxes + w, h = box[2] - box[0], box[3] - box[1] + + polygons = copy.deepcopy(polygons) + for p in polygons: + p[0::2] = p[0::2] - box[0] + p[1::2] = p[1::2] - box[1] + + # 2. Rescale the polygons to the new box size + # max() to avoid division by small number + ratio_h = mask_size / max(h, 0.1) + ratio_w = mask_size / max(w, 0.1) + + if ratio_h == ratio_w: + for p in polygons: + p *= ratio_h + else: + for p in polygons: + p[0::2] *= ratio_w + p[1::2] *= ratio_h + + # 3. Rasterize the polygons with coco api + mask = polygons_to_bitmask(polygons, mask_size, mask_size) + mask = torch.from_numpy(mask) + return mask + + +class BitMasks: + """ + This class stores the segmentation masks for all objects in one image, in + the form of bitmaps. + + Attributes: + tensor: bool Tensor of N,H,W, representing N instances in the image. + """ + + def __init__(self, tensor: Union[torch.Tensor, np.ndarray]): + """ + Args: + tensor: bool Tensor of N,H,W, representing N instances in the image. + """ + if isinstance(tensor, torch.Tensor): + tensor = tensor.to(torch.bool) + else: + tensor = torch.as_tensor(tensor, dtype=torch.bool, device=torch.device("cpu")) + assert tensor.dim() == 3, tensor.size() + self.image_size = tensor.shape[1:] + self.tensor = tensor + + @torch.jit.unused + def to(self, *args: Any, **kwargs: Any) -> "BitMasks": + return BitMasks(self.tensor.to(*args, **kwargs)) + + @property + def device(self) -> torch.device: + return self.tensor.device + + @torch.jit.unused + def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> "BitMasks": + """ + Returns: + BitMasks: Create a new :class:`BitMasks` by indexing. + + The following usage are allowed: + + 1. `new_masks = masks[3]`: return a `BitMasks` which contains only one mask. + 2. `new_masks = masks[2:10]`: return a slice of masks. + 3. `new_masks = masks[vector]`, where vector is a torch.BoolTensor + with `length = len(masks)`. Nonzero elements in the vector will be selected. + + Note that the returned object might share storage with this object, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return BitMasks(self.tensor[item].unsqueeze(0)) + m = self.tensor[item] + assert m.dim() == 3, "Indexing on BitMasks with {} returns a tensor with shape {}!".format(item, m.shape) + return BitMasks(m) + + @torch.jit.unused + def __iter__(self) -> torch.Tensor: + yield from self.tensor + + @torch.jit.unused + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.tensor)) + return s + + def __len__(self) -> int: + return self.tensor.shape[0] + + def nonempty(self) -> torch.Tensor: + """ + Find masks that are non-empty. + + Returns: + Tensor: a BoolTensor which represents + whether each mask is empty (False) or non-empty (True). + """ + return self.tensor.flatten(1).any(dim=1) + + @staticmethod + def from_polygon_masks( + polygon_masks: Union["PolygonMasks", List[List[np.ndarray]]], height: int, width: int + ) -> "BitMasks": + """ + Args: + polygon_masks (list[list[ndarray]] or PolygonMasks) + height, width (int) + """ + if isinstance(polygon_masks, PolygonMasks): + polygon_masks = polygon_masks.polygons + masks = [polygons_to_bitmask(p, height, width) for p in polygon_masks] + if len(masks): + return BitMasks(torch.stack([torch.from_numpy(x) for x in masks])) + else: + return BitMasks(torch.empty(0, height, width, dtype=torch.bool)) + + @staticmethod + def from_roi_masks(roi_masks: "ROIMasks", height: int, width: int) -> "BitMasks": + """ + Args: + roi_masks: + height, width (int): + """ + return roi_masks.to_bitmasks(height, width) + + def crop_and_resize(self, boxes: torch.Tensor, mask_size: int) -> torch.Tensor: + """ + Crop each bitmask by the given box, and resize results to (mask_size, mask_size). + This can be used to prepare training targets for Mask R-CNN. + It has less reconstruction error compared to rasterization with polygons. + However we observe no difference in accuracy, + but BitMasks requires more memory to store all the masks. + + Args: + boxes (Tensor): Nx4 tensor storing the boxes for each mask + mask_size (int): the size of the rasterized mask. + + Returns: + Tensor: + A bool tensor of shape (N, mask_size, mask_size), where + N is the number of predicted boxes for this image. + """ + assert len(boxes) == len(self), "{} != {}".format(len(boxes), len(self)) + device = self.tensor.device + + batch_inds = torch.arange(len(boxes), device=device).to(dtype=boxes.dtype)[:, None] + rois = torch.cat([batch_inds, boxes], dim=1) # Nx5 + + bit_masks = self.tensor.to(dtype=torch.float32) + rois = rois.to(device=device) + output = ( + ROIAlign((mask_size, mask_size), 1.0, 0, aligned=True).forward(bit_masks[:, None, :, :], rois).squeeze(1) + ) + output = output >= 0.5 + return output + + def get_bounding_boxes(self) -> Boxes: + """ + Returns: + Boxes: tight bounding boxes around bitmasks. + If a mask is empty, it's bounding box will be all zero. + """ + boxes = torch.zeros(self.tensor.shape[0], 4, dtype=torch.float32) + x_any = torch.any(self.tensor, dim=1) + y_any = torch.any(self.tensor, dim=2) + for idx in range(self.tensor.shape[0]): + x = torch.where(x_any[idx, :])[0] + y = torch.where(y_any[idx, :])[0] + if len(x) > 0 and len(y) > 0: + boxes[idx, :] = torch.as_tensor([x[0], y[0], x[-1] + 1, y[-1] + 1], dtype=torch.float32) + return Boxes(boxes) + + @staticmethod + def cat(bitmasks_list: List["BitMasks"]) -> "BitMasks": + """ + Concatenates a list of BitMasks into a single BitMasks + + Arguments: + bitmasks_list (list[BitMasks]) + + Returns: + BitMasks: the concatenated BitMasks + """ + assert isinstance(bitmasks_list, (list, tuple)) + assert len(bitmasks_list) > 0 + assert all(isinstance(bitmask, BitMasks) for bitmask in bitmasks_list) + + cat_bitmasks = type(bitmasks_list[0])(torch.cat([bm.tensor for bm in bitmasks_list], dim=0)) + return cat_bitmasks + + +class PolygonMasks: + """ + This class stores the segmentation masks for all objects in one image, in the form of polygons. + + Attributes: + polygons: list[list[ndarray]]. Each ndarray is a float64 vector representing a polygon. + """ + + def __init__(self, polygons: List[List[Union[torch.Tensor, np.ndarray]]]): + """ + Arguments: + polygons (list[list[np.ndarray]]): The first + level of the list correspond to individual instances, + the second level to all the polygons that compose the + instance, and the third level to the polygon coordinates. + The third level array should have the format of + [x0, y0, x1, y1, ..., xn, yn] (n >= 3). + """ + if not isinstance(polygons, list): + raise ValueError( + "Cannot create PolygonMasks: Expect a list of list of polygons per image. " + "Got '{}' instead.".format(type(polygons)) + ) + + def _make_array(t: Union[torch.Tensor, np.ndarray]) -> np.ndarray: + # Use float64 for higher precision, because why not? + # Always put polygons on CPU (self.to is a no-op) since they + # are supposed to be small tensors. + # May need to change this assumption if GPU placement becomes useful + if isinstance(t, torch.Tensor): + t = t.cpu().numpy() + return np.asarray(t).astype("float64") + + def process_polygons(polygons_per_instance: List[Union[torch.Tensor, np.ndarray]]) -> List[np.ndarray]: + if not isinstance(polygons_per_instance, list): + raise ValueError( + "Cannot create polygons: Expect a list of polygons per instance. " + "Got '{}' instead.".format(type(polygons_per_instance)) + ) + # transform each polygon to a numpy array + polygons_per_instance = [_make_array(p) for p in polygons_per_instance] + # for polygon in polygons_per_instance: + # if len(polygon) % 2 != 0 or len(polygon) < 6: + # raise ValueError(f"Cannot create a polygon from {len(polygon)} coordinates.") + return polygons_per_instance + + self.polygons: List[List[np.ndarray]] = [ + process_polygons(polygons_per_instance) for polygons_per_instance in polygons + ] + + def to(self, *args: Any, **kwargs: Any) -> "PolygonMasks": + return self + + @property + def device(self) -> torch.device: + return torch.device("cpu") + + def get_bounding_boxes(self) -> Boxes: + """ + Returns: + Boxes: tight bounding boxes around polygon masks. + """ + boxes = torch.zeros(len(self.polygons), 4, dtype=torch.float32) + for idx, polygons_per_instance in enumerate(self.polygons): + minxy = torch.as_tensor([float("inf"), float("inf")], dtype=torch.float32) + maxxy = torch.zeros(2, dtype=torch.float32) + for polygon in polygons_per_instance: + coords = torch.from_numpy(polygon).view(-1, 2).to(dtype=torch.float32) + minxy = torch.min(minxy, torch.min(coords, dim=0).values) + maxxy = torch.max(maxxy, torch.max(coords, dim=0).values) + boxes[idx, :2] = minxy + boxes[idx, 2:] = maxxy + return Boxes(boxes) + + def nonempty(self) -> torch.Tensor: + """ + Find masks that are non-empty. + + Returns: + Tensor: + a BoolTensor which represents whether each mask is empty (False) or not (True). + """ + keep = [1 if len(polygon) > 0 else 0 for polygon in self.polygons] + return torch.from_numpy(np.asarray(keep, dtype=np.bool)) + + def __getitem__(self, item: Union[int, slice, List[int], torch.BoolTensor]) -> "PolygonMasks": + """ + Support indexing over the instances and return a `PolygonMasks` object. + `item` can be: + + 1. An integer. It will return an object with only one instance. + 2. A slice. It will return an object with the selected instances. + 3. A list[int]. It will return an object with the selected instances, + correpsonding to the indices in the list. + 4. A vector mask of type BoolTensor, whose length is num_instances. + It will return an object with the instances whose mask is nonzero. + """ + if isinstance(item, int): + selected_polygons = [self.polygons[item]] + elif isinstance(item, slice): + selected_polygons = self.polygons[item] + elif isinstance(item, list): + selected_polygons = [self.polygons[i] for i in item] + elif isinstance(item, torch.Tensor): + # Polygons is a list, so we have to move the indices back to CPU. + if item.dtype == torch.bool: + assert item.dim() == 1, item.shape + item = item.nonzero().squeeze(1).cpu().numpy().tolist() + elif item.dtype in [torch.int32, torch.int64]: + item = item.cpu().numpy().tolist() + else: + raise ValueError("Unsupported tensor dtype={} for indexing!".format(item.dtype)) + selected_polygons = [self.polygons[i] for i in item] + return PolygonMasks(selected_polygons) + + def __iter__(self) -> Iterator[List[np.ndarray]]: + """ + Yields: + list[ndarray]: the polygons for one instance. + Each Tensor is a float64 vector representing a polygon. + """ + return iter(self.polygons) + + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.polygons)) + return s + + def __len__(self) -> int: + return len(self.polygons) + + def crop_and_resize(self, boxes: torch.Tensor, mask_size: int) -> torch.Tensor: + """ + Crop each mask by the given box, and resize results to (mask_size, mask_size). + This can be used to prepare training targets for Mask R-CNN. + + Args: + boxes (Tensor): Nx4 tensor storing the boxes for each mask + mask_size (int): the size of the rasterized mask. + + Returns: + Tensor: A bool tensor of shape (N, mask_size, mask_size), where + N is the number of predicted boxes for this image. + """ + assert len(boxes) == len(self), "{} != {}".format(len(boxes), len(self)) + + device = boxes.device + # Put boxes on the CPU, as the polygon representation is not efficient GPU-wise + # (several small tensors for representing a single instance mask) + boxes = boxes.to(torch.device("cpu")) + + results = [ + rasterize_polygons_within_box(poly, box.numpy(), mask_size) for poly, box in zip(self.polygons, boxes) + ] + """ + poly: list[list[float]], the polygons for one instance + box: a tensor of shape (4,) + """ + if len(results) == 0: + return torch.empty(0, mask_size, mask_size, dtype=torch.bool, device=device) + return torch.stack(results, dim=0).to(device=device) + + def area(self): + """ + Computes area of the mask. + Only works with Polygons, using the shoelace formula: + https://stackoverflow.com/questions/24467972/calculate-area-of-polygon-given-x-y-coordinates + + Returns: + Tensor: a vector, area for each instance + """ + + area = [] + for polygons_per_instance in self.polygons: + area_per_instance = 0 + for p in polygons_per_instance: + area_per_instance += polygon_area(p[0::2], p[1::2]) + area.append(area_per_instance) + + return torch.tensor(area) + + @staticmethod + def cat(polymasks_list: List["PolygonMasks"]) -> "PolygonMasks": + """ + Concatenates a list of PolygonMasks into a single PolygonMasks + + Arguments: + polymasks_list (list[PolygonMasks]) + + Returns: + PolygonMasks: the concatenated PolygonMasks + """ + assert isinstance(polymasks_list, (list, tuple)) + assert len(polymasks_list) > 0 + assert all(isinstance(polymask, PolygonMasks) for polymask in polymasks_list) + + cat_polymasks = type(polymasks_list[0])( + list(itertools.chain.from_iterable(pm.polygons for pm in polymasks_list)) + ) + return cat_polymasks + + +class ROIMasks: + """ + Represent masks by N smaller masks defined in some ROIs. Once ROI boxes are given, + full-image bitmask can be obtained by "pasting" the mask on the region defined + by the corresponding ROI box. + """ + + def __init__(self, tensor: torch.Tensor): + """ + Args: + tensor: (N, M, M) mask tensor that defines the mask within each ROI. + """ + if tensor.dim() != 3: + raise ValueError("ROIMasks must take a masks of 3 dimension.") + self.tensor = tensor + + def to(self, device: torch.device) -> "ROIMasks": + return ROIMasks(self.tensor.to(device)) + + @property + def device(self) -> device: + return self.tensor.device + + def __len__(self): + return self.tensor.shape[0] + + def __getitem__(self, item) -> "ROIMasks": + """ + Returns: + ROIMasks: Create a new :class:`ROIMasks` by indexing. + + The following usage are allowed: + + 1. `new_masks = masks[2:10]`: return a slice of masks. + 2. `new_masks = masks[vector]`, where vector is a torch.BoolTensor + with `length = len(masks)`. Nonzero elements in the vector will be selected. + + Note that the returned object might share storage with this object, + subject to Pytorch's indexing semantics. + """ + t = self.tensor[item] + if t.dim() != 3: + raise ValueError(f"Indexing on ROIMasks with {item} returns a tensor with shape {t.shape}!") + return ROIMasks(t) + + @torch.jit.unused + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.tensor)) + return s + + @torch.jit.unused + def to_bitmasks(self, boxes: torch.Tensor, height, width, threshold=0.5): + """ + Args: see documentation of :func:`paste_masks_in_image`. + """ + from detectron2.layers.mask_ops import _paste_masks_tensor_shape, paste_masks_in_image + + if torch.jit.is_tracing(): + if isinstance(height, torch.Tensor): + paste_func = _paste_masks_tensor_shape + else: + paste_func = paste_masks_in_image + else: + paste_func = retry_if_cuda_oom(paste_masks_in_image) + bitmasks = paste_func(self.tensor, boxes.tensor, (height, width), threshold=threshold) + return BitMasks(bitmasks) diff --git a/detectron2/structures/rotated_boxes.py b/detectron2/structures/rotated_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..19f049b513a2f0b17a15413b0ac08e98719a1f29 --- /dev/null +++ b/detectron2/structures/rotated_boxes.py @@ -0,0 +1,502 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List, Tuple + +import torch + +from detectron2.layers.rotated_boxes import pairwise_iou_rotated + +from .boxes import Boxes + + +class RotatedBoxes(Boxes): + """ + This structure stores a list of rotated boxes as a Nx5 torch.Tensor. + It supports some common methods about boxes + (`area`, `clip`, `nonempty`, etc), + and also behaves like a Tensor + (support indexing, `to(device)`, `.device`, and iteration over all boxes) + """ + + def __init__(self, tensor: torch.Tensor): + """ + Args: + tensor (Tensor[float]): a Nx5 matrix. Each row is + (x_center, y_center, width, height, angle), + in which angle is represented in degrees. + While there's no strict range restriction for it, + the recommended principal range is between [-180, 180) degrees. + + Assume we have a horizontal box B = (x_center, y_center, width, height), + where width is along the x-axis and height is along the y-axis. + The rotated box B_rot (x_center, y_center, width, height, angle) + can be seen as: + + 1. When angle == 0: + B_rot == B + 2. When angle > 0: + B_rot is obtained by rotating B w.r.t its center by :math:`|angle|` degrees CCW; + 3. When angle < 0: + B_rot is obtained by rotating B w.r.t its center by :math:`|angle|` degrees CW. + + Mathematically, since the right-handed coordinate system for image space + is (y, x), where y is top->down and x is left->right, the 4 vertices of the + rotated rectangle :math:`(yr_i, xr_i)` (i = 1, 2, 3, 4) can be obtained from + the vertices of the horizontal rectangle :math:`(y_i, x_i)` (i = 1, 2, 3, 4) + in the following way (:math:`\\theta = angle*\\pi/180` is the angle in radians, + :math:`(y_c, x_c)` is the center of the rectangle): + + .. math:: + + yr_i = \\cos(\\theta) (y_i - y_c) - \\sin(\\theta) (x_i - x_c) + y_c, + + xr_i = \\sin(\\theta) (y_i - y_c) + \\cos(\\theta) (x_i - x_c) + x_c, + + which is the standard rigid-body rotation transformation. + + Intuitively, the angle is + (1) the rotation angle from y-axis in image space + to the height vector (top->down in the box's local coordinate system) + of the box in CCW, and + (2) the rotation angle from x-axis in image space + to the width vector (left->right in the box's local coordinate system) + of the box in CCW. + + More intuitively, consider the following horizontal box ABCD represented + in (x1, y1, x2, y2): (3, 2, 7, 4), + covering the [3, 7] x [2, 4] region of the continuous coordinate system + which looks like this: + + .. code:: none + + O--------> x + | + | A---B + | | | + | D---C + | + v y + + Note that each capital letter represents one 0-dimensional geometric point + instead of a 'square pixel' here. + + In the example above, using (x, y) to represent a point we have: + + .. math:: + + O = (0, 0), A = (3, 2), B = (7, 2), C = (7, 4), D = (3, 4) + + We name vector AB = vector DC as the width vector in box's local coordinate system, and + vector AD = vector BC as the height vector in box's local coordinate system. Initially, + when angle = 0 degree, they're aligned with the positive directions of x-axis and y-axis + in the image space, respectively. + + For better illustration, we denote the center of the box as E, + + .. code:: none + + O--------> x + | + | A---B + | | E | + | D---C + | + v y + + where the center E = ((3+7)/2, (2+4)/2) = (5, 3). + + Also, + + .. math:: + + width = |AB| = |CD| = 7 - 3 = 4, + height = |AD| = |BC| = 4 - 2 = 2. + + Therefore, the corresponding representation for the same shape in rotated box in + (x_center, y_center, width, height, angle) format is: + + (5, 3, 4, 2, 0), + + Now, let's consider (5, 3, 4, 2, 90), which is rotated by 90 degrees + CCW (counter-clockwise) by definition. It looks like this: + + .. code:: none + + O--------> x + | B-C + | | | + | |E| + | | | + | A-D + v y + + The center E is still located at the same point (5, 3), while the vertices + ABCD are rotated by 90 degrees CCW with regard to E: + A = (4, 5), B = (4, 1), C = (6, 1), D = (6, 5) + + Here, 90 degrees can be seen as the CCW angle to rotate from y-axis to + vector AD or vector BC (the top->down height vector in box's local coordinate system), + or the CCW angle to rotate from x-axis to vector AB or vector DC (the left->right + width vector in box's local coordinate system). + + .. math:: + + width = |AB| = |CD| = 5 - 1 = 4, + height = |AD| = |BC| = 6 - 4 = 2. + + Next, how about (5, 3, 4, 2, -90), which is rotated by 90 degrees CW (clockwise) + by definition? It looks like this: + + .. code:: none + + O--------> x + | D-A + | | | + | |E| + | | | + | C-B + v y + + The center E is still located at the same point (5, 3), while the vertices + ABCD are rotated by 90 degrees CW with regard to E: + A = (6, 1), B = (6, 5), C = (4, 5), D = (4, 1) + + .. math:: + + width = |AB| = |CD| = 5 - 1 = 4, + height = |AD| = |BC| = 6 - 4 = 2. + + This covers exactly the same region as (5, 3, 4, 2, 90) does, and their IoU + will be 1. However, these two will generate different RoI Pooling results and + should not be treated as an identical box. + + On the other hand, it's easy to see that (X, Y, W, H, A) is identical to + (X, Y, W, H, A+360N), for any integer N. For example (5, 3, 4, 2, 270) would be + identical to (5, 3, 4, 2, -90), because rotating the shape 270 degrees CCW is + equivalent to rotating the same shape 90 degrees CW. + + We could rotate further to get (5, 3, 4, 2, 180), or (5, 3, 4, 2, -180): + + .. code:: none + + O--------> x + | + | C---D + | | E | + | B---A + | + v y + + .. math:: + + A = (7, 4), B = (3, 4), C = (3, 2), D = (7, 2), + + width = |AB| = |CD| = 7 - 3 = 4, + height = |AD| = |BC| = 4 - 2 = 2. + + Finally, this is a very inaccurate (heavily quantized) illustration of + how (5, 3, 4, 2, 60) looks like in case anyone wonders: + + .. code:: none + + O--------> x + | B\ + | / C + | /E / + | A / + | `D + v y + + It's still a rectangle with center of (5, 3), width of 4 and height of 2, + but its angle (and thus orientation) is somewhere between + (5, 3, 4, 2, 0) and (5, 3, 4, 2, 90). + """ + device = tensor.device if isinstance(tensor, torch.Tensor) else torch.device("cpu") + tensor = torch.as_tensor(tensor, dtype=torch.float32, device=device) + if tensor.numel() == 0: + # Use reshape, so we don't end up creating a new tensor that does not depend on + # the inputs (and consequently confuses jit) + tensor = tensor.reshape((0, 5)).to(dtype=torch.float32, device=device) + assert tensor.dim() == 2 and tensor.size(-1) == 5, tensor.size() + + self.tensor = tensor + + def clone(self) -> "RotatedBoxes": + """ + Clone the RotatedBoxes. + + Returns: + RotatedBoxes + """ + return RotatedBoxes(self.tensor.clone()) + + def to(self, device: torch.device): + # Boxes are assumed float32 and does not support to(dtype) + return RotatedBoxes(self.tensor.to(device=device)) + + def area(self) -> torch.Tensor: + """ + Computes the area of all the boxes. + + Returns: + torch.Tensor: a vector with areas of each box. + """ + box = self.tensor + area = box[:, 2] * box[:, 3] + return area + + def normalize_angles(self) -> None: + """ + Restrict angles to the range of [-180, 180) degrees + """ + self.tensor[:, 4] = (self.tensor[:, 4] + 180.0) % 360.0 - 180.0 + + def clip(self, box_size: Tuple[int, int], clip_angle_threshold: float = 1.0) -> None: + """ + Clip (in place) the boxes by limiting x coordinates to the range [0, width] + and y coordinates to the range [0, height]. + + For RRPN: + Only clip boxes that are almost horizontal with a tolerance of + clip_angle_threshold to maintain backward compatibility. + + Rotated boxes beyond this threshold are not clipped for two reasons: + + 1. There are potentially multiple ways to clip a rotated box to make it + fit within the image. + 2. It's tricky to make the entire rectangular box fit within the image + and still be able to not leave out pixels of interest. + + Therefore we rely on ops like RoIAlignRotated to safely handle this. + + Args: + box_size (height, width): The clipping box's size. + clip_angle_threshold: + Iff. abs(normalized(angle)) <= clip_angle_threshold (in degrees), + we do the clipping as horizontal boxes. + """ + h, w = box_size + + # normalize angles to be within (-180, 180] degrees + self.normalize_angles() + + idx = torch.where(torch.abs(self.tensor[:, 4]) <= clip_angle_threshold)[0] + + # convert to (x1, y1, x2, y2) + x1 = self.tensor[idx, 0] - self.tensor[idx, 2] / 2.0 + y1 = self.tensor[idx, 1] - self.tensor[idx, 3] / 2.0 + x2 = self.tensor[idx, 0] + self.tensor[idx, 2] / 2.0 + y2 = self.tensor[idx, 1] + self.tensor[idx, 3] / 2.0 + + # clip + x1.clamp_(min=0, max=w) + y1.clamp_(min=0, max=h) + x2.clamp_(min=0, max=w) + y2.clamp_(min=0, max=h) + + # convert back to (xc, yc, w, h) + self.tensor[idx, 0] = (x1 + x2) / 2.0 + self.tensor[idx, 1] = (y1 + y2) / 2.0 + # make sure widths and heights do not increase due to numerical errors + self.tensor[idx, 2] = torch.min(self.tensor[idx, 2], x2 - x1) + self.tensor[idx, 3] = torch.min(self.tensor[idx, 3], y2 - y1) + + def nonempty(self, threshold: float = 0.0) -> torch.Tensor: + """ + Find boxes that are non-empty. + A box is considered empty, if either of its side is no larger than threshold. + + Returns: + Tensor: a binary vector which represents + whether each box is empty (False) or non-empty (True). + """ + box = self.tensor + widths = box[:, 2] + heights = box[:, 3] + keep = (widths > threshold) & (heights > threshold) + return keep + + def __getitem__(self, item) -> "RotatedBoxes": + """ + Returns: + RotatedBoxes: Create a new :class:`RotatedBoxes` by indexing. + + The following usage are allowed: + + 1. `new_boxes = boxes[3]`: return a `RotatedBoxes` which contains only one box. + 2. `new_boxes = boxes[2:10]`: return a slice of boxes. + 3. `new_boxes = boxes[vector]`, where vector is a torch.ByteTensor + with `length = len(boxes)`. Nonzero elements in the vector will be selected. + + Note that the returned RotatedBoxes might share storage with this RotatedBoxes, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return RotatedBoxes(self.tensor[item].view(1, -1)) + b = self.tensor[item] + assert b.dim() == 2, "Indexing on RotatedBoxes with {} failed to return a matrix!".format(item) + return RotatedBoxes(b) + + def __len__(self) -> int: + return self.tensor.shape[0] + + def __repr__(self) -> str: + return "RotatedBoxes(" + str(self.tensor) + ")" + + def inside_box(self, box_size: Tuple[int, int], boundary_threshold: int = 0) -> torch.Tensor: + """ + Args: + box_size (height, width): Size of the reference box covering + [0, width] x [0, height] + boundary_threshold (int): Boxes that extend beyond the reference box + boundary by more than boundary_threshold are considered "outside". + + For RRPN, it might not be necessary to call this function since it's common + for rotated box to extend to outside of the image boundaries + (the clip function only clips the near-horizontal boxes) + + Returns: + a binary vector, indicating whether each box is inside the reference box. + """ + height, width = box_size + + cnt_x = self.tensor[..., 0] + cnt_y = self.tensor[..., 1] + half_w = self.tensor[..., 2] / 2.0 + half_h = self.tensor[..., 3] / 2.0 + a = self.tensor[..., 4] + c = torch.abs(torch.cos(a * math.pi / 180.0)) + s = torch.abs(torch.sin(a * math.pi / 180.0)) + # This basically computes the horizontal bounding rectangle of the rotated box + max_rect_dx = c * half_w + s * half_h + max_rect_dy = c * half_h + s * half_w + + inds_inside = ( + (cnt_x - max_rect_dx >= -boundary_threshold) + & (cnt_y - max_rect_dy >= -boundary_threshold) + & (cnt_x + max_rect_dx < width + boundary_threshold) + & (cnt_y + max_rect_dy < height + boundary_threshold) + ) + + return inds_inside + + def get_centers(self) -> torch.Tensor: + """ + Returns: + The box centers in a Nx2 array of (x, y). + """ + return self.tensor[:, :2] + + def scale(self, scale_x: float, scale_y: float) -> None: + """ + Scale the rotated box with horizontal and vertical scaling factors + Note: when scale_factor_x != scale_factor_y, + the rotated box does not preserve the rectangular shape when the angle + is not a multiple of 90 degrees under resize transformation. + Instead, the shape is a parallelogram (that has skew) + Here we make an approximation by fitting a rotated rectangle to the parallelogram. + """ + self.tensor[:, 0] *= scale_x + self.tensor[:, 1] *= scale_y + theta = self.tensor[:, 4] * math.pi / 180.0 + c = torch.cos(theta) + s = torch.sin(theta) + + # In image space, y is top->down and x is left->right + # Consider the local coordintate system for the rotated box, + # where the box center is located at (0, 0), and the four vertices ABCD are + # A(-w / 2, -h / 2), B(w / 2, -h / 2), C(w / 2, h / 2), D(-w / 2, h / 2) + # the midpoint of the left edge AD of the rotated box E is: + # E = (A+D)/2 = (-w / 2, 0) + # the midpoint of the top edge AB of the rotated box F is: + # F(0, -h / 2) + # To get the old coordinates in the global system, apply the rotation transformation + # (Note: the right-handed coordinate system for image space is yOx): + # (old_x, old_y) = (s * y + c * x, c * y - s * x) + # E(old) = (s * 0 + c * (-w/2), c * 0 - s * (-w/2)) = (-c * w / 2, s * w / 2) + # F(old) = (s * (-h / 2) + c * 0, c * (-h / 2) - s * 0) = (-s * h / 2, -c * h / 2) + # After applying the scaling factor (sfx, sfy): + # E(new) = (-sfx * c * w / 2, sfy * s * w / 2) + # F(new) = (-sfx * s * h / 2, -sfy * c * h / 2) + # The new width after scaling tranformation becomes: + + # w(new) = |E(new) - O| * 2 + # = sqrt[(sfx * c * w / 2)^2 + (sfy * s * w / 2)^2] * 2 + # = sqrt[(sfx * c)^2 + (sfy * s)^2] * w + # i.e., scale_factor_w = sqrt[(sfx * c)^2 + (sfy * s)^2] + # + # For example, + # when angle = 0 or 180, |c| = 1, s = 0, scale_factor_w == scale_factor_x; + # when |angle| = 90, c = 0, |s| = 1, scale_factor_w == scale_factor_y + self.tensor[:, 2] *= torch.sqrt((scale_x * c) ** 2 + (scale_y * s) ** 2) + + # h(new) = |F(new) - O| * 2 + # = sqrt[(sfx * s * h / 2)^2 + (sfy * c * h / 2)^2] * 2 + # = sqrt[(sfx * s)^2 + (sfy * c)^2] * h + # i.e., scale_factor_h = sqrt[(sfx * s)^2 + (sfy * c)^2] + # + # For example, + # when angle = 0 or 180, |c| = 1, s = 0, scale_factor_h == scale_factor_y; + # when |angle| = 90, c = 0, |s| = 1, scale_factor_h == scale_factor_x + self.tensor[:, 3] *= torch.sqrt((scale_x * s) ** 2 + (scale_y * c) ** 2) + + # The angle is the rotation angle from y-axis in image space to the height + # vector (top->down in the box's local coordinate system) of the box in CCW. + # + # angle(new) = angle_yOx(O - F(new)) + # = angle_yOx( (sfx * s * h / 2, sfy * c * h / 2) ) + # = atan2(sfx * s * h / 2, sfy * c * h / 2) + # = atan2(sfx * s, sfy * c) + # + # For example, + # when sfx == sfy, angle(new) == atan2(s, c) == angle(old) + self.tensor[:, 4] = torch.atan2(scale_x * s, scale_y * c) * 180 / math.pi + + @classmethod + def cat(cls, boxes_list: List["RotatedBoxes"]) -> "RotatedBoxes": + """ + Concatenates a list of RotatedBoxes into a single RotatedBoxes + + Arguments: + boxes_list (list[RotatedBoxes]) + + Returns: + RotatedBoxes: the concatenated RotatedBoxes + """ + assert isinstance(boxes_list, (list, tuple)) + if len(boxes_list) == 0: + return cls(torch.empty(0)) + assert all([isinstance(box, RotatedBoxes) for box in boxes_list]) + + # use torch.cat (v.s. layers.cat) so the returned boxes never share storage with input + cat_boxes = cls(torch.cat([b.tensor for b in boxes_list], dim=0)) + return cat_boxes + + @property + def device(self) -> torch.device: + return self.tensor.device + + @torch.jit.unused + def __iter__(self): + """ + Yield a box as a Tensor of shape (5,) at a time. + """ + yield from self.tensor + + +def pairwise_iou(boxes1: RotatedBoxes, boxes2: RotatedBoxes) -> None: + """ + Given two lists of rotated boxes of size N and M, + compute the IoU (intersection over union) + between **all** N x M pairs of boxes. + The box order must be (x_center, y_center, width, height, angle). + + Args: + boxes1, boxes2 (RotatedBoxes): + two `RotatedBoxes`. Contains N & M rotated boxes, respectively. + + Returns: + Tensor: IoU, sized [N,M]. + """ + + return pairwise_iou_rotated(boxes1.tensor, boxes2.tensor) diff --git a/detectron2/tracking/__init__.py b/detectron2/tracking/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..21078ae822b04b71dbd8b056b5993d173eaf6bff --- /dev/null +++ b/detectron2/tracking/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .base_tracker import ( # noqa + BaseTracker, + build_tracker_head, + TRACKER_HEADS_REGISTRY, +) +from .bbox_iou_tracker import BBoxIOUTracker # noqa +from .hungarian_tracker import BaseHungarianTracker # noqa +from .iou_weighted_hungarian_bbox_iou_tracker import ( # noqa + IOUWeightedHungarianBBoxIOUTracker, +) +from .utils import create_prediction_pairs # noqa +from .vanilla_hungarian_bbox_iou_tracker import VanillaHungarianBBoxIOUTracker # noqa + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/detectron2/tracking/base_tracker.py b/detectron2/tracking/base_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..a8872f71692bd3b372603e8608264c934faadb84 --- /dev/null +++ b/detectron2/tracking/base_tracker.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. +from detectron2.config import configurable +from detectron2.utils.registry import Registry + +from ..config.config import CfgNode as CfgNode_ +from ..structures import Instances + +TRACKER_HEADS_REGISTRY = Registry("TRACKER_HEADS") +TRACKER_HEADS_REGISTRY.__doc__ = """ +Registry for tracking classes. +""" + + +class BaseTracker(object): + """ + A parent class for all trackers + """ + + @configurable + def __init__(self, **kwargs): + self._prev_instances = None # (D2)instances for previous frame + self._matched_idx = set() # indices in prev_instances found matching + self._matched_ID = set() # idendities in prev_instances found matching + self._untracked_prev_idx = set() # indices in prev_instances not found matching + self._id_count = 0 # used to assign new id + + @classmethod + def from_config(cls, cfg: CfgNode_): + raise NotImplementedError("Calling BaseTracker::from_config") + + def update(self, predictions: Instances) -> Instances: + """ + Args: + predictions: D2 Instances for predictions of the current frame + Return: + D2 Instances for predictions of the current frame with ID assigned + + _prev_instances and instances will have the following fields: + .pred_boxes (shape=[N, 4]) + .scores (shape=[N,]) + .pred_classes (shape=[N,]) + .pred_keypoints (shape=[N, M, 3], Optional) + .pred_masks (shape=List[2D_MASK], Optional) 2D_MASK: shape=[H, W] + .ID (shape=[N,]) + + N: # of detected bboxes + H and W: height and width of 2D mask + """ + raise NotImplementedError("Calling BaseTracker::update") + + +def build_tracker_head(cfg: CfgNode_) -> BaseTracker: + """ + Build a tracker head from `cfg.TRACKER_HEADS.TRACKER_NAME`. + + Args: + cfg: D2 CfgNode, config file with tracker information + Return: + tracker object + """ + name = cfg.TRACKER_HEADS.TRACKER_NAME + tracker_class = TRACKER_HEADS_REGISTRY.get(name) + return tracker_class(cfg) diff --git a/detectron2/tracking/bbox_iou_tracker.py b/detectron2/tracking/bbox_iou_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..09d5ddc3ef782abc33b44d8e747e93be604e6d88 --- /dev/null +++ b/detectron2/tracking/bbox_iou_tracker.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. +import copy +from typing import List + +import numpy as np +import torch + +from detectron2.config import configurable +from detectron2.structures import Boxes, Instances +from detectron2.structures.boxes import pairwise_iou + +from ..config.config import CfgNode as CfgNode_ +from .base_tracker import TRACKER_HEADS_REGISTRY, BaseTracker + + +@TRACKER_HEADS_REGISTRY.register() +class BBoxIOUTracker(BaseTracker): + """ + A bounding box tracker to assign ID based on IoU between current and previous instances + """ + + @configurable + def __init__( + self, + *, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + track_iou_threshold: float = 0.5, + **kwargs, + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + track_iou_threshold: iou threshold, below this number a bbox pair is removed + from tracking + """ + super().__init__(**kwargs) + self._video_height = video_height + self._video_width = video_width + self._max_num_instances = max_num_instances + self._max_lost_frame_count = max_lost_frame_count + self._min_box_rel_dim = min_box_rel_dim + self._min_instance_period = min_instance_period + self._track_iou_threshold = track_iou_threshold + + @classmethod + def from_config(cls, cfg: CfgNode_): + """ + Old style initialization using CfgNode + + Args: + cfg: D2 CfgNode, config file + Return: + dictionary storing arguments for __init__ method + """ + assert "VIDEO_HEIGHT" in cfg.TRACKER_HEADS + assert "VIDEO_WIDTH" in cfg.TRACKER_HEADS + video_height = cfg.TRACKER_HEADS.get("VIDEO_HEIGHT") + video_width = cfg.TRACKER_HEADS.get("VIDEO_WIDTH") + max_num_instances = cfg.TRACKER_HEADS.get("MAX_NUM_INSTANCES", 200) + max_lost_frame_count = cfg.TRACKER_HEADS.get("MAX_LOST_FRAME_COUNT", 0) + min_box_rel_dim = cfg.TRACKER_HEADS.get("MIN_BOX_REL_DIM", 0.02) + min_instance_period = cfg.TRACKER_HEADS.get("MIN_INSTANCE_PERIOD", 1) + track_iou_threshold = cfg.TRACKER_HEADS.get("TRACK_IOU_THRESHOLD", 0.5) + return { + "_target_": "detectron2.tracking.bbox_iou_tracker.BBoxIOUTracker", + "video_height": video_height, + "video_width": video_width, + "max_num_instances": max_num_instances, + "max_lost_frame_count": max_lost_frame_count, + "min_box_rel_dim": min_box_rel_dim, + "min_instance_period": min_instance_period, + "track_iou_threshold": track_iou_threshold, + } + + def update(self, instances: Instances) -> Instances: + """ + See BaseTracker description + """ + instances = self._initialize_extra_fields(instances) + if self._prev_instances is not None: + # calculate IoU of all bbox pairs + iou_all = pairwise_iou( + boxes1=instances.pred_boxes, + boxes2=self._prev_instances.pred_boxes, + ) + # sort IoU in descending order + bbox_pairs = self._create_prediction_pairs(instances, iou_all) + # assign previous ID to current bbox if IoU > track_iou_threshold + self._reset_fields() + for bbox_pair in bbox_pairs: + idx = bbox_pair["idx"] + prev_id = bbox_pair["prev_id"] + if ( + idx in self._matched_idx + or prev_id in self._matched_ID + or bbox_pair["IoU"] < self._track_iou_threshold + ): + continue + instances.ID[idx] = prev_id + instances.ID_period[idx] = bbox_pair["prev_period"] + 1 + instances.lost_frame_count[idx] = 0 + self._matched_idx.add(idx) + self._matched_ID.add(prev_id) + self._untracked_prev_idx.remove(bbox_pair["prev_idx"]) + instances = self._assign_new_id(instances) + instances = self._merge_untracked_instances(instances) + self._prev_instances = copy.deepcopy(instances) + return instances + + def _create_prediction_pairs(self, instances: Instances, iou_all: np.ndarray) -> List: + """ + For all instances in previous and current frames, create pairs. For each + pair, store index of the instance in current frame predcitions, index in + previous predictions, ID in previous predictions, IoU of the bboxes in this + pair, period in previous predictions. + + Args: + instances: D2 Instances, for predictions of the current frame + iou_all: IoU for all bboxes pairs + Return: + A list of IoU for all pairs + """ + bbox_pairs = [] + for i in range(len(instances)): + for j in range(len(self._prev_instances)): + bbox_pairs.append( + { + "idx": i, + "prev_idx": j, + "prev_id": self._prev_instances.ID[j], + "IoU": iou_all[i, j], + "prev_period": self._prev_instances.ID_period[j], + } + ) + return bbox_pairs + + def _initialize_extra_fields(self, instances: Instances) -> Instances: + """ + If input instances don't have ID, ID_period, lost_frame_count fields, + this method is used to initialize these fields. + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances with extra fields added + """ + if not instances.has("ID"): + instances.set("ID", [None] * len(instances)) + if not instances.has("ID_period"): + instances.set("ID_period", [None] * len(instances)) + if not instances.has("lost_frame_count"): + instances.set("lost_frame_count", [None] * len(instances)) + if self._prev_instances is None: + instances.ID = list(range(len(instances))) + self._id_count += len(instances) + instances.ID_period = [1] * len(instances) + instances.lost_frame_count = [0] * len(instances) + return instances + + def _reset_fields(self): + """ + Before each uodate call, reset fields first + """ + self._matched_idx = set() + self._matched_ID = set() + self._untracked_prev_idx = set(range(len(self._prev_instances))) + + def _assign_new_id(self, instances: Instances) -> Instances: + """ + For each untracked instance, assign a new id + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances with new ID assigned + """ + untracked_idx = set(range(len(instances))).difference(self._matched_idx) + for idx in untracked_idx: + instances.ID[idx] = self._id_count + self._id_count += 1 + instances.ID_period[idx] = 1 + instances.lost_frame_count[idx] = 0 + return instances + + def _merge_untracked_instances(self, instances: Instances) -> Instances: + """ + For untracked previous instances, under certain condition, still keep them + in tracking and merge with the current instances. + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances merging current instances and instances from previous + frame decided to keep tracking + """ + untracked_instances = Instances( + image_size=instances.image_size, + pred_boxes=[], + pred_classes=[], + scores=[], + ID=[], + ID_period=[], + lost_frame_count=[], + ) + prev_bboxes = list(self._prev_instances.pred_boxes) + prev_classes = list(self._prev_instances.pred_classes) + prev_scores = list(self._prev_instances.scores) + prev_ID_period = self._prev_instances.ID_period + if instances.has("pred_masks"): + untracked_instances.set("pred_masks", []) + prev_masks = list(self._prev_instances.pred_masks) + if instances.has("pred_keypoints"): + untracked_instances.set("pred_keypoints", []) + prev_keypoints = list(self._prev_instances.pred_keypoints) + if instances.has("pred_keypoint_heatmaps"): + untracked_instances.set("pred_keypoint_heatmaps", []) + prev_keypoint_heatmaps = list(self._prev_instances.pred_keypoint_heatmaps) + for idx in self._untracked_prev_idx: + x_left, y_top, x_right, y_bot = prev_bboxes[idx] + if ( + (1.0 * (x_right - x_left) / self._video_width < self._min_box_rel_dim) + or (1.0 * (y_bot - y_top) / self._video_height < self._min_box_rel_dim) + or self._prev_instances.lost_frame_count[idx] >= self._max_lost_frame_count + or prev_ID_period[idx] <= self._min_instance_period + ): + continue + untracked_instances.pred_boxes.append(list(prev_bboxes[idx].numpy())) + untracked_instances.pred_classes.append(int(prev_classes[idx])) + untracked_instances.scores.append(float(prev_scores[idx])) + untracked_instances.ID.append(self._prev_instances.ID[idx]) + untracked_instances.ID_period.append(self._prev_instances.ID_period[idx]) + untracked_instances.lost_frame_count.append(self._prev_instances.lost_frame_count[idx] + 1) + if instances.has("pred_masks"): + untracked_instances.pred_masks.append(prev_masks[idx].numpy().astype(np.uint8)) + if instances.has("pred_keypoints"): + untracked_instances.pred_keypoints.append(prev_keypoints[idx].numpy().astype(np.uint8)) + if instances.has("pred_keypoint_heatmaps"): + untracked_instances.pred_keypoint_heatmaps.append( + prev_keypoint_heatmaps[idx].numpy().astype(np.float32) + ) + untracked_instances.pred_boxes = Boxes(torch.FloatTensor(untracked_instances.pred_boxes)) + untracked_instances.pred_classes = torch.IntTensor(untracked_instances.pred_classes) + untracked_instances.scores = torch.FloatTensor(untracked_instances.scores) + if instances.has("pred_masks"): + untracked_instances.pred_masks = torch.IntTensor(untracked_instances.pred_masks) + if instances.has("pred_keypoints"): + untracked_instances.pred_keypoints = torch.IntTensor(untracked_instances.pred_keypoints) + if instances.has("pred_keypoint_heatmaps"): + untracked_instances.pred_keypoint_heatmaps = torch.FloatTensor(untracked_instances.pred_keypoint_heatmaps) + + return Instances.cat( + [ + instances, + untracked_instances, + ] + ) diff --git a/detectron2/tracking/hungarian_tracker.py b/detectron2/tracking/hungarian_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..2b448a887a819128c6ed46a5276e07568efd273c --- /dev/null +++ b/detectron2/tracking/hungarian_tracker.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. +import copy +from typing import Dict + +import numpy as np +import torch +from scipy.optimize import linear_sum_assignment + +from detectron2.config import configurable +from detectron2.structures import Boxes, Instances + +from ..config.config import CfgNode as CfgNode_ +from .base_tracker import BaseTracker + + +class BaseHungarianTracker(BaseTracker): + """ + A base class for all Hungarian trackers + """ + + @configurable + def __init__( + self, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + **kwargs + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + """ + super().__init__(**kwargs) + self._video_height = video_height + self._video_width = video_width + self._max_num_instances = max_num_instances + self._max_lost_frame_count = max_lost_frame_count + self._min_box_rel_dim = min_box_rel_dim + self._min_instance_period = min_instance_period + + @classmethod + def from_config(cls, cfg: CfgNode_) -> Dict: + raise NotImplementedError("Calling HungarianTracker::from_config") + + def build_cost_matrix(self, instances: Instances, prev_instances: Instances) -> np.ndarray: + raise NotImplementedError("Calling HungarianTracker::build_matrix") + + def update(self, instances: Instances) -> Instances: + if instances.has("pred_keypoints"): + raise NotImplementedError("Need to add support for keypoints") + instances = self._initialize_extra_fields(instances) + if self._prev_instances is not None: + self._untracked_prev_idx = set(range(len(self._prev_instances))) + cost_matrix = self.build_cost_matrix(instances, self._prev_instances) + matched_idx, matched_prev_idx = linear_sum_assignment(cost_matrix) + instances = self._process_matched_idx(instances, matched_idx, matched_prev_idx) + instances = self._process_unmatched_idx(instances, matched_idx) + instances = self._process_unmatched_prev_idx(instances, matched_prev_idx) + self._prev_instances = copy.deepcopy(instances) + return instances + + def _initialize_extra_fields(self, instances: Instances) -> Instances: + """ + If input instances don't have ID, ID_period, lost_frame_count fields, + this method is used to initialize these fields. + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances with extra fields added + """ + if not instances.has("ID"): + instances.set("ID", [None] * len(instances)) + if not instances.has("ID_period"): + instances.set("ID_period", [None] * len(instances)) + if not instances.has("lost_frame_count"): + instances.set("lost_frame_count", [None] * len(instances)) + if self._prev_instances is None: + instances.ID = list(range(len(instances))) + self._id_count += len(instances) + instances.ID_period = [1] * len(instances) + instances.lost_frame_count = [0] * len(instances) + return instances + + def _process_matched_idx( + self, instances: Instances, matched_idx: np.ndarray, matched_prev_idx: np.ndarray + ) -> Instances: + assert matched_idx.size == matched_prev_idx.size + for i in range(matched_idx.size): + instances.ID[matched_idx[i]] = self._prev_instances.ID[matched_prev_idx[i]] + instances.ID_period[matched_idx[i]] = self._prev_instances.ID_period[matched_prev_idx[i]] + 1 + instances.lost_frame_count[matched_idx[i]] = 0 + return instances + + def _process_unmatched_idx(self, instances: Instances, matched_idx: np.ndarray) -> Instances: + untracked_idx = set(range(len(instances))).difference(set(matched_idx)) + for idx in untracked_idx: + instances.ID[idx] = self._id_count + self._id_count += 1 + instances.ID_period[idx] = 1 + instances.lost_frame_count[idx] = 0 + return instances + + def _process_unmatched_prev_idx(self, instances: Instances, matched_prev_idx: np.ndarray) -> Instances: + untracked_instances = Instances( + image_size=instances.image_size, + pred_boxes=[], + pred_masks=[], + pred_classes=[], + scores=[], + ID=[], + ID_period=[], + lost_frame_count=[], + ) + prev_bboxes = list(self._prev_instances.pred_boxes) + prev_classes = list(self._prev_instances.pred_classes) + prev_scores = list(self._prev_instances.scores) + prev_ID_period = self._prev_instances.ID_period + if instances.has("pred_masks"): + prev_masks = list(self._prev_instances.pred_masks) + untracked_prev_idx = set(range(len(self._prev_instances))).difference(set(matched_prev_idx)) + for idx in untracked_prev_idx: + x_left, y_top, x_right, y_bot = prev_bboxes[idx] + if ( + (1.0 * (x_right - x_left) / self._video_width < self._min_box_rel_dim) + or (1.0 * (y_bot - y_top) / self._video_height < self._min_box_rel_dim) + or self._prev_instances.lost_frame_count[idx] >= self._max_lost_frame_count + or prev_ID_period[idx] <= self._min_instance_period + ): + continue + untracked_instances.pred_boxes.append(list(prev_bboxes[idx].numpy())) + untracked_instances.pred_classes.append(int(prev_classes[idx])) + untracked_instances.scores.append(float(prev_scores[idx])) + untracked_instances.ID.append(self._prev_instances.ID[idx]) + untracked_instances.ID_period.append(self._prev_instances.ID_period[idx]) + untracked_instances.lost_frame_count.append(self._prev_instances.lost_frame_count[idx] + 1) + if instances.has("pred_masks"): + untracked_instances.pred_masks.append(prev_masks[idx].numpy().astype(np.uint8)) + + untracked_instances.pred_boxes = Boxes(torch.FloatTensor(untracked_instances.pred_boxes)) + untracked_instances.pred_classes = torch.IntTensor(untracked_instances.pred_classes) + untracked_instances.scores = torch.FloatTensor(untracked_instances.scores) + if instances.has("pred_masks"): + untracked_instances.pred_masks = torch.IntTensor(untracked_instances.pred_masks) + else: + untracked_instances.remove("pred_masks") + + return Instances.cat( + [ + instances, + untracked_instances, + ] + ) diff --git a/detectron2/tracking/iou_weighted_hungarian_bbox_iou_tracker.py b/detectron2/tracking/iou_weighted_hungarian_bbox_iou_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..59012d38c22217c6e3bdf636988480b6d2b43431 --- /dev/null +++ b/detectron2/tracking/iou_weighted_hungarian_bbox_iou_tracker.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. + +from typing import List + +import numpy as np + +from detectron2.config import CfgNode as CfgNode_ +from detectron2.config import configurable + +from .base_tracker import TRACKER_HEADS_REGISTRY +from .vanilla_hungarian_bbox_iou_tracker import VanillaHungarianBBoxIOUTracker + + +@TRACKER_HEADS_REGISTRY.register() +class IOUWeightedHungarianBBoxIOUTracker(VanillaHungarianBBoxIOUTracker): + """ + A tracker using IoU as weight in Hungarian algorithm, also known + as Munkres or Kuhn-Munkres algorithm + """ + + @configurable + def __init__( + self, + *, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + track_iou_threshold: float = 0.5, + **kwargs, + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + track_iou_threshold: iou threshold, below this number a bbox pair is removed + from tracking + """ + super().__init__( + video_height=video_height, + video_width=video_width, + max_num_instances=max_num_instances, + max_lost_frame_count=max_lost_frame_count, + min_box_rel_dim=min_box_rel_dim, + min_instance_period=min_instance_period, + track_iou_threshold=track_iou_threshold, + ) + + @classmethod + def from_config(cls, cfg: CfgNode_): + """ + Old style initialization using CfgNode + + Args: + cfg: D2 CfgNode, config file + Return: + dictionary storing arguments for __init__ method + """ + assert "VIDEO_HEIGHT" in cfg.TRACKER_HEADS + assert "VIDEO_WIDTH" in cfg.TRACKER_HEADS + video_height = cfg.TRACKER_HEADS.get("VIDEO_HEIGHT") + video_width = cfg.TRACKER_HEADS.get("VIDEO_WIDTH") + max_num_instances = cfg.TRACKER_HEADS.get("MAX_NUM_INSTANCES", 200) + max_lost_frame_count = cfg.TRACKER_HEADS.get("MAX_LOST_FRAME_COUNT", 0) + min_box_rel_dim = cfg.TRACKER_HEADS.get("MIN_BOX_REL_DIM", 0.02) + min_instance_period = cfg.TRACKER_HEADS.get("MIN_INSTANCE_PERIOD", 1) + track_iou_threshold = cfg.TRACKER_HEADS.get("TRACK_IOU_THRESHOLD", 0.5) + return { + "_target_": "detectron2.tracking.iou_weighted_hungarian_bbox_iou_tracker.IOUWeightedHungarianBBoxIOUTracker", # noqa + "video_height": video_height, + "video_width": video_width, + "max_num_instances": max_num_instances, + "max_lost_frame_count": max_lost_frame_count, + "min_box_rel_dim": min_box_rel_dim, + "min_instance_period": min_instance_period, + "track_iou_threshold": track_iou_threshold, + } + + def assign_cost_matrix_values(self, cost_matrix: np.ndarray, bbox_pairs: List) -> np.ndarray: + """ + Based on IoU for each pair of bbox, assign the associated value in cost matrix + + Args: + cost_matrix: np.ndarray, initialized 2D array with target dimensions + bbox_pairs: list of bbox pair, in each pair, iou value is stored + Return: + np.ndarray, cost_matrix with assigned values + """ + for pair in bbox_pairs: + # assign (-1 * IoU) for above threshold pairs, algorithms will minimize cost + cost_matrix[pair["idx"]][pair["prev_idx"]] = -1 * pair["IoU"] + return cost_matrix diff --git a/detectron2/tracking/utils.py b/detectron2/tracking/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..56565647f0df952ef4c410c11665a64549a68767 --- /dev/null +++ b/detectron2/tracking/utils.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +from typing import List + +import numpy as np + +from detectron2.structures import Instances + + +def create_prediction_pairs( + instances: Instances, + prev_instances: Instances, + iou_all: np.ndarray, + threshold: float = 0.5, +) -> List: + """ + Args: + instances: predictions from current frame + prev_instances: predictions from previous frame + iou_all: 2D numpy array containing iou for each bbox pair + threshold: below the threshold, doesn't consider the pair of bbox is valid + Return: + List of bbox pairs + """ + bbox_pairs = [] + for i in range(len(instances)): + for j in range(len(prev_instances)): + if iou_all[i, j] < threshold: + continue + bbox_pairs.append( + { + "idx": i, + "prev_idx": j, + "prev_id": prev_instances.ID[j], + "IoU": iou_all[i, j], + "prev_period": prev_instances.ID_period[j], + } + ) + return bbox_pairs + + +LARGE_COST_VALUE = 100000 diff --git a/detectron2/tracking/vanilla_hungarian_bbox_iou_tracker.py b/detectron2/tracking/vanilla_hungarian_bbox_iou_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..61a73a47514cad61cf3c03137ce4a2cfd1b457f6 --- /dev/null +++ b/detectron2/tracking/vanilla_hungarian_bbox_iou_tracker.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. + +from typing import List + +import numpy as np + +from detectron2.config import CfgNode as CfgNode_ +from detectron2.config import configurable +from detectron2.structures import Instances +from detectron2.structures.boxes import pairwise_iou +from detectron2.tracking.utils import LARGE_COST_VALUE, create_prediction_pairs + +from .base_tracker import TRACKER_HEADS_REGISTRY +from .hungarian_tracker import BaseHungarianTracker + + +@TRACKER_HEADS_REGISTRY.register() +class VanillaHungarianBBoxIOUTracker(BaseHungarianTracker): + """ + Hungarian algo based tracker using bbox iou as metric + """ + + @configurable + def __init__( + self, + *, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + track_iou_threshold: float = 0.5, + **kwargs, + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + track_iou_threshold: iou threshold, below this number a bbox pair is removed + from tracking + """ + super().__init__( + video_height=video_height, + video_width=video_width, + max_num_instances=max_num_instances, + max_lost_frame_count=max_lost_frame_count, + min_box_rel_dim=min_box_rel_dim, + min_instance_period=min_instance_period, + ) + self._track_iou_threshold = track_iou_threshold + + @classmethod + def from_config(cls, cfg: CfgNode_): + """ + Old style initialization using CfgNode + + Args: + cfg: D2 CfgNode, config file + Return: + dictionary storing arguments for __init__ method + """ + assert "VIDEO_HEIGHT" in cfg.TRACKER_HEADS + assert "VIDEO_WIDTH" in cfg.TRACKER_HEADS + video_height = cfg.TRACKER_HEADS.get("VIDEO_HEIGHT") + video_width = cfg.TRACKER_HEADS.get("VIDEO_WIDTH") + max_num_instances = cfg.TRACKER_HEADS.get("MAX_NUM_INSTANCES", 200) + max_lost_frame_count = cfg.TRACKER_HEADS.get("MAX_LOST_FRAME_COUNT", 0) + min_box_rel_dim = cfg.TRACKER_HEADS.get("MIN_BOX_REL_DIM", 0.02) + min_instance_period = cfg.TRACKER_HEADS.get("MIN_INSTANCE_PERIOD", 1) + track_iou_threshold = cfg.TRACKER_HEADS.get("TRACK_IOU_THRESHOLD", 0.5) + return { + "_target_": "detectron2.tracking.vanilla_hungarian_bbox_iou_tracker.VanillaHungarianBBoxIOUTracker", # noqa + "video_height": video_height, + "video_width": video_width, + "max_num_instances": max_num_instances, + "max_lost_frame_count": max_lost_frame_count, + "min_box_rel_dim": min_box_rel_dim, + "min_instance_period": min_instance_period, + "track_iou_threshold": track_iou_threshold, + } + + def build_cost_matrix(self, instances: Instances, prev_instances: Instances) -> np.ndarray: + """ + Build the cost matrix for assignment problem + (https://en.wikipedia.org/wiki/Assignment_problem) + + Args: + instances: D2 Instances, for current frame predictions + prev_instances: D2 Instances, for previous frame predictions + + Return: + the cost matrix in numpy array + """ + assert instances is not None and prev_instances is not None + # calculate IoU of all bbox pairs + iou_all = pairwise_iou( + boxes1=instances.pred_boxes, + boxes2=self._prev_instances.pred_boxes, + ) + bbox_pairs = create_prediction_pairs( + instances, self._prev_instances, iou_all, threshold=self._track_iou_threshold + ) + # assign large cost value to make sure pair below IoU threshold won't be matched + cost_matrix = np.full((len(instances), len(prev_instances)), LARGE_COST_VALUE) + return self.assign_cost_matrix_values(cost_matrix, bbox_pairs) + + def assign_cost_matrix_values(self, cost_matrix: np.ndarray, bbox_pairs: List) -> np.ndarray: + """ + Based on IoU for each pair of bbox, assign the associated value in cost matrix + + Args: + cost_matrix: np.ndarray, initialized 2D array with target dimensions + bbox_pairs: list of bbox pair, in each pair, iou value is stored + Return: + np.ndarray, cost_matrix with assigned values + """ + for pair in bbox_pairs: + # assign -1 for IoU above threshold pairs, algorithms will minimize cost + cost_matrix[pair["idx"]][pair["prev_idx"]] = -1 + return cost_matrix diff --git a/detectron2/utils/README.md b/detectron2/utils/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9765b24a730b77556104187ac3ef5439ab0859fd --- /dev/null +++ b/detectron2/utils/README.md @@ -0,0 +1,5 @@ +# Utility functions + +This folder contain utility functions that are not used in the +core library, but are useful for building models or training +code using the config system. diff --git a/detectron2/utils/__init__.py b/detectron2/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9020c2df23e2af280b7bb168b996ae9eaf312eb8 --- /dev/null +++ b/detectron2/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. diff --git a/detectron2/utils/analysis.py b/detectron2/utils/analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..de0d38a77e4d3b6c53e0f6bf23703db1c40f1ff1 --- /dev/null +++ b/detectron2/utils/analysis.py @@ -0,0 +1,185 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# -*- coding: utf-8 -*- + +import typing +from typing import Any, List + +import fvcore +from fvcore.nn import activation_count, flop_count, parameter_count, parameter_count_table +from torch import nn + +from detectron2.export import TracingAdapter + +__all__ = [ + "activation_count_operators", + "flop_count_operators", + "parameter_count_table", + "parameter_count", + "FlopCountAnalysis", +] + +FLOPS_MODE = "flops" +ACTIVATIONS_MODE = "activations" + + +# Some extra ops to ignore from counting, including elementwise and reduction ops +_IGNORED_OPS = { + "aten::add", + "aten::add_", + "aten::argmax", + "aten::argsort", + "aten::batch_norm", + "aten::constant_pad_nd", + "aten::div", + "aten::div_", + "aten::exp", + "aten::log2", + "aten::max_pool2d", + "aten::meshgrid", + "aten::mul", + "aten::mul_", + "aten::neg", + "aten::nonzero_numpy", + "aten::reciprocal", + "aten::repeat_interleave", + "aten::rsub", + "aten::sigmoid", + "aten::sigmoid_", + "aten::softmax", + "aten::sort", + "aten::sqrt", + "aten::sub", + "torchvision::nms", # TODO estimate flop for nms +} + + +class FlopCountAnalysis(fvcore.nn.FlopCountAnalysis): + """ + Same as :class:`fvcore.nn.FlopCountAnalysis`, but supports detectron2 models. + """ + + def __init__(self, model, inputs): + """ + Args: + model (nn.Module): + inputs (Any): inputs of the given model. Does not have to be tuple of tensors. + """ + wrapper = TracingAdapter(model, inputs, allow_non_tensor=True) + super().__init__(wrapper, wrapper.flattened_inputs) + self.set_op_handle(**{k: None for k in _IGNORED_OPS}) + + +def flop_count_operators(model: nn.Module, inputs: list) -> typing.DefaultDict[str, float]: + """ + Implement operator-level flops counting using jit. + This is a wrapper of :func:`fvcore.nn.flop_count` and adds supports for standard + detection models in detectron2. + Please use :class:`FlopCountAnalysis` for more advanced functionalities. + + Note: + The function runs the input through the model to compute flops. + The flops of a detection model is often input-dependent, for example, + the flops of box & mask head depends on the number of proposals & + the number of detected objects. + Therefore, the flops counting using a single input may not accurately + reflect the computation cost of a model. It's recommended to average + across a number of inputs. + + Args: + model: a detectron2 model that takes `list[dict]` as input. + inputs (list[dict]): inputs to model, in detectron2's standard format. + Only "image" key will be used. + supported_ops (dict[str, Handle]): see documentation of :func:`fvcore.nn.flop_count` + + Returns: + Counter: Gflop count per operator + """ + old_train = model.training + model.eval() + ret = FlopCountAnalysis(model, inputs).by_operator() + model.train(old_train) + return {k: v / 1e9 for k, v in ret.items()} + + +def activation_count_operators(model: nn.Module, inputs: list, **kwargs) -> typing.DefaultDict[str, float]: + """ + Implement operator-level activations counting using jit. + This is a wrapper of fvcore.nn.activation_count, that supports standard detection models + in detectron2. + + Note: + The function runs the input through the model to compute activations. + The activations of a detection model is often input-dependent, for example, + the activations of box & mask head depends on the number of proposals & + the number of detected objects. + + Args: + model: a detectron2 model that takes `list[dict]` as input. + inputs (list[dict]): inputs to model, in detectron2's standard format. + Only "image" key will be used. + + Returns: + Counter: activation count per operator + """ + return _wrapper_count_operators(model=model, inputs=inputs, mode=ACTIVATIONS_MODE, **kwargs) + + +def _wrapper_count_operators(model: nn.Module, inputs: list, mode: str, **kwargs) -> typing.DefaultDict[str, float]: + # ignore some ops + supported_ops = {k: lambda *args, **kwargs: {} for k in _IGNORED_OPS} + supported_ops.update(kwargs.pop("supported_ops", {})) + kwargs["supported_ops"] = supported_ops + + assert len(inputs) == 1, "Please use batch size=1" + tensor_input = inputs[0]["image"] + inputs = [{"image": tensor_input}] # remove other keys, in case there are any + + old_train = model.training + if isinstance(model, (nn.parallel.distributed.DistributedDataParallel, nn.DataParallel)): + model = model.module + wrapper = TracingAdapter(model, inputs) + wrapper.eval() + if mode == FLOPS_MODE: + ret = flop_count(wrapper, (tensor_input,), **kwargs) + elif mode == ACTIVATIONS_MODE: + ret = activation_count(wrapper, (tensor_input,), **kwargs) + else: + raise NotImplementedError("Count for mode {} is not supported yet.".format(mode)) + # compatible with change in fvcore + if isinstance(ret, tuple): + ret = ret[0] + model.train(old_train) + return ret + + +def find_unused_parameters(model: nn.Module, inputs: Any) -> List[str]: + """ + Given a model, find parameters that do not contribute + to the loss. + + Args: + model: a model in training mode that returns losses + inputs: argument or a tuple of arguments. Inputs of the model + + Returns: + list[str]: the name of unused parameters + """ + assert model.training + for _, prm in model.named_parameters(): + prm.grad = None + + if isinstance(inputs, tuple): + losses = model(*inputs) + else: + losses = model(inputs) + + if isinstance(losses, dict): + losses = sum(losses.values()) + losses.backward() + + unused: List[str] = [] + for name, prm in model.named_parameters(): + if prm.grad is None: + unused.append(name) + prm.grad = None + return unused diff --git a/detectron2/utils/collect_env.py b/detectron2/utils/collect_env.py new file mode 100644 index 0000000000000000000000000000000000000000..5c4f5407d401e444b9454bd2d98d50744e3dc27e --- /dev/null +++ b/detectron2/utils/collect_env.py @@ -0,0 +1,233 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import importlib +import os +import re +import subprocess +import sys +from collections import defaultdict + +import numpy as np +import PIL +import torch +import torchvision +from tabulate import tabulate + +__all__ = ["collect_env_info"] + + +def collect_torch_env(): + try: + import torch.__config__ + + return torch.__config__.show() + except ImportError: + # compatible with older versions of pytorch + from torch.utils.collect_env import get_pretty_env_info + + return get_pretty_env_info() + + +def get_env_module(): + var_name = "DETECTRON2_ENV_MODULE" + return var_name, os.environ.get(var_name, "") + + +def detect_compute_compatibility(CUDA_HOME, so_file): + try: + cuobjdump = os.path.join(CUDA_HOME, "bin", "cuobjdump") + if os.path.isfile(cuobjdump): + output = subprocess.check_output("'{}' --list-elf '{}'".format(cuobjdump, so_file), shell=True) + output = output.decode("utf-8").strip().split("\n") + arch = [] + for line in output: + line = re.findall(r"\.sm_([0-9]*)\.", line)[0] + arch.append(".".join(line)) + arch = sorted(set(arch)) + return ", ".join(arch) + else: + return so_file + "; cannot find cuobjdump" + except Exception: + # unhandled failure + return so_file + + +def collect_env_info(): + has_gpu = torch.cuda.is_available() # true for both CUDA & ROCM + torch_version = torch.__version__ + + # NOTE that CUDA_HOME/ROCM_HOME could be None even when CUDA runtime libs are functional + from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME + + has_rocm = False + if (getattr(torch.version, "hip", None) is not None) and (ROCM_HOME is not None): + has_rocm = True + has_cuda = has_gpu and (not has_rocm) + + data = [] + data.append(("sys.platform", sys.platform)) # check-template.yml depends on it + data.append(("Python", sys.version.replace("\n", ""))) + data.append(("numpy", np.__version__)) + + try: + import detectron2 # noqa + + data.append(("detectron2", detectron2.__version__ + " @" + os.path.dirname(detectron2.__file__))) + except ImportError: + data.append(("detectron2", "failed to import")) + except AttributeError: + data.append(("detectron2", "imported a wrong installation")) + + try: + import detectron2._C as _C + except ImportError as e: + data.append(("detectron2._C", f"not built correctly: {e}")) + + # print system compilers when extension fails to build + if sys.platform != "win32": # don't know what to do for windows + try: + # this is how torch/utils/cpp_extensions.py choose compiler + cxx = os.environ.get("CXX", "c++") + cxx = subprocess.check_output("'{}' --version".format(cxx), shell=True) + cxx = cxx.decode("utf-8").strip().split("\n")[0] + except subprocess.SubprocessError: + cxx = "Not found" + data.append(("Compiler ($CXX)", cxx)) + + if has_cuda and CUDA_HOME is not None: + try: + nvcc = os.path.join(CUDA_HOME, "bin", "nvcc") + nvcc = subprocess.check_output("'{}' -V".format(nvcc), shell=True) + nvcc = nvcc.decode("utf-8").strip().split("\n")[-1] + except subprocess.SubprocessError: + nvcc = "Not found" + data.append(("CUDA compiler", nvcc)) + if has_cuda and sys.platform != "win32": + try: + so_file = importlib.util.find_spec("detectron2._C").origin + except (ImportError, AttributeError): + pass + else: + data.append(("detectron2 arch flags", detect_compute_compatibility(CUDA_HOME, so_file))) + else: + # print compilers that are used to build extension + data.append(("Compiler", _C.get_compiler_version())) + data.append(("CUDA compiler", _C.get_cuda_version())) # cuda or hip + if has_cuda and getattr(_C, "has_cuda", lambda: True)(): + data.append(("detectron2 arch flags", detect_compute_compatibility(CUDA_HOME, _C.__file__))) + + data.append(get_env_module()) + data.append(("PyTorch", torch_version + " @" + os.path.dirname(torch.__file__))) + data.append(("PyTorch debug build", torch.version.debug)) + + if not has_gpu: + has_gpu_text = "No: torch.cuda.is_available() == False" + else: + has_gpu_text = "Yes" + data.append(("GPU available", has_gpu_text)) + if has_gpu: + devices = defaultdict(list) + for k in range(torch.cuda.device_count()): + cap = ".".join((str(x) for x in torch.cuda.get_device_capability(k))) + name = torch.cuda.get_device_name(k) + f" (arch={cap})" + devices[name].append(str(k)) + for name, devids in devices.items(): + data.append(("GPU " + ",".join(devids), name)) + + if has_rocm: + msg = " - invalid!" if not (ROCM_HOME and os.path.isdir(ROCM_HOME)) else "" + data.append(("ROCM_HOME", str(ROCM_HOME) + msg)) + else: + try: + from torch.utils.collect_env import get_nvidia_driver_version + from torch.utils.collect_env import run as _run + + data.append(("Driver version", get_nvidia_driver_version(_run))) + except Exception: + pass + msg = " - invalid!" if not (CUDA_HOME and os.path.isdir(CUDA_HOME)) else "" + data.append(("CUDA_HOME", str(CUDA_HOME) + msg)) + + cuda_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None) + if cuda_arch_list: + data.append(("TORCH_CUDA_ARCH_LIST", cuda_arch_list)) + data.append(("Pillow", PIL.__version__)) + + try: + data.append( + ( + "torchvision", + str(torchvision.__version__) + " @" + os.path.dirname(torchvision.__file__), + ) + ) + if has_cuda: + try: + torchvision_C = importlib.util.find_spec("torchvision._C").origin + msg = detect_compute_compatibility(CUDA_HOME, torchvision_C) + data.append(("torchvision arch flags", msg)) + except (ImportError, AttributeError): + data.append(("torchvision._C", "Not found")) + except AttributeError: + data.append(("torchvision", "unknown")) + + try: + import fvcore + + data.append(("fvcore", fvcore.__version__)) + except (ImportError, AttributeError): + pass + + try: + import iopath + + data.append(("iopath", iopath.__version__)) + except (ImportError, AttributeError): + pass + + try: + import cv2 + + data.append(("cv2", cv2.__version__)) + except (ImportError, AttributeError): + data.append(("cv2", "Not found")) + env_str = tabulate(data) + "\n" + env_str += collect_torch_env() + return env_str + + +def test_nccl_ops(): + num_gpu = torch.cuda.device_count() + if os.access("/tmp", os.W_OK): + import torch.multiprocessing as mp + + dist_url = "file:///tmp/nccl_tmp_file" + print("Testing NCCL connectivity ... this should not hang.") + mp.spawn(_test_nccl_worker, nprocs=num_gpu, args=(num_gpu, dist_url), daemon=False) + print("NCCL succeeded.") + + +def _test_nccl_worker(rank, num_gpu, dist_url): + import torch.distributed as dist + + dist.init_process_group(backend="NCCL", init_method=dist_url, rank=rank, world_size=num_gpu) + dist.barrier(device_ids=[rank]) + + +if __name__ == "__main__": + try: + from detectron2.utils.collect_env import collect_env_info as f + + print(f()) + except ImportError: + print(collect_env_info()) + + if torch.cuda.is_available(): + num_gpu = torch.cuda.device_count() + for k in range(num_gpu): + device = f"cuda:{k}" + try: + x = torch.tensor([1, 2.0], dtype=torch.float32) + x = x.to(device) + except Exception as e: + print(f"Unable to copy tensor to device={device}: {e}. " "Your CUDA environment is broken.") + if num_gpu > 1: + test_nccl_ops() diff --git a/detectron2/utils/colormap.py b/detectron2/utils/colormap.py new file mode 100644 index 0000000000000000000000000000000000000000..b749cc91d64fc851eece4b8837e504e863d3dbd0 --- /dev/null +++ b/detectron2/utils/colormap.py @@ -0,0 +1,159 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +An awesome colormap for really neat visualizations. +Copied from Detectron, and removed gray colors. +""" + +import random + +import numpy as np + +__all__ = ["colormap", "random_color", "random_colors"] + +# fmt: off +# RGB: +_COLORS = np.array( + [ + 0.000, 0.447, 0.741, + 0.850, 0.325, 0.098, + 0.929, 0.694, 0.125, + 0.494, 0.184, 0.556, + 0.466, 0.674, 0.188, + 0.301, 0.745, 0.933, + 0.635, 0.078, 0.184, + 0.300, 0.300, 0.300, + 0.600, 0.600, 0.600, + 1.000, 0.000, 0.000, + 1.000, 0.500, 0.000, + 0.749, 0.749, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 1.000, + 0.667, 0.000, 1.000, + 0.333, 0.333, 0.000, + 0.333, 0.667, 0.000, + 0.333, 1.000, 0.000, + 0.667, 0.333, 0.000, + 0.667, 0.667, 0.000, + 0.667, 1.000, 0.000, + 1.000, 0.333, 0.000, + 1.000, 0.667, 0.000, + 1.000, 1.000, 0.000, + 0.000, 0.333, 0.500, + 0.000, 0.667, 0.500, + 0.000, 1.000, 0.500, + 0.333, 0.000, 0.500, + 0.333, 0.333, 0.500, + 0.333, 0.667, 0.500, + 0.333, 1.000, 0.500, + 0.667, 0.000, 0.500, + 0.667, 0.333, 0.500, + 0.667, 0.667, 0.500, + 0.667, 1.000, 0.500, + 1.000, 0.000, 0.500, + 1.000, 0.333, 0.500, + 1.000, 0.667, 0.500, + 1.000, 1.000, 0.500, + 0.000, 0.333, 1.000, + 0.000, 0.667, 1.000, + 0.000, 1.000, 1.000, + 0.333, 0.000, 1.000, + 0.333, 0.333, 1.000, + 0.333, 0.667, 1.000, + 0.333, 1.000, 1.000, + 0.667, 0.000, 1.000, + 0.667, 0.333, 1.000, + 0.667, 0.667, 1.000, + 0.667, 1.000, 1.000, + 1.000, 0.000, 1.000, + 1.000, 0.333, 1.000, + 1.000, 0.667, 1.000, + 0.333, 0.000, 0.000, + 0.500, 0.000, 0.000, + 0.667, 0.000, 0.000, + 0.833, 0.000, 0.000, + 1.000, 0.000, 0.000, + 0.000, 0.167, 0.000, + 0.000, 0.333, 0.000, + 0.000, 0.500, 0.000, + 0.000, 0.667, 0.000, + 0.000, 0.833, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 0.167, + 0.000, 0.000, 0.333, + 0.000, 0.000, 0.500, + 0.000, 0.000, 0.667, + 0.000, 0.000, 0.833, + 0.000, 0.000, 1.000, + 0.000, 0.000, 0.000, + 0.143, 0.143, 0.143, + 0.857, 0.857, 0.857, + 1.000, 1.000, 1.000 + ] +).astype(np.float32).reshape(-1, 3) +# fmt: on + + +def colormap(rgb=False, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + + Returns: + ndarray: a float32 array of Nx3 colors, in range [0, 255] or [0, 1] + """ + assert maximum in [255, 1], maximum + c = _COLORS * maximum + if not rgb: + c = c[:, ::-1] + return c + + +def random_color(rgb=False, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + + Returns: + ndarray: a vector of 3 numbers + """ + idx = np.random.randint(0, len(_COLORS)) + ret = _COLORS[idx] * maximum + if not rgb: + ret = ret[::-1] + return ret + + +def random_colors(N, rgb=False, maximum=255): + """ + Args: + N (int): number of unique colors needed + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + + Returns: + ndarray: a list of random_color + """ + indices = random.sample(range(len(_COLORS)), N) + ret = [_COLORS[i] * maximum for i in indices] + if not rgb: + ret = [x[::-1] for x in ret] + return ret + + +if __name__ == "__main__": + import cv2 + + size = 100 + H, W = 10, 10 + canvas = np.random.rand(H * size, W * size, 3).astype("float32") + for h in range(H): + for w in range(W): + idx = h * W + w + if idx >= len(_COLORS): + break + canvas[h * size : (h + 1) * size, w * size : (w + 1) * size] = _COLORS[idx] + cv2.imshow("a", canvas) + cv2.waitKey(0) diff --git a/detectron2/utils/comm.py b/detectron2/utils/comm.py new file mode 100644 index 0000000000000000000000000000000000000000..e763643024dfa51f83dab194a1233d1a680e290b --- /dev/null +++ b/detectron2/utils/comm.py @@ -0,0 +1,200 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +This file contains primitives for multi-gpu communication. +This is useful when doing distributed training. +""" + +import functools + +import numpy as np +import torch +import torch.distributed as dist + +_LOCAL_PROCESS_GROUP = None +""" +A torch process group which only includes processes that on the same machine as the current process. +This variable is set when processes are spawned by `launch()` in "engine/launch.py". +""" + + +def get_world_size() -> int: + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank() -> int: + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + assert ( + _LOCAL_PROCESS_GROUP is not None + ), "Local process group is not created! Please use launch() to spawn processes!" + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, + i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def is_main_process() -> bool: + return get_rank() == 0 + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when + using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + if dist.get_backend() == dist.Backend.NCCL: + # This argument is needed to avoid warnings. + # It's valid only for NCCL backend. + dist.barrier(device_ids=[torch.cuda.current_device()]) + else: + dist.barrier() + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + else: + return dist.group.WORLD + + +def all_gather(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() # use CPU group by default, to reduce GPU RAM usage. + world_size = dist.get_world_size(group) + if world_size == 1: + return [data] + + output = [None for _ in range(world_size)] + dist.all_gather_object(output, data, group=group) + return output + + +def gather(data, dst=0, group=None): + """ + Run gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + dst (int): destination rank + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: on dst, a list of data gathered from each rank. Otherwise, + an empty list. + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + world_size = dist.get_world_size(group=group) + if world_size == 1: + return [data] + rank = dist.get_rank(group=group) + + if rank == dst: + output = [None for _ in range(world_size)] + dist.gather_object(data, output, dst=dst, group=group) + return output + else: + dist.gather_object(data, None, dst=dst, group=group) + return [] + + +def shared_random_seed(): + """ + Returns: + int: a random number that is the same across all workers. + If workers need a shared RNG, they can use this shared seed to + create one. + + All workers must call this function, otherwise it will deadlock. + """ + ints = np.random.randint(2**31) + all_ints = all_gather(ints) + return all_ints[0] + + +def reduce_dict(input_dict, average=True): + """ + Reduce the values in the dictionary from all processes so that process with rank + 0 has the reduced results. + + Args: + input_dict (dict): inputs to be reduced. All the values must be scalar CUDA Tensor. + average (bool): whether to do average or sum + + Returns: + a dict with the same keys as input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.reduce(values, dst=0) + if dist.get_rank() == 0 and average: + # only main process gets accumulated, so only divide by + # world_size in this case + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict diff --git a/detectron2/utils/develop.py b/detectron2/utils/develop.py new file mode 100644 index 0000000000000000000000000000000000000000..f968f1a016d97b1692a37b5c272855006215fe00 --- /dev/null +++ b/detectron2/utils/develop.py @@ -0,0 +1,60 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +"""Utilities for developers only. +These are not visible to users (not automatically imported). And should not +appeared in docs.""" + +# adapted from https://github.com/tensorpack/tensorpack/blob/master/tensorpack/utils/develop.py + + +def create_dummy_class(klass, dependency, message=""): + """ + When a dependency of a class is not available, create a dummy class which throws ImportError + when used. + + Args: + klass (str): name of the class. + dependency (str): name of the dependency. + message: extra message to print + Returns: + class: a class object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass) + if message: + err = err + " " + message + + class _DummyMetaClass(type): + # throw error on class attribute access + def __getattr__(_, __): + raise ImportError(err) + + class _Dummy(object, metaclass=_DummyMetaClass): + # throw error on constructor + def __init__(self, *args, **kwargs): + raise ImportError(err) + + return _Dummy + + +def create_dummy_func(func, dependency, message=""): + """ + When a dependency of a function is not available, create a dummy function which throws + ImportError when used. + + Args: + func (str): name of the function. + dependency (str or list[str]): name(s) of the dependency. + message: extra message to print + Returns: + function: a function object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func) + if message: + err = err + " " + message + + if isinstance(dependency, (list, tuple)): + dependency = ",".join(dependency) + + def _dummy(*args, **kwargs): + raise ImportError(err) + + return _dummy diff --git a/detectron2/utils/env.py b/detectron2/utils/env.py new file mode 100644 index 0000000000000000000000000000000000000000..9b02bb9a7cc231a2eaf865eabab6614991b0444a --- /dev/null +++ b/detectron2/utils/env.py @@ -0,0 +1,166 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import importlib +import importlib.util +import logging +import os +import random +import sys +from datetime import datetime + +import numpy as np +import torch + +__all__ = ["seed_all_rng"] + + +TORCH_VERSION = tuple(int(x) for x in torch.__version__.split(".")[:2]) +""" +PyTorch version as a tuple of 2 ints. Useful for comparison. +""" + + +DOC_BUILDING = os.getenv("_DOC_BUILDING", False) # set in docs/conf.py +""" +Whether we're building documentation. +""" + + +def seed_all_rng(seed=None): + """ + Set the random seed for the RNG in torch, numpy and python. + + Args: + seed (int): if None, will use a strong random seed. + """ + if seed is None: + seed = os.getpid() + int(datetime.now().strftime("%S%f")) + int.from_bytes(os.urandom(2), "big") + logger = logging.getLogger(__name__) + logger.info("Using a generated random seed {}".format(seed)) + np.random.seed(seed) + torch.manual_seed(seed) + random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + + +# from https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +def _import_file(module_name, file_path, make_importable=False): + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if make_importable: + sys.modules[module_name] = module + return module + + +def _configure_libraries(): + """ + Configurations for some libraries. + """ + # An environment option to disable `import cv2` globally, + # in case it leads to negative performance impact + disable_cv2 = int(os.environ.get("DETECTRON2_DISABLE_CV2", False)) + if disable_cv2: + sys.modules["cv2"] = None + else: + # Disable opencl in opencv since its interaction with cuda often has negative effects + # This envvar is supported after OpenCV 3.4.0 + os.environ["OPENCV_OPENCL_RUNTIME"] = "disabled" + try: + import cv2 + + if int(cv2.__version__.split(".")[0]) >= 3: + cv2.ocl.setUseOpenCL(False) + except ModuleNotFoundError: + # Other types of ImportError, if happened, should not be ignored. + # Because a failed opencv import could mess up address space + # https://github.com/skvark/opencv-python/issues/381 + pass + + def get_version(module, digit=2): + return tuple(map(int, module.__version__.split(".")[:digit])) + + # fmt: off + assert get_version(torch) >= (1, 4), "Requires torch>=1.4" + import fvcore + assert get_version(fvcore, 3) >= (0, 1, 2), "Requires fvcore>=0.1.2" + import yaml + assert get_version(yaml) >= (5, 1), "Requires pyyaml>=5.1" + # fmt: on + + +_ENV_SETUP_DONE = False + + +def setup_environment(): + """Perform environment setup work. The default setup is a no-op, but this + function allows the user to specify a Python source file or a module in + the $DETECTRON2_ENV_MODULE environment variable, that performs + custom setup work that may be necessary to their computing environment. + """ + global _ENV_SETUP_DONE + if _ENV_SETUP_DONE: + return + _ENV_SETUP_DONE = True + + _configure_libraries() + + custom_module_path = os.environ.get("DETECTRON2_ENV_MODULE") + + if custom_module_path: + setup_custom_environment(custom_module_path) + else: + # The default setup is a no-op + pass + + +def setup_custom_environment(custom_module): + """ + Load custom environment setup by importing a Python source file or a + module, and run the setup function. + """ + if custom_module.endswith(".py"): + module = _import_file("detectron2.utils.env.custom_module", custom_module) + else: + module = importlib.import_module(custom_module) + assert hasattr(module, "setup_environment") and callable(module.setup_environment), ( + "Custom environment module defined in {} does not have the " "required callable attribute 'setup_environment'." + ).format(custom_module) + module.setup_environment() + + +def fixup_module_metadata(module_name, namespace, keys=None): + """ + Fix the __qualname__ of module members to be their exported api name, so + when they are referenced in docs, sphinx can find them. Reference: + https://github.com/python-trio/trio/blob/6754c74eacfad9cc5c92d5c24727a2f3b620624e/trio/_util.py#L216-L241 + """ + if not DOC_BUILDING: + return + seen_ids = set() + + def fix_one(qualname, name, obj): + # avoid infinite recursion (relevant when using + # typing.Generic, for example) + if id(obj) in seen_ids: + return + seen_ids.add(id(obj)) + + mod = getattr(obj, "__module__", None) + if mod is not None and (mod.startswith(module_name) or mod.startswith("fvcore.")): + obj.__module__ = module_name + # Modules, unlike everything else in Python, put fully-qualitied + # names into their __name__ attribute. We check for "." to avoid + # rewriting these. + if hasattr(obj, "__name__") and "." not in obj.__name__: + obj.__name__ = name + obj.__qualname__ = qualname + if isinstance(obj, type): + for attr_name, attr_value in obj.__dict__.items(): + fix_one(objname + "." + attr_name, attr_name, attr_value) + + if keys is None: + keys = namespace.keys() + for objname in keys: + if not objname.startswith("_"): + obj = namespace[objname] + fix_one(objname, objname, obj) diff --git a/detectron2/utils/events.py b/detectron2/utils/events.py new file mode 100644 index 0000000000000000000000000000000000000000..72f2f0fdc7b8bbb577f0bf5f79cd1c7e9fa77230 --- /dev/null +++ b/detectron2/utils/events.py @@ -0,0 +1,483 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import datetime +import json +import logging +import os +import time +from collections import defaultdict +from contextlib import contextmanager +from typing import Optional + +import torch +from fvcore.common.history_buffer import HistoryBuffer + +from detectron2.utils.file_io import PathManager + +__all__ = [ + "get_event_storage", + "JSONWriter", + "TensorboardXWriter", + "CommonMetricPrinter", + "EventStorage", +] + +_CURRENT_STORAGE_STACK = [] + + +def get_event_storage(): + """ + Returns: + The :class:`EventStorage` object that's currently being used. + Throws an error if no :class:`EventStorage` is currently enabled. + """ + assert len( + _CURRENT_STORAGE_STACK + ), "get_event_storage() has to be called inside a 'with EventStorage(...)' context!" + return _CURRENT_STORAGE_STACK[-1] + + +class EventWriter: + """ + Base class for writers that obtain events from :class:`EventStorage` and process them. + """ + + def write(self): + raise NotImplementedError + + def close(self): + pass + + +class JSONWriter(EventWriter): + """ + Write scalars to a json file. + + It saves scalars as one json per line (instead of a big json) for easy parsing. + + Examples parsing such a json file: + :: + $ cat metrics.json | jq -s '.[0:2]' + [ + { + "data_time": 0.008433341979980469, + "iteration": 19, + "loss": 1.9228371381759644, + "loss_box_reg": 0.050025828182697296, + "loss_classifier": 0.5316952466964722, + "loss_mask": 0.7236229181289673, + "loss_rpn_box": 0.0856662318110466, + "loss_rpn_cls": 0.48198649287223816, + "lr": 0.007173333333333333, + "time": 0.25401854515075684 + }, + { + "data_time": 0.007216215133666992, + "iteration": 39, + "loss": 1.282649278640747, + "loss_box_reg": 0.06222952902317047, + "loss_classifier": 0.30682939291000366, + "loss_mask": 0.6970193982124329, + "loss_rpn_box": 0.038663312792778015, + "loss_rpn_cls": 0.1471673548221588, + "lr": 0.007706666666666667, + "time": 0.2490077018737793 + } + ] + + $ cat metrics.json | jq '.loss_mask' + 0.7126231789588928 + 0.689423680305481 + 0.6776131987571716 + ... + + """ + + def __init__(self, json_file, window_size=20): + """ + Args: + json_file (str): path to the json file. New data will be appended if the file exists. + window_size (int): the window size of median smoothing for the scalars whose + `smoothing_hint` are True. + """ + self._file_handle = PathManager.open(json_file, "a") + self._window_size = window_size + self._last_write = -1 + + def write(self): + storage = get_event_storage() + to_save = defaultdict(dict) + + for k, (v, iter) in storage.latest_with_smoothing_hint(self._window_size).items(): + # keep scalars that have not been written + if iter <= self._last_write: + continue + to_save[iter][k] = v + if len(to_save): + all_iters = sorted(to_save.keys()) + self._last_write = max(all_iters) + + for itr, scalars_per_iter in to_save.items(): + scalars_per_iter["iteration"] = itr + self._file_handle.write(json.dumps(scalars_per_iter, sort_keys=True) + "\n") + self._file_handle.flush() + try: + os.fsync(self._file_handle.fileno()) + except AttributeError: + pass + + def close(self): + self._file_handle.close() + + +class TensorboardXWriter(EventWriter): + """ + Write all scalars to a tensorboard file. + """ + + def __init__(self, log_dir: str, window_size: int = 20, **kwargs): + """ + Args: + log_dir (str): the directory to save the output events + window_size (int): the scalars will be median-smoothed by this window size + + kwargs: other arguments passed to `torch.utils.tensorboard.SummaryWriter(...)` + """ + self._window_size = window_size + from torch.utils.tensorboard import SummaryWriter + + self._writer = SummaryWriter(log_dir, **kwargs) + self._last_write = -1 + + def write(self): + storage = get_event_storage() + new_last_write = self._last_write + for k, (v, iter) in storage.latest_with_smoothing_hint(self._window_size).items(): + if iter > self._last_write: + self._writer.add_scalar(k, v, iter) + new_last_write = max(new_last_write, iter) + self._last_write = new_last_write + + # storage.put_{image,histogram} is only meant to be used by + # tensorboard writer. So we access its internal fields directly from here. + if len(storage._vis_data) >= 1: + for img_name, img, step_num in storage._vis_data: + self._writer.add_image(img_name, img, step_num) + # Storage stores all image data and rely on this writer to clear them. + # As a result it assumes only one writer will use its image data. + # An alternative design is to let storage store limited recent + # data (e.g. only the most recent image) that all writers can access. + # In that case a writer may not see all image data if its period is long. + storage.clear_images() + + if len(storage._histograms) >= 1: + for params in storage._histograms: + self._writer.add_histogram_raw(**params) + storage.clear_histograms() + + def close(self): + if hasattr(self, "_writer"): # doesn't exist when the code fails at import + self._writer.close() + + +class CommonMetricPrinter(EventWriter): + """ + Print **common** metrics to the terminal, including + iteration time, ETA, memory, all losses, and the learning rate. + It also applies smoothing using a window of 20 elements. + + It's meant to print common metrics in common ways. + To print something in more customized ways, please implement a similar printer by yourself. + """ + + def __init__(self, max_iter: Optional[int] = None, window_size: int = 20): + """ + Args: + max_iter: the maximum number of iterations to train. + Used to compute ETA. If not given, ETA will not be printed. + window_size (int): the losses will be median-smoothed by this window size + """ + self.logger = logging.getLogger(__name__) + self._max_iter = max_iter + self._window_size = window_size + self._last_write = None # (step, time) of last call to write(). Used to compute ETA + + def _get_eta(self, storage) -> Optional[str]: + if self._max_iter is None: + return "" + iteration = storage.iter + try: + eta_seconds = storage.history("time").median(1000) * (self._max_iter - iteration - 1) + storage.put_scalar("eta_seconds", eta_seconds, smoothing_hint=False) + return str(datetime.timedelta(seconds=int(eta_seconds))) + except KeyError: + # estimate eta on our own - more noisy + eta_string = None + if self._last_write is not None: + estimate_iter_time = (time.perf_counter() - self._last_write[1]) / (iteration - self._last_write[0]) + eta_seconds = estimate_iter_time * (self._max_iter - iteration - 1) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + self._last_write = (iteration, time.perf_counter()) + return eta_string + + def write(self): + storage = get_event_storage() + iteration = storage.iter + if iteration == self._max_iter: + # This hook only reports training progress (loss, ETA, etc) but not other data, + # therefore do not write anything after training succeeds, even if this method + # is called. + return + + try: + data_time = storage.history("data_time").avg(20) + except KeyError: + # they may not exist in the first few iterations (due to warmup) + # or when SimpleTrainer is not used + data_time = None + try: + iter_time = storage.history("time").global_avg() + except KeyError: + iter_time = None + try: + lr = "{:.5g}".format(storage.history("lr").latest()) + except KeyError: + lr = "N/A" + + eta_string = self._get_eta(storage) + + if torch.cuda.is_available(): + max_mem_mb = torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 + else: + max_mem_mb = None + + # NOTE: max_mem is parsed by grep in "dev/parse_results.sh" + self.logger.info( + " {eta}iter: {iter} {losses} {time}{data_time}lr: {lr} {memory}".format( + eta=f"eta: {eta_string} " if eta_string else "", + iter=iteration, + losses=" ".join( + [ + "{}: {:.4g}".format(k, v.median(self._window_size)) + for k, v in storage.histories().items() + if "loss" in k + ] + ), + time="time: {:.4f} ".format(iter_time) if iter_time is not None else "", + data_time="data_time: {:.4f} ".format(data_time) if data_time is not None else "", + lr=lr, + memory="max_mem: {:.0f}M".format(max_mem_mb) if max_mem_mb is not None else "", + ) + ) + + +class EventStorage: + """ + The user-facing class that provides metric storage functionalities. + + In the future we may add support for storing / logging other types of data if needed. + """ + + def __init__(self, start_iter=0): + """ + Args: + start_iter (int): the iteration number to start with + """ + self._history = defaultdict(HistoryBuffer) + self._smoothing_hints = {} + self._latest_scalars = {} + self._iter = start_iter + self._current_prefix = "" + self._vis_data = [] + self._histograms = [] + + def put_image(self, img_name, img_tensor): + """ + Add an `img_tensor` associated with `img_name`, to be shown on + tensorboard. + + Args: + img_name (str): The name of the image to put into tensorboard. + img_tensor (torch.Tensor or numpy.array): An `uint8` or `float` + Tensor of shape `[channel, height, width]` where `channel` is + 3. The image format should be RGB. The elements in img_tensor + can either have values in [0, 1] (float32) or [0, 255] (uint8). + The `img_tensor` will be visualized in tensorboard. + """ + self._vis_data.append((img_name, img_tensor, self._iter)) + + def put_scalar(self, name, value, smoothing_hint=True): + """ + Add a scalar `value` to the `HistoryBuffer` associated with `name`. + + Args: + smoothing_hint (bool): a 'hint' on whether this scalar is noisy and should be + smoothed when logged. The hint will be accessible through + :meth:`EventStorage.smoothing_hints`. A writer may ignore the hint + and apply custom smoothing rule. + + It defaults to True because most scalars we save need to be smoothed to + provide any useful signal. + """ + name = self._current_prefix + name + history = self._history[name] + value = float(value) + history.update(value, self._iter) + self._latest_scalars[name] = (value, self._iter) + + existing_hint = self._smoothing_hints.get(name) + if existing_hint is not None: + assert existing_hint == smoothing_hint, "Scalar {} was put with a different smoothing_hint!".format(name) + else: + self._smoothing_hints[name] = smoothing_hint + + def put_scalars(self, *, smoothing_hint=True, **kwargs): + """ + Put multiple scalars from keyword arguments. + + Examples: + + storage.put_scalars(loss=my_loss, accuracy=my_accuracy, smoothing_hint=True) + """ + for k, v in kwargs.items(): + self.put_scalar(k, v, smoothing_hint=smoothing_hint) + + def put_histogram(self, hist_name, hist_tensor, bins=1000): + """ + Create a histogram from a tensor. + + Args: + hist_name (str): The name of the histogram to put into tensorboard. + hist_tensor (torch.Tensor): A Tensor of arbitrary shape to be converted + into a histogram. + bins (int): Number of histogram bins. + """ + ht_min, ht_max = hist_tensor.min().item(), hist_tensor.max().item() + + # Create a histogram with PyTorch + hist_counts = torch.histc(hist_tensor, bins=bins) + hist_edges = torch.linspace(start=ht_min, end=ht_max, steps=bins + 1, dtype=torch.float32) + + # Parameter for the add_histogram_raw function of SummaryWriter + hist_params = dict( + tag=hist_name, + min=ht_min, + max=ht_max, + num=len(hist_tensor), + sum=float(hist_tensor.sum()), + sum_squares=float(torch.sum(hist_tensor**2)), + bucket_limits=hist_edges[1:].tolist(), + bucket_counts=hist_counts.tolist(), + global_step=self._iter, + ) + self._histograms.append(hist_params) + + def history(self, name): + """ + Returns: + HistoryBuffer: the scalar history for name + """ + ret = self._history.get(name, None) + if ret is None: + raise KeyError("No history metric available for {}!".format(name)) + return ret + + def histories(self): + """ + Returns: + dict[name -> HistoryBuffer]: the HistoryBuffer for all scalars + """ + return self._history + + def latest(self): + """ + Returns: + dict[str -> (float, int)]: mapping from the name of each scalar to the most + recent value and the iteration number its added. + """ + return self._latest_scalars + + def latest_with_smoothing_hint(self, window_size=20): + """ + Similar to :meth:`latest`, but the returned values + are either the un-smoothed original latest value, + or a median of the given window_size, + depend on whether the smoothing_hint is True. + + This provides a default behavior that other writers can use. + """ + result = {} + for k, (v, itr) in self._latest_scalars.items(): + result[k] = ( + self._history[k].median(window_size) if self._smoothing_hints[k] else v, + itr, + ) + return result + + def smoothing_hints(self): + """ + Returns: + dict[name -> bool]: the user-provided hint on whether the scalar + is noisy and needs smoothing. + """ + return self._smoothing_hints + + def step(self): + """ + User should either: (1) Call this function to increment storage.iter when needed. Or + (2) Set `storage.iter` to the correct iteration number before each iteration. + + The storage will then be able to associate the new data with an iteration number. + """ + self._iter += 1 + + @property + def iter(self): + """ + Returns: + int: The current iteration number. When used together with a trainer, + this is ensured to be the same as trainer.iter. + """ + return self._iter + + @iter.setter + def iter(self, val): + self._iter = int(val) + + @property + def iteration(self): + # for backward compatibility + return self._iter + + def __enter__(self): + _CURRENT_STORAGE_STACK.append(self) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert _CURRENT_STORAGE_STACK[-1] == self + _CURRENT_STORAGE_STACK.pop() + + @contextmanager + def name_scope(self, name): + """ + Yields: + A context within which all the events added to this storage + will be prefixed by the name scope. + """ + old_prefix = self._current_prefix + self._current_prefix = name.rstrip("/") + "/" + yield + self._current_prefix = old_prefix + + def clear_images(self): + """ + Delete all the stored images for visualization. This should be called + after images are written to tensorboard. + """ + self._vis_data = [] + + def clear_histograms(self): + """ + Delete all the stored histograms for visualization. + This should be called after histograms are written to tensorboard. + """ + self._histograms = [] diff --git a/detectron2/utils/file_io.py b/detectron2/utils/file_io.py new file mode 100644 index 0000000000000000000000000000000000000000..46ee4ec31d04eee77976ff3edbbf84762a3409ed --- /dev/null +++ b/detectron2/utils/file_io.py @@ -0,0 +1,37 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler +from iopath.common.file_io import PathManager as PathManagerBase + +__all__ = ["PathManager", "PathHandler"] + + +PathManager = PathManagerBase() +""" +This is a detectron2 project-specific PathManager. +We try to stay away from global PathManager in fvcore as it +introduces potential conflicts among other libraries. +""" + + +class Detectron2Handler(PathHandler): + """ + Resolve anything that's hosted under detectron2's namespace. + """ + + PREFIX = "detectron2://" + S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" + + def _get_supported_prefixes(self): + return [self.PREFIX] + + def _get_local_path(self, path, **kwargs): + name = path[len(self.PREFIX) :] + return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name, **kwargs) + + def _open(self, path, mode="r", **kwargs): + return PathManager.open(self._get_local_path(path), mode, **kwargs) + + +PathManager.register_handler(HTTPURLHandler()) +PathManager.register_handler(OneDrivePathHandler()) +PathManager.register_handler(Detectron2Handler()) diff --git a/detectron2/utils/logger.py b/detectron2/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..c7d175fcc342ae1c84de08ee3c6e859a76b746e3 --- /dev/null +++ b/detectron2/utils/logger.py @@ -0,0 +1,234 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import atexit +import functools +import logging +import os +import sys +import time +from collections import Counter + +import torch +from tabulate import tabulate +from termcolor import colored + +from detectron2.utils.file_io import PathManager + +__all__ = ["setup_logger", "log_first_n", "log_every_n", "log_every_n_seconds"] + + +class _ColorfulFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + self._root_name = kwargs.pop("root_name") + "." + self._abbrev_name = kwargs.pop("abbrev_name", "") + if len(self._abbrev_name): + self._abbrev_name = self._abbrev_name + "." + super(_ColorfulFormatter, self).__init__(*args, **kwargs) + + def formatMessage(self, record): + record.name = record.name.replace(self._root_name, self._abbrev_name) + log = super(_ColorfulFormatter, self).formatMessage(record) + if record.levelno == logging.WARNING: + prefix = colored("WARNING", "red", attrs=["blink"]) + elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: + prefix = colored("ERROR", "red", attrs=["blink", "underline"]) + else: + return log + return prefix + " " + log + + +@functools.lru_cache() # so that calling setup_logger multiple times won't add many handlers +def setup_logger(output=None, distributed_rank=0, *, color=True, name="detectron2", abbrev_name=None): + """ + Initialize the detectron2 logger and set its verbosity level to "DEBUG". + + Args: + output (str): a file name or a directory to save log. If None, will not save log file. + If ends with ".txt" or ".log", assumed to be a file name. + Otherwise, logs will be saved to `output/log.txt`. + name (str): the root module name of this logger + abbrev_name (str): an abbreviation of the module, to avoid long names in logs. + Set to "" to not log the root module in logs. + By default, will abbreviate "detectron2" to "d2" and leave other + modules unchanged. + + Returns: + logging.Logger: a logger + """ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + if abbrev_name is None: + abbrev_name = "d2" if name == "detectron2" else name + + plain_formatter = logging.Formatter("[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%m/%d %H:%M:%S") + # stdout logging: master only + if distributed_rank == 0: + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(logging.DEBUG) + if color: + formatter = _ColorfulFormatter( + colored("[%(asctime)s %(name)s]: ", "green") + "%(message)s", + datefmt="%m/%d %H:%M:%S", + root_name=name, + abbrev_name=str(abbrev_name), + ) + else: + formatter = plain_formatter + ch.setFormatter(formatter) + logger.addHandler(ch) + + # file logging: all workers + if output is not None: + if output.endswith(".txt") or output.endswith(".log"): + filename = output + else: + filename = os.path.join(output, "log.txt") + if distributed_rank > 0: + filename = filename + ".rank{}".format(distributed_rank) + PathManager.mkdirs(os.path.dirname(filename)) + + fh = logging.StreamHandler(_cached_log_stream(filename)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(plain_formatter) + logger.addHandler(fh) + + return logger + + +# cache the opened file object, so that different calls to `setup_logger` +# with the same file name can safely write to the same file. +@functools.lru_cache(maxsize=None) +def _cached_log_stream(filename): + # use 1K buffer if writing to cloud storage + io = PathManager.open(filename, "a", buffering=1024 if "://" in filename else -1) + atexit.register(io.close) + return io + + +""" +Below are some other convenient logging methods. +They are mainly adopted from +https://github.com/abseil/abseil-py/blob/master/absl/logging/__init__.py +""" + + +def _find_caller(): + """ + Returns: + str: module name of the caller + tuple: a hashable key to be used to identify different callers + """ + frame = sys._getframe(2) + while frame: + code = frame.f_code + if os.path.join("utils", "logger.") not in code.co_filename: + mod_name = frame.f_globals["__name__"] + if mod_name == "__main__": + mod_name = "detectron2" + return mod_name, (code.co_filename, frame.f_lineno, code.co_name) + frame = frame.f_back + + +_LOG_COUNTER = Counter() +_LOG_TIMER = {} + + +def log_first_n(lvl, msg, n=1, *, name=None, key="caller"): + """ + Log only for the first n times. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + key (str or tuple[str]): the string(s) can be one of "caller" or + "message", which defines how to identify duplicated logs. + For example, if called with `n=1, key="caller"`, this function + will only log the first call from the same caller, regardless of + the message content. + If called with `n=1, key="message"`, this function will log the + same content only once, even if they are called from different places. + If called with `n=1, key=("caller", "message")`, this function + will not log only if the same caller has logged the same message before. + """ + if isinstance(key, str): + key = (key,) + assert len(key) > 0 + + caller_module, caller_key = _find_caller() + hash_key = () + if "caller" in key: + hash_key = hash_key + caller_key + if "message" in key: + hash_key = hash_key + (msg,) + + _LOG_COUNTER[hash_key] += 1 + if _LOG_COUNTER[hash_key] <= n: + logging.getLogger(name or caller_module).log(lvl, msg) + + +def log_every_n(lvl, msg, n=1, *, name=None): + """ + Log once per n times. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + """ + caller_module, key = _find_caller() + _LOG_COUNTER[key] += 1 + if n == 1 or _LOG_COUNTER[key] % n == 1: + logging.getLogger(name or caller_module).log(lvl, msg) + + +def log_every_n_seconds(lvl, msg, n=1, *, name=None): + """ + Log no more than once per n seconds. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + """ + caller_module, key = _find_caller() + last_logged = _LOG_TIMER.get(key, None) + current_time = time.time() + if last_logged is None or current_time - last_logged >= n: + logging.getLogger(name or caller_module).log(lvl, msg) + _LOG_TIMER[key] = current_time + + +def create_small_table(small_dict): + """ + Create a small table using the keys of small_dict as headers. This is only + suitable for small dictionaries. + + Args: + small_dict (dict): a result dictionary of only a few items. + + Returns: + str: the table as a string. + """ + keys, values = tuple(zip(*small_dict.items())) + table = tabulate( + [values], + headers=keys, + tablefmt="pipe", + floatfmt=".3f", + stralign="center", + numalign="center", + ) + return table + + +def _log_api_usage(identifier: str): + """ + Internal function used to log the usage of different detectron2 components + inside facebook's infra. + """ + torch._C._log_api_usage_once("detectron2." + identifier) diff --git a/detectron2/utils/memory.py b/detectron2/utils/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..4a22fe1b5c65e5dd2ec944ab6f6c96d67b04a37e --- /dev/null +++ b/detectron2/utils/memory.py @@ -0,0 +1,85 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from contextlib import contextmanager +from functools import wraps + +import torch + +__all__ = ["retry_if_cuda_oom"] + + +@contextmanager +def _ignore_torch_cuda_oom(): + """ + A context which ignores CUDA OOM exception from pytorch. + """ + try: + yield + except RuntimeError as e: + # NOTE: the string may change? + if "CUDA out of memory. " in str(e): + pass + else: + raise + + +def retry_if_cuda_oom(func): + """ + Makes a function retry itself after encountering + pytorch's CUDA OOM error. + It will first retry after calling `torch.cuda.empty_cache()`. + + If that still fails, it will then retry by trying to convert inputs to CPUs. + In this case, it expects the function to dispatch to CPU implementation. + The return values may become CPU tensors as well and it's user's + responsibility to convert it back to CUDA tensor if needed. + + Args: + func: a stateless callable that takes tensor-like objects as arguments + + Returns: + a callable which retries `func` if OOM is encountered. + + Examples: + :: + output = retry_if_cuda_oom(some_torch_function)(input1, input2) + # output may be on CPU even if inputs are on GPU + + Note: + 1. When converting inputs to CPU, it will only look at each argument and check + if it has `.device` and `.to` for conversion. Nested structures of tensors + are not supported. + + 2. Since the function might be called more than once, it has to be + stateless. + """ + + def maybe_to_cpu(x): + try: + like_gpu_tensor = x.device.type == "cuda" and hasattr(x, "to") + except AttributeError: + like_gpu_tensor = False + if like_gpu_tensor: + return x.to(device="cpu") + else: + return x + + @wraps(func) + def wrapped(*args, **kwargs): + with _ignore_torch_cuda_oom(): + return func(*args, **kwargs) + + # Clear cache and retry + torch.cuda.empty_cache() + with _ignore_torch_cuda_oom(): + return func(*args, **kwargs) + + # Try on CPU. This slows down the code significantly, therefore print a notice. + logger = logging.getLogger(__name__) + logger.info("Attempting to copy inputs of {} to CPU due to CUDA OOM".format(str(func))) + new_args = (maybe_to_cpu(x) for x in args) + new_kwargs = {k: maybe_to_cpu(v) for k, v in kwargs.items()} + return func(*new_args, **new_kwargs) + + return wrapped diff --git a/detectron2/utils/registry.py b/detectron2/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..9cb88d8b1f70c1371ce709787ae773e63fdaea04 --- /dev/null +++ b/detectron2/utils/registry.py @@ -0,0 +1,61 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import pydoc +from typing import Any + +from fvcore.common.registry import Registry # for backward compatibility. + +""" +``Registry`` and `locate` provide ways to map a string (typically found +in config files) to callable objects. +""" + +__all__ = ["Registry", "locate"] + + +def _convert_target_to_string(t: Any) -> str: + """ + Inverse of ``locate()``. + + Args: + t: any object with ``__module__`` and ``__qualname__`` + """ + module, qualname = t.__module__, t.__qualname__ + + # Compress the path to this object, e.g. ``module.submodule._impl.class`` + # may become ``module.submodule.class``, if the later also resolves to the same + # object. This simplifies the string, and also is less affected by moving the + # class implementation. + module_parts = module.split(".") + for k in range(1, len(module_parts)): + prefix = ".".join(module_parts[:k]) + candidate = f"{prefix}.{qualname}" + try: + if locate(candidate) is t: + return candidate + except ImportError: + pass + return f"{module}.{qualname}" + + +def locate(name: str) -> Any: + """ + Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``, + such as "module.submodule.class_name". + + Raise Exception if it cannot be found. + """ + obj = pydoc.locate(name) + + # Some cases (e.g. torch.optim.sgd.SGD) not handled correctly + # by pydoc.locate. Try a private function from hydra. + if obj is None: + try: + # from hydra.utils import get_method - will print many errors + from hydra.utils import _locate + except ImportError as e: + raise ImportError(f"Cannot dynamically locate object {name}!") from e + else: + obj = _locate(name) # it raises if fails + + return obj diff --git a/detectron2/utils/serialize.py b/detectron2/utils/serialize.py new file mode 100644 index 0000000000000000000000000000000000000000..0b38862804b70cf1159a9bc93acdef73c184d883 --- /dev/null +++ b/detectron2/utils/serialize.py @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import cloudpickle + + +class PicklableWrapper(object): + """ + Wrap an object to make it more picklable, note that it uses + heavy weight serialization libraries that are slower than pickle. + It's best to use it only on closures (which are usually not picklable). + + This is a simplified version of + https://github.com/joblib/joblib/blob/master/joblib/externals/loky/cloudpickle_wrapper.py + """ + + def __init__(self, obj): + while isinstance(obj, PicklableWrapper): + # Wrapping an object twice is no-op + obj = obj._obj + self._obj = obj + + def __reduce__(self): + s = cloudpickle.dumps(self._obj) + return cloudpickle.loads, (s,) + + def __call__(self, *args, **kwargs): + return self._obj(*args, **kwargs) + + def __getattr__(self, attr): + # Ensure that the wrapped object can be used seamlessly as the previous object. + if attr not in ["_obj"]: + return getattr(self._obj, attr) + return getattr(self, attr) diff --git a/detectron2/utils/testing.py b/detectron2/utils/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..2ee73f692e23e0a6752f3a9278192c89790617fa --- /dev/null +++ b/detectron2/utils/testing.py @@ -0,0 +1,145 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import io +import os +import tempfile + +import numpy as np +import torch + +from detectron2 import model_zoo +from detectron2.config import CfgNode, LazyConfig, instantiate +from detectron2.data import DatasetCatalog +from detectron2.data.detection_utils import read_image +from detectron2.modeling import build_model +from detectron2.structures import Boxes, Instances, ROIMasks +from detectron2.utils.file_io import PathManager + +""" +Internal utilities for tests. Don't use except for writing tests. +""" + + +def get_model_no_weights(config_path): + """ + Like model_zoo.get, but do not load any weights (even pretrained) + """ + cfg = model_zoo.get_config(config_path) + if isinstance(cfg, CfgNode): + if not torch.cuda.is_available(): + cfg.MODEL.DEVICE = "cpu" + return build_model(cfg) + else: + return instantiate(cfg.model) + + +def random_boxes(num_boxes, max_coord=100, device="cpu"): + """ + Create a random Nx4 boxes tensor, with coordinates < max_coord. + """ + boxes = torch.rand(num_boxes, 4, device=device) * (max_coord * 0.5) + boxes.clamp_(min=1.0) # tiny boxes cause numerical instability in box regression + # Note: the implementation of this function in torchvision is: + # boxes[:, 2:] += torch.rand(N, 2) * 100 + # but it does not guarantee non-negative widths/heights constraints: + # boxes[:, 2] >= boxes[:, 0] and boxes[:, 3] >= boxes[:, 1]: + boxes[:, 2:] += boxes[:, :2] + return boxes + + +def get_sample_coco_image(tensor=True): + """ + Args: + tensor (bool): if True, returns 3xHxW tensor. + else, returns a HxWx3 numpy array. + + Returns: + an image, in BGR color. + """ + try: + file_name = DatasetCatalog.get("coco_2017_val_100")[0]["file_name"] + if not PathManager.exists(file_name): + raise FileNotFoundError() + except IOError: + # for public CI to run + file_name = PathManager.get_local_path("http://images.cocodataset.org/train2017/000000000009.jpg") + ret = read_image(file_name, format="BGR") + if tensor: + ret = torch.from_numpy(np.ascontiguousarray(ret.transpose(2, 0, 1))) + return ret + + +def convert_scripted_instances(instances): + """ + Convert a scripted Instances object to a regular :class:`Instances` object + """ + assert hasattr(instances, "image_size"), f"Expect an Instances object, but got {type(instances)}!" + ret = Instances(instances.image_size) + for name in instances._field_names: + val = getattr(instances, "_" + name, None) + if val is not None: + ret.set(name, val) + return ret + + +def assert_instances_allclose(input, other, *, rtol=1e-5, msg="", size_as_tensor=False): + """ + Args: + input, other (Instances): + size_as_tensor: compare image_size of the Instances as tensors (instead of tuples). + Useful for comparing outputs of tracing. + """ + if not isinstance(input, Instances): + input = convert_scripted_instances(input) + if not isinstance(other, Instances): + other = convert_scripted_instances(other) + + if not msg: + msg = "Two Instances are different! " + else: + msg = msg.rstrip() + " " + + size_error_msg = msg + f"image_size is {input.image_size} vs. {other.image_size}!" + if size_as_tensor: + assert torch.equal(torch.tensor(input.image_size), torch.tensor(other.image_size)), size_error_msg + else: + assert input.image_size == other.image_size, size_error_msg + fields = sorted(input.get_fields().keys()) + fields_other = sorted(other.get_fields().keys()) + assert fields == fields_other, msg + f"Fields are {fields} vs {fields_other}!" + + for f in fields: + val1, val2 = input.get(f), other.get(f) + if isinstance(val1, (Boxes, ROIMasks)): + # boxes in the range of O(100) and can have a larger tolerance + assert torch.allclose(val1.tensor, val2.tensor, atol=100 * rtol), msg + f"Field {f} differs too much!" + elif isinstance(val1, torch.Tensor): + if val1.dtype.is_floating_point: + mag = torch.abs(val1).max().cpu().item() + assert torch.allclose(val1, val2, atol=mag * rtol), msg + f"Field {f} differs too much!" + else: + assert torch.equal(val1, val2), msg + f"Field {f} is different!" + else: + raise ValueError(f"Don't know how to compare type {type(val1)}") + + +def reload_script_model(module): + """ + Save a jit module and load it back. + Similar to the `getExportImportCopy` function in torch/testing/ + """ + buffer = io.BytesIO() + torch.jit.save(module, buffer) + buffer.seek(0) + return torch.jit.load(buffer) + + +def reload_lazy_config(cfg): + """ + Save an object by LazyConfig.save and load it back. + This is used to test that a config still works the same after + serialization/deserialization. + """ + with tempfile.TemporaryDirectory(prefix="detectron2") as d: + fname = os.path.join(d, "d2_cfg_test.yaml") + LazyConfig.save(cfg, fname) + return LazyConfig.load(fname) diff --git a/detectron2/utils/video_visualizer.py b/detectron2/utils/video_visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..3e46152719832a269415b0956edeead2a2bd4225 --- /dev/null +++ b/detectron2/utils/video_visualizer.py @@ -0,0 +1,268 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List + +import numpy as np +import pycocotools.mask as mask_util + +from detectron2.structures import Instances +from detectron2.utils.visualizer import ( + ColorMode, + Visualizer, + _create_text_labels, + _PanopticPrediction, +) + +from .colormap import random_color, random_colors + + +class _DetectedInstance: + """ + Used to store data about detected objects in video frame, + in order to transfer color to objects in the future frames. + + Attributes: + label (int): + bbox (tuple[float]): + mask_rle (dict): + color (tuple[float]): RGB colors in range (0, 1) + ttl (int): time-to-live for the instance. For example, if ttl=2, + the instance color can be transferred to objects in the next two frames. + """ + + __slots__ = ["label", "bbox", "mask_rle", "color", "ttl"] + + def __init__(self, label, bbox, mask_rle, color, ttl): + self.label = label + self.bbox = bbox + self.mask_rle = mask_rle + self.color = color + self.ttl = ttl + + +class VideoVisualizer: + def __init__(self, metadata, instance_mode=ColorMode.IMAGE): + """ + Args: + metadata (MetadataCatalog): image metadata. + """ + self.metadata = metadata + self._old_instances = [] + assert instance_mode in [ + ColorMode.IMAGE, + ColorMode.IMAGE_BW, + ], "Other mode not supported yet." + self._instance_mode = instance_mode + self._max_num_instances = self.metadata.get("max_num_instances", 74) + self._assigned_colors = {} + self._color_pool = random_colors(self._max_num_instances, rgb=True, maximum=1) + self._color_idx_set = set(range(len(self._color_pool))) + + def draw_instance_predictions(self, frame, predictions): + """ + Draw instance-level prediction results on an image. + + Args: + frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. + predictions (Instances): the output of an instance detection/segmentation + model. Following fields will be used to draw: + "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). + + Returns: + output (VisImage): image object with visualizations. + """ + frame_visualizer = Visualizer(frame, self.metadata) + num_instances = len(predictions) + if num_instances == 0: + return frame_visualizer.output + + boxes = predictions.pred_boxes.tensor.numpy() if predictions.has("pred_boxes") else None + scores = predictions.scores if predictions.has("scores") else None + classes = predictions.pred_classes.numpy() if predictions.has("pred_classes") else None + keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None + colors = predictions.COLOR if predictions.has("COLOR") else [None] * len(predictions) + periods = predictions.ID_period if predictions.has("ID_period") else None + period_threshold = self.metadata.get("period_threshold", 0) + visibilities = [True] * len(predictions) if periods is None else [x > period_threshold for x in periods] + + if predictions.has("pred_masks"): + masks = predictions.pred_masks + # mask IOU is not yet enabled + # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) + # assert len(masks_rles) == num_instances + else: + masks = None + + if not predictions.has("COLOR"): + if predictions.has("ID"): + colors = self._assign_colors_by_id(predictions) + else: + # ToDo: clean old assign color method and use a default tracker to assign id + detected = [ + _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=colors[i], ttl=8) + for i in range(num_instances) + ] + colors = self._assign_colors(detected) + + labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) + + if self._instance_mode == ColorMode.IMAGE_BW: + # any() returns uint8 tensor + frame_visualizer.output.reset_image( + frame_visualizer._create_grayscale_image((masks.any(dim=0) > 0).numpy() if masks is not None else None) + ) + alpha = 0.3 + else: + alpha = 0.5 + + labels = None if labels is None else [y[0] for y in filter(lambda x: x[1], zip(labels, visibilities))] # noqa + assigned_colors = ( + None if colors is None else [y[0] for y in filter(lambda x: x[1], zip(colors, visibilities))] + ) # noqa + frame_visualizer.overlay_instances( + boxes=None if masks is not None else boxes[visibilities], # boxes are a bit distracting + masks=None if masks is None else masks[visibilities], + labels=labels, + keypoints=None if keypoints is None else keypoints[visibilities], + assigned_colors=assigned_colors, + alpha=alpha, + ) + + return frame_visualizer.output + + def draw_sem_seg(self, frame, sem_seg, area_threshold=None): + """ + Args: + sem_seg (ndarray or Tensor): semantic segmentation of shape (H, W), + each value is the integer label. + area_threshold (Optional[int]): only draw segmentations larger than the threshold + """ + # don't need to do anything special + frame_visualizer = Visualizer(frame, self.metadata) + frame_visualizer.draw_sem_seg(sem_seg, area_threshold=None) + return frame_visualizer.output + + def draw_panoptic_seg_predictions(self, frame, panoptic_seg, segments_info, area_threshold=None, alpha=0.5): + frame_visualizer = Visualizer(frame, self.metadata) + pred = _PanopticPrediction(panoptic_seg, segments_info, self.metadata) + + if self._instance_mode == ColorMode.IMAGE_BW: + frame_visualizer.output.reset_image(frame_visualizer._create_grayscale_image(pred.non_empty_mask())) + + # draw mask for all semantic segments first i.e. "stuff" + for mask, sinfo in pred.semantic_masks(): + category_idx = sinfo["category_id"] + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]] + except AttributeError: + mask_color = None + + frame_visualizer.draw_binary_mask( + mask, + color=mask_color, + text=self.metadata.stuff_classes[category_idx], + alpha=alpha, + area_threshold=area_threshold, + ) + + all_instances = list(pred.instance_masks()) + if len(all_instances) == 0: + return frame_visualizer.output + # draw mask for all instances second + masks, sinfo = list(zip(*all_instances)) + num_instances = len(masks) + masks_rles = mask_util.encode(np.asarray(np.asarray(masks).transpose(1, 2, 0), dtype=np.uint8, order="F")) + assert len(masks_rles) == num_instances + + category_ids = [x["category_id"] for x in sinfo] + detected = [ + _DetectedInstance(category_ids[i], bbox=None, mask_rle=masks_rles[i], color=None, ttl=8) + for i in range(num_instances) + ] + colors = self._assign_colors(detected) + labels = [self.metadata.thing_classes[k] for k in category_ids] + + frame_visualizer.overlay_instances( + boxes=None, + masks=masks, + labels=labels, + keypoints=None, + assigned_colors=colors, + alpha=alpha, + ) + return frame_visualizer.output + + def _assign_colors(self, instances): + """ + Naive tracking heuristics to assign same color to the same instance, + will update the internal state of tracked instances. + + Returns: + list[tuple[float]]: list of colors. + """ + + # Compute iou with either boxes or masks: + is_crowd = np.zeros((len(instances),), dtype=np.bool) + if instances[0].bbox is None: + assert instances[0].mask_rle is not None + # use mask iou only when box iou is None + # because box seems good enough + rles_old = [x.mask_rle for x in self._old_instances] + rles_new = [x.mask_rle for x in instances] + ious = mask_util.iou(rles_old, rles_new, is_crowd) + threshold = 0.5 + else: + boxes_old = [x.bbox for x in self._old_instances] + boxes_new = [x.bbox for x in instances] + ious = mask_util.iou(boxes_old, boxes_new, is_crowd) + threshold = 0.6 + if len(ious) == 0: + ious = np.zeros((len(self._old_instances), len(instances)), dtype="float32") + + # Only allow matching instances of the same label: + for old_idx, old in enumerate(self._old_instances): + for new_idx, new in enumerate(instances): + if old.label != new.label: + ious[old_idx, new_idx] = 0 + + matched_new_per_old = np.asarray(ious).argmax(axis=1) + max_iou_per_old = np.asarray(ious).max(axis=1) + + # Try to find match for each old instance: + extra_instances = [] + for idx, inst in enumerate(self._old_instances): + if max_iou_per_old[idx] > threshold: + newidx = matched_new_per_old[idx] + if instances[newidx].color is None: + instances[newidx].color = inst.color + continue + # If an old instance does not match any new instances, + # keep it for the next frame in case it is just missed by the detector + inst.ttl -= 1 + if inst.ttl > 0: + extra_instances.append(inst) + + # Assign random color to newly-detected instances: + for inst in instances: + if inst.color is None: + inst.color = random_color(rgb=True, maximum=1) + self._old_instances = instances[:] + extra_instances + return [d.color for d in instances] + + def _assign_colors_by_id(self, instances: Instances) -> List: + colors = [] + untracked_ids = set(self._assigned_colors.keys()) + for id in instances.ID: + if id in self._assigned_colors: + colors.append(self._color_pool[self._assigned_colors[id]]) + untracked_ids.remove(id) + else: + assert len(self._color_idx_set) >= 1, f"Number of id exceeded maximum, \ + max = {self._max_num_instances}" + idx = self._color_idx_set.pop() + color = self._color_pool[idx] + self._assigned_colors[id] = idx + colors.append(color) + for id in untracked_ids: + self._color_idx_set.add(self._assigned_colors[id]) + del self._assigned_colors[id] + return colors diff --git a/detectron2/utils/visualizer.py b/detectron2/utils/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..6d0cb2d456b46fc9064aadda28917d4d169efee2 --- /dev/null +++ b/detectron2/utils/visualizer.py @@ -0,0 +1,1232 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import colorsys +import logging +import math +from enum import Enum, unique + +import cv2 +import matplotlib as mpl +import matplotlib.colors as mplc +import matplotlib.figure as mplfigure +import numpy as np +import pycocotools.mask as mask_util +import torch +from matplotlib.backends.backend_agg import FigureCanvasAgg +from PIL import Image + +from detectron2.data import MetadataCatalog +from detectron2.structures import BitMasks, Boxes, BoxMode, Keypoints, PolygonMasks, RotatedBoxes +from detectron2.utils.file_io import PathManager + +from .colormap import random_color + +logger = logging.getLogger(__name__) + +__all__ = ["ColorMode", "VisImage", "Visualizer"] + + +_SMALL_OBJECT_AREA_THRESH = 1000 +_LARGE_MASK_AREA_THRESH = 120000 +_OFF_WHITE = (1.0, 1.0, 240.0 / 255) +_BLACK = (0, 0, 0) +_RED = (1.0, 0, 0) + +_KEYPOINT_THRESHOLD = 0.05 + + +@unique +class ColorMode(Enum): + """ + Enum of different color modes to use for instance visualizations. + """ + + IMAGE = 0 + """ + Picks a random color for every instance and overlay segmentations with low opacity. + """ + SEGMENTATION = 1 + """ + Let instances of the same category have similar colors + (from metadata.thing_colors), and overlay them with + high opacity. This provides more attention on the quality of segmentation. + """ + IMAGE_BW = 2 + """ + Same as IMAGE, but convert all areas without masks to gray-scale. + Only available for drawing per-instance mask predictions. + """ + + +class GenericMask: + """ + Attribute: + polygons (list[ndarray]): list[ndarray]: polygons for this mask. + Each ndarray has format [x, y, x, y, ...] + mask (ndarray): a binary mask + """ + + def __init__(self, mask_or_polygons, height, width): + self._mask = self._polygons = self._has_holes = None + self.height = height + self.width = width + + m = mask_or_polygons + if isinstance(m, dict): + # RLEs + assert "counts" in m and "size" in m + if isinstance(m["counts"], list): # uncompressed RLEs + h, w = m["size"] + assert h == height and w == width + m = mask_util.frPyObjects(m, h, w) + self._mask = mask_util.decode(m)[:, :] + return + + if isinstance(m, list): # list[ndarray] + self._polygons = [np.asarray(x).reshape(-1) for x in m] + return + + if isinstance(m, np.ndarray): # assumed to be a binary mask + assert m.shape[1] != 2, m.shape + assert m.shape == ( + height, + width, + ), f"mask shape: {m.shape}, target dims: {height}, {width}" + self._mask = m.astype("uint8") + return + + raise ValueError("GenericMask cannot handle object {} of type '{}'".format(m, type(m))) + + @property + def mask(self): + if self._mask is None: + self._mask = self.polygons_to_mask(self._polygons) + return self._mask + + @property + def polygons(self): + if self._polygons is None: + self._polygons, self._has_holes = self.mask_to_polygons(self._mask) + return self._polygons + + @property + def has_holes(self): + if self._has_holes is None: + if self._mask is not None: + self._polygons, self._has_holes = self.mask_to_polygons(self._mask) + else: + self._has_holes = False # if original format is polygon, does not have holes + return self._has_holes + + def mask_to_polygons(self, mask): + # cv2.RETR_CCOMP flag retrieves all the contours and arranges them to a 2-level + # hierarchy. External contours (boundary) of the object are placed in hierarchy-1. + # Internal contours (holes) are placed in hierarchy-2. + # cv2.CHAIN_APPROX_NONE flag gets vertices of polygons from contours. + mask = np.ascontiguousarray(mask) # some versions of cv2 does not support incontiguous arr + res = cv2.findContours(mask.astype("uint8"), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + hierarchy = res[-1] + if hierarchy is None: # empty mask + return [], False + has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0 + res = res[-2] + res = [x.flatten() for x in res] + # These coordinates from OpenCV are integers in range [0, W-1 or H-1]. + # We add 0.5 to turn them into real-value coordinate space. A better solution + # would be to first +0.5 and then dilate the returned polygon by 0.5. + res = [x + 0.5 for x in res if len(x) >= 6] + return res, has_holes + + def polygons_to_mask(self, polygons): + rle = mask_util.frPyObjects(polygons, self.height, self.width) + rle = mask_util.merge(rle) + return mask_util.decode(rle)[:, :] + + def area(self): + return self.mask.sum() + + def bbox(self): + p = mask_util.frPyObjects(self.polygons, self.height, self.width) + p = mask_util.merge(p) + bbox = mask_util.toBbox(p) + bbox[2] += bbox[0] + bbox[3] += bbox[1] + return bbox + + +class _PanopticPrediction: + """ + Unify different panoptic annotation/prediction formats + """ + + def __init__(self, panoptic_seg, segments_info, metadata=None): + if segments_info is None: + assert metadata is not None + # If "segments_info" is None, we assume "panoptic_img" is a + # H*W int32 image storing the panoptic_id in the format of + # category_id * label_divisor + instance_id. We reserve -1 for + # VOID label. + label_divisor = metadata.label_divisor + segments_info = [] + for panoptic_label in np.unique(panoptic_seg.numpy()): + if panoptic_label == -1: + # VOID region. + continue + pred_class = panoptic_label // label_divisor + isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() + segments_info.append( + { + "id": int(panoptic_label), + "category_id": int(pred_class), + "isthing": bool(isthing), + } + ) + del metadata + + self._seg = panoptic_seg + + self._sinfo = {s["id"]: s for s in segments_info} # seg id -> seg info + segment_ids, areas = torch.unique(panoptic_seg, sorted=True, return_counts=True) + areas = areas.numpy() + sorted_idxs = np.argsort(-areas) + self._seg_ids, self._seg_areas = segment_ids[sorted_idxs], areas[sorted_idxs] + self._seg_ids = self._seg_ids.tolist() + for sid, area in zip(self._seg_ids, self._seg_areas): + if sid in self._sinfo: + self._sinfo[sid]["area"] = float(area) + + def non_empty_mask(self): + """ + Returns: + (H, W) array, a mask for all pixels that have a prediction + """ + empty_ids = [] + for id in self._seg_ids: + if id not in self._sinfo: + empty_ids.append(id) + if len(empty_ids) == 0: + return np.zeros(self._seg.shape, dtype=np.uint8) + assert len(empty_ids) == 1, ">1 ids corresponds to no labels. This is currently not supported" + return (self._seg != empty_ids[0]).numpy().astype(np.bool) + + def semantic_masks(self): + for sid in self._seg_ids: + sinfo = self._sinfo.get(sid) + if sinfo is None or sinfo["isthing"]: + # Some pixels (e.g. id 0 in PanopticFPN) have no instance or semantic predictions. + continue + yield (self._seg == sid).numpy().astype(np.bool), sinfo + + def instance_masks(self): + for sid in self._seg_ids: + sinfo = self._sinfo.get(sid) + if sinfo is None or not sinfo["isthing"]: + continue + mask = (self._seg == sid).numpy().astype(np.bool) + if mask.sum() > 0: + yield mask, sinfo + + +def _create_text_labels(classes, scores, class_names, is_crowd=None): + """ + Args: + classes (list[int] or None): + scores (list[float] or None): + class_names (list[str] or None): + is_crowd (list[bool] or None): + + Returns: + list[str] or None + """ + labels = None + if classes is not None: + if class_names is not None and len(class_names) > 0: + labels = [class_names[i] for i in classes] + else: + labels = [str(i) for i in classes] + if scores is not None: + if labels is None: + labels = ["{:.0f}%".format(s * 100) for s in scores] + else: + labels = ["{} {:.0f}%".format(l, s * 100) for l, s in zip(labels, scores)] + if labels is not None and is_crowd is not None: + labels = [l + ("|crowd" if crowd else "") for l, crowd in zip(labels, is_crowd)] + return labels + + +class VisImage: + def __init__(self, img, scale=1.0): + """ + Args: + img (ndarray): an RGB image of shape (H, W, 3) in range [0, 255]. + scale (float): scale the input image + """ + self.img = img + self.scale = scale + self.width, self.height = img.shape[1], img.shape[0] + self._setup_figure(img) + + def _setup_figure(self, img): + """ + Args: + Same as in :meth:`__init__()`. + + Returns: + fig (matplotlib.pyplot.figure): top level container for all the image plot elements. + ax (matplotlib.pyplot.Axes): contains figure elements and sets the coordinate system. + """ + fig = mplfigure.Figure(frameon=False) + self.dpi = fig.get_dpi() + # add a small 1e-2 to avoid precision lost due to matplotlib's truncation + # (https://github.com/matplotlib/matplotlib/issues/15363) + fig.set_size_inches( + (self.width * self.scale + 1e-2) / self.dpi, + (self.height * self.scale + 1e-2) / self.dpi, + ) + self.canvas = FigureCanvasAgg(fig) + # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig) + ax = fig.add_axes([0.0, 0.0, 1.0, 1.0]) + ax.axis("off") + self.fig = fig + self.ax = ax + self.reset_image(img) + + def reset_image(self, img): + """ + Args: + img: same as in __init__ + """ + img = img.astype("uint8") + self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation="nearest") + + def save(self, filepath): + """ + Args: + filepath (str): a string that contains the absolute path, including the file name, where + the visualized image will be saved. + """ + self.fig.savefig(filepath) + + def get_image(self): + """ + Returns: + ndarray: + the visualized image of shape (H, W, 3) (RGB) in uint8 type. + The shape is scaled w.r.t the input image using the given `scale` argument. + """ + canvas = self.canvas + s, (width, height) = canvas.print_to_buffer() + # buf = io.BytesIO() # works for cairo backend + # canvas.print_rgba(buf) + # width, height = self.width, self.height + # s = buf.getvalue() + + buffer = np.frombuffer(s, dtype="uint8") + + img_rgba = buffer.reshape(height, width, 4) + rgb, alpha = np.split(img_rgba, [3], axis=2) + return rgb.astype("uint8") + + +class Visualizer: + """ + Visualizer that draws data about detection/segmentation on images. + + It contains methods like `draw_{text,box,circle,line,binary_mask,polygon}` + that draw primitive objects to images, as well as high-level wrappers like + `draw_{instance_predictions,sem_seg,panoptic_seg_predictions,dataset_dict}` + that draw composite data in some pre-defined style. + + Note that the exact visualization style for the high-level wrappers are subject to change. + Style such as color, opacity, label contents, visibility of labels, or even the visibility + of objects themselves (e.g. when the object is too small) may change according + to different heuristics, as long as the results still look visually reasonable. + + To obtain a consistent style, you can implement custom drawing functions with the + abovementioned primitive methods instead. If you need more customized visualization + styles, you can process the data yourself following their format documented in + tutorials (:doc:`/tutorials/models`, :doc:`/tutorials/datasets`). This class does not + intend to satisfy everyone's preference on drawing styles. + + This visualizer focuses on high rendering quality rather than performance. It is not + designed to be used for real-time applications. + """ + + # TODO implement a fast, rasterized version using OpenCV + + def __init__(self, img_rgb, metadata=None, scale=1.0, instance_mode=ColorMode.IMAGE): + """ + Args: + img_rgb: a numpy array of shape (H, W, C), where H and W correspond to + the height and width of the image respectively. C is the number of + color channels. The image is required to be in RGB format since that + is a requirement of the Matplotlib library. The image is also expected + to be in the range [0, 255]. + metadata (Metadata): dataset metadata (e.g. class names and colors) + instance_mode (ColorMode): defines one of the pre-defined style for drawing + instances on an image. + """ + self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8) + if metadata is None: + metadata = MetadataCatalog.get("__nonexist__") + self.metadata = metadata + self.output = VisImage(self.img, scale=scale) + self.cpu_device = torch.device("cpu") + + # too small texts are useless, therefore clamp to 9 + self._default_font_size = max(np.sqrt(self.output.height * self.output.width) // 90, 10 // scale) + self._instance_mode = instance_mode + self.keypoint_threshold = _KEYPOINT_THRESHOLD + + def draw_instance_predictions(self, predictions): + """ + Draw instance-level prediction results on an image. + + Args: + predictions (Instances): the output of an instance detection/segmentation + model. Following fields will be used to draw: + "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). + + Returns: + output (VisImage): image object with visualizations. + """ + boxes = predictions.pred_boxes if predictions.has("pred_boxes") else None + scores = predictions.scores if predictions.has("scores") else None + classes = predictions.pred_classes.tolist() if predictions.has("pred_classes") else None + labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) + keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None + + if predictions.has("pred_masks"): + masks = np.asarray(predictions.pred_masks) + masks = [GenericMask(x, self.output.height, self.output.width) for x in masks] + else: + masks = None + + if self._instance_mode == ColorMode.SEGMENTATION and self.metadata.get("thing_colors"): + colors = [self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) for c in classes] + alpha = 0.8 + else: + colors = None + alpha = 0.5 + + if self._instance_mode == ColorMode.IMAGE_BW: + self.output.reset_image( + self._create_grayscale_image( + (predictions.pred_masks.any(dim=0) > 0).numpy() if predictions.has("pred_masks") else None + ) + ) + alpha = 0.3 + + self.overlay_instances( + masks=masks, + boxes=boxes, + labels=labels, + keypoints=keypoints, + assigned_colors=colors, + alpha=alpha, + ) + return self.output + + def draw_sem_seg(self, sem_seg, area_threshold=None, alpha=0.8): + """ + Draw semantic segmentation predictions/labels. + + Args: + sem_seg (Tensor or ndarray): the segmentation of shape (H, W). + Each value is the integer label of the pixel. + area_threshold (int): segments with less than `area_threshold` are not drawn. + alpha (float): the larger it is, the more opaque the segmentations are. + + Returns: + output (VisImage): image object with visualizations. + """ + if isinstance(sem_seg, torch.Tensor): + sem_seg = sem_seg.numpy() + labels, areas = np.unique(sem_seg, return_counts=True) + sorted_idxs = np.argsort(-areas).tolist() + labels = labels[sorted_idxs] + for label in filter(lambda l: l < len(self.metadata.stuff_classes), labels): + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[label]] + except (AttributeError, IndexError): + mask_color = None + + binary_mask = (sem_seg == label).astype(np.uint8) + text = self.metadata.stuff_classes[label] + self.draw_binary_mask( + binary_mask, + color=mask_color, + edge_color=_OFF_WHITE, + text=text, + alpha=alpha, + area_threshold=area_threshold, + ) + return self.output + + def draw_panoptic_seg(self, panoptic_seg, segments_info, area_threshold=None, alpha=0.7): + """ + Draw panoptic prediction annotations or results. + + Args: + panoptic_seg (Tensor): of shape (height, width) where the values are ids for each + segment. + segments_info (list[dict] or None): Describe each segment in `panoptic_seg`. + If it is a ``list[dict]``, each dict contains keys "id", "category_id". + If None, category id of each pixel is computed by + ``pixel // metadata.label_divisor``. + area_threshold (int): stuff segments with less than `area_threshold` are not drawn. + + Returns: + output (VisImage): image object with visualizations. + """ + pred = _PanopticPrediction(panoptic_seg, segments_info, self.metadata) + + if self._instance_mode == ColorMode.IMAGE_BW: + self.output.reset_image(self._create_grayscale_image(pred.non_empty_mask())) + + # draw mask for all semantic segments first i.e. "stuff" + for mask, sinfo in pred.semantic_masks(): + category_idx = sinfo["category_id"] + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]] + except AttributeError: + mask_color = None + + text = self.metadata.stuff_classes[category_idx] + self.draw_binary_mask( + mask, + color=mask_color, + edge_color=_OFF_WHITE, + text=text, + alpha=alpha, + area_threshold=area_threshold, + ) + + # draw mask for all instances second + all_instances = list(pred.instance_masks()) + if len(all_instances) == 0: + return self.output + masks, sinfo = list(zip(*all_instances)) + category_ids = [x["category_id"] for x in sinfo] + + try: + scores = [x["score"] for x in sinfo] + except KeyError: + scores = None + labels = _create_text_labels( + category_ids, scores, self.metadata.thing_classes, [x.get("iscrowd", 0) for x in sinfo] + ) + + try: + colors = [self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) for c in category_ids] + except AttributeError: + colors = None + self.overlay_instances(masks=masks, labels=labels, assigned_colors=colors, alpha=alpha) + + return self.output + + draw_panoptic_seg_predictions = draw_panoptic_seg # backward compatibility + + def draw_dataset_dict(self, dic): + """ + Draw annotations/segmentaions in Detectron2 Dataset format. + + Args: + dic (dict): annotation/segmentation data of one image, in Detectron2 Dataset format. + + Returns: + output (VisImage): image object with visualizations. + """ + annos = dic.get("annotations", None) + if annos: + if "segmentation" in annos[0]: + masks = [x["segmentation"] for x in annos] + else: + masks = None + if "keypoints" in annos[0]: + keypts = [x["keypoints"] for x in annos] + keypts = np.array(keypts).reshape(len(annos), -1, 3) + else: + keypts = None + + boxes = [ + BoxMode.convert(x["bbox"], x["bbox_mode"], BoxMode.XYXY_ABS) if len(x["bbox"]) == 4 else x["bbox"] + for x in annos + ] + + colors = None + category_ids = [x["category_id"] for x in annos] + if self._instance_mode == ColorMode.SEGMENTATION and self.metadata.get("thing_colors"): + colors = [self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) for c in category_ids] + names = self.metadata.get("thing_classes", None) + labels = _create_text_labels( + category_ids, + scores=None, + class_names=names, + is_crowd=[x.get("iscrowd", 0) for x in annos], + ) + self.overlay_instances(labels=labels, boxes=boxes, masks=masks, keypoints=keypts, assigned_colors=colors) + + sem_seg = dic.get("sem_seg", None) + if sem_seg is None and "sem_seg_file_name" in dic: + with PathManager.open(dic["sem_seg_file_name"], "rb") as f: + sem_seg = Image.open(f) + sem_seg = np.asarray(sem_seg, dtype="uint8") + if sem_seg is not None: + self.draw_sem_seg(sem_seg, area_threshold=0, alpha=0.5) + + pan_seg = dic.get("pan_seg", None) + if pan_seg is None and "pan_seg_file_name" in dic: + with PathManager.open(dic["pan_seg_file_name"], "rb") as f: + pan_seg = Image.open(f) + pan_seg = np.asarray(pan_seg) + from panopticapi.utils import rgb2id + + pan_seg = rgb2id(pan_seg) + if pan_seg is not None: + segments_info = dic["segments_info"] + pan_seg = torch.tensor(pan_seg) + self.draw_panoptic_seg(pan_seg, segments_info, area_threshold=0, alpha=0.5) + return self.output + + def overlay_instances( + self, + *, + boxes=None, + labels=None, + masks=None, + keypoints=None, + assigned_colors=None, + alpha=0.5, + ): + """ + Args: + boxes (Boxes, RotatedBoxes or ndarray): either a :class:`Boxes`, + or an Nx4 numpy array of XYXY_ABS format for the N objects in a single image, + or a :class:`RotatedBoxes`, + or an Nx5 numpy array of (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image, + labels (list[str]): the text to be displayed for each instance. + masks (masks-like object): Supported types are: + + * :class:`detectron2.structures.PolygonMasks`, + :class:`detectron2.structures.BitMasks`. + * list[list[ndarray]]: contains the segmentation masks for all objects in one image. + The first level of the list corresponds to individual instances. The second + level to all the polygon that compose the instance, and the third level + to the polygon coordinates. The third level should have the format of + [x0, y0, x1, y1, ..., xn, yn] (n >= 3). + * list[ndarray]: each ndarray is a binary mask of shape (H, W). + * list[dict]: each dict is a COCO-style RLE. + keypoints (Keypoint or array like): an array-like object of shape (N, K, 3), + where the N is the number of instances and K is the number of keypoints. + The last dimension corresponds to (x, y, visibility or score). + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = 0 + if boxes is not None: + boxes = self._convert_boxes(boxes) + num_instances = len(boxes) + if masks is not None: + masks = self._convert_masks(masks) + if num_instances: + assert len(masks) == num_instances + else: + num_instances = len(masks) + if keypoints is not None: + if num_instances: + assert len(keypoints) == num_instances + else: + num_instances = len(keypoints) + keypoints = self._convert_keypoints(keypoints) + if labels is not None: + assert len(labels) == num_instances + if assigned_colors is None: + assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + if num_instances == 0: + return self.output + if boxes is not None and boxes.shape[1] == 5: + return self.overlay_rotated_instances(boxes=boxes, labels=labels, assigned_colors=assigned_colors) + + # Display in largest to smallest order to reduce occlusion. + areas = None + if boxes is not None: + areas = np.prod(boxes[:, 2:] - boxes[:, :2], axis=1) + elif masks is not None: + areas = np.asarray([x.area() for x in masks]) + + if areas is not None: + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] if boxes is not None else None + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + masks = [masks[idx] for idx in sorted_idxs] if masks is not None else None + assigned_colors = [assigned_colors[idx] for idx in sorted_idxs] + keypoints = keypoints[sorted_idxs] if keypoints is not None else None + + for i in range(num_instances): + color = assigned_colors[i] + if boxes is not None: + self.draw_box(boxes[i], edge_color=color) + + if masks is not None: + for segment in masks[i].polygons: + self.draw_polygon(segment.reshape(-1, 2), color, alpha=alpha) + + if labels is not None: + # first get a box + if boxes is not None: + x0, y0, x1, y1 = boxes[i] + text_pos = (x0, y0) # if drawing boxes, put text on the box corner. + horiz_align = "left" + elif masks is not None: + # skip small mask without polygon + if len(masks[i].polygons) == 0: + continue + + x0, y0, x1, y1 = masks[i].bbox() + + # draw text in the center (defined by median) when box is not drawn + # median is less sensitive to outliers. + text_pos = np.median(masks[i].mask.nonzero(), axis=1)[::-1] + horiz_align = "center" + else: + continue # drawing the box confidence for keypoints isn't very useful. + # for small objects, draw text at the side to avoid occlusion + instance_area = (y1 - y0) * (x1 - x0) + if instance_area < _SMALL_OBJECT_AREA_THRESH * self.output.scale or y1 - y0 < 40 * self.output.scale: + if y1 >= self.output.height - 5: + text_pos = (x1, y0) + else: + text_pos = (x0, y1) + + height_ratio = (y1 - y0) / np.sqrt(self.output.height * self.output.width) + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + font_size = np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) * 0.5 * self._default_font_size + self.draw_text( + labels[i], + text_pos, + color=lighter_color, + horizontal_alignment=horiz_align, + font_size=font_size, + ) + + # draw keypoints + if keypoints is not None: + for keypoints_per_instance in keypoints: + self.draw_and_connect_keypoints(keypoints_per_instance) + + return self.output + + def overlay_rotated_instances(self, boxes=None, labels=None, assigned_colors=None): + """ + Args: + boxes (ndarray): an Nx5 numpy array of + (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image. + labels (list[str]): the text to be displayed for each instance. + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = len(boxes) + + if assigned_colors is None: + assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + if num_instances == 0: + return self.output + + # Display in largest to smallest order to reduce occlusion. + if boxes is not None: + areas = boxes[:, 2] * boxes[:, 3] + + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + colors = [assigned_colors[idx] for idx in sorted_idxs] + + for i in range(num_instances): + self.draw_rotated_box_with_label( + boxes[i], edge_color=colors[i], label=labels[i] if labels is not None else None + ) + + return self.output + + def draw_and_connect_keypoints(self, keypoints): + """ + Draws keypoints of an instance and follows the rules for keypoint connections + to draw lines between appropriate keypoints. This follows color heuristics for + line color. + + Args: + keypoints (Tensor): a tensor of shape (K, 3), where K is the number of keypoints + and the last dimension corresponds to (x, y, probability). + + Returns: + output (VisImage): image object with visualizations. + """ + visible = {} + keypoint_names = self.metadata.get("keypoint_names") + for idx, keypoint in enumerate(keypoints): + + # draw keypoint + x, y, prob = keypoint + if prob > self.keypoint_threshold: + self.draw_circle((x, y), color=_RED) + if keypoint_names: + keypoint_name = keypoint_names[idx] + visible[keypoint_name] = (x, y) + + if self.metadata.get("keypoint_connection_rules"): + for kp0, kp1, color in self.metadata.keypoint_connection_rules: + if kp0 in visible and kp1 in visible: + x0, y0 = visible[kp0] + x1, y1 = visible[kp1] + color = tuple(x / 255.0 for x in color) + self.draw_line([x0, x1], [y0, y1], color=color) + + # draw lines from nose to mid-shoulder and mid-shoulder to mid-hip + # Note that this strategy is specific to person keypoints. + # For other keypoints, it should just do nothing + try: + ls_x, ls_y = visible["left_shoulder"] + rs_x, rs_y = visible["right_shoulder"] + mid_shoulder_x, mid_shoulder_y = (ls_x + rs_x) / 2, (ls_y + rs_y) / 2 + except KeyError: + pass + else: + # draw line from nose to mid-shoulder + nose_x, nose_y = visible.get("nose", (None, None)) + if nose_x is not None: + self.draw_line([nose_x, mid_shoulder_x], [nose_y, mid_shoulder_y], color=_RED) + + try: + # draw line from mid-shoulder to mid-hip + lh_x, lh_y = visible["left_hip"] + rh_x, rh_y = visible["right_hip"] + except KeyError: + pass + else: + mid_hip_x, mid_hip_y = (lh_x + rh_x) / 2, (lh_y + rh_y) / 2 + self.draw_line([mid_hip_x, mid_shoulder_x], [mid_hip_y, mid_shoulder_y], color=_RED) + return self.output + + """ + Primitive drawing functions: + """ + + def draw_text( + self, + text, + position, + *, + font_size=None, + color="g", + horizontal_alignment="center", + rotation=0, + ): + """ + Args: + text (str): class label + position (tuple): a tuple of the x and y coordinates to place text on image. + font_size (int, optional): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color: color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + horizontal_alignment (str): see `matplotlib.text.Text` + rotation: rotation angle in degrees CCW + + Returns: + output (VisImage): image object with text drawn. + """ + if not font_size: + font_size = self._default_font_size + + # since the text background is dark, we don't want the text to be dark + color = np.maximum(list(mplc.to_rgb(color)), 0.2) + color[np.argmax(color)] = max(0.8, np.max(color)) + + x, y = position + self.output.ax.text( + x, + y, + text, + size=font_size * self.output.scale, + family="sans-serif", + bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"}, + verticalalignment="top", + horizontalalignment=horizontal_alignment, + color=color, + zorder=10, + rotation=rotation, + ) + return self.output + + def draw_box(self, box_coord, alpha=0.5, edge_color="g", line_style="-"): + """ + Args: + box_coord (tuple): a tuple containing x0, y0, x1, y1 coordinates, where x0 and y0 + are the coordinates of the image's top left corner. x1 and y1 are the + coordinates of the image's bottom right corner. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + edge_color: color of the outline of the box. Refer to `matplotlib.colors` + for full list of formats that are accepted. + line_style (string): the string to use to create the outline of the boxes. + + Returns: + output (VisImage): image object with box drawn. + """ + x0, y0, x1, y1 = box_coord + width = x1 - x0 + height = y1 - y0 + + linewidth = max(self._default_font_size / 4, 1) + + self.output.ax.add_patch( + mpl.patches.Rectangle( + (x0, y0), + width, + height, + fill=False, + edgecolor=edge_color, + linewidth=linewidth * self.output.scale, + alpha=alpha, + linestyle=line_style, + ) + ) + return self.output + + def draw_rotated_box_with_label(self, rotated_box, alpha=0.5, edge_color="g", line_style="-", label=None): + """ + Draw a rotated box with label on its top-left corner. + + Args: + rotated_box (tuple): a tuple containing (cnt_x, cnt_y, w, h, angle), + where cnt_x and cnt_y are the center coordinates of the box. + w and h are the width and height of the box. angle represents how + many degrees the box is rotated CCW with regard to the 0-degree box. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + edge_color: color of the outline of the box. Refer to `matplotlib.colors` + for full list of formats that are accepted. + line_style (string): the string to use to create the outline of the boxes. + label (string): label for rotated box. It will not be rendered when set to None. + + Returns: + output (VisImage): image object with box drawn. + """ + cnt_x, cnt_y, w, h, angle = rotated_box + area = w * h + # use thinner lines when the box is small + linewidth = self._default_font_size / (6 if area < _SMALL_OBJECT_AREA_THRESH * self.output.scale else 3) + + theta = angle * math.pi / 180.0 + c = math.cos(theta) + s = math.sin(theta) + rect = [(-w / 2, h / 2), (-w / 2, -h / 2), (w / 2, -h / 2), (w / 2, h / 2)] + # x: left->right ; y: top->down + rotated_rect = [(s * yy + c * xx + cnt_x, c * yy - s * xx + cnt_y) for (xx, yy) in rect] + for k in range(4): + j = (k + 1) % 4 + self.draw_line( + [rotated_rect[k][0], rotated_rect[j][0]], + [rotated_rect[k][1], rotated_rect[j][1]], + color=edge_color, + linestyle="--" if k == 1 else line_style, + linewidth=linewidth, + ) + + if label is not None: + text_pos = rotated_rect[1] # topleft corner + + height_ratio = h / np.sqrt(self.output.height * self.output.width) + label_color = self._change_color_brightness(edge_color, brightness_factor=0.7) + font_size = np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) * 0.5 * self._default_font_size + self.draw_text(label, text_pos, color=label_color, font_size=font_size, rotation=angle) + + return self.output + + def draw_circle(self, circle_coord, color, radius=3): + """ + Args: + circle_coord (list(int) or tuple(int)): contains the x and y coordinates + of the center of the circle. + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + radius (int): radius of the circle. + + Returns: + output (VisImage): image object with box drawn. + """ + x, y = circle_coord + self.output.ax.add_patch(mpl.patches.Circle(circle_coord, radius=radius, fill=True, color=color)) + return self.output + + def draw_line(self, x_data, y_data, color, linestyle="-", linewidth=None): + """ + Args: + x_data (list[int]): a list containing x values of all the points being drawn. + Length of list should match the length of y_data. + y_data (list[int]): a list containing y values of all the points being drawn. + Length of list should match the length of x_data. + color: color of the line. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + linestyle: style of the line. Refer to `matplotlib.lines.Line2D` + for a full list of formats that are accepted. + linewidth (float or None): width of the line. When it's None, + a default value will be computed and used. + + Returns: + output (VisImage): image object with line drawn. + """ + if linewidth is None: + linewidth = self._default_font_size / 3 + linewidth = max(linewidth, 1) + self.output.ax.add_line( + mpl.lines.Line2D( + x_data, + y_data, + linewidth=linewidth * self.output.scale, + color=color, + linestyle=linestyle, + ) + ) + return self.output + + def draw_binary_mask(self, binary_mask, color=None, *, edge_color=None, text=None, alpha=0.5, area_threshold=10): + """ + Args: + binary_mask (ndarray): numpy array of shape (H, W), where H is the image height and + W is the image width. Each value in the array is either a 0 or 1 value of uint8 + type. + color: color of the mask. Refer to `matplotlib.colors` for a full list of + formats that are accepted. If None, will pick a random color. + edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a + full list of formats that are accepted. + text (str): if None, will be drawn on the object + alpha (float): blending efficient. Smaller values lead to more transparent masks. + area_threshold (float): a connected component smaller than this area will not be shown. + + Returns: + output (VisImage): image object with mask drawn. + """ + if color is None: + color = random_color(rgb=True, maximum=1) + color = mplc.to_rgb(color) + + has_valid_segment = False + binary_mask = binary_mask.astype("uint8") # opencv needs uint8 + mask = GenericMask(binary_mask, self.output.height, self.output.width) + shape2d = (binary_mask.shape[0], binary_mask.shape[1]) + + if not mask.has_holes: + # draw polygons for regular masks + for segment in mask.polygons: + area = mask_util.area(mask_util.frPyObjects([segment], shape2d[0], shape2d[1])) + if area < (area_threshold or 0): + continue + has_valid_segment = True + segment = segment.reshape(-1, 2) + self.draw_polygon(segment, color=color, edge_color=edge_color, alpha=alpha) + else: + # TODO: Use Path/PathPatch to draw vector graphics: + # https://stackoverflow.com/questions/8919719/how-to-plot-a-complex-polygon + rgba = np.zeros(shape2d + (4,), dtype="float32") + rgba[:, :, :3] = color + rgba[:, :, 3] = (mask.mask == 1).astype("float32") * alpha + has_valid_segment = True + self.output.ax.imshow(rgba, extent=(0, self.output.width, self.output.height, 0)) + + if text is not None and has_valid_segment: + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + self._draw_text_in_mask(binary_mask, text, lighter_color) + return self.output + + def draw_soft_mask(self, soft_mask, color=None, *, text=None, alpha=0.5): + """ + Args: + soft_mask (ndarray): float array of shape (H, W), each value in [0, 1]. + color: color of the mask. Refer to `matplotlib.colors` for a full list of + formats that are accepted. If None, will pick a random color. + text (str): if None, will be drawn on the object + alpha (float): blending efficient. Smaller values lead to more transparent masks. + + Returns: + output (VisImage): image object with mask drawn. + """ + if color is None: + color = random_color(rgb=True, maximum=1) + color = mplc.to_rgb(color) + + shape2d = (soft_mask.shape[0], soft_mask.shape[1]) + rgba = np.zeros(shape2d + (4,), dtype="float32") + rgba[:, :, :3] = color + rgba[:, :, 3] = soft_mask * alpha + self.output.ax.imshow(rgba, extent=(0, self.output.width, self.output.height, 0)) + + if text is not None: + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + binary_mask = (soft_mask > 0.5).astype("uint8") + self._draw_text_in_mask(binary_mask, text, lighter_color) + return self.output + + def draw_polygon(self, segment, color, edge_color=None, alpha=0.5): + """ + Args: + segment: numpy array of shape Nx2, containing all the points in the polygon. + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a + full list of formats that are accepted. If not provided, a darker shade + of the polygon color will be used instead. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + + Returns: + output (VisImage): image object with polygon drawn. + """ + if edge_color is None: + # make edge color darker than the polygon color + if alpha > 0.8: + edge_color = self._change_color_brightness(color, brightness_factor=-0.7) + else: + edge_color = color + edge_color = mplc.to_rgb(edge_color) + (1,) + + polygon = mpl.patches.Polygon( + segment, + fill=True, + facecolor=mplc.to_rgb(color) + (alpha,), + edgecolor=edge_color, + linewidth=max(self._default_font_size // 15 * self.output.scale, 1), + ) + self.output.ax.add_patch(polygon) + return self.output + + """ + Internal methods: + """ + + def _jitter(self, color): + """ + Randomly modifies given color to produce a slightly different color than the color given. + + Args: + color (tuple[double]): a tuple of 3 elements, containing the RGB values of the color + picked. The values in the list are in the [0.0, 1.0] range. + + Returns: + jittered_color (tuple[double]): a tuple of 3 elements, containing the RGB values of the + color after being jittered. The values in the list are in the [0.0, 1.0] range. + """ + color = mplc.to_rgb(color) + vec = np.random.rand(3) + # better to do it in another color space + vec = vec / np.linalg.norm(vec) * 0.5 + res = np.clip(vec + color, 0, 1) + return tuple(res) + + def _create_grayscale_image(self, mask=None): + """ + Create a grayscale version of the original image. + The colors in masked area, if given, will be kept. + """ + img_bw = self.img.astype("f4").mean(axis=2) + img_bw = np.stack([img_bw] * 3, axis=2) + if mask is not None: + img_bw[mask] = self.img[mask] + return img_bw + + def _change_color_brightness(self, color, brightness_factor): + """ + Depending on the brightness_factor, gives a lighter or darker color i.e. a color with + less or more saturation than the original color. + + Args: + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + brightness_factor (float): a value in [-1.0, 1.0] range. A lightness factor of + 0 will correspond to no change, a factor in [-1.0, 0) range will result in + a darker color and a factor in (0, 1.0] range will result in a lighter color. + + Returns: + modified_color (tuple[double]): a tuple containing the RGB values of the + modified color. Each value in the tuple is in the [0.0, 1.0] range. + """ + assert brightness_factor >= -1.0 and brightness_factor <= 1.0 + color = mplc.to_rgb(color) + polygon_color = colorsys.rgb_to_hls(*mplc.to_rgb(color)) + modified_lightness = polygon_color[1] + (brightness_factor * polygon_color[1]) + modified_lightness = 0.0 if modified_lightness < 0.0 else modified_lightness + modified_lightness = 1.0 if modified_lightness > 1.0 else modified_lightness + modified_color = colorsys.hls_to_rgb(polygon_color[0], modified_lightness, polygon_color[2]) + return modified_color + + def _convert_boxes(self, boxes): + """ + Convert different format of boxes to an NxB array, where B = 4 or 5 is the box dimension. + """ + if isinstance(boxes, Boxes) or isinstance(boxes, RotatedBoxes): + return boxes.tensor.detach().numpy() + else: + return np.asarray(boxes) + + def _convert_masks(self, masks_or_polygons): + """ + Convert different format of masks or polygons to a tuple of masks and polygons. + + Returns: + list[GenericMask]: + """ + + m = masks_or_polygons + if isinstance(m, PolygonMasks): + m = m.polygons + if isinstance(m, BitMasks): + m = m.tensor.numpy() + if isinstance(m, torch.Tensor): + m = m.numpy() + ret = [] + for x in m: + if isinstance(x, GenericMask): + ret.append(x) + else: + ret.append(GenericMask(x, self.output.height, self.output.width)) + return ret + + def _draw_text_in_mask(self, binary_mask, text, color): + """ + Find proper places to draw text given a binary mask. + """ + # TODO sometimes drawn on wrong objects. the heuristics here can improve. + _num_cc, cc_labels, stats, centroids = cv2.connectedComponentsWithStats(binary_mask, 8) + if stats[1:, -1].size == 0: + return + largest_component_id = np.argmax(stats[1:, -1]) + 1 + + # draw text on the largest component, as well as other very large components. + for cid in range(1, _num_cc): + if cid == largest_component_id or stats[cid, -1] > _LARGE_MASK_AREA_THRESH: + # median is more stable than centroid + # center = centroids[largest_component_id] + center = np.median((cc_labels == cid).nonzero(), axis=1)[::-1] + self.draw_text(text, center, color=color) + + def _convert_keypoints(self, keypoints): + if isinstance(keypoints, Keypoints): + keypoints = keypoints.tensor + keypoints = np.asarray(keypoints) + return keypoints + + def get_output(self): + """ + Returns: + output (VisImage): the image output containing the visualizations added + to the image. + """ + return self.output diff --git a/diff_ras/__init__.py b/diff_ras/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..684a75c81d303454d117a37a91469d5cc141a3fd --- /dev/null +++ b/diff_ras/__init__.py @@ -0,0 +1 @@ +from polygon import SoftPolygon diff --git a/diff_ras/polygon.py b/diff_ras/polygon.py new file mode 100644 index 0000000000000000000000000000000000000000..7d97865e5719cac94f664dc839ada2e6cd40d68d --- /dev/null +++ b/diff_ras/polygon.py @@ -0,0 +1,182 @@ +import native_rasterizer +import torch +import torch.nn as nn +from torch.autograd import Function + +MODE_BOUNDARY = "boundary" +MODE_MASK = "mask" +MODE_HARD_MASK = "hard_mask" + +MODE_MAPPING = {MODE_BOUNDARY: 0, MODE_MASK: 1, MODE_HARD_MASK: 2} + + +class SoftPolygonFunction(Function): + @staticmethod + def forward(ctx, vertices, width, height, inv_smoothness=1.0, mode=MODE_BOUNDARY): + ctx.width = width + ctx.height = height + ctx.inv_smoothness = inv_smoothness + ctx.mode = MODE_MAPPING[mode] + + vertices = vertices.clone() + ctx.device = vertices.device + ctx.batch_size, ctx.number_vertices = vertices.shape[:2] + + rasterized = torch.FloatTensor(ctx.batch_size, ctx.height, ctx.width).fill_(0.0).to(device=ctx.device) + + contribution_map = torch.IntTensor(ctx.batch_size, ctx.height, ctx.width).fill_(0).to(device=ctx.device) + rasterized, contribution_map = native_rasterizer.forward_rasterize( + vertices, rasterized, contribution_map, width, height, inv_smoothness, ctx.mode + ) + ctx.save_for_backward(vertices, rasterized, contribution_map) + + return rasterized # , contribution_map + + @staticmethod + def backward(ctx, grad_output): + vertices, rasterized, contribution_map = ctx.saved_tensors + + grad_output = grad_output.contiguous() + + # grad_vertices = torch.FloatTensor( + # ctx.batch_size, ctx.height, ctx.width, ctx.number_vertices, 2).fill_(0.0).to(device=ctx.device) + grad_vertices = torch.FloatTensor(ctx.batch_size, ctx.number_vertices, 2).fill_(0.0).to(device=ctx.device) + grad_vertices = native_rasterizer.backward_rasterize( + vertices, + rasterized, + contribution_map, + grad_output, + grad_vertices, + ctx.width, + ctx.height, + ctx.inv_smoothness, + ctx.mode, + ) + + return grad_vertices, None, None, None, None + + +class SoftPolygon(nn.Module): + MODES = [MODE_BOUNDARY, MODE_MASK, MODE_HARD_MASK] + + def __init__(self, inv_smoothness=1.0, mode=MODE_BOUNDARY): + super(SoftPolygon, self).__init__() + + self.inv_smoothness = inv_smoothness + + if mode not in SoftPolygon.MODES: + raise ValueError("invalid mode: {0}".format(mode)) + + self.mode = mode + + def forward(self, vertices, width, height, p, color=False): + return SoftPolygonFunction.apply(vertices, width, height, self.inv_smoothness, self.mode) + + +def pnp(vertices, width, height): + device = vertices.device + batch_size = vertices.size(0) + polygon_dimension = vertices.size(1) + + y_index = torch.arange(0, height).to(device) + x_index = torch.arange(0, width).to(device) + + grid_y, grid_x = torch.meshgrid(y_index, x_index) + xp = grid_x.unsqueeze(0).repeat(batch_size, 1, 1).float() + yp = grid_y.unsqueeze(0).repeat(batch_size, 1, 1).float() + + result = torch.zeros((batch_size, height, width)).bool().to(device) + + j = polygon_dimension - 1 + for vn in range(polygon_dimension): + from_x = vertices[:, vn, 0].unsqueeze(-1).unsqueeze(-1).repeat(1, height, width) + from_y = vertices[:, vn, 1].unsqueeze(-1).unsqueeze(-1).repeat(1, height, width) + + to_x = vertices[:, j, 0].unsqueeze(-1).unsqueeze(-1).repeat(1, height, width) + to_y = vertices[:, j, 1].unsqueeze(-1).unsqueeze(-1).repeat(1, height, width) + + has_condition = torch.logical_and( + (from_y > yp) != (to_y > yp), xp < ((to_x - from_x) * (yp - from_y) / (to_y - from_y) + from_x) + ) + + if has_condition.any(): + result[has_condition] = ~result[has_condition] + + j = vn + + signed_result = -torch.ones((batch_size, height, width), device=device) + signed_result[result] = 1.0 + + return signed_result + + +# used for verification purposes only. +class SoftPolygonPyTorch(nn.Module): + def __init__(self, inv_smoothness=1.0): + super(SoftPolygonPyTorch, self).__init__() + + self.inv_smoothness = inv_smoothness + + # vertices is N x P x 2 + def forward(self, vertices, width, height, p, color=False): + device = vertices.device + batch_size = vertices.size(0) + polygon_dimension = vertices.size(1) + + inside_outside = pnp(vertices, width, height) + + # discrete points we will sample from. + y_index = torch.arange(0, height).to(device) + x_index = torch.arange(0, width).to(device) + + grid_y, grid_x = torch.meshgrid(y_index, x_index) + grid_x = grid_x.unsqueeze(0).repeat(batch_size, 1, 1).float() + grid_y = grid_y.unsqueeze(0).repeat(batch_size, 1, 1).float() + + # do this "per dimension" + distance_segments = [] + over_segments = [] + for from_index in range(polygon_dimension): + segment_result = torch.zeros((batch_size, height, width)).to(device) + from_vertex = vertices[:, from_index].unsqueeze(-1).unsqueeze(-1) + + if from_index == (polygon_dimension - 1): + to_vertex = vertices[:, 0].unsqueeze(-1).unsqueeze(-1) + else: + to_vertex = vertices[:, from_index + 1].unsqueeze(-1).unsqueeze(-1) + + x2_sub_x1 = to_vertex[:, 0] - from_vertex[:, 0] + y2_sub_y1 = to_vertex[:, 1] - from_vertex[:, 1] + square_segment_length = x2_sub_x1 * x2_sub_x1 + y2_sub_y1 * y2_sub_y1 + 0.00001 + + # figure out if this is a major/minor segment (todo?) + x_sub_x1 = grid_x - from_vertex[:, 0] + y_sub_y1 = grid_y - from_vertex[:, 1] + x_sub_x2 = grid_x - to_vertex[:, 0] + y_sub_y2 = grid_y - to_vertex[:, 1] + + # dot between the given point and first vertex and first vertex and second vertex. + dot = ((x_sub_x1 * x2_sub_x1) + (y_sub_y1 * y2_sub_y1)) / square_segment_length + + # needlessly computed sometimes. + x_proj = grid_x - (from_vertex[:, 0] + dot * x2_sub_x1) + y_proj = grid_y - (from_vertex[:, 1] + dot * y2_sub_y1) + + from_closest = dot < 0 + to_closest = dot > 1 + interior_closest = (dot >= 0) & (dot <= 1) + + segment_result[from_closest] = x_sub_x1[from_closest] ** 2 + y_sub_y1[from_closest] ** 2 + segment_result[to_closest] = x_sub_x2[to_closest] ** 2 + y_sub_y2[to_closest] ** 2 + segment_result[interior_closest] = x_proj[interior_closest] ** 2 + y_proj[interior_closest] ** 2 + + distance_map = -segment_result + distance_segments.append(distance_map) + + signed_map = torch.sigmoid(-distance_map * inside_outside / self.inv_smoothness) + over_segments.append(signed_map) + + F_max, F_arg = torch.max(torch.stack(distance_segments, dim=-1), dim=-1) + F_theta = torch.gather(torch.stack(over_segments, dim=-1), dim=-1, index=F_arg.unsqueeze(-1))[..., 0] + + return F_theta diff --git a/diff_ras/rasterize_cuda.cpp b/diff_ras/rasterize_cuda.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cc3f8c0f86798900ee0134a9bde5e5c8fc01ba89 --- /dev/null +++ b/diff_ras/rasterize_cuda.cpp @@ -0,0 +1,49 @@ +#include +#include +#include + +#include "rasterize_cuda_kernel.h" + +// C++ interface +#define CHECK_CUDA(x) TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::vector forward_rasterize( + at::Tensor vertices, + at::Tensor rasterized, + at::Tensor contribution_map, + int width, + int height, + float inv_smoothness, + int mode) { + CHECK_INPUT(vertices); + CHECK_INPUT(rasterized); + CHECK_INPUT(contribution_map); + + return forward_rasterize_cuda(vertices, rasterized, contribution_map, width, height, inv_smoothness, mode); +} + +at::Tensor backward_rasterize( + at::Tensor vertices, + at::Tensor rasterized, + at::Tensor contribution_map, + at::Tensor grad_output, + at::Tensor grad_vertices, + int width, + int height, + float inv_smoothness, + int mode) { + CHECK_INPUT(vertices); + CHECK_INPUT(rasterized); + CHECK_INPUT(contribution_map); + CHECK_INPUT(grad_output); + CHECK_INPUT(grad_vertices); + + return backward_rasterize_cuda(vertices, rasterized, contribution_map, grad_output, grad_vertices, width, height, inv_smoothness, mode); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward_rasterize", &forward_rasterize, "forward rasterize (CUDA)"); + m.def("backward_rasterize", &backward_rasterize, "backward rasterize (CUDA)"); +} diff --git a/diff_ras/rasterize_cuda_kernel.cu b/diff_ras/rasterize_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..83364097cb5f6d899458be371b01c83bae07fea0 --- /dev/null +++ b/diff_ras/rasterize_cuda_kernel.cu @@ -0,0 +1,483 @@ +#include + +#include +#include + +#include "rasterize_cuda_kernel.h" + +const int CUDA_NUM_THREADS = 1024; + +inline int GET_BLOCKS(const int N){ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + +// for the older gpus atomicAdd with double arguments does not exist +#if __CUDA_ARCH__ < 600 and defined(__CUDA_ARCH__) +static __inline__ __device__ double atomicAdd(double* address, double val) { + unsigned long long int* address_as_ull = (unsigned long long int*)address; + unsigned long long int old = *address_as_ull, assumed; + do { + assumed = old; + old = atomicCAS(address_as_ull, assumed, + __double_as_longlong(val + __longlong_as_double(assumed))); + // Note: uses integer comparison to avoid hang in case of NaN (since NaN != NaN) } while (assumed != old); + } while (assumed != old); + return __longlong_as_double(old); +} +#endif + +#define GET_DIRECT_4d(data, x0, x1, x2, x3, sd0, sd1, sd2, sd3) \ + ((data)[(x0) * (sd0) + (x1) * (sd1) + (x2) * (sd2) + (x3) * (sd3)]) + +#define ADD_ATOMIC_4d(data, x0, x1, x2, x3, sd0, sd1, sd2, sd3, v) \ + atomicAdd( data + (x0) * (sd0) + (x1) * (sd1) + (x2) * (sd2) + (x3) * (sd3), v ) +#define ADD_ATOMIC_5d(data, x0, x1, x2, x3, x4, sd0, sd1, sd2, sd3, sd4, v) \ + atomicAdd( data + (x0) * (sd0) + (x1) * (sd1) + (x2) * (sd2) + (x3) * (sd3) + (x4)*(sd4), v ) +#define SET_DIRECT_4d(data, x0, x1, x2, x3, sd0, sd1, sd2, sd3, v) \ + ((data)[(x0) * (sd0) + (x1) * (sd1) + (x2) * (sd2) + (x3) * (sd3)]) = v + +#define GET_DIRECT_3d(data, x0, x1, x2, sd0, sd1, sd2) \ + ((data)[(x0) * (sd0) + (x1) * (sd1) + (x2) * (sd2)]) + +#define SET_DIRECT_3d(data, x0, x1, x2, sd0, sd1, sd2, v) \ + ((data)[(x0) * (sd0) + (x1) * (sd1) + (x2) * (sd2) ]) = v + +#define GET_DIRECT_5d(data, x0, x1, x2, x3, x4, stride0, stride1, stride2, stride3, stride4) \ + ((data)[(x0)*(stride0)+(x1)*(stride1)+(x2)*(stride2)+(x3)*(stride3)+(x4)*(stride4)]) + +#define SET_DIRECT_5d(data, x0, x1, x2, x3, x4, stride0, stride1, stride2, stride3, stride4, value) \ + ((data)[(x0)*(stride0)+(x1)*(stride1)+(x2)*(stride2)+(x3)*(stride3)+(x4)*(stride4)] = (value)) + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); i += blockDim.x * gridDim.x) + + +const int MODE_BOUNDARY = 0; +const int MODE_MASK = 1; +const int MODE_HARD_MASK = 2; + +template +__global__ void inside_outside_cuda_kernel( + const scalar_t* __restrict__ vertices, + int batch_size, + int number_vertices, + scalar_t* rasterized, + int height, + int width) { + // 1-D array of 1-D blocks. + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= batch_size * width * height) { + return; + } + + const int w = width; + const int h = height; + const int nv = number_vertices; + + // batch index. + const int bi = i / (w * h); + + // pixel number (linear index) + const int pn = i % (w * h); + const int yp = pn / w; + const int xp = pn % w; + + // cast a ray: William Randolph Franklin. + int j = 0; + scalar_t c = 0; + for (int vn = 0, j = nv - 1; vn < nv; j = vn++) { + scalar_t from_x; + scalar_t from_y; + scalar_t to_x; + scalar_t to_y; + + from_x = vertices[bi * (nv * 2) + vn * 2]; + from_y = vertices[bi * (nv * 2) + vn * 2 + 1]; + to_x = vertices[bi * (nv * 2) + j * 2]; + to_y = vertices[bi * (nv * 2) + j * 2 + 1]; + + if (((from_y > yp) != (to_y > yp)) && (xp < (to_x - from_x) * (yp - from_y) / (to_y - from_y) + from_x)) { + c = !c; + } + } + + rasterized[i] = c == 0 ? -1.0 : 1.0; +} + +template +__global__ void forward_rasterize_cuda_kernel( + const scalar_t* __restrict__ vertices, + int batch_size, + int number_vertices, + scalar_t* rasterized, + int* contribution_map, + int height, + int width, + float inv_smoothness, + int mode) { + // 1-D array of 1-D blocks. + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= batch_size * width * height) { + return; + } + + const int w = width; + const int h = height; + const int nv = number_vertices; + + // batch index. + const int bi = i / (w * h); + + // pixel number (linear index) + const int pn = i % (w * h); + const int yp = pn / w; + const int xp = pn % w; + + // go through each vertex. + // at some point, we'll need to record + // which segment contributed the most + // for backwards pass. + scalar_t max_contribution = -2147483647; + int max_vertex_number = -1; + for (int vn = 0; vn < nv; vn++) { + int from_index; + int to_index; + scalar_t from_x; + scalar_t from_y; + scalar_t to_x; + scalar_t to_y; + scalar_t x2_sub_x1; + scalar_t y2_sub_y1; + scalar_t square_segment_length; + scalar_t x_sub_x1; + scalar_t y_sub_y1; + scalar_t x_sub_x2; + scalar_t y_sub_y2; + scalar_t dot; + scalar_t x_proj; + scalar_t y_proj; + scalar_t contribution; + + // grid_x, grid_y = xp, yp. + from_index = vn; + to_index = (vn + 1) % number_vertices; + + from_x = vertices[bi * (nv * 2) + from_index * 2]; + from_y = vertices[bi * (nv * 2) + from_index * 2 + 1]; + + to_x = vertices[bi * (nv * 2) + to_index * 2]; + to_y = vertices[bi * (nv * 2) + to_index * 2 + 1]; + + x2_sub_x1 = to_x - from_x; + y2_sub_y1 = to_y - from_y; + + square_segment_length = x2_sub_x1 * x2_sub_x1 + y2_sub_y1 * y2_sub_y1 + 0.00001; + + x_sub_x1 = xp - from_x; + y_sub_y1 = yp - from_y; + x_sub_x2 = xp - to_x; + y_sub_y2 = yp - to_y; + + dot = ((x_sub_x1 * x2_sub_x1) + (y_sub_y1 * y2_sub_y1)) / square_segment_length; + x_proj = xp - (from_x + dot * x2_sub_x1); + y_proj = yp - (from_y + dot * y2_sub_y1); + + // Does it matter here to compute the squared distance or true Euclidean distance? + if (dot < 0) { + contribution = pow(x_sub_x1, 2) + pow(y_sub_y1, 2); + } + else if (dot > 1) { + contribution = pow(x_sub_x2, 2) + pow(y_sub_y2, 2); + } + else { + contribution = pow(x_proj, 2) + pow(y_proj, 2); + } + + // we need contribution to be a decreasing function. + // if (mode == MODE_MASK) { + // // sign * -dist + // contribution = 1.0 / (1.0 + exp(-rasterized[i] * contribution / inv_smoothness)); + // } + // else if (mode == MODE_HARD_MASK) { + // // map the inside outside map to 0 or 1.0. + // // technically, we don't need this preceeding loop. + // contribution = rasterized[i] < 0 ? 0.0 : 1.0; + // } + // else { + // contribution = exp(-contribution / inv_smoothness); + // } + + contribution = -contribution; + + if (contribution > max_contribution) { + max_contribution = contribution; + max_vertex_number = vn; + } + } + + if (mode == MODE_MASK) { + // sign * -dist + max_contribution = 1.0 / (1.0 + exp(rasterized[i] * max_contribution / inv_smoothness)); + } + else if (mode == MODE_HARD_MASK) { + // map the inside outside map to 0 or 1.0. + // technically, we don't need this preceeding loop. + max_contribution = rasterized[i] < 0 ? 0.0 : 1.0; + } + else { + max_contribution = exp(max_contribution / inv_smoothness); + } + + rasterized[i] = max_contribution; + contribution_map[i] = max_vertex_number; +} + +template +__global__ void backward_rasterize_cuda_kernel( + const scalar_t* __restrict__ vertices, + const scalar_t* __restrict__ rasterized, + const int* __restrict__ contribution_map, + const scalar_t* __restrict__ grad_output, + scalar_t* grad_vertices, + int batch_size, + int number_vertices, + int width, + int height, + float inv_smoothness) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= batch_size * width * height) { + return; + } + + const int w = width; + const int h = height; + const int nv = number_vertices; + + // batch index. + const int bi = i / (w * h); + + // pixel number (linear index) + const int pn = i % (w * h); + const int yp = pn / w; + const int xp = pn % w; + + // produce dR/dv. + // since we use max over all vertices, we only need + // to apply it to single vertex. + int vn; + int from_index; + int to_index; + scalar_t from_x; + scalar_t from_y; + scalar_t to_x; + scalar_t to_y; + scalar_t x2_sub_x1; + scalar_t y2_sub_y1; + scalar_t square_segment_length; + scalar_t x_sub_x1; + scalar_t y_sub_y1; + scalar_t x_sub_x2; + scalar_t y_sub_y2; + scalar_t dot; + scalar_t x_proj; + scalar_t y_proj; + scalar_t grad_x1 = 0.0; + scalar_t grad_y1 = 0.0; + scalar_t grad_x2 = 0.0; + scalar_t grad_y2 = 0.0; + + scalar_t in_out = rasterized[i] >= 0.5 ? 1.0 : -1.0; + + vn = contribution_map[i]; + from_index = vn; + to_index = (vn + 1) % nv; + + // determine how we computed the distance to this segment. + from_x = vertices[bi * (nv * 2) + from_index * 2]; + from_y = vertices[bi * (nv * 2) + from_index * 2 + 1]; + + to_x = vertices[bi * (nv * 2) + to_index * 2]; + to_y = vertices[bi * (nv * 2) + to_index * 2 + 1]; + + x2_sub_x1 = to_x - from_x; + y2_sub_y1 = to_y - from_y; + + // grad: + // dX1 = 2 * x2_sub_x1 * -1 + // dX2 = 2 * x2_sub_x1 + // dY1 = 2 * y2_sub_y1 * -1 + // dY2 = 2 * y2_sub_y1 + // possible this could NaN? + square_segment_length = x2_sub_x1 * x2_sub_x1 + y2_sub_y1 * y2_sub_y1 + 0.00001; + + x_sub_x1 = xp - from_x; + y_sub_y1 = yp - from_y; + x_sub_x2 = xp - to_x; + y_sub_y2 = yp - to_y; + + // grad numer: + // dX1 = -1 * x2_sub_x1 + -1 * x_sub_x1 + // dX2 = x_sub_x1 + scalar_t dot_num = ((x_sub_x1 * x2_sub_x1) + (y_sub_y1 * y2_sub_y1)); + dot = dot_num / square_segment_length; + x_proj = xp - (from_x + dot * x2_sub_x1); + y_proj = yp - (from_y + dot * y2_sub_y1); + + // negative sign? + if (dot < 0) { + // contribution = exp(-((xp - from_x) ** 2 + (yp - from_y) ** 2 / inv_smoothness) + // grad_x1 = (rasterized[i] * 2 * x_sub_x1) / inv_smoothness; + // grad_y1 = (rasterized[i] * 2 * y_sub_y1) / inv_smoothness; + // grad_x1 = in_out * rasterized[i] * (1.0 - rasterized[i]) * 2 * x_sub_x1 / inv_smoothness; + // grad_y1 = in_out * rasterized[i] * (1.0 - rasterized[i]) * 2 * y_sub_y1 / inv_smoothness; + grad_x1 = in_out * rasterized[i] * (1.0 - rasterized[i]) * -2 * x_sub_x1 / inv_smoothness; + grad_y1 = in_out * rasterized[i] * (1.0 - rasterized[i]) * -2 * y_sub_y1 / inv_smoothness; + } + else if (dot > 1) { + // contribution = exp(-((xp - to_x) ** 2 + (yp - to_y) ** 2) / inv_smoothness) + // grad_x2 = (rasterized[i] * 2 * x_sub_x2) / inv_smoothness; + // grad_y2 = (rasterized[i] * 2 * y_sub_y2) / inv_smoothness; + grad_x2 = in_out * rasterized[i] * (1.0 - rasterized[i]) * -2 * x_sub_x2 / inv_smoothness; + grad_y2 = in_out * rasterized[i] * (1.0 - rasterized[i]) * -2 * y_sub_y2 / inv_smoothness; + } + else { + // contribution = exp(-(xp - from_x) ** 2 / inv_smoothness) + scalar_t ss_x1 = -2.0 * x2_sub_x1; + scalar_t ss_x2 = 2.0 * x2_sub_x1; + scalar_t ss_y1 = -2.0 * y2_sub_y1; + scalar_t ss_y2 = 2.0 * y2_sub_y1; + + scalar_t dot_x1 = (square_segment_length * (-x2_sub_x1 - x_sub_x1) - dot_num * ss_x1) / pow(square_segment_length, 2); + scalar_t dot_x2 = (square_segment_length * x_sub_x1 - dot_num * ss_x2) / pow(square_segment_length, 2); + scalar_t dot_y1 = (square_segment_length * (-y2_sub_y1 - y_sub_y1) - dot_num * ss_y1) / pow(square_segment_length, 2); + scalar_t dot_y2 = (square_segment_length * y_sub_y1 - dot_num * ss_y2) / pow(square_segment_length, 2); + + // d/dx() + scalar_t x_proj_x1 = -1 - dot_x1 * x2_sub_x1 + dot; + scalar_t x_proj_x2 = -(dot_x2 * x2_sub_x1 + dot); + + scalar_t y_proj_y1 = -1 - dot_y1 * y2_sub_y1 + dot; + scalar_t y_proj_y2 = -(dot_y2 * y2_sub_y1 + dot); + + // we also need mixed. + scalar_t y_proj_x1 = -dot_x1 * y2_sub_y1; + scalar_t y_proj_x2 = -dot_x2 * y2_sub_y1; + scalar_t x_proj_y1 = -dot_y1 * x2_sub_x1; + scalar_t x_proj_y2 = -dot_y2 * x2_sub_x1; + + // - as well? + grad_x1 = in_out * rasterized[i] * (1.0 - rasterized[i]) * (2.0 * x_proj * x_proj_x1 + 2.0 * y_proj * y_proj_x1) / inv_smoothness; + grad_x2 = in_out * rasterized[i] * (1.0 - rasterized[i]) * (2.0 * x_proj * x_proj_x2 + 2.0 * y_proj * y_proj_x2) / inv_smoothness; + grad_y1 = in_out * rasterized[i] * (1.0 - rasterized[i]) * (2.0 * x_proj * x_proj_y1 + 2.0 * y_proj * y_proj_y1) / inv_smoothness; + grad_y2 = in_out * rasterized[i] * (1.0 - rasterized[i]) * (2.0 * x_proj * x_proj_y2 + 2.0 * y_proj * y_proj_y2) / inv_smoothness; + + // grad_x1 = -rasterized[i] * (2.0 * x_proj * x_proj_x1 + 2.0 * y_proj * y_proj_x1) / inv_smoothness; + // grad_x2 = -rasterized[i] * (2.0 * x_proj * x_proj_x2 + 2.0 * y_proj * y_proj_x2) / inv_smoothness; + // grad_y1 = -rasterized[i] * (2.0 * x_proj * x_proj_y1 + 2.0 * y_proj * y_proj_y1) / inv_smoothness; + // grad_y2 = -rasterized[i] * (2.0 * x_proj * x_proj_y2 + 2.0 * y_proj * y_proj_y2) / inv_smoothness; + } + + // apply the input gradients. + grad_x1 = grad_x1 * grad_output[i]; + grad_x2 = grad_x2 * grad_output[i]; + grad_y1 = grad_y1 * grad_output[i]; + grad_y2 = grad_y2 * grad_output[i]; + + // grad_vertices[bi * (height * width * nv * 2) + yp * (width * nv * 2) + xp * (nv * 2) + from_index * 2] = grad_x1; + // grad_vertices[bi * (height * width * nv * 2) + yp * (width * nv * 2) + xp * (nv * 2) + from_index * 2 + 1] = grad_y1; + // grad_vertices[bi * (height * width * nv * 2) + yp * (width * nv * 2) + xp * (nv * 2) + to_index * 2] = grad_x2; + // grad_vertices[bi * (height * width * nv * 2) + yp * (width * nv * 2) + xp * (nv * 2) + to_index * 2 + 1] = grad_y2; + + // unsure if should be deferencing. + atomicAdd(grad_vertices + bi * nv * 2 + from_index * 2, grad_x1); + atomicAdd(grad_vertices + bi * nv * 2 + from_index * 2 + 1, grad_y1); + atomicAdd(grad_vertices + bi * nv * 2 + to_index * 2, grad_x2); + atomicAdd(grad_vertices + bi * nv * 2 + to_index * 2 + 1, grad_y2); +} + +std::vector forward_rasterize_cuda( + at::Tensor vertices, + at::Tensor rasterized, + at::Tensor contribution_map, + int width, + int height, + float inv_smoothness, + int mode) { + const auto batch_size = vertices.size(0); + const auto number_vertices = vertices.size(1); + const int threads = 512; + + // each block processes some 512 sized chunk of the output image. + const dim3 blocks ((batch_size * width * height - 1) / threads + 1); + + if ((mode == MODE_MASK) || (mode == MODE_HARD_MASK)) { + // determine whether each point is inside or outside. + AT_DISPATCH_FLOATING_TYPES(vertices.type(), "inside_outside_cuda", ([&] { + inside_outside_cuda_kernel<<>>( + vertices.data(), + batch_size, + number_vertices, + rasterized.data(), + height, + width); + })); + } + + if (mode != MODE_HARD_MASK) { + AT_DISPATCH_FLOATING_TYPES(vertices.type(), "forward_rasterize_cuda", ([&] { + forward_rasterize_cuda_kernel<<>>( + vertices.data(), + batch_size, + number_vertices, + rasterized.data(), + contribution_map.data(), + height, + width, + inv_smoothness, + mode); + })); + } + + cudaError_t err = cudaGetLastError(); + err = cudaGetLastError(); + if (err != cudaSuccess) + printf("Error in forward_rasterize: %s\n", cudaGetErrorString(err)); + + return { rasterized, contribution_map }; +} + +at::Tensor backward_rasterize_cuda( + at::Tensor vertices, + at::Tensor rasterized, + at::Tensor contribution_map, + at::Tensor grad_output, + at::Tensor grad_vertices, + int width, + int height, + float inv_smoothness, + int mode) { + const auto batch_size = vertices.size(0); + const auto number_vertices = vertices.size(1); + const int threads = 512; + const dim3 blocks ((batch_size * width * height - 1) / threads + 1); + + AT_DISPATCH_FLOATING_TYPES(vertices.type(), "backward_rasterize_cuda", ([&] { + backward_rasterize_cuda_kernel<<>>( + vertices.data(), + rasterized.data(), + contribution_map.data(), + grad_output.data(), + grad_vertices.data(), + batch_size, + number_vertices, + width, + height, + inv_smoothness); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + printf("Error in backward_rasterize: %s\n", cudaGetErrorString(err)); + + return grad_vertices; +} diff --git a/diff_ras/rasterize_cuda_kernel.h b/diff_ras/rasterize_cuda_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..152ddb5cb6037182aa09d55dcbb4bede7d1185f4 --- /dev/null +++ b/diff_ras/rasterize_cuda_kernel.h @@ -0,0 +1,26 @@ +#ifdef __cplusplus +extern "C" { + #endif + + // CUDA forward declarations + std::vector forward_rasterize_cuda(at::Tensor vertices, + at::Tensor rasterized, + at::Tensor contribution_map, + int width, + int height, + float inv_smoothness, + int mode); + + at::Tensor backward_rasterize_cuda(at::Tensor vertices, + at::Tensor rasterized, + at::Tensor contribution_map, + at::Tensor grad_output, + at::Tensor grad_vertices, + int width, + int height, + float inv_smoothness, + int mode); + + #ifdef __cplusplus +} +#endif diff --git a/diff_ras/setup.py b/diff_ras/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..3c9b3d3eec101d8e199700aa16abc755692393f3 --- /dev/null +++ b/diff_ras/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +setup( + name="rasterizer", + ext_modules=[ + CUDAExtension( + "native_rasterizer", + [ + "rasterize_cuda.cpp", + "rasterize_cuda_kernel.cu", + ], + ), + ], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/engine.py b/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..7a963282afb3554551235649b2e7188fbbef23b2 --- /dev/null +++ b/engine.py @@ -0,0 +1,936 @@ +import copy +import json +import math +import os +import sys +from typing import Iterable + +import cv2 +import numpy as np +import torch +from shapely.geometry import Polygon + +import util.misc as utils + +# Add evaluations folder relative to this file's location +sys.path.append(os.path.join(os.path.dirname(__file__), "evaluations")) +from rplan_eval.Evaluator import Evaluator_RPlan +from s3d_floorplan_eval.DataRW.S3DRW import S3DRW +from s3d_floorplan_eval.DataRW.wrong_annotatios import wrong_s3d_annotations_list +from s3d_floorplan_eval.Evaluator.Evaluator import Evaluator +from s3d_floorplan_eval.options import MCSSOptions + +from datasets import get_dataset_class_labels +from util.eval_utils import compute_f1 +from util.plot_utils import ( + concat_floorplan_maps, + plot_density_map, + plot_floorplan_with_regions, + plot_semantic_rich_floorplan_opencv, + sort_polygons_by_matching, +) +from util.poly_ops import pad_gt_polys + +options = MCSSOptions() +opts = options.parse() + + +def train_one_epoch( + model: torch.nn.Module, + criterion: torch.nn.Module, + data_loader: Iterable, + optimizer: torch.optim.Optimizer, + device: torch.device, + epoch: int, + max_norm: float = 0, + poly2seq: bool = False, + ema_model=None, + **kwargs, +): + """ + Trains the model for one epoch using the provided data loader, criterion, and optimizer. + This function iterates over the data loader, computes losses, performs backpropagation, + applies gradient clipping if specified, updates the model parameters, and optionally + updates an EMA model. It logs various metrics including loss, learning rate, and gradient norm. + Args: + model (torch.nn.Module): The neural network model to be trained. + criterion (torch.nn.Module): The loss criterion used to compute the loss. + data_loader (Iterable): An iterable data loader yielding batches of inputs and extras. + optimizer (torch.optim.Optimizer): The optimizer used to update model parameters. + device (torch.device): The device (CPU or GPU) on which to perform computations. + epoch (int): The current epoch number, used for logging. + max_norm (float, optional): Maximum norm for gradient clipping. If 0, no clipping is applied. Defaults to 0. + poly2seq (bool, optional): If True, uses batched_extras as room_targets and passes them to the model. Defaults to False. + ema_model (optional): Exponential moving average model to update, if provided. Defaults to None. + **kwargs: Additional keyword arguments, such as 'drop_rate' for padding ground truth polygons. + Returns: + dict: A dictionary containing the global averages of logged metrics (e.g., loss, lr, grad_norm). + """ + + model.train() + criterion.train() + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value:.6f}")) + metric_logger.add_meter("grad_norm", utils.SmoothedValue(window_size=1, fmt="{value:.2f}")) + header = "Epoch: [{}]".format(epoch) + print_freq = 10 + model_obj = model if not hasattr(model, "module") else model.module + + for batched_inputs, batched_extras in metric_logger.log_every(data_loader, print_freq, header): + samples = [x["image"].to(device) for x in batched_inputs] + gt_instances = [x["instances"].to(device) for x in batched_inputs] + if not poly2seq: + room_targets = pad_gt_polys( + gt_instances, + model_obj.num_queries_per_poly, + samples[0].shape[1], + drop_rate=kwargs.get("drop_rate", 0.0), + device=device, + ) + outputs = model(samples) + else: + for key in batched_extras.keys(): + batched_extras[key] = batched_extras[key].to(device) + room_targets = batched_extras + outputs = model(samples, batched_extras) + + loss_dict = criterion(outputs, room_targets) + weight_dict = criterion.weight_dict + losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + + loss_dict_unscaled = {f"{k}_unscaled": v for k, v in loss_dict.items()} + loss_dict_scaled = {k: v * weight_dict[k] for k, v in loss_dict.items() if k in weight_dict} + losses_reduced_scaled = sum(loss_dict_scaled.values()) + + loss_value = losses_reduced_scaled.item() + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + print(loss_dict) + sys.exit(1) + + optimizer.zero_grad() + losses.backward() + if max_norm > 0: + grad_total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) + else: + grad_total_norm = utils.get_total_grad_norm(model.parameters(), max_norm) + optimizer.step() + if ema_model is not None: + utils.update_ema(ema_model, model.module, 0.999) + + metric_logger.update(loss=loss_value, **loss_dict_scaled, **loss_dict_unscaled) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + metric_logger.update(grad_norm=grad_total_norm) + + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def evaluate( + model, + criterion, + dataset_name, + data_loader, + device, + plot_density=False, + output_dir=None, + epoch=None, + poly2seq: bool = False, + add_cls_token=False, + per_token_sem_loss=False, + wd_as_line=True, +): + """ + Evaluates the model on a given dataset during training, computing losses and various metrics such as room IoU, + precision, recall, corner precision/recall, angle precision/recall, and semantic metrics if applicable. + + This function supports two evaluation modes: + - Non-poly2seq mode (poly2seq=False): RoomFormer evaluation with fg_mask filtering + - Poly2seq mode (poly2seq=True): Raster2Seq evaluation with sequence-based predictions + + Args: + model: The neural network model to evaluate. + criterion: The loss criterion used for evaluation. + dataset_name (str): Name of the dataset (e.g., "stru3d", "cubicasa", "r2g", "waffle"). + data_loader: DataLoader providing batches of input data. + device: The device (e.g., CPU or GPU) to run the evaluation on. + plot_density (bool, optional): If True, plots a density map for the last sample. Defaults to False. + output_dir (str, optional): Directory to save output plots if plot_density is True. Defaults to None. + epoch (int, optional): Current epoch number, used in plot filenames. Defaults to None. + poly2seq (bool, optional): If True, uses sequence-based prediction mode with forward_inference. + If False, uses mask-based filtering with fg_mask. Defaults to False. + add_cls_token (bool, optional): If True, accounts for class tokens in sequence processing. + Only used when poly2seq=True. Defaults to False. + per_token_sem_loss (bool, optional): If True, computes semantic loss per token using voting. + Only used when poly2seq=True. Defaults to False. + wd_as_line (bool, optional): If True, treats windows/doors as lines based on corner count. + If False, uses semantic class. Only used when poly2seq=True. Defaults to True. + + Returns: + dict: A dictionary containing averaged evaluation statistics, including losses and metrics like room_iou, + room_prec, room_rec, corner_prec, corner_rec, angles_prec, angles_rec, and semantic metrics if applicable. + """ + + model.eval() + criterion.eval() + + if dataset_name == "stru3d": + door_window_index = [16, 17] + elif dataset_name == "cubicasa": + door_window_index = [10, 9] + else: + door_window_index = [] + + metric_logger = utils.MetricLogger(delimiter=" ") + header = "Test:" + model_obj = model if not hasattr(model, "module") else model.module + + for batched_inputs, batched_extras in metric_logger.log_every(data_loader, 10, header): + samples = [x["image"].to(device) for x in batched_inputs] + scene_ids = [x["image_id"] for x in batched_inputs] + gt_instances = [x["instances"].to(device) for x in batched_inputs] + + if not poly2seq: + room_targets = pad_gt_polys( + gt_instances, model_obj.num_queries_per_poly, samples[0].shape[1], drop_rate=0.0, device=device + ) + outputs = model(samples) + else: + for key in batched_extras.keys(): + batched_extras[key] = batched_extras[key].to(device) + room_targets = batched_extras + outputs = model(samples, batched_extras) + + image_size = samples[0].size(2) + loss_dict = criterion(outputs, room_targets) + weight_dict = criterion.weight_dict + + if poly2seq: + outputs = model_obj.forward_inference(samples) + pred_corners = outputs["gen_out"] + fg_mask = None + else: + pred_logits = outputs["pred_logits"] + pred_corners = outputs["pred_coords"] + fg_mask = torch.sigmoid(pred_logits) > 0.5 # select valid corners + + bs = outputs["pred_logits"].shape[0] + + if "pred_room_logits" in outputs: + pred_room_logits = outputs["pred_room_logits"] + prob = torch.nn.functional.softmax(pred_room_logits, -1) + _, pred_room_label = prob[..., :-1].max(-1) + + # process per scene + for i in range(bs): + # Prepare ground truth data + gt_polys, gt_polys_types = [], [] + gt_window_doors = [] + gt_window_doors_types = [] + for gt_poly, gt_id in zip( + gt_instances[i].gt_masks.polygons, gt_instances[i].gt_classes.detach().cpu().tolist() + ): + gt_poly = gt_poly[0].reshape(-1, 2).astype(np.int32) + if gt_id in door_window_index: + gt_window_doors.append(gt_poly) + gt_window_doors_types.append(gt_id) + else: + gt_polys.append(gt_poly) + gt_polys_types.append(gt_id) + + # Create evaluator based on dataset + if dataset_name == "stru3d": + if int(scene_ids[i]) in wrong_s3d_annotations_list: + continue + curr_opts = copy.deepcopy(opts) + curr_opts.scene_id = "scene_0" + str(scene_ids[i]) + curr_data_rw = S3DRW(curr_opts, mode="online_eval") + evaluator = Evaluator(curr_data_rw, curr_opts, disable_overlapping_filter=poly2seq) + elif dataset_name in ["cubicasa", "r2g", "waffle"]: + evaluator = Evaluator_RPlan(disable_overlapping_filter=poly2seq, wd_as_line=wd_as_line) + + print("Running Evaluation for scene %s" % scene_ids[i]) + + room_polys = [] + semantic_rich = "pred_room_logits" in outputs + + if semantic_rich: + room_types = [] + window_doors = [] + window_doors_types = [] + + scene_outputs = _process_predictions( + pred_corners, + i, + semantic_rich, + poly2seq, + fg_mask, + image_size, + pred_room_label if semantic_rich else None, + pred_room_logits if semantic_rich else None, + add_cls_token, + per_token_sem_loss, + wd_as_line, + door_window_index, + dataset_name, + ) + room_polys = scene_outputs["room_polys"] + room_types = scene_outputs["room_types"] + window_doors = scene_outputs["window_doors"] + window_doors_types = scene_outputs["window_doors_types"] + pred_room_label_per_scene = scene_outputs["pred_room_label_per_scene"] + + if dataset_name == "stru3d": + if not semantic_rich: + quant_result_dict_scene = evaluator.evaluate_scene(room_polys=room_polys) + else: + quant_result_dict_scene = evaluator.evaluate_scene( + room_polys=room_polys, + room_types=room_types, + window_door_lines=window_doors, + window_door_lines_types=window_doors_types, + ) + elif dataset_name in ["cubicasa", "r2g", "waffle"]: + if not semantic_rich: + quant_result_dict_scene = evaluator.evaluate_scene( + room_polys=room_polys, + gt_polys=gt_polys, + room_types=None, + gt_polys_types=gt_polys_types, + img_size=(image_size, image_size), + ) + else: + quant_result_dict_scene = evaluator.evaluate_scene( + room_polys=room_polys, + gt_polys=gt_polys, + room_types=room_types, + gt_polys_types=gt_polys_types, + window_door_lines=window_doors, + gt_window_doors_list=gt_window_doors, + window_door_lines_types=window_doors_types, + gt_window_doors_type_list=gt_window_doors_types, + img_size=(image_size, image_size), + ) + + if "room_iou" in quant_result_dict_scene: + metric_logger.update(room_iou=quant_result_dict_scene["room_iou"]) + + metric_logger.update(room_prec=quant_result_dict_scene["room_prec"]) + metric_logger.update(room_rec=quant_result_dict_scene["room_rec"]) + metric_logger.update(corner_prec=quant_result_dict_scene["corner_prec"]) + metric_logger.update(corner_rec=quant_result_dict_scene["corner_rec"]) + metric_logger.update(angles_prec=quant_result_dict_scene["angles_prec"]) + metric_logger.update(angles_rec=quant_result_dict_scene["angles_rec"]) + + if semantic_rich: + metric_logger.update(room_sem_prec=quant_result_dict_scene["room_sem_prec"]) + metric_logger.update(room_sem_rec=quant_result_dict_scene["room_sem_rec"]) + metric_logger.update(window_door_prec=quant_result_dict_scene["window_door_prec"]) + metric_logger.update(window_door_rec=quant_result_dict_scene["window_door_rec"]) + + # plot last sample + if plot_density and len(room_polys) > 0: + pred_room_map = plot_density_map(samples[i], image_size, room_polys, pred_room_label_per_scene) + cv2.imwrite(os.path.join(output_dir, "{}_pred_room_map_{}.png".format(scene_ids[i], epoch)), pred_room_map) + plot_density = False # only plot once + + loss_dict_scaled = {k: v * weight_dict[k] for k, v in loss_dict.items() if k in weight_dict} + loss_dict_unscaled = {f"{k}_unscaled": v for k, v in loss_dict.items()} + metric_logger.update(loss=sum(loss_dict_scaled.values()), **loss_dict_scaled, **loss_dict_unscaled) + + print("Averaged stats:", metric_logger) + + stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + return stats + + +def _process_predictions( + pred_corners, + i, + semantic_rich, + poly2seq, + fg_mask, + image_size, + pred_room_label, + pred_room_logits, + add_cls_token, + per_token_sem_loss, + wd_as_line, + door_window_index, + dataset_name, +): + """ + Processes predictions for room layouts, extracting polygons for rooms, windows, and doors. + This function handles two main modes: non-poly2seq (mask-based) and poly2seq (sequence-based). + It filters and validates predicted corners based on various conditions, computes room types + and window/door types if semantic information is rich, and ensures polygons meet area thresholds. + + Args: + pred_corners (list or tensor): Predicted corner coordinates for scenes or sequences. + i (int): Index of the current scene in the batch. + semantic_rich (bool): Whether to include semantic information (room types, window/door types). + poly2seq (bool): Whether the predictions are in sequence format (poly2seq mode). + fg_mask (tensor): Foreground masks for valid corners per room (used in non-poly2seq mode). + image_size (int): Size of the image (used for scaling coordinates). + pred_room_label (tensor): Predicted room labels for the scene. + pred_room_logits (tensor): Logits for room predictions. + add_cls_token (int): Number of CLS tokens added in sequences (used in poly2seq mode). + per_token_sem_loss (bool): Whether to compute semantic loss per token (affects class aggregation). + wd_as_line (bool): Whether to treat windows/doors as lines (affects classification logic). + door_window_index (list or dict): Indices or mapping for door/window classes. + dataset_name (str): Name of the dataset being processed. + + Returns: + dict: A dictionary containing: + - "room_polys" (list of np.ndarray): List of valid room polygons (scaled and rounded). + - "room_types" (list or None): List of room types if semantic_rich, else None. + - "window_doors" (list or None): List of window/door polygons if semantic_rich, else None. + - "window_doors_types" (list or None): List of window/door types if semantic_rich, else None. + - "pred_room_label_per_scene" (list): Processed room labels for the scene. + """ + + np_softmax = lambda x: np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True) + pred_corners_per_scene = pred_corners[i] + room_polys = [] + + if semantic_rich: + room_types = [] + window_doors = [] + window_doors_types = [] + + pred_room_label_per_scene = pred_room_label[i].cpu().numpy() + pred_room_logit_per_scene = pred_room_logits[i].cpu().numpy() + + # Process predictions based on mode + if not poly2seq: + fg_mask_per_scene = fg_mask[i] + + # process per room + for j in range(fg_mask_per_scene.shape[0]): + fg_mask_per_room = fg_mask_per_scene[j] + pred_corners_per_room = pred_corners_per_scene[j] + valid_corners_per_room = pred_corners_per_room[fg_mask_per_room] + if len(valid_corners_per_room) > 0: + corners = (valid_corners_per_room * (image_size - 1)).cpu().numpy() + corners = np.around(corners).astype(np.int32) + + if not semantic_rich: + # only regular rooms + if len(corners) >= 4 and Polygon(corners).area >= 100: + room_polys.append(corners) + else: + # regular rooms + if len(corners) >= 3 and Polygon(corners).area >= 100: + room_polys.append(corners) + room_types.append(pred_room_label_per_scene[j]) + # window / door + elif len(corners) == 2: + window_doors.append(corners) + window_doors_types.append(pred_room_label_per_scene[j]) + + if not semantic_rich: + pred_room_label_per_scene = len(room_polys) * [-1] + else: + # poly2seq mode: sequence-based processing + all_room_polys = [] + tmp = [] + all_length_list = [0] + + for j in range(len(pred_corners_per_scene)): + if isinstance(pred_corners_per_scene[j], int): + if pred_corners_per_scene[j] == 2 and tmp: # sep + all_room_polys.append(tmp) + all_length_list.append(len(tmp) + 1 + add_cls_token) + tmp = [] + continue + tmp.append(pred_corners_per_scene[j]) + + if len(tmp): + all_room_polys.append(tmp) + all_length_list.append(len(tmp) + 1 + add_cls_token) + + start_poly_indices = np.cumsum(all_length_list) + + final_pred_classes = [] + for j, poly in enumerate(all_room_polys): + if len(poly) < 2: + continue + corners = np.array(poly, dtype=np.float32) * (image_size - 1) + corners = np.around(corners).astype(np.int32) + + if not semantic_rich: + # only regular rooms + if len(corners) >= 4 and Polygon(corners).area >= 100: + room_polys.append(corners) + else: + if per_token_sem_loss: + pred_classes, counts = np.unique( + pred_room_label_per_scene[start_poly_indices[j] : start_poly_indices[j + 1]][:-1], + return_counts=True, + ) + pred_class = pred_classes[np.argmax(counts)] + pred_logit = pred_room_logit_per_scene[start_poly_indices[j] : start_poly_indices[j + 1]][:-1] + else: + pred_class = pred_room_label_per_scene[ + start_poly_indices[j + 1] - 1 + ] # get last cls token in the seq + final_pred_classes.append(pred_class) + + if wd_as_line: + # regular rooms + if len(corners) >= 3 and Polygon(corners).area >= 100: + room_polys.append(corners) + room_types.append(pred_class) + # window / door + elif len(corners) == 2: + window_doors.append(corners) + if ( + door_window_index is not None + and pred_class not in door_window_index + and dataset_name != "r2g" + ): + wd_prob = np_softmax(pred_logit[:, door_window_index].sum(0)) + pred_class = door_window_index[wd_prob.argmax()] + window_doors_types.append(pred_class) + else: + # regular rooms + if door_window_index is not None and pred_class not in door_window_index: + room_polys.append(corners) + room_types.append(pred_class) + else: + window_doors.append(corners) + window_doors_types.append(pred_class) + + if not semantic_rich: + pred_room_label_per_scene = len(all_room_polys) * [-1] + else: + pred_room_label_per_scene = final_pred_classes + + return { + "room_polys": room_polys, + "room_types": room_types if semantic_rich else None, + "window_doors": window_doors if semantic_rich else None, + "window_doors_types": window_doors_types if semantic_rich else None, + "pred_room_label_per_scene": pred_room_label_per_scene, + } + + +@torch.no_grad() +def evaluate_floor( + model, + dataset_name, + data_loader, + device, + output_dir, + plot_pred=True, + plot_density=True, + plot_gt=True, + semantic_rich=False, + save_pred=False, + poly2seq: bool = False, + add_cls_token=False, + per_token_sem_loss=False, + iou_thres=0.5, + wd_as_line=True, +): + """ + Evaluate the model on a given dataset at testing. + + This function supports two evaluation modes: + - Non-poly2seq mode (poly2seq=False): RoomFormer evaluation with fg_mask filtering + - Poly2seq mode (poly2seq=True): Raster2Seq evaluation with sequence-based predictions + + This function processes the dataset in batches, performs inference using the provided model, + and computes quantitative metrics such as precision, recall, and F1-score for rooms, corners, + and angles. It supports semantic-rich evaluation including room types and window/door detection + for specific datasets. Optionally, it can plot predicted floorplans, density maps, and save + predictions as JSON files. + + Args: + model: The trained model to evaluate (e.g., a PyTorch model). + dataset_name (str): Name of the dataset (e.g., "stru3d", "cubicasa", "r2g", "waffle"). + data_loader: PyTorch DataLoader providing batched inputs and extras. + device: Device to run the model on (e.g., torch.device('cuda') or 'cpu'). + output_dir (str): Directory path to save output files (plots, JSONs, results). + plot_pred (bool, optional): If True, plot and save predicted floorplans. Default is True. + plot_density (bool, optional): If True, plot and save density maps. Default is True. + plot_gt (bool, optional): If True, save ground truth images. Default is True. + semantic_rich (bool, optional): If True, perform semantic evaluation including room types and window/door lines. Default is False. + save_pred (bool, optional): If True, save predicted polygons and metrics as JSON files. Default is False. + poly2seq (bool, optional): If True, uses sequence-based prediction mode with forward_inference. + If False, uses mask-based filtering with fg_mask. Defaults to False. + add_cls_token (bool, optional): If True, accounts for class tokens in sequence processing. + Only used when poly2seq=True. Defaults to False. + per_token_sem_loss (bool, optional): If True, computes semantic loss per token using voting. + Only used when poly2seq=True. Defaults to False. + iou_thres (float, optional): IoU threshold for matching predictions to ground truth. Default is 0.5. + wd_as_line (bool, optional): If True, treats windows/doors as lines based on corner count. + If False, uses semantic class. Only used when poly2seq=True. Defaults to True. + + Returns: + None. The function prints the aggregated quantitative results to the console and saves them to 'results.txt' in output_dir. + """ + model.eval() + + if dataset_name == "stru3d": + door_window_index = [16, 17] + elif dataset_name == "cubicasa": + door_window_index = [10, 9] + elif dataset_name == "waffle": + door_window_index = [] # [1, 2] + else: + door_window_index = [] + + metric_category = ["room", "corner", "angles"] + if semantic_rich: + metric_category += ["room_sem", "window_door"] + + quant_result_dict = None + scene_counter = 0 + merge = False # Only used in poly2seq mode for merged plot output + + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + for batched_inputs, batched_extras in data_loader: + samples = [x["image"].to(device) for x in batched_inputs] + scene_ids = [x["image_id"] for x in batched_inputs] + gt_instances = [x["instances"].to(device) for x in batched_inputs] + + image_size = samples[0].size(2) + + # Get predictions based on mode + if poly2seq: + outputs = model.forward_inference(samples) + pred_corners = outputs["gen_out"] + fg_mask = None + else: + outputs = model(samples) + pred_logits = outputs["pred_logits"] + pred_corners = outputs["pred_coords"] + fg_mask = torch.sigmoid(pred_logits) > 0.5 # select valid corners + + bs = outputs["pred_logits"].shape[0] + + if "pred_room_logits" in outputs: + pred_room_logits = outputs["pred_room_logits"] + prob = torch.nn.functional.softmax(pred_room_logits, -1) + _, pred_room_label = prob[..., :-1].max(-1) + + # process per scene + for i in range(bs): + # Prepare ground truth data + gt_polys, gt_polys_types = [], [] + gt_window_doors = [] + gt_window_doors_types = [] + for gt_poly, gt_id in zip( + gt_instances[i].gt_masks.polygons, gt_instances[i].gt_classes.detach().cpu().tolist() + ): + gt_poly = gt_poly[0].reshape(-1, 2).astype(np.int32) + if gt_id in door_window_index: + gt_window_doors.append(gt_poly) + gt_window_doors_types.append(gt_id) + else: + gt_polys.append(gt_poly) + gt_polys_types.append(gt_id) + + # Create evaluator based on dataset + if dataset_name == "stru3d": + if int(scene_ids[i]) in wrong_s3d_annotations_list: + continue + curr_opts = copy.deepcopy(opts) + curr_opts.scene_id = "scene_0" + str(scene_ids[i]) + curr_data_rw = S3DRW(curr_opts, mode="test") + evaluator = Evaluator(curr_data_rw, curr_opts, disable_overlapping_filter=poly2seq) + elif dataset_name in ["cubicasa", "waffle", "r2g"]: + evaluator = Evaluator_RPlan( + disable_overlapping_filter=poly2seq, iou_thres=iou_thres, wd_as_line=wd_as_line + ) + + print("Running Evaluation for scene %s" % scene_ids[i]) + + scene_outputs = _process_predictions( + pred_corners, + i, + semantic_rich, + poly2seq, + fg_mask, + image_size, + pred_room_label if semantic_rich else None, + pred_room_logits if semantic_rich else None, + add_cls_token, + per_token_sem_loss, + wd_as_line, + door_window_index, + dataset_name, + ) + room_polys = scene_outputs["room_polys"] + room_types = scene_outputs["room_types"] + window_doors = scene_outputs["window_doors"] + window_doors_types = scene_outputs["window_doors_types"] + pred_room_label_per_scene = scene_outputs["pred_room_label_per_scene"] + + if dataset_name == "stru3d": + if not semantic_rich: + quant_result_dict_scene = evaluator.evaluate_scene(room_polys=room_polys) + else: + quant_result_dict_scene = evaluator.evaluate_scene( + room_polys=room_polys, + room_types=room_types, + window_door_lines=window_doors, + window_door_lines_types=window_doors_types, + ) + elif dataset_name in ["cubicasa", "waffle", "r2g"]: + if not semantic_rich: + quant_result_dict_scene = evaluator.evaluate_scene( + room_polys=room_polys, + gt_polys=gt_polys, + room_types=None, + gt_polys_types=gt_polys_types, + img_size=(image_size, image_size), + ) + else: + quant_result_dict_scene = evaluator.evaluate_scene( + room_polys=room_polys, + gt_polys=gt_polys, + room_types=room_types, + gt_polys_types=gt_polys_types, + window_door_lines=window_doors, + gt_window_doors_list=gt_window_doors, + window_door_lines_types=window_doors_types, + gt_window_doors_type_list=gt_window_doors_types, + img_size=(image_size, image_size), + ) + + if quant_result_dict is None: + quant_result_dict = quant_result_dict_scene + else: + for k in quant_result_dict.keys(): + quant_result_dict[k] += quant_result_dict_scene[k] + + scene_counter += 1 + + # plot regular room floorplan + gt_room_polys = [np.array(poly) for poly in gt_polys] + room_polys = [np.array(poly) for poly in room_polys] + + if "gt_polys_sorted_indcs" in quant_result_dict_scene: + gt_polys_sorted_indcs = quant_result_dict_scene["gt_polys_sorted_indcs"] + del quant_result_dict_scene["gt_polys_sorted_indcs"] + gt_room_polys = [gt_room_polys[ind] for ind in gt_polys_sorted_indcs] + + if "pred2gt_indices" in quant_result_dict_scene: + pred2gt_indices = quant_result_dict_scene["pred2gt_indices"] + del quant_result_dict_scene["pred2gt_indices"] + room_polys, gt_room_polys, pred_mask, gt_mask = sort_polygons_by_matching( + pred2gt_indices, room_polys, gt_room_polys + ) + else: + pred_mask, gt_mask = None, None + + prec, rec = quant_result_dict_scene["room_prec"], quant_result_dict_scene["room_rec"] + f1 = 2 * prec * rec / (prec + rec + 1e-5) + missing_rate = quant_result_dict_scene["room_missing_ratio"] + plot_statistics = { + "f1": f1, + "prec": prec, + "rec": rec, + "missing_rate": missing_rate, + "num_preds": len(room_polys), + "num_gt": len(gt_polys), + "num_matched_preds": sum([x != -1 for x in pred2gt_indices]), + } + + if plot_pred: + gt_floorplan_map = plot_floorplan_with_regions( + gt_room_polys, matching_labels=gt_mask, base_scale=image_size, scale=1024 + ) + floorplan_map = plot_floorplan_with_regions( + room_polys, matching_labels=pred_mask, base_scale=image_size, scale=1024 + ) + if not merge: + cv2.imwrite(os.path.join(output_dir, "{}_pred_floorplan.png".format(scene_ids[i])), floorplan_map) + cv2.imwrite(os.path.join(output_dir, "{}_gt_floorplan.png".format(scene_ids[i])), gt_floorplan_map) + else: + concatenated_map = concat_floorplan_maps(gt_floorplan_map, floorplan_map, plot_statistics) + cv2.imwrite( + os.path.join(output_dir, "{}_pred_floorplan.png".format(scene_ids[i])), concatenated_map + ) + + if semantic_rich: + _, ID2CLASS_LABEL = get_dataset_class_labels(dataset_name) + floorplan_map = plot_semantic_rich_floorplan_opencv( + zip(room_polys + window_doors, room_types + window_doors_types), + os.path.join(output_dir, "{}_pred_floorplan_sem.png".format(scene_ids[i])), + door_window_index=door_window_index, + semantics_label_mapping=ID2CLASS_LABEL, + img_w=image_size, + img_h=image_size, + scale=1, + plot_text=False, + ) + + if save_pred: + # Save room_polys as JSON + json_path = os.path.join(output_dir, "jsons", "{}.json".format(str(scene_ids[i]).zfill(5))) + os.makedirs(os.path.dirname(json_path), exist_ok=True) + polys_list = [poly.astype(float).tolist() for poly in room_polys] + if semantic_rich: + polys_list += [window_door.astype(float).tolist() for window_door in window_doors] + types_list = room_types + window_doors_types + else: + types_list = [-1] * len(polys_list) + + output_json = [ + { + "image_id": str(scene_ids[i]).zfill(5), + "segmentation": polys_list[instance_id], + "category_id": int(types_list[instance_id]), + "id": instance_id, + } + for instance_id in range(len(polys_list)) + ] + with open(json_path, "w") as json_file: + json.dump(output_json, json_file) + + json_result_path = os.path.join( + output_dir, "result_jsons", "{}.json".format(str(scene_ids[i]).zfill(5)) + ) + new_quant_result_dict_scene = compute_f1(copy.deepcopy(quant_result_dict_scene), metric_category) + os.makedirs(os.path.dirname(json_result_path), exist_ok=True) + with open(json_result_path, "w") as json_file: + json.dump(new_quant_result_dict_scene, json_file) + + if plot_gt: + gt_image = np.transpose(samples[i].cpu().numpy(), (1, 2, 0)) + gt_image = (gt_image * 255).astype(np.uint8) + cv2.imwrite(os.path.join(output_dir, "{}_gt.png".format(scene_ids[i])), gt_image) + + if plot_density: + pred_room_map = plot_density_map( + samples[i], image_size, room_polys, pred_room_label_per_scene, plot_text=False + ) + gt_room_map = plot_density_map(samples[i], image_size, gt_polys, gt_polys_types, plot_text=False) + + if merge: + concatenated_map = concat_floorplan_maps(gt_room_map, pred_room_map, plot_statistics) + cv2.imwrite( + os.path.join(output_dir, "{}_pred_room_map.png".format(scene_ids[i])), concatenated_map + ) + else: + cv2.imwrite(os.path.join(output_dir, "{}_pred_room_map.png".format(scene_ids[i])), pred_room_map) + cv2.imwrite(os.path.join(output_dir, "{}_gt_room_map.png".format(scene_ids[i])), gt_room_map) + + for k in quant_result_dict.keys(): + quant_result_dict[k] /= float(scene_counter) + quant_result_dict = compute_f1(quant_result_dict, metric_category) + + print("*************************************************") + print(quant_result_dict) + print("*************************************************") + + with open(os.path.join(output_dir, "results.txt"), "w") as file: + file.write(json.dumps(quant_result_dict)) + + +def generate( + model, + samples, + semantic_rich=False, + poly2seq: bool = False, + use_cache=True, + per_token_sem_loss=False, + drop_wd=False, +): + """ + Generate room polygons and labels from model predictions. + + This function supports two generation modes: + - Non-poly2seq mode (poly2seq=False): RoomFormer generation with fg_mask filtering + - Poly2seq mode (poly2seq=True): Raster2Seq generation with sequence-based predictions + + Args: + model: The trained model to use for inference. + samples: Input image samples (list of tensors). + semantic_rich (bool, optional): If True, predict room types and window/door elements. Default is False. + poly2seq (bool, optional): If True, uses sequence-based prediction mode with forward_inference. + If False, uses mask-based filtering with fg_mask. Defaults to False. + use_cache (bool, optional): If True, use caching in forward_inference. Only used when poly2seq=True. Default is True. + per_token_sem_loss (bool, optional): If True, computes semantic class using voting across tokens. + Only used when poly2seq=True. Default is False. + drop_wd (bool, optional): If True, exclude window/door elements from output. Default is False. + + Returns: + dict: Dictionary containing: + - 'room': List of room polygons (and optionally window/door lines) per scene + - 'labels': List of class labels per scene + """ + + model.eval() + image_size = samples[0].size(2) + + # Get predictions based on mode + if poly2seq: + outputs = model.forward_inference(samples, use_cache) + pred_corners = outputs["gen_out"] + fg_mask = None + else: + outputs = model(samples) + pred_corners = outputs["pred_coords"] + pred_logits = outputs["pred_logits"] + fg_mask = torch.sigmoid(pred_logits) > 0.5 # select valid corners + + bs = outputs["pred_logits"].shape[0] + + if "pred_room_logits" in outputs: + pred_room_logits = outputs["pred_room_logits"] + prob = torch.nn.functional.softmax(pred_room_logits, -1) + _, pred_room_label = prob[..., :-1].max(-1) + + outputs = [] + output_classes = [] + + # process per scene + for i in range(bs): + room_polys = [] + + if semantic_rich: + room_types = [] + window_doors = [] + window_doors_types = [] + else: + window_doors = None + room_types = None + + scene_outputs = _process_predictions( + pred_corners, + i, + semantic_rich, + poly2seq, + fg_mask, + image_size, + pred_room_label if semantic_rich else None, + pred_room_logits if semantic_rich else None, + False, + per_token_sem_loss, + wd_as_line=True, + door_window_index=None, + dataset_name=None, + ) + room_polys = scene_outputs["room_polys"] + room_types = scene_outputs["room_types"] + window_doors = scene_outputs["window_doors"] + window_doors_types = scene_outputs["window_doors_types"] + + if not drop_wd and window_doors: + outputs.append(room_polys + window_doors) + output_classes.append(room_types + window_doors_types) + else: + outputs.append(room_polys) + output_classes.append(room_types) + + out_dict = {"room": outputs, "labels": output_classes} + return out_dict diff --git a/eval.py b/eval.py new file mode 100644 index 0000000000000000000000000000000000000000..c116641f9ba1f087f084c85e181211cdfe8811c8 --- /dev/null +++ b/eval.py @@ -0,0 +1,314 @@ +import argparse +import copy +import os +import random +from pathlib import Path + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader +from tqdm import trange + +from datasets import build_dataset +from engine import evaluate_floor, generate +from models import build_model + + +def get_args_parser(): + parser = argparse.ArgumentParser("Raster2Seq evaluation script", add_help=False) + parser.add_argument("--batch_size", default=10, type=int) + + parser.add_argument("--debug", action="store_true") + parser.add_argument("--input_channels", default=1, type=int) + parser.add_argument("--image_norm", action="store_true") + parser.add_argument("--eval_every_epoch", type=int, default=20) + parser.add_argument("--ckpt_every_epoch", type=int, default=20) + parser.add_argument("--label_smoothing", type=float, default=0.0) + parser.add_argument("--ignore_index", type=int, default=-1) + parser.add_argument("--image_size", type=int, default=256) + parser.add_argument("--ema4eval", action="store_true") + parser.add_argument("--measure_time", action="store_true") + parser.add_argument("--disable_sampling_cache", action="store_true") + parser.add_argument("--use_anchor", action="store_true") + parser.add_argument("--drop_wd", action="store_true") + parser.add_argument("--iou_thres", type=float, default=0.5) + parser.add_argument("--disable_sem_rich", action="store_true") + parser.add_argument("--wd_only", action="store_true") + parser.add_argument("--disable_image_transform", action="store_true") + parser.add_argument("--num_subset_images", type=int, default=-1) + parser.add_argument("--converter_version", type=str, default="v1") + parser.add_argument("--inject_cls_embed", action="store_true") + + # raster2seq + parser.add_argument("--poly2seq", action="store_true") + parser.add_argument("--seq_len", type=int, default=1024) + parser.add_argument("--num_bins", type=int, default=64) + parser.add_argument("--pre_decoder_pos_embed", action="store_true") + parser.add_argument("--learnable_dec_pe", action="store_true") + parser.add_argument("--dec_qkv_proj", action="store_true") + parser.add_argument("--dec_attn_concat_src", action="store_true") + parser.add_argument("--per_token_sem_loss", action="store_true") + parser.add_argument("--add_cls_token", action="store_true") + + # backbone + parser.add_argument("--backbone", default="resnet50", type=str, help="Name of the convolutional backbone to use") + parser.add_argument("--lr_backbone", default=0, type=float) + parser.add_argument( + "--dilation", + action="store_true", + help="If true, we replace stride with dilation in the last convolutional block (DC5)", + ) + parser.add_argument( + "--position_embedding", + default="sine", + type=str, + choices=("sine", "learned"), + help="Type of positional embedding to use on top of the image features", + ) + parser.add_argument("--position_embedding_scale", default=2 * np.pi, type=float, help="position / size * scale") + parser.add_argument("--num_feature_levels", default=4, type=int, help="number of feature levels") + + # Transformer + parser.add_argument("--enc_layers", default=6, type=int, help="Number of encoding layers in the transformer") + parser.add_argument("--dec_layers", default=6, type=int, help="Number of decoding layers in the transformer") + parser.add_argument( + "--dim_feedforward", + default=1024, + type=int, + help="Intermediate size of the feedforward layers in the transformer blocks", + ) + parser.add_argument( + "--hidden_dim", default=256, type=int, help="Size of the embeddings (dimension of the transformer)" + ) + parser.add_argument("--dropout", default=0.1, type=float, help="Dropout applied in the transformer") + parser.add_argument( + "--nheads", default=8, type=int, help="Number of attention heads inside the transformer's attentions" + ) + parser.add_argument( + "--num_queries", + default=800, + type=int, + help="Number of query slots (num_polys * max. number of corner per poly)", + ) + parser.add_argument("--num_polys", default=20, type=int, help="Number of maximum number of room polygons") + parser.add_argument("--dec_n_points", default=4, type=int) + parser.add_argument("--enc_n_points", default=4, type=int) + + parser.add_argument( + "--query_pos_type", + default="sine", + type=str, + choices=("static", "sine", "none"), + help="Type of query pos in decoder - \ + 1. static: same setting with DETR and Deformable-DETR, the query_pos is the same for all layers \ + 2. sine: since embedding from reference points (so if references points update, query_pos also \ + 3. none: remove query_pos", + ) + parser.add_argument( + "--with_poly_refine", + default=True, + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + parser.add_argument( + "--masked_attn", + default=False, + action="store_true", + help="if true, the query in one room will not be allowed to attend other room", + ) + parser.add_argument( + "--semantic_classes", + default=-1, + type=int, + help="Number of classes for semantically-rich floorplan: \ + 1. default -1 means non-semantic floorplan \ + 2. 19 for Structured3D: 16 room types + 1 door + 1 window + 1 empty", + ) + parser.add_argument( + "--disable_poly_refine", + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + + # aux + parser.add_argument( + "--no_aux_loss", + dest="aux_loss", + action="store_true", + help="Disables auxiliary decoding losses (loss at each layer)", + ) + + # dataset parameters + parser.add_argument("--dataset_name", default="stru3d") + parser.add_argument("--dataset_root", default="data/stru3d", type=str) + parser.add_argument("--eval_set", default="test", type=str) + + parser.add_argument("--device", default="cuda", help="device to use for training / testing") + parser.add_argument("--num_workers", default=2, type=int) + parser.add_argument("--seed", default=42, type=int) + parser.add_argument("--checkpoint", default="checkpoints/roomformer_scenecad.pth", help="resume from checkpoint") + parser.add_argument("--output_dir", default="eval_stru3d", help="path where to save result") + + # visualization options + parser.add_argument("--plot_pred", default=True, type=bool, help="plot predicted floorplan") + parser.add_argument( + "--plot_density", default=True, type=bool, help="plot predicited room polygons overlaid on the density map" + ) + parser.add_argument("--plot_gt", default=True, type=bool, help="plot ground truth floorplan") + parser.add_argument("--save_pred", action="store_true", help="save_pred") + + return parser + + +def main(args): + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + # build dataset and dataloader + dataset_eval = build_dataset(image_set=args.eval_set, args=args) + + tokenizer = None + if args.poly2seq: + args.vocab_size = dataset_eval.get_vocab_size() + tokenizer = dataset_eval.get_tokenizer() + + # overfit one sample + if args.debug: + dataset_eval = torch.utils.data.Subset(dataset_eval, [2]) + dataset_eval[0] + + if args.num_subset_images > 0 and args.num_subset_images < len(dataset_eval): + dataset_eval = torch.utils.data.Subset(dataset_eval, range(args.num_subset_images)) + + sampler_eval = torch.utils.data.SequentialSampler(dataset_eval) + + def trivial_batch_collator(batch): + """ + A batch collator that does nothing. + """ + return batch, None + + data_loader_eval = DataLoader( + dataset_eval, + args.batch_size, + sampler=sampler_eval, + drop_last=False, + collate_fn=trivial_batch_collator, + num_workers=args.num_workers, + pin_memory=True, + ) + + # build model + model = build_model(args, train=False, tokenizer=tokenizer) + model.to(device) + + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print("number of params:", n_parameters) + + for n, p in model.named_parameters(): + print(n) + + output_dir = Path(args.output_dir) + + checkpoint = torch.load(args.checkpoint, map_location="cpu") + if args.ema4eval: + ckpt_state_dict = copy.deepcopy(checkpoint["ema"]) + else: + ckpt_state_dict = copy.deepcopy(checkpoint["model"]) + for key, value in checkpoint["model"].items(): + if key.startswith("module."): + ckpt_state_dict[key[7:]] = checkpoint["model"][key] + del ckpt_state_dict[key] + missing_keys, unexpected_keys = model.load_state_dict(ckpt_state_dict, strict=False) + unexpected_keys = [k for k in unexpected_keys if not (k.endswith("total_params") or k.endswith("total_ops"))] + if len(missing_keys) > 0: + print("Missing Keys: {}".format(missing_keys)) + if len(unexpected_keys) > 0: + print("Unexpected Keys: {}".format(unexpected_keys)) + + # disable grad + for param in model.parameters(): + param.requires_grad = False + + if args.measure_time: + # images = torch.rand(args.batch_size, 3, args.image_size, args.image_size).to(device) + images = ( + torch.from_numpy(np.array(Image.open("data/coco_s3d_bw/val/03006.png").convert("RGB"))) + .permute(2, 0, 1) + .unsqueeze(0) + .to(device) + / 255.0 + ) + # INIT LOGGERS + starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + repetitions = 50 + timings = np.zeros((repetitions, 1)) + if args.poly2seq: + model = torch.compile(model) # compile model is not compatible with RoomFormer + # GPU-WARM-UP + for _ in trange(10, desc="GPU-WARM-UP"): + if not args.poly2seq: + _ = model(images) + else: + _ = model.forward_inference(images) + # MEASURE PERFORMANCE + with torch.no_grad(): + for rep in trange(repetitions): + starter.record() + _ = generate( + model, + images, + semantic_rich=args.semantic_classes > 0, + use_cache=True, + per_token_sem_loss=args.per_token_sem_loss, + drop_wd=args.drop_wd, + poly2seq=args.poly2seq, + ) + ender.record() + # WAIT FOR GPU SYNC + torch.cuda.synchronize() + curr_time = starter.elapsed_time(ender) + timings[rep] = curr_time + mean_syn = np.sum(timings) / repetitions + std_syn = np.std(timings) + print("Inference time: {:.2f}+/-{:.2f}ms".format(mean_syn, std_syn)) + exit(0) + + # save_dir = os.path.join(os.path.dirname(args.checkpoint), output_dir) + # save_dir = os.path.join(output_dir, os.path.dirname(args.checkpoint).split('/')[-1]) + save_dir = output_dir + os.makedirs(save_dir, exist_ok=True) + evaluate_floor( + model, + args.dataset_name, + data_loader_eval, + device, + save_dir, + plot_pred=args.plot_pred, + plot_density=args.plot_density, + plot_gt=args.plot_gt, + semantic_rich=(args.semantic_classes > 0 and not args.disable_sem_rich), + save_pred=args.save_pred, + per_token_sem_loss=args.per_token_sem_loss, + iou_thres=args.iou_thres, + poly2seq=args.poly2seq, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Raster2Seq evaluation script", parents=[get_args_parser()]) + args = parser.parse_args() + + if args.debug: + args.batch_size = 1 + if args.disable_poly_refine: + args.with_poly_refine = False + + main(args) diff --git a/eval_seg.py b/eval_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..9717946ac5926a26f162018c259e6e1037e694ba --- /dev/null +++ b/eval_seg.py @@ -0,0 +1,216 @@ +import inspect +import json +import os +import sys +from os.path import isfile, join, realpath +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader +from tqdm import tqdm + +sys.path.append(os.path.join(os.path.dirname(__file__), "evaluations")) +from clipseg_eval.general_utils import ( + AttributeDict, + filter_args, + get_attribute, + score_config_from_cli_args, +) + +sys.path.append(str(Path(__file__).resolve().parent.parent)) +from datasets import build_dataset +from detectron2.data.detection_utils import annotations_to_instances + +DATASET_CACHE = dict() + + +def load_model( + checkpoint_id, weights_file=None, strict=True, model_args="from_config", with_config=False, ignore_weights=False +): + + config = json.load(open(join("logs", checkpoint_id, "config.json"))) + + if model_args != "from_config" and type(model_args) != dict: + raise ValueError('model_args must either be "from_config" or a dictionary of values') + + model_cls = get_attribute(config["model"]) + + # load model + if model_args == "from_config": + _, model_args, _ = filter_args(config, inspect.signature(model_cls).parameters) + + model = model_cls(**model_args) + + if weights_file is None: + weights_file = realpath(join("logs", checkpoint_id, "weights.pth")) + else: + weights_file = realpath(join("logs", checkpoint_id, weights_file)) + + if isfile(weights_file) and not ignore_weights: + weights = torch.load(weights_file) + for _, w in weights.items(): + assert not torch.any(torch.isnan(w)), "weights contain NaNs" + model.load_state_dict(weights, strict=strict) + else: + if not ignore_weights: + raise FileNotFoundError(f"model checkpoint {weights_file} was not found") + + if with_config: + return model, config + + return model + + +def read_pred_json(json_file_path, image_size=(256, 256), mask_format="bitmask"): + # Read and parse the JSON file + with open(json_file_path, "r") as file: + predictions = json.load(file) + for i, p in enumerate(predictions): + predictions[i]["segmentation"] = [np.array(p["segmentation"]).flatten()] + + pred = annotations_to_instances(predictions, image_size, mask_format, no_boxes=True) + return pred + + +def compute_shift2(model, datasets, seed=123, repetitions=1): + """computes shift""" + + model.eval() + model.cuda() + + import random + + random.seed(seed) + + preds, gts = [], [] + for i_dataset, dataset in enumerate(datasets): + loader = DataLoader(dataset, batch_size=1, num_workers=0, shuffle=False, drop_last=False) + + max_iterations = int(repetitions * len(dataset.dataset.data_list)) + + with torch.no_grad(): + i = [] + for i_all, (data_x, data_y) in enumerate(loader): + data_x = [v.cuda(non_blocking=True) if v is not None else v for v in data_x] + data_y = [v.cuda(non_blocking=True) if v is not None else v for v in data_y] + + (pred,) = model(data_x[0], data_x[1], data_x[2]) + preds += [pred.detach()] + gts += [data_y] + + i += 1 + if max_iterations and i >= max_iterations: + break + + from metrics import FixedIntervalMetrics + + n_values = 25 # 51 + thresholds = np.linspace(0, 1, n_values)[1:-1] + metric = FixedIntervalMetrics(resize_pred=True, sigmoid=True, n_values=n_values) + + for p, y in zip(preds, gts): + metric.add(p.unsqueeze(1), y) + + best_idx = np.argmax(metric.value()["fgiou_scores"]) + best_thresh = thresholds[best_idx] + + return best_thresh + + +def get_cached_pascal_pfe(split, config): + from datasets.pfe_dataset import PFEPascalWrapper + + try: + dataset = DATASET_CACHE[(split, config.image_size, config.label_support, config.mask)] + except KeyError: + dataset = PFEPascalWrapper( + mode="val", split=split, mask=config.mask, image_size=config.image_size, label_support=config.label_support + ) + DATASET_CACHE[(split, config.image_size, config.label_support, config.mask)] = dataset + return dataset + + +def main(): + config, train_checkpoint_id = score_config_from_cli_args() + + metrics = score(config, train_checkpoint_id, None) + print(metrics) + + +def score(config, train_checkpoint_id, train_config): + config = AttributeDict(config) + print(config) + + metric_args = dict() + + if "threshold" in config: + if config.metric.split(".")[-1] == "SkLearnMetrics": + metric_args["threshold"] = config.threshold + + if "resize_to" in config: + metric_args["resize_to"] = config.resize_to + + if "sigmoid" in config: + metric_args["sigmoid"] = config.sigmoid + + if "custom_threshold" in config: + metric_args["custom_threshold"] = config.custom_threshold + + if config.test_dataset == "waffle": + coco_dataset = build_dataset(image_set="test", args=config) + coco_dataset[0] + + def trivial_batch_collator(batch): + """ + A batch collator that does nothing. + """ + return batch + + loader = DataLoader( + coco_dataset, + batch_size=config.batch_size, + num_workers=2, + shuffle=False, + drop_last=False, + collate_fn=trivial_batch_collator, + ) + metric = get_attribute(config.metric)(resize_pred=False, n_values=25, **metric_args) + + shift = config.shift if "shift" in config else 0 + pred_json_root = config.pred_json_root + + with torch.no_grad(): + i = 0 + for i_all, batch_data in enumerate(tqdm(loader)): + image_path = batch_data[0]["file_name"] + data_y = batch_data[0]["instances"].gt_masks.tensor[None, ...] + gt_classes = batch_data[0]["instances"].gt_classes[None, ...] + interior_mask = gt_classes == 0 + data_y = data_y[interior_mask][None, ...] + data_y = torch.sum(data_y, dim=1, keepdim=True).clamp(0, 1) # Shape: Bx1xHxW + + pred = read_pred_json( + os.path.join(pred_json_root, os.path.basename(image_path).split(".")[0] + ".json"), + image_size=(config.image_size, config.image_size), + mask_format=config.mask_format, + ) + if len(pred) == 0: + pred = torch.zeros_like(data_y) + else: + pred = pred.gt_masks.tensor[None, ...] + pred = torch.sum(pred, dim=1, keepdim=True).clamp(0, 1) # Shape: Bx1xHxW + metric.add(pred + shift, data_y) + + i += 1 + if config.max_iterations and i >= config.max_iterations: + break + + key_prefix = config["name"] if "name" in config else "coco" + + print(metric.scores()) + return {key_prefix: metric.scores()} + + +if __name__ == "__main__": + main() diff --git a/evaluations/clipseg_eval/config.yaml b/evaluations/clipseg_eval/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d6c1de20c730bea3bdff681d1de09ff6ae019d6 --- /dev/null +++ b/evaluations/clipseg_eval/config.yaml @@ -0,0 +1,41 @@ + +test_configuration_common: + normalize: True + image_size: 256 + metric: clipseg_eval.metrics.FixedIntervalMetricsWithMatching + batch_size: 1 + test_dataset: waffle + sigmoid: False + # max_iterations: 250 + custom_threshold: 0.25 + + dataset_root: data/waffle_benchmark_processed/ + semantic_classes: 3 + dataset_name: waffle + image_norm: False + poly2seq: False + num_bins: 32 + seq_len: 512 + add_cls_token: False + per_token_class: True + input_channels: 3 + mask_format: bitmask + pred_json_root: + +test_configuration: + name: coco + mask: text + +columns: [name, +pas_t_fgiou_best, pas_t_miou_best, pas_t_fgiou_ct, +pas_h_fgiou_best, pas_h_miou_best, pas_h_fgiou_ct, +pas_h2_fgiou_best, pas_h2_miou_best, pas_h2_fgiou_ct, pas_h2_fgiou_best_t, +train_loss, duration, date +] + +individual_configurations: + +- {name: roomformer, remove_classes: [pas5i, 0], negative_prob: 0.0, test_configuration: {splits: [0], custom_threshold: 0.25}} +- {name: autoroom, remove_classes: [pas5i, 1], negative_prob: 0.0, test_configuration: {splits: [0], custom_threshold: 0.25}} + +# baseline \ No newline at end of file diff --git a/evaluations/clipseg_eval/general_utils.py b/evaluations/clipseg_eval/general_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2ba74107ed5d8342d3ad12fa5239811c3be22068 --- /dev/null +++ b/evaluations/clipseg_eval/general_utils.py @@ -0,0 +1,271 @@ +import inspect +import json +import os +import sys +from os.path import basename, dirname, expanduser, isdir, isfile, join, realpath +from shutil import copy + +import torch +import yaml + + +class Logger(object): + def __getattr__(self, k): + return print + + +log = Logger() + + +def training_config_from_cli_args(): + experiment_name = sys.argv[1] + experiment_id = int(sys.argv[2]) + + yaml_config = yaml.load(open(f"experiments/{experiment_name}"), Loader=yaml.SafeLoader) + + config = yaml_config["configuration"] + config = {**config, **yaml_config["individual_configurations"][experiment_id]} + config = AttributeDict(config) + return config + + +def score_config_from_cli_args(): + experiment_name = sys.argv[1] + experiment_id = int(sys.argv[2]) + + yaml_config = yaml.load(open(experiment_name), Loader=yaml.SafeLoader) + + config = yaml_config["test_configuration_common"] + + config = {**config, **yaml_config["test_configuration"]} + if "test_configuration" in yaml_config["individual_configurations"][experiment_id]: + config = {**config, **yaml_config["individual_configurations"][experiment_id]["test_configuration"]} + + train_checkpoint_id = yaml_config["individual_configurations"][experiment_id]["name"] + config["pred_json_root"] = sys.argv[3] + + config = AttributeDict(config) + return config, train_checkpoint_id + + +def get_from_repository( + local_name, repo_files, integrity_check=None, repo_dir="~/dataset_repository", local_dir="~/datasets" +): + """copies files from repository to local folder. + + repo_files: list of filenames or list of tuples [filename, target path] + + e.g. get_from_repository('MyDataset', [['data/dataset1.tar', 'other/path/ds03.tar']) + will create a folder 'MyDataset' in local_dir, and extract the content of + '/data/dataset1.tar' to /MyDataset/other/path. + """ + + local_dir = realpath(join(expanduser(local_dir), local_name)) + + dataset_exists = True + + # check if folder is available + if not isdir(local_dir): + dataset_exists = False + + if integrity_check is not None: + try: + integrity_ok = integrity_check(local_dir) + except BaseException: + integrity_ok = False + + if integrity_ok: + log.hint("Passed custom integrity check") + else: + log.hint("Custom integrity check failed") + + dataset_exists = dataset_exists and integrity_ok + + if not dataset_exists: + repo_dir = realpath(expanduser(repo_dir)) + + for i, filename in enumerate(repo_files): + if type(filename) == str: + origin, target = filename, filename + archive_target = join(local_dir, basename(origin)) + extract_target = join(local_dir) + else: + origin, target = filename + archive_target = join(local_dir, dirname(target), basename(origin)) + extract_target = join(local_dir, dirname(target)) + + archive_origin = join(repo_dir, origin) + + log.hint(f"copy: {archive_origin} to {archive_target}") + + # make sure the path exists + os.makedirs(dirname(archive_target), exist_ok=True) + + if os.path.isfile(archive_target): + # only copy if size differs + if os.path.getsize(archive_target) != os.path.getsize(archive_origin): + log.hint( + f"file exists but filesize differs: target {os.path.getsize(archive_target)} vs. origin {os.path.getsize(archive_origin)}" + ) + copy(archive_origin, archive_target) + else: + copy(archive_origin, archive_target) + + extract_archive(archive_target, extract_target, noarchive_ok=True) + + # concurrent processes might have deleted the file + if os.path.isfile(archive_target): + os.remove(archive_target) + + +def extract_archive(filename, target_folder=None, noarchive_ok=False): + from subprocess import PIPE, run + + if filename.endswith(".tgz") or filename.endswith(".tar"): + command = f"tar -xf {filename}" + command += f" -C {target_folder}" if target_folder is not None else "" + elif filename.endswith(".tar.gz"): + command = f"tar -xzf {filename}" + command += f" -C {target_folder}" if target_folder is not None else "" + elif filename.endswith("zip"): + command = f"unzip {filename}" + command += f" -d {target_folder}" if target_folder is not None else "" + else: + if noarchive_ok: + return + else: + raise ValueError(f"unsuppored file ending of {filename}") + + log.hint(command) + result = run(command.split(), stdout=PIPE, stderr=PIPE) + if result.returncode != 0: + print(result.stdout, result.stderr) + + +class AttributeDict(dict): + """ + An extended dictionary that allows access to elements as atttributes and counts + these accesses. This way, we know if some attributes were never used. + """ + + def __init__(self, *args, **kwargs): + from collections import Counter + + super().__init__(*args, **kwargs) + self.__dict__["counter"] = Counter() + + def __getitem__(self, k): + self.__dict__["counter"][k] += 1 + return super().__getitem__(k) + + def __getattr__(self, k): + self.__dict__["counter"][k] += 1 + return super().get(k) + + def __setattr__(self, k, v): + return super().__setitem__(k, v) + + def __delattr__(self, k, v): + return super().__delitem__(k, v) + + def unused_keys(self, exceptions=()): + return [k for k in super().keys() if self.__dict__["counter"][k] == 0 and k not in exceptions] + + def assume_no_unused_keys(self, exceptions=()): + if len(self.unused_keys(exceptions=exceptions)) > 0: + log.warning("Unused keys:", self.unused_keys(exceptions=exceptions)) + + +def get_attribute(name): + import importlib + + if name is None: + raise ValueError("The provided attribute is None") + + name_split = name.split(".") + mod = importlib.import_module(".".join(name_split[:-1])) + return getattr(mod, name_split[-1]) + + +def filter_args(input_args, default_args): + + updated_args = {k: input_args[k] if k in input_args else v for k, v in default_args.items()} + used_args = {k: v for k, v in input_args.items() if k in default_args} + unused_args = {k: v for k, v in input_args.items() if k not in default_args} + + return AttributeDict(updated_args), AttributeDict(used_args), AttributeDict(unused_args) + + +def load_model(checkpoint_id, weights_file=None, strict=True, model_args="from_config", with_config=False): + + config = json.load(open(join("logs", checkpoint_id, "config.json"))) + + if model_args != "from_config" and type(model_args) != dict: + raise ValueError('model_args must either be "from_config" or a dictionary of values') + + model_cls = get_attribute(config["model"]) + + # load model + if model_args == "from_config": + _, model_args, _ = filter_args(config, inspect.signature(model_cls).parameters) + + model = model_cls(**model_args) + + if weights_file is None: + weights_file = realpath(join("logs", checkpoint_id, "weights.pth")) + else: + weights_file = realpath(join("logs", checkpoint_id, weights_file)) + + if isfile(weights_file): + weights = torch.load(weights_file) + for _, w in weights.items(): + assert not torch.any(torch.isnan(w)), "weights contain NaNs" + model.load_state_dict(weights, strict=strict) + else: + raise FileNotFoundError(f"model checkpoint {weights_file} was not found") + + if with_config: + return model, config + + return model + + +class TrainingLogger(object): + def __init__(self, model, log_dir, config=None, *args): + super().__init__() + self.model = model + self.base_path = join(f"logs/{log_dir}") if log_dir is not None else None + + os.makedirs("logs/", exist_ok=True) + os.makedirs(self.base_path, exist_ok=True) + + if config is not None: + json.dump(config, open(join(self.base_path, "config.json"), "w")) + + def iter(self, i, **kwargs): + if i % 100 == 0 and "loss" in kwargs: + loss = kwargs["loss"] + print(f"iteration {i}: loss {loss:.4f}") + + def save_weights(self, only_trainable=False, weight_file="weights.pth"): + if self.model is None: + raise AttributeError( + "You need to provide a model reference when initializing TrainingTracker to save weights." + ) + + weights_path = join(self.base_path, weight_file) + + weight_dict = self.model.state_dict() + + if only_trainable: + weight_dict = {n: weight_dict[n] for n, p in self.model.named_parameters() if p.requires_grad} + + torch.save(weight_dict, weights_path) + log.info(f"Saved weights to {weights_path}") + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + """automatically stop processes if used in a context manager""" + pass diff --git a/evaluations/clipseg_eval/metrics.py b/evaluations/clipseg_eval/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..9ec87577ad9ba2dfd6a45526019186cdd94657ec --- /dev/null +++ b/evaluations/clipseg_eval/metrics.py @@ -0,0 +1,476 @@ +from collections import defaultdict + +import numpy as np +import torch +from clipseg_eval.general_utils import log +from torch.nn import functional as nnf + + +class BaseMetric(object): + def __init__( + self, metric_names, pred_range=None, gt_index=0, pred_index=0, eval_intermediate=True, eval_validation=True + ): + self._names = tuple(metric_names) + self._eval_intermediate = eval_intermediate + self._eval_validation = eval_validation + + self._pred_range = pred_range + self._pred_index = pred_index + self._gt_index = gt_index + + self.predictions = [] + self.ground_truths = [] + + def eval_intermediate(self): + return self._eval_intermediate + + def eval_validation(self): + return self._eval_validation + + def names(self): + return self._names + + def add(self, predictions, ground_truth): + raise NotImplementedError + + def value(self): + raise NotImplementedError + + def scores(self): + # similar to value but returns dict + value = self.value() + if isinstance(value, dict): + return value + else: + assert type(value) in {list, tuple} + return list(zip(self.names(), self.value())) + + def _get_pred_gt(self, predictions, ground_truth): + pred = predictions[self._pred_index] + gt = ground_truth[self._gt_index] + + if self._pred_range is not None: + pred = pred[:, self._pred_range[0] : self._pred_range[1]] + + return pred, gt + + +class FixedIntervalMetrics(BaseMetric): + def __init__( + self, sigmoid=False, ignore_mask=False, resize_to=None, resize_pred=None, n_values=51, custom_threshold=None + ): + + super().__init__( + ( + "ap", + "best_fgiou", + "best_miou", + "fgiou0.5", + "fgiou0.1", + "mean_iou_0p5", + "mean_iou_0p1", + "best_biniou", + "biniou_0.5", + "fgiou_thresh", + ) + ) + self.intersections = [] + self.unions = [] + # self.threshold = threshold + self.sigmoid = sigmoid + self.resize_to = resize_to + self.resize_pred = resize_pred # resize prediction to match ground truth + self.class_count = defaultdict(lambda: 0) + self.per_class = defaultdict(lambda: [0, 0]) + self.ignore_mask = ignore_mask + self.custom_threshold = custom_threshold + + self.scores_ap = [] + self.scores_iou = [] + self.gts, self.preds = [], [] + self.classes = [] + + # [1:-1] ignores 0 and 1 + self.threshold_values = np.linspace(0, 1, n_values)[1:-1] + + self.metrics = dict(tp=[], fp=[], fn=[], tn=[]) + + def add(self, pred, gt): + + pred_batch = pred[0].cpu() + + if self.sigmoid: + pred_batch = torch.sigmoid(pred_batch) + + gt_batch = gt[0].cpu() + mask_batch = ( + gt[1] if len(gt) > 1 and not self.ignore_mask and gt[1].numel() > 0 else ([None] * len(pred_batch)) + ) + cls_batch = gt[2] if len(gt) > 2 else [None] * len(pred_batch) + + if self.resize_to is not None: + gt_batch = nnf.interpolate(gt_batch, self.resize_to, mode="nearest") + pred_batch = nnf.interpolate(pred_batch, self.resize_to, mode="bilinear", align_corners=False) + + if isinstance(cls_batch, torch.Tensor): + cls_batch = cls_batch.cpu().numpy().tolist() + + assert len(gt_batch) == len(pred_batch) == len(cls_batch), ( + f"{len(gt_batch)} {len(pred_batch)} {len(cls_batch)}" + ) + + for predictions, ground_truth, mask, cls in zip(pred_batch, gt_batch, mask_batch, cls_batch): + if self.resize_pred: + predictions = nnf.interpolate( + predictions.unsqueeze(0).float(), + size=ground_truth.size()[-2:], + mode="bilinear", + align_corners=True, + ) + + p = predictions.flatten() + g = ground_truth.flatten() + + assert len(p) == len(g) + + if mask is not None: + m = mask.flatten().bool() + p = p[m] + g = g[m] + + p_sorted = p.sort() + p = p_sorted.values + g = g[p_sorted.indices] + + tps, fps, fns, tns = [], [], [], [] + for thresh in self.threshold_values: + valid = torch.where(p > thresh)[0] + if len(valid) > 0: + n = int(valid[0]) + else: + n = len(g) + + fn = int(g[:n].sum()) + tp = int(g[n:].sum()) + fns += [fn] + tns += [n - fn] + tps += [tp] + fps += [len(g) - n - tp] + + self.metrics["tp"] += [tps] + self.metrics["fp"] += [fps] + self.metrics["fn"] += [fns] + self.metrics["tn"] += [tns] + + self.classes += [cls.item() if isinstance(cls, torch.Tensor) else cls] + + def value(self): + + import time + + t_start = time.time() + + if set(self.classes) == set([None]): + all_classes = None + log.warning("classes were not provided, cannot compute mIoU") + else: + all_classes = set(int(c) for c in self.classes) + # log.info(f'compute metrics for {len(all_classes)} classes') + + summed = { + k: [ + sum([self.metrics[k][i][j] for i in range(len(self.metrics[k]))]) + for j in range(len(self.threshold_values)) + ] + for k in self.metrics.keys() + } + + if all_classes is not None: + assert len(self.classes) == len(self.metrics["tp"]) == len(self.metrics["fn"]) + # group by class + metrics_by_class = {c: {k: [] for k in self.metrics.keys()} for c in all_classes} + for i in range(len(self.metrics["tp"])): + for k in self.metrics.keys(): + metrics_by_class[self.classes[i]][k] += [self.metrics[k][i]] + + # sum over all instances within the classes + summed_by_cls = { + k: {c: np.array(metrics_by_class[c][k]).sum(0).tolist() for c in all_classes} + for k in self.metrics.keys() + } + + # Compute average precision + + assert (np.array(summed["fp"]) + np.array(summed["tp"])).sum(), "no predictions is made" + + # only consider values where a prediction is made + precisions = [ + summed["tp"][j] / (1 + summed["tp"][j] + summed["fp"][j]) + for j in range(len(self.threshold_values)) + if summed["tp"][j] + summed["fp"][j] > 0 + ] + recalls = [ + summed["tp"][j] / (1 + summed["tp"][j] + summed["fn"][j]) + for j in range(len(self.threshold_values)) + if summed["tp"][j] + summed["fp"][j] > 0 + ] + + # remove duplicate recall-precision-pairs (and sort by recall value) + recalls, precisions = zip(*sorted(list(set(zip(recalls, precisions))), key=lambda x: x[0])) + + from scipy.integrate import simps + + ap = simps(precisions, recalls) + + # Compute best IoU + fgiou_scores = [ + summed["tp"][j] / (1 + summed["tp"][j] + summed["fp"][j] + summed["fn"][j]) + for j in range(len(self.threshold_values)) + ] + + biniou_scores = [ + 0.5 * (summed["tp"][j] / (1 + summed["tp"][j] + summed["fp"][j] + summed["fn"][j])) + + 0.5 * (summed["tn"][j] / (1 + summed["tn"][j] + summed["fn"][j] + summed["fp"][j])) + for j in range(len(self.threshold_values)) + ] + + # index_0p5 = self.threshold_values.tolist().index(0.5) + # index_0p1 = self.threshold_values.tolist().index(0.1) + # index_0p2 = self.threshold_values.tolist().index(0.2) + # index_0p3 = self.threshold_values.tolist().index(0.3) + + if self.custom_threshold is not None: + index_ct = self.threshold_values.tolist().index(self.custom_threshold) + + if all_classes is not None: + # mean IoU + mean_ious = [ + np.mean( + [ + summed_by_cls["tp"][c][j] + / (1 + summed_by_cls["tp"][c][j] + summed_by_cls["fp"][c][j] + summed_by_cls["fn"][c][j]) + for c in all_classes + ] + ) + for j in range(len(self.threshold_values)) + ] + + mean_iou_dict = { + "miou_best": max(mean_ious) if all_classes is not None else None, + # 'miou_0.5': mean_ious[index_0p5] if all_classes is not None else None, + # 'miou_0.1': mean_ious[index_0p1] if all_classes is not None else None, + # 'miou_0.2': mean_ious[index_0p2] if all_classes is not None else None, + # 'miou_0.3': mean_ious[index_0p3] if all_classes is not None else None, + "miou_best_t": self.threshold_values[np.argmax(mean_ious)], + "mean_iou_ct": ( + mean_ious[index_ct] if all_classes is not None and self.custom_threshold is not None else None + ), + "mean_iou_scores": mean_ious, + } + + print( + f"metric computation on {(len(all_classes) if all_classes is not None else 'no')} classes took {time.time() - t_start:.1f}s" + ) + + return { + "ap": ap, + # fgiou + "fgiou_best": max(fgiou_scores), + # 'fgiou_0.5': fgiou_scores[index_0p5], + # 'fgiou_0.1': fgiou_scores[index_0p1], + # 'fgiou_0.2': fgiou_scores[index_0p2], + # 'fgiou_0.3': fgiou_scores[index_0p3], + "fgiou_best_t": self.threshold_values[np.argmax(fgiou_scores)], + # mean iou + # biniou + "biniou_best": max(biniou_scores), + # 'biniou_0.5': biniou_scores[index_0p5], + # 'biniou_0.1': biniou_scores[index_0p1], + # 'biniou_0.2': biniou_scores[index_0p2], + # 'biniou_0.3': biniou_scores[index_0p3], + "biniou_best_t": self.threshold_values[np.argmax(biniou_scores)], + # custom threshold + "fgiou_ct": fgiou_scores[index_ct] if self.custom_threshold is not None else None, + "biniou_ct": biniou_scores[index_ct] if self.custom_threshold is not None else None, + "ct": self.custom_threshold, + # statistics + "fgiou_scores": fgiou_scores, + "biniou_scores": biniou_scores, + "precision_recall_curve": sorted(list(set(zip(recalls, precisions)))), + "summed_statistics": summed, + "summed_by_cls_statistics": summed_by_cls, + **mean_iou_dict, + } + + # ('ap', 'best_fgiou', 'best_miou', 'fgiou0.5', 'fgiou0.1', 'mean_iou_0p5', 'mean_iou_0p1', 'best_biniou', 'biniou_0.5', 'fgiou_thresh' + + # return ap, best_fgiou, best_mean_iou, iou_0p5, iou_0p1, mean_iou_0p5, mean_iou_0p1, best_biniou, biniou0p5, best_fgiou_thresh, {'summed': summed, 'summed_by_cls': summed_by_cls} + + +class FixedIntervalMetricsWithMatching(FixedIntervalMetrics): + def __init__( + self, sigmoid=False, ignore_mask=False, resize_to=None, resize_pred=None, n_values=51, custom_threshold=None + ): + + super().__init__(sigmoid, ignore_mask, resize_to, resize_pred, n_values, custom_threshold) + self.threshold_values = np.array([0.5]) # np.linspace(0, 1, n_values)[1:-1] + self.metrics = dict(tp=[], fp=[], fn=[], tn=[]) + + def add(self, pred, gt): + + pred_batch = pred[0].cpu() + + if self.sigmoid: + pred_batch = torch.sigmoid(pred_batch) + + gt_batch = gt[0].cpu() + mask_batch = ( + gt[1] if len(gt) > 1 and not self.ignore_mask and gt[1].numel() > 0 else ([None] * len(pred_batch)) + ) + cls_batch = gt[2] if len(gt) > 2 else [None] * len(pred_batch) + + if self.resize_to is not None: + gt_batch = nnf.interpolate(gt_batch, self.resize_to, mode="nearest") + pred_batch = nnf.interpolate(pred_batch, self.resize_to, mode="bilinear", align_corners=False) + + if isinstance(cls_batch, torch.Tensor): + cls_batch = cls_batch.cpu().numpy().tolist() + + assert len(gt_batch) == len(pred_batch) == len(cls_batch), ( + f"{len(gt_batch)} {len(pred_batch)} {len(cls_batch)}" + ) + + for predictions, ground_truth, mask, cls in zip(pred_batch, gt_batch, mask_batch, cls_batch): + if self.resize_pred: + predictions = nnf.interpolate( + predictions.unsqueeze(0).float(), + size=ground_truth.size()[-2:], + mode="bilinear", + align_corners=True, + ) + + p = predictions.flatten() + g = ground_truth.flatten() + + assert len(p) == len(g) + + if mask is not None: + m = mask.flatten().bool() + p = p[m] + g = g[m] + + p_sorted = p.sort() + p = p_sorted.values + g = g[p_sorted.indices] + + tps, fps, fns, tns = [], [], [], [] + for thresh in self.threshold_values: + valid = torch.where(p > thresh)[0] + if len(valid) > 0: + n = int(valid[0]) + else: + n = len(g) + + fn = int(g[:n].sum()) + tp = int(g[n:].sum()) + fns += [fn] + tns += [n - fn] + tps += [tp] + fps += [len(g) - n - tp] + + self.metrics["tp"] += [tps] + self.metrics["fp"] += [fps] + self.metrics["fn"] += [fns] + self.metrics["tn"] += [tns] + + self.classes += [cls.item() if isinstance(cls, torch.Tensor) else cls] + + def value(self): + + import time + + t_start = time.time() + + if set(self.classes) == set([None]): + all_classes = None + log.warning("classes were not provided, cannot compute mIoU") + else: + all_classes = set(int(c) for c in self.classes) + log.info(f"compute metrics for {len(all_classes)} classes") + + summed = { + k: [ + sum([self.metrics[k][i][j] for i in range(len(self.metrics[k]))]) + for j in range(len(self.threshold_values)) + ] + for k in self.metrics.keys() + } + + if all_classes is not None: + assert len(self.classes) == len(self.metrics["tp"]) == len(self.metrics["fn"]) + # group by class + metrics_by_class = {c: {k: [] for k in self.metrics.keys()} for c in all_classes} + for i in range(len(self.metrics["tp"])): + for k in self.metrics.keys(): + metrics_by_class[self.classes[i]][k] += [self.metrics[k][i]] + + # # sum over all instances within the classes + # summed_by_cls = { + # k: {c: np.array(metrics_by_class[c][k]).sum(0).tolist() for c in all_classes} + # for k in self.metrics.keys() + # } + + # Compute average precision + + assert (np.array(summed["fp"]) + np.array(summed["tp"])).sum(), "no predictions is made" + + # only consider values where a prediction is made + precisions = [ + summed["tp"][j] / (1 + summed["tp"][j] + summed["fp"][j]) + for j in range(len(self.threshold_values)) + if summed["tp"][j] + summed["fp"][j] > 0 + ] + recalls = [ + summed["tp"][j] / (1 + summed["tp"][j] + summed["fn"][j]) + for j in range(len(self.threshold_values)) + if summed["tp"][j] + summed["fp"][j] > 0 + ] + + # remove duplicate recall-precision-pairs (and sort by recall value) + recalls, precisions = zip(*sorted(list(set(zip(recalls, precisions))), key=lambda x: x[0])) + + from scipy.integrate import simps + + ap = simps(precisions, recalls) + + # Compute best IoU + fgiou_scores = [ + summed["tp"][j] / (1 + summed["tp"][j] + summed["fp"][j] + summed["fn"][j]) + for j in range(len(self.threshold_values)) + ] + + biniou_scores = [ + 0.5 * (summed["tp"][j] / (1 + summed["tp"][j] + summed["fp"][j] + summed["fn"][j])) + + 0.5 * (summed["tn"][j] / (1 + summed["tn"][j] + summed["fn"][j] + summed["fp"][j])) + for j in range(len(self.threshold_values)) + ] + + print( + f"metric computation on {(len(all_classes) if all_classes is not None else 'no')} classes took {time.time() - t_start:.1f}s" + ) + + return { + "ap": ap, + # fgiou + "fgiou_best": max(fgiou_scores), + "fgiou_best_t": self.threshold_values[np.argmax(fgiou_scores)], + # mean iou + # biniou + "biniou_best": max(biniou_scores), + "biniou_best_t": self.threshold_values[np.argmax(biniou_scores)], + # statistics + "fgiou_scores": fgiou_scores, + "biniou_scores": biniou_scores, + "precision_recall_curve": sorted(list(set(zip(recalls, precisions)))), + "summed_statistics": summed, + } diff --git a/evaluations/rplan_eval/Evaluator.py b/evaluations/rplan_eval/Evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..c2a9d1c5759386f670c2585312979d6c9d33f888 --- /dev/null +++ b/evaluations/rplan_eval/Evaluator.py @@ -0,0 +1,586 @@ +""" +This is a hack implementation for evaluation on RPlan + +Mostly copy-paste from Evaluator.py (from MonteFloor) with small modification +""" + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch + +corner_metric_thresh = 10 # 10,20 +angle_metric_thresh = 5 + +# colormap_255 = [[i, i, i] for i in range(40)] + + +class Evaluator_RPlan: + def __init__(self, data_rw=None, options=None, disable_overlapping_filter=False, iou_thres=0.5, wd_as_line=True): + self.data_rw = data_rw + self.options = options + self.disable_overlapping_filter = disable_overlapping_filter + self.iou_thres = iou_thres + self.wd_as_line = wd_as_line + + self.device = torch.device("cuda") + + def polygonize_mask(self, mask, degree, return_mask=True): + h, w = mask.shape[0], mask.shape[1] + mask = mask + + room_mask = 255 * (mask == 1) + room_mask = room_mask.astype(np.uint8) + room_mask_inv = 255 - room_mask + + ret, thresh = cv2.threshold(room_mask_inv, 250, 255, cv2.THRESH_BINARY_INV) + + contours, hierarchy = cv2.findContours(thresh, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + + cnt = contours[0] + max_area = cv2.contourArea(cnt) + + for cont in contours: + if cv2.contourArea(cont) > max_area: + cnt = cont + max_area = cv2.contourArea(cont) + + epsilon = degree * cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, epsilon, True) + approx = approx.astype(np.int32).reshape((-1, 2)) + + if return_mask: + room_filled_map = np.zeros((h, w)) + cv2.fillPoly(room_filled_map, [approx], color=1.0) + + return approx, room_filled_map + else: + return approx + + def print_res_str_for_latex(self, quant_result_dict): + + str_fields = "" + str_values = "" + + avg_value_prec = 0 + avg_value_rec = 0 + for k_ind, k in enumerate(quant_result_dict.keys()): + str_fields += " & " + k + str_values += " & %.2f " % quant_result_dict[k] + + if k_ind % 2 == 0: + avg_value_prec += quant_result_dict[k] / 3 + else: + avg_value_rec += quant_result_dict[k] / 3 + + str_fields += "tm_prec & tm_rec" + + str_values += " & %.2f " % avg_value_prec + str_values += " & %.2f " % avg_value_rec + + str_fields += " \\\\" + str_values += " \\\\" + + print(str_fields) + print(str_values) + + def calc_gradient(self, room_map): + grad_x = np.abs(room_map[:, 1:] - room_map[:, :-1]) + grad_y = np.abs(room_map[1:] - room_map[:-1]) + + grad_xy = np.zeros_like(room_map) + grad_xy[1:] = grad_y + grad_xy[:, 1:] = np.maximum(grad_x, grad_xy[:, 1:]) + + plt.figure() + plt.axis("off") + plt.imshow(grad_xy, cmap="gray") + # plt.show() + plt.savefig("grad.png", bbox_inches="tight") + + plt.figure() + plt.axis("off") + plt.imshow(room_map, cmap="gray") + # plt.show() + plt.savefig("joint_mask.png", bbox_inches="tight") + assert False + + def evaluate_scene( + self, + room_polys, + gt_polys, + room_types, + gt_polys_types, + window_door_lines=None, + gt_window_doors_list=None, + window_door_lines_types=None, + gt_window_doors_type_list=None, + show=False, + name="ours", + dataset_type="s3d", + img_size=(256, 256), + ): + + gt_polys_list = [np.concatenate([poly, poly[None, 0]]) for poly in gt_polys] + room_polys = [np.concatenate([poly, poly[None, 0]]) for poly in room_polys] + + ignore_mask_region = None + + # TODO: input img_size + quant_result_dict = self.get_quantitative( + gt_polys_list, + gt_polys_types, + gt_window_doors_list, + gt_window_doors_type_list, + ignore_mask_region, + room_polys, + room_types, + window_door_lines, + window_door_lines_types, + None, + img_size, + dataset_type=dataset_type, + ) + + return quant_result_dict + + def get_quantitative( + self, + gt_polys, + gt_polys_types, + gt_window_doors, + gt_window_doors_types, + ignore_mask_region, + pred_polys=None, + pred_types=None, + pred_window_doors=None, + pred_window_doors_types=None, + masks_list=None, + img_size=(256, 256), + dataset_type="s3d", + ): + def get_room_metric(): + pred_overlaps = [False] * len(pred_room_map_list) + if not self.disable_overlapping_filter: + for pred_ind1 in range(len(pred_room_map_list) - 1): + pred_map1 = pred_room_map_list[pred_ind1] + + for pred_ind2 in range(pred_ind1 + 1, len(pred_room_map_list)): + pred_map2 = pred_room_map_list[pred_ind2] + + if dataset_type == "s3d": + kernel = np.ones((5, 5), np.uint8) + else: + kernel = np.ones((3, 3), np.uint8) + + # todo: for our method, the rooms share corners and edges, need to check here + pred_map1_er = cv2.erode(pred_map1, kernel) + pred_map2_er = cv2.erode(pred_map2, kernel) + + intersection = (pred_map1_er + pred_map2_er) == 2 + # intersection = (pred_map1 + pred_map2) == 2 + + intersection_area = np.sum(intersection) + + if intersection_area >= 1: + pred_overlaps[pred_ind1] = True + pred_overlaps[pred_ind2] = True + + # import pdb; pdb.set_trace() + room_metric = [np.bool((1 - pred_overlaps[ind]) * pred2gt_exists[ind]) for ind in range(len(pred_polys))] + if pred_types is not None: + room_sem_metric = [ + np.bool((1 - pred_overlaps[ind]) * pred2gt_exists_sem[ind]) for ind in range(len(pred_polys)) + ] + else: + room_sem_metric = None + return room_metric, room_sem_metric + + def get_corner_metric(): + + room_corners_metric = [] + for pred_poly_ind, gt_poly_ind in enumerate(pred2gt_indices): + p_poly = pred_polys[pred_poly_ind][:-1] # Last vertex = First vertex + + p_poly_corner_metrics = [False] * p_poly.shape[0] + if not room_metric[pred_poly_ind]: + room_corners_metric += p_poly_corner_metrics + continue + + gt_poly = gt_polys[gt_poly_ind][:-1] + + # for v in p_poly: + # v_dists = np.linalg.norm(v[None,:] - gt_poly, axis=1, ord=2) + # v_min_dist = np.min(v_dists) + # + # v_tp = v_min_dist <= 10 + # room_corners_metric.append(v_tp) + + for v in gt_poly: + v_dists = np.linalg.norm(v[None, :] - p_poly, axis=1, ord=2) + v_min_dist_ind = np.argmin(v_dists) + v_min_dist = v_dists[v_min_dist_ind] + + if not p_poly_corner_metrics[v_min_dist_ind]: + v_tp = v_min_dist <= corner_metric_thresh + p_poly_corner_metrics[v_min_dist_ind] = v_tp + + room_corners_metric += p_poly_corner_metrics + + return room_corners_metric + + def get_angle_metric(): + + def get_line_vector(p1, p2): + p1 = np.concatenate((p1, np.array([1]))) + p2 = np.concatenate((p2, np.array([1]))) + + line_vector = -np.cross(p1, p2) + + return line_vector + + def get_poly_orientation(my_poly): + angles_sum = 0 + for v_ind, _ in enumerate(my_poly): + if v_ind < len(my_poly) - 1: + v_sides = my_poly[[v_ind - 1, v_ind, v_ind, v_ind + 1], :] + else: + v_sides = my_poly[[v_ind - 1, v_ind, v_ind, 0], :] + + v1_vector = get_line_vector(v_sides[0], v_sides[1]) + v1_vector = v1_vector / (np.linalg.norm(v1_vector, ord=2) + 1e-4) + v2_vector = get_line_vector(v_sides[2], v_sides[3]) + v2_vector = v2_vector / (np.linalg.norm(v2_vector, ord=2) + 1e-4) + + orientation = (v_sides[1, 1] - v_sides[0, 1]) * (v_sides[3, 0] - v_sides[1, 0]) - ( + v_sides[3, 1] - v_sides[1, 1] + ) * (v_sides[1, 0] - v_sides[0, 0]) + + v1_vector_2d = v1_vector[:2] / (v1_vector[2] + 1e-4) + v2_vector_2d = v2_vector[:2] / (v2_vector[2] + 1e-4) + + v1_vector_2d = v1_vector_2d / (np.linalg.norm(v1_vector_2d, ord=2) + 1e-4) + v2_vector_2d = v2_vector_2d / (np.linalg.norm(v2_vector_2d, ord=2) + 1e-4) + + angle_cos = v1_vector_2d.dot(v2_vector_2d) + angle_cos = np.clip(angle_cos, -1, 1) + + # G.T. has clockwise orientation, remove minus in the equation + + angle = np.sign(orientation) * np.abs(np.arccos(angle_cos)) + angle_degree = angle * 180 / np.pi + + angles_sum += angle_degree + + return np.sign(angles_sum) + + def get_angle_v_sides(inp_v_sides, poly_orient): + v1_vector = get_line_vector(inp_v_sides[0], inp_v_sides[1]) + v1_vector = v1_vector / (np.linalg.norm(v1_vector, ord=2) + 1e-4) + v2_vector = get_line_vector(inp_v_sides[2], inp_v_sides[3]) + v2_vector = v2_vector / (np.linalg.norm(v2_vector, ord=2) + 1e-4) + + orientation = (inp_v_sides[1, 1] - inp_v_sides[0, 1]) * (inp_v_sides[3, 0] - inp_v_sides[1, 0]) - ( + inp_v_sides[3, 1] - inp_v_sides[1, 1] + ) * (inp_v_sides[1, 0] - inp_v_sides[0, 0]) + + v1_vector_2d = v1_vector[:2] / (v1_vector[2] + 1e-4) + v2_vector_2d = v2_vector[:2] / (v2_vector[2] + 1e-4) + + v1_vector_2d = v1_vector_2d / (np.linalg.norm(v1_vector_2d, ord=2) + 1e-4) + v2_vector_2d = v2_vector_2d / (np.linalg.norm(v2_vector_2d, ord=2) + 1e-4) + + angle_cos = v1_vector_2d.dot(v2_vector_2d) + angle_cos = np.clip(angle_cos, -1, 1) + + angle = poly_orient * np.sign(orientation) * np.arccos(angle_cos) + angle_degree = angle * 180 / np.pi + + return angle_degree + + room_angles_metric = [] + for pred_poly_ind, gt_poly_ind in enumerate(pred2gt_indices): + p_poly = pred_polys[pred_poly_ind][:-1] # Last vertex = First vertex + + p_poly_angle_metrics = [False] * p_poly.shape[0] + if not room_metric[pred_poly_ind]: + room_angles_metric += p_poly_angle_metrics + continue + + gt_poly = gt_polys[gt_poly_ind][:-1] + + gt_poly_orient = get_poly_orientation(gt_poly) + p_poly_orient = get_poly_orientation(p_poly) + + for v_gt_ind, v in enumerate(gt_poly): + v_dists = np.linalg.norm(v[None, :] - p_poly, axis=1, ord=2) + v_ind = np.argmin(v_dists) + v_min_dist = v_dists[v_ind] + + if v_min_dist > corner_metric_thresh: + continue + + if v_ind < len(p_poly) - 1: + v_sides = p_poly[[v_ind - 1, v_ind, v_ind, v_ind + 1], :] + else: + v_sides = p_poly[[v_ind - 1, v_ind, v_ind, 0], :] + + v_sides = v_sides.reshape((4, 2)) + pred_angle_degree = get_angle_v_sides(v_sides, p_poly_orient) + + # Note: replacing some variables with values from the g.t. poly + + if v_gt_ind < len(gt_poly) - 1: + v_sides = gt_poly[[v_gt_ind - 1, v_gt_ind, v_gt_ind, v_gt_ind + 1], :] + else: + v_sides = gt_poly[[v_gt_ind - 1, v_gt_ind, v_gt_ind, 0], :] + + v_sides = v_sides.reshape((4, 2)) + gt_angle_degree = get_angle_v_sides(v_sides, gt_poly_orient) + + angle_metric = np.abs(pred_angle_degree - gt_angle_degree) + p_poly_angle_metrics[v_ind] = angle_metric <= angle_metric_thresh + + room_angles_metric += p_poly_angle_metrics + + for am, cm in zip(room_angles_metric, corner_metric): + assert not (cm == False and am == True), "cm: %d am: %d" % (cm, am) + + return room_angles_metric + + def poly_map_sort_key(x): + return np.sum(x[1]) + + h, w = img_size + + gt_room_map_list = [] + for room_ind, poly in enumerate(gt_polys): + room_map = np.zeros((h, w)) + cv2.fillPoly(room_map, [poly], color=1.0) + + gt_room_map_list.append(room_map) + + gt_polys_sorted_indcs = [ + i[0] for i in sorted(enumerate(gt_room_map_list), key=poly_map_sort_key, reverse=True) + ] + + gt_polys = [gt_polys[ind] for ind in gt_polys_sorted_indcs] + gt_polys_types = [gt_polys_types[ind] for ind in gt_polys_sorted_indcs] + gt_room_map_list = [gt_room_map_list[ind] for ind in gt_polys_sorted_indcs] + + if pred_polys is not None: + pred_room_map_list = [] + for room_ind, poly in enumerate(pred_polys): + room_map = np.zeros((h, w)) + cv2.fillPoly(room_map, [poly], color=1.0) + + pred_room_map_list.append(room_map) + else: + pred_room_map_list = masks_list + + gt2pred_indices = [-1] * len(gt_polys) + gt2pred_exists = [False] * len(gt_polys) + + gt2pred_indices_sem = [-1] * len(gt_polys) + gt2pred_exists_sem = [False] * len(gt_polys) + + gt2pred_indices_wd = None if gt_window_doors is None else [-1] * len(gt_window_doors) + gt2pred_exists_wd = None if gt_window_doors is None else [False] * len(gt_window_doors) + + ### match predicted rooms to ground truth rooms + best_iou = 0.0 + for gt_ind, gt_map in enumerate(gt_room_map_list): + best_iou = 0.0 + best_ind = -1 + best_ind_sem = -1 + for pred_ind, pred_map in enumerate(pred_room_map_list): + intersection = (pred_map + gt_map) == 2 + union = (pred_map + gt_map) >= 1 + + iou = np.sum(intersection) / (np.sum(union) + 1) + + if iou > best_iou and iou > self.iou_thres: + best_iou = iou + best_ind = pred_ind + + if pred_types is not None: + if gt_polys_types[gt_ind] == pred_types[pred_ind]: + best_ind_sem = pred_ind + + gt2pred_indices[gt_ind] = best_ind + gt2pred_exists[gt_ind] = best_ind != -1 + + if pred_types is not None: + gt2pred_indices_sem[gt_ind] = best_ind_sem + gt2pred_exists_sem[gt_ind] = best_ind_sem != -1 + + ### match predicted window/door to ground truth window/door + if pred_window_doors_types is not None: + if self.wd_as_line: + for gt_ind, gt_wd in enumerate(gt_window_doors): + best_dist = 100000.0 + best_ind = -1 + + for pred_ind, pred_wd in enumerate(pred_window_doors): + dist_match1 = [ + np.linalg.norm(gt_wd[0] - pred_wd[0], axis=0, ord=2), + np.linalg.norm(gt_wd[1] - pred_wd[1], axis=0, ord=2), + ] + dist_match2 = [ + np.linalg.norm(gt_wd[0] - pred_wd[1], axis=0, ord=2), + np.linalg.norm(gt_wd[1] - pred_wd[0], axis=0, ord=2), + ] + + dist_match = dist_match1 if sum(dist_match1) < sum(dist_match2) else dist_match2 + + if ( + sum(dist_match) < best_dist + and dist_match[0] < corner_metric_thresh + and dist_match[1] < corner_metric_thresh + and gt_window_doors_types[gt_ind] == pred_window_doors_types[pred_ind] + ): + best_dist = sum(dist_match) + best_ind = pred_ind + + gt2pred_indices_wd[gt_ind] = best_ind + gt2pred_exists_wd[gt_ind] = best_ind != -1 + else: + gt_wd_map_list = [] + for wd_ind, poly in enumerate(gt_window_doors): + wd_map = np.zeros((h, w)) + cv2.fillPoly(wd_map, [poly], color=1.0) + gt_wd_map_list.append(wd_map) + + gt_wd_polys_sorted_indcs = [ + i[0] for i in sorted(enumerate(gt_wd_map_list), key=poly_map_sort_key, reverse=True) + ] + + gt_window_doors = [gt_window_doors[ind] for ind in gt_wd_polys_sorted_indcs] + gt_wd_map_list = [gt_wd_map_list[ind] for ind in gt_wd_polys_sorted_indcs] + + if pred_window_doors is not None: + pred_wd_map_list = [] + for wd_ind, poly in enumerate(pred_window_doors): + wd_map = np.zeros((h, w)) + cv2.fillPoly(wd_map, [poly], color=1.0) + + pred_wd_map_list.append(wd_map) + else: + pred_wd_map_list = masks_list + + for gt_ind, gt_map in enumerate(gt_wd_map_list): + best_iou = 0.0 + best_ind = -1 + for pred_ind, pred_map in enumerate(pred_wd_map_list): + intersection = (pred_map + gt_map) == 2 + union = (pred_map + gt_map) >= 1 + + iou = np.sum(intersection) / (np.sum(union) + 1) + + if iou > best_iou and iou > self.iou_thres: + best_iou = iou + best_ind = pred_ind + + gt2pred_indices_wd[gt_ind] = best_ind + gt2pred_exists_wd[gt_ind] = best_ind != -1 + + pred2gt_exists = [True if pred_ind in gt2pred_indices else False for pred_ind, _ in enumerate(pred_polys)] + pred2gt_indices = [ + gt2pred_indices.index(pred_ind) if pred_ind in gt2pred_indices else -1 + for pred_ind, _ in enumerate(pred_polys) + ] + room_missing_ratio = 1.0 - sum(pred2gt_exists) / float(len(gt_polys) + 1e-4) + + if pred_types is not None: + pred2gt_exists_sem = [ + True if pred_ind in gt2pred_indices_sem else False for pred_ind, _ in enumerate(pred_polys) + ] + + if pred_window_doors_types is not None: + pred2gt_exists_wd = [ + True if pred_ind in gt2pred_indices_wd else False for pred_ind, _ in enumerate(pred_window_doors) + ] + + room_metric, room_sem_metric = get_room_metric() + ###### metric for room WITHOUT considering type ###### + if len(pred_polys) == 0: + room_metric_prec = 0 + else: + room_metric_prec = sum(room_metric) / float(len(pred_polys)) + room_metric_rec = sum(room_metric) / float(len(gt_polys) + 1e-4) + ###### metric for room WITH considering type ###### + + ###### metric for room WITH considering type ###### + if pred_types is not None: + if len(pred_polys) == 0: + room_sem_metric_prec = 0 + else: + room_sem_metric_prec = sum(room_sem_metric) / float(len(pred_polys)) + room_sem_metric_rec = sum(room_sem_metric) / float(len(gt_polys) + 1e-4) + + ###### metric for window and door ###### + if pred_window_doors_types is not None: + if len(pred_window_doors) == 0: + window_door_metric_prec = 0 + else: + window_door_metric_prec = sum(pred2gt_exists_wd) / float(len(pred_window_doors)) + window_door_metric_rec = sum(pred2gt_exists_wd) / float(len(gt_window_doors) + 1e-4) + + ###### metric for corner ###### + corner_metric = get_corner_metric() + pred_corners_n = sum([poly.shape[0] - 1 for poly in pred_polys]) + gt_corners_n = sum([poly.shape[0] - 1 for poly in gt_polys]) + + if pred_corners_n > 0: + corner_metric_prec = sum(corner_metric) / float(pred_corners_n) + else: + corner_metric_prec = 0 + corner_metric_rec = sum(corner_metric) / float(gt_corners_n + 1e-4) + + ###### metric for angle ###### + angles_metric = get_angle_metric() + + if pred_corners_n > 0: + angles_metric_prec = sum(angles_metric) / float(pred_corners_n) + else: + angles_metric_prec = 0 + angles_metric_rec = sum(angles_metric) / float(gt_corners_n + 1e-4) + + # sanity check + assert room_metric_prec <= 1 + assert room_metric_rec <= 1 + assert corner_metric_prec <= 1 + assert corner_metric_rec <= 1 + assert angles_metric_prec <= 1 + assert angles_metric_rec <= 1 + + if pred_types is not None: + assert room_sem_metric_prec <= 1 + assert room_sem_metric_rec <= 1 + + if pred_window_doors_types is not None: + assert window_door_metric_prec <= 1 + assert window_door_metric_rec <= 1 + + result_dict = { + "room_iou": best_iou, + "room_prec": room_metric_prec, + "room_rec": room_metric_rec, + "corner_prec": corner_metric_prec, + "corner_rec": corner_metric_rec, + "angles_prec": angles_metric_prec, + "angles_rec": angles_metric_rec, + "room_missing_ratio": room_missing_ratio, + "pred2gt_indices": pred2gt_indices, + "gt_polys_sorted_indcs": gt_polys_sorted_indcs, + } + + if pred_types is not None: + result_dict["room_sem_prec"] = room_sem_metric_prec + result_dict["room_sem_rec"] = room_sem_metric_rec + + if pred_window_doors_types is not None: + result_dict["window_door_prec"] = window_door_metric_prec + result_dict["window_door_rec"] = window_door_metric_rec + + return result_dict diff --git a/evaluations/s3d_floorplan_eval/DataRW/DataRW.py b/evaluations/s3d_floorplan_eval/DataRW/DataRW.py new file mode 100644 index 0000000000000000000000000000000000000000..27c57ab0d52746df439856cbadca286c90b0f212 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/DataRW/DataRW.py @@ -0,0 +1,3 @@ +class DataRW: + def __init__(self, options): + pass diff --git a/evaluations/s3d_floorplan_eval/DataRW/S3DRW.py b/evaluations/s3d_floorplan_eval/DataRW/S3DRW.py new file mode 100644 index 0000000000000000000000000000000000000000..9eacd33c62e018401c6e1272825b7dfe384febab --- /dev/null +++ b/evaluations/s3d_floorplan_eval/DataRW/S3DRW.py @@ -0,0 +1,129 @@ +import os +import time + +import cv2 +import numpy as np +import torch +from s3d_floorplan_eval.DataRW.DataRW import DataRW +from s3d_floorplan_eval.S3DLoader.S3DLoader import S3DLoader + + +class S3DRW(DataRW): + def __init__(self, options, mode): + """ + Class for accessing FloorNet dataset related data + + :param options: + """ + # initialize the base class variables + super(DataRW, self).__init__() + + self.options = options + + self.dataset_path = options.dataset_path + self.scene_id = options.scene_id + + self.mcts_path = options.mcts_path + self.creation_time = int(time.time()) + + self.device = torch.device("cpu") + + # mode = "train" + # mode = "online_eval" + # For validation only + # self.loader = S3DLoader(options, 'online_eval').dataset + self.loader = S3DLoader(options, mode).dataset + + # gt_sample = iter(floornet_loader.dataset[int(self.scene_id)]) + # self.gt_sample = floornet_loader.load_sample(list(iter(floornet_loader.dataset))[int(self.scene_id)]) + + if mode == "online_eval": + scene_ind = int(self.scene_id[6:]) - 3000 + elif mode == "test": + scene_ind = int(self.scene_id[6:]) - 3250 + elif mode == "train": + scene_ind = int(self.scene_id[6:]) + else: + assert False + + self.gt_sample = self.loader[scene_ind] + self.gt_sample["density_map"] = torch.tensor(self.gt_sample["density_map"][None], device=self.device) + self.gt_sample["room_map"] = torch.tensor(self.gt_sample["room_map"][None, :, :, None], device=self.device) + self.gt_sample["wall_map"] = torch.tensor(self.gt_sample["wall_map"][None, :, :, None], device=self.device) + + self.density_map = self.gt_sample["density_map"][:, :, :, None] + + self.h, self.w = self.density_map.shape[1], self.density_map.shape[2] + + self.generate_input_map_from_props = self.generate_input_dict_from_room_props + + def get_gt_solution(self): + """ + Read top-view density map of the scene + + :return: + """ + img_path = os.path.join(self.dataset_path, str(self.scene_id) + "_density.png") + density_map = cv2.imread(img_path, cv2.IMREAD_ANYDEPTH | cv2.IMREAD_ANYCOLOR)[:, :, 0][None, :, :, None] + + density_map = torch.from_numpy(density_map).to(self.device) + + dm_min = torch.min(density_map) + dm_max = torch.max(density_map) + + density_map = (density_map - dm_min) / (dm_max - dm_min) + + return density_map.type(torch.cuda.FloatTensor) + + def polygonize_mask(self, pm, return_mask=True): + pm_np = pm.cpu().numpy() + + room_mask = 255 * (pm_np == 1) + room_mask = room_mask.astype(np.uint8) + room_mask_inv = 255 - room_mask + + ret, thresh = cv2.threshold(room_mask_inv, 250, 255, cv2.THRESH_BINARY_INV) + + contours, hierarchy = cv2.findContours(thresh, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + + cnt = contours[0] + max_area = cv2.contourArea(cnt) + + for cont in contours: + if cv2.contourArea(cont) > max_area: + cnt = cont + max_area = cv2.contourArea(cont) + + # define main island contour approx. and hull + epsilon = 0.01 * cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, epsilon, True) + approx = approx.astype(np.int32).reshape((1, -1, 2)) + + if return_mask: + room_filled_map = np.zeros((self.h, self.w)) + cv2.fillPoly(room_filled_map, approx, color=1.0) + + room_filled_map = torch.tensor(room_filled_map[:, :], dtype=torch.float32, device=self.device) + + return room_filled_map + else: + approx_tensor = torch.tensor(approx, device=self.device) + return approx_tensor + + def generate_input_dict_from_room_props(self, room_prop_list, score_function, use_thresh=False): + """ + + :param room_prop_list: + :type room_prop_list: list of FloorPlanRoomProp + :param score_function: + :return: + """ + + if score_function == "room_maskrcnn_iou": + inputs = self.generate_input_dict_for_room_maskrcnn_iou(room_prop_list) + elif score_function == "room_iou": + inputs = self.generate_input_dict_for_room_iou(room_prop_list, use_thresh=use_thresh) + else: + assert "generate_input_dict_from_room_props for %s not implemented" % score_function + + return inputs diff --git a/evaluations/s3d_floorplan_eval/DataRW/wrong_annotatios.py b/evaluations/s3d_floorplan_eval/DataRW/wrong_annotatios.py new file mode 100644 index 0000000000000000000000000000000000000000..4ae62706c04f328aab22df11a6c4f7d772d8ba68 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/DataRW/wrong_annotatios.py @@ -0,0 +1 @@ +wrong_s3d_annotations_list = [3261, 3271, 3276, 3296, 3342, 3387, 3398, 3466, 3496] diff --git a/evaluations/s3d_floorplan_eval/Evaluator/Evaluator.py b/evaluations/s3d_floorplan_eval/Evaluator/Evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..d01d64b9a685170a2c6f42931751dfabef50b43f --- /dev/null +++ b/evaluations/s3d_floorplan_eval/Evaluator/Evaluator.py @@ -0,0 +1,587 @@ +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch + +try: + np.bool = np.bool_ +except Exception: + np.bool = np.bool # for numpy 1.20 + +corner_metric_thresh = 10 +angle_metric_thresh = 5 + +# colormap_255 = [[i, i, i] for i in range(40)] + + +class Evaluator: + def __init__(self, data_rw, options, disable_overlapping_filter=False): + self.data_rw = data_rw + self.options = options + self.disable_overlapping_filter = disable_overlapping_filter + + self.device = torch.device("cuda") + + def polygonize_mask(self, mask, degree, return_mask=True): + h, w = mask.shape[0], mask.shape[1] + mask = mask + + room_mask = 255 * (mask == 1) + room_mask = room_mask.astype(np.uint8) + room_mask_inv = 255 - room_mask + + ret, thresh = cv2.threshold(room_mask_inv, 250, 255, cv2.THRESH_BINARY_INV) + + contours, hierarchy = cv2.findContours(thresh, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + + cnt = contours[0] + max_area = cv2.contourArea(cnt) + + for cont in contours: + if cv2.contourArea(cont) > max_area: + cnt = cont + max_area = cv2.contourArea(cont) + + epsilon = degree * cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, epsilon, True) + approx = approx.astype(np.int32).reshape((-1, 2)) + + if return_mask: + room_filled_map = np.zeros((h, w)) + cv2.fillPoly(room_filled_map, [approx], color=1.0) + + return approx, room_filled_map + else: + return approx + + def print_res_str_for_latex(self, quant_result_dict): + + str_fields = "" + str_values = "" + + avg_value_prec = 0 + avg_value_rec = 0 + for k_ind, k in enumerate(quant_result_dict.keys()): + str_fields += " & " + k + str_values += " & %.2f " % quant_result_dict[k] + + if k_ind % 2 == 0: + avg_value_prec += quant_result_dict[k] / 3 + else: + avg_value_rec += quant_result_dict[k] / 3 + + str_fields += "tm_prec & tm_rec" + + str_values += " & %.2f " % avg_value_prec + str_values += " & %.2f " % avg_value_rec + + str_fields += " \\\\" + str_values += " \\\\" + + print(str_fields) + print(str_values) + + def calc_gradient(self, room_map): + grad_x = np.abs(room_map[:, 1:] - room_map[:, :-1]) + grad_y = np.abs(room_map[1:] - room_map[:-1]) + + grad_xy = np.zeros_like(room_map) + grad_xy[1:] = grad_y + grad_xy[:, 1:] = np.maximum(grad_x, grad_xy[:, 1:]) + + plt.figure() + plt.axis("off") + plt.imshow(grad_xy, cmap="gray") + # plt.show() + plt.savefig("grad.png", bbox_inches="tight") + + plt.figure() + plt.axis("off") + plt.imshow(room_map, cmap="gray") + # plt.show() + plt.savefig("joint_mask.png", bbox_inches="tight") + assert False + + def evaluate_scene( + self, + room_polys, + room_types=None, + window_door_lines=None, + window_door_lines_types=None, + show=False, + name="ours", + dataset_type="s3d", + ): + + with torch.no_grad(): + joint_room_map = np.zeros((self.options.height, self.options.width)) + edge_map = np.zeros_like(joint_room_map) + + density_map = self.data_rw.density_map.cpu().numpy()[0] + img_size = (density_map.shape[0], density_map.shape[0]) + + for _, poly in enumerate(room_polys): + cv2.polylines(edge_map, [poly], isClosed=True, color=1.0) + cv2.fillPoly(joint_room_map, [poly], color=1.0) + + # Ground Truth + gt_polys_list = self.data_rw.gt_sample["polygons_list"] + gt_polys_list = [np.concatenate([poly, poly[None, 0]]) for poly in gt_polys_list] + gt_polys_type_list = self.data_rw.gt_sample["polygons_type_list"] + + gt_window_doors_list = self.data_rw.gt_sample["window_doors_list"] + gt_window_doors_type_list = self.data_rw.gt_sample["window_doors_type_list"] + + room_polys = [np.concatenate([poly, poly[None, 0]]) for poly in room_polys] + + ignore_mask_region = self.data_rw.gt_sample["wall_map"].cpu().numpy()[0, :, :, 0] + + img_size = (joint_room_map.shape[0], joint_room_map.shape[1]) + quant_result_dict = self.get_quantitative( + gt_polys_list, + gt_polys_type_list, + gt_window_doors_list, + gt_window_doors_type_list, + ignore_mask_region, + room_polys, + room_types, + window_door_lines, + window_door_lines_types, + None, + img_size, + dataset_type=dataset_type, + ) + + return quant_result_dict + + def get_quantitative( + self, + gt_polys, + gt_polys_types, + gt_window_doors, + gt_window_doors_types, + ignore_mask_region, + pred_polys=None, + pred_types=None, + pred_window_doors=None, + pred_window_doors_types=None, + masks_list=None, + img_size=(256, 256), + dataset_type="s3d", + ): + def get_room_metric(): + pred_overlaps = [False] * len(pred_room_map_list) + if not self.disable_overlapping_filter: + for pred_ind1 in range(len(pred_room_map_list) - 1): + pred_map1 = pred_room_map_list[pred_ind1] + + for pred_ind2 in range(pred_ind1 + 1, len(pred_room_map_list)): + pred_map2 = pred_room_map_list[pred_ind2] + + if dataset_type == "s3d": + kernel = np.ones((5, 5), np.uint8) + else: + kernel = np.ones((3, 3), np.uint8) + + # todo: for our method, the rooms share corners and edges, need to check here + pred_map1_er = cv2.erode(pred_map1, kernel) + pred_map2_er = cv2.erode(pred_map2, kernel) + + intersection = (pred_map1_er + pred_map2_er) == 2 + # intersection = (pred_map1 + pred_map2) == 2 + + intersection_area = np.sum(intersection) + + if intersection_area >= 1: + pred_overlaps[pred_ind1] = True + pred_overlaps[pred_ind2] = True + + # import pdb; pdb.set_trace() + room_metric = [np.bool((1 - pred_overlaps[ind]) * pred2gt_exists[ind]) for ind in range(len(pred_polys))] + if pred_types is not None: + room_sem_metric = [ + np.bool((1 - pred_overlaps[ind]) * pred2gt_exists_sem[ind]) for ind in range(len(pred_polys)) + ] + else: + room_sem_metric = None + return room_metric, room_sem_metric + + def get_corner_metric(): + + room_corners_metric = [] + for pred_poly_ind, gt_poly_ind in enumerate(pred2gt_indices): + p_poly = pred_polys[pred_poly_ind][:-1] # Last vertex = First vertex + + p_poly_corner_metrics = [False] * p_poly.shape[0] + if not room_metric[pred_poly_ind]: + room_corners_metric += p_poly_corner_metrics + continue + + gt_poly = gt_polys[gt_poly_ind][:-1] + + # for v in p_poly: + # v_dists = np.linalg.norm(v[None,:] - gt_poly, axis=1, ord=2) + # v_min_dist = np.min(v_dists) + # + # v_tp = v_min_dist <= 10 + # room_corners_metric.append(v_tp) + + for v in gt_poly: + v_dists = np.linalg.norm(v[None, :] - p_poly, axis=1, ord=2) + v_min_dist_ind = np.argmin(v_dists) + v_min_dist = v_dists[v_min_dist_ind] + + if not p_poly_corner_metrics[v_min_dist_ind]: + v_tp = v_min_dist <= corner_metric_thresh + p_poly_corner_metrics[v_min_dist_ind] = v_tp + + room_corners_metric += p_poly_corner_metrics + + return room_corners_metric + + def get_angle_metric(): + + def get_line_vector(p1, p2): + p1 = np.concatenate((p1, np.array([1]))) + p2 = np.concatenate((p2, np.array([1]))) + + line_vector = -np.cross(p1, p2) + + return line_vector + + def get_poly_orientation(my_poly): + angles_sum = 0 + for v_ind, _ in enumerate(my_poly): + if v_ind < len(my_poly) - 1: + v_sides = my_poly[[v_ind - 1, v_ind, v_ind, v_ind + 1], :] + else: + v_sides = my_poly[[v_ind - 1, v_ind, v_ind, 0], :] + + v1_vector = get_line_vector(v_sides[0], v_sides[1]) + v1_vector = v1_vector / (np.linalg.norm(v1_vector, ord=2) + 1e-4) + v2_vector = get_line_vector(v_sides[2], v_sides[3]) + v2_vector = v2_vector / (np.linalg.norm(v2_vector, ord=2) + 1e-4) + + orientation = (v_sides[1, 1] - v_sides[0, 1]) * (v_sides[3, 0] - v_sides[1, 0]) - ( + v_sides[3, 1] - v_sides[1, 1] + ) * (v_sides[1, 0] - v_sides[0, 0]) + + v1_vector_2d = v1_vector[:2] / (v1_vector[2] + 1e-4) + v2_vector_2d = v2_vector[:2] / (v2_vector[2] + 1e-4) + + v1_vector_2d = v1_vector_2d / (np.linalg.norm(v1_vector_2d, ord=2) + 1e-4) + v2_vector_2d = v2_vector_2d / (np.linalg.norm(v2_vector_2d, ord=2) + 1e-4) + + angle_cos = v1_vector_2d.dot(v2_vector_2d) + angle_cos = np.clip(angle_cos, -1, 1) + + # G.T. has clockwise orientation, remove minus in the equation + + angle = np.sign(orientation) * np.abs(np.arccos(angle_cos)) + angle_degree = angle * 180 / np.pi + + angles_sum += angle_degree + + return np.sign(angles_sum) + + def get_angle_v_sides(inp_v_sides, poly_orient): + v1_vector = get_line_vector(inp_v_sides[0], inp_v_sides[1]) + v1_vector = v1_vector / (np.linalg.norm(v1_vector, ord=2) + 1e-4) + v2_vector = get_line_vector(inp_v_sides[2], inp_v_sides[3]) + v2_vector = v2_vector / (np.linalg.norm(v2_vector, ord=2) + 1e-4) + + orientation = (inp_v_sides[1, 1] - inp_v_sides[0, 1]) * (inp_v_sides[3, 0] - inp_v_sides[1, 0]) - ( + inp_v_sides[3, 1] - inp_v_sides[1, 1] + ) * (inp_v_sides[1, 0] - inp_v_sides[0, 0]) + + v1_vector_2d = v1_vector[:2] / (v1_vector[2] + 1e-4) + v2_vector_2d = v2_vector[:2] / (v2_vector[2] + 1e-4) + + v1_vector_2d = v1_vector_2d / (np.linalg.norm(v1_vector_2d, ord=2) + 1e-4) + v2_vector_2d = v2_vector_2d / (np.linalg.norm(v2_vector_2d, ord=2) + 1e-4) + + angle_cos = v1_vector_2d.dot(v2_vector_2d) + angle_cos = np.clip(angle_cos, -1, 1) + + angle = poly_orient * np.sign(orientation) * np.arccos(angle_cos) + angle_degree = angle * 180 / np.pi + + return angle_degree + + room_angles_metric = [] + for pred_poly_ind, gt_poly_ind in enumerate(pred2gt_indices): + p_poly = pred_polys[pred_poly_ind][:-1] # Last vertex = First vertex + + p_poly_angle_metrics = [False] * p_poly.shape[0] + if not room_metric[pred_poly_ind]: + room_angles_metric += p_poly_angle_metrics + continue + + gt_poly = gt_polys[gt_poly_ind][:-1] + + # for v in p_poly: + # v_dists = np.linalg.norm(v[None,:] - gt_poly, axis=1, ord=2) + # v_min_dist = np.min(v_dists) + # + # v_tp = v_min_dist <= 10 + # room_corners_metric.append(v_tp) + + gt_poly_orient = get_poly_orientation(gt_poly) + p_poly_orient = get_poly_orientation(p_poly) + + for v_gt_ind, v in enumerate(gt_poly): + v_dists = np.linalg.norm(v[None, :] - p_poly, axis=1, ord=2) + v_ind = np.argmin(v_dists) + v_min_dist = v_dists[v_ind] + + if v_min_dist > corner_metric_thresh: + # room_angles_metric.append(False) + continue + + if v_ind < len(p_poly) - 1: + v_sides = p_poly[[v_ind - 1, v_ind, v_ind, v_ind + 1], :] + else: + v_sides = p_poly[[v_ind - 1, v_ind, v_ind, 0], :] + + v_sides = v_sides.reshape((4, 2)) + pred_angle_degree = get_angle_v_sides(v_sides, p_poly_orient) + + # Note: replacing some variables with values from the g.t. poly + + if v_gt_ind < len(gt_poly) - 1: + v_sides = gt_poly[[v_gt_ind - 1, v_gt_ind, v_gt_ind, v_gt_ind + 1], :] + else: + v_sides = gt_poly[[v_gt_ind - 1, v_gt_ind, v_gt_ind, 0], :] + + v_sides = v_sides.reshape((4, 2)) + gt_angle_degree = get_angle_v_sides(v_sides, gt_poly_orient) + + angle_metric = np.abs(pred_angle_degree - gt_angle_degree) + + # room_angles_metric.append(angle_metric < 5) + p_poly_angle_metrics[v_ind] = angle_metric <= angle_metric_thresh + + # if angle_metric > 5: + # print(v_gt_ind, angle_metric) + # print(pred_angle_degree, gt_angle_degree) + # input("?") + + room_angles_metric += p_poly_angle_metrics + + for am, cm in zip(room_angles_metric, corner_metric): + assert not (cm == False and am == True), "cm: %d am: %d" % (cm, am) + + return room_angles_metric + + def poly_map_sort_key(x): + return np.sum(x[1]) + + h, w = img_size + + gt_room_map_list = [] + for room_ind, poly in enumerate(gt_polys): + room_map = np.zeros((h, w)) + cv2.fillPoly(room_map, [poly], color=1.0) + + gt_room_map_list.append(room_map) + + gt_polys_sorted_indcs = [ + i[0] for i in sorted(enumerate(gt_room_map_list), key=poly_map_sort_key, reverse=True) + ] + + gt_polys = [gt_polys[ind] for ind in gt_polys_sorted_indcs] + gt_polys_types = [gt_polys_types[ind] for ind in gt_polys_sorted_indcs] + gt_room_map_list = [gt_room_map_list[ind] for ind in gt_polys_sorted_indcs] + + if pred_polys is not None: + pred_room_map_list = [] + for room_ind, poly in enumerate(pred_polys): + room_map = np.zeros((h, w)) + cv2.fillPoly(room_map, [poly], color=1.0) + + pred_room_map_list.append(room_map) + else: + pred_room_map_list = masks_list + + gt2pred_indices = [-1] * len(gt_polys) + gt2pred_exists = [False] * len(gt_polys) + + gt2pred_indices_sem = [-1] * len(gt_polys) + gt2pred_exists_sem = [False] * len(gt_polys) + + gt2pred_indices_wd = [-1] * len(gt_window_doors) + gt2pred_exists_wd = [False] * len(gt_window_doors) + + ### match predicted rooms to ground truth rooms + for gt_ind, gt_map in enumerate(gt_room_map_list): + best_iou = 0.0 + best_ind = -1 + best_ind_sem = -1 + for pred_ind, pred_map in enumerate(pred_room_map_list): + intersection = (1 - ignore_mask_region) * ((pred_map + gt_map) == 2) + union = (1 - ignore_mask_region) * ((pred_map + gt_map) >= 1) + # intersection = (pred_map + gt_map) == 2 + # union = (pred_map + gt_map) >= 1 + + iou = np.sum(intersection) / (np.sum(union) + 1) + + if iou > best_iou and iou > 0.5: + best_iou = iou + best_ind = pred_ind + + if pred_types is not None: + if gt_polys_types[gt_ind] == pred_types[pred_ind]: + best_ind_sem = pred_ind + + # plt.figure() + # plt.subplot(121) + # plt.imshow(pred_map) + # plt.subplot(122) + # plt.imshow(gt_map) + # plt.show() + # if best_ind == -1: + # plt.figure() + # plt.imshow(gt_map) + # plt.show() + + gt2pred_indices[gt_ind] = best_ind + gt2pred_exists[gt_ind] = best_ind != -1 + + if pred_types is not None: + gt2pred_indices_sem[gt_ind] = best_ind_sem + gt2pred_exists_sem[gt_ind] = best_ind_sem != -1 + + # if best_ind == -1: + # plt.figure() + # plt.imshow(gt_map) + # plt.show() + + ### match predicted window/door to ground truth window/door + if pred_window_doors_types is not None: + for gt_ind, gt_wd in enumerate(gt_window_doors): + best_dist = 100000.0 + best_ind = -1 + + for pred_ind, pred_wd in enumerate(pred_window_doors): + dist_match1 = [ + np.linalg.norm(gt_wd[0] - pred_wd[0], axis=0, ord=2), + np.linalg.norm(gt_wd[1] - pred_wd[1], axis=0, ord=2), + ] + dist_match2 = [ + np.linalg.norm(gt_wd[0] - pred_wd[1], axis=0, ord=2), + np.linalg.norm(gt_wd[1] - pred_wd[0], axis=0, ord=2), + ] + + dist_match = dist_match1 if sum(dist_match1) < sum(dist_match2) else dist_match2 + + if ( + sum(dist_match) < best_dist + and dist_match[0] < corner_metric_thresh + and dist_match[1] < corner_metric_thresh + and gt_window_doors_types[gt_ind] == pred_window_doors_types[pred_ind] + ): + best_dist = sum(dist_match) + best_ind = pred_ind + + gt2pred_indices_wd[gt_ind] = best_ind + gt2pred_exists_wd[gt_ind] = best_ind != -1 + + pred2gt_exists = [True if pred_ind in gt2pred_indices else False for pred_ind, _ in enumerate(pred_polys)] + pred2gt_indices = [ + gt2pred_indices.index(pred_ind) if pred_ind in gt2pred_indices else -1 + for pred_ind, _ in enumerate(pred_polys) + ] + + room_missing_ratio = 1.0 - sum(pred2gt_exists) / float(len(gt_polys) + 1e-4) + + if pred_types is not None: + pred2gt_exists_sem = [ + True if pred_ind in gt2pred_indices_sem else False for pred_ind, _ in enumerate(pred_polys) + ] + + if pred_window_doors_types is not None: + pred2gt_exists_wd = [ + True if pred_ind in gt2pred_indices_wd else False for pred_ind, _ in enumerate(pred_window_doors) + ] + + room_metric, room_sem_metric = get_room_metric() + ###### metric for room WITHOUT considering type ###### + if len(pred_polys) == 0: + room_metric_prec = 0 + else: + room_metric_prec = sum(room_metric) / float(len(pred_polys)) + room_metric_rec = sum(room_metric) / float(len(gt_polys)) + + ###### metric for room WITH considering type ###### + if pred_types is not None: + if len(pred_polys) == 0: + room_sem_metric_prec = 0 + else: + room_sem_metric_prec = sum(room_sem_metric) / float(len(pred_polys)) + room_sem_metric_rec = sum(room_sem_metric) / float(len(gt_polys)) + + ###### metric for window and door ###### + if pred_window_doors_types is not None: + if len(pred_window_doors) == 0: + window_door_metric_prec = 0 + else: + window_door_metric_prec = sum(pred2gt_exists_wd) / float(len(pred_window_doors)) + window_door_metric_rec = sum(pred2gt_exists_wd) / float(len(gt_window_doors)) + + ###### metric for corner ###### + corner_metric = get_corner_metric() + pred_corners_n = sum([poly.shape[0] - 1 for poly in pred_polys]) + gt_corners_n = sum([poly.shape[0] - 1 for poly in gt_polys]) + + if pred_corners_n > 0: + corner_metric_prec = sum(corner_metric) / float(pred_corners_n) + else: + corner_metric_prec = 0 + corner_metric_rec = sum(corner_metric) / float(gt_corners_n) + + ###### metric for angle ###### + angles_metric = get_angle_metric() + + if pred_corners_n > 0: + angles_metric_prec = sum(angles_metric) / float(pred_corners_n) + else: + angles_metric_prec = 0 + angles_metric_rec = sum(angles_metric) / float(gt_corners_n) + + # sanity check + assert room_metric_prec <= 1 + assert room_metric_rec <= 1 + assert corner_metric_prec <= 1 + assert corner_metric_rec <= 1 + assert angles_metric_prec <= 1 + assert angles_metric_rec <= 1 + + if pred_types is not None: + assert room_sem_metric_prec <= 1 + assert room_sem_metric_rec <= 1 + + if pred_window_doors_types is not None: + assert window_door_metric_prec <= 1 + assert window_door_metric_rec <= 1 + + result_dict = { + "room_prec": room_metric_prec, + "room_rec": room_metric_rec, + "corner_prec": corner_metric_prec, + "corner_rec": corner_metric_rec, + "angles_prec": angles_metric_prec, + "angles_rec": angles_metric_rec, + "room_missing_ratio": room_missing_ratio, + "pred2gt_indices": pred2gt_indices, + "gt_polys_sorted_indcs": gt_polys_sorted_indcs, + } + + if pred_types is not None: + result_dict["room_sem_prec"] = room_sem_metric_prec + result_dict["room_sem_rec"] = room_sem_metric_rec + + if pred_window_doors_types is not None: + result_dict["window_door_prec"] = window_door_metric_prec + result_dict["window_door_rec"] = window_door_metric_rec + + return result_dict diff --git a/evaluations/s3d_floorplan_eval/S3DLoader/S3DLoader.py b/evaluations/s3d_floorplan_eval/S3DLoader/S3DLoader.py new file mode 100644 index 0000000000000000000000000000000000000000..af48b815afeb8587a8c17495d836268c2a14b89e --- /dev/null +++ b/evaluations/s3d_floorplan_eval/S3DLoader/S3DLoader.py @@ -0,0 +1,319 @@ +import json +import os + +import cv2 +import numpy as np +import torch +from s3d_floorplan_eval.S3DLoader.s3d_utils import generate_floorplan, parse_floor_plan_polys +from torch.utils.data import DataLoader, Dataset + + +class S3DLoader(object): + def __init__(self, args, mode, generate_input_candidates=False): + self.mode = mode + self.seed = 8978 + np.random.seed(seed=self.seed) + + if hasattr(args, "network_mode"): + self.function_mode = args.network_mode + else: + self.function_mode = "S" + + if hasattr(args, "batch_size"): + self.batch_size = args.batch_size + else: + self.batch_size = 1 + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print("Selected device is:", device) + self.device = device + + if mode == "train": + self.dataset = self.create_dataset(args, mode, generate_input_candidates) + self.augment = True + self.data = DataLoader( + self.dataset, self.batch_size, drop_last=True, collate_fn=self.collate_fn, shuffle=True + ) + self.sample_n = len(self.dataset) + + elif mode == "online_eval" or mode == "test": + self.dataset = self.create_dataset(args, mode, generate_input_candidates) + self.augment = False + self.sample_n = len(self.dataset) + self.data = DataLoader(self.dataset, self.batch_size, drop_last=True, collate_fn=self.collate_fn) + + elif mode == "test": + self.dataset = self.create_dataset(args, mode) + self.augment = False + self.batch_size = 1 + self.sample_n = 20 + self.data = DataLoader( + self.dataset, self.batch_size, num_workers=1, drop_last=True, collate_fn=self.collate_fn + ) + + # elif mode == 'test': + # self.dataset = self.create_dataset(args, mode) + # self.augment = False + # + # self.data = DataLoader(self.dataset, + # 1, + # shuffle=False, + # num_workers=1) + # self.sample_n = 20 + + else: + print("mode should be one of 'train, test, online_eval'. Got {}".format(mode)) + + def collate_fn(self, samples): + + # wall_maps = [torch.tensor(s["wall_map"][None,:,:,None], device=self.device) for s in samples] + room_maps = [torch.tensor(s["room_map"][None, :, :, None], device=self.device) for s in samples] + input_maps = [torch.tensor(s["input_map"][None], device=self.device) for s in samples] + scores = [torch.tensor(s["score"][None], device=self.device) for s in samples] + + torch_sample = {} + torch_sample["room_map"] = torch.cat(room_maps, dim=0) + # torch_sample["wall_map"] = torch.cat(wall_maps, dim=0) + torch_sample["input_map"] = torch.cat(input_maps, dim=0) + torch_sample["score"] = torch.cat(scores, dim=0) + + for key, value in torch_sample.items(): + assert torch.all(torch_sample[key] == torch_sample[key]) + assert torch.all(torch.logical_not(torch.isinf(torch_sample[key]))) + + return torch_sample + + def create_dataset(self, args, mode, generate_input_candidates): + self.args = args + dataset_path = args.dataset_path + + if mode == "train": + scenes_path = os.path.join(dataset_path, "train") + dataset = S3DDataset( + args, + scenes_path, + None, + num_scenes=3000, + generate_input_candidates=generate_input_candidates, + mode=mode, + ) + + elif mode == "online_eval": + scenes_path = os.path.join(dataset_path, "val") + dataset = S3DDataset( + args, scenes_path, None, num_scenes=250, generate_input_candidates=generate_input_candidates, mode=mode + ) + elif mode == "test": + scenes_path = os.path.join(dataset_path, "test") + dataset = S3DDataset( + args, scenes_path, None, num_scenes=250, generate_input_candidates=generate_input_candidates, mode=mode + ) + + return dataset + + def load_sample(self, sample_batch): + """ + Identity function. Everything is already loaded in Dataset class for Structured 3D + :param sample_batch: + :return: + """ + return sample_batch + + +class S3DDataset(Dataset): + def __init__(self, options, scenes_path, score_gen, num_scenes, generate_input_candidates, mode): + print("Creating Structured3D Dataset with %d scenes..." % num_scenes) + self.options = options + self.score_gen = None + + self.mode = mode + + self.scenes_path = scenes_path + self.floor_data_folder_name = "" + + self.scenes_list = os.listdir(scenes_path) + self.scenes_list.sort() + + inv_scenes = [".DS_Store", "scene_01155", "scene_01852", "scene_01192", "scene_01816"] + self.scenes_list = [s for s in self.scenes_list if s not in inv_scenes] + self.scenes_list = self.scenes_list[:num_scenes] + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.device = device + + self.gen_input_candidates = generate_input_candidates + + def __getitem__(self, item): + scene_name = self.scenes_list[item] + sample = self.load_scene(scene_name) + + return sample + + def __len__(self): + return len(self.scenes_list) + + def load_density_map(self, sp): + """ + Load density map + + :param sp: + :return: + """ + density_path = os.path.join(sp, self.floor_data_folder_name, "density.png") + density_map = cv2.imread(density_path, cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH) / 255.0 + + if self.gen_input_candidates: + thresh = np.maximum(np.random.random(), 0.8) + density_map = np.minimum(density_map, thresh) / thresh + + if self.mode != "test": + pow = np.random.random() + pow = (1.5 - 1.0) * (pow - 1) + 1.5 + density_map = density_map**pow + + return density_map.astype(np.float32) + + def load_annotation(self, sp): + """ + Load annotation dict + + :param sp: + :return: + :rtype: dict + """ + anno_path = os.path.join(sp, self.floor_data_folder_name, "annotation_3d.json") + with open(anno_path, "r") as f: + anno_dict = json.load(f) + + return anno_dict + + def load_scene(self, scene_name): + """ + Load scene + + :param scene_name: + :return: + """ + + def cvt_tmp_sample_to_torch(): + torch_sample = {} + + room_map = torch.tensor(np.array(sample["room_map"]), device=self.device)[None] + + torch_sample["room_map"] = room_map + + if "input_map" in sample.keys(): + torch_sample["input_map"] = torch.tensor(np.array(sample["input_map"]), device=self.device)[None] + torch_sample["cand_inst"] = torch.tensor(np.array(sample["cand_inst"]), device=self.device)[None] + torch_sample["cand_confidence"] = torch.tensor( + np.array(sample["cand_confidence"]), device=self.device + )[None] + + else: + torch_sample["density_map"] = torch.tensor(np.array(sample["density_map"]), device=self.device)[None] + torch_sample["wall_map"] = torch.tensor(np.array(sample["wall_map"]), device=self.device)[None] + torch_sample["polygons_list"] = [ + torch.tensor(poly, device=self.device)[None] for poly in sample["polygons_list"] + ] + + return torch_sample + + sp = os.path.join(self.scenes_path, scene_name) + sample = {} + sample["scene_name"] = scene_name + + scene_anno = self.load_annotation(sp) + + # density_map = torch.tensor(np.array(density_map))[None] + density_map = self.load_density_map(sp) + + self.generate_room_map(sample, scene_anno, density_map) + + sample["density_map"] = density_map + + # import pdb; pdb.set_trace() + for key, value in sample.items(): + assert np.all(value == value), "%s contains NaN" % key + + # import matplotlib.pyplot as plt + # plt.figure() + # plt.subplot(131) + # plt.title(scene_name) + # plt.imshow(density_map) + # plt.subplot(132) + # plt.imshow(sample["room_map"]) + # plt.subplot(133) + # # plt.imshow(sample["input_map"][:,:,1]) + # # plt.imshow(sample["cand_inst"][:,:,0]) + # plt.show() + + return sample + + def generate_room_map(self, sample, annos, density_map): + """ + + :param density_map: + :param sample: + :param annos: + :return: + """ + + h, w = density_map.shape + + polys = parse_floor_plan_polys(annos) + + room_map, polygons_list, polygons_type_list = generate_floorplan( + annos, + polys, + h, + w, + ignore_types=["outwall", "door", "window"], + constant_color=False, + shuffle=self.gen_input_candidates, + ) + + room_map = cv2.dilate(room_map, np.ones((5, 5))) + + wall_map, _, _ = generate_floorplan( + annos, polys, h, w, ignore_types=[], include_types=["outwall"], constant_color=True + ) + wall_map *= room_map == 0 + + window_doors_map, window_doors_list, window_doors_type_list = generate_floorplan( + annos, + polys, + h, + w, + ignore_types=[], + include_types=["door", "window"], + fillpoly=False, + constant_color=True, + shuffle=self.gen_input_candidates, + ) + + sample["room_map"] = room_map.astype(np.float32) + sample["wall_map"] = wall_map.astype(np.float32) + + sample["polygons_list"] = polygons_list + sample["polygons_type_list"] = polygons_type_list + + sample["window_doors_list"] = window_doors_list + sample["window_doors_type_list"] = window_doors_type_list + + def generate_density(self, points, width=256, height=256): + image_res_tensor = torch.tensor([width, height], device=self.device).reshape(1, 1, 2) + + coordinates = torch.round(points[:, :, :2] * image_res_tensor) + coordinates = torch.minimum( + torch.maximum(coordinates, torch.zeros_like(image_res_tensor)), image_res_tensor - 1 + ).type(torch.cuda.LongTensor) + + density = torch.zeros((self.batch_size, height, width), dtype=torch.float, device=self.device) + + for i in range(self.batch_size): + unique_coordinates, counts = torch.unique(coordinates[i], return_counts=True, dim=0) + + density[i, unique_coordinates[:, 1], unique_coordinates[:, 0]] = counts.type(torch.cuda.FloatTensor) + density[i] = density[i] / torch.max(density[i]) + + return density diff --git a/evaluations/s3d_floorplan_eval/S3DLoader/poly_utils.py b/evaluations/s3d_floorplan_eval/S3DLoader/poly_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..16bf3e29782a25b15373693a642df66df5249612 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/S3DLoader/poly_utils.py @@ -0,0 +1,29 @@ +import numpy as np + + +def rotate_poly(poly, angle, flip_h): + """ + Rotate poly + + :param poly: + :return: + """ + + px, py = poly[:, 0], poly[:, 1] + + angle_rad = angle * np.pi / 180 + + qx = np.cos(angle_rad) * px - np.sin(angle_rad) * py + qy = np.sin(angle_rad) * px + np.cos(angle_rad) * py + + if flip_h: + qx = -qx + + rotated_poly = np.zeros_like(poly) + rotated_poly[:, 0] = qx + rotated_poly[:, 1] = qy + + # print("p", poly) + # print("r", rotated_poly) + + return rotated_poly diff --git a/evaluations/s3d_floorplan_eval/S3DLoader/s3d_utils.py b/evaluations/s3d_floorplan_eval/S3DLoader/s3d_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9eeffea14fa7a6ddabd2928cf22a4f786676f9f5 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/S3DLoader/s3d_utils.py @@ -0,0 +1,188 @@ +""" +This code is an adaptation that uses Structured 3D for the code base. + +Reference: https://github.com/bertjiazheng/Structured3D +""" + +import random + +import cv2 +import numpy as np +from shapely.geometry import Polygon + +type2id = { + "living room": 0, + "kitchen": 1, + "bedroom": 2, + "bathroom": 3, + "balcony": 4, + "corridor": 5, + "dining room": 6, + "study": 7, + "studio": 8, + "store room": 9, + "garden": 10, + "laundry room": 11, + "office": 12, + "basement": 13, + "garage": 14, + "undefined": 15, + "door": 16, + "window": 17, + "outwall": -1, +} + + +def parse_floor_plan_polys(annos): + planes = [] + for semantic in annos["semantics"]: + for planeID in semantic["planeID"]: + if annos["planes"][planeID]["type"] == "floor": + planes.append({"planeID": planeID, "type": semantic["type"]}) + + if semantic["type"] == "outwall": + outerwall_planes = semantic["planeID"] + + # extract hole vertices + lines_holes = [] + for semantic in annos["semantics"]: + if semantic["type"] in ["window", "door"]: + for planeID in semantic["planeID"]: + lines_holes.extend(np.where(np.array(annos["planeLineMatrix"][planeID]))[0].tolist()) + lines_holes = np.unique(lines_holes) + + # junctions on the floor + junctions = np.array([junc["coordinate"] for junc in annos["junctions"]]) + junction_floor = np.where(np.isclose(junctions[:, -1], 0))[0] + + # construct each polygon + polygons = [] + for plane in planes: + lineIDs = np.where(np.array(annos["planeLineMatrix"][plane["planeID"]]))[0].tolist() + junction_pairs = [np.where(np.array(annos["lineJunctionMatrix"][lineID]))[0].tolist() for lineID in lineIDs] + polygon = convert_lines_to_vertices(junction_pairs) + polygons.append([polygon[0], plane["type"]]) + + outerwall_floor = [] + for planeID in outerwall_planes: + lineIDs = np.where(np.array(annos["planeLineMatrix"][planeID]))[0].tolist() + lineIDs = np.setdiff1d(lineIDs, lines_holes) + junction_pairs = [np.where(np.array(annos["lineJunctionMatrix"][lineID]))[0].tolist() for lineID in lineIDs] + for start, end in junction_pairs: + if start in junction_floor and end in junction_floor: + outerwall_floor.append([start, end]) + + outerwall_polygon = convert_lines_to_vertices(outerwall_floor) + polygons.append([outerwall_polygon[0], "outwall"]) + + return polygons + + +def convert_lines_to_vertices(lines): + """ + convert line representation to polygon vertices + + """ + polygons = [] + lines = np.array(lines) + + polygon = None + while len(lines) != 0: + if polygon is None: + polygon = lines[0].tolist() + lines = np.delete(lines, 0, 0) + + lineID, juncID = np.where(lines == polygon[-1]) + vertex = lines[lineID[0], 1 - juncID[0]] + lines = np.delete(lines, lineID, 0) + + if vertex in polygon: + polygons.append(polygon) + polygon = None + else: + polygon.append(vertex) + + return polygons + + +def generate_floorplan( + annos, + polygons, + height, + width, + ignore_types, + include_types=None, + fillpoly=True, + constant_color=False, + shuffle=False, +): + """ + plot floorplan + + """ + + floor_map = np.zeros((height, width)) + + junctions = np.array([junc["coordinate"][:2] for junc in annos["junctions"]]) + + room_ind = 0 + if shuffle: + room_ind = np.random.randint(0, 2) + + polygons_list = [] + polygons_type_list = [] + for poly_ind, (polygon, poly_type) in enumerate(polygons): + if poly_type in ignore_types: + continue + if include_types is not None and poly_type not in include_types: + continue + + polygon = junctions[np.array(polygon)].astype(np.int32) + + poly_shapely = Polygon(polygon) + area = poly_shapely.area + + # assert area > 10 + # if area < 100: + # continue + if poly_type not in ["door", "window"] and area < 100: + continue + if poly_type in ["door", "window"] and area < 1: + continue + + if poly_type in ["door", "window"]: + assert polygon.shape[0] == 4 + midp_1 = (polygon[0] + polygon[1]) / 2 + midp_2 = (polygon[1] + polygon[2]) / 2 + midp_3 = (polygon[2] + polygon[3]) / 2 + midp_4 = (polygon[3] + polygon[0]) / 2 + + dist_1_3 = np.square(midp_1 - midp_3).sum() + dist_2_4 = np.square(midp_2 - midp_4).sum() + if dist_1_3 > dist_2_4: + polygon = np.row_stack([midp_1, midp_3]) + else: + polygon = np.row_stack([midp_2, midp_4]) + + polygons_list.append(polygon) + polygons_type_list.append(type2id[poly_type]) + + if shuffle: + random.shuffle(polygons_list) + for poly_ind, polygon in enumerate(polygons_list): + if shuffle: + room_ind += np.random.randint(1, 2) + else: + room_ind += 1 + + if fillpoly: + if constant_color: + clr = 1.0 + else: + clr = room_ind + cv2.fillPoly(floor_map, [polygon], color=clr) + else: + assert constant_color + cv2.polylines(floor_map, [polygon.astype(np.int32)], isClosed=True, color=1.0, thickness=3) + + return floor_map, polygons_list, polygons_type_list diff --git a/evaluations/s3d_floorplan_eval/convert_density.py b/evaluations/s3d_floorplan_eval/convert_density.py new file mode 100644 index 0000000000000000000000000000000000000000..13a923c8876fc98fb94e64f9b085d731cef05104 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/convert_density.py @@ -0,0 +1,19 @@ +import os + +import cv2 +import numpy as np + +source = "../Structured3D/montefloor_data/test/" +dst = "./viz_density" + +for dirname in sorted(os.listdir(source)): + density_path = os.path.join(source, dirname, "density.png") + density_img = cv2.imread(density_path) + density = 255 - density_img + out_path = os.path.join(dst, dirname + ".png") + out_alpha_path = os.path.join(dst, dirname + "_alpha.png") + alphas = np.zeros([density.shape[0], density.shape[1], 1], dtype=np.int32) + alphas[density_img.sum(axis=-1) > 0] = 255 + density_alpha = np.concatenate([density, alphas], axis=-1) + cv2.imwrite(out_path, density) + cv2.imwrite(out_alpha_path, density_alpha) diff --git a/evaluations/s3d_floorplan_eval/evaluate_solution.py b/evaluations/s3d_floorplan_eval/evaluate_solution.py new file mode 100644 index 0000000000000000000000000000000000000000..479db335c3d75f49337cf12bbb8a3221ddb56187 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/evaluate_solution.py @@ -0,0 +1,95 @@ +import copy +import os + +import numpy as np +from DataRW.S3DRW import S3DRW +from DataRW.wrong_annotatios import wrong_s3d_annotations_list +from Evaluator.Evaluator import Evaluator +from options import MCSSOptions +from planar_graph_utils import get_regions_from_pg + +room_polys_def = [ + np.array([[191, 150], [191, 70], [222, 70], [222, 150], [191, 150]]), + np.array([[232, 65], [232, 11], [202, 11], [202, 65], [232, 65]]), + np.array([[47, 50], [47, 150], [24, 150], [24, 50], [47, 50]]), + np.array([[199, 156], [199, 234], [146, 234], [146, 156], [199, 156]]), + np.array([[109, 184], [120, 184], [120, 156], [50, 156], [50, 234], [109, 234], [109, 184]]), + np.array([[110, 234], [144, 234], [144, 187], [110, 187], [110, 234]]), + np.array( + [ + [50, 50], + [50, 150], + [123, 150], + [123, 184], + [144, 184], + [144, 150], + [190, 150], + [190, 70], + [108, 70], + [108, 50], + [50, 50], + ] + ), +] + +# pg_base = 'results/npy_heat_s3d_256/' +pg_base = "results/test_gt/" +# pg_base = 'results/test_eval2/' + +options = MCSSOptions() +opts = options.parse() + +if __name__ == "__main__": + # data_rw = FloorNetRW(opts) + + if opts.scene_id == "val": + opts.scene_id = "scene_03250" # Temp. value + data_rw = S3DRW(opts) + scene_list = data_rw.loader.scenes_list + + quant_result_dict = None + quant_result_maskrcnn_dict = None + scene_counter = 0 + for scene_ind, scene in enumerate(scene_list): + if int(scene[6:]) in wrong_s3d_annotations_list: + continue + + print("------------") + curr_opts = copy.deepcopy(opts) + curr_opts.scene_id = scene + curr_data_rw = S3DRW(curr_opts) + print("Running Evaluation for scene %s" % scene) + + evaluator = Evaluator(curr_data_rw, curr_opts) + + # TODO load your room polygons into room_polys, list of polygons (n x 2) + # room_polys = np.array([[[0,0], [200, 0], [200, 200]]]) # Placeholder + + pg_path = os.path.join(pg_base, scene[6:] + ".npy") + example_pg = np.load(pg_path, allow_pickle=True).tolist() + example_pg["corners"] = example_pg["corners"][:8] + example_pg["edges"] = example_pg["edges"][:8] + regions = get_regions_from_pg(example_pg, corner_sorted=True) + # regions = [np.array(re, dtype=np.int32) for re in example_pg] + room_polys = regions + # room_polys = room_polys_def # Placeholder + + quant_result_dict_scene = evaluator.evaluate_scene(room_polys=room_polys) + + if quant_result_dict is None: + quant_result_dict = quant_result_dict_scene + else: + for k in quant_result_dict.keys(): + quant_result_dict[k] += quant_result_dict_scene[k] + + scene_counter += 1 + + # break + + for k in quant_result_dict.keys(): + quant_result_dict[k] /= float(scene_counter) + + print("Our: ", quant_result_dict) + + print("Ours") + evaluator.print_res_str_for_latex(quant_result_dict) diff --git a/evaluations/s3d_floorplan_eval/generate_html.py b/evaluations/s3d_floorplan_eval/generate_html.py new file mode 100644 index 0000000000000000000000000000000000000000..e16a9024bc1d7ca0de0a43b87d6ce5488946664f --- /dev/null +++ b/evaluations/s3d_floorplan_eval/generate_html.py @@ -0,0 +1,63 @@ +import os.path as osp + +import numpy as np + +head = """ + + + + +

+

+
+ +""" + +end = """ +
+
` + +""" + + +def writeHTML(out_path, results_dirs): + f = open(out_path, "w") + f.write(head + "\n") + f.write( + "" + ' ID ' + ' Input ' + ' HAWP ' + ' LETR ' + ' HEAT (Ours) ' + ' Ground-truth ' + "" + ) + + wrong_s3d_annotations_list = [3261, 3271, 3276, 3296, 3342, 3387, 3398, 3466, 3496] + file_ids = ["0{}".format(x) for x in range(3250, 3500) if x not in wrong_s3d_annotations_list] + permuted_ids = np.random.permutation(file_ids) + file_ids = permuted_ids[:100] + + for file_id in file_ids: + row_str = "" + row_str += " {} ".format(file_id) + for dir_idx, result_dir in enumerate(results_dirs): + if dir_idx == 0: + pred_filepath = osp.join(result_dir, "scene_{}_alpha.png".format(file_id)) + row_str += ' '.format(pred_filepath) + else: + pred_filepath = osp.join(result_dir, "{}.png".format(file_id)) + row_str += ' '.format(pred_filepath) + row_str += "" + f.write(row_str + "\n") + + f.write(end + "\n") + + +if __name__ == "__main__": + results_dirs = ["viz_density", "viz_hawp", "viz_letr", "viz_heat_th5", "viz_gt"] + + writeHTML(out_path="./indoor_qual.html", results_dirs=results_dirs) diff --git a/evaluations/s3d_floorplan_eval/options.py b/evaluations/s3d_floorplan_eval/options.py new file mode 100644 index 0000000000000000000000000000000000000000..cfa92387ccfcdfb8c5e907d62ef91a12cda3aaf6 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/options.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import, division, print_function + +import argparse +import os + +file_dir = os.path.dirname(__file__) # the directory that options.py resides in + + +class MCSSOptions: + def __init__(self): + self.parser = argparse.ArgumentParser(description="MCSSFloor options") + + # PATHS + self.parser.add_argument( + "--mcts_path", + type=str, + help="the name of the MonteFloorNet model", + default="/media/sinisa/Sinisa_hdd_data/Sinisa_Projects/corridor_localisation/experiments/MonteFloorNet_experiments/room_shape_experiments/Structured3D_test/", + ) + + self.parser.add_argument( + "--dataset_path", + type=str, + help="the name of the MonteFloorNet model", + default="evaluations/s3d_floorplan_eval/montefloor_data", + ) + + self.parser.add_argument( + "--dataset_type", + type=str, + help="the name of the dataset type", + choices=["floornet", "s3d", "fsp"], + default="s3d", + ) + self.parser.add_argument("--scene_id", type=str, help="the name of the scene", default="val") + + self.parser.add_argument("--min_scene_ind", type=int, help="the name of the scene", default=0) + self.parser.add_argument("--max_scene_ind", type=int, help="the name of the scene", default=251) + + # MonteFloorNet options + # self.parser.add_argument("--model_S_path", + # help="the name of the MonteFloorNet model", + # default="/home/sinisa/tmp/current_experiments/montefloornet_S_model_camera_ready16/best_models/weights_935") + + self.parser.add_argument("--height", type=int, help="input image height", default=256) + self.parser.add_argument("--width", type=int, help="input image width", default=256) + + def parse(self): + self.options, unknown = self.parser.parse_known_args() + return self.options diff --git a/evaluations/s3d_floorplan_eval/planar_graph_utils.py b/evaluations/s3d_floorplan_eval/planar_graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..70e09e9f5bef44527ea6ef8b1c97fa1e138531f3 --- /dev/null +++ b/evaluations/s3d_floorplan_eval/planar_graph_utils.py @@ -0,0 +1,351 @@ +import cv2 +import numpy as np +from scipy import ndimage + + +def extract_regions(adj_mat, corners, corner_sorted): + all_regions = list() + cur_idx = 0 + corners = corners.astype(np.int) + nb_orders = _sort_neighours(adj_mat, corners) + while cur_idx is not None: + regions = _get_regions_for_corner(cur_idx, adj_mat, nb_orders) + all_regions.extend(regions) + cur_idx = _get_new_start(adj_mat, cur_idx, corners) + + outwall_idx = get_outwall(all_regions, corners, corner_sorted) + all_regions.pop(outwall_idx) + + # all_regions = filter_regions(all_regions) # only used for drawing visualization + # return all_regions + + all_regions_coords = [corners[regions] for regions in all_regions] + return all_regions_coords + + +def get_outwall(all_regions, corners, corner_sorted): + """ + Find the outermost boundary loop, which should be discarded + """ + if corner_sorted: + regions_for_top_bot = np.nonzero([(0 in region and len(corners) - 1 in region) for region in all_regions])[0] + if len(regions_for_top_bot) == 1: + return regions_for_top_bot[0] + else: + areas = [_compute_region_area(corners[all_regions[idx]]) for idx in range(len(all_regions))] + max_idx = np.argmax(areas) + return max_idx + else: + areas = [_compute_region_area(corners[all_regions[idx]]) for idx in range(len(all_regions))] + max_idx = np.argmax(areas) + return max_idx + + +# def filter_regions(all_regions): +# areas = [_compute_region_area(corners[all_regions[idx]]) for idx in range(len(all_regions))] +# all_regions = [region for idx, region in enumerate(all_regions) if areas[idx] > 20] +# return all_regions + + +def _compute_region_area(region): + edge_map = np.zeros([256, 256]) + for idx, c in enumerate(region[:-1]): + cv2.line(edge_map, tuple(c), tuple(region[idx + 1]), 1, 3) + reverse_edge_map = 1 - edge_map + label, num_features = ndimage.label(reverse_edge_map) + if num_features < 2: + return 0 + # import pdb; pdb.set_trace() + # raise ValueError('Invalid region structure') + bg_label = label[0, 0] + num_labels = [(label == l).sum() for l in range(1, num_features + 1)] + num_labels[bg_label - 1] = 0 + room_label = np.argmax(num_labels) + 1 + area = (label == room_label).sum() + return area + + +def _get_regions_for_corner(cur_idx, adj_mat, nb_orders): + regions = list() + if adj_mat[cur_idx].sum() == 0: + assert ValueError("Zero-degree corner, should not reach here") + # elif adj_mat[cur_idx].sum() == 1: # remove the connection if only one neighbour + # other_idx = nb_orders[0] + # import pdb; pdb.set_trace() + # adj_mat[cur_idx, other_idx] = 0 + else: + v_s = cur_idx + know_v_q = False + while v_s is not None: + if not know_v_q: + v_p, v_q = _find_wedge_nbs(v_s, nb_orders, adj_mat) + if v_p is None: # cannot find proper wedge, remove this corner + adj_mat[v_s, :] = 0 + adj_mat[:, v_s] = 0 + break + else: + assert v_q is not None, "v_q should be known here" + v_p = _find_wedge_third_v(v_q, v_s, nb_orders, adj_mat, dir=-1) + if v_p is None: + adj_mat[v_s, :] = 0 + adj_mat[:, v_s] = 0 + break + cur_region = [ + v_p, + v_s, + ] + # try: + assert adj_mat[v_p, v_s] == 1, "Wrong connection matrix!" + # except: + # import pdb; pdb.set_trace() + adj_mat[v_p, v_s] = 0 + region_i = 0 + closed_polygon = False + while v_q is not None: # tracking the current region + cur_region.append(v_q) + assert adj_mat[v_s, v_q] == 1, "Wrong connection matrix!" + adj_mat[v_s, v_q] = 0 + # update the nb order list for the current v_s + if v_q == cur_region[0]: # get a closed polygon + closed_polygon = True + break + else: + v_p = cur_region[region_i + 1] + v_s = cur_region[region_i + 2] + v_q = _find_wedge_third_v(v_p, v_s, nb_orders, adj_mat, dir=1) + if v_q is None: + closed_polygon = False + break + region_i += 1 + + if closed_polygon: # find a closed region, keep iteration + regions.append(cur_region) + found_next = False + for temp_i in range(1, len(cur_region)): + if adj_mat[cur_region[temp_i], cur_region[temp_i - 1]] == 1: + found_next = True + v_s_idx = temp_i + break + if not found_next: + v_s = None + else: + v_s = cur_region[v_s_idx] + v_q = cur_region[v_s_idx - 1] + know_v_q = True + else: # no closed region, directly quit the search for the current v_s + break + return regions + + +def _find_wedge_nbs(v_s, nb_orders, adj_mat): + sorted_nbs = nb_orders[v_s] + start_idx = 0 + while True: + if start_idx == -len(sorted_nbs): + return None, None + v_p, v_q = sorted_nbs[start_idx], sorted_nbs[start_idx - 1] + if adj_mat[v_p, v_s] == 1 and adj_mat[v_s, v_q] == 1: + return v_p, v_q + else: + start_idx -= 1 + + +def _find_wedge_third_v(v1, v2, nb_orders, adj_mat, dir): + sorted_nbs = nb_orders[v2] + v1_idx = sorted_nbs.index(v1) + if dir == 1: + v3_idx = v1_idx - 1 + while adj_mat[v2, sorted_nbs[v3_idx]] == 0: + if sorted_nbs[v3_idx] == v1: + return None + v3_idx -= 1 + elif dir == -1: + v3_idx = v1_idx + 1 if v1_idx <= len(sorted_nbs) - 2 else 0 + while adj_mat[sorted_nbs[v3_idx], v2] == 0: + if sorted_nbs[v3_idx] == v1: + return None + v3_idx = v3_idx + 1 if v3_idx <= len(sorted_nbs) - 2 else 0 + else: + raise ValueError("unknown dir {}".format(dir)) + return sorted_nbs[v3_idx] + + +def _get_new_start(adj_mat, cur_idx, corners): + for i in range(cur_idx, len(corners)): + if adj_mat[i].sum() > 0: + return i + return None + + +def _sort_neighours(adj_mat, corners): + nb_orders = dict() + for idx, c in enumerate(corners): + nb_ids = np.nonzero(adj_mat[idx])[0] + nb_degrees = [_compute_degree(c, corners[other_idx]) for other_idx in nb_ids] + degree_ranks = np.argsort(nb_degrees) + sort_nb_ids = [nb_ids[i] for i in degree_ranks] + nb_orders[idx] = sort_nb_ids + return nb_orders + + +def _compute_degree(c1, c2): + vec = (c2[0] - c1[0], -(c2[1] - c1[1])) # note that the y direction should be flipped (image coord system) + cos = (vec[0] * 1 + vec[1] * 0) / np.sqrt(vec[0] ** 2 + vec[1] ** 2) + theta = np.arccos(cos) + if vec[1] < 0: + theta = np.pi * 2 - theta + return theta + + +def preprocess_pg(pg): + corners = pg["corners"] + edge_pairs = pg["edges"] + adj_mat = np.zeros([len(corners), len(corners)]) + for edge_pair in edge_pairs: + c1, c2 = edge_pair + adj_mat[c1][c2] = 1 + adj_mat[c2][c1] = 1 + + return corners, adj_mat + + +def cleanup_pg(pg): + corners = pg["corners"] + edge_pairs = pg["edges"] + adj_list = [[] for _ in range(len(corners))] + + for edge_pair in edge_pairs: + adj_list[edge_pair[0]].append(edge_pair[1]) + adj_list[edge_pair[1]].append(edge_pair[0]) + + for idx in range(len(corners)): + if len(adj_list[idx]) < 2: + _remove_corner(idx, adj_list) + + new_corners = list() + removed_ids = list() + old_to_new = dict() + counter = 0 + for c_i in range(len(adj_list)): + if len(adj_list[c_i]) > 0: + assert len(adj_list[c_i]) >= 2 + new_corners.append(corners[c_i]) + old_to_new[c_i] = counter + counter += 1 + else: + removed_ids.append(c_i) + + new_edges = list() + for c_i_1 in range(len(adj_list)): + for c_i_2 in adj_list[c_i_1]: + if c_i_1 < c_i_2: + new_edge = (old_to_new[c_i_1], old_to_new[c_i_2]) + new_edges.append(new_edge) + new_corners = np.array(new_corners) + new_edges = np.array(new_edges) + new_pg = { + "corners": new_corners, + "edges": new_edges, + } + return new_pg + + +def _remove_corner(idx, adj_list): + assert len(adj_list[idx]) <= 1 + if len(adj_list[idx]) == 0: + return + nbs = list(adj_list[idx]) + adj_list[idx].pop(0) + for nb in nbs: + adj_list[nb].remove(idx) + if len(adj_list[nb]) < 2: + _remove_corner(nb, adj_list) + + +def get_regions_from_pg(pg, corner_sorted): + pg = cleanup_pg(pg) + corners, adj_mat = preprocess_pg(pg) + if len(corners) == 0: + regions = [] + else: + regions = extract_regions(adj_mat, corners, corner_sorted) + return regions + + +def convert_annot(annot): + corners = np.array(list(annot.keys())) + corners_mapping = {tuple(c): idx for idx, c in enumerate(corners)} + edges = set() + for corner, connections in annot.items(): + idx_c = corners_mapping[tuple(corner)] + for other_c in connections: + idx_other_c = corners_mapping[tuple(other_c)] + if (idx_c, idx_other_c) not in edges and (idx_other_c, idx_c) not in edges: + edges.add((idx_c, idx_other_c)) + edges = np.array(list(edges)) + pg_data = {"corners": corners, "edges": edges} + return pg_data + + +colors_12 = [ + "#DCECC9", + "#B3DDCC", + "#8ACDCE", + "#62BED2", + "#46AACE", + "#3D91BE", + "#3677AE", + "#2D5E9E", + "#24448E", + "#1C2B7F", + "#162165", + "#11174B", +] + + +def plot_floorplan_with_regions(regions, corners, edges, scale): + colors = colors_12[:8] + + regions = [(region * scale / 256).round().astype(np.int) for region in regions] + corners = (corners * scale / 256).round().astype(np.int) + + # define the color map + room_colors = [colors[i % 8] for i in range(len(regions))] + + colorMap = [tuple(int(h[i : i + 2], 16) for i in (1, 3, 5)) for h in room_colors] + colorMap = np.asarray(colorMap) + if len(regions) > 0: + colorMap = np.concatenate([np.full(shape=(1, 3), fill_value=0), colorMap], axis=0).astype(np.uint8) + else: + colorMap = np.concatenate([np.full(shape=(1, 3), fill_value=0)], axis=0).astype(np.uint8) + # when using opencv, we need to flip, from RGB to BGR + colorMap = colorMap[:, ::-1] + + alpha_channels = np.zeros(colorMap.shape[0], dtype=np.uint8) + alpha_channels[1 : len(regions) + 1] = 150 + + colorMap = np.concatenate([colorMap, np.expand_dims(alpha_channels, axis=-1)], axis=-1) + + room_map = np.zeros([scale, scale]).astype(np.int32) + # sort regions + if len(regions) > 1: + avg_corner = [region.mean(axis=0) for region in regions] + ind = np.argsort(np.array(avg_corner)[:, 0], axis=0) + regions = np.array(regions)[ind] + + for idx, polygon in enumerate(regions): + cv2.fillPoly(room_map, [polygon], color=idx + 1) + + image = colorMap[room_map.reshape(-1)].reshape((scale, scale, 4)) + + pointColor = tuple((np.array([0.95, 0.3, 0.3, 1]) * 255).astype(np.uint8).tolist()) + for point in corners: + cv2.circle(image, tuple(point), color=pointColor, radius=12, thickness=-1) + cv2.circle(image, tuple(point), color=(255, 255, 255, 255), radius=6, thickness=-1) + + for edge in edges: + c1 = corners[edge[0]] + c2 = corners[edge[1]] + cv2.line(image, tuple(c1), tuple(c2), color=(0, 0, 0, 255), thickness=3) + + return image diff --git a/evaluations/s3d_floorplan_eval/visualize_npy.py b/evaluations/s3d_floorplan_eval/visualize_npy.py new file mode 100644 index 0000000000000000000000000000000000000000..51ec55edeef4dba95670430f4cdd5e06d93af45c --- /dev/null +++ b/evaluations/s3d_floorplan_eval/visualize_npy.py @@ -0,0 +1,33 @@ +import os + +import cv2 +import numpy as np +from planar_graph_utils import get_regions_from_pg, plot_floorplan_with_regions + +# example_pg = { +# (127, 20): [(20, 120), (234, 120)], +# (20, 120): [(127, 20), (234, 120), (20, 240)], +# (234, 120): [(127, 20), (20, 120), (234, 240)], +# (20, 240): [(20, 120), (234, 240)], +# (234, 240): [(234, 120), (20, 240)], +# } + +# pg_base = '../results/npy_heat_s3d_256/' +pg_base = "../results/test_gt/" +viz_base = "./viz_gt" +if not os.path.exists(viz_base): + os.makedirs(viz_base) + +for filename in sorted(os.listdir(pg_base)): + pg_path = os.path.join(pg_base, filename) + example_pg = np.load(pg_path, allow_pickle=True).tolist() + + corners = example_pg["corners"] + corners = corners.astype(np.int) + edges = example_pg["edges"] + + print("Processing file: {}".format(filename)) + regions = get_regions_from_pg(example_pg, corner_sorted=True) + print("num of extracted regions {}".format(len(regions))) + floorplan_image = plot_floorplan_with_regions(regions, corners, edges, scale=1000) + cv2.imwrite(os.path.join(viz_base, "{}.png".format(filename[:-4])), floorplan_image) diff --git a/main_ddp.py b/main_ddp.py new file mode 100644 index 0000000000000000000000000000000000000000..b8a62e7b2bbead972c00b2d046729dfcf148a18c --- /dev/null +++ b/main_ddp.py @@ -0,0 +1,624 @@ +import argparse +import copy +import datetime +import json +import os +import random +import time +from pathlib import Path + +import numpy as np +import torch +import torch.distributed as dist +from torch.nn import functional as F +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +import util.misc as utils +import wandb +from datasets import build_dataset +from engine import evaluate, train_one_epoch +from models import build_model + + +def get_args_parser(): + parser = argparse.ArgumentParser("Raster2Seq training script", add_help=False) + parser.add_argument("--lr", default=2e-4, type=float) + parser.add_argument("--lr_backbone_names", default=["backbone.0"], type=str, nargs="+") + parser.add_argument("--lr_backbone", default=2e-5, type=float) + parser.add_argument("--lr_linear_proj_names", default=["sampling_offsets"], type=str, nargs="+") + parser.add_argument("--lr_linear_proj_mult", default=0.1, type=float) + parser.add_argument("--batch_size", default=10, type=int) + parser.add_argument("--weight_decay", default=1e-4, type=float) + parser.add_argument("--epochs", default=500, type=int) + parser.add_argument("--lr_drop", default="400", type=str) + parser.add_argument("--clip_max_norm", default=0.1, type=float, help="gradient clipping max norm") + + parser.add_argument("--sgd", action="store_true") + + parser.add_argument("--input_channels", default=1, type=int) + parser.add_argument("--start_from_checkpoint", default="", help="resume from checkpoint") + parser.add_argument("--image_norm", action="store_true") + parser.add_argument("--debug", action="store_true") + parser.add_argument("--eval_every_epoch", type=int, default=20) + parser.add_argument("--ckpt_every_epoch", type=int, default=20) + parser.add_argument("--label_smoothing", type=float, default=0.0) + parser.add_argument("--ignore_index", type=int, default=-1) + parser.add_argument("--image_size", type=int, default=256) + parser.add_argument("--ema4eval", action="store_true") + parser.add_argument("--increase_cls_loss_coef", default=1.0, type=float) + parser.add_argument("--increase_cls_loss_coef_epoch_ratio", default=-1, type=float) + parser.add_argument("--use_anchor", action="store_true") + parser.add_argument("--disable_wd_as_line", action="store_true") + parser.add_argument("--wd_only", action="store_true") + parser.add_argument("--converter_version", type=str, default="v1") + parser.add_argument("--freeze_anchor", action="store_true") + parser.add_argument("--inject_cls_embed", action="store_true") + parser.add_argument( + "--random_drop_rate", type=float, default=0.0, help="randomly drop some polygons during training" + ) + + # raster2seq + parser.add_argument("--poly2seq", action="store_true") + parser.add_argument("--seq_len", type=int, default=1024) + parser.add_argument("--num_bins", type=int, default=64) + parser.add_argument("--pre_decoder_pos_embed", action="store_true") + parser.add_argument("--learnable_dec_pe", action="store_true") + parser.add_argument("--dec_qkv_proj", action="store_true") + parser.add_argument("--dec_attn_concat_src", action="store_true") + parser.add_argument("--per_token_sem_loss", action="store_true") + parser.add_argument("--add_cls_token", action="store_true") + parser.add_argument("--jointly_train", action="store_true") + + # parser.add_argument('--use_room_attn_at_last_dec_layer', default=False, action='store_true', help="use room-wise attention in last decoder layer") + + # backbone + parser.add_argument("--backbone", default="resnet50", type=str, help="Name of the convolutional backbone to use") + parser.add_argument( + "--dilation", + action="store_true", + help="If true, we replace stride with dilation in the last convolutional block (DC5)", + ) + parser.add_argument( + "--position_embedding", + default="sine", + type=str, + choices=("sine", "learned"), + help="Type of positional embedding to use on top of the image features", + ) + parser.add_argument("--position_embedding_scale", default=2 * np.pi, type=float, help="position / size * scale") + parser.add_argument("--num_feature_levels", default=4, type=int, help="number of feature levels") + + # Transformer + parser.add_argument("--enc_layers", default=6, type=int, help="Number of encoding layers in the transformer") + parser.add_argument("--dec_layers", default=6, type=int, help="Number of decoding layers in the transformer") + parser.add_argument( + "--dim_feedforward", + default=1024, + type=int, + help="Intermediate size of the feedforward layers in the transformer blocks", + ) + parser.add_argument( + "--hidden_dim", default=256, type=int, help="Size of the embeddings (dimension of the transformer)" + ) + parser.add_argument("--dropout", default=0.1, type=float, help="Dropout applied in the transformer") + parser.add_argument( + "--nheads", default=8, type=int, help="Number of attention heads inside the transformer's attentions" + ) + parser.add_argument( + "--num_queries", + default=800, + type=int, + help="Number of query slots (num_polys * max. number of corner per poly)", + ) + parser.add_argument("--num_polys", default=20, type=int, help="Number of maximum number of room polygons") + parser.add_argument("--dec_n_points", default=4, type=int) + parser.add_argument("--enc_n_points", default=4, type=int) + parser.add_argument( + "--query_pos_type", + default="sine", + type=str, + choices=("static", "sine", "none"), + help="Type of query pos in decoder - \ + 1. static: same setting with DETR and Deformable-DETR, the query_pos is the same for all layers \ + 2. sine: since embedding from reference points (so if references points update, query_pos also \ + 3. none: remove query_pos", + ) + parser.add_argument( + "--with_poly_refine", + default=True, + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + parser.add_argument( + "--masked_attn", + default=False, + action="store_true", + help="if true, the query in one room will not be allowed to attend other room", + ) + parser.add_argument( + "--semantic_classes", + default=-1, + type=int, + help="Number of classes for semantically-rich floorplan: \ + 1. default -1 means non-semantic floorplan \ + 2. 19 for Structured3D: 16 room types + 1 door + 1 window + 1 empty", + ) + parser.add_argument( + "--disable_poly_refine", + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + + # loss + parser.add_argument( + "--no_aux_loss", + dest="aux_loss", + action="store_true", + help="Disables auxiliary decoding losses (loss at each layer)", + ) + + # matcher + parser.add_argument("--set_cost_class", default=2, type=float, help="Class coefficient in the matching cost") + parser.add_argument("--set_cost_coords", default=5, type=float, help="L1 coords coefficient in the matching cost") + + # loss coefficients + parser.add_argument("--cls_loss_coef", default=2, type=float) + parser.add_argument("--room_cls_loss_coef", default=0.2, type=float) + parser.add_argument("--coords_loss_coef", default=5, type=float) + parser.add_argument("--raster_loss_coef", default=0, type=float) + + # dataset parameters + parser.add_argument("--dataset_name", default="stru3d") + parser.add_argument("--dataset_root", default="data/stru3d", type=str) + + parser.add_argument("--output_dir", default="output", help="path where to save, empty for no saving") + parser.add_argument("--device", default="cuda", help="device to use for training / testing") + parser.add_argument("--seed", default=42, type=int) + parser.add_argument("--resume", default="", help="resume from checkpoint") + parser.add_argument("--start_epoch", default=0, type=int, metavar="N", help="start epoch") + parser.add_argument("--num_workers", default=2, type=int) + parser.add_argument("--job_name", default="train_stru3d", type=str) + + return parser + + +def main(args): + + print("git:\n {}\n".format(utils.get_sha())) + + print(args) + # Setup DDP: + dist.init_process_group("nccl") + rank = dist.get_rank() + device = rank % torch.cuda.device_count() + seed = args.seed * dist.get_world_size() + rank + # fix the seed for reproducibility + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + torch.cuda.set_device(device) + print(f"Starting rank={rank}, seed={seed}, world_size={dist.get_world_size()}.") + + # setup wandb for logging + if rank == 0: + utils.setup_wandb() + wandb.init(project="Raster2Seq", resume="allow", id=args.run_name, dir="./wandb") + + # build dataset and dataloader + dataset_train = build_dataset(image_set="train", args=args) + dataset_val = build_dataset(image_set="val", args=args) + tokenizer = None + if args.poly2seq: + args.vocab_size = dataset_train.get_vocab_size() + tokenizer = dataset_train.get_tokenizer() + + # overfit one sample + if args.debug: + dataset_val = torch.utils.data.Subset(copy.deepcopy(dataset_val), [0]) + dataset_train = copy.deepcopy(dataset_val) + + sampler_train = DistributedSampler( + dataset_train, num_replicas=dist.get_world_size(), rank=rank, shuffle=True, seed=args.seed + ) + sampler_val = DistributedSampler( + dataset_val, num_replicas=dist.get_world_size(), rank=rank, shuffle=False, seed=args.seed + ) + + def trivial_batch_collator(batch): + """ + A batch collator that does nothing. + """ + if "target_seq" in batch[0]: + # Concatenate tensors for each key in the batch + delta_x1 = torch.stack([item["delta_x1"] for item in batch], dim=0) + delta_x2 = torch.stack([item["delta_x2"] for item in batch], dim=0) + delta_y1 = torch.stack([item["delta_y1"] for item in batch], dim=0) + delta_y2 = torch.stack([item["delta_y2"] for item in batch], dim=0) + seq11 = torch.stack([item["seq11"] for item in batch], dim=0) + seq21 = torch.stack([item["seq21"] for item in batch], dim=0) + seq12 = torch.stack([item["seq12"] for item in batch], dim=0) + seq22 = torch.stack([item["seq22"] for item in batch], dim=0) + target_seq = torch.stack([item["target_seq"] for item in batch], dim=0) + token_labels = torch.stack([item["token_labels"] for item in batch], dim=0) + mask = torch.stack([item["mask"] for item in batch], dim=0) + target_polygon_labels = torch.stack([item["target_polygon_labels"] for item in batch], dim=0) + # input_polygon_labels = torch.stack([item['input_polygon_labels'] for item in batch], dim=0) + + # Delete the keys from the batch + for item in batch: + del item["delta_x1"] + del item["delta_x2"] + del item["delta_y1"] + del item["delta_y2"] + del item["seq11"] + del item["seq21"] + del item["seq12"] + del item["seq22"] + del item["target_seq"] + del item["token_labels"] + del item["mask"] + del item["target_polygon_labels"] + # del item['input_polygon_labels'] + + # Return the concatenated batch + return batch, { + "delta_x1": delta_x1, + "delta_x2": delta_x2, + "delta_y1": delta_y1, + "delta_y2": delta_y2, + "seq11": seq11, + "seq21": seq21, + "seq12": seq12, + "seq22": seq22, + "target_seq": target_seq, + "token_labels": token_labels, + "mask": mask, + "target_polygon_labels": target_polygon_labels, + # 'input_polygon_labels': input_polygon_labels, + } + + return batch, None + + data_loader_train = DataLoader( + dataset_train, + args.batch_size, + shuffle=False, + sampler=sampler_train, + num_workers=args.num_workers, + collate_fn=trivial_batch_collator, + pin_memory=True, + drop_last=True, + ) + data_loader_val = DataLoader( + dataset_val, + args.batch_size, + shuffle=False, + sampler=sampler_val, + collate_fn=trivial_batch_collator, + num_workers=args.num_workers, + pin_memory=True, + drop_last=False, + ) + + # build model + model, criterion = build_model(args, tokenizer=tokenizer) + ema = copy.deepcopy(model).to(device) + utils.requires_grad(ema, False) + model = DDP(model.to(device), device_ids=[rank], find_unused_parameters=True) + + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print("number of params:", n_parameters) + + def match_name_keywords(n, name_keywords): + out = False + for b in name_keywords: + if b in n: + out = True + break + return out + + for n, p in model.named_parameters(): + print(n) + + if args.per_token_sem_loss and not args.jointly_train: + # disable gradient for model, except new classifier + for n, p in model.named_parameters(): + if "room_class_embed" in n: + p.requires_grad = True + else: + p.requires_grad = False + + param_dicts = [ + { + "params": [ + p + for n, p in model.named_parameters() + if not match_name_keywords(n, args.lr_backbone_names) + and not match_name_keywords(n, args.lr_linear_proj_names) + and p.requires_grad + ], + "lr": args.lr, + }, + { + "params": [ + p + for n, p in model.named_parameters() + if match_name_keywords(n, args.lr_backbone_names) and p.requires_grad + ], + "lr": args.lr_backbone, + }, + { + "params": [ + p + for n, p in model.named_parameters() + if match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad + ], + "lr": args.lr * args.lr_linear_proj_mult, + }, + ] + print(f"Rank {dist.get_rank()}: Model has {sum(p.numel() for p in model.parameters())} parameters") + + if args.sgd: + optimizer = torch.optim.SGD(param_dicts, lr=args.lr, momentum=0.9, weight_decay=args.weight_decay) + else: + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, weight_decay=args.weight_decay) + + if args.lr_drop: + lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, args.lr_drop) + else: + lr_scheduler = None + + output_dir = Path(args.output_dir) + if args.resume and os.path.exists(args.resume): + checkpoint = torch.load(args.resume, map_location="cpu") + for key, value in checkpoint["model"].items(): + if key.startswith("module."): + checkpoint[key[7:]] = checkpoint["model"][key] + del checkpoint[key] + missing_keys, unexpected_keys = model.module.load_state_dict(checkpoint["model"], strict=False) + unexpected_keys = [k for k in unexpected_keys if not (k.endswith("total_params") or k.endswith("total_ops"))] + if len(missing_keys) > 0: + print("Missing Keys: {}".format(missing_keys)) + raise ValueError("Missing keys in state_dict") + if len(unexpected_keys) > 0: + print("Unexpected Keys: {}".format(unexpected_keys)) + if "optimizer" in checkpoint and "lr_scheduler" in checkpoint and "epoch" in checkpoint: + p_groups = copy.deepcopy(optimizer.param_groups) + optimizer.load_state_dict(checkpoint["optimizer"]) + for pg, pg_old in zip(optimizer.param_groups, p_groups): + pg["lr"] = pg_old["lr"] + if "initial_lr" in pg_old: + pg["initial_lr"] = pg_old["initial_lr"] + # print(optimizer.param_groups) + if lr_scheduler is not None and checkpoint["lr_scheduler"] is not None: + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) + # todo: this is a hack for doing experiment that resume from checkpoint and also modify lr scheduler (e.g., decrease lr in advance). + args.override_resumed_lr_drop = False + if args.override_resumed_lr_drop: + print( + "Warning: (hack) args.override_resumed_lr_drop is set to True, so args.lr_drop would override lr_drop in resumed lr_scheduler." + ) + lr_scheduler.step_size = args.lr_drop + if lr_scheduler is not None: + lr_scheduler.base_lrs = list(map(lambda group: group["initial_lr"], optimizer.param_groups)) + + if lr_scheduler is not None: + lr_scheduler.step(lr_scheduler.last_epoch) + args.start_epoch = checkpoint["epoch"] + 1 + + # # check the resumed model + # test_stats = evaluate( + # model, criterion, args.dataset_name, data_loader_val, device, poly2seq=args.poly2seq + # ) + dist.barrier() + + if args.start_from_checkpoint: + checkpoint = torch.load(args.start_from_checkpoint, map_location="cpu")["model"] + for key, value in checkpoint.items(): + if key.startswith("class_embed"): + if checkpoint[key].size(0) != model.module.num_classes: + if "weight" in key: + checkpoint[key] = torch.cat( + [checkpoint[key], torch.zeros((1, checkpoint[key].size(1)), dtype=torch.float)], dim=0 + ) + else: + checkpoint[key] = torch.cat([checkpoint[key], torch.zeros([1], dtype=torch.float)], dim=0) + elif "token_embed" in key: + if checkpoint[key].size(0) != model.module.transformer.decoder.token_embed.weight.size(0): + checkpoint[key] = torch.cat( + [checkpoint[key], torch.zeros((1, checkpoint[key].size(1)), dtype=torch.float)], dim=0 + ) + elif "pos_embed" in key and checkpoint[key].shape[1] != model.module.transformer.pos_embed.shape[1]: + checkpoint[key] = model.module.transformer.pos_embed + elif "attention_mask" in key and checkpoint[key].shape[0] != model.module.attention_mask.shape[0]: + checkpoint[key] = model.module.attention_mask + elif key.startswith("input_proj") and key.endswith("weight"): + # only modify the conv layer + lidx, sub_lidx = int(key.split(".")[1]), int(key.split(".")[2]) + if sub_lidx != 0: + continue + tgt_size = model.module.input_proj[lidx][0].weight.size(2) + if tgt_size != checkpoint[key].size(2): + checkpoint[key] = F.interpolate( + checkpoint[key], size=(tgt_size, tgt_size), mode="bilinear", align_corners=False + ) + elif "sampling_offsets" in key: + diff_scale = model.module.transformer.encoder.layers[0].self_attn.sampling_offsets.weight.size( + 0 + ) // checkpoint[key].size(0) + if diff_scale > 1: + if ".weight" in key: + checkpoint[key] = checkpoint[key].repeat((diff_scale, 1)) + else: + checkpoint[key] = checkpoint[key].repeat((diff_scale,)) + elif "attention_weights" in key: + diff_scale = model.module.transformer.encoder.layers[0].self_attn.attention_weights.weight.size( + 0 + ) // checkpoint[key].size(0) + if diff_scale > 1: + if ".weight" in key: + checkpoint[key] = checkpoint[key].repeat((diff_scale, 1)) + else: + checkpoint[key] = checkpoint[key].repeat((diff_scale,)) + + missing_keys, unexpected_keys = model.module.load_state_dict(checkpoint, strict=False) + unexpected_keys = [k for k in unexpected_keys if not (k.endswith("total_params") or k.endswith("total_ops"))] + if len(missing_keys) > 0: + print("Missing Keys: {}".format(missing_keys)) + if len(unexpected_keys) > 0: + print("Unexpected Keys: {}".format(unexpected_keys)) + dist.barrier() + + # Prepare models for training: + utils.update_ema(ema, model.module, decay=0) # Ensure EMA is initialized with synced weights + ema.eval() + + print("Start training") + start_time = time.time() + for epoch in range(args.start_epoch, args.epochs): + sampler_train.set_epoch(epoch) + train_stats = train_one_epoch( + model, + criterion, + data_loader_train, + optimizer, + device, + epoch, + args.clip_max_norm, + args.poly2seq, + ema_model=ema, + drop_rate=args.random_drop_rate, + ) + if lr_scheduler is not None: + lr_scheduler.step() + + if epoch > int(args.increase_cls_loss_coef_epoch_ratio * args.epochs) and args.increase_cls_loss_coef > 1.0: + criterion._update_ce_coeff(args.increase_cls_loss_coef * args.cls_loss_coef) + + if (epoch + 1) in args.lr_drop or (epoch + 1) % args.ckpt_every_epoch == 0 or (epoch + 1) == args.epochs: + if rank == 0: + checkpoint_paths = [output_dir / "checkpoint.pth"] + # extra checkpoint before LR drop and every 20 epochs + checkpoint_paths.append(output_dir / f"checkpoint{epoch:04}.pth") + for checkpoint_path in checkpoint_paths: + torch.save( + { + "model": model.module.state_dict(), + "ema": ema.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": None if lr_scheduler is None else lr_scheduler.state_dict(), + "epoch": epoch, + "args": args, + }, + checkpoint_path, + ) + dist.barrier() + + log_stats = {**{f"train_{k}": v for k, v in train_stats.items()}, "epoch": epoch, "n_parameters": n_parameters} + + if rank == 0: + wandb.log({"epoch": epoch}) + wandb.log({"lr_rate": train_stats["lr"]}) + + train_log_dict = { + "train/loss": train_stats["loss"], + "train/loss_ce": train_stats["loss_ce"], + "train/loss_coords": train_stats["loss_coords"], + "train/loss_coords_unscaled": train_stats["loss_coords_unscaled"], + "train/cardinality_error": train_stats["cardinality_error_unscaled"], + } + + if args.semantic_classes > 0: + # need to log additional metrics for semantically-rich floorplans + train_log_dict["train/loss_ce_room"] = train_stats["loss_ce_room"] + else: + if "loss_raster" in train_stats: + # only apply the rasterization loss for non-semantic floorplans + train_log_dict["train/loss_raster"] = train_stats["loss_raster"] + + if rank == 0: + wandb.log(train_log_dict) + + # eval every 20 + if (epoch + 1) % args.eval_every_epoch == 0: + eval_model = model if not args.ema4eval else ema + test_stats = evaluate( + eval_model, + criterion, + args.dataset_name, + data_loader_val, + device, + plot_density=True, + output_dir=output_dir, + epoch=epoch, + poly2seq=args.poly2seq, + add_cls_token=args.add_cls_token, + per_token_sem_loss=args.per_token_sem_loss, + wd_as_line=not args.disable_wd_as_line, + ) + log_stats.update(**{f"test_{k}": v for k, v in test_stats.items()}) + + val_log_dict = { + "val/loss": test_stats["loss"], + "val/loss_ce": test_stats["loss_ce"], + "val/loss_coords": test_stats["loss_coords"], + "val/loss_coords_unscaled": test_stats["loss_coords_unscaled"], + "val/cardinality_error": test_stats["cardinality_error_unscaled"], + "val_metrics/room_prec": test_stats["room_prec"], + "val_metrics/room_rec": test_stats["room_rec"], + "val_metrics/corner_prec": test_stats["corner_prec"], + "val_metrics/corner_rec": test_stats["corner_rec"], + "val_metrics/angles_prec": test_stats["angles_prec"], + "val_metrics/angles_rec": test_stats["angles_rec"], + } + + if args.semantic_classes > 0: + # need to log additional metrics for semantically-rich floorplans + val_log_dict["val/loss_ce_room"] = test_stats["loss_ce_room"] + val_log_dict["val_metrics/room_sem_prec"] = test_stats["room_sem_prec"] + val_log_dict["val_metrics/room_sem_rec"] = test_stats["room_sem_rec"] + if "window_door_prec" in test_stats: + val_log_dict["val_metrics/window_door_prec"] = test_stats["window_door_prec"] + val_log_dict["val_metrics/window_door_rec"] = test_stats["window_door_rec"] + + else: + if "loss_raster" in test_stats: + # only apply the rasterization loss for non-semantic floorplans + val_log_dict["val/loss_raster"] = test_stats["loss_raster"] + + if "room_iou" in test_stats: + val_log_dict["val_metrics/room_iou"] = test_stats["room_iou"] + + if rank == 0: + wandb.log(val_log_dict) + + if args.output_dir: + with (output_dir / "log.txt").open("a") as f: + f.write(json.dumps(log_stats) + "\n") + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print("Training time {}".format(total_time_str)) + + dist.destroy_process_group() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Raster2Seq training script", parents=[get_args_parser()]) + args = parser.parse_args() + now = datetime.datetime.now() + # run_id = now.strftime("%Y-%m-%d-%H-%M-%S") + args.run_name = args.job_name # run_id+'_'+args.job_name + args.output_dir = os.path.join(args.output_dir, args.run_name) + + args.lr_drop = [] if len(args.lr_drop) == 0 else [int(x) for x in args.lr_drop.split(",")] + if args.debug: + args.batch_size = 1 + if args.disable_poly_refine: + args.with_poly_refine = False + + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + main(args) diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..51965feb3100a03405286cea88e30c5dd0577e2f --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------------------ +# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) +# ------------------------------------------------------------------------------------ + +from .raster2seq import build as build_v2 +from .roomformer import build + + +def build_model(args, train=True, tokenizer=None): + if not args.poly2seq: + return build(args, train) + return build_v2(args, train, tokenizer=tokenizer) diff --git a/models/backbone.py b/models/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..51780b3f2f35e9272027759c985ae0057100735d --- /dev/null +++ b/models/backbone.py @@ -0,0 +1,134 @@ +# ------------------------------------------------------------------------------------ +# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) +# ------------------------------------------------------------------------------------ + +""" +Backbone modules. +""" + +from typing import Dict, List + +import torch +import torch.nn.functional as F +import torchvision +from torch import nn +from torchvision.models._utils import IntermediateLayerGetter + +from util.misc import NestedTensor + +from .position_encoding import build_position_encoding + + +class FrozenBatchNorm2d(torch.nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + Copy-paste from torchvision.misc.ops with added eps before rqsrt, + without which any other models than torchvision.models.resnet[18,34,50,101] + produce nans. + """ + + def __init__(self, n, eps=1e-5): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer("weight", torch.ones(n)) + self.register_buffer("bias", torch.zeros(n)) + self.register_buffer("running_mean", torch.zeros(n)) + self.register_buffer("running_var", torch.ones(n)) + self.eps = eps + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + num_batches_tracked_key = prefix + "num_batches_tracked" + if num_batches_tracked_key in state_dict: + del state_dict[num_batches_tracked_key] + + super(FrozenBatchNorm2d, self)._load_from_state_dict( + state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ) + + def forward(self, x): + # move reshapes to the beginning + # to make it fuser-friendly + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + rv = self.running_var.reshape(1, -1, 1, 1) + rm = self.running_mean.reshape(1, -1, 1, 1) + eps = self.eps + scale = w * (rv + eps).rsqrt() + bias = b - rm * scale + return x * scale + bias + + +class BackboneBase(nn.Module): + def __init__(self, backbone: nn.Module, train_backbone: bool, return_interm_layers: bool): + super().__init__() + for name, parameter in backbone.named_parameters(): + if not train_backbone or "layer2" not in name and "layer3" not in name and "layer4" not in name: + parameter.requires_grad_(False) + if return_interm_layers: + return_layers = {"layer2": "0", "layer3": "1", "layer4": "2"} + self.strides = [8, 16, 32] + self.num_channels = [512, 1024, 2048] + else: + return_layers = {"layer4": "0"} + self.strides = [32] + self.num_channels = [2048] + self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) + + def forward(self, tensor_list: NestedTensor): + xs = self.body(tensor_list.tensors) + out: Dict[str, NestedTensor] = {} + for name, x in xs.items(): + m = tensor_list.mask + assert m is not None + mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0] + out[name] = NestedTensor(x, mask) + return out + + +class Backbone(BackboneBase): + """ResNet backbone with frozen BatchNorm.""" + + def __init__(self, name: str, train_backbone: bool, return_interm_layers: bool, dilation: bool, input_channels=1): + norm_layer = FrozenBatchNorm2d + backbone = getattr(torchvision.models, name)( + replace_stride_with_dilation=[False, False, dilation], pretrained=True, norm_layer=norm_layer + ) + # modify the first layer to compatible with single channel input + backbone.conv1 = nn.Conv2d(input_channels, 64, kernel_size=7, stride=2, padding=3, bias=False) + assert name not in ("resnet18", "resnet34"), "number of channels are hard coded" + super().__init__(backbone, train_backbone, return_interm_layers) + if dilation: + self.strides[-1] = self.strides[-1] // 2 + + +class Joiner(nn.Sequential): + def __init__(self, backbone, position_embedding): + super().__init__(backbone, position_embedding) + self.strides = backbone.strides + self.num_channels = backbone.num_channels + + def forward(self, tensor_list: NestedTensor): + xs = self[0](tensor_list) + out: List[NestedTensor] = [] + pos = [] + for name, x in sorted(xs.items()): + out.append(x) + + # position encoding + for x in out: + pos.append(self[1](x).to(x.tensors.dtype)) + + return out, pos + + +def build_backbone(args): + position_embedding = build_position_encoding(args) + train_backbone = args.lr_backbone > 0 + return_interm_layers = args.num_feature_levels > 1 + backbone = Backbone( + args.backbone, train_backbone, return_interm_layers, args.dilation, input_channels=args.input_channels + ) + model = Joiner(backbone, position_embedding) + return model diff --git a/models/deformable_points.py b/models/deformable_points.py new file mode 100644 index 0000000000000000000000000000000000000000..7a7d89ff0ac180992cf039e0d4ab59587522d105 --- /dev/null +++ b/models/deformable_points.py @@ -0,0 +1,114 @@ +import einops +import torch +import torch.nn.functional as F +from torch import nn + + +class LayerNormProxy(nn.Module): + def __init__(self, dim): + + super().__init__() + self.norm = nn.LayerNorm(dim) + + def forward(self, x): + + x = einops.rearrange(x, "b c h w -> b h w c") + x = self.norm(x) + return einops.rearrange(x, "b h w c -> b c h w") + + +class MSDeformablePoints(nn.Module): + def __init__( + self, + embed_dim, + n_levels, + n_heads, + offset_range_factor=-1, + ): + + super().__init__() + self.n_head_channels = embed_dim // n_heads + self.scale = self.n_head_channels**-0.5 + self.n_heads = n_heads + self.nc = self.n_head_channels * n_heads + self.offset_range_factor = offset_range_factor + + self.kernel_sizes = [(n_levels - 1 - i) * 2 + 1 for i in range(n_levels)] # [7, 5, 3, 1] + self.strides = [2 ** (n_levels - i) for i in range(n_levels)] # [16, 8, 4, 2] + + self.conv_offset = nn.ModuleList( + [ + nn.Sequential( + nn.Conv2d( + self.n_head_channels, + self.n_head_channels, + self.kernel_sizes[i], + self.strides[i], + self.kernel_sizes[i] // 2, + groups=self.n_heads, + ), + LayerNormProxy(self.n_head_channels), + nn.GELU(), + nn.Conv2d(self.n_head_channels, 2, 1, 1, 0, bias=False), + ) + for i in range(n_levels) + ] + ) + + self.proj_q = nn.ModuleList( + [nn.Conv2d(self.nc, self.nc, kernel_size=1, stride=1, padding=0) for _ in range(n_levels)] + ) + + @torch.no_grad() + def _get_ref_points(self, H_key, W_key, B, dtype, device): + + ref_y, ref_x = torch.meshgrid( + torch.linspace(0.5, H_key - 0.5, H_key, dtype=dtype, device=device), + torch.linspace(0.5, W_key - 0.5, W_key, dtype=dtype, device=device), + indexing="ij", + ) + ref = torch.stack((ref_y, ref_x), -1) + ref[..., 1].div_(W_key).mul_(2.0).sub_(1.0) + ref[..., 0].div_(H_key).mul_(2.0).sub_(1.0) + ref = ref[None, ...].expand(B * self.n_heads, -1, -1, -1) # [B*g, H, W, 2] + + return ref + + def forward(self, x, spatial_shapes, level_start_index): + B = x.size(0) + dtype, device = x.dtype, x.device + + x_list = x.split([H_ * W_ for H_, W_ in spatial_shapes], dim=1) + out = [] + for i in range(len(x_list)): + cur_x = x_list[i] + q = self.proj_q[i]( + einops.rearrange(cur_x, "b (h w) c -> b c h w", h=spatial_shapes[i][0], w=spatial_shapes[i][1]) + ) + q_off = einops.rearrange(q, "b (g c) h w -> (b g) c h w", g=self.n_heads, c=self.n_head_channels) + offset = self.conv_offset[i](q_off).contiguous() # [B*g, 2, Hg Wg] + Hk, Wk = offset.size(2), offset.size(3) + + if self.offset_range_factor >= 0: + offset_range = torch.tensor([1.0 / Hk, 1.0 / Wk], device=device).reshape(1, 2, 1, 1) + offset = offset.tanh().mul(offset_range).mul(self.offset_range_factor) + + offset = einops.rearrange(offset, "b two h w -> b h w two") + reference = self._get_ref_points(Hk, Wk, B, dtype, device) + + if self.offset_range_factor >= 0: + pos = offset + reference + else: + pos = (offset + reference).clamp(-1.0, +1.0) + + H, W = spatial_shapes[i] + x_sampled = F.grid_sample( + input=cur_x.reshape(B * self.n_heads, self.n_head_channels, H, W), # [B*g, Cg, H, W] + grid=pos[..., (1, 0)], # y, x -> x, y: [B*g, Hg, Wg, 2] + mode="bilinear", + align_corners=True, + ) # [B*g, Cg, Hg, Wg] + + x_sampled = einops.rearrange(x_sampled, "(B g) C H W -> B (H W) (g C)", B=B) + out.append(x_sampled) + return torch.cat(out, dim=1) diff --git a/models/deformable_transformer.py b/models/deformable_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..2bc23b78b9d2104f0ca17311ceaba0630361b580 --- /dev/null +++ b/models/deformable_transformer.py @@ -0,0 +1,439 @@ +# ------------------------------------------------------------------------------------ +# Original RoomFormer implementation (https://github.com/ywyue/RoomFormer.git) +# ------------------------------------------------------------------------------------ + +import copy +import math + +import torch +import torch.nn.functional as F +from torch import nn +from torch.nn.init import normal_ + +from models.ops.modules import MSDeformAttn +from util.misc import inverse_sigmoid + + +class MLP(nn.Module): + """Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +class DeformableTransformer(nn.Module): + def __init__( + self, + d_model=256, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=1024, + dropout=0.1, + activation="relu", + poly_refine=True, + return_intermediate_dec=False, + aux_loss=False, + num_feature_levels=4, + dec_n_points=4, + enc_n_points=4, + query_pos_type="none", + ): + super().__init__() + + self.d_model = d_model + self.nhead = nhead + + encoder_layer = DeformableTransformerEncoderLayer( + d_model, dim_feedforward, dropout, activation, num_feature_levels, nhead, enc_n_points + ) + self.encoder = DeformableTransformerEncoder(encoder_layer, num_encoder_layers) + + decoder_layer = DeformableTransformerDecoderLayer( + d_model, dim_feedforward, dropout, activation, num_feature_levels, nhead, dec_n_points + ) + self.decoder = DeformableTransformerDecoder( + decoder_layer, num_decoder_layers, poly_refine, return_intermediate_dec, aux_loss, query_pos_type + ) + + self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model)) + + if query_pos_type == "sine": + self.decoder.pos_trans = nn.Linear(d_model, d_model) + self.decoder.pos_trans_norm = nn.LayerNorm(d_model) + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MSDeformAttn): + m._reset_parameters() + normal_(self.level_embed) + + def get_valid_ratio(self, mask): + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def forward(self, srcs, masks, pos_embeds, query_embed=None, tgt=None, tgt_masks=None): + assert query_embed is not None + + # prepare input for encoder + src_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): + bs, c, h, w = src.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + src = src.flatten(2).transpose(1, 2) + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + src_flatten.append(src) + mask_flatten.append(mask) + src_flatten = torch.cat(src_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor(spatial_shapes, dtype=torch.long, device=src_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) + + # encoder + memory = self.encoder( + src_flatten, spatial_shapes, level_start_index, valid_ratios, lvl_pos_embed_flatten, mask_flatten + ) + + # prepare input for decoder + bs, _, c = memory.shape + + query_embed = query_embed.unsqueeze(0).expand(bs, -1, -1) + tgt = tgt.unsqueeze(0).expand(bs, -1, -1) + reference_points = query_embed.sigmoid() + init_reference_out = reference_points + + # decoder + hs, inter_references, inter_classes = self.decoder( + tgt, + reference_points, + memory, + src_flatten, + spatial_shapes, + level_start_index, + valid_ratios, + query_embed, + mask_flatten, + tgt_masks, + ) + + return hs, init_reference_out, inter_references, inter_classes + + +class DeformableTransformerEncoderLayer(nn.Module): + def __init__(self, d_model=256, d_ffn=1024, dropout=0.1, activation="relu", n_levels=4, n_heads=8, n_points=4): + super().__init__() + + # self attention + self.self_attn = MSDeformAttn(d_model, n_levels, n_heads, n_points) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation) + self.dropout2 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout3 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, src): + src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) + src = src + self.dropout3(src2) + src = self.norm2(src) + return src + + def forward(self, src, pos, reference_points, spatial_shapes, level_start_index, padding_mask=None): + # self attention + src2 = self.self_attn( + self.with_pos_embed(src, pos), reference_points, src, spatial_shapes, level_start_index, padding_mask + ) + src = src + self.dropout1(src2) + src = self.norm1(src) + + # ffn + src = self.forward_ffn(src) + + return src + + +class DeformableTransformerEncoder(nn.Module): + def __init__(self, encoder_layer, num_layers): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + ref_y, ref_x = torch.meshgrid( + torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device), + ) + ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_) + ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def forward(self, src, spatial_shapes, level_start_index, valid_ratios, pos=None, padding_mask=None): + output = src + reference_points = self.get_reference_points(spatial_shapes, valid_ratios, device=src.device) + for _, layer in enumerate(self.layers): + output = layer(output, pos, reference_points, spatial_shapes, level_start_index, padding_mask) + + return output + + +class DeformableTransformerDecoderLayer(nn.Module): + def __init__(self, d_model=256, d_ffn=1024, dropout=0.1, activation="relu", n_levels=4, n_heads=8, n_points=4): + super().__init__() + + # cross attention + self.cross_attn = MSDeformAttn(d_model, n_levels, n_heads, n_points) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # self attention + self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) + self.dropout2 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation) + self.dropout3 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout4 = nn.Dropout(dropout) + self.norm3 = nn.LayerNorm(d_model) + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, tgt): + tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout4(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward( + self, + tgt, + query_pos, + reference_points, + src, + src_spatial_shapes, + level_start_index, + src_padding_mask=None, + tgt_masks=None, + ): + # self attention + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn(q.transpose(0, 1), k.transpose(0, 1), tgt.transpose(0, 1), attn_mask=tgt_masks)[ + 0 + ].transpose(0, 1) + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + + # cross attention + tgt2 = self.cross_attn( + self.with_pos_embed(tgt, query_pos), + reference_points, + src, + src_spatial_shapes, + level_start_index, + src_padding_mask, + ) + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + + # ffn + tgt = self.forward_ffn(tgt) + + return tgt + + +class DeformableTransformerDecoder(nn.Module): + def __init__( + self, + decoder_layer, + num_layers, + poly_refine=True, + return_intermediate=False, + aux_loss=False, + query_pos_type="none", + ): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.poly_refine = poly_refine + self.return_intermediate = return_intermediate + self.aux_loss = aux_loss + self.query_pos_type = query_pos_type + + self.coords_embed = None + self.class_embed = None + self.pos_trans = None + self.pos_trans_norm = None + + def get_query_pos_embed(self, ref_points): + num_pos_feats = 128 + temperature = 10000 + scale = 2 * math.pi + + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=ref_points.device) + dim_t = temperature ** (2 * (dim_t // 2) / num_pos_feats) # [128] + # N, L, 2 + ref_points = ref_points * scale + # N, L, 2, 128 + pos = ref_points[:, :, :, None] / dim_t + # N, L, 256 + pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), dim=4).flatten(2) + return pos + + def forward( + self, + tgt, + reference_points, + src, + src_flatten, + src_spatial_shapes, + src_level_start_index, + src_valid_ratios, + query_pos=None, + src_padding_mask=None, + tgt_masks=None, + ): + output = tgt # [10, 800, 256] + + intermediate = [] + intermediate_reference_points = [] + intermediate_classes = [] + point_classes = torch.zeros(output.shape[:2]).unsqueeze(-1).to(output.device) + for lid, layer in enumerate(self.layers): + assert reference_points.shape[-1] == 2 + reference_points_input = reference_points[:, :, None] * src_valid_ratios[:, None] + + if self.query_pos_type == "sine": + query_pos = self.pos_trans_norm(self.pos_trans(self.get_query_pos_embed(reference_points))) + + elif self.query_pos_type == "none": + query_pos = None + + output = layer( + output, + query_pos, + reference_points_input, + src, + src_spatial_shapes, + src_level_start_index, + src_padding_mask, + tgt_masks, + ) + + # iterative polygon refinement + if self.poly_refine: + offset = self.coords_embed[lid](output) + assert reference_points.shape[-1] == 2 + new_reference_points = offset + new_reference_points = offset + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points + + # if not using iterative polygon refinement, just output the reference points decoded from the last layer + elif lid == len(self.layers) - 1: + offset = self.coords_embed[-1](output) + assert reference_points.shape[-1] == 2 + new_reference_points = offset + new_reference_points = offset + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points + + # If aux loss supervision, we predict classes label from each layer and supervise loss + if self.aux_loss: + point_classes = self.class_embed[lid](output) + # Otherwise, we only predict class label from the last layer + elif lid == len(self.layers) - 1: + point_classes = self.class_embed[-1](output) + + if self.return_intermediate: + intermediate.append(output) + intermediate_reference_points.append(reference_points) + intermediate_classes.append(point_classes) + + if self.return_intermediate: + return ( + torch.stack(intermediate), + torch.stack(intermediate_reference_points), + torch.stack(intermediate_classes), + ) + + return output, reference_points, point_classes + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(f"activation should be relu/gelu, not {activation}.") + + +def build_deforamble_transformer(args): + return DeformableTransformer( + d_model=args.hidden_dim, + nhead=args.nheads, + num_encoder_layers=args.enc_layers, + num_decoder_layers=args.dec_layers, + dim_feedforward=args.dim_feedforward, + dropout=args.dropout, + activation="relu", + poly_refine=args.with_poly_refine, + return_intermediate_dec=True, + aux_loss=args.aux_loss, + num_feature_levels=args.num_feature_levels, + dec_n_points=args.dec_n_points, + enc_n_points=args.enc_n_points, + query_pos_type=args.query_pos_type, + ) diff --git a/models/deformable_transformer_v2.py b/models/deformable_transformer_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..b22dc571b4de60502ac19c3f4de71111b4f70635 --- /dev/null +++ b/models/deformable_transformer_v2.py @@ -0,0 +1,632 @@ +import copy +import math + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from util.misc import inverse_sigmoid + +from .deformable_transformer import ( + DeformableTransformerEncoder, + DeformableTransformerEncoderLayer, + MSDeformAttn +) +from .kv_cache import KVCache, VCache + + +def Embedding(num_embeddings, embedding_dim, padding_idx=None, zero_init=False): + m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx) + nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5) + if padding_idx is not None: + nn.init.constant_(m.weight[padding_idx], 0) + if zero_init: + nn.init.constant_(m.weight, 0) + return m + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, seq_len): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + pos = np.arange(seq_len, dtype=np.float32) + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +class DeformableTransformer(nn.Module): + def __init__( + self, + d_model=256, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=1024, + dropout=0.1, + activation="relu", + poly_refine=True, + return_intermediate_dec=False, + aux_loss=False, + num_feature_levels=4, + dec_n_points=4, + enc_n_points=4, + query_pos_type="none", + vocab_size=None, + seq_len=1024, + pre_decoder_pos_embed=False, + learnable_dec_pe=False, + dec_attn_concat_src=False, + dec_qkv_proj=True, + pad_idx=None, + use_anchor=False, + inject_cls_embed=False, + ): + super().__init__() + + self.d_model = d_model + self.nhead = nhead + self.poly_refine = poly_refine + self.use_anchor = use_anchor + self.inject_cls_embed = inject_cls_embed + + encoder_layer = DeformableTransformerEncoderLayer( + d_model, dim_feedforward, dropout, activation, num_feature_levels, nhead, enc_n_points + ) + self.encoder = DeformableTransformerEncoder(encoder_layer, num_encoder_layers) + + decoder_layer = TransformerDecoderLayer( + d_model, + dim_feedforward, + dropout, + activation, + num_feature_levels, + nhead, + dec_n_points, + use_qkv_proj=(dec_qkv_proj and not dec_attn_concat_src), + ) + + self.decoder = TransformerDecoder( + decoder_layer, + num_decoder_layers, + poly_refine, + return_intermediate_dec, + aux_loss, + query_pos_type, + vocab_size, + pad_idx, + use_anchor=use_anchor, + ) + + self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model)) + + if query_pos_type == "sine" and (poly_refine or use_anchor): + self.decoder.pos_trans = nn.Linear(d_model, d_model) + self.decoder.pos_trans_norm = nn.LayerNorm(d_model) + + self.pre_decoder_pos_embed = pre_decoder_pos_embed + + self.pos_embed = nn.Parameter(torch.zeros(1, seq_len, d_model), requires_grad=learnable_dec_pe) + pos_embed = get_1d_sincos_pos_embed_from_grid(d_model, seq_len) + self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float().unsqueeze(0)) + + self.dec_attn_concat_src = dec_attn_concat_src + + if self.inject_cls_embed: + self.decoder.room_class_trans = nn.Sequential( + nn.Linear(d_model, d_model, bias=False), nn.LayerNorm(d_model) + ) + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MSDeformAttn): + m._reset_parameters() + nn.init.normal_(self.level_embed) + + def get_valid_ratio(self, mask): + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def _create_causal_attention_mask(self, seq_len): + """ + Creates a causal attention mask for a sequence of length `seq_len`. + """ + # Create an upper triangular matrix with 1s above the diagonal + mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) + # Invert the mask: 1 -> -inf (masked), 0 -> 0 (unmasked) + causal_mask = mask.masked_fill(mask == 1, float("-inf")).masked_fill(mask == 0, 0.0) + return causal_mask + + def forward( + self, + srcs, + masks, + pos_embeds, + query_embed=None, + tgt=None, + tgt_masks=None, + seq_kwargs=None, + force_simple_returns=False, + return_enc_cache=False, + enc_cache=None, + decode_token_pos=None, + ): + # assert query_embed is not None + + if enc_cache is None: + # prepare input for encoder + src_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): + bs, c, h, w = src.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + src = src.flatten(2).transpose(1, 2) + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + src_flatten.append(src) + mask_flatten.append(mask) + src_flatten = torch.cat(src_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor(spatial_shapes, dtype=torch.long, device=src_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) + + # encoder + memory = self.encoder( + src_flatten, spatial_shapes, level_start_index, valid_ratios, lvl_pos_embed_flatten, mask_flatten + ) + enc_cache_output = { + "memory": memory, + "spatial_shapes": spatial_shapes, + "level_start_index": level_start_index, + "valid_ratios": valid_ratios, + "mask_flatten": mask_flatten, + "src_flatten": src_flatten, + } + else: + memory, spatial_shapes, level_start_index, valid_ratios, mask_flatten = ( + enc_cache["memory"], + enc_cache["spatial_shapes"], + enc_cache["level_start_index"], + enc_cache["valid_ratios"], + enc_cache["mask_flatten"], + ) + src_flatten = enc_cache["src_flatten"] + enc_cache_output = enc_cache + + # prepare input for decoder + bs, _, c = memory.shape + + assert not (self.use_anchor and self.poly_refine), "use_anchor and poly_refine cannot be used together" + if self.poly_refine or self.use_anchor: + query_embed = query_embed.unsqueeze(0).expand(bs, -1, -1) + reference_points = query_embed.sigmoid() + query_pos = None # inferred from reference_points + else: + reference_points = None + query_pos = self.pos_embed + init_reference_out = reference_points + + if tgt_masks is None: + # make causal mask + if decode_token_pos is not None: + tgt_masks = torch.zeros(1, decode_token_pos.max() + 1, dtype=torch.float).to(memory.device) + else: + tgt_masks = self._create_causal_attention_mask(seq_kwargs["seq11"].shape[1]).to(memory.device) + + # decoder + hs, inter_references, inter_classes = self.decoder( + tgt, + reference_points, + memory, + src_flatten, + spatial_shapes, + level_start_index, + valid_ratios, + query_pos, + mask_flatten, + tgt_masks, + seq_kwargs, + force_simple_returns=force_simple_returns, + pre_decoder_pos_embed=self.pre_decoder_pos_embed, + attn_concat_src=self.dec_attn_concat_src, + decode_token_pos=decode_token_pos, + ) + if return_enc_cache: + return hs, init_reference_out, inter_references, inter_classes, enc_cache_output + return hs, init_reference_out, inter_references, inter_classes + + def _setup_caches(self, max_batch_size, max_seq_length, max_vision_length, model_dim, nhead, dtype, device): + for layer in self.decoder.layers: + layer.kv_cache = KVCache(max_batch_size, max_seq_length, model_dim, dtype).to(device) + layer.cross_attn.cache = VCache( + max_batch_size, max_vision_length, nhead, int(model_dim // nhead), dtype + ).to(device) + + +class TransformerDecoderLayer(nn.Module): + def __init__( + self, + d_model=256, + d_ffn=1024, + dropout=0.1, + activation="relu", + n_levels=4, + n_heads=8, + n_points=4, + use_qkv_proj=True, + ): + super().__init__() + self.d_model = d_model + + if use_qkv_proj: + self.attn_q = nn.Linear(d_model, d_model, bias=False) + self.attn_k = nn.Linear(d_model, d_model, bias=False) + self.attn_v = nn.Linear(d_model, d_model, bias=False) + else: + self.attn_q = nn.Identity() + self.attn_k = nn.Identity() + self.attn_v = nn.Identity() + + # attention + self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) + self.dropout2 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + # cross attention + self.cross_attn = MSDeformAttn(d_model, n_levels, n_heads, n_points) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation) + self.dropout3 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout4 = nn.Dropout(dropout) + self.norm3 = nn.LayerNorm(d_model) + + self.kv_cache = None + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos[:, : tensor.size(1)] + + def forward_ffn(self, tgt): + tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout4(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward( + self, + tgt, + query_pos, + reference_points, + src, + src_spatial_shapes, + level_start_index, + src_padding_mask=None, + tgt_masks=None, + attn_concat_src=False, + input_pos=None, + ): + + q = self.with_pos_embed(self.attn_q(tgt), query_pos) + # self attention + if self.kv_cache is not None and input_pos is not None: + k = self.attn_k(tgt) + v = self.attn_v(tgt) + k, v = self.kv_cache.update(input_pos, k, v) + else: + k = self.attn_k(tgt) + v = self.attn_v(tgt) + + if attn_concat_src: + k = torch.cat([src, k], dim=1) + v = torch.cat([src, v], dim=1) + tgt_masks = torch.cat([torch.zeros(q.size(1), src.size(1), device=q.device), tgt_masks], dim=1).to( + dtype=torch.float32 + ) + + tgt2 = self.self_attn(q.transpose(0, 1), k.transpose(0, 1), v.transpose(0, 1), attn_mask=tgt_masks)[ + 0 + ].transpose(0, 1) + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + + # cross attention + tgt2 = self.cross_attn( + self.with_pos_embed(tgt, query_pos), + reference_points, + src, + src_spatial_shapes, + level_start_index, + src_padding_mask, + use_cache=(input_pos is not None and input_pos[0] != 0), + ) # disable cache when processing first token + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + + # ffn + tgt = self.forward_ffn(tgt) + + return tgt, None + + +class TransformerDecoder(nn.Module): + def __init__( + self, + decoder_layer, + num_layers, + poly_refine=True, + return_intermediate=False, + aux_loss=False, + query_pos_type="none", + vocab_size=None, + pad_idx=None, + use_anchor=None, + ): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.poly_refine = poly_refine + self.return_intermediate = return_intermediate + self.aux_loss = aux_loss + self.query_pos_type = query_pos_type + + self.coords_embed = None + self.class_embed = None + self.pos_trans = None + self.pos_trans_norm = None + self.use_anchor = use_anchor + + self.room_class_embed = None + self.room_class_trans = None + + self.token_embed = Embedding(vocab_size, self.layers[0].d_model, padding_idx=pad_idx, zero_init=False) + + def _seq_embed(self, seq11, seq12, seq21, seq22, delta_x1, delta_x2, delta_y1, delta_y2): + # embedding [B, L, D] + e11 = self.token_embed(seq11) + e21 = self.token_embed(seq21) + e12 = self.token_embed(seq12) + e22 = self.token_embed(seq22) + + # bilinear interpolation [B, L, D] + out = ( + e11 * delta_x2[..., None] * delta_y2[..., None] + + e21 * delta_x1[..., None] * delta_y2[..., None] + + e12 * delta_x2[..., None] * delta_y1[..., None] + + e22 * delta_x1[..., None] * delta_y1[..., None] + ) + + return out + + def _add_cls_embed(self, x, input_cls_seq): + # Suppose class_labels is of shape [batch, seq_len] with integer class indices + one_hot = F.one_hot(input_cls_seq, num_classes=self.room_class_embed.out_features).float() + x = x + self.room_class_trans(self.room_class_embed[-1](one_hot)) + return x + + def get_query_pos_embed(self, ref_points): + num_pos_feats = 128 + temperature = 10000 + scale = 2 * math.pi + + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=ref_points.device) + dim_t = temperature ** (2 * (dim_t // 2) / num_pos_feats) # [128] + # N, L, 2 + ref_points = ref_points * scale + # N, L, 2, 128 + pos = ref_points[:, :, :, None] / dim_t + # N, L, 256 + pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), dim=4).flatten(2) + return pos + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos[:, : tensor.size(1)] + + def forward( + self, + tgt, + reference_points, + src, + src_flatten, + src_spatial_shapes, + src_level_start_index, + src_valid_ratios, + query_pos=None, + src_padding_mask=None, + tgt_masks=None, + seq_kwargs=None, + force_simple_returns=False, + pre_decoder_pos_embed=False, + attn_concat_src=False, + decode_token_pos=None, + ): + # print(seq_kwargs['seq11'].max(),seq_kwargs['seq21'].max(), seq_kwargs['seq12'].max(), seq_kwargs['seq22'].max()) + + output = self._seq_embed( + seq11=seq_kwargs["seq11"], + seq12=seq_kwargs["seq12"], + seq21=seq_kwargs["seq21"], + seq22=seq_kwargs["seq22"], + delta_x1=seq_kwargs["delta_x1"], + delta_x2=seq_kwargs["delta_x2"], + delta_y1=seq_kwargs["delta_y1"], + delta_y2=seq_kwargs["delta_y2"], + ) # [B, L, D] + + if decode_token_pos is not None: + if query_pos is not None: # if using abs pos_embed + query_pos = query_pos[:, decode_token_pos] + if reference_points is not None: + reference_points = reference_points[:, decode_token_pos : decode_token_pos + 1] + + if reference_points is None: + reference_points = torch.zeros(output.shape[0], output.shape[1], 2).to(output.device) + + # assert not(pre_decoder_pos_embed and self.poly_refine), 'pre_decoder_pos_embed and poly_refine cannot be used together' + + if pre_decoder_pos_embed: + # infer query_pos from reference_points + if (self.poly_refine or self.use_anchor) and self.query_pos_type == "sine": + query_pos = self.pos_trans_norm(self.pos_trans(self.get_query_pos_embed(reference_points))) + output = self.with_pos_embed(output, query_pos) + query_pos = None + + if self.room_class_trans is not None: + # add class embedding + output = self._add_cls_embed(output, seq_kwargs["input_polygon_labels"]) + + intermediate = [] + intermediate_reference_points = [] + intermediate_classes = [] + point_classes = torch.zeros(output.shape[0], output.shape[1], self.class_embed[0].out_features).to( + output.device + ) + for lid, layer in enumerate(self.layers): + if self.poly_refine or self.use_anchor: + assert reference_points.shape[-1] == 2 + reference_points_input = reference_points[:, :, None] * src_valid_ratios[:, None] + # disable adding query_pos for every layer + if not pre_decoder_pos_embed: + if self.query_pos_type == "sine": + query_pos = self.pos_trans_norm(self.pos_trans(self.get_query_pos_embed(reference_points))) + + elif self.query_pos_type == "none": + query_pos = None + else: + reference_points_input = None + output, src_tmp = layer( + output, + query_pos, + reference_points_input, + src, + src_spatial_shapes, + src_level_start_index, + src_padding_mask, + tgt_masks, + attn_concat_src=attn_concat_src, + input_pos=decode_token_pos, + ) + if src_tmp is not None: + src = src_tmp + + # iterative polygon refinement + if self.poly_refine: + offset = self.coords_embed[lid](output) + assert reference_points.shape[-1] == 2 + new_reference_points = offset + new_reference_points = offset + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points + + # if not using iterative polygon refinement, just output the reference points decoded from the last layer + elif lid == len(self.layers) - 1: + if self.use_anchor: + offset = self.coords_embed[-1](output) + assert reference_points.shape[-1] == 2 + new_reference_points = offset + new_reference_points = offset + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points + else: + reference_points = self.coords_embed[-1](output).sigmoid() + + # If aux loss supervision, we predict classes label from each layer and supervise loss + if self.aux_loss: + point_classes = self.class_embed[lid](output) + # Otherwise, we only predict class label from the last layer + elif lid == len(self.layers) - 1: + point_classes = self.class_embed[-1](output) + + if self.return_intermediate: + intermediate.append(output) + intermediate_reference_points.append(reference_points) + intermediate_classes.append(point_classes) + + if self.return_intermediate and not force_simple_returns: + return ( + torch.stack(intermediate), + torch.stack(intermediate_reference_points), + torch.stack(intermediate_classes), + ) + + return output, reference_points, point_classes + + +def _get_clones(module, N): + if isinstance(module, list): + return nn.ModuleList(module) + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(f"activation should be relu/gelu, not {activation}.") + + +def build_deforamble_transformer(args, pad_idx=None): + return DeformableTransformer( + d_model=args.hidden_dim, + nhead=args.nheads, + num_encoder_layers=args.enc_layers, + num_decoder_layers=args.dec_layers, + dim_feedforward=args.dim_feedforward, + dropout=args.dropout, + activation="relu", + poly_refine=args.with_poly_refine, + return_intermediate_dec=True, + aux_loss=args.aux_loss, + num_feature_levels=args.num_feature_levels, + dec_n_points=args.dec_n_points, + enc_n_points=args.enc_n_points, + query_pos_type=args.query_pos_type, + vocab_size=args.vocab_size, + seq_len=args.seq_len, + pre_decoder_pos_embed=args.pre_decoder_pos_embed, + learnable_dec_pe=args.learnable_dec_pe, + dec_attn_concat_src=args.dec_attn_concat_src, + dec_qkv_proj=args.dec_qkv_proj, + pad_idx=pad_idx, + use_anchor=args.use_anchor, + inject_cls_embed=getattr(args, "inject_cls_embed", False), + ) diff --git a/models/kv_cache.py b/models/kv_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..002976779dd0bc7eda7c3443ffce9c8daeb9f837 --- /dev/null +++ b/models/kv_cache.py @@ -0,0 +1,31 @@ +import torch +from torch import nn + + +class KVCache(nn.Module): + def __init__(self, max_batch_size, max_seq_length, model_dim, dtype): + super().__init__() + cache_shape = (max_batch_size, max_seq_length, model_dim) + self.register_buffer("k_cache", torch.zeros(cache_shape, dtype=dtype)) + self.register_buffer("v_cache", torch.zeros(cache_shape, dtype=dtype)) + + def update(self, input_pos, k_val, v_val): + # input_pos: [S], k_val: [B, S, H, D] + index = input_pos[0].long() + 1 + self.k_cache[:, input_pos, ...] = k_val + self.v_cache[:, input_pos, ...] = v_val + + return self.k_cache[:, :index], self.v_cache[:, :index] + + +class VCache(nn.Module): + def __init__(self, max_batch_size, max_seq_length, num_heads, head_dim, dtype): + super().__init__() + cache_shape = (max_batch_size, max_seq_length, num_heads, head_dim) + self.register_buffer("v_cache", torch.zeros(cache_shape, dtype=dtype)) + + def update(self, v_val): + self.v_cache = v_val + + def get(self): + return self.v_cache diff --git a/models/label_smoothing_loss.py b/models/label_smoothing_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..a7d677c1a40680f34df1cf06bce7f524fe991c66 --- /dev/null +++ b/models/label_smoothing_loss.py @@ -0,0 +1,33 @@ +import torch +import torch.nn.functional as F + + +def kl_loss(p, q): + p_loss = F.kl_div(p, torch.exp(q), reduction="sum") + q_loss = F.kl_div(q, torch.exp(p), reduction="sum") + loss = (p_loss + q_loss) / 2 + return loss + + +def label_smoothed_nll_loss( + logits, + target, + epsilon, + reduction="sum", +): + lprobs = F.log_softmax(logits, dim=-1) + if target.dim() == lprobs.dim() - 1: + target = target.unsqueeze(-1) + nll_loss = -lprobs.gather(dim=-1, index=target).squeeze(-1) + smooth_loss = -lprobs.sum(dim=-1, keepdim=True).squeeze(-1) + eps_i = epsilon / (lprobs.size(-1) - 1) + loss = (1.0 - epsilon - eps_i) * nll_loss + eps_i * smooth_loss + + ntokens = loss.numel() + nll_loss = nll_loss.sum() + + loss = loss.sum() + if reduction == "mean": + loss /= ntokens + + return loss # nll_loss, ntokens diff --git a/models/losses.py b/models/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..f7c0d2a2c9a4227e90e6994eaeab0a91478c660c --- /dev/null +++ b/models/losses.py @@ -0,0 +1,252 @@ +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from util.poly_ops import get_all_order_corners + +try: + from diff_ras.polygon import SoftPolygon +except ImportError: + SoftPolygon = None +from util.bf_utils import POLY_LOSS_REGISTRY, rasterize_instances + + +def custom_L1_loss(src_polys, target_polys, target_len): + """L1 loss for coordinates regression + We only calculate the loss between valid corners since we filter out invalid corners in final results + Args: + src_polys: Tensor of dim [num_target_polys, num_queries_per_poly*2] with the matched predicted polygons coordinates + target_polys: Tensor of dim [num_target_polys, num_queries_per_poly*2] with the target polygons coordinates + target_len: list of size num_target_polys, each element indicates 2 * num_corners of this poly + """ + total_loss = 0.0 + for i in range(target_polys.shape[0]): + tgt_poly_single = target_polys[i, : target_len[i]] + all_polys = get_all_order_corners(tgt_poly_single) + total_loss += torch.cdist(src_polys[i, : target_len[i]].unsqueeze(0), all_polys, p=1).min() + total_loss = total_loss / target_len.sum() + return total_loss + + +class ClippingStrategy(nn.Module): + def __init__(self, cfg, is_boundary=False): + super().__init__() + + self.register_buffer( + "laplacian", torch.tensor([-1, -1, -1, -1, 8, -1, -1, -1, -1], dtype=torch.float32).reshape(1, 1, 3, 3) + ) + + self.is_boundary = is_boundary + self.side_lengths = np.array([64, 64, 64, 64, 64, 64, 64, 64]).reshape(-1, 2) + + # not used. + def _extract_target_boundary(self, masks, shape): + boundary_targets = F.conv2d(masks.unsqueeze(1), self.laplacian, padding=1) + boundary_targets = boundary_targets.clamp(min=0) + boundary_targets[boundary_targets > 0.1] = 1 + boundary_targets[boundary_targets <= 0.1] = 0 + + # odd? only if the width doesn't match? + if boundary_targets.shape[-2:] != shape: + boundary_targets = F.interpolate(boundary_targets, shape, mode="nearest") + + return boundary_targets + + def forward(self, instances, clip_boxes=None, lid=0): + device = self.laplacian.device + + gt_masks = [] + + if clip_boxes is not None: + clip_boxes = torch.split(clip_boxes, [len(inst) for inst in instances], dim=0) + + for idx, instances_per_image in enumerate(instances): + if len(instances_per_image) == 0: + continue + + if clip_boxes is not None: + # todo, need to support rectangular boxes. + gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize( + clip_boxes[idx].detach(), self.side_lengths[lid][0] + ) + else: + gt_masks_per_image = instances_per_image.gt_masks.rasterize_no_crop(self.side_length).to(device) + + # A tensor of shape (N, M, M), N=#instances in the image; M=mask_side_len + gt_masks.append(gt_masks_per_image) + + return torch.cat(gt_masks).squeeze(1) + + +def dice_loss(input, target): + smooth = 1.0 + + iflat = input.reshape(-1) + tflat = target.reshape(-1) + intersection = (iflat * tflat).sum() + + return 1 - ((2.0 * intersection + smooth) / (iflat.sum() + tflat.sum() + smooth)) + + +def dice_loss_no_reduction(input, target): + smooth = 1.0 + + iflat = input.flatten(-2, -1) # [200, 4096] + tflat = target.flatten(-2, -1) # [200, 4096] + intersection = (iflat * tflat).sum(1) # [200] + + return 1 - ((2.0 * intersection + smooth) / (iflat.sum(1) + tflat.sum(1) + smooth)) + + +@POLY_LOSS_REGISTRY.register() +class MaskRasterizationLoss(nn.Module): + def __init__(self, cfg): + super().__init__() + + self.register_buffer( + "rasterize_at", torch.from_numpy(np.array([64, 64, 64, 64, 64, 64, 64, 64]).reshape(-1, 2)) + ) + # self.register_buffer("rasterize_at", torch.from_numpy(np.array([128, 128, 128, 128, 128, 128, 128, 128]).reshape(-1, 2))) + # self.register_buffer("rasterize_at", torch.from_numpy(np.array([256, 256, 256, 256, 256, 256, 256, 256]).reshape(-1, 2))) + self.inv_smoothness_schedule = (0.1,) + self.inv_smoothness = self.inv_smoothness_schedule[0] + self.inv_smoothness_iter = () + self.inv_smoothness_idx = 0 + self.iter = 0 + + # whether to invoke our own rasterizer in "hard" mode. + self.use_rasterized_gt = True + + self.pred_rasterizer = SoftPolygon(inv_smoothness=self.inv_smoothness, mode="mask") + self.clip_to_proposal = False + self.predict_in_box_space = True + + if self.clip_to_proposal or not self.use_rasterized_gt: + self.clipper = ClippingStrategy(cfg=None) + self.gt_rasterizer = None + else: + self.gt_rasterizer = SoftPolygon(inv_smoothness=1.0, mode="hard_mask") + + self.offset = 0.5 + self.loss_fn = dice_loss + self.name = "mask" + + def _create_targets(self, instances, clip_boxes=None, lid=0): + if self.clip_to_proposal or not self.use_rasterized_gt: + targets = self.clipper(instances, clip_boxes=clip_boxes, lid=lid) + else: + targets = rasterize_instances(self.gt_rasterizer, instances, self.rasterize_at) + + return targets + + def forward(self, preds, targets, target_len, lid=0): + + resolution = self.rasterize_at[lid] + + target_masks = [] + pred_masks = [] + for i in range(len(targets)): + # tgt_poly_single = targets[i, :target_len[i]].view(-1, 2).unsqueeze(0) + # pred_poly_single = preds[i, :target_len[i]].view(-1, 2).unsqueeze(0) + tgt_poly_single = targets[i][: target_len[i]].view(-1, 2).unsqueeze(0) + pred_poly_single = preds[i][: target_len[i]].view(-1, 2).unsqueeze(0) + + tgt_mask = self.gt_rasterizer( + tgt_poly_single * float(resolution[1].item()), resolution[1].item(), resolution[0].item(), 1.0 + ) + tgt_mask = (tgt_mask + 1) / 2 + + pred_mask = self.pred_rasterizer( + pred_poly_single * float(resolution[1].item()), resolution[1].item(), resolution[0].item(), 1.0 + ) + target_masks.append(tgt_mask) + pred_masks.append(pred_mask) + + pred_masks = torch.stack(pred_masks) + target_masks = torch.stack(target_masks) + + return self.loss_fn(pred_masks, target_masks) + + +class MaskRasterizationCost(nn.Module): + def __init__(self, cfg): + super().__init__() + + self.register_buffer( + "rasterize_at", torch.from_numpy(np.array([64, 64, 64, 64, 64, 64, 64, 64]).reshape(-1, 2)) + ) + # self.register_buffer("rasterize_at", torch.from_numpy(np.array([128, 128, 128, 128, 128, 128, 128, 128]).reshape(-1, 2))) + self.inv_smoothness_schedule = (0.1,) + self.inv_smoothness = self.inv_smoothness_schedule[0] + self.inv_smoothness_iter = () + self.inv_smoothness_idx = 0 + self.iter = 0 + + self.pred_rasterizer = SoftPolygon(inv_smoothness=self.inv_smoothness, mode="mask") + + # whether to invoke our own rasterizer in "hard" mode. + self.use_rasterized_gt = True + + self.gt_rasterizer = SoftPolygon(inv_smoothness=1.0, mode="hard_mask") + + self.offset = 0.5 + self.loss_fn = dice_loss_no_reduction + self.name = "mask" + + def mask_iou( + self, + mask1: torch.Tensor, + mask2: torch.Tensor, + ) -> torch.Tensor: + """ + Inputs: + mask1: NxHxW torch.float32. Consists of [0, 1] + mask2: NxHxW torch.float32. Consists of [0, 1] + Outputs: + ret: NxM torch.float32. Consists of [0 - 1] + """ + + N, H, W = mask1.shape + M, H, W = mask2.shape + + mask1 = mask1.view(N, H * W) + mask2 = mask2.view(M, H * W) + + intersection = torch.matmul(mask1, mask2.t()) + + area1 = mask1.sum(dim=1).view(1, -1) + area2 = mask2.sum(dim=1).view(1, -1) + + union = (area1.t() + area2) - intersection + + ret = torch.where( + union == 0, + torch.tensor(0.0, device=mask1.device), + intersection / union, + ) + + return ret + + def forward(self, preds, targets, target_len, lid=0): + resolution = self.rasterize_at[lid] + cost_mask = torch.zeros([preds.shape[0], targets.shape[0]], device=preds.device) + pred_masks = [] + + for i in range(targets.shape[0]): + tgt_poly_single = targets[i, : target_len[i]].view(-1, 2).unsqueeze(0) + pred_poly_all = preds[:, : target_len[i]].view(preds.shape[0], -1, 2) + + tgt_mask = self.gt_rasterizer( + tgt_poly_single * float(resolution[1].item()), resolution[1].item(), resolution[0].item(), 1.0 + ) + pred_masks = self.pred_rasterizer( + pred_poly_all * float(resolution[1].item()), resolution[1].item(), resolution[0].item(), 1.0 + ) + + tgt_mask = (tgt_mask + 1) / 2 + tgt_masks = tgt_mask.repeat(preds.shape[0], 1, 1) + + cost_mask[:, i] = self.loss_fn(tgt_masks, pred_masks) + + return cost_mask diff --git a/models/matcher.py b/models/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..6428afeec5a3651d89a5a92bfb3871ad879b8715 --- /dev/null +++ b/models/matcher.py @@ -0,0 +1,106 @@ +# ------------------------------------------------------------------------------------ +# Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR) +# ------------------------------------------------------------------------------------ + +""" +Modules to compute the matching cost and solve the corresponding LSAP. +""" + +import torch +from scipy.optimize import linear_sum_assignment +from torch import nn + +from util.poly_ops import get_all_order_corners + + +class HungarianMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + + We do the matching in polygon (room) level + """ + + def __init__(self, cost_class: float = 1, cost_coords: float = 1): + """Creates the matcher + + Params: + cost_class: This is the relative weight of the classification error in the matching cost + cost_coords: This is the relative weight of the L1 error of the polygon coordinates in the matching cost + """ + super().__init__() + self.cost_class = cost_class + self.cost_coords = cost_coords + assert cost_class != 0 or cost_coords != 0, "all costs cant be 0" + + def calculate_angles(self, polygon): + vect1 = polygon.roll(1, 0) - polygon + vect2 = polygon.roll(-1, 0) - polygon + cos_sim = ((vect1 * vect2).sum(1) + 1e-9) / ( + torch.norm(vect1, p=2, dim=1) * torch.norm(vect2, p=2, dim=1) + 1e-9 + ) + return cos_sim + + def calculate_src_angles(self, polygon): + vect1 = polygon.roll(1, 1) - polygon + vect2 = polygon.roll(-1, 1) - polygon + + cos_sim = ((vect1 * vect2).sum(-1) + 1e-9) / ( + torch.norm(vect1, p=2, dim=-1) * torch.norm(vect2, p=2, dim=-1) + 1e-9 + ) + + return cos_sim + + def forward(self, outputs, targets): + """Performs the matching + + Params: + outputs: This is a dict that contains at least these entries: + "pred_logits": Tensor of dim [batch_size, num_polys, num_queries_per_poly] with the classification logits + "pred_coords": Tensor of dim [batch_size, num_polys, num_queries_per_poly, 2] with the predicted polygons coordinates + + targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: + "labels": Tensor of dim [num_target_polys, num_queries_per_poly] (where num_target_polys is the number of ground-truth + polygons in the target) containing the class labels + "coords": Tensor of dim [num_target_polys, num_queries_per_poly * 2] containing the target polygons coordinates + + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order), max(index_i) = num_polys - 1 + - index_j is the indices of the corresponding selected targets (in order), max(index_j) = num_target_polys - 1 + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_polys, num_target_polys) + """ + with torch.no_grad(): + bs, num_polys = outputs["pred_logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + src_prob = outputs["pred_logits"].flatten(0, 1).sigmoid() + src_polys = outputs["pred_coords"].flatten(0, 1).flatten(1, 2) + + # Also concat the target labels and coords + tgt_ids = torch.cat([v["labels"] for v in targets]) + tgt_polys = torch.cat([v["coords"] for v in targets]) + tgt_len = torch.cat([v["lengths"] for v in targets]) + + # Compute the pair-wise classification cost. + # We just use the L1 distance between prediction probality and target labels (inc. no-object calss) + cost_class = torch.cdist(src_prob, tgt_ids, p=1) + + # Compute the L1 cost between coords + # Here we does not consider no-object corner in target since we filter out no-object corners in results + cost_coords = torch.zeros([src_polys.shape[0], tgt_polys.shape[0]], device=src_polys.device) + for i in range(tgt_polys.shape[0]): + tgt_polys_single = tgt_polys[i, : tgt_len[i]] + all_polys = get_all_order_corners(tgt_polys_single) + cost_coords[:, i] = torch.cdist(src_polys[:, : tgt_len[i]], all_polys, p=1).min(axis=-1)[0] + + # Final cost matrix + C = self.cost_coords * cost_coords + self.cost_class * cost_class + C = C.view(bs, num_polys, -1).cpu() + + sizes = [len(v["coords"]) for v in targets] + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +def build_matcher(args): + return HungarianMatcher(cost_class=args.set_cost_class, cost_coords=args.set_cost_coords) diff --git a/models/ops/functions/__init__.py b/models/ops/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a2197bda3199aa32cafc5b9d396479609853dd2 --- /dev/null +++ b/models/ops/functions/__init__.py @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn_func import MSDeformAttnFunction + diff --git a/models/ops/functions/ms_deform_attn_func.py b/models/ops/functions/ms_deform_attn_func.py new file mode 100644 index 0000000000000000000000000000000000000000..8c5df8cf5d23aca963eec6c1133c180b37289607 --- /dev/null +++ b/models/ops/functions/ms_deform_attn_func.py @@ -0,0 +1,61 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +import MultiScaleDeformableAttention as MSDA + + +class MSDeformAttnFunction(Function): + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): + ctx.im2col_step = im2col_step + output = MSDA.ms_deform_attn_forward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors + grad_value, grad_sampling_loc, grad_attn_weight = \ + MSDA.ms_deform_attn_backward( + value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) + + return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): + # for debug and test only, + # need to use cuda version instead + N_, S_, M_, D_ = value.shape + _, Lq_, M_, L_, P_, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for lid_, (H_, W_) in enumerate(value_spatial_shapes): + # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ + value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) + # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 + sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) + # N_*M_, D_, Lq_, P_ + sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, + mode='bilinear', padding_mode='zeros', align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) + attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) + return output.transpose(1, 2).contiguous() diff --git a/models/ops/make.sh b/models/ops/make.sh new file mode 100644 index 0000000000000000000000000000000000000000..106b685722bc6ed70a06bf04309e75e62f73a430 --- /dev/null +++ b/models/ops/make.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +python setup.py build install diff --git a/models/ops/modules/__init__.py b/models/ops/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f82cb1ad9d634a87b54ba6a71b58a230bcade5fe --- /dev/null +++ b/models/ops/modules/__init__.py @@ -0,0 +1,9 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from .ms_deform_attn import MSDeformAttn diff --git a/models/ops/modules/ms_deform_attn.py b/models/ops/modules/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..d22d2410a687fa3af50e0f17092f82d65455c2ba --- /dev/null +++ b/models/ops/modules/ms_deform_attn.py @@ -0,0 +1,127 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ + +from ..functions import MSDeformAttnFunction + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n-1) == 0) and n != 0 + + +class MSDeformAttn(nn.Module): + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """ + Multi-Scale Deformable Attention Module + :param d_model hidden dimension + :param n_levels number of feature levels + :param n_heads number of attention heads + :param n_points number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.im2col_step = 64 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self.cache = None + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.) + constant_(self.attention_weights.bias.data, 0.) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None, + use_cache=False): + """ + :param query (N, Length_{query}, C) + :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area + or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes + :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) + :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements + + :return output (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in + + if use_cache and self.cache is not None: + value = self.cache.get() + else: + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) + if self.cache is not None: + self.cache.update(value) + + sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points is not None: + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + else: + raise ValueError( + 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) + else: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = sampling_offsets / offset_normalizer[None, None, None, :, None, :] + output = MSDeformAttnFunction.apply( + value, input_spatial_shapes, input_level_start_index, sampling_locations, attention_weights, self.im2col_step) + output = self.output_proj(output) + return output \ No newline at end of file diff --git a/models/ops/setup.py b/models/ops/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..a0131bc21cf1b45b90fcf174e2c53e4c08e9c641 --- /dev/null +++ b/models/ops/setup.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + if torch.cuda.is_available() and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + raise NotImplementedError('Cuda is not availabel') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "MultiScaleDeformableAttention", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="MultiScaleDeformableAttention", + version="1.0", + author="Weijie Su", + url="https://github.com/fundamentalvision/Deformable-DETR", + description="PyTorch Wrapper for CUDA Functions of Multi-Scale Deformable Attention", + packages=find_packages(exclude=("configs", "tests",)), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/models/ops/src/cpu/ms_deform_attn_cpu.cpp b/models/ops/src/cpu/ms_deform_attn_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e1bf854de1f3860d20b6fef5c1a17817c268e70a --- /dev/null +++ b/models/ops/src/cpu/ms_deform_attn_cpu.cpp @@ -0,0 +1,41 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include +#include + + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/models/ops/src/cpu/ms_deform_attn_cpu.h b/models/ops/src/cpu/ms_deform_attn_cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..81b7b58a3d9502bbb684dc84687a526dedf94cae --- /dev/null +++ b/models/ops/src/cpu/ms_deform_attn_cpu.h @@ -0,0 +1,33 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + + diff --git a/models/ops/src/cuda/ms_deform_attn_cuda.cu b/models/ops/src/cuda/ms_deform_attn_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..d6d583647cce987196d5ad1968a8a365a379e774 --- /dev/null +++ b/models/ops/src/cuda/ms_deform_attn_cuda.cu @@ -0,0 +1,153 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include "cuda/ms_deform_im2col_cuda.cuh" + +#include +#include +#include +#include + + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto output = at::zeros({batch, num_query, num_heads, channels}, value.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] { + ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + columns.data()); + + })); + } + + output = output.view({batch, num_query, num_heads*channels}); + + return output; +} + + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto grad_value = at::zeros_like(value); + auto grad_sampling_loc = at::zeros_like(sampling_loc); + auto grad_attn_weight = at::zeros_like(attn_weight); + + const int batch_n = im2col_step_; + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] { + ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + grad_value.data() + n * im2col_step_ * per_value_size, + grad_sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + grad_attn_weight.data() + n * im2col_step_ * per_attn_weight_size); + + })); + } + + return { + grad_value, grad_sampling_loc, grad_attn_weight + }; +} \ No newline at end of file diff --git a/models/ops/src/cuda/ms_deform_attn_cuda.h b/models/ops/src/cuda/ms_deform_attn_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..c7ae53f99c820ce6193b608ad344550348a0b42c --- /dev/null +++ b/models/ops/src/cuda/ms_deform_attn_cuda.h @@ -0,0 +1,30 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + diff --git a/models/ops/src/cuda/ms_deform_im2col_cuda.cuh b/models/ops/src/cuda/ms_deform_im2col_cuda.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6bc2acb7aea0eab2e9e91e769a16861e1652c284 --- /dev/null +++ b/models/ops/src/cuda/ms_deform_im2col_cuda.cuh @@ -0,0 +1,1327 @@ +/*! +************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************** +* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) +* Copyright (c) 2018 Microsoft +************************************************************************** +*/ + +#include +#include +#include + +#include +#include + +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N, const int num_threads) +{ + return (N + num_threads - 1) / num_threads; +} + + +template +__device__ scalar_t ms_deform_attn_im2col_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_attn_weight = top_grad * val; + *grad_sampling_loc = width * grad_w_weight * top_grad_value; + *(grad_sampling_loc + 1) = height * grad_h_weight * top_grad_value; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear_gm(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_attn_weight, top_grad * val); + atomicAdd(grad_sampling_loc, width * grad_w_weight * top_grad_value); + atomicAdd(grad_sampling_loc + 1, height * grad_h_weight * top_grad_value); +} + + +template +__global__ void ms_deformable_im2col_gpu_kernel(const int n, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + scalar_t *data_col_ptr = data_col + index; + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + scalar_t col = 0; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const scalar_t *data_value_ptr = data_value + (data_value_ptr_init_offset + level_start_id * qid_stride); + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + col += ms_deform_attn_im2col_bilinear(data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col) * weight; + } + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockSize; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockSize/2; s>0; s>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + atomicAdd(grad_sampling_loc, cache_grad_sampling_loc[0]); + atomicAdd(grad_sampling_loc + 1, cache_grad_sampling_loc[1]); + atomicAdd(grad_attn_weight, cache_grad_attn_weight[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_gm(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear_gm( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + grad_sampling_loc, grad_attn_weight); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +void ms_deformable_im2col_cuda(cudaStream_t stream, + const scalar_t* data_value, + const int64_t* data_spatial_shapes, + const int64_t* data_level_start_index, + const scalar_t* data_sampling_loc, + const scalar_t* data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* data_col) +{ + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + const int num_threads = CUDA_NUM_THREADS; + ms_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_value, data_spatial_shapes, data_level_start_index, data_sampling_loc, data_attn_weight, + batch_size, spatial_size, num_heads, channels, num_levels, num_query, num_point, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void ms_deformable_col2im_cuda(cudaStream_t stream, + const scalar_t* grad_col, + const scalar_t* data_value, + const int64_t * data_spatial_shapes, + const int64_t * data_level_start_index, + const scalar_t * data_sampling_loc, + const scalar_t * data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int num_threads = (channels > CUDA_NUM_THREADS)?CUDA_NUM_THREADS:channels; + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + if (channels > 1024) + { + if ((channels & 1023) == 0) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_gm + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + else{ + switch(channels) + { + case 1: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 2: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 4: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 8: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 16: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 32: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 64: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 128: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 256: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 512: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 1024: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + default: + if (channels < 64) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} \ No newline at end of file diff --git a/models/ops/src/ms_deform_attn.h b/models/ops/src/ms_deform_attn.h new file mode 100644 index 0000000000000000000000000000000000000000..ac0ef2ec25f7d0ee51ca2d807b159ddf85652017 --- /dev/null +++ b/models/ops/src/ms_deform_attn.h @@ -0,0 +1,62 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "cpu/ms_deform_attn_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/ms_deform_attn_cuda.h" +#endif + + +at::Tensor +ms_deform_attn_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_forward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +ms_deform_attn_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_backward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/models/ops/src/vision.cpp b/models/ops/src/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2201f63a51dca16d0b31148ed2c9e8e47ec15bdc --- /dev/null +++ b/models/ops/src/vision.cpp @@ -0,0 +1,16 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include "ms_deform_attn.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward"); + m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward"); +} diff --git a/models/ops/test.py b/models/ops/test.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbf6d5547d131f01a8c5c28b76557bd27a9334b --- /dev/null +++ b/models/ops/test.py @@ -0,0 +1,89 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +from torch.autograd import gradcheck + +from functions.ms_deform_attn_func import MSDeformAttnFunction, ms_deform_attn_core_pytorch + + +N, M, D = 1, 2, 2 +Lq, L, P = 2, 2, 2 +shapes = torch.as_tensor([(6, 4), (3, 2)], dtype=torch.long).cuda() +level_start_index = torch.cat((shapes.new_zeros((1, )), shapes.prod(1).cumsum(0)[:-1])) +S = sum([(H*W).item() for H, W in shapes]) + + +torch.manual_seed(3) + + +@torch.no_grad() +def check_forward_equal_with_pytorch_double(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + output_pytorch = ms_deform_attn_core_pytorch(value.double(), shapes, sampling_locations.double(), attention_weights.double()).detach().cpu() + output_cuda = MSDeformAttnFunction.apply(value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step).detach().cpu() + fwdok = torch.allclose(output_cuda, output_pytorch) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + + print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_forward_equal_with_pytorch_float(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + output_pytorch = ms_deform_attn_core_pytorch(value, shapes, sampling_locations, attention_weights).detach().cpu() + output_cuda = MSDeformAttnFunction.apply(value, shapes, level_start_index, sampling_locations, attention_weights, im2col_step).detach().cpu() + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_gradient_numerical(channels=4, grad_value=True, grad_sampling_loc=True, grad_attn_weight=True): + + value = torch.rand(N, S, M, channels).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + func = MSDeformAttnFunction.apply + + value.requires_grad = grad_value + sampling_locations.requires_grad = grad_sampling_loc + attention_weights.requires_grad = grad_attn_weight + + gradok = gradcheck(func, (value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step)) + + print(f'* {gradok} check_gradient_numerical(D={channels})') + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_double() + check_forward_equal_with_pytorch_float() + + for channels in [30, 32, 64, 71, 1025, 2048, 3096]: + check_gradient_numerical(channels, True, True, True) + + + diff --git a/models/position_encoding.py b/models/position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..df3df7964a81b0d95212a0a05a7c7b51bc1d4f8a --- /dev/null +++ b/models/position_encoding.py @@ -0,0 +1,109 @@ +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ + +""" +Various positional encodings for the transformer. +""" + +import math + +import torch +from torch import nn + +from util.misc import NestedTensor + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + mask = tensor_list.mask + assert mask is not None + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = (y_embed - 0.5) / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = (x_embed - 0.5) / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) + pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + +class PositionEmbeddingLearned(nn.Module): + """ + Absolute pos embedding, learned. + """ + + def __init__(self, num_pos_feats=256): + super().__init__() + self.row_embed = nn.Embedding(50, num_pos_feats) + self.col_embed = nn.Embedding(50, num_pos_feats) + self.reset_parameters() + + def reset_parameters(self): + nn.init.uniform_(self.row_embed.weight) + nn.init.uniform_(self.col_embed.weight) + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + h, w = x.shape[-2:] + i = torch.arange(w, device=x.device) + j = torch.arange(h, device=x.device) + x_emb = self.col_embed(i) + y_emb = self.row_embed(j) + pos = ( + torch.cat( + [ + x_emb.unsqueeze(0).repeat(h, 1, 1), + y_emb.unsqueeze(1).repeat(1, w, 1), + ], + dim=-1, + ) + .permute(2, 0, 1) + .unsqueeze(0) + .repeat(x.shape[0], 1, 1, 1) + ) + return pos + + +def build_position_encoding(args): + N_steps = args.hidden_dim // 2 + if args.position_embedding in ("v2", "sine"): + # TODO find a better way of exposing other arguments + position_embedding = PositionEmbeddingSine(N_steps, normalize=True) + elif args.position_embedding in ("v3", "learned"): + position_embedding = PositionEmbeddingLearned(N_steps) + else: + raise ValueError(f"not supported {args.position_embedding}") + + return position_embedding diff --git a/models/raster2seq.py b/models/raster2seq.py new file mode 100644 index 0000000000000000000000000000000000000000..e0ef523b09452f4c4b227ba334caaa9c376391ea --- /dev/null +++ b/models/raster2seq.py @@ -0,0 +1,811 @@ +import copy +import math + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from datasets.poly_data import TokenType +from util.misc import NestedTensor, nested_tensor_from_tensor_list + +from .backbone import build_backbone + +from .deformable_transformer_v2 import build_deforamble_transformer +from .label_smoothing_loss import label_smoothed_nll_loss +from .losses import MaskRasterizationLoss + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class Raster2Seq(nn.Module): + """This is the RoomFormer module that performs floorplan reconstruction""" + + def __init__( + self, + backbone, + transformer, + num_classes, + num_queries, + num_polys, + num_feature_levels, + aux_loss=True, + with_poly_refine=False, + masked_attn=False, + semantic_classes=-1, + seq_len=1024, + tokenizer=None, + use_anchor=False, + patch_size=1, + freeze_anchor=False, + inject_cls_embed=False, + ): + """Initializes the model. + Parameters: + backbone: torch module of the backbone to be used. See backbone.py + transformer: torch module of the transformer architecture. See transformer.py + num_classes: number of object classes + num_queries: number of object queries, ie detection slot. This is the maximal number of possible corners + in a single image. + num_polys: maximal number of possible polygons in a single image. + num_queries/num_polys would be the maximal number of possible corners in a single polygon. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + with_poly_refine: iterative polygon refinement + """ + super().__init__() + self.num_queries = num_queries + self.num_polys = num_polys + assert num_queries % num_polys == 0 + self.transformer = transformer + hidden_dim = transformer.d_model + self.num_classes = num_classes + + self.class_embed = nn.Linear(hidden_dim, num_classes) + self.coords_embed = MLP(hidden_dim, hidden_dim, 2, 3) + self.num_feature_levels = num_feature_levels + self.tokenizer = tokenizer + self.seq_len = seq_len + self.patch_size = patch_size + self.inject_cls_embed = inject_cls_embed + + # self.tgt_embed = nn.Embedding(num_queries, hidden_dim) + if num_feature_levels > 1: + num_backbone_outs = len(backbone.strides) + input_proj_list = [] + for _ in range(num_backbone_outs): + in_channels = backbone.num_channels[_] + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, hidden_dim, kernel_size=patch_size, stride=patch_size, padding=0), + nn.GroupNorm(32, hidden_dim), + ) + ) + for _ in range(num_feature_levels - num_backbone_outs): + if patch_size == 1: + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, hidden_dim, kernel_size=3, stride=2, padding=1), + nn.GroupNorm(32, hidden_dim), + ) + ) + else: + input_proj_list.append( + nn.Sequential( + nn.Conv2d( + in_channels, hidden_dim, kernel_size=2 * patch_size, stride=2 * patch_size, padding=0 + ), + nn.GroupNorm(32, hidden_dim), + ) + ) + in_channels = hidden_dim + self.input_proj = nn.ModuleList(input_proj_list) + else: + self.input_proj = nn.ModuleList( + [ + nn.Sequential( + nn.Conv2d(backbone.num_channels[0], hidden_dim, kernel_size=1), + nn.GroupNorm(32, hidden_dim), + ) + ] + ) + self.backbone = backbone + self.aux_loss = aux_loss + self.with_poly_refine = with_poly_refine + + prior_prob = 0.01 + bias_value = -math.log((1 - prior_prob) / prior_prob) + self.class_embed.bias.data = torch.ones(num_classes) * bias_value + nn.init.constant_(self.coords_embed.layers[-1].weight.data, 0) + nn.init.constant_(self.coords_embed.layers[-1].bias.data, 0) + for proj in self.input_proj: + nn.init.xavier_uniform_(proj[0].weight, gain=1) + nn.init.constant_(proj[0].bias, 0) + + num_pred = transformer.decoder.num_layers + + if with_poly_refine: + self.class_embed = _get_clones(self.class_embed, num_pred) + self.coords_embed = _get_clones(self.coords_embed, num_pred) + nn.init.constant_(self.coords_embed[0].layers[-1].bias.data[2:], -2.0) + else: + nn.init.constant_(self.coords_embed.layers[-1].bias.data[2:], -2.0) + self.class_embed = nn.ModuleList([self.class_embed for _ in range(num_pred)]) + self.coords_embed = nn.ModuleList([self.coords_embed for _ in range(num_pred)]) + + if use_anchor or with_poly_refine: + self.query_embed = nn.Embedding(seq_len, 2) + self.query_embed.weight.requires_grad = not freeze_anchor + else: + self.query_embed = None + + self.transformer.decoder.coords_embed = self.coords_embed + self.transformer.decoder.class_embed = self.class_embed + + # Semantically-rich floorplan + self.room_class_embed = None + if semantic_classes > 0: + self.room_class_embed = nn.Linear(hidden_dim, semantic_classes) + if self.inject_cls_embed: + self.transformer.decoder.room_class_embed = self.room_class_embed + + # self.num_queries_per_poly = num_queries // num_polys + + # # The attention mask is used to prevent object queries in one polygon attending to another polygon, default false + # if masked_attn: + # self.attention_mask = torch.ones((num_queries, num_queries), dtype=torch.bool) + # for i in range(num_polys): + # self.attention_mask[i * self.num_queries_per_poly:(i + 1) * self.num_queries_per_poly, + # i * self.num_queries_per_poly:(i + 1) * self.num_queries_per_poly] = False + # else: + # self.attention_mask = None + + self.register_buffer("attention_mask", self._create_causal_attention_mask(seq_len)) + + def _create_causal_attention_mask(self, seq_len): + """ + Creates a causal attention mask for a sequence of length `seq_len`. + """ + # Create an upper triangular matrix with 1s above the diagonal + mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) + # Invert the mask: 1 -> -inf (masked), 0 -> 0 (unmasked) + causal_mask = mask.masked_fill(mask == 1, float("-inf")).masked_fill(mask == 0, 0.0) + return causal_mask + + def forward(self, samples: NestedTensor, seq_kwargs=None): + """The forward expects a NestedTensor, which consists of: + - samples.tensors: batched images, of shape [batch_size x C x H x W] + - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels + + It returns a dict with the following elements: + - "pred_logits": the classification logits (including no-object) for all queries. + Shape= [batch_size x num_queries x (num_classes + 1)] + - "pred_coords": The normalized corner coordinates for all queries, represented as + (x, y). These values are normalized in [0, 1], + relative to the size of each individual image (disregarding possible padding). + - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of + dictionnaries containing the two above keys for each decoder layer. + """ + if not isinstance(samples, NestedTensor): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.backbone(samples) + + srcs = [] + masks = [] + for l, feat in enumerate(features): + src, mask = feat.decompose() + src = self.input_proj[l](src) + srcs.append(src) + if self.patch_size != 1: + mask = F.interpolate(mask[None].float(), size=src.shape[-2:]).to(torch.bool)[0] + pos[l] = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + masks.append(mask) + assert mask is not None + if self.num_feature_levels > len(srcs): + _len_srcs = len(srcs) + for l in range(_len_srcs, self.num_feature_levels): + if l == _len_srcs: + src = self.input_proj[l](features[-1].tensors) + else: + src = self.input_proj[l](srcs[-1]) + m = samples.mask + mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0] + pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + srcs.append(src) + masks.append(mask) + pos.append(pos_l) + + query_embeds = None if self.query_embed is None else self.query_embed.weight + tgt_embeds = None + + hs, init_reference, inter_references, inter_classes = self.transformer( + srcs, masks, pos, query_embeds, tgt_embeds, self.attention_mask, seq_kwargs + ) + + outputs_class = inter_classes + outputs_coord = inter_references + + out = {"pred_logits": outputs_class[-1], "pred_coords": outputs_coord[-1]} + + if self.room_class_embed is not None: + outputs_room_class = self.room_class_embed(hs[-1]) + out = { + "pred_logits": outputs_class[-1], + "pred_coords": outputs_coord[-1], + "pred_room_logits": outputs_room_class, + } + + if self.aux_loss: + out["aux_outputs"] = self._set_aux_loss(outputs_class, outputs_coord) + + return out + + def _prepare_sequences(self, b): + prev_output_token_11 = [[self.tokenizer.bos] for _ in range(b)] + prev_output_token_12 = [[self.tokenizer.bos] for _ in range(b)] + prev_output_token_21 = [[self.tokenizer.bos] for _ in range(b)] + prev_output_token_22 = [[self.tokenizer.bos] for _ in range(b)] + delta_x1 = [[0] for _ in range(b)] + delta_y1 = [[0] for _ in range(b)] + delta_x2 = [[1] for _ in range(b)] + delta_y2 = [[1] for _ in range(b)] + + gen_out = [[] for _ in range(b)] + + if self.inject_cls_embed: + input_polygon_labels = [[self.semantic_classes - 1] for _ in range(b)] + else: + input_polygon_labels = [[-1] for _ in range(b)] # dummies values, not used in inference + + return ( + prev_output_token_11, + prev_output_token_12, + prev_output_token_21, + prev_output_token_22, + delta_x1, + delta_x2, + delta_y1, + delta_y2, + gen_out, + input_polygon_labels, + ) + + def forward_inference(self, samples: NestedTensor, use_cache=True): + """The forward expects a NestedTensor, which consists of: + - samples.tensors: batched images, of shape [batch_size x C x H x W] + - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels + + It returns a dict with the following elements: + - "pred_logits": the classification logits (including no-object) for all queries. + Shape= [batch_size x num_queries x (num_classes + 1)] + - "pred_coords": The normalized corner coordinates for all queries, represented as + (x, y). These values are normalized in [0, 1], + relative to the size of each individual image (disregarding possible padding). + - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of + dictionnaries containing the two above keys for each decoder layer. + """ + if not isinstance(samples, NestedTensor): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.backbone(samples) + + bs = samples.tensors.shape[0] + + srcs = [] + masks = [] + for l, feat in enumerate(features): + src, mask = feat.decompose() + src = self.input_proj[l](src) + srcs.append(src) + if self.patch_size != 1: + mask = F.interpolate(mask[None].float(), size=src.shape[-2:]).to(torch.bool)[0] + pos[l] = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + masks.append(mask) + assert mask is not None + if self.num_feature_levels > len(srcs): + _len_srcs = len(srcs) + for l in range(_len_srcs, self.num_feature_levels): + if l == _len_srcs: + src = self.input_proj[l](features[-1].tensors) + else: + src = self.input_proj[l](srcs[-1]) + m = samples.mask + mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0] + pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + srcs.append(src) + masks.append(mask) + pos.append(pos_l) + + ##### decoder part + if use_cache: + # kv cache for faster inference + max_src_len = sum([x.size(2) * x.size(3) for x in srcs]) # 1360 + self._setup_caches(bs, max_src_len) + + ( + prev_output_token_11, + prev_output_token_12, + prev_output_token_21, + prev_output_token_22, + delta_x1, + delta_x2, + delta_y1, + delta_y2, + gen_out, + input_polygon_labels, + ) = self._prepare_sequences(bs) + + query_embeds = None if self.query_embed is None else self.query_embed.weight + # tgt_embeds = self.tgt_embed.weight + tgt_embeds = None + enc_cache = None + + device = samples.tensors.device + num_bins = self.tokenizer.num_bins + min_len = 6 + max_len = self.tokenizer.seq_len + unfinish_flag = np.ones(bs) + + i = 0 + + output_hs_list = [] + while i < max_len and unfinish_flag.any(): + prev_output_tokens_11_tensor = torch.tensor(np.array(prev_output_token_11)[:, i : i + 1]).to(device).long() + prev_output_tokens_12_tensor = torch.tensor(np.array(prev_output_token_12)[:, i : i + 1]).to(device).long() + prev_output_tokens_21_tensor = torch.tensor(np.array(prev_output_token_21)[:, i : i + 1]).to(device).long() + prev_output_tokens_22_tensor = torch.tensor(np.array(prev_output_token_22)[:, i : i + 1]).to(device).long() + delta_x1_tensor = torch.tensor(np.array(delta_x1)[:, i : i + 1], dtype=torch.float32).to(device) + delta_x2_tensor = torch.tensor(np.array(delta_x2)[:, i : i + 1], dtype=torch.float32).to(device) + delta_y1_tensor = torch.tensor(np.array(delta_y1)[:, i : i + 1], dtype=torch.float32).to(device) + delta_y2_tensor = torch.tensor(np.array(delta_y2)[:, i : i + 1], dtype=torch.float32).to(device) + input_polygon_labels_tensor = torch.tensor( + np.array(input_polygon_labels)[:, i : i + 1], dtype=torch.long + ).to(device) + + seq_kwargs = { + "seq11": prev_output_tokens_11_tensor, + "seq12": prev_output_tokens_12_tensor, + "seq21": prev_output_tokens_21_tensor, + "seq22": prev_output_tokens_22_tensor, + "delta_x1": delta_x1_tensor, + "delta_x2": delta_x2_tensor, + "delta_y1": delta_y1_tensor, + "delta_y2": delta_y2_tensor, + "input_polygon_labels": input_polygon_labels_tensor, + } + + if not use_cache: + hs, _, reg_output, cls_output = self.transformer( + srcs, + masks, + pos, + query_embeds, + tgt_embeds, + None, + seq_kwargs, + force_simple_returns=True, + return_enc_cache=use_cache, + enc_cache=None, + decode_token_pos=None, + ) + output_hs_list.append(hs[:, i : i + 1]) + else: + decode_token_pos = torch.tensor([i], device=device, dtype=torch.long) + hs, _, reg_output, cls_output, enc_cache = self.transformer( + srcs, + masks, + pos, + query_embeds, + tgt_embeds, + None, + seq_kwargs, + force_simple_returns=True, + return_enc_cache=use_cache, + enc_cache=enc_cache, + decode_token_pos=decode_token_pos, + ) + output_hs_list.append(hs) + cls_type = torch.argmax(cls_output, 2) + # print(cls_type, torch.softmax(cls_output, dim=2)[:, :, cls_type], torch.topk(torch.softmax(cls_output, dim=2), k=3)) + for j in range(bs): + if unfinish_flag[j] == 1: # prediction is not finished + cls_j = cls_type[j, 0].item() + if cls_j == TokenType.coord.value or (cls_j == TokenType.eos.value and i < min_len): + output_j_x, output_j_y = reg_output[j, 0].detach().cpu().numpy() + output_j_x = min(output_j_x, 1) + output_j_y = min(output_j_y, 1) + + gen_out[j].append([output_j_x, output_j_y]) + + output_j_x = output_j_x * (num_bins - 1) + output_j_y = output_j_y * (num_bins - 1) + + output_j_x_floor = math.floor(output_j_x) + output_j_y_floor = math.floor(output_j_y) + output_j_x_ceil = math.ceil(output_j_x) + output_j_y_ceil = math.ceil(output_j_y) + + # tokenization + prev_output_token_11[j].append(output_j_x_floor * num_bins + output_j_y_floor) + prev_output_token_12[j].append(output_j_x_floor * num_bins + output_j_y_ceil) + prev_output_token_21[j].append(output_j_x_ceil * num_bins + output_j_y_floor) + prev_output_token_22[j].append(output_j_x_ceil * num_bins + output_j_y_ceil) + + delta_x = output_j_x - output_j_x_floor + delta_y = output_j_y - output_j_y_floor + + elif cls_j == TokenType.sep.value: + gen_out[j].append(2) + prev_output_token_11[j].append(self.tokenizer.sep) + prev_output_token_12[j].append(self.tokenizer.sep) + prev_output_token_21[j].append(self.tokenizer.sep) + prev_output_token_22[j].append(self.tokenizer.sep) + + delta_x = 0 + delta_y = 0 + + elif cls_j == TokenType.cls.value: + gen_out[j].append(-1) + prev_output_token_11[j].append(self.tokenizer.cls) + prev_output_token_12[j].append(self.tokenizer.cls) + prev_output_token_21[j].append(self.tokenizer.cls) + prev_output_token_22[j].append(self.tokenizer.cls) + delta_x = 0 + delta_y = 0 + + else: # eos is predicted and i >= min_len + unfinish_flag[j] = 0 + gen_out[j].append(-1) + prev_output_token_11[j].append(self.tokenizer.eos) + prev_output_token_12[j].append(self.tokenizer.eos) + prev_output_token_21[j].append(self.tokenizer.eos) + prev_output_token_22[j].append(self.tokenizer.eos) + delta_x = 0 + delta_y = 0 + + else: # prediction is finished + gen_out[j].append(-1) + prev_output_token_11[j].append(self.tokenizer.pad) + prev_output_token_12[j].append(self.tokenizer.pad) + prev_output_token_21[j].append(self.tokenizer.pad) + prev_output_token_22[j].append(self.tokenizer.pad) + delta_x = 0 + delta_y = 0 + delta_x1[j].append(delta_x) + delta_y1[j].append(delta_y) + delta_x2[j].append(1 - delta_x) + delta_y2[j].append(1 - delta_y) + i += 1 + + out = {"pred_logits": cls_output, "pred_coords": reg_output, "gen_out": gen_out} + + # hack implementation of room label prediction, not compatible with auxiliary loss + if self.room_class_embed is not None: + hs = torch.cat(output_hs_list, dim=1) + outputs_room_class = self.room_class_embed(hs) + out = { + "pred_logits": cls_output, + "pred_coords": reg_output, + "pred_room_logits": outputs_room_class, + "gen_out": gen_out, + "anchors": query_embeds.detach(), + } + + return out + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord): + return [{"pred_logits": a, "pred_coords": b} for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] + + def _setup_caches(self, max_bs, max_src_len): + self.transformer._setup_caches( + max_bs, + self.seq_len, + max_src_len, + self.transformer.d_model, + self.transformer.nhead, + self.transformer.level_embed.dtype, + device=self.transformer.level_embed.device, + ) + + +class SemHead(nn.Module): + def __init__(self, hidden_dim, num_classes): + super().__init__() + self.shared_layer = nn.Linear(hidden_dim, hidden_dim) + self.room_embed = nn.Linear(hidden_dim, num_classes - 2) + self.num_classes = num_classes + self.window_door_embed = nn.Linear(hidden_dim, 2) + + def forward(self, x): + x = F.normalize(torch.relu(self.shared_layer(x)), p=2, dim=-1, eps=1e-12) + room_out = self.room_embed(x) + window_door_out = self.window_door_embed(x) + out = torch.cat([room_out[:, :, :-1], window_door_out, room_out[:, :, -1:]], dim=-1) + return out.contiguous() + + +class SetCriterion(nn.Module): + """This class computes the loss for multiple polygons. + The process happens in two steps: + 1) we compute hungarian assignment between ground truth polygons and the outputs of the model + 2) we supervise each pair of matched ground-truth / prediction (supervise class and coords) + """ + + def __init__( + self, + num_classes, + semantic_classes, + matcher, + weight_dict, + losses, + label_smoothing=0.0, + per_token_sem_loss=False, + ): + """Create the criterion. + Parameters: + num_classes: number of classes for corner validity (binary) + semantic_classes: number of semantic classes for polygon (room type, door, window) + matcher: module able to compute a matching between targets and proposals + weight_dict: dict containing as key the names of the losses and as values their relative weight. + losses: list of all the losses to be applied. See get_loss for list of available losses. + """ + super().__init__() + self.num_classes = num_classes + self.semantic_classes = semantic_classes + self.matcher = matcher + self.weight_dict = weight_dict + self.losses = losses + self.label_smoothing = label_smoothing + self.per_token_sem_loss = per_token_sem_loss + + if "loss_raster" in self.weight_dict: + self.raster_loss = MaskRasterizationLoss(None) + + def _update_ce_coeff(self, loss_ce_coeff): + self.weight_dict["loss_ce"] = loss_ce_coeff + + def loss_labels(self, outputs, targets, indices): + """Classification loss (NLL) + targets dicts must contain the key "labels" + """ + assert "pred_logits" in outputs + src_logits = outputs["pred_logits"] + + target_classes = targets["token_labels"].to(src_logits.device) + mask = (target_classes != -1).bool() + loss_ce = label_smoothed_nll_loss( + src_logits[mask], target_classes[mask], epsilon=self.label_smoothing, reduction="mean" + ) + losses = {"loss_ce": loss_ce} + + if "pred_room_logits" in outputs: + room_src_logits = outputs["pred_room_logits"] + if not self.per_token_sem_loss: + mask = target_classes == 3 # cls token + room_target_classes = targets["target_polygon_labels"].to(room_src_logits.device) + loss_ce_room = label_smoothed_nll_loss( + room_src_logits[mask], + room_target_classes[room_target_classes != -1], + epsilon=self.label_smoothing, + reduction="mean", + ) + else: + room_target_classes = targets["target_polygon_labels"].to(room_src_logits.device) + loss_ce_room = label_smoothed_nll_loss( + room_src_logits[room_target_classes != -1], + room_target_classes[room_target_classes != -1], + epsilon=self.label_smoothing, + reduction="mean", + ) + + losses = {"loss_ce": loss_ce, "loss_ce_room": loss_ce_room} + + return losses + + @torch.no_grad() + def loss_cardinality(self, outputs, targets, indices): + """Compute the cardinality error, ie the absolute error in the number of predicted non-empty corners + This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients + """ + losses = {"cardinality_error": 0.0} + return losses + + def _extract_polygons(self, sequence, token_labels): + # sequence: [B, N, 2], token_labels: [B, N] + B, N = token_labels.shape + polygons = [] + + for b in range(B): + labels = token_labels[b] # [N] + coords = sequence[b] # [N, 2] + + # Find separator and EOS positions + sep_eos_mask = (labels == 1) | (labels == 2) + split_indices = torch.nonzero(sep_eos_mask, as_tuple=False).squeeze(-1) + + # Handle empty case + if len(split_indices) == 0: + # No separators found, treat entire sequence as one polygon + corner_mask = labels == 0 + if corner_mask.any(): + polygons.append(coords[corner_mask]) + continue + + # Create start and end indices + device = labels.device + starts = torch.cat([torch.tensor([0], device=device), split_indices[:-1] + 1]) + ends = split_indices + + # Extract polygons between separators + for s, e in zip(starts, ends): + if s < e: # Valid range + segment_labels = labels[s:e] + segment_coords = coords[s:e] + corner_mask = segment_labels == 0 + if corner_mask.any(): + polygons.append(segment_coords[corner_mask]) + + return polygons + + def loss_polys(self, outputs, targets, indices): + """Compute the losses related to the polygons: + 1. L1 loss for polygon coordinates + 2. Dice loss for polygon rasterizated binary masks + """ + assert "pred_coords" in outputs + src_poly = outputs["pred_coords"] + device = src_poly.device + token_labels = targets["token_labels"].to(device) + mask = (token_labels == 0).bool() + target_polys = targets["target_seq"].to(device) + + loss_coords = F.l1_loss(src_poly[mask], target_polys[mask]) + + losses = {} + losses["loss_coords"] = loss_coords + + # omit the rasterization loss for semantically-rich floorplan + if self.weight_dict.get("loss_raster", 0) > 0: + pred_poly_list = self._extract_polygons(src_poly, token_labels) + target_poly_list = self._extract_polygons(target_polys, token_labels) + loss_raster_mask = self.raster_loss( + pred_poly_list, + target_poly_list, + [len(x) for x in target_poly_list], + ) + losses["loss_raster"] = loss_raster_mask + + return losses + + def _get_src_permutation_idx(self, indices): + # permute predictions following indices + batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) + src_idx = torch.cat([src for (src, _) in indices]) + return batch_idx, src_idx + + def _get_tgt_permutation_idx(self, indices): + # permute targets following indices + batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) + tgt_idx = torch.cat([tgt for (_, tgt) in indices]) + return batch_idx, tgt_idx + + def get_loss(self, loss, outputs, targets, indices, **kwargs): + loss_map = {"labels": self.loss_labels, "cardinality": self.loss_cardinality, "polys": self.loss_polys} + assert loss in loss_map, f"do you really want to compute {loss} loss?" + return loss_map[loss](outputs, targets, indices, **kwargs) + + def forward(self, outputs, targets): + """This performs the loss computation. + Parameters: + outputs: dict of tensors, see the output specification of the model for the format + targets: list of dicts, such that len(targets) == batch_size. + The expected keys in each dict depends on the losses applied, see each loss' doc + """ + indices = None + + # Compute all the requested losses + losses = {} + for loss in self.losses: + kwargs = {} + losses.update(self.get_loss(loss, outputs, targets, indices, **kwargs)) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + if "aux_outputs" in outputs: + for i, aux_outputs in enumerate(outputs["aux_outputs"]): + for loss in self.losses: + l_dict = self.get_loss(loss, aux_outputs, targets, indices) + l_dict = {k + f"_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + if "enc_outputs" in outputs: + enc_outputs = outputs["enc_outputs"] + indices = self.matcher(enc_outputs, targets) + for loss in self.losses: + l_dict = self.get_loss(loss, enc_outputs, targets, indices) + l_dict = {k + "_enc": v for k, v in l_dict.items()} + losses.update(l_dict) + + return losses + + +class MLP(nn.Module): + """Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +def build(args, train=True, tokenizer=None): + num_classes = 3 if not args.add_cls_token else 4 # + if tokenizer is not None: + pad_idx = tokenizer.pad + + backbone = build_backbone(args) + transformer = build_deforamble_transformer(args, pad_idx=pad_idx) + model = Raster2Seq( + backbone, + transformer, + num_classes=num_classes, + num_queries=args.num_queries, + num_polys=args.num_polys, + num_feature_levels=args.num_feature_levels, + aux_loss=args.aux_loss, + with_poly_refine=args.with_poly_refine, + masked_attn=args.masked_attn, + semantic_classes=args.semantic_classes, + seq_len=args.seq_len, + tokenizer=tokenizer, + use_anchor=args.use_anchor, + patch_size=[1, 2][args.image_size == 512], # 1 for 256x256, 2 for 512x512 + freeze_anchor=getattr(args, "freeze_anchor", False), + inject_cls_embed=getattr(args, "inject_cls_embed", False), + ) + + if not train: + return model + + device = torch.device(args.device) + matcher = None # build_matcher(args) + weight_dict = { + "loss_ce": args.cls_loss_coef, + "loss_ce_room": args.room_cls_loss_coef, + "loss_coords": args.coords_loss_coef, + } + if args.raster_loss_coef > 0: + weight_dict["loss_raster"] = args.raster_loss_coef + weight_dict["loss_dir"] = 1 + + enc_weight_dict = {} + enc_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()}) + weight_dict.update(enc_weight_dict) + # TODO this is a hack + if args.aux_loss: + aux_weight_dict = {} + for i in range(args.dec_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + aux_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + losses = ["labels", "polys", "cardinality"] + # num_classes, matcher, weight_dict, losses + criterion = SetCriterion( + num_classes, + args.semantic_classes, + matcher, + weight_dict, + losses, + label_smoothing=args.label_smoothing, + per_token_sem_loss=args.per_token_sem_loss, + ) + criterion.to(device) + + return model, criterion diff --git a/models/roomformer.py b/models/roomformer.py new file mode 100644 index 0000000000000000000000000000000000000000..82ba358873d6baf6b655e7d9a918ce835e977529 --- /dev/null +++ b/models/roomformer.py @@ -0,0 +1,452 @@ +# ------------------------------------------------------------------------------------ +# Original RoomFormer implementation (https://github.com/ywyue/RoomFormer.git) +# ------------------------------------------------------------------------------------ + +import copy +import math + +import torch +import torch.nn.functional as F +from torch import nn + +from util.misc import NestedTensor, nested_tensor_from_tensor_list + +from .backbone import build_backbone +from .deformable_transformer import build_deforamble_transformer +from .losses import MaskRasterizationLoss, custom_L1_loss +from .matcher import build_matcher + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +class RoomFormer(nn.Module): + """This is the RoomFormer module that performs floorplan reconstruction""" + + def __init__( + self, + backbone, + transformer, + num_classes, + num_queries, + num_polys, + num_feature_levels, + aux_loss=True, + with_poly_refine=False, + masked_attn=False, + semantic_classes=-1, + patch_size=1, + ): + """Initializes the model. + Parameters: + backbone: torch module of the backbone to be used. See backbone.py + transformer: torch module of the transformer architecture. See transformer.py + num_classes: number of object classes + num_queries: number of object queries, ie detection slot. This is the maximal number of possible corners + in a single image. + num_polys: maximal number of possible polygons in a single image. + num_queries/num_polys would be the maximal number of possible corners in a single polygon. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + with_poly_refine: iterative polygon refinement + """ + super().__init__() + self.num_queries = num_queries + self.num_polys = num_polys + self.num_classes = num_classes + assert num_queries % num_polys == 0 + self.transformer = transformer + hidden_dim = transformer.d_model + self.class_embed = nn.Linear(hidden_dim, num_classes) + self.coords_embed = MLP(hidden_dim, hidden_dim, 2, 3) + self.num_feature_levels = num_feature_levels + self.patch_size = patch_size + + self.query_embed = nn.Embedding(num_queries, 2) + self.tgt_embed = nn.Embedding(num_queries, hidden_dim) + if num_feature_levels > 1: + num_backbone_outs = len(backbone.strides) + input_proj_list = [] + for _ in range(num_backbone_outs): + in_channels = backbone.num_channels[_] + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, hidden_dim, kernel_size=patch_size, stride=patch_size, padding=0), + nn.GroupNorm(32, hidden_dim), + ) + ) + for _ in range(num_feature_levels - num_backbone_outs): + if patch_size == 1: + input_proj_list.append( + nn.Sequential( + nn.Conv2d(in_channels, hidden_dim, kernel_size=3, stride=2, padding=1), + nn.GroupNorm(32, hidden_dim), + ) + ) + else: + input_proj_list.append( + nn.Sequential( + nn.Conv2d( + in_channels, hidden_dim, kernel_size=2 * patch_size, stride=2 * patch_size, padding=0 + ), + nn.GroupNorm(32, hidden_dim), + ) + ) + in_channels = hidden_dim + self.input_proj = nn.ModuleList(input_proj_list) + else: + self.input_proj = nn.ModuleList( + [ + nn.Sequential( + nn.Conv2d(backbone.num_channels[0], hidden_dim, kernel_size=1), + nn.GroupNorm(32, hidden_dim), + ) + ] + ) + self.backbone = backbone + self.aux_loss = aux_loss + self.with_poly_refine = with_poly_refine + + prior_prob = 0.01 + bias_value = -math.log((1 - prior_prob) / prior_prob) + self.class_embed.bias.data = torch.ones(num_classes) * bias_value + nn.init.constant_(self.coords_embed.layers[-1].weight.data, 0) + nn.init.constant_(self.coords_embed.layers[-1].bias.data, 0) + for proj in self.input_proj: + nn.init.xavier_uniform_(proj[0].weight, gain=1) + nn.init.constant_(proj[0].bias, 0) + + num_pred = transformer.decoder.num_layers + + if with_poly_refine: + self.class_embed = _get_clones(self.class_embed, num_pred) + self.coords_embed = _get_clones(self.coords_embed, num_pred) + nn.init.constant_(self.coords_embed[0].layers[-1].bias.data[2:], -2.0) + else: + nn.init.constant_(self.coords_embed.layers[-1].bias.data[2:], -2.0) + self.class_embed = nn.ModuleList([self.class_embed for _ in range(num_pred)]) + self.coords_embed = nn.ModuleList([self.coords_embed for _ in range(num_pred)]) + + self.transformer.decoder.coords_embed = self.coords_embed + self.transformer.decoder.class_embed = self.class_embed + + # Semantically-rich floorplan + self.room_class_embed = None + if semantic_classes > 0: + self.room_class_embed = nn.Linear(hidden_dim, semantic_classes) + + self.num_queries_per_poly = num_queries // num_polys + + # The attention mask is used to prevent object queries in one polygon attending to another polygon, default false + if masked_attn: + self.attention_mask = torch.ones((num_queries, num_queries), dtype=torch.bool) + for i in range(num_polys): + self.attention_mask[ + i * self.num_queries_per_poly : (i + 1) * self.num_queries_per_poly, + i * self.num_queries_per_poly : (i + 1) * self.num_queries_per_poly, + ] = False + else: + self.attention_mask = None + + def forward(self, samples: NestedTensor): + """The forward expects a NestedTensor, which consists of: + - samples.tensors: batched images, of shape [batch_size x C x H x W] + - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels + + It returns a dict with the following elements: + - "pred_logits": the classification logits (including no-object) for all queries. + Shape= [batch_size x num_queries x (num_classes + 1)] + - "pred_coords": The normalized corner coordinates for all queries, represented as + (x, y). These values are normalized in [0, 1], + relative to the size of each individual image (disregarding possible padding). + - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of + dictionnaries containing the two above keys for each decoder layer. + """ + if not isinstance(samples, NestedTensor): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.backbone(samples) + + bs = samples.tensors.shape[0] + srcs = [] + masks = [] + for l, feat in enumerate(features): + src, mask = feat.decompose() + src = self.input_proj[l](src) + srcs.append(src) + if self.patch_size != 1: + mask = F.interpolate(mask[None].float(), size=src.shape[-2:]).to(torch.bool)[0] + pos[l] = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + masks.append(mask) + assert mask is not None + if self.num_feature_levels > len(srcs): + _len_srcs = len(srcs) + for l in range(_len_srcs, self.num_feature_levels): + if l == _len_srcs: + src = self.input_proj[l](features[-1].tensors) + else: + src = self.input_proj[l](srcs[-1]) + m = samples.mask + mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0] + pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype) + srcs.append(src) + masks.append(mask) + pos.append(pos_l) + + query_embeds = self.query_embed.weight + tgt_embeds = self.tgt_embed.weight + + hs, init_reference, inter_references, inter_classes = self.transformer( + srcs, masks, pos, query_embeds, tgt_embeds, self.attention_mask + ) + + num_layer = hs.shape[0] + outputs_class = inter_classes.reshape(num_layer, bs, self.num_polys, self.num_queries_per_poly) + outputs_coord = inter_references.reshape(num_layer, bs, self.num_polys, self.num_queries_per_poly, 2) + + out = {"pred_logits": outputs_class[-1], "pred_coords": outputs_coord[-1]} + + # hack implementation of room label prediction, not compatible with auxiliary loss + if self.room_class_embed is not None: + outputs_room_class = self.room_class_embed( + hs[-1].view(bs, self.num_polys, self.num_queries_per_poly, -1).mean(axis=2) + ) + out = { + "pred_logits": outputs_class[-1], + "pred_coords": outputs_coord[-1], + "pred_room_logits": outputs_room_class, + } + + if self.aux_loss: + out["aux_outputs"] = self._set_aux_loss(outputs_class, outputs_coord) + + return out + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + return [{"pred_logits": a, "pred_coords": b} for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] + + +class SetCriterion(nn.Module): + """This class computes the loss for multiple polygons. + The process happens in two steps: + 1) we compute hungarian assignment between ground truth polygons and the outputs of the model + 2) we supervise each pair of matched ground-truth / prediction (supervise class and coords) + """ + + def __init__(self, num_classes, semantic_classes, matcher, weight_dict, losses, ignore_index=-1): + """Create the criterion. + Parameters: + num_classes: number of classes for corner validity (binary) + semantic_classes: number of semantic classes for polygon (room type, door, window) + matcher: module able to compute a matching between targets and proposals + weight_dict: dict containing as key the names of the losses and as values their relative weight. + losses: list of all the losses to be applied. See get_loss for list of available losses. + """ + super().__init__() + self.num_classes = num_classes + self.semantic_classes = semantic_classes + self.matcher = matcher + self.weight_dict = weight_dict + self.losses = losses + self.raster_loss = MaskRasterizationLoss(None) + self.ignore_index = ignore_index + + def loss_labels(self, outputs, targets, indices): + """Classification loss (NLL) + targets dicts must contain the key "labels" + """ + assert "pred_logits" in outputs + src_logits = outputs["pred_logits"] + + idx = self._get_src_permutation_idx(indices) + target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape, self.num_classes - 1, dtype=torch.float32, device=src_logits.device + ) + target_classes[idx] = target_classes_o + + loss_ce = F.binary_cross_entropy_with_logits(src_logits, target_classes) + + losses = {"loss_ce": loss_ce} + + # hack implementation of room label/door/window prediction + if "pred_room_logits" in outputs: + room_src_logits = outputs["pred_room_logits"] + room_target_classes_o = torch.cat([t["room_labels"][J] for t, (_, J) in zip(targets, indices)]) + room_target_classes = torch.full( + room_src_logits.shape[:2], self.semantic_classes - 1, dtype=torch.int64, device=room_src_logits.device + ) + room_target_classes[idx] = room_target_classes_o + loss_ce_room = F.cross_entropy( + room_src_logits.transpose(1, 2), room_target_classes, ignore_index=self.ignore_index + ) + losses = {"loss_ce": loss_ce, "loss_ce_room": loss_ce_room} + + return losses + + @torch.no_grad() + def loss_cardinality(self, outputs, targets, indices): + """Compute the cardinality error, ie the absolute error in the number of predicted non-empty corners + This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients + """ + pred_logits = outputs["pred_logits"] + device = pred_logits.device + tgt_lengths = torch.as_tensor([sum(v["lengths"]) for v in targets], device=device) / 2 + # Count the number of predictions that are NOT "no-object" (invalid corners) + card_pred = (pred_logits.sigmoid() > 0.5).flatten(1, 2).sum(1) + card_err = F.l1_loss(card_pred.float(), tgt_lengths.float()) + losses = {"cardinality_error": card_err} + return losses + + def loss_polys(self, outputs, targets, indices): + """Compute the losses related to the polygons: + 1. L1 loss for polygon coordinates + 2. Dice loss for polygon rasterizated binary masks + """ + assert "pred_coords" in outputs + idx = self._get_src_permutation_idx(indices) + src_polys = outputs["pred_coords"][idx] + target_polys = torch.cat([t["coords"][i] for t, (_, i) in zip(targets, indices)], dim=0) + target_len = torch.cat([t["lengths"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + loss_coords = custom_L1_loss(src_polys.flatten(1, 2), target_polys, target_len) + + losses = {} + losses["loss_coords"] = loss_coords + + # omit the rasterization loss for semantically-rich floorplan + if self.semantic_classes == -1: + loss_raster_mask = self.raster_loss(src_polys.flatten(1, 2), target_polys, target_len) + losses["loss_raster"] = loss_raster_mask + + return losses + + def _get_src_permutation_idx(self, indices): + # permute predictions following indices + batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) + src_idx = torch.cat([src for (src, _) in indices]) + return batch_idx, src_idx + + def _get_tgt_permutation_idx(self, indices): + # permute targets following indices + batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) + tgt_idx = torch.cat([tgt for (_, tgt) in indices]) + return batch_idx, tgt_idx + + def get_loss(self, loss, outputs, targets, indices, **kwargs): + loss_map = {"labels": self.loss_labels, "cardinality": self.loss_cardinality, "polys": self.loss_polys} + assert loss in loss_map, f"do you really want to compute {loss} loss?" + return loss_map[loss](outputs, targets, indices, **kwargs) + + def forward(self, outputs, targets): + """This performs the loss computation. + Parameters: + outputs: dict of tensors, see the output specification of the model for the format + targets: list of dicts, such that len(targets) == batch_size. + The expected keys in each dict depends on the losses applied, see each loss' doc + """ + outputs_without_aux = {k: v for k, v in outputs.items() if k != "aux_outputs" and k != "enc_outputs"} + + # Retrieve the matching between the outputs of the last layer and the targets + indices = self.matcher(outputs_without_aux, targets) + + # Compute all the requested losses + losses = {} + for loss in self.losses: + kwargs = {} + losses.update(self.get_loss(loss, outputs, targets, indices, **kwargs)) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + if "aux_outputs" in outputs: + for i, aux_outputs in enumerate(outputs["aux_outputs"]): + # indices = self.matcher(aux_outputs, targets) + for loss in self.losses: + l_dict = self.get_loss(loss, aux_outputs, targets, indices) + l_dict = {k + f"_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + if "enc_outputs" in outputs: + enc_outputs = outputs["enc_outputs"] + # bin_targets = copy.deepcopy(targets) + # for bt in bin_targets: + # bt['labels'] = torch.zeros_like(bt['labels']) + # indices = self.matcher(enc_outputs, bin_targets) + indices = self.matcher(enc_outputs, targets) + for loss in self.losses: + # l_dict = self.get_loss(loss, enc_outputs, bin_targets, indices) + l_dict = self.get_loss(loss, enc_outputs, targets, indices) + l_dict = {k + "_enc": v for k, v in l_dict.items()} + losses.update(l_dict) + + return losses + + +class MLP(nn.Module): + """Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +def build(args, train=True): + num_classes = 1 # valid or invalid corner + + backbone = build_backbone(args) + transformer = build_deforamble_transformer(args) + model = RoomFormer( + backbone, + transformer, + num_classes=num_classes, + num_queries=args.num_queries, + num_polys=args.num_polys, + num_feature_levels=args.num_feature_levels, + aux_loss=args.aux_loss, + with_poly_refine=args.with_poly_refine, + masked_attn=args.masked_attn, + semantic_classes=args.semantic_classes, + patch_size=1, # [1, 2][args.image_size == 512], # 1 for 256x256, 2 for 512x512 + ) + + if not train: + return model + + device = torch.device(args.device) + matcher = build_matcher(args) + weight_dict = { + "loss_ce": args.cls_loss_coef, + "loss_ce_room": args.room_cls_loss_coef, + "loss_coords": args.coords_loss_coef, + "loss_raster": args.raster_loss_coef, + } + weight_dict["loss_dir"] = 1 + + enc_weight_dict = {} + enc_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()}) + weight_dict.update(enc_weight_dict) + # TODO this is a hack + if args.aux_loss: + aux_weight_dict = {} + for i in range(args.dec_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + aux_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + losses = ["labels", "polys", "cardinality"] + # num_classes, matcher, weight_dict, losses + criterion = SetCriterion( + num_classes, args.semantic_classes, matcher, weight_dict, losses, ignore_index=args.ignore_index + ) + criterion.to(device) + + return model, criterion diff --git a/plot_floor.py b/plot_floor.py new file mode 100644 index 0000000000000000000000000000000000000000..94b8cb54d5b2f010881a0c9d74a8e7b448bdc87a --- /dev/null +++ b/plot_floor.py @@ -0,0 +1,593 @@ +import argparse +import os +import random +from collections import defaultdict +from pathlib import Path + +import cv2 +import numpy as np +import plotly.graph_objects as go +import torch +from torch.utils.data import DataLoader + +from datasets import build_dataset +from datasets.data_utils import sort_polygons +from util.plot_utils import ( + CC5K_LABEL, + S3D_LABEL, + auto_crop_whitespace, + plot_room_map, + plot_semantic_rich_floorplan_opencv, + plot_semantic_rich_floorplan_tight, +) + + +def unnormalize_image(x): + mean = np.array([0.485, 0.456, 0.406]) + std = np.array([0.229, 0.224, 0.225]) + return x * std + mean + + +def plot_gt_floor( + args, + data_loader, + device, + output_dir, + plot_gt=True, + semantic_rich=False, + dataset_name="cubicasa", + crop_white_space=False, +): + if not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + semantics_label_mapping = None + if args.dataset_name == "stru3d": + door_window_index = [16, 17] + semantics_label_mapping = S3D_LABEL + elif args.dataset_name == "cubicasa": + door_window_index = [10, 9] + semantics_label_mapping = CC5K_LABEL + elif args.dataset_name == "waffle": + door_window_index = [1, 2] + else: + door_window_index = [] + + for batched_inputs, _ in data_loader: + samples = [x["image"].to(device) for x in batched_inputs] + scene_ids = [x["image_id"] for x in batched_inputs] + gt_instances = [x["instances"].to(device) for x in batched_inputs] + + # draw GT map + if plot_gt: + for i, gt_inst in enumerate(gt_instances): + image = np.transpose((samples[i] * 255).cpu().numpy(), [1, 2, 0]) + if not semantic_rich: + # plot regular room floorplan + gt_polys = [] + density_map = np.transpose((samples[i] * 255).cpu().numpy(), [1, 2, 0]) + density_map = np.repeat(density_map, 3, axis=2) + + for j, poly in enumerate(gt_inst.gt_masks.polygons): + corners = poly[0].reshape(-1, 2) + if len(corners) < 3: + continue + gt_polys.append(corners) + + gt_room_polys = [np.array(r) for r in gt_polys] + gt_polygons_labels = gt_inst.gt_classes.cpu().numpy() + + gt_sem_rich = [] + for poly, poly_type in zip(gt_room_polys, gt_polygons_labels): + gt_sem_rich.append([poly, poly_type]) + + if args.plot_engine == "opencv": + gt_sem_rich_path: str = os.path.join( + output_dir, "{}_floor.png".format(str(scene_ids[i]).zfill(5)) + ) + gt_floorplan_map = plot_semantic_rich_floorplan_opencv( + gt_sem_rich, + None, + door_window_index=door_window_index, + img_w=args.image_size * args.image_scale, + img_h=args.image_size * args.image_scale, + semantics_label_mapping=semantics_label_mapping, + plot_text=False, + scale=args.image_scale, + is_sem=True, + one_color=args.one_color, + is_bw=args.is_bw, + ) + if crop_white_space: + image = cv2.resize( + image, + (args.image_scale * args.image_size, args.image_scale * args.image_size), + interpolation=cv2.INTER_NEAREST, + ) + image, cropped_box = auto_crop_whitespace(image) + _x, _y, _w, _h = [ele for ele in cropped_box] + gt_floorplan_map = gt_floorplan_map[_y : _y + _h, _x : _x + _w].copy() + cv2.imwrite(gt_sem_rich_path, gt_floorplan_map, [cv2.IMWRITE_PNG_COMPRESSION, 0]) + else: + gt_sem_rich_path = os.path.join(output_dir, "{}.png".format(str(scene_ids[i]).zfill(5))) + plot_semantic_rich_floorplan_tight( + gt_sem_rich, + gt_sem_rich_path, + None, + None, + plot_text=False, + is_bw=args.is_bw, + door_window_index=door_window_index, + img_w=args.image_size * args.image_scale, + img_h=args.image_size * args.image_scale, + ) + else: + # plot semantically-rich floorplan + gt_polygons_labels = gt_inst.gt_classes.cpu().numpy() + gt_polygons = gt_inst.gt_masks.polygons + + gt_polygons, sorted_indices = sort_polygons(gt_polygons) + gt_polygons_labels = [gt_polygons_labels[_idx] for _idx in sorted_indices] + + gt_sem_rich = [] + for j, (poly, poly_label) in enumerate(zip(gt_polygons, gt_polygons_labels)): + # if gt_inst.gt_classes.cpu().numpy()[j] not in [1, 9, 11]: + # continue + corners = poly[0].reshape(-1, 2).astype(np.int32) + # corners_flip_y = corners.copy() + # corners_flip_y[:,1] = 255 - corners_flip_y[:,1] + # corners = corners_flip_y + gt_sem_rich.append([corners, poly_label]) + + if args.plot_engine == "opencv": + gt_sem_rich_path = os.path.join(output_dir, "{}_floor.png".format(str(scene_ids[i]).zfill(5))) + gt_floorplan_map = plot_semantic_rich_floorplan_opencv( + gt_sem_rich, + None, + door_window_index=door_window_index, + semantics_label_mapping=semantics_label_mapping, + scale=args.image_scale, + img_w=args.image_size * args.image_scale, + img_h=args.image_size * args.image_scale, + is_bw=args.is_bw, + plot_text=False, + one_color=args.one_color, + ) + + if crop_white_space: + image, cropped_box = auto_crop_whitespace(image) + _x, _y, _w, _h = [ele * args.image_scale for ele in cropped_box] + gt_floorplan_map = gt_floorplan_map[_y : _y + _h, _x : _x + _w].copy() + cv2.imwrite(gt_sem_rich_path, gt_floorplan_map, [cv2.IMWRITE_PNG_COMPRESSION, 0]) + else: + gt_sem_rich_path = os.path.join(output_dir, "{}.png".format(str(scene_ids[i]).zfill(5))) + plot_semantic_rich_floorplan_tight( + gt_sem_rich, + gt_sem_rich_path, + None, + None, + plot_text=False, + is_bw=args.is_bw, + door_window_index=door_window_index, + img_w=args.image_size * args.image_scale, + img_h=args.image_size * args.image_scale, + ) + + +def plot_polys(data_loader, device, output_dir): + if not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + for batched_inputs, _ in data_loader: + samples = [x["image"].to(device) for x in batched_inputs] + scene_ids = [x["image_id"] for x in batched_inputs] + gt_instances = [x["instances"].to(device) for x in batched_inputs] + + for i in range(len(samples)): + density_map = np.transpose((samples[i]).cpu().numpy(), [1, 2, 0]) + if density_map.shape[2] == 3: + density_map = density_map * 255 + else: + density_map = np.repeat(density_map, 3, axis=2) * 255 + pred_room_map = np.zeros(density_map.shape).astype(np.uint8) + + room_polys = gt_instances[i].gt_masks.polygons + room_ids = gt_instances[i].gt_classes.detach().cpu().numpy() + for poly, poly_id in zip(room_polys, room_ids): + poly = poly[0].reshape(-1, 2).astype(np.int32) + pred_room_map = plot_room_map(poly, pred_room_map, poly_id) + + # Blend the overlay with the density map using alpha blending + alpha = 0.6 # Adjust for desired transparency + pred_room_map = cv2.addWeighted( + density_map.astype(np.uint8), alpha, pred_room_map.astype(np.uint8), 1 - alpha, 0 + ) + + # # plot predicted polygon overlaid on the density map + # pred_room_map = np.clip(pred_room_map + density_map, 0, 255) + cv2.imwrite(os.path.join(output_dir, "{}_pred_room_map.png".format(scene_ids[i])), pred_room_map) + + +def plot_gt_image(data_loader, device, output_dir, crop_white_space=False): + if not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + for batched_inputs, _ in data_loader: + samples = [x["image"].to(device) for x in batched_inputs] + scene_ids = [x["image_id"] for x in batched_inputs] + + for i in range(len(samples)): + density_map = np.transpose((samples[i]).cpu().numpy(), [1, 2, 0]) + if density_map.shape[2] == 3: + density_map = density_map * 255 + else: + density_map = np.repeat(density_map, 3, axis=2) * 255 + + if crop_white_space: + density_map = cv2.resize( + density_map, + (args.image_scale * args.image_size, args.image_scale * args.image_size), + interpolation=cv2.INTER_NEAREST, + ) + density_map, _ = auto_crop_whitespace(image=density_map, color_invert=True) + + cv2.imwrite(os.path.join(output_dir, "{}_gt_image.png".format(scene_ids[i])), density_map) + + +def plot_histogram(count_dict, title, output_path, bin_size=10): + # Group keys into bins based on the bin_size + binned_count_dict = {} + for key, value in count_dict.items(): + bin_key = (key // bin_size) * bin_size # Determine the bin for the key + binned_count_dict[bin_key] = binned_count_dict.get(bin_key, 0) + value + + # Sort the bins + binned_keys = sorted(binned_count_dict.keys()) + binned_values = [binned_count_dict[key] for key in binned_keys] + + # Determine the maximum value for the y-axis + max_y = max(binned_values) + # Adjust y-axis ticks dynamically for large ranges + tick_interval = max(1, max_y // 10) # Divide the range into 10 intervals + tickvals_y = list(range(0, max_y + tick_interval, tick_interval)) + + # Determine tick values for x-axis dynamically + tickvals_x = binned_keys # Use the binned keys as tick values + + fig = go.Figure( + data=[ + go.Bar( + x=binned_keys, + y=binned_values, + text=binned_values, + textposition="outside", + marker=dict(color="blue"), + width=0.5, + ) + ] + ) + + fig.update_layout( + title={ + "text": f"Histogram of {title}", + "font": {"size": 30}, # Increase title font size + "x": 0.5, # Center the title + }, + xaxis_title={"text": f"Number of {title}", "font": {"size": 24}}, # Increase x-axis label font size + yaxis_title={"text": "Frequency", "font": {"size": 24}}, # Increase y-axis label font size + xaxis=dict( + tickmode="array", # Use custom tick values + tickvals=tickvals_x, + ticktext=[f"{x}-{x + bin_size - 1}" for x in binned_keys], + tickfont=dict(size=20), # Increase x-axis tick font size + ), + yaxis=dict( + tickvals=tickvals_y, # Set custom tick values + ticktext=[str(val) for val in tickvals_y], # Set custom tick labels + tickfont=dict(size=20), # Increase y-axis tick font size + ), + template="plotly_white", + # bargap=0.5, # Add gap between bars (0.5 = 50% of bar width) + # Increase figure width for a long x-axis + width=max(600, 30 * len(binned_keys)), # Dynamic width based on number of bars + ) + # Save the figure as an image + fig.write_image(output_path, scale=3) + print(f"Figure saved to {output_path}") + + # fig.show() + + +def loop_data(data_loader, eval_set, device, output_dir): + max_num_points = -1 + max_num_polys = -1 + count_pts_dict = defaultdict(lambda: 0) + count_room_dict = defaultdict(lambda: 0) + count_length_dict = defaultdict(lambda: 0) + for batched_inputs, batched_extras in data_loader: + samples = [x["image"].to(device) for x in batched_inputs] + gt_instances = [x["instances"].to(device) for x in batched_inputs] + for i in range(len(samples)): + if batched_extras is not None: + t = (batched_extras["token_labels"][i] == 0).sum().item() + count_length_dict[t] += 1 + room_polys = gt_instances[i].gt_masks.polygons + room_ids = gt_instances[i].gt_classes.detach().cpu().numpy() + count_room_dict[len(room_ids)] += 1 + for poly, poly_id in zip(room_polys, room_ids): + poly = poly[0].reshape(-1, 2).astype(np.int32) + count_pts_dict[len(poly)] += 1 + if len(poly) > max_num_points: + max_num_points = len(poly) + if len(room_ids) > max_num_polys: + max_num_polys = len(room_ids) + + print(f"Max pts: {max_num_points}, Max polys: {max_num_polys}") + + plot_histogram( + count_pts_dict, "Points in Polygons", os.path.join(output_dir, f"{eval_set}_polygon_histogram.png"), bin_size=5 + ) + plot_histogram( + count_room_dict, + "Rooms in Floorplan image", + os.path.join(output_dir, f"{eval_set}_room_histogram.png"), + bin_size=5, + ) + plot_histogram( + count_length_dict, + "Corners in Floorplan image", + os.path.join(output_dir, f"{eval_set}_seqlen_histogram.png"), + bin_size=30, + ) + + +def get_args_parser(): + parser = argparse.ArgumentParser("Raster2Seq plotting script", add_help=False) + parser.add_argument("--batch_size", default=10, type=int) + + parser.add_argument("--debug", action="store_true") + parser.add_argument("--image_size", type=int, default=256) + parser.add_argument("--wd_only", action="store_true") + parser.add_argument("--drop_wd", action="store_true", help="disable Windor & Door in the plots") + parser.add_argument( + "--crop_white_space", action="store_true", help="remove redundant whitespace from the rendering" + ) + parser.add_argument("--image_scale", type=int, default=1, help="adjust rendering resolution of the plots") + parser.add_argument("--one_color", action="store_true", help="use single color for every room (i.e. yellow)") + parser.add_argument("--is_bw", action="store_true", help="plot floorplan as binary image") + parser.add_argument("--plot_engine", type=str, default="opencv") + parser.add_argument( + "--compute_stats", + action="store_true", + help="compute statistics of the dataset (e.g. max_num_pts, max_num_polys) " + "and plot histogram for counting number of Points, Rooms, Corners", + ) + # poly2seq + parser.add_argument("--poly2seq", action="store_true") + parser.add_argument("--seq_len", type=int, default=1024) + parser.add_argument("--num_bins", type=int, default=64) + parser.add_argument("--add_cls_token", action="store_true") + parser.add_argument("--per_token_sem_loss", action="store_true") + + # backbone + parser.add_argument("--input_channels", default=1, type=int) + parser.add_argument("--backbone", default="resnet50", type=str, help="Name of the convolutional backbone to use") + parser.add_argument("--lr_backbone", default=0, type=float) + parser.add_argument( + "--dilation", + action="store_true", + help="If true, we replace stride with dilation in the last convolutional block (DC5)", + ) + parser.add_argument( + "--position_embedding", + default="sine", + type=str, + choices=("sine", "learned"), + help="Type of positional embedding to use on top of the image features", + ) + parser.add_argument("--position_embedding_scale", default=2 * np.pi, type=float, help="position / size * scale") + parser.add_argument("--num_feature_levels", default=4, type=int, help="number of feature levels") + + parser.add_argument("--image_norm", action="store_true") + parser.add_argument("--disable_image_transform", action="store_true") + + # Transformer + parser.add_argument("--enc_layers", default=6, type=int, help="Number of encoding layers in the transformer") + parser.add_argument("--dec_layers", default=6, type=int, help="Number of decoding layers in the transformer") + parser.add_argument( + "--dim_feedforward", + default=1024, + type=int, + help="Intermediate size of the feedforward layers in the transformer blocks", + ) + parser.add_argument( + "--hidden_dim", default=256, type=int, help="Size of the embeddings (dimension of the transformer)" + ) + parser.add_argument("--dropout", default=0.1, type=float, help="Dropout applied in the transformer") + parser.add_argument( + "--nheads", default=8, type=int, help="Number of attention heads inside the transformer's attentions" + ) + parser.add_argument( + "--num_queries", + default=800, + type=int, + help="Number of query slots (num_polys * max. number of corner per poly)", + ) + parser.add_argument("--num_polys", default=20, type=int, help="Number of maximum number of room polygons") + parser.add_argument("--dec_n_points", default=4, type=int) + parser.add_argument("--enc_n_points", default=4, type=int) + + parser.add_argument( + "--query_pos_type", + default="sine", + type=str, + choices=("static", "sine", "none"), + help="Type of query pos in decoder - \ + 1. static: same setting with DETR and Deformable-DETR, the query_pos is the same for all layers \ + 2. sine: since embedding from reference points (so if references points update, query_pos also \ + 3. none: remove query_pos", + ) + parser.add_argument( + "--with_poly_refine", + default=True, + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + parser.add_argument( + "--masked_attn", + default=False, + action="store_true", + help="if true, the query in one room will not be allowed to attend other room", + ) + parser.add_argument( + "--semantic_classes", + default=-1, + type=int, + help="Number of classes for semantically-rich floorplan: \ + 1. default -1 means non-semantic floorplan \ + 2. 19 for Structured3D: 16 room types + 1 door + 1 window + 1 empty", + ) + parser.add_argument( + "--use_room_attn_at_last_dec_layer", + default=False, + action="store_true", + help="use room-wise attention in last decoder layer", + ) + + # aux + parser.add_argument( + "--no_aux_loss", + dest="aux_loss", + action="store_true", + help="Disables auxiliary decoding losses (loss at each layer)", + ) + + # dataset parameters + parser.add_argument("--dataset_name", default="stru3d") + parser.add_argument("--dataset_root", default="data/stru3d", type=str) + parser.add_argument("--eval_set", default="test", type=str) + + parser.add_argument("--device", default="cuda", help="device to use for training / testing") + parser.add_argument("--num_workers", default=2, type=int) + parser.add_argument("--seed", default=42, type=int) + parser.add_argument("--checkpoint", default="checkpoints/roomformer_scenecad.pth", help="resume from checkpoint") + parser.add_argument("--output_dir", default="eval_stru3d", help="path where to save result") + + # visualization options + parser.add_argument( + "--plot_density", + default=False, + action="store_true", + help="plot predicited room polygons overlaid on the density map", + ) + parser.add_argument("--plot_gt", default=False, action="store_true", help="plot ground truth floorplan") + parser.add_argument("--plot_gt_image", default=False, action="store_true", help="plot ground truth image") + + return parser + + +def main(args): + + device = "cpu" # torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + # build dataset and dataloader + dataset_eval = build_dataset(image_set=args.eval_set, args=args) + # for test + if args.debug: + dataset_eval = torch.utils.data.Subset(dataset_eval, [7]) # list(range(0, args.batch_size, 1)) + sampler_eval = torch.utils.data.SequentialSampler(dataset_eval) + + def trivial_batch_collator(batch): + """ + A batch collator that does nothing. + """ + if "target_seq" in batch[0]: + # Concatenate tensors for each key in the batch + delta_x1 = torch.stack([item["delta_x1"] for item in batch], dim=0) + delta_x2 = torch.stack([item["delta_x2"] for item in batch], dim=0) + delta_y1 = torch.stack([item["delta_y1"] for item in batch], dim=0) + delta_y2 = torch.stack([item["delta_y2"] for item in batch], dim=0) + seq11 = torch.stack([item["seq11"] for item in batch], dim=0) + seq21 = torch.stack([item["seq21"] for item in batch], dim=0) + seq12 = torch.stack([item["seq12"] for item in batch], dim=0) + seq22 = torch.stack([item["seq22"] for item in batch], dim=0) + target_seq = torch.stack([item["target_seq"] for item in batch], dim=0) + token_labels = torch.stack([item["token_labels"] for item in batch], dim=0) + mask = torch.stack([item["mask"] for item in batch], dim=0) + + # Delete the keys from the batch + for item in batch: + del item["delta_x1"] + del item["delta_x2"] + del item["delta_y1"] + del item["delta_y2"] + del item["seq11"] + del item["seq21"] + del item["seq12"] + del item["seq22"] + del item["target_seq"] + del item["token_labels"] + del item["mask"] + + # Return the concatenated batch + return batch, { + "delta_x1": delta_x1, + "delta_x2": delta_x2, + "delta_y1": delta_y1, + "delta_y2": delta_y2, + "seq11": seq11, + "seq21": seq21, + "seq12": seq12, + "seq22": seq22, + "target_seq": target_seq, + "token_labels": token_labels, + "mask": mask, + } + + return batch, None + + data_loader_eval = DataLoader( + dataset_eval, + args.batch_size, + sampler=sampler_eval, + drop_last=False, + collate_fn=trivial_batch_collator, + num_workers=args.num_workers, + pin_memory=True, + ) + output_dir = Path(args.output_dir) + + save_dir = output_dir # os.path.join(os.path.dirname(args.checkpoint), output_dir) + os.makedirs(save_dir, exist_ok=True) + + if args.plot_gt: + plot_gt_floor( + args, + data_loader_eval, + device, + save_dir, + plot_gt=args.plot_gt, + semantic_rich=args.semantic_classes > 0, + crop_white_space=args.crop_white_space, + ) + + if args.plot_density: + plot_polys(data_loader_eval, device, save_dir) + + if args.plot_gt_image: + plot_gt_image(data_loader_eval, device, save_dir, crop_white_space=args.crop_white_space) + + if args.compute_stats: + loop_data(data_loader_eval, args.eval_set, device, save_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Raster2Seq plotting script", parents=[get_args_parser()]) + args = parser.parse_args() + + main(args) diff --git a/predict.py b/predict.py new file mode 100644 index 0000000000000000000000000000000000000000..a422b80363da48c48596e15e8483754f826bbeee --- /dev/null +++ b/predict.py @@ -0,0 +1,441 @@ +import argparse +import copy +import json +import os +import random + +import cv2 +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from tqdm import tqdm, trange + +from datasets.discrete_tokenizer import DiscreteTokenizer +from datasets.transforms import ResizeAndPad +from detectron2.data import transforms as T +from engine import generate, plot_density_map +from models import build_model +from util.plot_utils import CC5K_LABEL, S3D_LABEL, auto_crop_whitespace, plot_semantic_rich_floorplan_opencv + + +class ImageDataset(Dataset): + def __init__(self, image_paths, num_image_channels=3, transform=None): + """ + Args: + image_paths (list): List of image file paths. + transform (callable, optional): Optional transform to be applied on an image. + """ + self.image_paths = image_paths + self.transform = transform + self.num_image_channels = num_image_channels + + def __len__(self): + return len(self.image_paths) + + def _expand_image_dims(self, x): + if len(x.shape) == 2: + exp_img = np.expand_dims(x, 0) + else: + exp_img = x.transpose((2, 0, 1)) # (h,w,c) -> (c,h,w) + return exp_img + + def __getitem__(self, idx): + """ + Args: + idx (int): Index of the image to fetch. + + Returns: + torch.Tensor: Transformed image tensor. + """ + img_path = self.image_paths[idx] + if self.num_image_channels == 3: + image = np.array(Image.open(img_path).convert("RGB")) # Ensure 3-channel RGB + else: + image = np.array(Image.open(img_path)) # Ensure 1-channel RGB + if self.transform: + aug_input = T.AugInput(image) + _ = self.transform(aug_input) + image = aug_input.image + + image = (1 / 255) * torch.as_tensor(np.array(self._expand_image_dims(image))) + return { + "file_name": img_path, + "image": image, + } + + +def get_args_parser(): + parser = argparse.ArgumentParser("Raster2Seq prediction script", add_help=False) + parser.add_argument("--batch_size", default=10, type=int) + + parser.add_argument("--debug", action="store_true") + parser.add_argument("--input_channels", default=1, type=int) + parser.add_argument("--image_norm", action="store_true") + parser.add_argument("--eval_every_epoch", type=int, default=20) + parser.add_argument("--ckpt_every_epoch", type=int, default=20) + parser.add_argument("--label_smoothing", type=float, default=0.0) + parser.add_argument("--ignore_index", type=int, default=-1) + parser.add_argument("--image_size", type=int, default=256) + parser.add_argument("--ema4eval", action="store_true") + parser.add_argument("--measure_time", action="store_true") + parser.add_argument("--disable_sampling_cache", action="store_true") + parser.add_argument("--use_anchor", action="store_true") + parser.add_argument("--drop_wd", action="store_true") + parser.add_argument("--plot_text", action="store_true") + parser.add_argument("--image_scale", type=int, default=2) + parser.add_argument("--one_color", action="store_true") + parser.add_argument("--crop_white_space", action="store_true") + + # raster2seq + parser.add_argument("--poly2seq", action="store_true") + parser.add_argument("--seq_len", type=int, default=1024) + parser.add_argument("--num_bins", type=int, default=64) + parser.add_argument("--pre_decoder_pos_embed", action="store_true") + parser.add_argument("--learnable_dec_pe", action="store_true") + parser.add_argument("--dec_qkv_proj", action="store_true") + parser.add_argument("--dec_attn_concat_src", action="store_true") + parser.add_argument("--per_token_sem_loss", action="store_true") + parser.add_argument("--add_cls_token", action="store_true") + + # backbone + parser.add_argument("--backbone", default="resnet50", type=str, help="Name of the convolutional backbone to use") + parser.add_argument("--lr_backbone", default=0, type=float) + parser.add_argument( + "--dilation", + action="store_true", + help="If true, we replace stride with dilation in the last convolutional block (DC5)", + ) + parser.add_argument( + "--position_embedding", + default="sine", + type=str, + choices=("sine", "learned"), + help="Type of positional embedding to use on top of the image features", + ) + parser.add_argument("--position_embedding_scale", default=2 * np.pi, type=float, help="position / size * scale") + parser.add_argument("--num_feature_levels", default=4, type=int, help="number of feature levels") + + # Transformer + parser.add_argument("--enc_layers", default=6, type=int, help="Number of encoding layers in the transformer") + parser.add_argument("--dec_layers", default=6, type=int, help="Number of decoding layers in the transformer") + parser.add_argument( + "--dim_feedforward", + default=1024, + type=int, + help="Intermediate size of the feedforward layers in the transformer blocks", + ) + parser.add_argument( + "--hidden_dim", default=256, type=int, help="Size of the embeddings (dimension of the transformer)" + ) + parser.add_argument("--dropout", default=0.1, type=float, help="Dropout applied in the transformer") + parser.add_argument( + "--nheads", default=8, type=int, help="Number of attention heads inside the transformer's attentions" + ) + parser.add_argument( + "--num_queries", + default=800, + type=int, + help="Number of query slots (num_polys * max. number of corner per poly)", + ) + parser.add_argument("--num_polys", default=20, type=int, help="Number of maximum number of room polygons") + parser.add_argument("--dec_n_points", default=4, type=int) + parser.add_argument("--enc_n_points", default=4, type=int) + + parser.add_argument( + "--query_pos_type", + default="sine", + type=str, + choices=("static", "sine", "none"), + help="Type of query pos in decoder - \ + 1. static: same setting with DETR and Deformable-DETR, the query_pos is the same for all layers \ + 2. sine: since embedding from reference points (so if references points update, query_pos also \ + 3. none: remove query_pos", + ) + parser.add_argument( + "--with_poly_refine", + default=True, + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + parser.add_argument( + "--masked_attn", + default=False, + action="store_true", + help="if true, the query in one room will not be allowed to attend other room", + ) + parser.add_argument( + "--semantic_classes", + default=-1, + type=int, + help="Number of classes for semantically-rich floorplan: \ + 1. default -1 means non-semantic floorplan \ + 2. 19 for Structured3D: 16 room types + 1 door + 1 window + 1 empty", + ) + parser.add_argument( + "--disable_poly_refine", + action="store_true", + help="iteratively refine reference points (i.e. positional part of polygon queries)", + ) + + # aux + parser.add_argument( + "--no_aux_loss", + dest="aux_loss", + action="store_true", + help="Disables auxiliary decoding losses (loss at each layer)", + ) + + # dataset parameters + parser.add_argument("--dataset_name", default="stru3d") + parser.add_argument("--dataset_root", default="data/stru3d", type=str) + parser.add_argument("--eval_set", default="test", type=str) + + parser.add_argument("--device", default="cuda", help="device to use for training / testing") + parser.add_argument("--num_workers", default=2, type=int) + parser.add_argument("--seed", default=42, type=int) + parser.add_argument("--checkpoint", default="checkpoints/roomformer_scenecad.pth", help="resume from checkpoint") + parser.add_argument("--output_dir", default="eval_stru3d", help="path where to save result") + + # visualization options + parser.add_argument("--plot_pred", default=True, type=bool, help="plot predicted floorplan") + parser.add_argument( + "--plot_density", default=True, type=bool, help="plot predicited room polygons overlaid on the density map" + ) + parser.add_argument("--plot_gt", default=False, type=bool, help="plot ground truth floorplan") + parser.add_argument("--save_pred", action="store_true", help="save_pred") + + return parser + + +def get_image_paths_from_directory(directory_path): + """ + Load all images from the specified directory. + + Args: + directory_path (str): Path to the directory containing images. + + Returns: + list: A list of PIL Image objects. + """ + paths = [] + valid_extensions = (".jpg", ".jpeg", ".png", ".bmp", ".tiff") # Add more extensions if needed + + # Iterate through all files in the directory + for root, _, files in os.walk(directory_path): + for filename in files: + if filename.lower().endswith(valid_extensions): # Check for valid image extensions + file_path = os.path.join(root, filename) + paths.append(file_path) + + return paths + + +def main(args): + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + image_paths = get_image_paths_from_directory(args.dataset_root) + data_transform = T.AugmentationList( + [ + ResizeAndPad((args.image_size, args.image_size), pad_value=255), + ] + ) + dataset_eval = ImageDataset(image_paths, num_image_channels=args.input_channels, transform=data_transform) + + tokenizer = None + if args.poly2seq: + tokenizer = DiscreteTokenizer(args.num_bins, args.seq_len, add_cls=args.add_cls_token) + args.vocab_size = len(tokenizer) + + # overfit one sample + if args.debug: + idx = 0 + for i, x in enumerate(dataset_eval): + if "3252" in x["file_name"]: + idx = i + dataset_eval = torch.utils.data.Subset(dataset_eval, [idx]) + + sampler_eval = torch.utils.data.SequentialSampler(dataset_eval) + data_loader_eval = DataLoader( + dataset_eval, + args.batch_size, + sampler=sampler_eval, + drop_last=False, + num_workers=args.num_workers, + pin_memory=True, + ) + + # build model + model = build_model(args, train=False, tokenizer=tokenizer) + model.to(device) + + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print("number of params:", n_parameters) + + checkpoint = torch.load(args.checkpoint, map_location="cpu") + if args.ema4eval: + ckpt_state_dict = copy.deepcopy(checkpoint["ema"]) + else: + ckpt_state_dict = copy.deepcopy(checkpoint["model"]) + for key, value in checkpoint["model"].items(): + if key.startswith("module."): + ckpt_state_dict[key[7:]] = checkpoint["model"][key] + del ckpt_state_dict[key] + missing_keys, unexpected_keys = model.load_state_dict(ckpt_state_dict, strict=False) + unexpected_keys = [k for k in unexpected_keys if not (k.endswith("total_params") or k.endswith("total_ops"))] + if len(missing_keys) > 0: + print("Missing Keys: {}".format(missing_keys)) + if len(unexpected_keys) > 0: + print("Unexpected Keys: {}".format(unexpected_keys)) + + # disable grad + for param in model.parameters(): + param.requires_grad = False + + save_dir = os.path.join(args.output_dir, os.path.dirname(args.checkpoint).split("/")[-1]) + os.makedirs(save_dir, exist_ok=True) + + semantics_label_mapping = None + if args.dataset_name == "stru3d": + door_window_index = [16, 17] + semantics_label_mapping = S3D_LABEL + elif args.dataset_name == "cubicasa": + door_window_index = [10, 9] + semantics_label_mapping = CC5K_LABEL + elif args.dataset_name == "waffle": + door_window_index = [1, 2] + else: + door_window_index = [] + + if args.measure_time: + images = torch.rand(args.batch_size, 3, args.image_size, args.image_size).to(device) + if args.poly2seq: + model = torch.compile(model) # compile model is not compatible with RoomFormer + # GPU-WARM-UP + for _ in trange(10, desc="GPU-WARM-UP"): + if not args.poly2seq: + _ = model(images) + else: + _ = model.forward_inference(images) + + # INIT LOGGERS + starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + + total_time = 0.0 + for batch_images in tqdm(data_loader_eval): + starter.record() + x = batch_images["image"].to(device) + filenames = batch_images["file_name"] + outputs = generate( + model, + x, + semantic_rich=args.semantic_classes > 0, + use_cache=True, + per_token_sem_loss=args.per_token_sem_loss, + drop_wd=args.drop_wd, + poly2seq=args.poly2seq, + ) + ender.record() + torch.cuda.synchronize() + total_time += starter.elapsed_time(ender) / len(data_loader_eval) + + pred_rooms = outputs["room"] + pred_labels = outputs["labels"] + + image_size = x.shape[-2] + for j, (pred_rm, pred_cls) in enumerate(zip(pred_rooms, pred_labels)): + if pred_cls is None: + pred_cls = [-1] * len(pred_rm) + fn = os.path.basename(filenames[j]).split(".")[0] + pred_room_map = plot_density_map( + x[j], + image_size, + pred_rm, + pred_cls, + plot_text=args.plot_text, + ) + + floorplan_map = plot_semantic_rich_floorplan_opencv( + zip(pred_rm, pred_cls), + None, + door_window_index=door_window_index, + semantics_label_mapping=semantics_label_mapping, + plot_text=args.plot_text, + one_color=args.one_color, + is_sem=args.semantic_classes > 0, + img_w=image_size * args.image_scale, + img_h=image_size * args.image_scale, + scale=args.image_scale, + ) + + image = x[j].permute(1, 2, 0).cpu().numpy() * 255 + if args.crop_white_space: + image = cv2.resize( + image, + (args.image_scale * args.image_size, args.image_scale * args.image_size), + interpolation=cv2.INTER_NEAREST, + ) + image, cropped_box = auto_crop_whitespace(image) + _x, _y, _w, _h = [ele for ele in cropped_box] + + floorplan_map = floorplan_map[_y : _y + _h, _x : _x + _w].copy() + + # Ensure images are not empty before saving + if pred_room_map is not None and pred_room_map.size > 0: + cv2.imwrite(os.path.join(save_dir, "{}_pred_room_map.png".format(fn)), pred_room_map) + else: + print("Warning: pred_room_map is empty, skipping save.") + + if floorplan_map is not None and floorplan_map.size > 0: + cv2.imwrite(os.path.join(save_dir, "{}_pred_floorplan.png".format(fn)), floorplan_map) + else: + print("Warning: floorplan_map is empty, skipping save.") + + if image is not None and image.size > 0: + cv2.imwrite(os.path.join(save_dir, "{}.png".format(fn)), image) + else: + print("Warning: image is empty, skipping save.") + + if args.save_pred: + # Save room_polys as JSON + json_path = os.path.join(save_dir, "jsons", "{}.json".format(fn)) + npy_path = os.path.join(save_dir, "npy", "{}.npy".format(fn)) + os.makedirs(os.path.dirname(json_path), exist_ok=True) + os.makedirs(os.path.dirname(npy_path), exist_ok=True) + polys_list = [poly.astype(float).tolist() for poly in pred_rm] + types_list = pred_cls + + output_json = [ + { + "image_id": fn, + "segmentation": polys_list[instance_id], + "category_id": int(types_list[instance_id]), + "id": instance_id, + } + for instance_id in range(len(polys_list)) + ] + with open(json_path, "w") as json_file: + json.dump(output_json, json_file) + + polys_list = [np.array(poly).reshape(-1, 2) for poly in polys_list] + np.save(npy_path, np.array(polys_list, dtype=object)) + + print(f"Total inference time: {total_time:.2f} ms") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser("Raster2Seq prediction script", parents=[get_args_parser()]) + args = parser.parse_args() + + if args.debug: + args.batch_size = 1 + if args.disable_poly_refine: + args.with_poly_refine = False + + main(args) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..d5e831aecaee598a29ebc096352f936d7875c96d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[tool.ruff] +line-length = 119 +target-version = "py38" +exclude = [ + ".git", + ".venv", + "venv", + "__pycache__", + "build", + "dist", + "*.egg-info", + "data", + "output", + "detectron2", + "vis_utils", + "rl", +] + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] +ignore = ["E203", "E501", "E741", "E731", "E712", "E721", "E402", "W605"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" + +[tool.ruff.lint.isort] +known-first-party = ["FIRSTPARTY"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..97816028c6bc3d442bc27a9dd2c04f904fb17ffc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +opencv-python +# numpy==1.21.0 +matplotlib==3.6.2 +imageio==2.19.3 +wandb==0.12.19 +scipy==1.8.1 +fvcore +cloudpickle==2.1.0 +omegaconf==2.2.2 +fairscale==0.4.6 +timm==0.5.4 +shapely==1.8.2 +tqdm==4.64.0 +pycocotools +descartes +webcolors +drawsvg +cairosvg +plyfile +scikit-image==0.19.0 +svgpathtools==1.2.4 +svgwrite==1.2.1 \ No newline at end of file diff --git a/tools/cross_eval_cc5k.sh b/tools/cross_eval_cc5k.sh new file mode 100644 index 0000000000000000000000000000000000000000..e3885dfe871ce78544ede14752a20b36027ed71a --- /dev/null +++ b/tools/cross_eval_cc5k.sh @@ -0,0 +1,26 @@ +DATA=data/coco_cubicasa5k_nowalls_v4-1_refined +SPLIT=test + +NAME=r2g_cc5k_${SPLIT}_preds +SAVE_DIR=cross_eval_outputs/${NAME} +CKPT=checkpoints/r2g_sem_res256_ep0549.pth + +python eval.py \ + --dataset_name=cubicasa \ + --dataset_root=${DATA}/ \ + --eval_set=${SPLIT} \ + --checkpoint=${CKPT} \ + --output_dir=${SAVE_DIR} \ + --semantic_classes=13 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ + --disable_sem_rich \ + --drop_wd \ No newline at end of file diff --git a/tools/cross_eval_r2g.sh b/tools/cross_eval_r2g.sh new file mode 100644 index 0000000000000000000000000000000000000000..2ef89f9e62ba5677f444822421dfd0f8ba7333f8 --- /dev/null +++ b/tools/cross_eval_r2g.sh @@ -0,0 +1,26 @@ +DATA=data/R2G_hr_dataset_processed_v1 +SPLIT=test + +NAME=cc5k_r2g_${SPLIT}_preds +SAVE_DIR=cross_eval_outputs/${NAME} +CKPT=checkpoints/cc5k_sem_res256_ep0499.pth + +python eval.py \ + --dataset_name=r2g \ + --dataset_root=${DATA}/ \ + --eval_set=${SPLIT} \ + --checkpoint=${CKPT} \ + --output_dir=${SAVE_DIR} \ + --semantic_classes=12 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ + --disable_sem_rich \ + --drop_wd \ No newline at end of file diff --git a/tools/cross_eval_waffle.sh b/tools/cross_eval_waffle.sh new file mode 100644 index 0000000000000000000000000000000000000000..fc7972704b4a0b0abf688237c5765d446833c099 --- /dev/null +++ b/tools/cross_eval_waffle.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +DATA=data/waffle_benchmark_processed/ +SPLIT=test + +##### Trained on CubiCasa5K dataset, test on WAFFLE +NAME=cc5k_waffle_${SPLIT}_preds +SAVE_DIR=cross_eval_outputs/${NAME} +CKPT=checkpoints/cc5k_sem_res256_ep0499.pth +python predict.py \ + --dataset_root=${DATA}/${SPLIT} \ + --checkpoint=${CKPT} \ + --output_dir=${SAVE_DIR} \ + --semantic_classes=12 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ + --drop_wd \ + --one_color + +python eval_seg.py evaluations/clipseg_eval/config.yaml 0 \ + ${SAVE_DIR}/checkpoints/jsons + +##### Trained on Raster2Graph dataset, test on WAFFLE +NAME=r2g_waffle_${SPLIT}_preds +SAVE_DIR=cross_eval_outputs/${NAME} +CKPT=checkpoints/r2g_sem_res256_ep0549.pth +python predict.py \ + --dataset_root=${DATA}/${SPLIT} \ + --checkpoint=${CKPT} \ + --output_dir=${SAVE_DIR} \ + --semantic_classes=13 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ + --drop_wd \ + --one_color + +python eval_seg.py evaluations/clipseg_eval/config.yaml 0 \ + ${SAVE_DIR}/checkpoints/jsons \ No newline at end of file diff --git a/tools/download_checkpoints.sh b/tools/download_checkpoints.sh new file mode 100644 index 0000000000000000000000000000000000000000..8d1d8b50992c9aa391395a7b32e91a67f3bff67b --- /dev/null +++ b/tools/download_checkpoints.sh @@ -0,0 +1,18 @@ +# Structured3D +gdown --fuzzy https://drive.google.com/file/d/1aC2vyR_2ct7DNR5nGL4o2bkGRpZr0qVE/view?usp=sharing -O checkpoints +# CubiCasa5K +gdown --fuzzy https://drive.google.com/file/d/1NcKzVSkfvHs97aE48-9bFBMlyz1Wfkea/view?usp=sharing -O checkpoints +# Raster2Graph +gdown --fuzzy https://drive.google.com/file/d/15xCgv0a8Na5QZEGptPvG0ZnKnpB3pXqk/view?usp=sharing -O checkpoints + +# Semantic Structured3D +gdown --fuzzy https://drive.google.com/file/d/1OsqFzusl7qH0H1Od3yY_SziBCjnW6FPo/view?usp=sharing -O checkpoints +# Semantic CubiCasa5K +gdown --fuzzy https://drive.google.com/file/d/1M32HlYwXw-4Q_uajSCvpbF31UFPzQVHP/view?usp=sharing -O checkpoints +# Semantic Raster2Graph +gdown --fuzzy https://drive.google.com/file/d/1ZI4hs1iKk2y84S2dpsDHb40wCOPOGrtj/view?usp=sharing -O checkpoints + + +# # Optional: Structured3D-DensityMap +# gdown --fuzzy https://drive.google.com/file/d/1NcKzVSkfvHs97aE48-9bFBMlyz1Wfkea/view?usp=sharing -O checkpoints +# gdown --fuzzy https://drive.google.com/file/d/1M32HlYwXw-4Q_uajSCvpbF31UFPzQVHP/view?usp=sharing -O checkpoints \ No newline at end of file diff --git a/tools/eval_cc5k_finetune.sh b/tools/eval_cc5k_finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f2faa7de2c2790b945d5d71e341117cd7ed80ff --- /dev/null +++ b/tools/eval_cc5k_finetune.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CKPT=checkpoints/cc5k_sem_res256_ep0499.pth +python eval.py --dataset_name=cubicasa \ + --dataset_root=data/coco_cubicasa5k_nowalls_v4-1_refined/ \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/cc5k_sem_results/ \ + --semantic_classes=12 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ No newline at end of file diff --git a/tools/eval_cc5k_pretrain.sh b/tools/eval_cc5k_pretrain.sh new file mode 100644 index 0000000000000000000000000000000000000000..0d93c6b84d79e4c155a17fe5532e798c330b150b --- /dev/null +++ b/tools/eval_cc5k_pretrain.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CKPT=checkpoints/cc5k_res256_ep0499.pth +python eval.py --dataset_name=cubicasa \ + --dataset_root=data/coco_cubicasa5k_nowalls_v4-1_refined/ \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/cc5k_results/ \ + --semantic_classes=-1 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --save_pred \ No newline at end of file diff --git a/tools/eval_r2g_finetune.sh b/tools/eval_r2g_finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..07d5d78289998362990392f55fdf490212dd6d36 --- /dev/null +++ b/tools/eval_r2g_finetune.sh @@ -0,0 +1,19 @@ +# !/usr/bin/env bash + +CKPT=checkpoints/r2g_sem_res256_ep0549.pth +python eval.py --dataset_name=r2g \ + --dataset_root=data/R2G_hr_dataset_processed_v1 \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/r2g_sem_results \ + --semantic_classes=13 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ No newline at end of file diff --git a/tools/eval_r2g_pretrain.sh b/tools/eval_r2g_pretrain.sh new file mode 100644 index 0000000000000000000000000000000000000000..06fe3dbd7799cfe50ed2d54b583e3cebbe2cc94e --- /dev/null +++ b/tools/eval_r2g_pretrain.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CKPT=checkpoints/r2g_res256_ep0849.pth +python eval.py --dataset_name=r2g \ + --dataset_root=data/R2G_hr_dataset_processed_v1 \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/r2g_results/ \ + --semantic_classes=-1 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --save_pred \ No newline at end of file diff --git a/tools/eval_r2g_res512_finetune.sh b/tools/eval_r2g_res512_finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..8b96139e94e88fc7cae9be83c85f214421800c21 --- /dev/null +++ b/tools/eval_r2g_res512_finetune.sh @@ -0,0 +1,20 @@ +# !/usr/bin/env bash + +CKPT=checkpoints/r2g_sem_res512_ep0749.pth +python eval.py --dataset_name=r2g \ + --dataset_root=data/R2G_hr_dataset_processed_v1 \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/r2g_res512_sem_results \ + --semantic_classes=13 \ + --input_channels 3 \ + --poly2seq \ + --image_size 512 \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ No newline at end of file diff --git a/tools/eval_s3d_density_finetune.sh b/tools/eval_s3d_density_finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..f8cb53553a88232e753ba72645d1775ab95a9bae --- /dev/null +++ b/tools/eval_s3d_density_finetune.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CKPT=checkpoints/s3dd_sem_res256_ep0699.pth +python eval.py --dataset_name=stru3d \ + --dataset_root=data/stru3d \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/s3dd_sem_results \ + --semantic_classes=19 \ + --input_channels 1 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --per_token_sem_loss \ + --save_pred \ No newline at end of file diff --git a/tools/eval_s3d_density_pretrain.sh b/tools/eval_s3d_density_pretrain.sh new file mode 100644 index 0000000000000000000000000000000000000000..c0947ab9bc7c32634077509357fcf1bc852ec8d7 --- /dev/null +++ b/tools/eval_s3d_density_pretrain.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CKPT=checkpoints/s3dd_res256_ep0499.pth +python eval.py --dataset_name=stru3d \ + --dataset_root=data/stru3d \ + --eval_set=test \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/s3dd_results/ \ + --semantic_classes=-1 \ + --input_channels=1 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --ema4eval \ + --use_anchor \ + --save_pred \ No newline at end of file diff --git a/tools/eval_s3d_finetune.sh b/tools/eval_s3d_finetune.sh new file mode 100644 index 0000000000000000000000000000000000000000..e476dfe67fa8437ef6fb9d90cb802c33494a216e --- /dev/null +++ b/tools/eval_s3d_finetune.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DATA=data/coco_s3d_bw +FOLDER=test + +CKPT=checkpoints/s3dbw_sem_res256_ep0449.pth +python eval.py --dataset_name=stru3d \ + --dataset_root=${DATA} \ + --eval_set=${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/s3dbw_sem_results \ + --semantic_classes=19 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --use_anchor \ + --per_token_sem_loss \ + --ema4eval \ + --save_pred \ No newline at end of file diff --git a/tools/eval_s3d_pretrain.sh b/tools/eval_s3d_pretrain.sh new file mode 100644 index 0000000000000000000000000000000000000000..8ee00b4e07379fc9e8592682e372b4bd371a58f3 --- /dev/null +++ b/tools/eval_s3d_pretrain.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DATA=data/coco_s3d_bw/ +FOLDER=test +CKPT=checkpoints/s3dbw_res256_ep1349.pth + +python eval.py \ + --dataset_name=stru3d \ + --dataset_root=${DATA} \ + --eval_set=${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=eval_outputs/s3dbw_results/ \ + --semantic_classes=-1 \ + --input_channels=3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --use_anchor \ + --ema4eval \ + --save_pred \ No newline at end of file diff --git a/tools/finetune_cc5k.sh b/tools/finetune_cc5k.sh new file mode 100644 index 0000000000000000000000000000000000000000..3a9c30519abbe89a691f0a94f5fa7253430724dd --- /dev/null +++ b/tools/finetune_cc5k.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=14156 +NUM_GPUS=1 + +CLS_COEFF=5 +COO_COEFF=20 +SEM_COEFF=1 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 + +DATA=data/coco_cubicasa5k_nowalls_v4-1_refined/ +JOB=cc5k_sem_res256 +PRETRAIN=checkpoints/cc5k_res256_ep0499.pth # or save_models/cc5k_res256/checkpoint0499.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online torchrun --nproc_per_node=1 --master_port=$MASTER_PORT main_ddp.py --dataset_name=cubicasa \ + --dataset_root=${DATA} \ + --semantic_classes=12 \ + --job_name=${JOB} \ + --batch_size 56 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch=50 \ + --eval_every_epoch=50 \ + --label_smoothing 0.1 \ + --epochs 500 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --room_cls_loss_coef ${SEM_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --jointly_train \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ No newline at end of file diff --git a/tools/finetune_r2g.sh b/tools/finetune_r2g.sh new file mode 100644 index 0000000000000000000000000000000000000000..db49121c2afb171a98306551705a324ded9b64c1 --- /dev/null +++ b/tools/finetune_r2g.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=25157 +NUM_GPUS=2 + +CLS_COEFF=5 +COO_COEFF=20 +SEM_COEFF=1 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 + +DATA=data/R2G_hr_dataset_processed_v1 +JOB=r2g_sem_res256 +PRETRAIN=checkpoints/r2g_res256_ep0849.pth # or saved_models/r2g_res256/checkpoint0849.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online torchrun --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=r2g \ + --dataset_root=${DATA} \ + --semantic_classes=13 \ + --job_name=${JOB} \ + --batch_size 56 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len $SEQ_LEN \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch=50 \ + --eval_every_epoch=50 \ + --label_smoothing 0.1 \ + --epochs 550 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --room_cls_loss_coef ${SEM_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --jointly_train \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ No newline at end of file diff --git a/tools/finetune_r2g_res512.sh b/tools/finetune_r2g_res512.sh new file mode 100644 index 0000000000000000000000000000000000000000..3e6b5303468e11309cbda3d839b70e0d57d276a0 --- /dev/null +++ b/tools/finetune_r2g_res512.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=25157 +NUM_GPUS=2 + +CLS_COEFF=5 +COO_COEFF=20 +SEM_COEFF=1 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 + +DATA=data/R2G_hr_dataset_processed_v1 +JOB=r2g_sem_res512 +PRETRAIN=checkpoints/r2g_res256_ep0849.pth # or saved_models/r2g_res256/checkpoint0849.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online torchrun --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=r2g \ + --dataset_root=${DATA} \ + --semantic_classes=13 \ + --job_name=${JOB} \ + --batch_size 32 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len $SEQ_LEN \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch=50 \ + --eval_every_epoch=50 \ + --label_smoothing 0.1 \ + --epochs 750 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --room_cls_loss_coef ${SEM_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --jointly_train \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ + --image_size 512 \ No newline at end of file diff --git a/tools/finetune_s3d.sh b/tools/finetune_s3d.sh new file mode 100644 index 0000000000000000000000000000000000000000..a4114ada362116abd5862e935ba2095fa592baee --- /dev/null +++ b/tools/finetune_s3d.sh @@ -0,0 +1,45 @@ +#!/bin/bash +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=13566 +NUM_GPUS=1 + +CLS_COEFF=1 +COO_COEFF=20 +SEM_COEFF=1 +SEQ_LEN=512 +CONVERTER=v3 + +JOB=s3dbw_sem_res256 +PRETRAIN=checkpoints/s3dbw_res256_ep1349.pth # or save_models/s3dbw_res256/checkpoint1349.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online python -m torch.distributed.run --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=stru3d \ + --dataset_root=data/coco_s3d_bw \ + --semantic_classes=19 \ + --job_name=${JOB} \ + --batch_size 32 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins 32 \ + --ckpt_every_epoch 50 \ + --eval_every_epoch 20 \ + --lr 2e-4 \ + --lr_backbone 2e-5 \ + --label_smoothing 0.1 \ + --epochs 450 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --room_cls_loss_coef ${SEM_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --jointly_train \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ No newline at end of file diff --git a/tools/finetune_s3d_density.sh b/tools/finetune_s3d_density.sh new file mode 100644 index 0000000000000000000000000000000000000000..ca9e6586aa2b383e591689047ea079368f3bfa19 --- /dev/null +++ b/tools/finetune_s3d_density.sh @@ -0,0 +1,45 @@ +#!/bin/bash +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=13563 +NUM_GPUS=1 + +SEM_COEFF=1 +CLS_COEFF=2 +COO_COEFF=20 +SEQ_LEN=512 +CONVERTER=v3 + +JOB=s3dd_sem_res256 +PRETRAIN=checkpoints/s3dd_res256_ep0499.pth # or save_models/s3dd_res256/checkpoint0499.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online python -m torch.distributed.run --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=stru3d \ + --dataset_root=data/stru3d \ + --semantic_classes=19 \ + --job_name=${JOB} \ + --batch_size 32 \ + --input_channels=1 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins 32 \ + --ckpt_every_epoch 50 \ + --eval_every_epoch 20 \ + --lr 2e-4 \ + --lr_backbone 2e-5 \ + --label_smoothing 0.0 \ + --epochs 700 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --room_cls_loss_coef ${SEM_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --jointly_train \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ No newline at end of file diff --git a/tools/predict_cc5k.sh b/tools/predict_cc5k.sh new file mode 100644 index 0000000000000000000000000000000000000000..7e4785147c4400d5d2a743860a5b0f1e4499f12a --- /dev/null +++ b/tools/predict_cc5k.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DATA=data/coco_cubicasa5k_nowalls_v4-1_refined/ +FOLDER=test +CKPT=checkpoints/cc5k_sem_res256_ep0499.pth + +python predict.py \ + --dataset_name=cubicasa \ + --dataset_root=${DATA}/${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=pred_outputs/cc5k_${FOLDER}_preds \ + --semantic_classes=12 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --use_anchor \ + --ema4eval \ + --save_pred \ No newline at end of file diff --git a/tools/predict_r2g.sh b/tools/predict_r2g.sh new file mode 100644 index 0000000000000000000000000000000000000000..56ec707409a51379660267607a87e790501e10bc --- /dev/null +++ b/tools/predict_r2g.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DATA=data/R2G_hr_dataset_processed_v1 +FOLDER=test +CKPT=checkpoints/r2g_sem_res256_ep0549.pth + +python predict.py \ + --dataset_name=r2g \ + --dataset_root=${DATA}/${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=pred_outputs/r2g_${FOLDER}_preds \ + --semantic_classes=13 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --use_anchor \ + --ema4eval \ + --save_pred \ No newline at end of file diff --git a/tools/predict_s3d.sh b/tools/predict_s3d.sh new file mode 100644 index 0000000000000000000000000000000000000000..d1d24190c338a1ce00dcffbb7c06279f784cba7a --- /dev/null +++ b/tools/predict_s3d.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +DATA=data/coco_s3d_bw/ +FOLDER=test +CKPT=checkpoints/s3dbw_sem_res256_ep0449.pth + +python predict.py \ + --dataset_name=stru3d \ + --dataset_root=${DATA}/${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=pred_outputs/s3d_${FOLDER}_preds \ + --semantic_classes=19 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --per_token_sem_loss \ + --use_anchor \ + --ema4eval \ + --save_pred \ No newline at end of file diff --git a/tools/predict_s3d_density.sh b/tools/predict_s3d_density.sh new file mode 100644 index 0000000000000000000000000000000000000000..2eb451feb67a578bba9d24ffb63f32ee24483d13 --- /dev/null +++ b/tools/predict_s3d_density.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +DATA=data/stru3d/ +FOLDER=test +CKPT=checkpoints/s3dd_sem_res256_ep0699.pth + +python predict.py \ + --dataset_name=stru3d \ + --dataset_root=${DATA}/${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=pred_outputs/s3dd_${FOLDER}_preds \ + --semantic_classes=19 \ + --input_channels 1 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --use_anchor \ + --ema4eval \ + --save_pred \ No newline at end of file diff --git a/tools/predict_waffle.sh b/tools/predict_waffle.sh new file mode 100644 index 0000000000000000000000000000000000000000..b0f068763f6dc9c737d273855b4d5a96e5206877 --- /dev/null +++ b/tools/predict_waffle.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +DATA=data/waffle/data/original_size_images/ +FOLDER=00000 +CKPT=checkpoints/cc5k_sem_res256_ep0499.pth + +python predict.py \ + --dataset_root=${DATA}/${FOLDER} \ + --checkpoint=${CKPT} \ + --output_dir=pred_outputs/waffle_raster${FOLDER}_preds \ + --semantic_classes=12 \ + --input_channels 3 \ + --poly2seq \ + --seq_len 512 \ + --num_bins 32 \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --use_anchor \ + --ema4eval \ + --per_token_sem_loss \ + --drop_wd \ + --save_pred \ + --one_color \ No newline at end of file diff --git a/tools/pretrain_cc5k.sh b/tools/pretrain_cc5k.sh new file mode 100644 index 0000000000000000000000000000000000000000..361c8630060d7144970a376e6d089bc9f83107a2 --- /dev/null +++ b/tools/pretrain_cc5k.sh @@ -0,0 +1,40 @@ +#!/bin/bash +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=14159 +NUM_GPUS=1 + +CLS_COEFF=5 +COO_COEFF=20 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 + +JOB=cc5k_res256 +PRETRAIN=checkpoints/s3dbw_res256_ep1349.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online torchrun --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=cubicasa \ + --dataset_root=data/coco_cubicasa5k_nowalls_v4-1_refined/ \ + --semantic_classes=-1 \ + --job_name=${JOB} \ + --batch_size 56 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch=50 \ + --eval_every_epoch=50 \ + --label_smoothing 0.1 \ + --epochs 500 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ No newline at end of file diff --git a/tools/pretrain_r2g.sh b/tools/pretrain_r2g.sh new file mode 100644 index 0000000000000000000000000000000000000000..8e0d1257b7f211370a5fb4b243e35ec010e6a46f --- /dev/null +++ b/tools/pretrain_r2g.sh @@ -0,0 +1,39 @@ +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=24259 +NUM_GPUS=2 + +CLS_COEFF=5 +COO_COEFF=20 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 + +JOB=r2g_res256 +PRETRAIN=checkpoints/s3dbw_res256_ep1349.pth +OUTPUT_DIR=save_models + +WANDB_MODE=online torchrun --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=r2g \ + --dataset_root=data/R2G_hr_dataset_processed_v1 \ + --semantic_classes=-1 \ + --job_name=${JOB} \ + --batch_size 64 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch=50 \ + --eval_every_epoch=50 \ + --label_smoothing 0.1 \ + --epochs 850 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --converter_version ${CONVERTER} \ + --use_anchor \ + --start_from_checkpoint ${PRETRAIN} \ No newline at end of file diff --git a/tools/pretrain_s3d.sh b/tools/pretrain_s3d.sh new file mode 100644 index 0000000000000000000000000000000000000000..71133d43bc80dc771d115a24a4eda0e40b5f9f15 --- /dev/null +++ b/tools/pretrain_s3d.sh @@ -0,0 +1,40 @@ +#!/bin/bash +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=13476 +NUM_GPUS=1 + +CLS_COEFF=1 +COO_COEFF=20 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 + +JOB=s3dbw_res256 +OUTPUT_DIR=save_models + +WANDB_MODE=online python -m torch.distributed.run --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=stru3d \ + --dataset_root=data/coco_s3d_bw \ + --semantic_classes=-1 \ + --job_name=${JOB} \ + --batch_size 32 \ + --input_channels=3 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch 50 \ + --eval_every_epoch 20 \ + --lr 2e-4 \ + --lr_backbone 2e-5 \ + --label_smoothing 0.0 \ + --epochs 1400 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --converter_version ${CONVERTER} \ + --use_anchor \ No newline at end of file diff --git a/tools/pretrain_s3d_density.sh b/tools/pretrain_s3d_density.sh new file mode 100644 index 0000000000000000000000000000000000000000..796669a45f306643b1d0c61d90920bf803c068d6 --- /dev/null +++ b/tools/pretrain_s3d_density.sh @@ -0,0 +1,39 @@ +#!/bin/bash +export NCCL_P2P_LEVEL=NVL + +MASTER_PORT=23472 +NUM_GPUS=1 + +CLS_COEFF=2 +COO_COEFF=20 +SEQ_LEN=512 +NUM_BINS=32 +CONVERTER=v3 +JOB=s3dd_res256 +OUTPUT_DIR=save_models + +WANDB_MODE=online python -m torch.distributed.run --nproc_per_node=${NUM_GPUS} --master_port=$MASTER_PORT main_ddp.py --dataset_name=stru3d \ + --dataset_root=data/stru3d \ + --semantic_classes=-1 \ + --job_name=${JOB} \ + --batch_size 32 \ + --input_channels=1 \ + --output_dir ${OUTPUT_DIR} \ + --poly2seq \ + --seq_len ${SEQ_LEN} \ + --num_bins ${NUM_BINS} \ + --ckpt_every_epoch 50 \ + --eval_every_epoch 20 \ + --lr 2e-4 \ + --lr_backbone 2e-5 \ + --label_smoothing 0.0 \ + --epochs 500 \ + --lr_drop '' \ + --cls_loss_coef ${CLS_COEFF} \ + --coords_loss_coef ${COO_COEFF} \ + --resume ${OUTPUT_DIR}/${JOB}/checkpoint.pth \ + --ema4eval \ + --disable_poly_refine \ + --dec_attn_concat_src \ + --converter_version ${CONVERTER} \ + --use_anchor \ No newline at end of file diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/util/bf_utils.py b/util/bf_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3c7644bc26fe3acf32139a85359c567ffebedd0 --- /dev/null +++ b/util/bf_utils.py @@ -0,0 +1,146 @@ +import copy +import itertools +import math + +import numpy as np +import torch +from torch import nn + +from detectron2.utils.registry import Registry + +# need an easier place to avoid circular dependencies. +POLY_LOSS_REGISTRY = Registry("POLY_LOSS") +POLY_LOSS_REGISTRY.__doc__ = """ +Registry for loss computations on predicted polygons. +""" + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +def clip_and_normalize_polygons(polys, inf_value=2.01): + min_x, _ = polys[:, :, 0].min(dim=-1) + min_y, _ = polys[:, :, 1].min(dim=-1) + + polys[torch.isinf(polys)] = -np.inf + max_x, _ = polys[:, :, 0].max(dim=-1) + max_y, _ = polys[:, :, 1].max(dim=-1) + + polys[torch.isinf(polys)] = inf_value + + min_xy = torch.stack((min_x, min_y), dim=-1) + max_xy = torch.stack((max_x, max_y), dim=-1) - min_xy + + polys = (polys - min_xy.unsqueeze(1)) / max_xy.unsqueeze(1) + + return polys + + +def pad_polygons(polys): + count = len(polys) + max_vertices = max([len(p) for p in polys]) + pad_count = [max_vertices - len(p) for p in polys] + + # add between the first and second vertices. + xs = [np.linspace(polys[i][0][0] + 0.00001, polys[i][1][0] - 0.00001, num=pad_count[i]) for i in range(count)] + ys = [np.linspace(polys[i][0][1] + 0.00001, polys[i][1][1] - 0.00001, num=pad_count[i]) for i in range(count)] + + xys = [np.stack((xs[i], ys[i]), axis=-1) for i in range(count)] + polys = [np.concatenate((polys[i][:1], xys[i], polys[i][1:])) for i in range(count)] + + return np.stack(polys) + + +def rasterize_instances(rasterizer, instances, shape, offset=0.0): + if shape[0] != shape[1]: + raise ValueError("expected square") + + device = instances[0].gt_boxes.device + all_polygons = clip_and_normalize_polygons( + torch.from_numpy( + pad_polygons( + list( + itertools.chain.from_iterable( + [[p[0].reshape(-1, 2) for p in inst.gt_masks.polygons] for inst in instances] + ) + ) + ) + ) + .float() + .to(device) + ) + + # to me it seems the offset would need to be in _pixel_ space? + return rasterizer(all_polygons * float(shape[1].item()) + offset, shape[1].item(), shape[0].item(), 1.0) + + +def get_union_box(p, box): + # compute the enclosing box. + all_points = torch.cat((p, box.view(-1, 2, 2)), dim=-2) + min_xy = torch.min(all_points, dim=-2)[0] + max_xy = torch.max(all_points, dim=-2)[0] + + return torch.cat((min_xy, max_xy), dim=-1) + + +def sample_ellipse_fast(x, y, r1, r2, count=32, dt=0.01): + batch_size, num_el = r1.shape + device = r1.device + num_integrals = int(round(2 * math.pi / dt)) + + thetas = dt * torch.arange(num_integrals, device=device).unsqueeze(0).unsqueeze(0).repeat(batch_size, num_el, 1) + thetas_c = torch.cumsum(thetas, dim=-1) + dpt = torch.sqrt((r1.unsqueeze(-1) * torch.sin(thetas_c)) ** 2 + (r2.unsqueeze(-1) * torch.cos(thetas_c)) ** 2) + circumference = dpt.sum(dim=-1) + + run = torch.cumsum( + torch.sqrt( + (r1.unsqueeze(-1) * torch.sin(thetas + dt)) ** 2 + (r2.unsqueeze(-1) * torch.cos(thetas + dt)) ** 2 + ), + dim=-1, + ) + sub = (count * run) / circumference.unsqueeze(-1) + + # OK, now find the smallest point >= 0..count-1 + counts = ( + torch.arange(count, device=device) + .unsqueeze(0) + .unsqueeze(0) + .unsqueeze(0) + .repeat(batch_size, num_el, num_integrals, 1) + ) + diff = sub.unsqueeze(dim=-1) - counts + diff[diff < 0] = 10000.0 + + idx = diff.argmin(dim=2) + thetas = torch.gather(thetas + dt, -1, idx) + + xy = torch.stack( + ( + x.unsqueeze(-1) + r1.unsqueeze(-1) * torch.cos(thetas), + y.unsqueeze(-1) + r2.unsqueeze(-1) * torch.sin(thetas), + ), + dim=-1, + ) + + return xy + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) diff --git a/util/eval_utils.py b/util/eval_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..05b0ed1768e27ec31c8e381423aa0cbeb0ea086d --- /dev/null +++ b/util/eval_utils.py @@ -0,0 +1,7 @@ +def compute_f1(quant_result_dict, metric_category): + for metric in metric_category: + prec = quant_result_dict[metric + "_prec"] + rec = quant_result_dict[metric + "_rec"] + f1 = 2 * prec * rec / (prec + rec + 1e-5) + quant_result_dict[metric + "_f1"] = f1 + return quant_result_dict \ No newline at end of file diff --git a/util/misc.py b/util/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..89a3b9e4b73a2b2054467f4a23124aa94c5848e7 --- /dev/null +++ b/util/misc.py @@ -0,0 +1,568 @@ +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ + +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" + +import datetime +import getpass +import os +import pickle +import subprocess +import time +from collections import OrderedDict, defaultdict, deque +from typing import List, Optional + +import torch +import torch.distributed as dist + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision +from torch import Tensor + +if float(torchvision.__version__.split(".")[1]) < 5: + import math + + from torchvision.ops.misc import _NewEmptyTensorOp + + def _check_size_scale_factor(dim, size, scale_factor): + # type: (int, Optional[List[int]], Optional[float]) -> None + if size is None and scale_factor is None: + raise ValueError("either size or scale_factor should be defined") + if size is not None and scale_factor is not None: + raise ValueError("only one of size or scale_factor should be defined") + if not (scale_factor is not None and len(scale_factor) != dim): + raise ValueError( + "scale_factor shape must match input shape. Input is {}D, scale_factor size is {}".format( + dim, len(scale_factor) + ) + ) + + def _output_size(dim, input, size, scale_factor): + # type: (int, Tensor, Optional[List[int]], Optional[float]) -> List[int] + assert dim == 2 + _check_size_scale_factor(dim, size, scale_factor) + if size is not None: + return size + # if dim is not 2 or scale_factor is iterable use _ntuple instead of concat + assert scale_factor is not None and isinstance(scale_factor, (int, float)) + scale_factors = [scale_factor, scale_factor] + # math.floor might return float in py2.7 + return [int(math.floor(input.size(i + 2) * scale_factors[i])) for i in range(dim)] + +elif float(torchvision.__version__.split(".")[1]) < 7: + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, avg=self.avg, global_avg=self.global_avg, max=self.max, value=self.value + ) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [header, "[{0" + space_fmt + "}/{1}]", "eta: {eta}", "{meters}", "time: {time}", "data: {data}"] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print("{} Total time: {} ({:.4f} s / it)".format(header, total_time_str, total_time / len(iterable))) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device, non_blocking=False): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device, non_blocking=non_blocking) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device, non_blocking=non_blocking) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def record_stream(self, *args, **kwargs): + self.tensors.record_stream(*args, **kwargs) + if self.mask is not None: + self.mask.record_stream(*args, **kwargs) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def get_local_size(): + if not is_dist_avail_and_initialized(): + return 1 + return int(os.environ["LOCAL_SIZE"]) + + +def get_local_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return int(os.environ["LOCAL_RANK"]) + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + args.dist_url = "env://" + os.environ["LOCAL_SIZE"] = str(torch.cuda.device_count()) + elif "SLURM_PROCID" in os.environ: + proc_id = int(os.environ["SLURM_PROCID"]) + ntasks = int(os.environ["SLURM_NTASKS"]) + node_list = os.environ["SLURM_NODELIST"] + num_gpus = torch.cuda.device_count() + addr = subprocess.getoutput("scontrol show hostname {} | head -n1".format(node_list)) + os.environ["MASTER_PORT"] = os.environ.get("MASTER_PORT", "29500") + os.environ["MASTER_ADDR"] = addr + os.environ["WORLD_SIZE"] = str(ntasks) + os.environ["RANK"] = str(proc_id) + os.environ["LOCAL_RANK"] = str(proc_id % num_gpus) + os.environ["LOCAL_SIZE"] = str(num_gpus) + args.dist_url = "env://" + args.world_size = ntasks + args.rank = proc_id + args.gpu = proc_id % num_gpus + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if float(torchvision.__version__[:3]) < 0.7: + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + if float(torchvision.__version__[:3]) < 0.5: + return _NewEmptyTensorOp.apply(input, output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) + + +def get_total_grad_norm(parameters, norm_type=2): + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + device = parameters[0].grad.device + total_norm = torch.norm( + torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), norm_type + ) + return total_norm + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def setup_wandb(): + """ + Set up weight and bias + """ + keys_folder = "wandb_keys" + if not os.path.exists(keys_folder): + os.mkdir(keys_folder) + + username = getpass.getuser() + print(username) + wandb_key_path = keys_folder + "/" + username + "_wandb.key" + if not os.path.exists(wandb_key_path): + wandb_key = input( + "[You need to firstly setup and login wandb] Please enter your wandb key (https://wandb.ai/authorize):" + ) + with open(wandb_key_path, "w") as fh: + fh.write(wandb_key) + else: + print("wandb key already set") + os.system('export WANDB_API_KEY=$(cat "' + wandb_key_path + '")') + + +@torch.no_grad() +def update_ema(ema_model, model, decay=0.9999): + """ + Step the EMA model towards the current model. + """ + ema_params = OrderedDict(ema_model.named_parameters()) + model_params = OrderedDict(model.named_parameters()) + + for name, param in model_params.items(): + # TODO: Consider applying only to params that require_grad to avoid small numerical changes of pos_embed + ema_params[name].mul_(decay).add_(param.data, alpha=1 - decay) + + +def requires_grad(model, flag=True): + """ + Set requires_grad flag for all parameters in a model. + """ + for p in model.parameters(): + p.requires_grad = flag diff --git a/util/plot_utils.py b/util/plot_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eebeaf4db639dc50382a263816b36e1b5b8e3227 --- /dev/null +++ b/util/plot_utils.py @@ -0,0 +1,1468 @@ +""" +Utilities for floorplan visualization. +""" + +import math + +import cv2 +import matplotlib.patches as mpatches +import matplotlib.pyplot as plt +import numpy as np +from descartes.patch import PolygonPatch +from imageio import imsave +from matplotlib.cm import get_cmap +from matplotlib.colors import to_hex +from PIL import ImageColor +from plotly.colors import qualitative +from shapely.geometry import LineString, Polygon + +colors_12 = [ + "#e6194b", + "#3cb44b", + "#ffe119", + "#0082c8", + "#f58230", + "#911eb4", + "#46f0f0", + "#f032e6", + "#d2f53c", + "#fabebe", + "#008080", + "#e6beff", + "#aa6e28", + "#fffac8", + "#800000", + "#aaffc3", + "#808000", + "#ffd7b4", +] + +semantics_cmap = { + 0: "#e6194b", + 1: "#3cb44b", + 2: "#ffe119", + 3: "#0082c8", + 4: "#f58230", + 5: "#911eb4", + 6: "#46f0f0", + 7: "#f032e6", + 8: "#d2f53c", + 9: "#fabebe", + 10: "#008080", + 11: "#e6beff", + 12: "#aa6e28", + 13: "#fffac8", + 14: "#800000", + 15: "#aaffc3", + 16: "#808000", + 17: "#ffd7b4", +} + +S3D_LABEL = { + 0: "Living Room", + 1: "Kitchen", + 2: "Bedroom", + 3: "Bathroom", + 4: "Balcony", + 5: "Corridor", + 6: "Dining room", + 7: "Study", + 8: "Studio", + 9: "Store room", + 10: "Garden", + 11: "Laundry room", + 12: "Office", + 13: "Basement", + 14: "Garage", + 15: "Misc.", + 16: "Door", + 17: "Window", +} + +CC5K_LABEL = { + 0: "Outdoor", + 1: "Kitchen", + 2: "Living Room", + 3: "Bed Room", + 4: "Bath", + 5: "Entry", + 6: "Storage", + 7: "Garage", + 8: "Undefined", + 9: "Window", + 10: "Door", +} + +R2G_LABEL = { + 0: "unknown", + 1: "living_room", + 2: "kitchen", + 3: "bedroom", + 4: "bathroom", + 5: "restroom", + 6: "balcony", + 7: "closet", + 8: "corridor", + 9: "washing_room", + 10: "PS", + 11: "outside", +} + +BLUE = "#6699cc" +GRAY = "#999999" +DARKGRAY = "#333333" +YELLOW = "#ffcc33" +GREEN = "#339933" +RED = "#ff3333" +BLACK = "#000000" + + +def auto_crop_whitespace(image, color_invert=True): + # Convert to grayscale if not already + if len(image.shape) == 3: + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + else: + gray = image.copy() + + # Invert the image so floorplan is white and background is black + if color_invert: + _, binary = cv2.threshold(255 - gray, 1, 255, cv2.THRESH_BINARY) + else: + _, binary = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY) + + # Find non-zero (non-white) content + coords = cv2.findNonZero(binary) + x, y, w, h = cv2.boundingRect(coords) + + # Crop image + cropped_image = image[y : y + h, x : x + w].copy() + + # if polygons is None: + # return cropped_image, None + + # # Shift polygon coordinates + # shifted_polygons = [ + # [(px - x, py - y) for (px, py) in poly] + # for poly in polygons + # ] + return cropped_image, [x, y, w, h] # shifted_polygons + + +def plot_floorplan_with_regions( + regions, + corners=None, + edges=None, + base_scale=256, + scale=256, + matching_labels=None, + regions_type=None, + plot_text=False, + semantics_label_mapping=None, +): + """Draw floorplan map where different colors indicate different rooms""" + # cmap = get_cmap('tab20', 20) # nipy_spectral + # colors = [cmap(x) for x in np.linspace(0, 1, 21)] # colors = colors_12 + colors = list(qualitative.Set3) + list(qualitative.Dark2) # qualitative.Light24 + rgb_string_to_tuple = lambda rgb_string: tuple(float(x) / 255 for x in rgb_string.strip("rgb()").split(",")) + colors = [rgb_string_to_tuple(x) for x in colors] + # colors = [to_rgb(x) for x in colors] + gray_color = tuple(c / 255.0 for c in (255, 255, 255, 255)) + + regions = [(region * scale / base_scale).round().astype(np.int32) for region in regions] + + # Ensure room_colors contains valid hex strings + if matching_labels is None: + room_colors = [to_hex(colors[i % len(colors)]) for i in range(len(regions))] + else: + room_colors = [ + to_hex(colors[i % len(colors)]) if matching_labels[i] else to_hex(gray_color[:3]) + for i in range(len(regions)) + ] + + # colorMap = [tuple(int(h[i:i + 2], 16) for i in (1, 3, 5)) for h in room_colors] + # colorMap = np.asarray(colorMap) + colorMap = np.array([ImageColor.getrgb(h) for h in room_colors], dtype=np.uint8) + if len(regions) > 0: + colorMap = np.concatenate([np.full(shape=(1, 3), fill_value=0), colorMap], axis=0).astype(np.uint8) + else: + colorMap = np.concatenate([np.full(shape=(1, 3), fill_value=0)], axis=0).astype(np.uint8) + # when using opencv, we need to flip, from RGB to BGR + colorMap = colorMap[:, ::-1] + + alpha_channels = np.zeros(colorMap.shape[0], dtype=np.uint8) + alpha_channels[1 : len(regions) + 1] = 150 + + colorMap = np.concatenate([colorMap, np.expand_dims(alpha_channels, axis=-1)], axis=-1) + + room_map = np.zeros([scale, scale]).astype(np.int32) + # # sort regions + # if len(regions) > 1: + # avg_corner = [region.mean(axis=0) for region in regions] + # ind = np.argsort(np.square(np.array(avg_corner)).sum(axis=1), axis=0) + # regions = [regions[_idx] for _idx in ind] # np.array(regions)[ind] + + for idx, polygon in enumerate(regions): + cv2.fillPoly(room_map, [polygon], color=idx + 1) + + image = colorMap[room_map.reshape(-1)].reshape((scale, scale, 4)) + + pointColor = (0, 0, 0, 255) + lineColor = (0, 0, 0, 255) + + for region in regions: + for i, point in enumerate(region): + if i == len(region) - 1: + cv2.line(image, tuple(point), tuple(region[0]), color=lineColor, thickness=5) + else: + cv2.line(image, tuple(point), tuple(region[i + 1]), color=lineColor, thickness=5) + + for region in regions: + for i, point in enumerate(region): + cv2.circle(image, tuple(point), color=pointColor, radius=12, thickness=-1) + cv2.circle(image, tuple(point), color=(255, 255, 255, 0), radius=6, thickness=-1) + + if plot_text: + font_scale = 1.0 + text_padding = 1 + # Add room labels + for points, poly_type in zip(regions, regions_type): + # Calculate the centroid for text placement + M = cv2.moments(points) + if M["m00"] != 0: # Avoid division by zero + centroid_x = int(M["m10"] / M["m00"]) + centroid_y = int(M["m01"] / M["m00"]) + + # Get room label + label = semantics_label_mapping[poly_type] + + # Get text size for centering and background + text_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1)[0] + + # Calculate text background rectangle + text_x = centroid_x - text_size[0] // 2 + text_y = centroid_y + text_size[1] // 2 + + # Create background for text + rect_top_left = (text_x - text_padding, text_y - text_size[1] - text_padding) + rect_bottom_right = (text_x + text_size[0] + text_padding, text_y + text_padding) + + # Draw semi-transparent white background for text + background = image.copy() + cv2.rectangle(background, rect_top_left, rect_bottom_right, (255, 255, 255), -1) + + # Blend the background + cv2.addWeighted(background, 0.4, image, 0.6, 0, image) + + # cv2.rectangle(image, rect_top_left, rect_bottom_right, + # (255, 255, 255), -1) + + # Draw the text + cv2.putText( + image, + label, + (text_x, text_y), + cv2.FONT_HERSHEY_SIMPLEX, + font_scale, + (0, 0, 0), # Black text + 1, # Thickness + cv2.LINE_AA, # Anti-aliased text + ) + + return image + + +def plot_score_map(corner_map, scores): + """Draw score map overlaid on the density map""" + score_map = np.zeros([356, 356, 3]) + score_map[100:, 50:306] = corner_map + cv2.putText( + score_map, + "room_prec: " + str(round(scores["room_prec"] * 100, 1)), + (20, 30), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (252, 252, 0), + 1, + cv2.LINE_AA, + ) + cv2.putText( + score_map, + "room_rec: " + str(round(scores["room_rec"] * 100, 1)), + (190, 30), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (252, 252, 0), + 1, + cv2.LINE_AA, + ) + cv2.putText( + score_map, + "corner_prec: " + str(round(scores["corner_prec"] * 100, 1)), + (20, 55), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (0, 255, 255), + 1, + cv2.LINE_AA, + ) + cv2.putText( + score_map, + "corner_rec: " + str(round(scores["corner_rec"] * 100, 1)), + (190, 55), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (0, 255, 255), + 1, + cv2.LINE_AA, + ) + cv2.putText( + score_map, + "angles_prec: " + str(round(scores["angles_prec"] * 100, 1)), + (20, 80), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (0, 255, 0), + 1, + cv2.LINE_AA, + ) + cv2.putText( + score_map, + "angles_rec: " + str(round(scores["angles_rec"] * 100, 1)), + (190, 80), + cv2.FONT_HERSHEY_SIMPLEX, + 0.55, + (0, 255, 0), + 1, + cv2.LINE_AA, + ) + + return score_map + + +def plot_room_map(preds, room_map, room_id=0, im_size=256, plot_text=True): + """Draw room polygons overlaid on the density map""" + centroid_x = int(np.mean(preds[:, 0])) + centroid_y = int(np.mean(preds[:, 1])) + + # Get text size to create a background box + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 0.3 + thickness = 1 + text = str(room_id) + (text_width, text_height), baseline = cv2.getTextSize(text, font, font_scale, thickness) + border_color = (252, 252, 0) + + for i, corner in enumerate(preds): + if i == len(preds) - 1: + cv2.line( + room_map, + (round(corner[0]), round(corner[1])), + (round(preds[0][0]), round(preds[0][1])), + border_color, + 2, + ) + else: + cv2.line( + room_map, + (round(corner[0]), round(corner[1])), + (round(preds[i + 1][0]), round(preds[i + 1][1])), + border_color, + 2, + ) + cv2.circle(room_map, (round(corner[0]), round(corner[1])), 2, (0, 0, 255), 2) + # cv2.putText(room_map, str(i), (round(corner[0]), round(corner[1])), cv2.FONT_HERSHEY_SIMPLEX, + # 0.4, (0, 255, 0), 1, cv2.LINE_AA) + + # Draw white background box with transparency + # overlay = room_map.copy() + # cv2.addWeighted(overlay, 0.7, room_map, 0.3, 0, room_map) # 70% opacity + + # Draw text + if plot_text: + cv2.rectangle( + room_map, + (centroid_x - text_width // 2 - 2, centroid_y - text_height // 2 - 2), + (centroid_x + text_width // 2 + 2, centroid_y + text_height // 2 + 2), + (255, 255, 255), # (0, 0, 0), + -1, + ) # Filled rectangle + cv2.putText( + room_map, + text, + (centroid_x - text_width // 2, centroid_y + text_height // 2), + font, + font_scale, + (0, 100, 0), + thickness, + ) + + return room_map + + +def plot_density_map(sample, image_size, room_polys, pred_room_label_per_scene, plot_text=True): + if not isinstance(sample, np.ndarray): + density_map = np.transpose(sample.cpu().numpy(), [1, 2, 0]) + else: + density_map = sample + if density_map.shape[2] == 3: + density_map = density_map * (image_size - 1) + else: + density_map = np.repeat(density_map, 3, axis=2) * (image_size - 1) + pred_room_map = np.zeros([image_size, image_size, 3]) + + for room_poly, room_id in zip(room_polys, pred_room_label_per_scene): + pred_room_map = plot_room_map(room_poly, pred_room_map, room_id, im_size=image_size, plot_text=plot_text) + + alpha = 0.4 # Adjust for desired transparency + pred_room_map = cv2.addWeighted(density_map.astype(np.uint8), alpha, pred_room_map.astype(np.uint8), 1 - alpha, 0) + return pred_room_map + + +def plot_anno(img, annos, save_path, transformed=False, draw_poly=True, draw_bbx=True, thickness=2): + """Visualize annotation""" + img = np.repeat(np.expand_dims(img, 2), 3, axis=2) + num_inst = len(annos) + + bbx_color = (0, 255, 0) + # poly_color = (0, 255, 0) + for j in range(num_inst): + if draw_bbx: + bbox = annos[j]["bbox"] + if transformed: + start_point = (round(bbox[0]), round(bbox[1])) + end_point = (round(bbox[2]), round(bbox[3])) + else: + start_point = (round(bbox[0]), round(bbox[1])) + end_point = (round(bbox[0] + bbox[2]), round(bbox[1] + bbox[3])) + # Blue color in BGR + img = cv2.rectangle(img, start_point, end_point, bbx_color, thickness) + + if draw_poly: + verts = annos[j]["segmentation"][0] + if isinstance(verts, list): + verts = np.array(verts) + verts = verts.reshape(-1, 2) + + for i, corner in enumerate(verts): + if i == len(verts) - 1: + cv2.line( + img, + (round(corner[0]), round(corner[1])), + (round(verts[0][0]), round(verts[0][1])), + (0, 252, 252), + 1, + ) + else: + cv2.line( + img, + (round(corner[0]), round(corner[1])), + (round(verts[i + 1][0]), round(verts[i + 1][1])), + (0, 252, 252), + 1, + ) + cv2.circle(img, (round(corner[0]), round(corner[1])), 2, (255, 0, 0), 2) + cv2.putText( + img, + str(i), + (round(corner[0]), round(corner[1])), + cv2.FONT_HERSHEY_SIMPLEX, + 0.4, + (0, 255, 0), + 1, + cv2.LINE_AA, + ) + + imsave(save_path, img) + + +def plot_coords(ax, ob, color=BLACK, zorder=1, alpha=1, linewidth=1): + x, y = ob.xy + ax.plot(x, y, color=color, zorder=zorder, alpha=alpha, linewidth=linewidth, solid_joinstyle="miter") + + +def plot_corners(ax, ob, color=BLACK, zorder=1, alpha=1): + x, y = ob.xy + ax.scatter(x, y, color=color, marker="o") + + +def get_angle(p1, p2): + """Get the angle of this line with the horizontal axis.""" + dx = p2[0] - p1[0] + dy = p2[1] - p1[1] + theta = math.atan2(dy, dx) + angle = math.degrees(theta) # angle is in (-180, 180] + if angle < 0: + angle = 360 + angle + return angle + + +def filled_arc(e1, e2, direction, radius, ax, color): + """Draw arc for door""" + angle = get_angle(e1, e2) + if direction == "counterclock": + theta1 = angle + theta2 = angle + 90.0 + else: + theta1 = angle - 90.0 + theta2 = angle + circ = mpatches.Wedge(e1, radius, theta1, theta2, fill=True, color=color, linewidth=1, ec="#000000") + ax.add_patch(circ) + + +def plot_semantic_rich_floorplan(polygons, file_name, prec=None, rec=None): + """plot semantically-rich floorplan (i.e. with additional room label, door, window)""" + + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + + polygons_windows = [] + polygons_doors = [] + + # Iterate over rooms to draw black outline + for poly, poly_type in polygons: + if len(poly) > 2: + polygon = Polygon(poly) + if poly_type != 16 and poly_type != 17: + plot_coords(ax, polygon.exterior, alpha=1.0, linewidth=10) + + # Iterate over all predicted polygons (rooms, doors, windows) + for poly, poly_type in polygons: + if poly_type == "outqwall": # unclear what is this? + pass + elif poly_type == 16: # Door + door_length = math.dist(poly[0], poly[1]) + polygons_doors.append([poly, poly_type, door_length]) + elif poly_type == 17: # Window + polygons_windows.append([poly, poly_type]) + else: # regular room + polygon = Polygon(poly) + patch = PolygonPatch(polygon, facecolor="#FFFFFF", alpha=1.0, linewidth=0) + ax.add_patch(patch) + patch = PolygonPatch( + polygon, + facecolor=semantics_cmap[poly_type], + alpha=0.5, + linewidth=1, + capstyle="round", + edgecolor="#000000FF", + ) + ax.add_patch(patch) + ax.text( + np.mean(poly[:, 0]), + np.mean(poly[:, 1]), + S3D_LABEL[poly_type], + fontsize=6, + horizontalalignment="center", + verticalalignment="center", + bbox=dict(facecolor="white", alpha=0.7), + ) + + # Compute door size statistics (median) + door_median_size = np.median([door_length for (_, _, door_length) in polygons_doors]) + + # Draw doors + for poly, poly_type, door_size in polygons_doors: + door_size_y = np.abs(poly[0, 1] - poly[1, 1]) + door_size_x = np.abs(poly[0, 0] - poly[1, 0]) + if door_size_y > door_size_x: + if poly[1, 1] > poly[0, 1]: + e1 = poly[0] + e2 = poly[1] + else: + e1 = poly[1] + e2 = poly[0] + + if door_size < door_median_size * 1.5: + filled_arc(e1, e2, "clock", door_size, ax, "white") + else: + filled_arc(e1, e2, "clock", door_size / 2, ax, "white") + filled_arc(e2, e1, "counterclock", door_size / 2, ax, "white") + + else: + if poly[1, 0] > poly[0, 0]: + e1 = poly[1] + e2 = poly[0] + else: + e1 = poly[0] + e2 = poly[1] + + if door_size < door_median_size * 1.5: + filled_arc(e1, e2, "counterclock", door_size, ax, "white") + else: + filled_arc(e1, e2, "counterclock", door_size / 2, ax, "white") + filled_arc(e2, e1, "clock", door_size / 2, ax, "white") + + # Draw windows + for line, line_type in polygons_windows: + line = LineString(line) + poly = line.buffer(1.5, cap_style=2) + if poly.is_empty: + continue + patch = PolygonPatch(poly, facecolor="#FFFFFF", alpha=1.0, linewidth=1, linestyle="dashed") + ax.add_patch(patch) + + title = "" + if prec is not None: + title = "prec: " + str(round(prec * 100, 1)) + ", rec: " + str(round(rec * 100, 1)) + plt.title(file_name.split("/")[-1] + " " + title) + plt.axis("equal") + plt.axis("off") + + print(f">>> {file_name}") + # fig.savefig(file_name[:-3]+'svg', dpi=fig.dpi, format='svg') + fig.savefig(file_name, dpi=fig.dpi) + + +def plot_semantic_rich_floorplan_tight( + polygons, + file_name, + prec=None, + rec=None, + plot_text=True, + is_bw=False, + door_window_index=[16, 17], + img_w=256, + img_h=256, +): + """plot semantically-rich floorplan (i.e. with additional room label, door, window)""" + + # fig = plt.figure() + # ax = fig.add_subplot(1, 1, 1) + + # Set figure size to exactly 256x256 pixels + dpi = 100 # Standard screen DPI + figsize = (img_w / dpi, img_h / dpi) # Convert pixels to inches + + # Create square figure with fixed size + fig = plt.figure(figsize=figsize, dpi=dpi) + ax = fig.add_axes([0, 0, 1, 1]) + + # Set equal aspect ratio and the limits to exactly match the coordinate space + ax.set_aspect("equal") + ax.set_xlim(0, img_w - 1) # 255 + ax.set_ylim(0, img_h - 1) # 255 + + polygons_windows = [] + polygons_doors = [] + + # Iterate over rooms to draw black outline + for poly, poly_type in polygons: + if len(poly) > 2: + polygon = Polygon(poly) + + if poly_type not in door_window_index: + plot_coords(ax, polygon.exterior, alpha=1.0, linewidth=10) + + # Iterate over all predicted polygons (rooms, doors, windows) + for poly, poly_type in polygons: + if poly_type == "outqwall": # unclear what is this? + pass + elif poly_type == door_window_index[0]: # Door + door_length = math.dist(poly[0], poly[1]) + polygons_doors.append([poly, poly_type, door_length]) + elif poly_type == door_window_index[1]: # Window + polygons_windows.append([poly, poly_type]) + else: # regular room + if len(poly) < 3: + continue + polygon = Polygon(poly) + patch = PolygonPatch(polygon, facecolor="#FFFFFF", alpha=1.0, linewidth=0) + ax.add_patch(patch) + if not is_bw: + patch = PolygonPatch( + polygon, + facecolor=semantics_cmap[poly_type], + alpha=0.5, + linewidth=1, + capstyle="round", + edgecolor="#000000FF", + ) + ax.add_patch(patch) + if plot_text: + ax.text( + np.mean(poly[:, 0]), + np.mean(poly[:, 1]), + S3D_LABEL[poly_type], + size=6, + horizontalalignment="center", + verticalalignment="center", + ) + + # Compute door size statistics (median) + door_median_size = np.median([door_length for (_, _, door_length) in polygons_doors]) + + # Draw doors + for poly, poly_type, door_size in polygons_doors: + door_size_y = np.abs(poly[0, 1] - poly[1, 1]) + door_size_x = np.abs(poly[0, 0] - poly[1, 0]) + if door_size_y > door_size_x: + if poly[1, 1] > poly[0, 1]: + e1 = poly[0] + e2 = poly[1] + else: + e1 = poly[1] + e2 = poly[0] + + if door_size < door_median_size * 1.5: + filled_arc(e1, e2, "clock", door_size, ax, "white") + else: + filled_arc(e1, e2, "clock", door_size / 2, ax, "white") + filled_arc(e2, e1, "counterclock", door_size / 2, ax, "white") + + else: + if poly[1, 0] > poly[0, 0]: + e1 = poly[1] + e2 = poly[0] + else: + e1 = poly[0] + e2 = poly[1] + + if door_size < door_median_size * 1.5: + filled_arc(e1, e2, "counterclock", door_size, ax, "white") + else: + filled_arc(e1, e2, "counterclock", door_size / 2, ax, "white") + filled_arc(e2, e1, "clock", door_size / 2, ax, "white") + + # Draw windows + for line, line_type in polygons_windows: + line = LineString(line) + poly = line.buffer(1.5, cap_style=2) + if poly.is_empty: + continue + patch = PolygonPatch(poly, facecolor="#FFFFFF", alpha=1.0, linewidth=1, linestyle="dashed") + ax.add_patch(patch) + + if plot_text: + title = "" + if prec is not None: + title = "prec: " + str(round(prec * 100, 1)) + ", rec: " + str(round(rec * 100, 1)) + plt.title(file_name.split("/")[-1] + " " + title) + + # plt.axis('equal') + plt.axis("off") + + print(f">>> {file_name}") + # fig.savefig(file_name[:-3]+'svg', dpi=fig.dpi, format='svg') + if is_bw: + plt.set_cmap(get_cmap("gray")) + + fig.savefig(file_name, dpi=dpi, bbox_inches="tight", pad_inches=0) + + +def plot_semantic_rich_floorplan_nicely( + polygons, + file_name, + prec=None, + rec=None, + plot_text=True, + is_bw=False, + door_window_index=[16, 17], + img_w=256, + img_h=256, + semantics_label_mapping=S3D_LABEL, +): + """plot semantically-rich floorplan (i.e. with additional room label, door, window)""" + + # Set figure size to exactly 256x256 pixels + dpi = 150 # Standard screen DPI + figsize = (img_w / dpi, img_h / dpi) # Convert pixels to inches + + # Create square figure with fixed size + fig = plt.figure(figsize=figsize, dpi=dpi, frameon=False) + ax = fig.add_axes([0, 0, 1, 1]) + + # Set equal aspect ratio and the limits to exactly match the coordinate space + # ax.set_aspect('equal') + # ax.set_xlim(0, img_w - 1) + # ax.set_ylim(0, img_h - 1) + + # Disable autoscaling + ax.autoscale(False) + + # Disable adjusting automatically + plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) + + polygons_windows = [] + polygons_doors = [] + + # Iterate over rooms to draw black outline + for poly, poly_type in polygons: + if len(poly) > 2: + polygon = Polygon(poly) + + if poly_type not in door_window_index: + plot_coords(ax, polygon.exterior, alpha=1.0, linewidth=2) + + # Iterate over all predicted polygons (rooms, doors, windows) + for poly, poly_type in polygons: + if poly_type == door_window_index[0]: # Door + door_length = math.dist(poly[0], poly[1]) + polygons_doors.append([poly, poly_type, door_length]) + elif poly_type == door_window_index[1]: # Window + polygons_windows.append([poly, poly_type]) + else: # regular room + polygon = Polygon(poly) + patch = PolygonPatch(polygon, facecolor="#FFFFFF", alpha=1.0, linewidth=0) + ax.add_patch(patch) + if not is_bw: + patch = PolygonPatch( + polygon, + facecolor=semantics_cmap[poly_type], + alpha=0.5, + linewidth=1, + capstyle="round", + edgecolor="#000000FF", + ) + ax.add_patch(patch) + if plot_text: + ax.text( + np.mean(poly[:, 0]), + np.mean(poly[:, 1]), + semantics_label_mapping[poly_type], + fontsize=6, + ha="center", + va="center", + bbox=dict(facecolor="white", alpha=0.7), + ) + + # Compute door size statistics (median) + door_median_size = np.median([door_length for (_, _, door_length) in polygons_doors]) + + # Draw doors + for poly, poly_type, door_size in polygons_doors: + door_size_y = np.abs(poly[0, 1] - poly[1, 1]) + door_size_x = np.abs(poly[0, 0] - poly[1, 0]) + if door_size_y > door_size_x: + if poly[1, 1] > poly[0, 1]: + e1 = poly[0] + e2 = poly[1] + else: + e1 = poly[1] + e2 = poly[0] + + if door_size < door_median_size * 1.5: + filled_arc(e1, e2, "clock", door_size, ax, "white") + else: + filled_arc(e1, e2, "clock", door_size / 2, ax, "white") + filled_arc(e2, e1, "counterclock", door_size / 2, ax, "white") + + else: + if poly[1, 0] > poly[0, 0]: + e1 = poly[1] + e2 = poly[0] + else: + e1 = poly[0] + e2 = poly[1] + + if door_size < door_median_size * 1.5: + filled_arc(e1, e2, "counterclock", door_size, ax, "white") + else: + filled_arc(e1, e2, "counterclock", door_size / 2, ax, "white") + filled_arc(e2, e1, "clock", door_size / 2, ax, "white") + + # Draw windows + for line, line_type in polygons_windows: + line = LineString(line) + poly = line.buffer(1.5, cap_style=2) + if poly.is_empty: + continue + patch = PolygonPatch(poly, facecolor="#FFFFFF", alpha=1.0, linewidth=1, linestyle="dashed") + ax.add_patch(patch) + + if plot_text: + title = "" + if prec is not None: + title = "prec: " + str(round(prec * 100, 1)) + ", rec: " + str(round(rec * 100, 1)) + plt.title(file_name.split("/")[-1] + " " + title) + + print(f">>> {file_name}") + plt.axis("equal") + plt.axis("off") + # fig.savefig(file_name[:-3]+'svg', dpi=fig.dpi, format='svg') + if is_bw: + plt.set_cmap(get_cmap("gray")) + fig.savefig(file_name, bbox_inches="tight", pad_inches=0) + plt.close() + + +def plot_semantic_rich_floorplan_opencv( + polygons, + file_name, + img_w=256, + img_h=256, + door_window_index=[16, 17], + semantics_label_mapping=S3D_LABEL, + is_bw=False, + plot_text=True, + one_color=False, + scale=1, + is_sem=True, +): + """ + Plot semantically-rich floorplan using OpenCV with improved quality. + + Args: + polygons (list): A list of polygons, where each polygon is a list of (x, y) coordinates. + file_name (str): Path to save the output image. + img_w (int): Width of the output image. + img_h (int): Height of the output image. + door_window_index (list): Indices for door and window types. + semantics_label_mapping (dict): Mapping from polygon type to semantic label. + is_bw (bool): If True, use black and white colors only. + line_thickness (int): Thickness of lines for polygons and doors/windows. + text_padding (int): Padding around text labels. + font_scale (float): Scale factor for text size. + room_alpha (float): Transparency for room colors (0.0-1.0). + anti_aliasing (bool): Whether to use anti-aliasing for lines. + """ + line_thickness = 2 + text_padding = 1 + font_scale = 0.25 + room_alpha = 0.6 + + if scale != 1: + new_polygons = [] + for poly, poly_label in polygons: + poly = (poly * scale).round().astype(np.int32) + new_polygons.append([poly, poly_label]) + polygons = new_polygons + + if one_color: + colors = ["#FFD700"] + else: + colors = list(qualitative.Set3) + list(qualitative.Dark2) + rgb_string_to_tuple = lambda rgb_string: tuple(float(x) / 255 for x in rgb_string.strip("rgb()").split(",")) + colors = [to_hex(rgb_string_to_tuple(x)) for x in colors] + + # Create a white background image (more conventional for floorplans) + if is_bw: + image = np.ones((img_h, img_w), dtype=np.uint8) * 255 # White grayscale image + else: + image = np.ones((img_h, img_w, 3), dtype=np.uint8) * 255 # White RGB image + + # Create a separate layer for room colors + overlay = image.copy() + + # Track polygons for each type for proper layering + room_polygons = [] + door_polygons = [] + window_polygons = [] + + # Sort polygons by type + for poly, poly_type in polygons: + if len(poly) < 2: # Skip invalid polygons + continue + + points = np.array(poly, dtype=np.int32) + + if door_window_index and poly_type == door_window_index[0]: # Door + door_polygons.append((points, poly_type)) + elif door_window_index and poly_type == door_window_index[1]: # Window + window_polygons.append((points, poly_type)) + else: # Room + room_polygons.append((points, poly_type)) + + # Draw rooms first (bottom layer) + for room_id, (points, poly_type) in enumerate(room_polygons): + # Fill room with color + if not is_bw: + # Get RGB color from semantics_cmap and convert from RGB to BGR for OpenCV + if not is_sem: + rgb_color = ImageColor.getcolor(colors[room_id % len(colors)], "RGB") + else: + rgb_color = ImageColor.getcolor(colors[poly_type % len(colors)], "RGB") + + bgr_color = (rgb_color[2], rgb_color[1], rgb_color[0]) + cv2.fillPoly(overlay, [points], color=bgr_color) + else: + # Use light gray for rooms in BW mode + cv2.fillPoly(overlay, [points], color=(240, 240, 240)) + + # Draw room outline + line_type = cv2.LINE_AA + cv2.polylines(image, [points], isClosed=True, color=(0, 0, 0), thickness=line_thickness, lineType=line_type) + + # Blend overlay with transparency + cv2.addWeighted(overlay, room_alpha, image, 1 - room_alpha, 0, image) + + # Draw doors with proper styling + for points, _ in door_polygons: + if len(points) >= 2: + # For doors, we can improve by drawing arcs to represent swing + # Here we draw them as thick lines with distinctive color + door_color = (100, 100, 100) if is_bw else (0, 0, 255) # Gray for BW, Red for RGB + line_type = cv2.LINE_AA + cv2.polylines( + image, [points], isClosed=False, color=door_color, thickness=line_thickness * 2, lineType=line_type + ) + + # Draw windows with dashed styling + for points, _ in window_polygons: + if len(points) >= 2: + window_color = (150, 150, 150) if is_bw else (255, 0, 0) # Gray for BW, Blue for RGB + + # Create dashed line effect for windows + if len(points) == 2: + # For a simple line window + pt1, pt2 = points[0], points[1] + dash_length = 5 + + # Calculate line parameters + length = np.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2) + if length > 0: + num_dashes = max(2, int(length / (2 * dash_length))) + + for i in range(num_dashes): + start_ratio = i / num_dashes + end_ratio = (i + 0.5) / num_dashes + + start_x = int(pt1[0] + (pt2[0] - pt1[0]) * start_ratio) + start_y = int(pt1[1] + (pt2[1] - pt1[1]) * start_ratio) + end_x = int(pt1[0] + (pt2[0] - pt1[0]) * end_ratio) + end_y = int(pt1[1] + (pt2[1] - pt1[1]) * end_ratio) + + line_type = cv2.LINE_AA + cv2.line( + image, + (start_x, start_y), + (end_x, end_y), + window_color, + thickness=line_thickness, + lineType=line_type, + ) + else: + # For multi-point windows + line_type = cv2.LINE_AA + cv2.polylines( + image, [points], isClosed=True, color=window_color, thickness=line_thickness, lineType=line_type + ) + + if plot_text: + # Add room labels + for i, (points, poly_type) in enumerate(room_polygons): + # if i > 1: continue # TODO:test + # Calculate the centroid for text placement + M = cv2.moments(points) + if M["m00"] != 0: # Avoid division by zero + centroid_x = int(M["m10"] / M["m00"]) + centroid_y = int(M["m01"] / M["m00"]) + + # Get room label + label = semantics_label_mapping[poly_type] + + # Get text size for centering and background + text_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1)[0] + + # Calculate text background rectangle + text_x = centroid_x - text_size[0] // 2 + text_y = centroid_y + text_size[1] // 2 + + # Create background for text + rect_top_left = (text_x - text_padding, text_y - text_size[1] - text_padding) + rect_bottom_right = (text_x + text_size[0] + text_padding, text_y + text_padding) + + # Draw semi-transparent white background for text + background = image.copy() + cv2.rectangle(background, rect_top_left, rect_bottom_right, (255, 255, 255), -1) + + # Blend the background + cv2.addWeighted(background, 0.7, image, 0.3, 0, image) + + # Draw the text + cv2.putText( + image, + label, + (text_x, text_y), + cv2.FONT_HERSHEY_SIMPLEX, + font_scale, + (0, 0, 0), # Black text + 1, # Thickness + cv2.LINE_AA, # Anti-aliased text + ) + + # Add border around the image for better framing + # cv2.rectangle(image, (0, 0), (img_w-1, img_h-1), (0, 0, 0), 1, cv2.LINE_AA) + + # Save with high quality + if file_name is not None: + if is_bw: + cv2.imwrite(file_name, image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) + else: + cv2.imwrite(file_name, image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) + print(f"Saved improved floorplan to {file_name}") + + return image + + +def draw_dashed_line(image, pt1, pt2, color, thickness, dash_length=10): + """Draw a dashed line between two points.""" + # Calculate the Euclidean distance between the points + dist = np.linalg.norm(np.array(pt2) - np.array(pt1)) + # Calculate the number of dashes + num_dashes = int(dist // dash_length) + # Calculate the direction vector + direction = (np.array(pt2) - np.array(pt1)) / dist + for i in range(num_dashes): + start = pt1 + direction * (i * dash_length) + end = pt1 + direction * ((i + 0.5) * dash_length) + cv2.line(image, tuple(start.astype(int)), tuple(end.astype(int)), color, thickness) + + +def draw_dashed_polyline(image, points, color, thickness, dash_length=10, gap_length=5): + """ + Draws a dashed polyline with evenly spaced dashes along the entire path. + + Parameters: + - image: The image on which to draw. + - points: List of points defining the polyline. + - color: Color of the dashes (BGR tuple). + - thickness: Thickness of the dashes. + - dash_length: Length of each dash. + - gap_length: Length of the gap between dashes. + """ + if len(points) < 2: + return + + # Convert points to numpy array for vectorized operations + pts = np.array(points, dtype=np.float32) + + # Calculate the total length of the polyline + segment_lengths = np.linalg.norm(pts[1:] - pts[:-1], axis=1) + total_length = np.sum(segment_lengths) + + # Determine number of dashes + pattern_length = dash_length + gap_length + num_dashes = int(total_length // pattern_length) + + # Generate dash start positions along the total length + dash_positions = np.arange(0, num_dashes * pattern_length, pattern_length) + + # Initialize variables to track the current segment + seg_idx = 0 + seg_start = pts[0] + seg_end = pts[1] + seg_length = segment_lengths[0] + seg_vector = (seg_end - seg_start) / seg_length + seg_pos = 0.0 # Position along the current segment + + for pos in dash_positions: + # Advance to the segment containing the current dash + while seg_pos + seg_length < pos: + seg_pos += seg_length + seg_idx += 1 + if seg_idx >= len(pts) - 1: + return + seg_start = pts[seg_idx] + seg_end = pts[seg_idx + 1] + seg_length = segment_lengths[seg_idx] + seg_vector = (seg_end - seg_start) / seg_length + + # Calculate start and end points of the dash + offset = pos - seg_pos + start_point = seg_start + seg_vector * offset + end_offset = min(dash_length, seg_length - offset) + end_point = start_point + seg_vector * end_offset + + # Draw the dash + cv2.line( + image, tuple(np.round(start_point).astype(int)), tuple(np.round(end_point).astype(int)), color, thickness + ) + + +def plot_semantic_rich_floorplan_opencv_figure( + polygons, + file_name, + img_w=256, + img_h=256, + door_window_index=[16, 17], + semantics_label_mapping=S3D_LABEL, + is_bw=False, + plot_text=True, + one_color=False, +): + """ + Plot semantically-rich floorplan using OpenCV with improved quality. + + Args: + polygons (list): A list of polygons, where each polygon is a list of (x, y) coordinates. + file_name (str): Path to save the output image. + img_w (int): Width of the output image. + img_h (int): Height of the output image. + door_window_index (list): Indices for door and window types. + semantics_label_mapping (dict): Mapping from polygon type to semantic label. + is_bw (bool): If True, use black and white colors only. + line_thickness (int): Thickness of lines for polygons and doors/windows. + text_padding (int): Padding around text labels. + font_scale (float): Scale factor for text size. + room_alpha (float): Transparency for room colors (0.0-1.0). + anti_aliasing (bool): Whether to use anti-aliasing for lines. + """ + line_thickness = 2 + text_padding = 1 + font_scale = 1.0 + room_alpha = 0.6 + + if img_w != 256: + new_polygons = [] + for poly, poly_label in polygons: + poly = (poly * img_w / 256).round().astype(np.int32) + new_polygons.append([poly, poly_label]) + polygons = new_polygons + + if one_color: + colors = ["#FFD700"] + else: + # colors = [to_hex(x) for x in qualitative.Light24] + # TODO + colors = ["#FFFFFF"] * len(qualitative.Light24) + colors[polygons[0][1]] = "#FF9616" # red + colors[polygons[1][1]] = "#FE00CE" # green + + # cmap = get_cmap('tab20', 20) + # colors = [to_hex(cmap(x)) for x in np.linspace(0, 1, 20)] # Convert to hex + # Create a white background image (more conventional for floorplans) + if is_bw: + image = np.ones((img_h, img_w), dtype=np.uint8) * 255 # White grayscale image + else: + image = np.ones((img_h, img_w, 3), dtype=np.uint8) * 255 # White RGB image + + # Create a separate layer for room colors + overlay = image.copy() + + # Track polygons for each type for proper layering + room_polygons = [] + door_polygons = [] + window_polygons = [] + + # Sort polygons by type + for poly, poly_type in polygons: + if len(poly) < 2: # Skip invalid polygons + continue + + points = np.array(poly, dtype=np.int32) + + if poly_type == door_window_index[0]: # Door + door_polygons.append((points, poly_type)) + elif poly_type == door_window_index[1]: # Window + window_polygons.append((points, poly_type)) + else: # Room + room_polygons.append((points, poly_type)) + + # Draw rooms first (bottom layer) + for room_id, (points, poly_type) in enumerate(room_polygons): + # TODO:test + if room_id > 1: + poly_type = room_polygons[0][1] + 1 + + # Fill room with color + if not is_bw: + # Get RGB color from semantics_cmap and convert from RGB to BGR for OpenCV + # if not plot_text: + # rgb_color = ImageColor.getcolor(colors[room_id % len(colors)], "RGB") + # else: + # rgb_color = ImageColor.getcolor(colors[poly_type % len(colors)], "RGB") + # TODO + rgb_color = ImageColor.getcolor(colors[poly_type % len(colors)], "RGB") + bgr_color = (rgb_color[2], rgb_color[1], rgb_color[0]) + + cv2.fillPoly(overlay, [points], color=bgr_color) + else: + # Use light gray for rooms in BW mode + cv2.fillPoly(overlay, [points], color=(240, 240, 240)) + + # # Draw room outline + # if room_id > 1: + # # Draw dashed room outline + # for i in range(len(points)): + # pt1 = points[i] + # pt2 = points[(i + 1) % len(points)] # Wrap around to the first point + # # draw_dashed_line(image, pt1, pt2, color=(0, 0, 0), thickness=line_thickness, dash_length=10) + # draw_dashed_polyline(image, points, color=(0, 0, 0), thickness=line_thickness, dash_length=5, gap_length=5) + # else: + line_type = cv2.LINE_AA + cv2.polylines(image, [points], isClosed=True, color=(0, 0, 0), thickness=line_thickness, lineType=line_type) + + # Blend overlay with transparency + cv2.addWeighted(overlay, room_alpha, image, 1 - room_alpha, 0, image) + + # Draw doors with proper styling + for points, _ in door_polygons: + if len(points) >= 2: + # For doors, we can improve by drawing arcs to represent swing + # Here we draw them as thick lines with distinctive color + door_color = (100, 100, 100) if is_bw else (0, 0, 255) # Gray for BW, Red for RGB + line_type = cv2.LINE_AA + cv2.polylines( + image, [points], isClosed=False, color=door_color, thickness=line_thickness * 2, lineType=line_type + ) + + # Draw windows with dashed styling + for points, _ in window_polygons: + if len(points) >= 2: + window_color = (150, 150, 150) if is_bw else (255, 0, 0) # Gray for BW, Blue for RGB + + # Create dashed line effect for windows + if len(points) == 2: + # For a simple line window + pt1, pt2 = points[0], points[1] + dash_length = 5 + + # Calculate line parameters + length = np.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2) + if length > 0: + num_dashes = max(2, int(length / (2 * dash_length))) + + for i in range(num_dashes): + start_ratio = i / num_dashes + end_ratio = (i + 0.5) / num_dashes + + start_x = int(pt1[0] + (pt2[0] - pt1[0]) * start_ratio) + start_y = int(pt1[1] + (pt2[1] - pt1[1]) * start_ratio) + end_x = int(pt1[0] + (pt2[0] - pt1[0]) * end_ratio) + end_y = int(pt1[1] + (pt2[1] - pt1[1]) * end_ratio) + + line_type = cv2.LINE_AA + cv2.line( + image, + (start_x, start_y), + (end_x, end_y), + window_color, + thickness=line_thickness, + lineType=line_type, + ) + else: + # For multi-point windows + line_type = cv2.LINE_AA + cv2.polylines( + image, [points], isClosed=True, color=window_color, thickness=line_thickness, lineType=line_type + ) + + if plot_text: + # Add room labels + for i, (points, poly_type) in enumerate(room_polygons): + if i > 1: + continue # TODO:test + # Calculate the centroid for text placement + M = cv2.moments(points) + if M["m00"] != 0: # Avoid division by zero + centroid_x = int(M["m10"] / M["m00"]) + centroid_y = int(M["m01"] / M["m00"]) + + # Get room label + label = semantics_label_mapping[poly_type] + + # Get text size for centering and background + text_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, font_scale, 1)[0] + + # Calculate text background rectangle + text_x = centroid_x - text_size[0] // 2 + text_y = centroid_y + text_size[1] // 2 + + # Create background for text + rect_top_left = (text_x - text_padding, text_y - text_size[1] - text_padding) + rect_bottom_right = (text_x + text_size[0] + text_padding, text_y + text_padding) + + # Draw semi-transparent white background for text + background = image.copy() + cv2.rectangle(background, rect_top_left, rect_bottom_right, (255, 255, 255), -1) + + # Blend the background + cv2.addWeighted(background, 0.7, image, 0.3, 0, image) + + # Draw the text + cv2.putText( + image, + label, + (text_x, text_y), + cv2.FONT_HERSHEY_SIMPLEX, + font_scale, + (0, 0, 0), # Black text + 1, # Thickness + cv2.LINE_AA, # Anti-aliased text + ) + + # Add border around the image for better framing + # cv2.rectangle(image, (0, 0), (img_w-1, img_h-1), (0, 0, 0), 1, cv2.LINE_AA) + + # Save with high quality + if is_bw: + cv2.imwrite(file_name, image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) + else: + cv2.imwrite(file_name, image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) + + print(f"Saved improved floorplan to {file_name}") + + return image # Return the image for optional further processing or visualization + + +def sort_polygons_by_matching(matching_pred2gt, pred_polygons, gt_polygons): + """ + Sorts pred_polygons and gt_polygons based on the matching indices. + + Args: + matching_pred2gt (list): List of matching indices from pred to gt. + pred_polygons (list): List of predicted polygons. + gt_polygons (list): List of ground truth polygons. + + Returns: + tuple: (sorted_pred_polygons, sorted_gt_polygons) + """ + sorted_pred_polygons = [] # Keep the order of pred_polygons as is + sorted_gt_polygons = [] + pred_mask = [] + gt_mask = [] + remaining_pred_polygons = [] + + for i, match_idx in enumerate(matching_pred2gt): + if match_idx == -1: + # sorted_gt_polygons.append(None) # No match, insert placeholder + remaining_pred_polygons.append(pred_polygons[i]) + continue + else: + sorted_pred_polygons.append(pred_polygons[i]) + sorted_gt_polygons.append(gt_polygons[match_idx]) + gt_mask.append(1) + pred_mask.append(1) + + sorted_pred_polygons.extend(remaining_pred_polygons) + pred_mask.extend([0] * len(remaining_pred_polygons)) + + for i in range(len(gt_polygons)): + if i not in matching_pred2gt: + sorted_gt_polygons.append(gt_polygons[i]) + gt_mask.append(0) + + return sorted_pred_polygons, sorted_gt_polygons, pred_mask, gt_mask + + +def concat_floorplan_maps(gt_floorplan_map, floorplan_map, plot_statistics={}): + pad_color = (0, 0, 0) if gt_floorplan_map.shape[2] == 3 else (0, 0, 0, 0) + padding = np.full((gt_floorplan_map.shape[0], 10, gt_floorplan_map.shape[2]), pad_color, dtype=np.uint8) + # Concatenate pred_room_map, padding, and gt_room_map + concatenated_map = cv2.hconcat([gt_floorplan_map, padding, floorplan_map]) + top_padding = np.full((100, concatenated_map.shape[1], concatenated_map.shape[2]), pad_color, dtype=np.uint8) + + # Add text for f1 and missing_rate + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 1 + font_color = (255, 255, 255) if gt_floorplan_map.shape[2] == 3 else (0, 0, 255, 255) # White text + thickness = 2 + line_type = cv2.LINE_AA + + # Position for the text + text_f1 = ( + f"F1: {plot_statistics['f1']:.2f}, Prec: {plot_statistics['prec']:.2f}, Rec: {plot_statistics['rec']:.2f}" + ) + text_missing_rate = f"Missing Rate: {plot_statistics['missing_rate']:.2f}, {plot_statistics['num_preds']}/{plot_statistics['num_matched_preds']}/{plot_statistics['num_gt']}" + text_position_f1 = (10, 30) # Position within the top padding + text_position_missing_rate = (10, 70) # Adjusted position for the second line + + # Overlay text on the top padding + cv2.putText(top_padding, text_f1, text_position_f1, font, font_scale, font_color, thickness, line_type) + cv2.putText( + top_padding, text_missing_rate, text_position_missing_rate, font, font_scale, font_color, thickness, line_type + ) + + # Concatenate the top padding with the concatenated_map + final_map = cv2.vconcat([top_padding, concatenated_map]) + return final_map diff --git a/util/poly_ops.py b/util/poly_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2c2a08c76c0adf6b0d10c9ce23029c63bcf2fa12 --- /dev/null +++ b/util/poly_ops.py @@ -0,0 +1,91 @@ +""" +Utilities for polygon manipulation. +""" + +import numpy as np +import torch + + +def is_clockwise(points): + """Check whether a sequence of points is clockwise ordered""" + # points is a list of 2d points. + assert len(points) > 0 + s = 0.0 + for p1, p2 in zip(points, points[1:] + [points[0]]): + s += (p2[0] - p1[0]) * (p2[1] + p1[1]) + return s > 0.0 + + +def resort_corners(corners): + """Resort a sequence of corners so that the first corner starts + from upper-left and counterclockwise ordered in image + """ + corners = corners.reshape(-1, 2) + x_y_square_sum = corners[:, 0] ** 2 + corners[:, 1] ** 2 + start_corner_idx = np.argmin(x_y_square_sum) + + corners_sorted = np.concatenate([corners[start_corner_idx:], corners[:start_corner_idx]]) + + ## sort points clockwise (counterclockwise in image) + if not is_clockwise(corners_sorted[:, :2].tolist()): + corners_sorted[1:] = np.flip(corners_sorted[1:], 0) + + return corners_sorted.reshape(-1) + + +def get_all_order_corners(corners): + """Get all possible permutation of a polygon""" + length = int(len(corners) / 2) + all_corners = torch.stack([corners.roll(i * 2) for i in range(length)]) + return all_corners + + +def pad_gt_polys(gt_instances, num_queries_per_poly, image_size, drop_rate, device=None): + """Pad the ground truth polygons so that they have a uniform length""" + + thr_length = num_queries_per_poly * 2 + room_targets = [] + # padding ground truth on-fly + for gt_inst in gt_instances: + room_dict = {} + room_corners = [] + corner_labels = [] + corner_lengths = [] + + for i, poly in enumerate(gt_inst.gt_masks.polygons): + corners = torch.from_numpy(poly[0]).to(device) + corners = torch.clip(corners, 0, image_size - 1) / (image_size - 1) + + # automatically skip the polygon if it is too long + if len(corners) > thr_length: + continue + + corner_lengths.append(len(corners)) + + corners_pad = torch.zeros(num_queries_per_poly * 2, device=device) + corners_pad[: len(corners)] = corners + + labels = torch.ones(int(len(corners) / 2), dtype=torch.int64).to(device) + labels_pad = torch.zeros(num_queries_per_poly, device=device) + labels_pad[: len(labels)] = labels + room_corners.append(corners_pad) + corner_labels.append(labels_pad) + + room_classes = gt_inst.gt_classes + if drop_rate > 0.0: + keep_indices = np.where(np.random.rand(len(room_corners)) >= drop_rate)[0].tolist() + if len(keep_indices) > 0: # Only apply drop if we have something left + room_corners = [room_corners[i] for i in keep_indices] + corner_labels = [corner_labels[i] for i in keep_indices] + corner_lengths = [corner_lengths[i] for i in keep_indices] + room_classes = gt_inst.gt_classes[keep_indices] + + room_dict = { + "coords": torch.stack(room_corners).to(device), + "labels": torch.stack(corner_labels).to(device), + "lengths": torch.tensor(corner_lengths, device=device), + "room_labels": room_classes.to(device), + } + room_targets.append(room_dict) + + return room_targets