Spaces:
Build error
Build error
File size: 8,250 Bytes
117d838 5a3810e 117d838 5a3810e 117d838 8d91aab 117d838 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | from collections import defaultdict
from itertools import chain
from typing import Dict, List, Tuple, TypedDict
import datasets as ds
import evaluate
import numpy as np
import numpy.typing as npt
from scipy.optimize import linear_sum_assignment
class Layout(TypedDict):
bboxes: npt.NDArray[np.float64]
categories: npt.NDArray[np.int64]
_DESCRIPTION = """\
Compute the maximum IoU between two sets of layouts.
"""
_KWARGS_DESCRIPTION = """\
Args:
layouts1 (`list` of `dict`): A list of dictionaries representing layouts including `list` of `bboxes` (float) and `list` of `categories` (int).
layouts2 (`list` of `dict`): A list of dictionaries representing layouts including `list` of `bboxes` (float) and `list` of `categories` (int).
Returns:
float: The maximum IoU score.
Examples:
Example 1: Single processing
>>> metric = evaluate.load("creative-graphic-design/layout-maximum-iou")
>>> num_samples, num_categories = 24, 4
>>> layout1 = {
>>> "bboxes": np.random.rand(num_samples, num_categories),
>>> "categories": np.random.randint(0, num_categories, size=(num_samples,)),
>>> }
>>> layout2 = {
>>> "bboxes": np.random.rand(num_samples, num_categories),
>>> "categories": np.random.randint(0, num_categories, size=(num_samples,)),
>>> }
>>> metric.add(layouts1=layout1, layouts2=layout2)
>>> print(metric.compute())
Example 2: Batch processing
>>> metric = evaluate.load("creative-graphic-design/layout-maximum-iou")
>>> batch_size, num_samples, num_categories = 512, 24, 4
>>> layouts1 = [
>>> {
>>> "bboxes": np.random.rand(num_samples, num_categories),
>>> "categories": np.random.randint(0, num_categories, size=(num_samples,)),
>>> }
>>> for _ in range(batch_size)
>>> ]
>>> layouts2 = [
>>> {
>>> "bboxes": np.random.rand(num_samples, num_categories),
>>> "categories": np.random.randint(0, num_categories, size=(num_samples,)),
>>> }
>>> for _ in range(batch_size)
>>> ]
>>> metric.add_batch(layouts1=layouts1, layouts2=layouts2)
>>> print(metric.compute())
"""
_CITATION = """\
@inproceedings{kikuchi2021constrained,
title={Constrained graphic layout generation via latent optimization},
author={Kikuchi, Kotaro and Simo-Serra, Edgar and Otani, Mayu and Yamaguchi, Kota},
booktitle={Proceedings of the 29th ACM International Conference on Multimedia},
pages={88--96},
year={2021}
}
"""
def convert_xywh_to_ltrb(
batch_bbox: npt.NDArray[np.float64],
) -> Tuple[
npt.NDArray[np.float64],
npt.NDArray[np.float64],
npt.NDArray[np.float64],
npt.NDArray[np.float64],
]:
xc, yc, w, h = batch_bbox
x1 = xc - w / 2
y1 = yc - h / 2
x2 = xc + w / 2
y2 = yc + h / 2
return (x1, y1, x2, y2)
def _compute_iou(
bbox1: npt.NDArray[np.float64],
bbox2: npt.NDArray[np.float64],
generalized: bool = False,
):
# shape: bbox1 (N, 4), bbox2 (N, 4)
assert bbox1.shape[0] == bbox2.shape[0]
assert bbox1.shape[1] == bbox1.shape[1] == 4
l1, t1, r1, b1 = convert_xywh_to_ltrb(bbox1.T)
l2, t2, r2, b2 = convert_xywh_to_ltrb(bbox2.T)
a1, a2 = (r1 - l1) * (b1 - t1), (r2 - l2) * (b2 - t2)
# intersection
l_max = np.maximum(l1, l2)
r_min = np.minimum(r1, r2)
t_max = np.maximum(t1, t2)
b_min = np.minimum(b1, b2)
cond = (l_max < r_min) & (t_max < b_min)
ai = np.where(cond, (r_min - l_max) * (b_min - t_max), np.zeros_like(a1[0]))
au = a1 + a2 - ai
iou = ai / au
if not generalized:
return iou
# outer region
l_min = np.minimum(l1, l2)
r_max = np.maximum(r1, r2)
t_min = np.minimum(t1, t2)
b_max = np.maximum(b1, b2)
ac = (r_max - l_min) * (b_max - t_min)
giou = iou - (ac - au) / ac
return giou
def _compute_maximum_iou_for_layout(layout1: Layout, layout2: Layout):
score = 0.0
bi, ci = layout1["bboxes"], layout1["categories"]
bj, cj = layout2["bboxes"], layout2["categories"]
N = len(bi)
for c in list(set(ci.tolist())):
_bi = bi[np.where(ci == c)]
_bj = bj[np.where(cj == c)]
n = len(_bi)
ii, jj = np.meshgrid(range(n), range(n))
ii, jj = ii.flatten(), jj.flatten()
iou = _compute_iou(_bi[ii], _bj[jj]).reshape(n, n)
# Note: maximize is supported only when scipy >= 1.4
ii, jj = linear_sum_assignment(iou, maximize=True)
score += iou[ii, jj].sum().item()
return score / N
def _compute_maximum_iou(
layouts_1_and_2: Tuple[List[Layout], List[Layout]],
) -> npt.NDArray[np.float64]:
assert len(layouts_1_and_2) == 2
layouts1, layouts2 = layouts_1_and_2
N, M = len(layouts1), len(layouts2)
ii, jj = np.meshgrid(range(N), range(M))
ii, jj = ii.flatten(), jj.flatten()
scores = np.asarray(
[
_compute_maximum_iou_for_layout(layouts1[i], layouts2[j])
for i, j in zip(ii, jj)
]
)
scores = scores.reshape(N, M)
ii, jj = linear_sum_assignment(scores, maximize=True)
return scores[ii, jj]
def _get_cond_to_layouts(layouts: List[Layout]) -> Dict[str, List[Layout]]:
out = defaultdict(list)
for layout in layouts:
bboxes = layout["bboxes"]
categories = layout["categories"]
# e.g., [18, 2, 1, 20, 0, 0, 0, 0, 0, 9, 9, 5, 0, 5, 0, 0]
# -> "[0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 5, 5, 9, 9, 18, 20]"
cond_key = str(sorted(categories))
categories = np.array(categories)
layout_dict: Layout = {
"bboxes": np.asarray(bboxes),
"categories": np.asarray(categories),
}
out[cond_key].append(layout_dict)
return out
def compute_maximum_iou(args):
return [_compute_maximum_iou(a) for a in args]
class LayoutMaximumIoU(evaluate.Metric):
def _info(self) -> evaluate.EvaluationModuleInfo:
return evaluate.EvaluationModuleInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=ds.Features(
{
"layouts1": {
"bboxes": ds.Sequence(ds.Sequence((ds.Value("float64")))),
"categories": ds.Sequence(ds.Value("int64")),
},
"layouts2": {
"bboxes": ds.Sequence(ds.Sequence((ds.Value("float64")))),
"categories": ds.Sequence(ds.Value("int64")),
},
}
),
codebase_urls=[
"https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L206-L247",
"https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L250-L297",
"https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L300-L314",
"https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L317-L329",
"https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L332-L340",
"https://github.com/CyberAgentAILab/layout-dm/blob/main/src/trainer/trainer/helpers/metric.py#L343-L371",
],
)
def _compute(
self,
*,
layouts1: List[Layout],
layouts2: List[Layout],
) -> float:
c2bl_1 = _get_cond_to_layouts(layouts1)
keys_1 = set(c2bl_1.keys())
c2bl_2 = _get_cond_to_layouts(layouts2)
keys_2 = set(c2bl_2.keys())
keys = list(keys_1.intersection(keys_2))
args = [(c2bl_1[key], c2bl_2[key]) for key in keys]
# to check actual number of layouts for evaluation
# ans = 0
# for x in args:
# ans += len(x[0])
scores = compute_maximum_iou(args)
scores = np.asarray(list(chain.from_iterable(scores)))
return scores.mean().item() if len(scores) != 0 else 0.0
|