fastai/widgets/image_cleaner.py

from ..torch_core import *
from ..basic_train import *
from ..basic_data import *
from ..vision.data import *
from ..vision.transform import *
from ..vision.image import *
from ..callbacks.hooks import *
from ..layers import *
from ipywidgets import widgets, Layout
from IPython.display import clear_output, display

__all__ = ['DatasetFormatter', 'ImageCleaner']

class DatasetFormatter():
    "Returns a dataset with the appropriate format and file indices to be displayed."
    @classmethod
    def from_toplosses(cls, learn, n_imgs=None, **kwargs):
        "Gets indices with top losses."
        train_ds, train_idxs = cls.get_toplosses_idxs(learn, n_imgs, **kwargs)
        return train_ds, train_idxs

    @classmethod
    def get_toplosses_idxs(cls, learn, n_imgs, **kwargs):
        "Sorts `ds_type` dataset by top losses and returns dataset and sorted indices."
        dl = learn.data.fix_dl
        if not n_imgs: n_imgs = len(dl.dataset)
        _,_,top_losses = learn.get_preds(ds_type=DatasetType.Fix, with_loss=True)
        idxs = torch.topk(top_losses, n_imgs)[1]
        return cls.padded_ds(dl.dataset, **kwargs), idxs

    def padded_ds(ll_input, size=(250, 300), resize_method=ResizeMethod.CROP, padding_mode='zeros', **kwargs):
        "For a LabelList `ll_input`, resize each image to `size` using `resize_method` and `padding_mode`."
        return ll_input.transform(tfms=crop_pad(), size=size, resize_method=resize_method, padding_mode=padding_mode)
    
    @classmethod
    def from_similars(cls, learn, layer_ls:list=[0, 7, 2], **kwargs):
        "Gets the indices for the most similar images."
        train_ds, train_idxs = cls.get_similars_idxs(learn, layer_ls, **kwargs)
        return train_ds, train_idxs

    @classmethod
    def get_similars_idxs(cls, learn, layer_ls, **kwargs):
        "Gets the indices for the most similar images in `ds_type` dataset"
        hook = hook_output(learn.model[layer_ls[0]][layer_ls[1]][layer_ls[2]])
        dl = learn.data.fix_dl

        ds_actns = cls.get_actns(learn, hook=hook, dl=dl, **kwargs)
        similarities = cls.comb_similarity(ds_actns, ds_actns, **kwargs)
        idxs = cls.sort_idxs(similarities)
        return cls.padded_ds(dl, **kwargs), idxs

    @staticmethod
    def get_actns(learn, hook:Hook, dl:DataLoader, pool=AdaptiveConcatPool2d, pool_dim:int=4, **kwargs):
        "Gets activations at the layer specified by `hook`, applies `pool` of dim `pool_dim` and concatenates"
        print('Getting activations...')

        actns = []
        learn.model.eval()
        with torch.no_grad():
            for (xb,yb) in progress_bar(dl):
                learn.model(xb)
                actns.append((hook.stored).cpu())

        if pool:
            pool = pool(pool_dim)
            return pool(torch.cat(actns)).view(len(dl.x),-1)
        else: return torch.cat(actns).view(len(dl.x),-1)


    @staticmethod
    def comb_similarity(t1: torch.Tensor, t2: torch.Tensor, **kwargs):
        # https://github.com/pytorch/pytorch/issues/11202
        "Computes the similarity function between each embedding of `t1` and `t2` matrices."
        print('Computing similarities...')

        w1 = t1.norm(p=2, dim=1, keepdim=True)
        w2 = w1 if t2 is t1 else t2.norm(p=2, dim=1, keepdim=True)

        t = torch.mm(t1, t2.t()) / (w1 * w2.t()).clamp(min=1e-8)
        return torch.tril(t, diagonal=-1) 

    def largest_indices(arr, n):
        "Returns the `n` largest indices from a numpy array `arr`."
        #https://stackoverflow.com/questions/6910641/how-do-i-get-indices-of-n-maximum-values-in-a-numpy-array
        flat = arr.flatten()
        indices = np.argpartition(flat, -n)[-n:]
        indices = indices[np.argsort(-flat[indices])]
        return np.unravel_index(indices, arr.shape)

    @classmethod
    def sort_idxs(cls, similarities):
        "Sorts `similarities` and return the indexes in pairs ordered by highest similarity."
        idxs = cls.largest_indices(similarities, len(similarities))
        idxs = [(idxs[0][i], idxs[1][i]) for i in range(len(idxs[0]))]
        return [e for l in idxs for e in l]

class ImageCleaner():
    "Displays images for relabeling or deletion and saves changes in `path` as 'cleaned.csv'."
    def __init__(self, dataset, fns_idxs, path, batch_size:int=5, duplicates=False):
        self._all_images,self._batch = [],[]
        self._path = Path(path)
        self._batch_size = batch_size
        if duplicates: self._batch_size = 2
        self._duplicates = duplicates
        self._labels = dataset.classes
        self._all_images = self.create_image_list(dataset, fns_idxs)
        self._csv_dict = {dataset.x.items[i]: dataset.y[i] for i in range(len(dataset))}
        self._deleted_fns = []
        self._skipped = 0
        self.render()

    @classmethod
    def make_img_widget(cls, img, layout=Layout(), format='jpg'):
        "Returns an image widget for specified file name `img`."
        return widgets.Image(value=img, format=format, layout=layout)

    @classmethod
    def make_button_widget(cls, label, file_path=None, handler=None, style=None, layout=Layout(width='auto')):
        "Return a Button widget with specified `handler`."
        btn = widgets.Button(description=label, layout=layout)
        if handler is not None: btn.on_click(handler)
        if style is not None: btn.button_style = style
        btn.file_path = file_path
        btn.flagged_for_delete = False
        return btn

    @classmethod
    def make_dropdown_widget(cls, description='Description', options=['Label 1', 'Label 2'], value='Label 1',
                            file_path=None, layout=Layout(), handler=None):
        "Return a Dropdown widget with specified `handler`."
        dd = widgets.Dropdown(description=description, options=options, value=value, layout=layout)
        if file_path is not None: dd.file_path = file_path
        if handler is not None: dd.observe(handler, names=['value'])
        return dd

    @classmethod
    def make_horizontal_box(cls, children, layout=Layout()):
        "Make a horizontal box with `children` and `layout`."
        return widgets.HBox(children, layout=layout)

    @classmethod
    def make_vertical_box(cls, children, layout=Layout(), duplicates=False):
        "Make a vertical box with `children` and `layout`."
        if not duplicates: return widgets.VBox(children, layout=layout)
        else: return widgets.VBox([children[0], children[2]], layout=layout)

    def create_image_list(self, dataset, fns_idxs):
        "Create a list of images, filenames and labels but first removing files that are not supposed to be displayed."
        items = dataset.x.items
        if self._duplicates:
            chunked_idxs = chunks(fns_idxs, 2)
            chunked_idxs = [chunk for chunk in chunked_idxs if Path(items[chunk[0]]).is_file() and Path(items[chunk[1]]).is_file()]
            return  [(dataset.x[i]._repr_jpeg_(), items[i], self._labels[dataset.y[i].data]) for chunk in chunked_idxs for i in chunk]
        else:
            return [(dataset.x[i]._repr_jpeg_(), items[i], self._labels[dataset.y[i].data]) for i in fns_idxs if
                    Path(items[i]).is_file()]

    def relabel(self, change):
        "Relabel images by moving from parent dir with old label `class_old` to parent dir with new label `class_new`."
        class_new,class_old,file_path = change.new,change.old,change.owner.file_path
        fp = Path(file_path)
        parent = fp.parents[1]
        self._csv_dict[fp] = class_new

    def next_batch(self, _):
        "Handler for 'Next Batch' button click. Delete all flagged images and renders next batch."
        for img_widget, delete_btn, fp, in self._batch:
            fp = delete_btn.file_path
            if (delete_btn.flagged_for_delete == True):
                self.delete_image(fp)
                self._deleted_fns.append(fp)
        self._all_images = self._all_images[self._batch_size:]
        self.empty_batch()
        self.render()

    def on_delete(self, btn):
        "Flag this image as delete or keep."
        btn.button_style = "" if btn.flagged_for_delete else "danger"
        btn.flagged_for_delete = not btn.flagged_for_delete

    def empty_batch(self): self._batch[:] = []

    def delete_image(self, file_path):
        del self._csv_dict[file_path]

    def empty(self):
        return len(self._all_images) == 0

    def get_widgets(self, duplicates):
        "Create and format widget set."
        widgets = []
        for (img,fp,human_readable_label) in self._all_images[:self._batch_size]:
            img_widget = self.make_img_widget(img, layout=Layout(height='250px', width='300px'))
            dropdown = self.make_dropdown_widget(description='', options=self._labels, value=human_readable_label,
                                                 file_path=fp, handler=self.relabel, layout=Layout(width='auto'))
            delete_btn = self.make_button_widget('Delete', file_path=fp, handler=self.on_delete)
            widgets.append(self.make_vertical_box([img_widget, dropdown, delete_btn],
                                                  layout=Layout(width='auto', height='300px',
                                                      overflow_x="hidden"), duplicates=duplicates))
            self._batch.append((img_widget, delete_btn, fp))
        return widgets

    def batch_contains_deleted(self):
        "Check if current batch contains already deleted images."
        if not self._duplicates: return False
        imgs = [self._all_images[:self._batch_size][0][1], self._all_images[:self._batch_size][1][1]]
        return any(img in self._deleted_fns for img in imgs)

    def write_csv(self):
        # Get first element's file path so we write CSV to same directory as our data
        csv_path = self._path/'cleaned.csv'
        with open(csv_path, 'w') as f:
            csv_writer = csv.writer(f)
            csv_writer.writerow(['name','label'])
            for pair in self._csv_dict.items():
                pair = [os.path.relpath(pair[0], self._path), pair[1]]
                csv_writer.writerow(pair)
        return csv_path

    def render(self):
        "Re-render Jupyter cell for batch of images."
        clear_output()
        self.write_csv()
        if self.empty() and self._skipped>0:
            return display(f'No images to show :). {self._skipped} pairs were '
                    f'skipped since at least one of the images was deleted by the user.')
        elif self.empty():
            return display('No images to show :)')
        if self.batch_contains_deleted():
            self.next_batch(None)
            self._skipped += 1
        else:
            display(self.make_horizontal_box(self.get_widgets(self._duplicates)))
            display(self.make_button_widget('Next Batch', handler=self.next_batch, style="primary"))