Upload Files

app.py

@@ -0,0 +1,82 @@
+import gradio as gr
+import numpy as np
+
+from src.part1 import create_Gaussian_kernel, create_hybrid_image
+
+input_pictures_examples = [
+    ["./data/1a_dog.bmp", "./data/1b_cat.bmp", 5],
+    ["./data/2a_motorcycle.bmp", "./data/2b_bicycle.bmp", 5],
+    ["./data/3a_plane.bmp", "./data/3b_bird.bmp", 5],
+    ["./data/4a_einstein.bmp", "./data/4b_marilyn.bmp", 5],
+    ["./data/5a_submarine.bmp", "./data/5b_fish.bmp", 5],
+    ["./data/Message+1.png", "./data/Message+2.png", 5],
+]
+
+
+def action_hybrid_image(a, b, c):
+    # check input is None
+    if a is None or b is None or c is None:
+        return None, None, None
+
+    l, h, lh = create_hybrid_image(a / 255, b / 255, c / 255)
+
+    l, h, lh = (l * 255).astype(np.uint8), ((h + 0.5) * 255).astype(np.uint8), (lh * 255).astype(np.uint8),
+
+    return l, h, lh, lh, lh, lh, lh, lh
+
+
+with gr.Blocks(theme=gr.themes.Soft()) as app:
+    gr.Markdown("## Input")
+    with gr.Row() as uploader:
+        # allow user to upload 2 images TODO: Limit picture size, button: exchange pictures
+        image1 = gr.Image(label="Image 1 (Low Frequency)", interactive=True)
+
+        # Gaussian Kernel
+        with gr.Column() as k:
+            # allow user to select a cutoff frequency
+            cutoff = gr.Slider(1, 7, 5, step=1, label="Cutoff Frequency")
+            # display the kernel
+            kernel = gr.Image(label="Kernel", image_mode="L")
+
+        image2 = gr.Image(label="Image 2 (High Frequency)", interactive=True)
+
+        cutoff.release(create_Gaussian_kernel,
+                       inputs=cutoff, outputs=kernel, api_name="draw_gaussian_kernel")
+
+    submit = gr.Button("Submit", variant="primary")
+
+    gr.Markdown("## Output")
+    # output
+    with gr.Row() as output:
+        low_frequencies = gr.Image(label="Low Frequencies")
+        hybrid_image = gr.Image(label="Hybrid Image")
+        high_frequencies = gr.Image(label="High Frequencies")
+
+    gr.Markdown("## Outputs In Different Sizes")
+    with gr.Row() as output:
+        hybrid_large = gr.Image(label="Large", show_label=False).style(height=400, width=400)
+        hybrid_medium = gr.Image(label="Medium", show_label=False).style(height=300, width=300)
+        hybrid_small = gr.Image(label="Small", show_label=False).style(height=200, width=200)
+        hybrid_tiny = gr.Image(label="Tiny", show_label=False).style(height=100, width=100)
+        hybrid_super_tiny = gr.Image(label="Super Tiny", show_label=False).style(height=50, width=50)
+
+    submit.click(
+        action_hybrid_image,
+        inputs=[image1, image2, kernel],
+        outputs=[
+            low_frequencies, high_frequencies, hybrid_image,
+            hybrid_large, hybrid_medium, hybrid_small, hybrid_tiny, hybrid_super_tiny
+        ]
+    )
+
+    gr.Markdown("## Use Examples")
+
+    gr.Examples(
+        examples=input_pictures_examples,
+        inputs=[image1, image2, cutoff],
+        # outputs=txt_3,
+        # fn=combine,
+        # cache_examples=True, # cache examples to local storage
+    )
+
+app.launch(server_port=3030)

requirements.txt

@@ -0,0 +1,5 @@
+torch
+torchvision
+numpy
+matplotlib
+Pillow

src/cutoff_frequencies.txt

@@ -0,0 +1,5 @@
+7
+5
+5
+3
+4

src/cutoff_frequencies_temp.txt

@@ -0,0 +1,5 @@
+7
+7
+7
+7
+7

src/datasets.py

@@ -0,0 +1,132 @@
+import numpy as np
+import os
+import torch
+import torch.utils.data as data
+import torchvision.transforms as transforms
+
+from PIL import Image
+from typing import List, Tuple
+
+
+def make_dataset(path: str) -> Tuple[List[str], List[str]]:
+    """
+    Creates a dataset of paired images from a directory.
+
+    The dataset should be partitioned into two sets: one contains images that
+    will have the low pass filter applied, and the other contains images that
+    will have the high pass filter applied.
+
+    Args
+    - path: string specifying the directory containing images
+    Returns
+    - images_a: list of strings specifying the paths to the images in set A,
+        in lexicographically-sorted order
+    - images_b: list of strings specifying the paths to the images in set B,
+        in lexicographically-sorted order
+    """
+    images_a = []
+    images_b = []
+
+    all_images = os.listdir(path)
+    for image in all_images:
+        if image[1] == 'a':
+            images_a.append(os.path.join(path, image))
+        else:
+            images_b.append(os.path.join(path, image))
+    images_a, images_b = np.sort(images_a), np.sort(images_b)
+    return images_a, images_b
+
+
+def get_cutoff_frequencies(path: str) -> List[int]:
+    """
+    Gets the cutoff frequencies corresponding to each pair of images.
+
+    The cutoff frequencies are the values you discovered from experimenting in
+    part 1.
+
+    Args
+    - path: string specifying the path to the .txt file with cutoff frequency
+        values
+    Returns
+    - cutoff_frequencies: numpy array of ints. The array should have the same
+        length as the number of image pairs in the dataset
+    """
+
+    cutoff_frequencies = []
+    freq_file = open(path, 'r')
+    freqs = freq_file.readlines()
+    for freq in freqs:
+        freq = int(freq.strip())
+        cutoff_frequencies.append(freq)
+    cutoff_frequencies = np.array(cutoff_frequencies)
+    return cutoff_frequencies
+
+
+class HybridImageDataset(data.Dataset):
+    """Hybrid images dataset."""
+
+    def __init__(self, image_dir: str, cf_file: str) -> None:
+        """
+        HybridImageDataset class constructor.
+
+        You must replace self.transform with the appropriate transform from
+        torchvision.transforms that converts a PIL image to a torch Tensor. You can
+        specify additional transforms (e.g. image resizing) if you want to, but
+        it's not necessary for the images we provide you since each pair has the
+        same dimensions.
+
+        Args:
+        - image_dir: string specifying the directory containing images
+        - cf_file: string specifying the path to the .txt file with cutoff
+        frequency values
+        """
+        images_a, images_b = make_dataset(image_dir)
+        cutoff_frequencies = get_cutoff_frequencies(cf_file)
+
+        self.transform = transforms.Compose([transforms.ToTensor()])
+
+        self.images_a = images_a
+        self.images_b = images_b
+        self.cutoff_frequencies = cutoff_frequencies
+
+    def __len__(self) -> int:
+        """Returns number of pairs of images in dataset."""
+        return len(self.images_a)
+
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor, int]:
+        """
+        Returns the pair of images and corresponding cutoff frequency value at
+        index `idx`.
+
+        Since self.images_a and self.images_b contain paths to the images, you
+        should read the images here and normalize the pixels to be between 0 and 1.
+        Make sure you transpose the dimensions so that image_a and image_b are of
+        shape (c, m, n) instead of the typical (m, n, c), and convert them to
+        torch Tensors.
+
+        Args
+        - idx: int specifying the index at which data should be retrieved
+        Returns
+        - image_a: Tensor of shape (c, m, n)
+        - image_b: Tensor of shape (c, m, n)
+        - cutoff_frequency: int specifying the cutoff frequency corresponding to
+        (image_a, image_b) pair
+
+        HINTS:
+        - You should use the PIL library to read images
+        - You will use self.transform to convert the PIL image to a torch Tensor
+        """
+
+        image_a_dir = self.images_a[idx]
+        image_b_dir = self.images_b[idx]
+        cutoff_frequency = self.cutoff_frequencies[idx]
+
+        image_a = Image.open(image_a_dir)
+        image_b = Image.open(image_b_dir)
+        pixels_a = np.array(image_a) / 255.0
+        pixels_b = np.array(image_b) / 255.0
+
+        image_a = self.transform(pixels_a).float()
+        image_b = self.transform(pixels_b).float()
+
+        return image_a, image_b, cutoff_frequency

src/models.py

@@ -0,0 +1,117 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from src.part1 import create_Gaussian_kernel
+
+
+class HybridImageModel(nn.Module):
+    def __init__(self):
+        """
+        Initializes an instance of the HybridImageModel class.
+        """
+        super(HybridImageModel, self).__init__()
+
+    def get_kernel(self, cutoff_frequency: int) -> torch.Tensor:
+        """
+        Returns a Gaussian kernel using the specified cutoff frequency.
+
+        PyTorch requires the kernel to be of a particular shape in order to apply
+        it to an image. Specifically, the kernel needs to be of shape (c, 1, k, k)
+        where c is the # channels in the image. Start by getting a 2D Gaussian
+        kernel using your implementation from Part 1, which will be of shape
+        (k, k). Then, let's say you have an RGB image, you will need to turn this
+        into a Tensor of shape (3, 1, k, k) by stacking the Gaussian kernel 3
+        times.
+
+        Args
+        - cutoff_frequency: int specifying cutoff_frequency
+        Returns
+        - kernel: Tensor of shape (c, 1, k, k) where c is # channels
+
+        HINTS:
+        - You will use the create_Gaussian_kernel() function from part1.py in this
+        function.
+        - Since the # channels may differ across each image in the dataset, make
+        sure you don't hardcode the dimensions you reshape the kernel to. There
+        is a variable defined in this class to give you channel information.
+        - You can use np.reshape() to change the dimensions of a numpy array.
+        - You can use np.tile() to repeat a numpy array along specified axes.
+        - You can use torch.Tensor() to convert numpy arrays to torch Tensors.
+        """
+
+        kernel = create_Gaussian_kernel(cutoff_frequency)
+        c = self.n_channels
+        k = kernel.shape[0]
+
+        kernel = np.reshape(kernel, (1, k ** 2))
+        kernel = np.tile(kernel, c)
+        kernel = np.reshape(kernel, (c, 1, k, k))
+        kernel = torch.Tensor(kernel)
+        return kernel
+
+    def low_pass(self, x, kernel):
+        """
+        Applies low pass filter to the input image.
+
+        Args:
+        - x: Tensor of shape (b, c, m, n) where b is batch size
+        - kernel: low pass filter to be applied to the image
+        Returns:
+        - filtered_image: Tensor of shape (b, c, m, n)
+
+        HINT:
+        - You should use the 2d convolution operator from torch.nn.functional.
+        - Make sure to pad the image appropriately (it's a parameter to the
+        convolution function you should use here!).
+        - Pass self.n_channels as the value to the "groups" parameter of the
+        convolution function. This represents the # of channels that the filter
+        will be applied to.
+        """
+
+        k = kernel.shape[2]
+        filtered_image = F.conv2d(input=x.float(),
+                                  weight=kernel,
+                                  padding=k // 2,
+                                  groups=self.n_channels)
+        return filtered_image
+
+    def forward(self, image1, image2, cutoff_frequency):
+        """
+        Takes two images and creates a hybrid image. Returns the low frequency
+        content of image1, the high frequency content of image 2, and the hybrid
+        image.
+
+        Args
+        - image1: Tensor of shape (b, c, m, n)
+        - image2: Tensor of shape (b, c, m, n)
+        - cutoff_frequency: Tensor of shape (b)
+        Returns:
+        - low_frequencies: Tensor of shape (b, c, m, n)
+        - high_frequencies: Tensor of shape (b, c, m, n)
+        - hybrid_image: Tensor of shape (b, c, m, n)
+
+        HINTS:
+        - You will use the get_kernel() function and your low_pass() function in
+        this function.
+        - Similar to Part 1, you can get just the high frequency content of an
+        image by removing its low frequency content.
+        - Don't forget to make sure to clip the pixel values >=0 and <=1. You can
+        use torch.clamp().
+        - If you want to use images with different dimensions, you should resize
+        them in the HybridImageDataset class using torchvision.transforms.
+        """
+        self.n_channels = image1.shape[1]
+
+        kernel = self.get_kernel(int(cutoff_frequency.item()))
+
+        low_freq_1 = self.low_pass(image1, kernel)
+        low_freq_2 = self.low_pass(image2, kernel)
+
+        high_freq_2 = image2.float() - low_freq_2
+
+        hybrid_image = torch.clamp(low_freq_1 + high_freq_2, 0, 1)
+        low_frequencies = low_freq_1
+        high_frequencies = high_freq_2
+        return low_frequencies, high_frequencies, hybrid_image

src/part1.py

@@ -0,0 +1,127 @@
+import numpy as np
+
+
+def create_Gaussian_kernel(cutoff_frequency):
+    """
+        Returns a 2D Gaussian kernel using the specified filter size standard
+        deviation and cutoff frequency.
+
+        The kernel should have:
+        - shape (k, k) where k = cutoff_frequency * 4 + 1
+        - mean = floor(k / 2)
+        - standard deviation = cutoff_frequency
+        - values that sum to 1
+
+        Args:
+        - cutoff_frequency: an int controlling how much low frequency to leave in
+          the image.
+        Returns:
+        - kernel: numpy nd-array of shape (k, k)
+
+        HINT:
+        - The 2D Gaussian kernel here can be calculated as the outer product of two
+          vectors with values populated from evaluating the 1D Gaussian PDF at each
+          corrdinate.
+    """
+    k = cutoff_frequency * 4 + 1
+    mean = np.floor(k / 2)
+    std = cutoff_frequency
+    gauss_1d = np.zeros((k, 1))
+
+    total = 0
+    index = 0
+    for x in range(-int(mean), int(mean) + 1):
+        x1 = 1 / np.sqrt(2 * np.pi * std ** 2)
+        x2 = np.exp(-(x ** 2) / (2 * std ** 2))
+        g = x1 * x2
+        gauss_1d[index] = g
+        index += 1
+        total += g
+    kernel = np.outer(gauss_1d, gauss_1d) / total ** 2
+    return kernel
+
+
+def my_imfilter(image, filter):
+    """
+    Apply a filter to an image. Return the filtered image.
+
+    Args
+    - image: numpy nd-array of shape (m, n, c)
+    - filter: numpy nd-array of shape (k, j)
+    Returns
+    - filtered_image: numpy nd-array of shape (m, n, c)
+
+    HINTS:
+    - You may not use any libraries that do the work for you. Using numpy to work
+    with matrices is fine and encouraged. Using OpenCV or similar to do the
+    filtering for you is not allowed.
+    - I encourage you to try implementing this naively first, just be aware that
+    it may take an absurdly long time to run. You will need to get a function
+    that takes a reasonable amount of time to run so that the TAs can verify
+    your code works.
+    """
+    m = image.shape[0]
+    n = image.shape[1]
+    c = image.shape[2]
+
+    padding_height = filter.shape[0] // 2
+    padding_width = filter.shape[1] // 2
+
+    # padding manually
+    padded_image = np.zeros((m + padding_height * 2, n + padding_width * 2, c))
+    padded_image[padding_height:padding_height + m, padding_width:padding_width + n, :] = image
+
+    # convolution
+    filtered_image = np.zeros((m, n, c))
+
+    for a in range(0, c):
+        for i in range(0, m):
+            for j in range(0, n):
+                x = np.multiply(padded_image[i: i + filter.shape[0], j:j + filter.shape[1], a], filter)
+                filtered_image[i, j, a] = x.sum()
+
+    return filtered_image
+
+
+def create_hybrid_image(image1, image2, filter):
+    """
+    Takes two images and a low-pass filter and creates a hybrid image. Returns
+    the low frequency content of image1, the high frequency content of image 2,
+    and the hybrid image.
+
+    Args
+    - image1: numpy nd-array of dim (m, n, c)
+    - image2: numpy nd-array of dim (m, n, c)
+    - filter: numpy nd-array of dim (x, y)
+    Returns
+    - low_frequencies: numpy nd-array of shape (m, n, c)
+    - high_frequencies: numpy nd-array of shape (m, n, c)
+    - hybrid_image: numpy nd-array of shape (m, n, c)
+
+    HINTS:
+    - You will use your my_imfilter function in this function.
+    - You can get just the high frequency content of an image by removing its low
+      frequency content. Think about how to do this in mathematical terms.
+    - Don't forget to make sure the pixel values of the hybrid image are between
+      0 and 1. This is known as 'clipping'.
+    - If you want to use images with different dimensions, you should resize them
+      in the notebook code.
+    """
+    assert image1.shape[0] == image2.shape[0]
+    assert image1.shape[1] == image2.shape[1]
+    assert image1.shape[2] == image2.shape[2]
+    assert filter.shape[0] <= image1.shape[0]
+    assert filter.shape[1] <= image1.shape[1]
+    assert filter.shape[0] % 2 == 1
+    assert filter.shape[1] % 2 == 1
+
+    image1_low = my_imfilter(image1, filter)
+    image2_low = my_imfilter(image2, filter)
+    image2_high = image2 - image2_low
+
+    hybrid_image = np.clip(image1_low + image2_high, 0, 1)
+
+    low_frequencies = image1_low
+    high_frequencies = image2_high
+
+    return low_frequencies, high_frequencies, hybrid_image

src/unit_test.py

@@ -0,0 +1,584 @@
+#!/usr/bin/python3
+
+import numpy as np
+from pathlib import Path
+import torch
+import sys
+
+sys.path.append("../")
+from src.part1 import my_imfilter
+from src.datasets import HybridImageDataset
+from src.models import HybridImageModel, create_Gaussian_kernel
+from src.utils import (
+    vis_image_scales_numpy,
+    im2single,
+    single2im,
+    load_image,
+    save_image,
+    write_objects_to_file
+)
+
+ROOT = Path(__file__).resolve().parent.parent  # ../..
+
+"""
+Even size kernels are not required for this project, so we exclude this test case.
+"""
+
+
+def get_dog_img():
+    """
+    """
+    dog_img_fpath = f'{ROOT}/data/1a_dog.bmp'
+    dog_img = load_image(dog_img_fpath)
+    return dog_img
+
+
+def test_dataloader_len():
+    """
+    Check dataloader __len__ for correct size (should be 5 pairs of images).
+    """
+    img_dir = f'{ROOT}/data'
+    cut_off_file = f'{ROOT}/cutoff_frequencies.txt'
+    hid = HybridImageDataset(img_dir, cut_off_file)
+    assert len(hid) == 5
+
+
+def test_dataloader_get_item():
+    """
+    Verify that __getitem__ is implemented correctly, for the first dog/cat entry.
+    """
+    img_dir = f'{ROOT}/data'
+    cut_off_file = f'{ROOT}/cutoff_frequencies.txt'
+    hid = HybridImageDataset(img_dir, cut_off_file)
+
+    first_item = hid[0]
+    dog_img, cat_img, cutoff = first_item
+
+    gt_size = [3, 361, 410]
+    # low frequency should be 1a_dog.bmp, high freq should be cat
+    assert [dog_img.shape[i] for i in range(3)] == gt_size
+    assert [cat_img.shape[i] for i in range(3)] == gt_size
+
+    # ground truth values
+    dog_img_crop = torch.tensor(
+        [
+            [[0.4784, 0.4745],
+             [0.5255, 0.5176]],
+
+            [[0.4627, 0.4667],
+             [0.5098, 0.5137]],
+
+            [[0.4588, 0.4706],
+             [0.5059, 0.5059]]
+        ]
+    )
+    assert torch.allclose(dog_img[:, 100:102, 100:102], dog_img_crop, atol=1e-3)
+    assert 0. < cutoff < 1000.
+
+
+def test_low_pass_filter_square_kernel():
+    """
+        Allow students to use arbitrary padding types without penalty.
+    """
+    dog_img = get_dog_img()
+    img_h, img_w, _ = dog_img.shape
+    low_pass_filter = create_Gaussian_kernel(cutoff_frequency=7)
+    k_h, k_w = low_pass_filter.shape
+    student_filtered_img = my_imfilter(dog_img, low_pass_filter)
+
+    # Exclude the border pixels.
+    student_filtered_img_interior = student_filtered_img[k_h:img_h - k_h, k_w:img_w - k_w]
+    assert np.allclose(158332.02, student_filtered_img_interior.sum())
+
+
+def test_random_filter_nonsquare_kernel():
+    """
+        Test a non-square filter (that is not a low-pass filter).
+    """
+    image = np.array(range(10 * 15 * 3), dtype=np.uint8)
+    image = image.reshape(10, 15, 3)
+    image = image.astype(np.float32)
+    kernel = np.array(range(3 * 5), dtype=np.float32).reshape(3, 5) / 15
+    img_h, img_w, _ = image.shape
+
+    student_output = my_imfilter(image, kernel)
+
+    h_center = img_h // 2
+    w_center = img_w // 2
+
+    gt_center_crop = np.array(
+        [
+            [[1542.0001, 1549., 1556.0001],
+             [1563., 1569.9999, 1577.0001]],
+
+            [[832.99994, 840.00006, 847.],
+             [854., 861., 868.0001]]
+        ], dtype=np.float32
+    )
+
+    student_center_crop = student_output[h_center - 1:h_center + 1, w_center - 1:w_center + 1]
+    assert np.allclose(student_center_crop, gt_center_crop, atol=1e-3)
+
+    student_filtered_interior = student_output[1:img_h - 1, 3:img_w - 3, :]
+    assert np.allclose(student_filtered_interior.sum(), 194196.0, atol=1e-1)
+
+
+def test_random_filter_square_kernel():
+    """
+        Test a square filter (that is not a low-pass filter).
+    """
+    image = np.array(range(4 * 5 * 3), dtype=np.uint8)
+    image = image.reshape(4, 5, 3)
+    image = image.astype(np.float32)
+    kernel = np.array(range(3 * 3), dtype=np.float32).reshape(3, 3) / 9
+    img_h, img_w, _ = image.shape
+
+    student_output = my_imfilter(image, kernel)
+
+    student_filtered_interior = student_output[1:img_h - 1, 1:img_w - 1, :]
+    gt_interior_values = np.array(
+        [
+            [[104., 108., 112.],
+             [116., 120.00001, 124.],
+             [128., 132., 136.]],
+
+            [[164., 168.00002, 172.],
+             [176., 180., 184.],
+             [188.00002, 192., 196.]]
+        ], dtype=np.float32
+    )
+    assert np.allclose(student_filtered_interior, gt_interior_values)
+
+
+def verify_low_freq_sq_kernel_np(image1, kernel, low_frequencies) -> bool:
+    """
+        Interactive test to be used in IPython notebook, that will print out
+        test result, and return value can also be queried for success (true).
+
+        Args:
+        -	image1
+        -	kernel
+        -	low_frequencies
+
+        Returns:
+        -	Boolean indicating success.
+    """
+    gt_image1 = load_image(f'{ROOT}/data/1a_dog.bmp')
+    if not np.allclose(image1, gt_image1):
+        print('Please pass in the dog image `1a_dog.bmp` as the `image1` argument.')
+        return False
+
+    img_h, img_w, _ = image1.shape
+    k_h, k_w = kernel.shape
+    # Exclude the border pixels.
+    low_freq_interior = low_frequencies[k_h:img_h - k_h, k_w:img_w - k_w]
+    correct_sum = np.allclose(158332.02, low_freq_interior.sum())
+
+    # ground truth values
+    gt_low_freq_crop = np.array(
+        [
+            [[0.53500533, 0.523871, 0.5142517],
+             [0.5367106, 0.526209, 0.51830757]],
+
+            [[0.53472066, 0.5236291, 0.5149963],
+             [0.5368732, 0.5264317, 0.5193449]]
+        ], dtype=np.float32
+    )
+
+    # H,W,C order in Numpy
+    correct_crop = np.allclose(low_frequencies[100:102, 100:102, :], gt_low_freq_crop, atol=1e-3)
+    if correct_sum and correct_crop:
+        print('Success! Low frequencies values are correct.')
+        return True
+    else:
+        print('Low frequencies values are not correct, please double check your implementation.')
+        return False
+
+
+## Purely for visualization/debugging ########
+# plt.subplot(1,2,1)
+# plt.imshow(image1)
+
+# plt.subplot(1,2,2)
+# plt.imshow(low_frequencies)
+# plt.show()
+##############################################
+
+
+def verify_high_freq_sq_kernel_np(image2, kernel, high_frequencies) -> bool:
+    """
+        Interactive test to be used in IPython notebook, that will print out
+        test result, and return value can also be queried for success (true).
+
+        Args:
+        -	image2: Array representing the cat image (1b_cat.bmp)
+        -	kernel: Low pass kernel (2d Gaussian)
+        -	high_frequencies: High frequencies of image2 (output of high-pass filter)
+
+        Returns:
+        -	retval: Boolean indicating success.
+    """
+    gt_image2 = load_image(f'{ROOT}/data/1b_cat.bmp')
+    if not np.allclose(image2, gt_image2):
+        print('Please pass in the cat image `1b_cat.bmp` as the `image2` argument.')
+        return False
+
+    img_h, img_w, _ = image2.shape
+    k_h, k_w = kernel.shape
+    # Exclude the border pixels.
+    high_freq_interior = high_frequencies[k_h:img_h - k_h, k_w:img_w - k_w]
+    correct_sum = np.allclose(12.029784, high_freq_interior.sum(), atol=1e-2)
+
+    # ground truth values
+    gt_high_freq_crop = np.array(
+        [
+            [[7.9535842e-03, 2.9861331e-02, 3.0958146e-02],
+             [-7.6553226e-03, 2.2351682e-02, 2.7430430e-02]],
+
+            [[1.5485287e-02, 3.3503681e-02, 3.0706093e-02],
+             [-6.8724155e-05, 3.3921897e-02, 3.1234175e-02]]
+        ], dtype=np.float32
+    )
+
+    # H,W,C order in Numpy
+    correct_crop = np.allclose(high_frequencies[100:102, 100:102, :], gt_high_freq_crop, atol=1e-3)
+    if correct_sum and correct_crop:
+        print('Success! High frequencies values are correct.')
+        return True
+    else:
+        print('High frequencies values are not correct, please double check your implementation.')
+        return False
+
+
+## Purely for visualization/debugging ########
+# plt.subplot(1,2,1)
+# plt.imshow(image2)
+
+# plt.subplot(1,2,2)
+# high_frequencies += 0.5 # np.clip(high_frequencies, 0., 1.0)
+# plt.imshow(high_frequencies)
+# plt.show()
+##############################################
+
+
+def verify_hybrid_image_np(image1, image2, kernel, hybrid_image) -> bool:
+    """
+        Interactive test to be used in IPython notebook, that will print out
+        test result, and return value can also be queried for success (true).
+
+        Args:
+        -	image1
+        -	image2
+        -	kernel
+        -	hybrid_image
+
+        Returns:
+        -	Boolean indicating success.
+    """
+    gt_image1 = load_image(f'{ROOT}/data/1a_dog.bmp')
+    if not np.allclose(image1, gt_image1):
+        print('Please pass in the dog image `1a_dog.bmp` as the `image1` argument.')
+        return False
+
+    gt_image2 = load_image(f'{ROOT}/data/1b_cat.bmp')
+    if not np.allclose(image2, gt_image2):
+        print('Please pass in the cat image `1b_cat.bmp` as the `image2` argument.')
+        return False
+
+    img_h, img_w, _ = image2.shape
+    k_h, k_w = kernel.shape
+    # Exclude the border pixels.
+    hybrid_interior = hybrid_image[k_h:img_h - k_h, k_w:img_w - k_w]
+    correct_sum = np.allclose(158339.52, hybrid_interior.sum())
+
+    # ground truth values
+    gt_hybrid_crop = np.array(
+        [
+            [[0.5429589, 0.55373234, 0.5452099],
+             [0.5290553, 0.5485607, 0.545738]],
+
+            [[0.55020595, 0.55713284, 0.5457024],
+             [0.5368045, 0.5603536, 0.5505791]]
+        ], dtype=np.float32
+    )
+
+    # H,W,C order in Numpy
+    correct_crop = np.allclose(hybrid_image[100:102, 100:102, :], gt_hybrid_crop, atol=1e-3)
+    if correct_sum and correct_crop:
+        print('Success! Hybrid image values are correct.')
+        return True
+    else:
+        print('Hybrid image values are not correct, please double check your implementation.')
+        return False
+
+
+## Purely for debugging/visualization ##
+# plt.imshow(hybrid_image)
+# plt.show()
+########################################
+
+
+def verify_gaussian_kernel(kernel, cutoff_frequency) -> bool:
+    """
+        Interactive test to be used in IPython notebook, that will print out
+        test result, and return value can also be queried for success (true).
+
+        Args:
+        -	kernel
+        -	cutoff_frequency
+
+        Returns:
+        -	Boolean indicating success.
+    """
+    if cutoff_frequency != 7:
+        print('Please change the cutoff_frequency back to 7 and rerun this test')
+        return False
+    if kernel.shape != (29, 29):
+        print('The kernel is not the correct size')
+        return False
+
+    kernel_h, kernel_w = kernel.shape
+    gt_kernel_crop = np.array(
+        [
+            [0.00323564, 0.00333623, 0.00337044, 0.00333623],
+            [0.00333623, 0.00343993, 0.00347522, 0.00343993],
+            [0.00337044, 0.00347522, 0.00351086, 0.00347522],
+            [0.00333623, 0.00343993, 0.00347522, 0.00343993]
+        ]
+    )
+
+    h_center = kernel_h // 2
+    w_center = kernel_w // 2
+    student_kernel_crop = kernel[h_center - 2:h_center + 2, w_center - 2:w_center + 2]
+
+    correct_crop = np.allclose(gt_kernel_crop, student_kernel_crop, atol=1e-7)
+    correct_sum = np.allclose(kernel.sum(), 1.0, atol=1e-3)
+    correct_vals = correct_crop and correct_sum
+
+    if correct_vals:
+        print('Success -- kernel values are correct.')
+        return True
+    else:
+        print('Kernel values are not correct.')
+        return False
+
+
+def test_pytorch_low_pass_filter_square_kernel():
+    """
+    Test the low pass filter, but not the output of the forward() pass.
+    """
+    hi_model = HybridImageModel()
+    img_dir = f'{ROOT}/data'
+    cut_off_file = f'{ROOT}/cutoff_frequencies_temp.txt'
+
+    # Dump to a file
+    cutoff_freqs = [7, 7, 7, 7, 7]
+    write_objects_to_file(fpath=cut_off_file, obj_list=cutoff_freqs)
+    hi_dataset = HybridImageDataset(img_dir, cut_off_file)
+
+    # should be the dog image
+    img_a, img_b, cutoff_freq = hi_dataset[0]
+    # turn CHW into NCHW
+    img_a = img_a.unsqueeze(0)
+
+    hi_model.n_channels = 3
+    kernel = hi_model.get_kernel(cutoff_freq)
+    pytorch_low_freq = hi_model.low_pass(img_a, kernel)
+
+    assert list(pytorch_low_freq.shape) == [1, 3, 361, 410]
+    assert isinstance(pytorch_low_freq, torch.Tensor)
+
+    # crop from pytorch_output[:,:,20:22,20:22]
+    gt_crop = torch.tensor(
+        [
+            [
+                [[0.7941, 0.7989],
+                 [0.7906, 0.7953]],
+
+                [[0.9031, 0.9064],
+                 [0.9021, 0.9052]],
+
+                [[0.9152, 0.9173],
+                 [0.9168, 0.9187]]
+            ]
+        ], dtype=torch.float32
+    )
+    assert torch.allclose(pytorch_low_freq[:, :, 20:22, 20:22], gt_crop, atol=1e-3)
+
+    # ground truth element sum
+    assert np.allclose(pytorch_low_freq.numpy().sum(), 209926.3481)
+
+
+def verify_low_freq_sq_kernel_pytorch(image_a, model, cutoff_freq, low_frequencies) -> bool:
+    """
+        Test the output of the forward pass.
+
+        Args:
+        -	image_a
+        -	model
+        -	cutoff_freq
+        -	low_frequencies
+
+        Returns:
+        -	None
+    """
+    if not isinstance(cutoff_freq, torch.Tensor) or not torch.allclose(cutoff_freq, torch.Tensor([7])):
+        print('Please pass a Pytorch tensor containing `7` as the cutoff frequency.')
+        return False
+
+    img_a_val_sum = float(image_a.sum())
+    if not np.allclose(img_a_val_sum, 215154.9531):
+        print('Please pass in the dog image `1a_dog.bmp` as the `image_a` argument.')
+        return False
+
+    gt_low_freq_crop = torch.tensor(
+        [
+            [[0.5350, 0.5367],
+             [0.5347, 0.5369]],
+
+            [[0.5239, 0.5262],
+             [0.5236, 0.5264]],
+
+            [[0.5143, 0.5183],
+             [0.5150, 0.5193]]
+        ]
+    )
+    correct_crop = torch.allclose(gt_low_freq_crop, low_frequencies[0, :, 100:102, 100:102], atol=1e-3)
+
+    img_h = image_a.shape[2]
+    img_w = image_a.shape[3]
+    kernel = model.get_kernel(int(cutoff_freq))
+    if not isinstance(kernel, torch.Tensor):
+        print('Kernel is not a torch tensor')
+        return False
+
+    gt_kernel_sz_list = [3, 1, 29, 29]
+    kernel_sz_list = [int(val) for val in kernel.shape]
+
+    if gt_kernel_sz_list != kernel_sz_list:
+        print('Kernel is not the correct size')
+        return False
+
+    k_h = kernel.shape[2]
+    k_w = kernel.shape[3]
+
+    # Exclude the border pixels.
+    low_freq_interior = low_frequencies[0, :, k_h:img_h - k_h, k_w:img_w - k_w]
+    correct_sum = np.allclose(158332.06, float(low_freq_interior.sum()), atol=1)
+
+    if correct_sum and correct_crop:
+        print('Success! Pytorch low frequencies values are correct.')
+        return True
+    else:
+        print('Pytorch low frequencies values are not correct, please double check your implementation.')
+        return False
+
+
+def verify_high_freq_sq_kernel_pytorch(image_b, model, cutoff_freq, high_frequencies) -> bool:
+    """
+        Test the output of the forward pass.
+
+        Args:
+        -	image_b
+        -	model
+        -	cutoff_freq
+        -	high_frequencies
+
+        Returns:
+        -	None
+    """
+    if not isinstance(cutoff_freq, torch.Tensor) or not torch.allclose(cutoff_freq, torch.Tensor([7])):
+        print('Please pass a Pytorch tensor containing `7` as the cutoff frequency.')
+        return False
+
+    img_b_val_sum = float(image_b.sum())
+    if not np.allclose(img_b_val_sum, 230960.1875, atol=5.0):
+        print('Please pass in the cat image `1b_cat.bmp` as the `image_b` argument.')
+        return False
+
+    gt_high_freq_crop = torch.tensor(
+        [
+            [[7.9527e-03, -7.6560e-03],
+             [1.5484e-02, -6.9082e-05]],
+
+            [[2.9861e-02, 2.2352e-02],
+             [3.3504e-02, 3.3922e-02]],
+
+            [[3.0958e-02, 2.7430e-02],
+             [3.0706e-02, 3.1234e-02]]
+        ]
+    )
+    correct_crop = torch.allclose(gt_high_freq_crop, high_frequencies[0, :, 100:102, 100:102], atol=1e-3)
+
+    img_h = image_b.shape[2]
+    img_w = image_b.shape[3]
+    kernel = model.get_kernel(int(cutoff_freq))
+    if not isinstance(kernel, torch.Tensor):
+        print('Kernel is not a torch tensor')
+        return False
+
+    gt_kernel_sz_list = [3, 1, 29, 29]
+    kernel_sz_list = [int(val) for val in kernel.shape]
+
+    if gt_kernel_sz_list != kernel_sz_list:
+        print('Kernel is not the correct size')
+        return False
+
+    k_h = kernel.shape[2]
+    k_w = kernel.shape[3]
+
+    # Exclude the border pixels.
+    high_freq_interior = high_frequencies[0, :, k_h:img_h - k_h, k_w:img_w - k_w]
+    correct_sum = np.allclose(12.012651, float(high_freq_interior.sum()), atol=1e-1)
+
+    if correct_sum and correct_crop:
+        print('Success! Pytorch high frequencies values are correct.')
+        return True
+    else:
+        print('Pytorch high frequencies values are not correct, please double check your implementation.')
+        return False
+
+
+def verify_hybrid_image_pytorch(image_a, image_b, model, cutoff_freq, hybrid_image) -> bool:
+    """
+        Test the output of the forward pass.
+
+        Args:
+        -	image_a
+        -	image_b
+        -	model
+        -	cutoff_freq
+        -	hybrid_image
+
+        Returns:
+        -	None
+    """
+    _, _, img_h, img_w = image_b.shape
+    kernel = model.get_kernel(int(cutoff_freq))
+    _, _, k_h, k_w = kernel.shape
+
+    # Exclude the border pixels.
+    hybrid_interior = hybrid_image[0, :, k_h:img_h - k_h, k_w:img_w - k_w]
+    correct_sum = np.allclose(158339.5469, hybrid_interior.sum(), atol=1e-2)
+
+    # ground truth values
+    gt_hybrid_crop = torch.tensor(
+        [
+            [[0.5430, 0.5291],
+             [0.5502, 0.5368]],
+
+            [[0.5537, 0.5486],
+             [0.5571, 0.5604]],
+
+            [[0.5452, 0.5457],
+             [0.5457, 0.5506]]
+        ]
+    )
+    # H,W,C order in Numpy
+    correct_crop = torch.allclose(hybrid_image[0, :, 100:102, 100:102], gt_hybrid_crop, atol=1e-3)
+    if correct_sum and correct_crop:
+        print('Success! Pytorch hybrid image values are correct.')
+        return True
+    else:
+        print('Pytorch hybrid image values are not correct, please double check your implementation.')
+        return False

src/utils.py

@@ -0,0 +1,174 @@
+import copy
+import numpy as np
+import PIL
+
+from typing import Any, List, Tuple
+
+
+def PIL_resize(img: np.ndarray, size: Tuple[int, int]) -> np.ndarray:
+    """
+      Args:
+      - img: Array representing an image
+      - size: Tuple representing new desired (width, height)
+
+      Returns:
+      - img
+    """
+    img = numpy_arr_to_PIL_image(img, scale_to_255=True)
+    img = img.resize(size)
+    img = PIL_image_to_numpy_arr(img)
+    return img
+
+
+def PIL_image_to_numpy_arr(img, downscale_by_255=True):
+    """
+      Args:
+      - img
+      - downscale_by_255
+
+      Returns:
+      - img
+    """
+    img = np.asarray(img)
+    img = img.astype(np.float32)
+    if downscale_by_255:
+        img /= 255
+    return img
+
+
+def vis_image_scales_numpy(image: np.ndarray) -> np.ndarray:
+    """
+      This function will display an image at different scales (zoom factors). The
+      original image will appear at the far left, and then the image will
+      iteratively be shrunk by 2x in each image to the right.
+
+      This is a particular effective way to simulate the perspective effect, as
+      if viewing an image at different distances. We thus use it to visualize
+      hybrid images, which represent a combination of two images, as described
+      in the SIGGRAPH 2006 paper "Hybrid Images" by Oliva, Torralba, Schyns.
+
+      Args:
+      - image: Array of shape (H, W, C)
+
+      Returns:
+      - img_scales: Array of shape (M, K, C) representing horizontally stacked
+        images, growing smaller from left to right.
+        K = W + int(1/2 W + 1/4 W + 1/8 W + 1/16 W) + (5 * 4)
+    """
+    original_height = image.shape[0]
+    original_width = image.shape[1]
+    num_colors = 1 if image.ndim == 2 else 3
+    img_scales = np.copy(image)
+    cur_image = np.copy(image)
+
+    scales = 5
+    scale_factor = 0.5
+    padding = 5
+
+    new_h = original_height
+    new_w = original_width
+
+    for scale in range(2, scales + 1):
+        # add padding
+        img_scales = np.hstack((img_scales,
+                                np.ones((original_height, padding, num_colors), dtype=np.float32))
+                               )
+
+        new_h = int(scale_factor * new_h)
+        new_w = int(scale_factor * new_w)
+        # downsample image iteratively
+        cur_image = PIL_resize(cur_image, size=(new_w, new_h))
+
+        # pad the top to append to the output
+        h_pad = original_height - cur_image.shape[0]
+        pad = np.ones((h_pad, cur_image.shape[1], num_colors), dtype=np.float32)
+        tmp = np.vstack((pad, cur_image))
+        img_scales = np.hstack((img_scales, tmp))
+
+    return img_scales
+
+
+def im2single(im: np.ndarray) -> np.ndarray:
+    """
+      Args:
+      - img: uint8 array of shape (m,n,c) or (m,n) and in range [0,255]
+
+      Returns:
+      - im: float or double array of identical shape and in range [0,1]
+    """
+    im = im.astype(np.float32) / 255
+    return im
+
+
+def single2im(im: np.ndarray) -> np.ndarray:
+    """
+      Args:
+      - im: float or double array of shape (m,n,c) or (m,n) and in range [0,1]
+
+      Returns:
+      - im: uint8 array of identical shape and in range [0,255]
+    """
+    im *= 255
+    im = im.astype(np.uint8)
+    return im
+
+
+def numpy_arr_to_PIL_image(img: np.ndarray, scale_to_255: False) -> PIL.Image:
+    """
+      Args:
+      - img: in [0,1]
+
+      Returns:
+      - img in [0,255]
+
+    """
+    if scale_to_255:
+        img *= 255
+    return PIL.Image.fromarray(np.uint8(img))
+
+
+def load_image(path: str) -> np.ndarray:
+    """
+      Args:
+      - path: string representing a file path to an image
+
+      Returns:
+      - float or double array of shape (m,n,c) or (m,n) and in range [0,1],
+        representing an RGB image
+    """
+    pil_img = PIL.Image.open(path)
+    img = PIL_image_to_numpy_arr(pil_img, False)
+    img = im2single(img)
+    return img
+
+
+def save_image(path: str, im: np.ndarray) -> bool:
+    """
+      Args:
+      - path: string representing a file path to an image
+      - img: numpy array
+
+      Returns:
+      - retval indicating write success
+    """
+    img = copy.deepcopy(im)
+    img = single2im(img)
+    pil_img = numpy_arr_to_PIL_image(img, scale_to_255=False)
+    return pil_img.save(path)
+
+
+def write_objects_to_file(fpath: str, obj_list: List[Any]):
+    """
+      If the list contents are float or int, convert them to strings.
+      Separate with carriage return.
+
+      Args:
+      - fpath: string representing path to a file
+      - obj_list: List of strings, floats, or integers to be written out to a file, one per line.
+
+      Returns:
+      - None
+    """
+    obj_list = [str(obj) + '\n' for obj in obj_list]
+    with open(fpath, 'w') as f:
+        f.writelines(obj_list)

webpage_martix.py

@@ -0,0 +1,90 @@
+import gradio as gr
+import numpy as np
+from tqdm import trange
+
+from src.part1 import create_Gaussian_kernel, create_hybrid_image
+
+input_pictures_examples = [
+    ["./data/1a_dog.bmp", "./data/1b_cat.bmp"],
+    ["./data/2a_motorcycle.bmp", "./data/2b_bicycle.bmp"],
+    ["./data/3a_plane.bmp", "./data/3b_bird.bmp"],
+    ["./data/4a_einstein.bmp", "./data/4b_marilyn.bmp"],
+    ["./data/5a_submarine.bmp", "./data/5b_fish.bmp"],
+    ["./data/Message+1.png", "./data/Message+2.png", 5],
+]
+
+
+def action_hybrid_image(a, b, c):
+    # check input is None
+    if a is None or b is None or c is None:
+        return None, None, None
+
+    l, h, lh = create_hybrid_image(a / 255, b / 255, c / 255)
+
+    return (l * 255).astype(np.uint8), ((h + 0.5) * 255).astype(np.uint8), (lh * 255).astype(np.uint8)
+
+
+def action_hybrid_matrix(a, b):
+    result_matrix = []
+
+    for i in trange(1, 8):
+        kernel = create_Gaussian_kernel(i)
+        l, h, lh = action_hybrid_image(a, b, kernel)
+        result_matrix.append(lh)
+
+    for i in trange(1, 8):
+        kernel = create_Gaussian_kernel(i)
+        l, h, lh = action_hybrid_image(b, a, kernel)
+        result_matrix.append(lh)
+
+    return result_matrix
+
+
+with gr.Blocks() as app:
+    gr.Markdown("## Input")
+    with gr.Row() as uploader:
+        # allow user to upload 2 images TODO: Limit picture size, button: exchange pictures
+        image1 = gr.Image(label="Image 1", interactive=True)
+
+        image2 = gr.Image(label="Image 2", interactive=True)
+
+    submit = gr.Button("Submit")
+
+    gr.Markdown("## Output")
+    # output
+    with gr.Row() as output:
+        h_1_2_1 = gr.Image(label="Hybrid Image_121")
+        h_1_2_2 = gr.Image(label="Hybrid Image_122")
+        h_1_2_3 = gr.Image(label="Hybrid Image_123")
+        h_1_2_4 = gr.Image(label="Hybrid Image_124")
+        h_1_2_5 = gr.Image(label="Hybrid Image_125")
+        h_1_2_6 = gr.Image(label="Hybrid Image_126")
+        h_1_2_7 = gr.Image(label="Hybrid Image_127")
+
+    with gr.Row() as exchanged:
+        h_2_1_1 = gr.Image(label="Hybrid Image_211")
+        h_2_1_2 = gr.Image(label="Hybrid Image_212")
+        h_2_1_3 = gr.Image(label="Hybrid Image_213")
+        h_2_1_4 = gr.Image(label="Hybrid Image_214")
+        h_2_1_5 = gr.Image(label="Hybrid Image_215")
+        h_2_1_6 = gr.Image(label="Hybrid Image_216")
+        h_2_1_7 = gr.Image(label="Hybrid Image_217")
+
+    submit.click(
+        action_hybrid_matrix,
+        inputs=[image1, image2],
+        outputs=[h_1_2_1, h_1_2_2, h_1_2_3, h_1_2_4, h_1_2_5, h_1_2_6, h_1_2_7,
+                 h_2_1_1, h_2_1_2, h_2_1_3, h_2_1_4, h_2_1_5, h_2_1_6, h_2_1_7]
+    )
+
+    gr.Markdown("## Use Examples")
+
+    gr.Examples(
+        examples=input_pictures_examples,
+        inputs=[image1, image2],
+        # outputs=txt_3,
+        # fn=combine,
+        # cache_examples=True, # cache examples to local storage
+    )
+
+app.launch(server_port=3030)