codes/calibration.py

"""This module is for calibration of the Owleye. The module includes the code to collect data from the user,
while they are looking at the white point. The molude contains one class called Clb. To understand this module, you should know
about Mediapipe landmark detection."""


import numpy as np
import cv2
import time
from codes.base import eyeing as ey
import os
from datetime import datetime
if os.name == "nt":
    import winsound
elif os.name == "posix":
    pass
from sklearn.utils import shuffle
import math


INFO = ("", "M", 25, "Email: ")      # The information that goes to information.txt
CALIBRATION_GRID = (4, 200, 6, 100)         # Calibration grid

# Class for calibration
class Clb(object):
    running = True

    @staticmethod
    def create_grid(clb_grid):
        """
        This method creates the desired grid points.

        Parameters:
            clb_grid: A list
            
        Returns:
            points: A list that contains n lists
        """
        point_ratio = 0.012
        if len(clb_grid) == 2:
            # For going through just rows
            rows = clb_grid[0]
            points_in_row = clb_grid[1]
            points = []
            dy_rows = (1 - rows * point_ratio) / (rows - 1)
            dx = (1 - points_in_row * point_ratio) / (points_in_row - 1)

            for j in range(rows):
                if j == 0:
                    p_y = j * (point_ratio + dy_rows) + 4.0 * point_ratio / 3.0
                elif j == rows-1:
                    p_y = j * (point_ratio + dy_rows) - point_ratio / 3.0
                else:
                    p_y = j * (point_ratio + dy_rows) + point_ratio / 2
                smp_in_p = []
                for i in range(points_in_row):
                    if i == 0:
                        p_x = i * (point_ratio + dx) + point_ratio
                    elif i == points_in_row - 1:
                        p_x = i * (point_ratio + dx)
                    else:
                        p_x = i * (point_ratio + dx) + point_ratio / 2
                    smp_in_p.append([p_x, p_y])
                if j % 2 == 0:
                    points.append(smp_in_p)
                else:
                    smp_in_p.reverse()
                    points.append(smp_in_p)

        elif len(clb_grid) == 3:
            # For appearing stationary (not moving)
            rows = clb_grid[0]
            cols = clb_grid[1]
            smp_in_pnt = clb_grid[2]
            points = []
            dy = (1 - rows * point_ratio) / (rows - 1)
            dx = (1 - cols * point_ratio) / (cols - 1)

            for j in range(rows):
                if j == 0:
                    p_y = j * (point_ratio + dy) + 4.0 * point_ratio / 3.0
                elif j == rows - 1:
                    p_y = j * (point_ratio + dy) - point_ratio / 3.0
                else:
                    p_y = j * (point_ratio + dy) + point_ratio / 2
                for i in range(cols):
                    if i == 0:
                        p_x = i * (point_ratio + dx) + point_ratio
                    elif i == cols - 1:
                        p_x = i * (point_ratio + dx)
                    else:
                        p_x = i * (point_ratio + dx) + point_ratio / 2
                    smp_in_p = []
                    for k in range(smp_in_pnt):
                        smp_in_p.append([p_x, p_y])
                    points.append(smp_in_p)

        elif len(clb_grid) == 4:
            # For going through rows and columns. It is suggested
            rows = clb_grid[0]
            points_in_row = clb_grid[1]
            cols = clb_grid[2]
            points_in_col = clb_grid[3]
            points = []

            d_rows = (1 - rows * point_ratio) / (rows - 1)
            dx = (1 - points_in_row * point_ratio) / (points_in_row - 1)
            d_cols = (1 - cols * point_ratio) / (cols - 1)
            dy = (1 - points_in_col * point_ratio) / (points_in_col - 1)

            for j in range(rows):
                if j == 0:
                    p_y = j * (point_ratio + d_rows) + 4.0 * point_ratio / 3.0
                elif j == rows - 1:
                    p_y = j * (point_ratio + d_rows) - point_ratio / 3.0
                else:
                    p_y = j * (point_ratio + d_rows) + point_ratio / 2
                smp_in_p = []
                for i in range(points_in_row):
                    if i == 0:
                        p_x = i * (point_ratio + dx) + point_ratio
                    elif i == points_in_row - 1:
                        p_x = i * (point_ratio + dx)
                    else:
                        p_x = i * (point_ratio + dx) + point_ratio / 2
                    smp_in_p.append([p_x, p_y])
                if j % 2 == 0:
                    points.append(smp_in_p)
                else:
                    smp_in_p.reverse()
                    points.append(smp_in_p)
            for i in range(cols):
                if i == 0:
                    p_x = i * (point_ratio + d_cols) + point_ratio
                elif i == cols - 1:
                    p_x = i * (point_ratio + d_cols)
                else:
                    p_x = i * (point_ratio + d_cols) + point_ratio / 2
                smp_in_p = []
                for j in range(points_in_col):
                    if j == 0:
                        p_y = j * (point_ratio + dy) + 4.0 * point_ratio / 3.0
                    elif j == points_in_col - 1:
                        p_y = j * (point_ratio + dy) - point_ratio / 3.0
                    else:
                        p_y = j * (point_ratio + dy) + point_ratio / 2
                    smp_in_p.append([p_x, p_y])
                if i % 2 == 0:
                    points.append(smp_in_p)
                else:
                    smp_in_p.reverse()
                    points.append(smp_in_p)

        else:
            print("\nPlease Enter a vector with length of 2-4!!")
            points = None
            quit()

        return points


    def et(self, num, camera_id=0, info=INFO, clb_grid=CALIBRATION_GRID):
        """
        Collecting the data (inputs and outputs of the models)
        
        Parameters:
            num: Subject's number
            camera_id: Camera ID
            info: Subject's information
            clb_grid: Calibration grid
        
        Retruns:
            None
        
        """
        
        print("\nCalibration started!")
        
        # Some interactions with user
        name, descriptions = info
        tx0 = [["Follow WHITE point", (0.05, 0.25), 1.5, ey.RED, 3],
        ["SPACE --> start", (0.05, 0.5), 1.5, ey.RED, 3],
        ["ESC --> Stop", (0.05, 0.75), 1.5, ey.RED, 3]]
        run_app = True

        sbj_dir = ey.subjects_dir + f"{num}/"
        if os.path.exists(sbj_dir):
            tx1 = [["There is a subject in", (0.05, 0.2), 1.3, ey.RED, 2],
            [f"{sbj_dir}.", (0.05, 0.4), 1.3, ey.RED, 2],
            ["Do you want to", (0.05, 0.6), 1.3, ey.RED, 2],
            ["remove it (y/n)?", (0.05, 0.8), 1.3, ey.RED, 2]]

            win_name = "Subject exists"
            ey.big_win(win_name, 0)
            ey.show_clb_win(win_name, texts=tx1, win_color=ey.WHITE)
            button = cv2.waitKey(0)
            if button == 27 or (button == ord("q")) or (button == ord("Q")) or (button == ord("n")) or (button == ord("N")):
                run_app = False
            cv2.destroyWindow(win_name)

        if run_app:
            sbj_dir = ey.create_dir([sbj_dir])
            clb_points = self.create_grid(clb_grid)

            # Some landmarks needed for calculation of face vectors
            some_landmarks_ids = ey.get_some_landmarks_ids()

            (
                frame_size,
                camera_matrix,
                dst_cof,
                pcf
            ) = ey.get_camera_properties(camera_id)

            face_mesh = ey.get_mesh()

            fps_vec = []
            t_mat = []
            eyes_mat = []
            inp_scalars_mat = []
            points_loc_mat = []
            eyes_ratio_mat = []

            t0 = time.perf_counter()
            cap = ey.get_camera(camera_id, frame_size)
            ey.pass_frames(cap, 100)

            win_name = "Information"
            ey.big_win(win_name, math.floor(len(ey.monitors) / 2)*ey.monitors[0].width)
            ey.show_clb_win(win_name, texts=tx0, win_color=ey.WHITE)
            cv2.waitKey(10000)
            cv2.destroyWindow(win_name)

            # Going through monitors
            for (i_m, m) in enumerate(ey.monitors):
                win_name = f"Calibration-{i_m}"
                ey.big_win(win_name, i_m * m.width)
                # Going to each series of points (for example, one row of points)
                for item in clb_points:
                    if not self.running and (i_m != 0):
                        break
                    t_vec = []
                    eyes_vec = []
                    inp_scalars_vec = []
                    points_loc_vec = []
                    eyes_ratio_vec = []
                    
                    pnt = item[0]
                    ey.show_clb_win(win_name, pnt, win_color=ey.GRAY)
                    button = cv2.waitKey(0)
                    if button == 27 or (button == ord("q")) or (button == ord("Q")):
                        break
                    elif button == ord(' '):
                        ey.pass_frames(cap)
                        t1 = time.perf_counter()
                        s = len(item)

                        # Going through each point in each series
                        for pnt in item:
                            ey.show_clb_win(win_name, pnt)
                            button = cv2.waitKey(1)
                            if button == 27:
                                break
                            while True:
                                frame_success, frame, frame_rgb = ey.get_frame(cap)     # Get image
                                if frame_success:
                                    results = face_mesh.process(frame_rgb) # Predicting the landmarks using image
                                    
                                    # Getting the inputs of the models
                                    (
                                        features_success,
                                        _,
                                        eyes_frame_gray,
                                        features_vector,
                                        eyes_ratio,
                                        _
                                    ) = ey.get_model_inputs(
                                        frame,
                                        frame_rgb,
                                        results,
                                        camera_matrix,
                                        pcf,
                                        frame_size,
                                        dst_cof,
                                        some_landmarks_ids
                                    )
                                    if features_success:
                                        # Putting the inputs of the models into lists
                                        t_vec.append(round(time.perf_counter() - t1, 3))
                                        eyes_vec.append(eyes_frame_gray)
                                        inp_scalars_vec.append(features_vector)
                                        points_loc_vec.append([(pnt[0] + i_m)/len(ey.monitors), pnt[1]])
                                        eyes_ratio_vec.append(eyes_ratio)
                                        break
                            if not self.running:
                                break
                        fps_vec.append(ey.get_time(s, t1))
                        t_mat.append(np.array(t_vec))
                        eyes_mat.append(np.array(eyes_vec))
                        inp_scalars_mat.append(np.array(inp_scalars_vec))
                        points_loc_mat.append(np.array(points_loc_vec))
                        eyes_ratio_mat.append(np.array(eyes_ratio_vec))
                        
                    if not self.running:
                        break
                    if button == 27 or (button == ord("q")) or (button == ord("Q")):
                        break
                if button == 27 or (button == ord("q")) or (button == ord("Q")):
                    break
                cv2.destroyWindow(win_name)
            cap.release()
            cv2.destroyAllWindows()

            if button != 27 and (button != ord("q")) and (button != ord("Q")):
                ey.get_time(0, t0, True)
                print(f"Mean FPS : {np.array(fps_vec).mean()}")

                f = open(sbj_dir + "Information.txt", "w+")
                f.write(name + "\n" + descriptions + "\n" + str(datetime.now())[:16])
                f.close()

                et_dir = ey.create_dir([sbj_dir, ey.CLB])
                ey.save([t_mat, eyes_mat, inp_scalars_mat, points_loc_mat, eyes_ratio_mat], et_dir, [ey.T, ey.X1, ey.X2, ey.Y, ey.ER])

        else:
            self.running = False


    @staticmethod
    def make_io(num, data_out):
        """
        Mixing the data of calibration and out looking, to create a dataset of in-out
        
        Parameters:
            data_out: data of user's looking at outside of the screen
        
        Returns:
            None
        """
        sbj_dir = ey.create_dir([ey.subjects_dir, f"{num}"])
        et_dir = ey.create_dir([sbj_dir, ey.CLB])

        x1_et0, x2_et0 = ey.load(et_dir, [ey.X1, ey.X2])
        x1_et = []
        x2_et = []
        for (x1_vec, x2_vec) in zip(x1_et0, x2_et0):
            for (x10, x20) in zip(x1_vec, x2_vec):
                x1_et.append(x10)
                x2_et.append(x20)
        x1_et = np.array(x1_et)
        x2_et = np.array(x2_et)

        x1_o, x2_o, y_o = data_out
        smp_in_cls = int(x1_o.shape[0])

        x1_et_shf, x2_et_shf = shuffle(x1_et, x2_et)

        x1_i, x2_i = x1_et_shf[:smp_in_cls], x2_et_shf[:smp_in_cls]
        y_i = np.zeros((smp_in_cls,))

        x1_io = [np.concatenate((x1_i, x1_o))]
        x2_io = [np.concatenate((x2_i, x2_o))]
        y_io = [np.concatenate((y_i, y_o))]

        io_dir = ey.create_dir([sbj_dir, ey.IO])
        ey.save([x1_io, x2_io, y_io], io_dir, [ey.X1, ey.X2, ey.Y])


    def out(self, num, camera_id=0, n_smp_in_cls=300):
        """
        Collecting data while the user is looking out of the screen
        
        Parameters:
            num: Subject number
            camera_id: Camera ID
            n_smp_in_cls: The number of samples for each class
        
        Returns:
            None
        """
        print("Getting out data...")
        out_class_num = 1

        some_landmarks_ids = ey.get_some_landmarks_ids()

        (
            frame_size,
            camera_matrix,
            dst_cof,
            pcf
        ) = ey.get_camera_properties(camera_id)

        face_mesh = ey.get_mesh()

        t0 = time.perf_counter()
        eyes_data_gray = []
        vector_inputs = []
        output_class = []
        fps_vec = []
        cap = ey.get_camera(camera_id, frame_size)
        ey.pass_frames(cap, 100)
        tx0 = [["Look everywhere ", (0.05, 0.25), 1.3, ey.RED, 3],
        ["'out' of screen", (0.05, 0.5), 1.3, ey.RED, 3],
        ["SPACE --> start sampling", (0.05, 0.75), 1.3, ey.RED, 3]]

        win_name = "out of screen"
        ey.big_win(win_name, 0)
        ey.show_clb_win(win_name, texts=tx0, win_color=ey.WHITE)
        button = cv2.waitKey(0)
        if button == 27 or (button == ord("q")) or (button == ord("Q")):
            quit()
        cv2.destroyWindow(win_name)
        i = 0
        ey.pass_frames(cap)
        t1 = time.perf_counter()

        # Going through frames
        while True:
            frame_success, frame, frame_rgb = ey.get_frame(cap)
            if frame_success:
                # Predicting the face landmarks
                results = face_mesh.process(frame_rgb)

                # Calculating the face features
                (
                    features_success,
                    _,
                    eyes_frame_gray,
                    features_vector,
                    _,
                    _
                ) = ey.get_model_inputs(
                    frame,
                    frame_rgb,
                    results,
                    camera_matrix,
                    pcf,
                    frame_size,
                    dst_cof,
                    some_landmarks_ids
                )
                if features_success:
                    eyes_data_gray.append(eyes_frame_gray)
                    vector_inputs.append(features_vector)
                    output_class.append(out_class_num)

                    i += 1
                    if i == n_smp_in_cls:
                        break
        fps_vec.append(ey.get_time(i, t1))
        print("Data collected")
        if os.name == "nt":
            winsound.PlaySound("SystemExit", winsound.SND_ALIAS)

        cap.release()
        cv2.destroyAllWindows()
        ey.get_time(0, t0, True)
        print(f"Mean FPS : {np.array(fps_vec).mean()}")

        x1 = np.array(eyes_data_gray)
        x2 = np.array(vector_inputs)
        y = np.array(output_class)

        print("Data collection finished!")

        self.make_io(num, [x1, x2, y])

    def calculate_threshold(self, num, camera_id=0):
        """
        Calculating the blinking threshold automatically. Here we collect data and tell the user to blink during a certain time.
        Then we gain the maximum value for thier eye movement velocity and it's considered as a blink. So, we tune the threshold
        base don that.

        Parameters:
            num: subject number
            camera_id: Camera ID
        
        Returns:
            None
        """

        print("\nGetting eyes ratio...")
        tx0 = [["Look somewhere", (0.02, 0.3), 1.1, ey.RED, 2],
        ["SPACE --> start/pause", (0.02, 0.6), 1.1, ey.RED, 2]]
        tx1 = [["Blink", (0.39, 0.5), 1.6, ey.RED, 3]]
        some_landmarks_ids = ey.get_some_landmarks_ids()

        (
            frame_size,
            camera_matrix,
            dst_cof,
            pcf
        ) = ey.get_camera_properties(camera_id)

        face_mesh = ey.get_mesh()

        eyes_ratio_mat = []
        t_mat = []
        t0 = time.perf_counter()
        cap = ey.get_camera(camera_id, frame_size)
        ey.pass_frames(cap, 100)

        # Going through frames, if the user pressed 'SPACE', the program will be paused, if they press 'q', the program will be stopped.
        i = 0
        while self.running:
            win_name = f"Calibration-{i}"
            ey.big_win(win_name, math.floor(len(ey.monitors) / 2)*ey.monitors[0].width)

            eyes_ratio_vec = []
            t_vec = []
            ey.show_clb_win(win_name, win_color=ey.WHITE, texts=tx0)

            button = cv2.waitKey(0)
            if (button == ord('q')) or (button == ord('Q')) or (button == 27):
                break
            elif button == ord(' '):
                ey.pass_frames(cap)
                t1 = time.perf_counter()
                while self.running:
                    ey.show_clb_win(win_name, texts=tx1, win_color=ey.GRAY)
                    button = cv2.waitKey(1)
                    if (button == ord('q')) or (button == ord('Q')) or (button == 27) or (button == ord(' ')):
                        break
                    frame_success, frame, frame_rgb = ey.get_frame(cap)
                    if frame_success:
                        # Predicting the face landmarks
                        results = face_mesh.process(frame_rgb)

                        # Calculating the face features
                        (
                            features_success,
                            _,
                            _,
                            _,
                            eyes_ratio,
                            _
                        ) = ey.get_model_inputs(
                            frame,
                            frame_rgb,
                            results,
                            camera_matrix,
                            pcf,
                            frame_size,
                            dst_cof,
                            some_landmarks_ids,
                            False,

                        )
                        if features_success:
                            t_vec.append(round(time.perf_counter() - t1, 3))
                            eyes_ratio_vec.append(eyes_ratio)
                            
                    if not self.running:
                        break
            t_mat.append(np.array(t_vec))
            eyes_ratio_mat.append(np.array(eyes_ratio_vec))
            if (button == ord('q')) or (button == ord('Q')) or (button == 27):
                break
            if not self.running:
                break
            cv2.destroyWindow(win_name)
        cap.release()
        cv2.destroyAllWindows()
        ey.get_time(0, t0, True)

        eyes_ratio_v_mat = ey.get_blinking(t_mat, eyes_ratio_mat)[0]

        offered_threshold = ey.DEFAULT_BLINKING_THRESHOLD
        if len(eyes_ratio_v_mat) > 1:
            max_values = []
            for eyes_ratio_v_vec in eyes_ratio_v_mat:
                max_values.append(eyes_ratio_v_vec.max())
            offered_threshold = min(max_values) * 0.99
        else:
            if eyes_ratio_v_mat:
                offered_threshold = eyes_ratio_v_mat[0].max() * 0.6
        print(f"Offered Threshold: {offered_threshold}")
            
        er_dir = ey.create_dir([ey.subjects_dir, f"{num}", ey.ER])

        ey.save([t_mat, eyes_ratio_mat, offered_threshold], er_dir, [ey.T, ey.ER, "oth_app"])