|
|
"""This module contains Smp class to collect inputs from the user. To understand this module you should know about Mediapipe library."""
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
import cv2
|
|
|
import time
|
|
|
from codes.base import eyeing as ey
|
|
|
from codes.calibration import Clb
|
|
|
import math
|
|
|
import random
|
|
|
from datetime import datetime
|
|
|
|
|
|
|
|
|
class Smp(object):
|
|
|
running = True
|
|
|
|
|
|
def sampling(self, num, camera_id=0, gui=True):
|
|
|
"""
|
|
|
Collecting inputs (eyes images and face vectors) from the user as the main sampling data.
|
|
|
|
|
|
Parameters:
|
|
|
num: subject number
|
|
|
camera_id: Camera ID
|
|
|
gui: If it's False, for having the ability to stop the program, a little window will be appeared. So, you can press "q" to stop it. If it's True, you can stop program using GUI.
|
|
|
|
|
|
Returns:
|
|
|
None
|
|
|
"""
|
|
|
|
|
|
face_saving_time = 80
|
|
|
return_face1 = True
|
|
|
win_name = "Sampling"
|
|
|
little_win_name = "smp"
|
|
|
tx0 = [["Sampling", (0.25, 0.5), 2, ey.RED, 3]]
|
|
|
tx1 = [["SPACE --> start/pause", (0.05, 0.3), 1.5, ey.RED, 3],
|
|
|
["ESC --> Stop", (0.05, 0.6), 1.6, ey.RED, 3]]
|
|
|
|
|
|
some_landmarks_ids = ey.get_some_landmarks_ids()
|
|
|
|
|
|
(
|
|
|
frame_size,
|
|
|
camera_matrix,
|
|
|
dst_cof,
|
|
|
pcf
|
|
|
) = ey.get_camera_properties(camera_id)
|
|
|
|
|
|
face_mesh = ey.get_mesh()
|
|
|
|
|
|
cap = ey.get_camera(camera_id, frame_size)
|
|
|
ey.pass_frames(cap, 100)
|
|
|
|
|
|
print("Sampling started...")
|
|
|
t_mat = []
|
|
|
sys_time_mat = []
|
|
|
eyes_mat = []
|
|
|
inp_scalars_mat = []
|
|
|
eyes_ratio_mat = []
|
|
|
face_vec = []
|
|
|
fps_vec = []
|
|
|
|
|
|
ey.big_win(win_name, math.floor(len(ey.monitors) / 2)*ey.monitors[0].width)
|
|
|
ey.show_clb_win(win_name, win_color=ey.WHITE, texts=tx0)
|
|
|
cv2.waitKey(4000)
|
|
|
cv2.destroyWindow(win_name)
|
|
|
|
|
|
t0 = time.perf_counter()
|
|
|
while self.running:
|
|
|
j = 0
|
|
|
ey.big_win(win_name, math.floor(len(ey.monitors) / 2)*ey.monitors[0].width)
|
|
|
ey.show_clb_win(win_name, win_color=ey.WHITE, texts=tx1)
|
|
|
button = cv2.waitKey(0)
|
|
|
cv2.destroyWindow(win_name)
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
elif button == ord(' '):
|
|
|
t_vec = []
|
|
|
sys_time_vec = []
|
|
|
eyes_vec = []
|
|
|
inp_scalars_vec = []
|
|
|
eyes_ratio_vec = []
|
|
|
t1 = time.perf_counter()
|
|
|
while self.running:
|
|
|
frame_success, frame, frame_rgb = ey.get_frame(cap)
|
|
|
if frame_success:
|
|
|
return_face = False
|
|
|
if ((time.perf_counter() - t0) < face_saving_time) and return_face1:
|
|
|
return_face = True
|
|
|
|
|
|
results = face_mesh.process(frame_rgb)
|
|
|
|
|
|
|
|
|
(
|
|
|
features_success,
|
|
|
_,
|
|
|
eyes_frame_gray,
|
|
|
features_vector,
|
|
|
eyes_ratio,
|
|
|
face_img
|
|
|
) = ey.get_model_inputs(
|
|
|
frame,
|
|
|
frame_rgb,
|
|
|
results,
|
|
|
camera_matrix,
|
|
|
pcf,
|
|
|
frame_size,
|
|
|
dst_cof,
|
|
|
some_landmarks_ids,
|
|
|
return_face=return_face
|
|
|
)
|
|
|
if features_success:
|
|
|
t_vec.append(round(time.perf_counter() - t0, 3))
|
|
|
sys_time_vec.append(str(datetime.now())[-15:-3])
|
|
|
eyes_vec.append(eyes_frame_gray)
|
|
|
inp_scalars_vec.append(features_vector)
|
|
|
eyes_ratio_vec.append(eyes_ratio)
|
|
|
if return_face:
|
|
|
face_vec.append(face_img)
|
|
|
j += 1
|
|
|
if not gui:
|
|
|
ey.show_clb_win(little_win_name, win_color=ey.RED, win_size=(50, 50))
|
|
|
button = cv2.waitKey(1)
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27) or (button == ord(' ')):
|
|
|
break
|
|
|
fps_vec.append(ey.get_time(j, t1, True))
|
|
|
t_mat.append(np.array(t_vec))
|
|
|
sys_time_mat.append(sys_time_vec)
|
|
|
eyes_mat.append(np.array(eyes_vec))
|
|
|
inp_scalars_mat.append(np.array(inp_scalars_vec))
|
|
|
eyes_ratio_mat.append(np.array(eyes_ratio_vec))
|
|
|
|
|
|
if not gui:
|
|
|
cv2.destroyWindow(little_win_name)
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
return_face1 = False
|
|
|
|
|
|
print("Sampling finished")
|
|
|
ey.get_time(0, t0, True)
|
|
|
print(f"Mean FPS : {np.array(fps_vec).mean()}")
|
|
|
|
|
|
cv2.destroyAllWindows()
|
|
|
cap.release()
|
|
|
|
|
|
smp_dir = ey.create_dir([ey.subjects_dir, f"{num}", ey.SMP])
|
|
|
ey.save(
|
|
|
[t_mat, sys_time_mat, eyes_mat, inp_scalars_mat, eyes_ratio_mat, [np.array(face_vec)]],
|
|
|
smp_dir,
|
|
|
[ey.T, "sys_time", ey.X1, ey.X2, ey.ER, ey.FV])
|
|
|
|
|
|
|
|
|
def accuracy(self, num, camera_id=0, clb_grid=(2, 2, 10)):
|
|
|
"""
|
|
|
Collecting inputs (eyes images and face vectors) from the user to assess the accuracy of the models. It shows a point in a grid
|
|
|
and the user must look at that. So, with the true outputs (locations of the point) and the predicted outputs, it's possible to
|
|
|
evaluate the models. The evaulation part is in the eye_track.py module.
|
|
|
|
|
|
Parameters:
|
|
|
num: subject number
|
|
|
camera_id: Camera ID
|
|
|
clb_grid: The grid that you want to use for comparison
|
|
|
|
|
|
Returns:
|
|
|
None
|
|
|
"""
|
|
|
|
|
|
|
|
|
tx0 = [["Track WHITE point", (0.05, 0.25), 1.5, ey.RED, 3],
|
|
|
["SPACE --> start", (0.05, 0.5), 1.5, ey.RED, 3],
|
|
|
["ESC --> Stop", (0.05, 0.75), 1.5, ey.RED, 3]]
|
|
|
clb_points = Clb().create_grid(clb_grid)
|
|
|
|
|
|
some_landmarks_ids = ey.get_some_landmarks_ids()
|
|
|
|
|
|
(
|
|
|
frame_size,
|
|
|
camera_matrix,
|
|
|
dst_cof,
|
|
|
pcf
|
|
|
) = ey.get_camera_properties(camera_id)
|
|
|
|
|
|
face_mesh = ey.get_mesh()
|
|
|
|
|
|
i = 0
|
|
|
fps_vec = []
|
|
|
t_mat = []
|
|
|
eyes_mat = []
|
|
|
inp_scalars_mat = []
|
|
|
points_loc_mat = []
|
|
|
eyes_ratio_mat = []
|
|
|
cap = ey.get_camera(camera_id, frame_size)
|
|
|
ey.pass_frames(cap, 100)
|
|
|
t0 = time.perf_counter()
|
|
|
|
|
|
win_name = "Information"
|
|
|
ey.big_win(win_name, math.floor(len(ey.monitors) / 2)*ey.monitors[0].width)
|
|
|
ey.show_clb_win(win_name, texts=tx0, win_color=ey.WHITE)
|
|
|
cv2.waitKey(10000)
|
|
|
cv2.destroyWindow(win_name)
|
|
|
for (i_m, m) in enumerate(ey.monitors):
|
|
|
if not self.running:
|
|
|
break
|
|
|
win_name = f"Calibration-{i_m}"
|
|
|
ey.big_win(win_name, i_m * m.width)
|
|
|
for item in clb_points:
|
|
|
if not self.running and (i_m != 0):
|
|
|
break
|
|
|
pnt = item[0]
|
|
|
t_vec = []
|
|
|
eyes_vec = []
|
|
|
inp_scalars_vec = []
|
|
|
points_loc_vec = []
|
|
|
eyes_ratio_vec = []
|
|
|
ey.show_clb_win(win_name, pnt, win_color=ey.GRAY)
|
|
|
|
|
|
button = cv2.waitKey(0)
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
elif button == ord(' '):
|
|
|
ey.pass_frames(cap)
|
|
|
t1 = time.perf_counter()
|
|
|
s = len(item)
|
|
|
for pnt in item:
|
|
|
ey.show_clb_win(win_name, pnt)
|
|
|
button = cv2.waitKey(1)
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
while True:
|
|
|
frame_success, frame, frame_rgb = ey.get_frame(cap)
|
|
|
if frame_success:
|
|
|
results = face_mesh.process(frame_rgb)
|
|
|
(
|
|
|
features_success,
|
|
|
_,
|
|
|
eyes_frame_gray,
|
|
|
features_vector,
|
|
|
eyes_ratio,
|
|
|
_
|
|
|
) = ey.get_model_inputs(
|
|
|
frame,
|
|
|
frame_rgb,
|
|
|
results,
|
|
|
camera_matrix,
|
|
|
pcf,
|
|
|
frame_size,
|
|
|
dst_cof,
|
|
|
some_landmarks_ids
|
|
|
)
|
|
|
if features_success:
|
|
|
t_vec.append(round(time.perf_counter() - t1, 3))
|
|
|
eyes_vec.append(eyes_frame_gray)
|
|
|
inp_scalars_vec.append(features_vector)
|
|
|
points_loc_vec.append([(pnt[0] + i_m)/len(ey.monitors), pnt[1]])
|
|
|
eyes_ratio_vec.append(eyes_ratio)
|
|
|
i += 1
|
|
|
break
|
|
|
if not self.running:
|
|
|
break
|
|
|
fps_vec.append(ey.get_time(s, t1))
|
|
|
t_mat.append(np.array(t_vec))
|
|
|
eyes_mat.append(np.array(eyes_vec))
|
|
|
inp_scalars_mat.append(np.array(inp_scalars_vec))
|
|
|
points_loc_mat.append(np.array(points_loc_vec))
|
|
|
eyes_ratio_mat.append(np.array(eyes_ratio_vec))
|
|
|
|
|
|
if not self.running:
|
|
|
break
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
cv2.destroyWindow(win_name)
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
cap.release()
|
|
|
|
|
|
cv2.destroyAllWindows()
|
|
|
ey.get_time(0, t0, True)
|
|
|
print(f"Mean FPS : {np.array(fps_vec).mean()}")
|
|
|
|
|
|
acc_dir = ey.create_dir([ey.subjects_dir, f"{num}", ey.ACC])
|
|
|
ey.save(
|
|
|
[t_mat, eyes_mat, inp_scalars_mat, points_loc_mat, eyes_ratio_mat],
|
|
|
acc_dir,
|
|
|
[ey.T, ey.X1, ey.X2, ey.Y, ey.ER])
|
|
|
print("Accuracy data collected!")
|
|
|
|
|
|
|
|
|
def latency(self, num, camera_id=0):
|
|
|
""" Collecting data to assessing latency. The user should look at the left and right side of the screen when they see red or blue colors.
|
|
|
|
|
|
Parameters:
|
|
|
num: Subject number
|
|
|
camera_id: Camera ID
|
|
|
|
|
|
Returns:
|
|
|
None
|
|
|
"""
|
|
|
tx1 = [["SPACE --> start", (0.05, 0.2), 1.3, ey.BLACK, 2],
|
|
|
[f"ESC --> stop", (0.05, 0.4), 1.3, ey.BLACK, 2],
|
|
|
["RED --> Left", (0.05, 0.6), 1.3, ey.RED, 2],
|
|
|
["BLUE --> Right", (0.05, 0.8), 1.3, ey.BLUE, 2]]
|
|
|
some_landmarks_ids = ey.get_some_landmarks_ids()
|
|
|
|
|
|
(
|
|
|
frame_size,
|
|
|
camera_matrix,
|
|
|
dst_cof,
|
|
|
pcf
|
|
|
) = ey.get_camera_properties(camera_id)
|
|
|
|
|
|
face_mesh = ey.get_mesh()
|
|
|
|
|
|
fps_vec = []
|
|
|
t_mat = []
|
|
|
eyes_mat = []
|
|
|
inp_scalars_mat = []
|
|
|
cap = ey.get_camera(camera_id, frame_size)
|
|
|
ey.pass_frames(cap, 100)
|
|
|
t0 = time.perf_counter()
|
|
|
|
|
|
win_name = "Information"
|
|
|
ey.big_win(win_name, math.floor(len(ey.monitors) / 2) * ey.monitors[0].width)
|
|
|
ey.show_clb_win(win_name, texts=tx1, win_color=ey.WHITE)
|
|
|
button = cv2.waitKey(0)
|
|
|
if button == ord(' '):
|
|
|
cv2.destroyWindow(win_name)
|
|
|
win_name = "Latency"
|
|
|
time.sleep(2)
|
|
|
for j in range(6):
|
|
|
if not self.running:
|
|
|
break
|
|
|
t_vec = []
|
|
|
eyes_vec = []
|
|
|
inp_scalars_vec = []
|
|
|
i = 0
|
|
|
t1 = time.perf_counter()
|
|
|
dt = random.random()*3
|
|
|
ey.big_win(win_name, math.floor(len(ey.monitors) / 2) * ey.monitors[0].width)
|
|
|
if j % 2 == 0:
|
|
|
win_color = ey.BLUE
|
|
|
else:
|
|
|
win_color = ey.RED
|
|
|
ey.show_clb_win(win_name, win_color=win_color)
|
|
|
cv2.waitKey(ey.LATENCY_WAITING_TIME)
|
|
|
cv2.destroyWindow(win_name)
|
|
|
while (time.perf_counter()-t1) < (3 + dt):
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
while True:
|
|
|
frame_success, frame, frame_rgb = ey.get_frame(cap)
|
|
|
if frame_success:
|
|
|
results = face_mesh.process(frame_rgb)
|
|
|
(
|
|
|
features_success,
|
|
|
_,
|
|
|
eyes_frame_gray,
|
|
|
features_vector,
|
|
|
eyes_ratio,
|
|
|
_
|
|
|
) = ey.get_model_inputs(
|
|
|
frame,
|
|
|
frame_rgb,
|
|
|
results,
|
|
|
camera_matrix,
|
|
|
pcf,
|
|
|
frame_size,
|
|
|
dst_cof,
|
|
|
some_landmarks_ids
|
|
|
)
|
|
|
if features_success:
|
|
|
t_vec.append(round(time.perf_counter() - t1, 3))
|
|
|
eyes_vec.append(eyes_frame_gray)
|
|
|
inp_scalars_vec.append(features_vector)
|
|
|
i += 1
|
|
|
break
|
|
|
if not self.running:
|
|
|
break
|
|
|
fps_vec.append(ey.get_time(i, t1))
|
|
|
t_mat.append(np.array(t_vec))
|
|
|
eyes_mat.append(np.array(eyes_vec))
|
|
|
inp_scalars_mat.append(np.array(inp_scalars_vec))
|
|
|
|
|
|
if not self.running:
|
|
|
break
|
|
|
if (button == ord('q')) or (button == ord('Q')) or (button == 27):
|
|
|
break
|
|
|
cap.release()
|
|
|
|
|
|
ey.get_time(0, t0, True)
|
|
|
print(f"Mean FPS : {np.array(fps_vec).mean()}")
|
|
|
|
|
|
ltn_dir = ey.create_dir([ey.subjects_dir, f"{num}", ey.LTN])
|
|
|
ey.save(
|
|
|
[t_mat, eyes_mat, inp_scalars_mat],
|
|
|
ltn_dir,
|
|
|
[ey.T, ey.X1, ey.X2])
|
|
|
print("Latency data collected!") |
