import os import json import numpy as np import torch from tqdm.auto import tqdm from models import CNN from configs import CnnConfig from utils import load_json import cv2 import glob def replace_nan(x, y): x = 0.0 if np.isnan(x) else x y = 0.0 if np.isnan(y) else y return x, y def draw_hands( image, hand_x, hand_y, connections, connection_color, thickness, point_color, frame_length, frame_width, ): for connection in connections: x0 = hand_x[connection[0]] * frame_width y0 = hand_y[connection[0]] * frame_length x1 = hand_x[connection[1]] * frame_width y1 = hand_y[connection[1]] * frame_length x0, y0 = replace_nan(x0, y0) x1, y1 = replace_nan(x1, y1) cv2.line( image, (int(x0), int(y0)), (int(x1), int(y1)), connection_color, thickness, ) for x, y in zip(hand_x, hand_y): x, y = replace_nan(x, y) cv2.circle(image, (int(x), int(y)), thickness, point_color, thickness) return image def draw_pose( image, pose_x, pose_y, links, connection_color, thickness, point_color, frame_length, frame_width, ): for link in links: x0 = pose_x[link[0]] * frame_width y0 = pose_y[link[0]] * frame_length x1 = pose_x[link[1]] * frame_width y1 = pose_y[link[1]] * frame_length x0, y0 = replace_nan(x0, y0) x1, y1 = replace_nan(x1, y1) cv2.line( image, (int(x0), int(y0)), (int(x1), int(y1)), connection_color, thickness, ) cv2.circle(image, (int(x0), int(y0)), thickness, point_color, thickness) cv2.circle(image, (int(x1), int(y1)), thickness, point_color, thickness) return image def cnn_feat(file_path, save_dir): video_record = load_json(file_path) FRAME_LENGTH = 1080 FRAME_WIDTH = 1920 POINT_COLOR = (255, 0, 0) CONNECTION_COLOR = (0, 255, 0) THICKNESS = 2 connections = [ (0, 1), (1, 2), (2, 3), (3, 4), (5, 6), (6, 7), (7, 8), (9, 10), (10, 11), (11, 12), (13, 14), (14, 15), (15, 16), (17, 18), (18, 19), (19, 20), (0, 5), (5, 9), (9, 13), (13, 17), (0, 17), ] links = [ (11, 12), (11, 23), (12, 24), (23, 24), (11, 13), (13, 15), (12, 14), (14, 16), (15, 21), (15, 17), (17, 19), (19, 15), (22, 16), (16, 18), (18, 20), (16, 20), ] model = CNN(CnnConfig) features = np.empty((0, CnnConfig.output_dim)) assert video_record["n_frames"] > 0, "Number of frames should be greater than zero" for i in range(video_record["n_frames"]): image = np.zeros((FRAME_LENGTH, FRAME_WIDTH, 3), np.uint8) pose_x = video_record["pose_x"][i] pose_y = video_record["pose_y"][i] hand1_x = video_record["hand1_x"][i] hand1_y = video_record["hand1_y"][i] hand2_x = video_record["hand2_x"][i] hand2_y = video_record["hand2_y"][i] if hand1_x[0] != 0: image = draw_hands( image, hand1_x, hand1_y, connections, CONNECTION_COLOR, THICKNESS, POINT_COLOR, FRAME_LENGTH, FRAME_WIDTH, ) if hand2_x[0] != 0: image = draw_hands( image, hand2_x, hand2_y, connections, CONNECTION_COLOR, THICKNESS, POINT_COLOR, FRAME_LENGTH, FRAME_WIDTH, ) image = draw_pose( image, pose_x, pose_y, links, CONNECTION_COLOR, THICKNESS, POINT_COLOR, FRAME_LENGTH, FRAME_WIDTH, ) image = image.astype(np.float32) / 255 image = cv2.resize(image, (224, 224)) feat = model(torch.FloatTensor(image).permute(2, 0, 1).unsqueeze(0)) features = np.vstack([features, feat.numpy()]) save_path = os.path.join(save_dir, video_record["uid"] + ".npy") np.save(save_path, features) def runner(args, mode): json_files = sorted( glob.glob( os.path.join(args.data_dir, f"{args.dataset}_{mode}_keypoints", "*.json") ) ) save_dir = os.path.join(args.data_dir, f"{args.dataset}_{mode}_features") if not os.path.exists(save_dir): os.mkdir(save_dir) saved_files = glob.glob(os.path.join(save_dir, "*.npy")) if len(saved_files) != len(json_files): for file_path in tqdm(json_files, desc=f"Saving CNN features - {mode} files"): cnn_feat(file_path, save_dir) else: print(mode, "CNN features already exist!") def save_cnn_features(args): runner(args, mode="train") runner(args, mode="val") runner(args, mode="test")