gitignore ipynb txt

home.ipynb

@@ -0,0 +1,305 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting utools.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile utools.py\n",
+    "import tflite_runtime.interpreter as tflite   \n",
+    "import tflite_runtime\n",
+    "import numpy as np\n",
+    "ROWS_PER_FRAME=543\n",
+    "def load_relevant_data_subset(df):\n",
+    "    data_columns = ['x', 'y', 'z']\n",
+    "    data=df[data_columns]\n",
+    "    n_frames = int(len(data) / ROWS_PER_FRAME)#单个文件的总帧数\n",
+    "    data = data.values.reshape(n_frames, ROWS_PER_FRAME, len(data_columns))\n",
+    "    return data.astype(np.float32)\n",
+    "\n",
+    "def mark_pred(model_path_1,aa):\n",
+    "    interpreter = tflite.Interpreter(model_path_1)\n",
+    "    found_signatures = list(interpreter.get_signature_list().keys())\n",
+    "    prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n",
+    "    output_1 = prediction_fn(inputs=aa)\n",
+    "    return output_1\n",
+    "\n",
+    "def softmax(x, axis=None):\n",
+    "    x_exp = np.exp(x - np.max(x, axis=axis, keepdims=True))\n",
+    "    return x_exp / np.sum(x_exp, axis=axis, keepdims=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting model.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile model.py\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import os\n",
+    "import shutil\n",
+    "from datetime import datetime\n",
+    "from timeit import default_timer as timer\n",
+    "from utools import load_relevant_data_subset,mark_pred\n",
+    "from utools import softmax\n",
+    "import mediapipe as mp\n",
+    "import cv2\n",
+    "import json\n",
+    "N=3\n",
+    "\n",
+    "ROWS_PER_FRAME=543\n",
+    "with open('sign_to_prediction_index_map_cn.json', 'r') as f:\n",
+    "    person_dict = json.load(f)\n",
+    "inverse_dict=dict([val,key] for key,val in person_dict.items())\n",
+    "\n",
+    "\n",
+    "def r_holistic(video_path):\n",
+    "    mp_drawing = mp.solutions.drawing_utils\n",
+    "    mp_drawing_styles = mp.solutions.drawing_styles\n",
+    "    mp_holistic = mp.solutions.holistic\n",
+    "    frame_number = 0\n",
+    "    frame = []\n",
+    "    type_ = []\n",
+    "    index = []\n",
+    "    x = []\n",
+    "    y = []\n",
+    "    z = []\n",
+    "    cap=cv2.VideoCapture(video_path)\n",
+    "    frame_width = int(cap.get(3))\n",
+    "    frame_height = int(cap.get(4))\n",
+    "    fps = int(cap.get(cv2.CAP_PROP_FPS))\n",
+    "    frame_size = (frame_width, frame_height)\n",
+    "    fourcc = cv2.VideoWriter_fourcc(*\"VP80\") #cv2.VideoWriter_fourcc('H.264')\n",
+    "    output_video = \"output_recorded_holistic.webm\"\n",
+    "    out = cv2.VideoWriter(output_video, fourcc, int(fps/N), frame_size)\n",
+    "    with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5) as holistic:\n",
+    "        n=0\n",
+    "        while cap.isOpened():\n",
+    "            frame_number+=1\n",
+    "            n+=1\n",
+    "            ret, image = cap.read()\n",
+    "            if not ret:\n",
+    "                break\n",
+    "            if n%N==0:\n",
+    "                image.flags.writeable = False\n",
+    "                image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)\n",
+    "                #mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=RGB_frame)\n",
+    "                results = holistic.process(image)\n",
+    "\n",
+    "                # Draw landmark annotation on the image.\n",
+    "                image.flags.writeable = True\n",
+    "                image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
+    "                mp_drawing.draw_landmarks(\n",
+    "                    image,\n",
+    "                    results.face_landmarks,\n",
+    "                    mp_holistic.FACEMESH_CONTOURS,\n",
+    "                    landmark_drawing_spec=None,\n",
+    "                    connection_drawing_spec=mp_drawing_styles\n",
+    "                    .get_default_face_mesh_contours_style())\n",
+    "                mp_drawing.draw_landmarks(\n",
+    "                    image,\n",
+    "                    results.pose_landmarks,\n",
+    "                    mp_holistic.POSE_CONNECTIONS,\n",
+    "                    landmark_drawing_spec=mp_drawing_styles\n",
+    "                    .get_default_pose_landmarks_style())\n",
+    "                # Flip the image horizontally for a selfie-view display.\n",
+    "                #if cv2.waitKey(5) & 0xFF == 27:\n",
+    "                out.write(image)\n",
+    "                \n",
+    "                if(results.face_landmarks is None):\n",
+    "                    for i in range(468):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"face\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(None)\n",
+    "                        y.append(None)\n",
+    "                        z.append(None)\n",
+    "                else:\n",
+    "                    for ind,val in enumerate(results.face_landmarks.landmark):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"face\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(val.x)\n",
+    "                        y.append(val.y)\n",
+    "                        z.append(val.z)\n",
+    "                #left hand\n",
+    "                if(results.left_hand_landmarks is None):\n",
+    "                    for i in range(21):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"left_hand\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(None)\n",
+    "                        y.append(None)\n",
+    "                        z.append(None)\n",
+    "                else:\n",
+    "                    for ind,val in enumerate(results.left_hand_landmarks.landmark):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"left_hand\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(val.x)\n",
+    "                        y.append(val.y)\n",
+    "                        z.append(val.z)\n",
+    "                #pose\n",
+    "                if(results.pose_landmarks is None):\n",
+    "                    for i in range(33):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"pose\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(None)\n",
+    "                        y.append(None)\n",
+    "                        z.append(None)\n",
+    "                else:\n",
+    "                    for ind,val in enumerate(results.pose_landmarks.landmark):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"pose\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(val.x)\n",
+    "                        y.append(val.y)\n",
+    "                        z.append(val.z)\n",
+    "                #right hand\n",
+    "                if(results.right_hand_landmarks is None):\n",
+    "                    for i in range(21):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"right_hand\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(None)\n",
+    "                        y.append(None)\n",
+    "                        z.append(None)\n",
+    "                else:\n",
+    "                    for ind,val in enumerate(results.right_hand_landmarks.landmark):\n",
+    "                        frame.append(frame_number)\n",
+    "                        type_.append(\"right_hand\")\n",
+    "                        index.append(ind)\n",
+    "                        x.append(val.x)\n",
+    "                        y.append(val.y)\n",
+    "                        z.append(val.z)\n",
+    "        #break\n",
+    "    cap.release()\n",
+    "    out.release()\n",
+    "    cv2.destroyAllWindows()\n",
+    "    df1 = pd.DataFrame({\n",
+    "            \"frame\" : frame,\n",
+    "            \"type\"  : type_,\n",
+    "            \"landmark_index\" : index,\n",
+    "            \"x\" : x,\n",
+    "            \"y\" : y,\n",
+    "            \"z\" : z\n",
+    "        })\n",
+    "    aa=load_relevant_data_subset(df1)\n",
+    "    model_path_1='model_1.tflite'\n",
+    "    model_path_2='model_2.tflite'\n",
+    "    model_path_3='model_3.tflite'\n",
+    "    #interpreter = tflite.Interpreter(model_path_1)\n",
+    "    #found_signatures = list(interpreter.get_signature_list().keys())\n",
+    "    #prediction_fn = interpreter.get_signature_runner(\"serving_default\")\n",
+    "    output_1 = mark_pred(model_path_1,aa)\n",
+    "    output_2 = mark_pred(model_path_2,aa)\n",
+    "    output_3 = mark_pred(model_path_3,aa)\n",
+    "    output=softmax(output_1['outputs'])+softmax(output_2['outputs'])+softmax(output_3['outputs'])\n",
+    "    sign = output.argmax()\n",
+    "    lb = inverse_dict.get(sign)\n",
+    "    yield output_video,lb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting app.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile app.py\n",
+    "\n",
+    "import gradio as gr\n",
+    "from model import r_holistic\n",
+    "\n",
+    "title='手语动作分类'\n",
+    "description = \"此分类模型可以识别250个[ASL](https://www.lifeprint.com/)手语动作\\\n",
+    "    并将其转化为特定的标签, 标签列表见链接[sign_to_prediction_index_map.json](sign_to_prediction_index_map.json), \\\n",
+    "    大家可以使用示例视频进行测试, 也可以根据列表下载或模拟相应的手语视频测试输出.\\\n",
+    "    \\n工作流程:\\\n",
+    "    \\n  1. landmark提取, 我使用了[ MediaPipe Holistic Solution](https://ai.google.dev/edge/mediapipe/solutions/vision/holistic_landmarker)进行landmark提取.\\\n",
+    "    \\n  2. 利用landmark进行手语识别, 我使用了自己搭建并训练的模型, 主体框架为cnn和transform，此模型在测试数据集上精度在90%以上.\"\n",
+    "\n",
+    "output_video_file = gr.Video(label=\"landmark输出\")\n",
+    "output_text=gr.Textbox(label=\"手语预测结果\")\n",
+    "slider_1=gr.Slider(0,1,label='detection_confidence')\n",
+    "slider_2=gr.Slider(0,1,label='tracking_confidence')\n",
+    "\n",
+    "iface = gr.Interface(\n",
+    "  fn=r_holistic,\n",
+    "  inputs=[gr.Video(sources=None, label=\"手语视频片段\")],\n",
+    "  outputs= [output_video_file,output_text],\n",
+    "  title=title, \n",
+    "  description=description,\n",
+    "  examples=['book.mp4','book2.mp4','chair1.mp4','chair2.mp4'],\n",
+    "  #cache_examples=True,\n",
+    "  )  #[\"hand-land-mark-video/01.mp4\",\"hand-land-mark-video/02.mp4\"]\n",
+    "  \n",
+    "\n",
+    "iface.launch(share=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "myenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}