Upload 20 files

.gitattributes

@@ -44,3 +44,4 @@ examples/displayed/fine_keypoints.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/displayed/help_keypoints.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/displayed/no_keypoints.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/displayed/woman_keypoints.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/displayed/book_keypoints.mp4 filter=lfs diff=lfs merge=lfs -text

example_keypoint_adder.py

@@ -0,0 +1,307 @@
+"""
+example_keypoint_adder.py
+Manages example videos and their precomputed keypoint JSON files and rendered keypoint videos.
+Ensures that each example entry has:
+1. video_path: Path to the input ASL video.
+2. keypoints_path: Path to the extracted keypoints JSON.
+3. keypoints_video_path: Path to the video with keypoints rendered on top.
+
+Can be run as a standalone script to batch-process all examples or add new ones.
+"""
+
+import os
+import sys
+import json
+import argparse
+import numpy as np
+
+# Ensure UTF-8 stdout
+if hasattr(sys.stdout, 'reconfigure'):
+    try:
+        sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+    except Exception:
+        pass
+
+from predict_pose import create_mediapipe_models, predict_pose, load_video_cv
+from visualize_pose import render_keypoints_video
+
+
+def to_json_serializable(obj):
+    """Recursively converts NumPy arrays, numbers, and tensors to JSON-serializable Python types."""
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    if isinstance(obj, (np.float32, np.float64, np.float16)):
+        return float(obj)
+    if isinstance(obj, (np.int32, np.int64, np.int16, np.int8, np.uint8)):
+        return int(obj)
+    if isinstance(obj, dict):
+        return {str(k): to_json_serializable(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [to_json_serializable(v) for v in obj]
+    return obj
+
+
+def load_examples_json(json_path="examples.json"):
+    """Loads examples configuration from a JSON file."""
+    if not os.path.exists(json_path):
+        return []
+    try:
+        with open(json_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+            if isinstance(data, list):
+                return data
+            elif isinstance(data, dict) and "examples" in data:
+                return data["examples"]
+            return []
+    except Exception as e:
+        print(f"Error loading {json_path}: {e}")
+        return []
+
+
+def save_examples_json(examples, json_path="examples.json"):
+    """Saves examples configuration to a JSON file."""
+    try:
+        with open(json_path, "w", encoding="utf-8") as f:
+            json.dump(examples, f, indent=2, ensure_ascii=False)
+        print(f"Saved {len(examples)} examples to {json_path}")
+    except Exception as e:
+        print(f"Error saving {json_path}: {e}")
+
+
+def process_example(
+    video_path,
+    title=None,
+    keypoints_path=None,
+    keypoints_video_path=None,
+    pose_models=None,
+    force=False,
+    checkpoint_folder="checkpoints/pose/"
+):
+    """
+    Checks if keypoints JSON and rendered keypoints video exist for a video.
+    If not (or if force=True), extracts keypoints and renders the visualization video.
+    Returns the updated example dictionary.
+    """
+    video_path = os.path.normpath(video_path)
+    if not os.path.exists(video_path):
+        print(f"[Warning] Video file not found: {video_path}")
+        return None
+
+    video_dir = os.path.dirname(video_path)
+    video_stem = os.path.splitext(os.path.basename(video_path))[0]
+    
+    if not title:
+        title = video_stem.replace("_", " ").capitalize()
+
+    if not keypoints_path:
+        keypoints_path = os.path.join(video_dir, f"{video_stem}_keypoints.json")
+    keypoints_path = os.path.normpath(keypoints_path)
+
+    if not keypoints_video_path:
+        keypoints_video_path = os.path.join(video_dir, f"{video_stem}_keypoints.mp4")
+    keypoints_video_path = os.path.normpath(keypoints_video_path)
+
+    kp_exists = os.path.exists(keypoints_path) and os.path.getsize(keypoints_path) > 0
+    vid_exists = os.path.exists(keypoints_video_path) and os.path.getsize(keypoints_video_path) > 0
+
+    if kp_exists and vid_exists and not force:
+        print(f"[OK] Keypoints and video already exist for '{title}' ({video_path})")
+        return {
+            "title": title,
+            "video_path": video_path.replace("\\", "/"),
+            "keypoints_path": keypoints_path.replace("\\", "/"),
+            "keypoints_video_path": keypoints_video_path.replace("\\", "/"),
+        }
+
+    print(f"\n[Processing] Generating keypoints for '{title}' ({video_path})...")
+
+    # Load pose models if not yet loaded
+    if pose_models is None:
+        print("  Initializing MediaPipe & YOLO models...")
+        pose_models = create_mediapipe_models(checkpoint_folder)
+
+    # Load video frames
+    video_frames, fps = load_video_cv(video_path)
+    if fps is None or fps <= 0 or np.isnan(fps):
+        fps = 25.0
+    print(f"  Loaded {len(video_frames)} frames @ {fps:.1f} FPS")
+
+    # Predict pose & keypoints (includes 512x512 scaling for WLASL)
+    pose_results = predict_pose(video_frames, pose_models)
+
+    # Attach bounding boxes to keypoints dict for each frame
+    kps_list = pose_results.get("keypoints", [])
+    bbox_face_list = pose_results.get("bbox_face", [])
+    bbox_lh_list = pose_results.get("bbox_left_hand", [])
+    bbox_rh_list = pose_results.get("bbox_right_hand", [])
+
+    for i, kp in enumerate(kps_list):
+        if i < len(bbox_face_list):
+            kp['bbox_face'] = bbox_face_list[i]
+        if i < len(bbox_lh_list):
+            kp['bbox_left_hand'] = bbox_lh_list[i]
+        if i < len(bbox_rh_list):
+            kp['bbox_right_hand'] = bbox_rh_list[i]
+
+    # Save keypoints to JSON (strip heavy raw image arrays to keep JSON file lightweight ~50KB and ultra-fast)
+    heavy_image_keys = {'images', 'cropped_images', 'cropped_left_hand', 'cropped_right_hand', 'cropped_face'}
+    clean_pose_results = {k: v for k, v in pose_results.items() if k not in heavy_image_keys}
+
+    os.makedirs(os.path.dirname(os.path.abspath(keypoints_path)), exist_ok=True)
+    serializable_results = to_json_serializable(clean_pose_results)
+    with open(keypoints_path, "w", encoding="utf-8") as f:
+        json.dump(serializable_results, f)
+    print(f"  [Saved] Lightweight Keypoints JSON -> {keypoints_path}")
+
+    # Render keypoint overlay video
+    rendered_path = render_keypoints_video(
+        video_frames=video_frames,
+        keypoints_list=kps_list,
+        fps=fps,
+        output_path=keypoints_video_path,
+    )
+    print(f"  [Saved] Rendered Keypoints Video -> {rendered_path}")
+
+    return {
+        "title": title,
+        "video_path": video_path.replace("\\", "/"),
+        "keypoints_path": keypoints_path.replace("\\", "/"),
+        "keypoints_video_path": keypoints_video_path.replace("\\", "/"),
+    }
+
+
+def ensure_all_examples(
+    json_path="examples.json",
+    default_dir="examples/displayed",
+    force=False,
+    checkpoint_folder="checkpoints/pose/"
+):
+    """
+    Ensures all examples in `examples.json` (or found in default_dir)
+    have precomputed keypoints JSON and rendered MP4 video.
+    """
+    existing_examples = load_examples_json(json_path)
+
+    # If examples.json doesn't exist or is empty, discover .mp4 files in default_dir
+    if not existing_examples and os.path.exists(default_dir):
+        print(f"Discovering example videos in {default_dir}...")
+        for fname in sorted(os.listdir(default_dir)):
+            if fname.lower().endswith(".mp4") and not fname.endswith("_keypoints.mp4"):
+                vpath = os.path.join(default_dir, fname)
+                stem = os.path.splitext(fname)[0]
+                existing_examples.append({
+                    "title": stem.capitalize(),
+                    "video_path": vpath.replace("\\", "/"),
+                    "keypoints_path": os.path.join(default_dir, f"{stem}_keypoints.json").replace("\\", "/"),
+                    "keypoints_video_path": os.path.join(default_dir, f"{stem}_keypoints.mp4").replace("\\", "/"),
+                })
+
+    if not existing_examples:
+        print("No examples found to process.")
+        return []
+
+    pose_models = None
+    # Check if any model processing is needed before initializing heavy models
+    needs_processing = force or any(
+        not (
+            os.path.exists(ex.get("keypoints_path", "")) and 
+            os.path.exists(ex.get("keypoints_video_path", "")) and
+            os.path.getsize(ex.get("keypoints_path", "")) > 0 and
+            os.path.getsize(ex.get("keypoints_video_path", "")) > 0
+        )
+        for ex in existing_examples
+    )
+
+    if needs_processing:
+        print("Initializing pose estimation models for processing...")
+        pose_models = create_mediapipe_models(checkpoint_folder)
+
+    updated_examples = []
+    for ex in existing_examples:
+        vpath = ex.get("video_path", "")
+        title = ex.get("title", "")
+        kpath = ex.get("keypoints_path", "")
+        kvpath = ex.get("keypoints_video_path", "")
+
+        res = process_example(
+            video_path=vpath,
+            title=title,
+            keypoints_path=kpath,
+            keypoints_video_path=kvpath,
+            pose_models=pose_models,
+            force=force,
+            checkpoint_folder=checkpoint_folder,
+        )
+        if res:
+            updated_examples.append(res)
+        else:
+            updated_examples.append(ex)
+
+    save_examples_json(updated_examples, json_path)
+    return updated_examples
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Ensure all examples have precomputed keypoints JSON and rendered pose videos."
+    )
+    parser.add_argument(
+        "--json",
+        default="examples.json",
+        help="Path to examples.json (default: examples.json)"
+    )
+    parser.add_argument(
+        "--dir",
+        default="examples/displayed",
+        help="Directory to scan for example videos if JSON is empty (default: examples/displayed)"
+    )
+    parser.add_argument(
+        "--add",
+        default=None,
+        help="Add a specific video path to examples.json and generate its keypoints"
+    )
+    parser.add_argument(
+        "--title",
+        default=None,
+        help="Optional title/label when adding a single video"
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Force re-generation of keypoints and videos even if they exist"
+    )
+    args = parser.parse_args()
+
+    if args.add:
+        print(f"Adding single video: {args.add}")
+        examples = load_examples_json(args.json)
+        pose_models = create_mediapipe_models("checkpoints/pose/")
+        res = process_example(
+            video_path=args.add,
+            title=args.title,
+            pose_models=pose_models,
+            force=args.force
+        )
+        if res:
+            # Check if entry with same video_path already exists
+            norm_new = os.path.normpath(res["video_path"])
+            exists_idx = -1
+            for idx, ex in enumerate(examples):
+                if os.path.normpath(ex.get("video_path", "")) == norm_new:
+                    exists_idx = idx
+                    break
+            if exists_idx >= 0:
+                examples[exists_idx] = res
+            else:
+                examples.append(res)
+            save_examples_json(examples, args.json)
+    else:
+        ensure_all_examples(
+            json_path=args.json,
+            default_dir=args.dir,
+            force=args.force
+        )
+
+
+if __name__ == "__main__":
+    main()

examples.json

@@ -0,0 +1,38 @@
+[
+  {
+    "title": "Book",
+    "video_path": "examples/displayed/book.mp4",
+    "keypoints_path": "examples/displayed/book_keypoints.json",
+    "keypoints_video_path": "examples/displayed/book_keypoints.mp4"
+  },
+  {
+    "title": "Deaf",
+    "video_path": "examples/displayed/deaf.mp4",
+    "keypoints_path": "examples/displayed/deaf_keypoints.json",
+    "keypoints_video_path": "examples/displayed/deaf_keypoints.mp4"
+  },
+  {
+    "title": "Fine",
+    "video_path": "examples/displayed/fine.mp4",
+    "keypoints_path": "examples/displayed/fine_keypoints.json",
+    "keypoints_video_path": "examples/displayed/fine_keypoints.mp4"
+  },
+  {
+    "title": "Help",
+    "video_path": "examples/displayed/help.mp4",
+    "keypoints_path": "examples/displayed/help_keypoints.json",
+    "keypoints_video_path": "examples/displayed/help_keypoints.mp4"
+  },
+  {
+    "title": "No",
+    "video_path": "examples/displayed/no.mp4",
+    "keypoints_path": "examples/displayed/no_keypoints.json",
+    "keypoints_video_path": "examples/displayed/no_keypoints.mp4"
+  },
+  {
+    "title": "Woman",
+    "video_path": "examples/displayed/woman.mp4",
+    "keypoints_path": "examples/displayed/woman_keypoints.json",
+    "keypoints_video_path": "examples/displayed/woman_keypoints.mp4"
+  }
+]