Initial deployment of trained model weights and configs
Browse files- .gitattributes +1 -0
- README.md +198 -0
- model.py +439 -0
- model.safetensors +3 -0
- pose.jpg +3 -0
- requirements.txt +7 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
pose.jpg filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- en
|
| 5 |
+
tags:
|
| 6 |
+
- pose-estimation
|
| 7 |
+
- human-pose-estimation
|
| 8 |
+
- safetensors
|
| 9 |
+
- keypoint-detection
|
| 10 |
+
- computer-vision
|
| 11 |
+
- pytorch
|
| 12 |
+
- efficientnet
|
| 13 |
+
- coco
|
| 14 |
+
- mmpose
|
| 15 |
+
|
| 16 |
+
datasets:
|
| 17 |
+
- coco
|
| 18 |
+
- mpii
|
| 19 |
+
- crowdpose
|
| 20 |
+
- ochuman
|
| 21 |
+
metrics:
|
| 22 |
+
- ap
|
| 23 |
+
- ar
|
| 24 |
+
library_name: pytorch
|
| 25 |
+
pipeline_tag: keypoint-detection
|
| 26 |
+
---
|
| 27 |
+
|
| 28 |
+
# EfficientNet-B5 Pose Estimation
|
| 29 |
+
|
| 30 |
+
A 2D human pose estimation model trained at **DeKUT-DSAIL** using the [MMPose](https://github.com/open-mmlab/mmpose) framework. Predicts **17 COCO keypoints** from a single cropped person image.
|
| 31 |
+
|
| 32 |
+
| Property | Value |
|
| 33 |
+
|---|---|
|
| 34 |
+
| Backbone | EfficientNet-B5 |
|
| 35 |
+
| Attention Neck | None |
|
| 36 |
+
| Parameters | ~40 M |
|
| 37 |
+
| Input Size | 192 × 256 |
|
| 38 |
+
| Output | Heatmaps (17, 64, 48) |
|
| 39 |
+
|
| 40 |
+
---
|
| 41 |
+
|
| 42 |
+
## Evaluation Results
|
| 43 |
+
|
| 44 |
+
Evaluated on **COCO 2017 val** using OKS-based metrics (top-down, GT bounding boxes).
|
| 45 |
+
|
| 46 |
+
| Metric | Score |
|
| 47 |
+
|---|---|
|
| 48 |
+
| **COCO AP** | **0.713** |
|
| 49 |
+
| **COCO AR** | **0.748** |
|
| 50 |
+
|
| 51 |
+
---
|
| 52 |
+
|
| 53 |
+
## Repository Files
|
| 54 |
+
|
| 55 |
+
```
|
| 56 |
+
model.safetensors # Model weights (safetensors format)
|
| 57 |
+
model.py # Self-contained PoseEstimator inference helper
|
| 58 |
+
requirements.txt # Python dependencies
|
| 59 |
+
pose.jpg # Example test image
|
| 60 |
+
README.md # This model card
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
---
|
| 64 |
+
|
| 65 |
+
## Quick Start
|
| 66 |
+
|
| 67 |
+
### Step 1 — Clone the repository
|
| 68 |
+
|
| 69 |
+
```bash
|
| 70 |
+
git clone https://huggingface.co/DeKUT-DSAIL/efficientnet_b5_coco_256x192
|
| 71 |
+
cd efficientnet_b5_coco_256x192
|
| 72 |
+
```
|
| 73 |
+
|
| 74 |
+
### Step 2 — Create a virtual environment
|
| 75 |
+
|
| 76 |
+
**Linux / macOS**
|
| 77 |
+
```bash
|
| 78 |
+
python -m venv venv
|
| 79 |
+
source venv/bin/activate
|
| 80 |
+
```
|
| 81 |
+
|
| 82 |
+
**Windows (Command Prompt)**
|
| 83 |
+
```cmd
|
| 84 |
+
python -m venv venv
|
| 85 |
+
venv\Scripts\activate.bat
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
**Windows (PowerShell)**
|
| 89 |
+
```powershell
|
| 90 |
+
python -m venv venv
|
| 91 |
+
venv\Scripts\Activate.ps1
|
| 92 |
+
```
|
| 93 |
+
|
| 94 |
+
### Step 3 — Install dependencies
|
| 95 |
+
|
| 96 |
+
```bash
|
| 97 |
+
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
|
| 98 |
+
pip install -r requirements.txt
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
> **GPU users:** Replace the PyTorch URL with your CUDA version.
|
| 102 |
+
> See [pytorch.org/get-started](https://pytorch.org/get-started/locally/).
|
| 103 |
+
|
| 104 |
+
### Step 4 — Run inference
|
| 105 |
+
|
| 106 |
+
```python
|
| 107 |
+
import cv2
|
| 108 |
+
from model import PoseEstimator
|
| 109 |
+
|
| 110 |
+
estimator = PoseEstimator("DeKUT-DSAIL/efficientnet_b5_coco_256x192")
|
| 111 |
+
|
| 112 |
+
image = cv2.imread("pose.jpg")
|
| 113 |
+
keypoints, scores = estimator.predict(image)
|
| 114 |
+
|
| 115 |
+
print("Keypoints shape:", keypoints.shape) # (N, 17, 2)
|
| 116 |
+
print("Scores shape: ", scores.shape) # (N, 17, 1)
|
| 117 |
+
|
| 118 |
+
annotated = estimator.visualize(image, keypoints, scores, score_threshold=0.3)
|
| 119 |
+
cv2.imwrite("output.jpg", annotated)
|
| 120 |
+
print("Saved output.jpg")
|
| 121 |
+
```
|
| 122 |
+
|
| 123 |
+
---
|
| 124 |
+
|
| 125 |
+
## Input / Output Specification
|
| 126 |
+
|
| 127 |
+
| Property | Value |
|
| 128 |
+
|---|---|
|
| 129 |
+
| Input size | `(1, 3, 256, 192)` — RGB, channel-first |
|
| 130 |
+
| Normalisation | Mean `[0.485, 0.456, 0.406]` / Std `[0.229, 0.224, 0.225]` |
|
| 131 |
+
| Output | Heatmaps `(N, 17, 64, 48)` |
|
| 132 |
+
| Keypoints | COCO 17-joint format |
|
| 133 |
+
|
| 134 |
+
### COCO 17 Keypoints
|
| 135 |
+
|
| 136 |
+
| Index | Name | Index | Name |
|
| 137 |
+
|---|---|---|---|
|
| 138 |
+
| 0 | nose | 9 | left_wrist |
|
| 139 |
+
| 1 | left_eye | 10 | right_wrist |
|
| 140 |
+
| 2 | right_eye | 11 | left_hip |
|
| 141 |
+
| 3 | left_ear | 12 | right_hip |
|
| 142 |
+
| 4 | right_ear | 13 | left_knee |
|
| 143 |
+
| 5 | left_shoulder | 14 | right_knee |
|
| 144 |
+
| 6 | right_shoulder | 15 | left_ankle |
|
| 145 |
+
| 7 | left_elbow | 16 | right_ankle |
|
| 146 |
+
| 8 | right_elbow | | |
|
| 147 |
+
|
| 148 |
+
---
|
| 149 |
+
|
| 150 |
+
## Training Details
|
| 151 |
+
|
| 152 |
+
Trained using [MMPose](https://github.com/open-mmlab/mmpose) on the following datasets:
|
| 153 |
+
|
| 154 |
+
| Dataset | Link |
|
| 155 |
+
|---|---|
|
| 156 |
+
| **COCO 2017** | [cocodataset.org](https://cocodataset.org/#keypoints-2017) |
|
| 157 |
+
| **MPII Human Pose** | [mpii.is.tue.mpg.de](http://human-pose.mpi-inf.mpg.de/) |
|
| 158 |
+
| **CrowdPose** | [GitHub](https://github.com/Jeff-sjtu/CrowdPose) |
|
| 159 |
+
| **OCHuman** | [GitHub](https://github.com/liruilong940607/OCHumanApi) |
|
| 160 |
+
|
| 161 |
+
| Parameter | Value |
|
| 162 |
+
|---|---|
|
| 163 |
+
| Optimizer | AdamW |
|
| 164 |
+
| Learning rate | 1 × 10⁻³ |
|
| 165 |
+
| LR schedule | Multi-step decay |
|
| 166 |
+
| Batch size | 64 |
|
| 167 |
+
| Epochs | 210 |
|
| 168 |
+
| Input size | 192 × 256 |
|
| 169 |
+
| Loss | MSE on heatmaps + knowledge distillation loss |
|
| 170 |
+
|
| 171 |
+
---
|
| 172 |
+
|
| 173 |
+
## Architecture
|
| 174 |
+
|
| 175 |
+
```
|
| 176 |
+
Input Image (3, 256, 192)
|
| 177 |
+
│
|
| 178 |
+
▼
|
| 179 |
+
EfficientNet-B5 Backbone
|
| 180 |
+
│
|
| 181 |
+
│
|
| 182 |
+
▼
|
| 183 |
+
HeatmapHead (3× deconv + 1×1 conv)
|
| 184 |
+
│
|
| 185 |
+
▼
|
| 186 |
+
Output Heatmaps (17, 64, 48)
|
| 187 |
+
```
|
| 188 |
+
|
| 189 |
+
---
|
| 190 |
+
|
| 191 |
+
## Developed by
|
| 192 |
+
|
| 193 |
+
**DeKUT-DSAIL** — Dedan Kimathi University of Technology
|
| 194 |
+
|
| 195 |
+
- Framework: PyTorch / MMPose
|
| 196 |
+
- Model type: 2D Human Pose Estimation
|
| 197 |
+
- Task: Keypoint Detection
|
| 198 |
+
- License: Apache 2.0
|
model.py
ADDED
|
@@ -0,0 +1,439 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from typing import Optional, Sequence, Tuple
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.distributed as dist
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
from torch import Tensor
|
| 11 |
+
import warnings
|
| 12 |
+
import logging
|
| 13 |
+
import numpy as np
|
| 14 |
+
import cv2
|
| 15 |
+
from safetensors.torch import load_file
|
| 16 |
+
from huggingface_hub import hf_hub_download
|
| 17 |
+
|
| 18 |
+
class _EfficientNetBackbone(nn.Module):
|
| 19 |
+
_LAST_CHANNELS: dict[str, int] = {
|
| 20 |
+
'b0': 1280, 'b1': 1280, 'b2': 1408,
|
| 21 |
+
'b3': 1536, 'b4': 1792, 'b5': 2048,
|
| 22 |
+
'b6': 2304, 'b7': 2560,
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
variant: str = 'b5',
|
| 28 |
+
pretrained: bool = False,
|
| 29 |
+
out_indices: Tuple[int, ...] = (8,),
|
| 30 |
+
frozen_stages: int = -1,
|
| 31 |
+
norm_eval: bool = False,
|
| 32 |
+
) -> None:
|
| 33 |
+
super().__init__()
|
| 34 |
+
|
| 35 |
+
variant = variant.lower()
|
| 36 |
+
assert variant in self._LAST_CHANNELS, (
|
| 37 |
+
f"Unknown EfficientNet variant '{variant}'. "
|
| 38 |
+
f"Choose from {list(self._LAST_CHANNELS)}."
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
self.variant = variant
|
| 42 |
+
self.out_indices = out_indices
|
| 43 |
+
self.frozen_stages = frozen_stages
|
| 44 |
+
self.norm_eval = norm_eval
|
| 45 |
+
self.out_channels = self._LAST_CHANNELS[variant]
|
| 46 |
+
import torchvision.models as tvm
|
| 47 |
+
|
| 48 |
+
weights_arg = 'DEFAULT' if pretrained else None
|
| 49 |
+
builder = getattr(tvm, f'efficientnet_{variant}')
|
| 50 |
+
is_dist = dist.is_available() and dist.is_initialized()
|
| 51 |
+
local_rank = int(os.environ.get('LOCAL_RANK', 0))
|
| 52 |
+
if is_dist and local_rank != 0:
|
| 53 |
+
dist.barrier()
|
| 54 |
+
tv_model = builder(weights=weights_arg)
|
| 55 |
+
if is_dist and local_rank == 0:
|
| 56 |
+
dist.barrier()
|
| 57 |
+
|
| 58 |
+
self.features: nn.Sequential = tv_model.features
|
| 59 |
+
self.classifier = tv_model.classifier
|
| 60 |
+
|
| 61 |
+
self._freeze_stages()
|
| 62 |
+
|
| 63 |
+
def _freeze_stages(self) -> None:
|
| 64 |
+
for i, layer in enumerate(self.features):
|
| 65 |
+
if i <= self.frozen_stages:
|
| 66 |
+
layer.eval()
|
| 67 |
+
for param in layer.parameters():
|
| 68 |
+
param.requires_grad = False
|
| 69 |
+
|
| 70 |
+
def train(self, mode: bool = True) -> 'EfficientNetBackbone':
|
| 71 |
+
super().train(mode)
|
| 72 |
+
self._freeze_stages()
|
| 73 |
+
if mode and self.norm_eval:
|
| 74 |
+
for m in self.modules():
|
| 75 |
+
if isinstance(m, (nn.BatchNorm2d, nn.SyncBatchNorm)):
|
| 76 |
+
m.eval()
|
| 77 |
+
return self
|
| 78 |
+
|
| 79 |
+
def forward(self, x: Tensor) -> Tuple[Tensor, ...]:
|
| 80 |
+
outs = []
|
| 81 |
+
for i, layer in enumerate(self.features):
|
| 82 |
+
x = layer(x)
|
| 83 |
+
if i in self.out_indices:
|
| 84 |
+
outs.append(x)
|
| 85 |
+
return tuple(outs)
|
| 86 |
+
|
| 87 |
+
class HeatmapHead(nn.Module):
|
| 88 |
+
def __init__(
|
| 89 |
+
self,
|
| 90 |
+
in_channels: int,
|
| 91 |
+
out_channels: int,
|
| 92 |
+
deconv_out_channels: Sequence[int] = (256, 256, 256),
|
| 93 |
+
deconv_kernel_sizes: Sequence[int] = (4, 4, 4),
|
| 94 |
+
conv_out_channels: Optional[Sequence[int]] = None,
|
| 95 |
+
conv_kernel_sizes: Optional[Sequence[int]] = None,
|
| 96 |
+
final_kernel_size: int = 1,
|
| 97 |
+
) -> None:
|
| 98 |
+
super().__init__()
|
| 99 |
+
|
| 100 |
+
self.in_channels = in_channels
|
| 101 |
+
self.out_channels = out_channels
|
| 102 |
+
|
| 103 |
+
if deconv_out_channels:
|
| 104 |
+
assert len(deconv_out_channels) == len(deconv_kernel_sizes), (
|
| 105 |
+
"'deconv_out_channels' and 'deconv_kernel_sizes' must have "
|
| 106 |
+
"equal length."
|
| 107 |
+
)
|
| 108 |
+
self.deconv_layers = self._make_deconv_layers(
|
| 109 |
+
in_channels, deconv_out_channels, deconv_kernel_sizes
|
| 110 |
+
)
|
| 111 |
+
in_channels = deconv_out_channels[-1]
|
| 112 |
+
else:
|
| 113 |
+
self.deconv_layers = nn.Identity()
|
| 114 |
+
|
| 115 |
+
if conv_out_channels:
|
| 116 |
+
assert conv_kernel_sizes is not None and len(
|
| 117 |
+
conv_out_channels) == len(conv_kernel_sizes), (
|
| 118 |
+
"'conv_out_channels' and 'conv_kernel_sizes' must have "
|
| 119 |
+
"equal length."
|
| 120 |
+
)
|
| 121 |
+
self.conv_layers = self._make_conv_layers(
|
| 122 |
+
in_channels, conv_out_channels, conv_kernel_sizes
|
| 123 |
+
)
|
| 124 |
+
in_channels = conv_out_channels[-1]
|
| 125 |
+
else:
|
| 126 |
+
self.conv_layers = nn.Identity()
|
| 127 |
+
|
| 128 |
+
pad = (final_kernel_size - 1) // 2
|
| 129 |
+
self.final_layer = nn.Conv2d(
|
| 130 |
+
in_channels, out_channels,
|
| 131 |
+
kernel_size=final_kernel_size,
|
| 132 |
+
padding=pad,
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
self._init_weights()
|
| 136 |
+
|
| 137 |
+
def _init_weights(self) -> None:
|
| 138 |
+
for m in self.modules():
|
| 139 |
+
if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
|
| 140 |
+
nn.init.normal_(m.weight, std=0.001)
|
| 141 |
+
if m.bias is not None:
|
| 142 |
+
nn.init.zeros_(m.bias)
|
| 143 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 144 |
+
nn.init.ones_(m.weight)
|
| 145 |
+
nn.init.zeros_(m.bias)
|
| 146 |
+
|
| 147 |
+
@staticmethod
|
| 148 |
+
def _make_deconv_layers(
|
| 149 |
+
in_channels: int,
|
| 150 |
+
out_channels_list: Sequence[int],
|
| 151 |
+
kernel_sizes: Sequence[int],
|
| 152 |
+
) -> nn.Sequential:
|
| 153 |
+
layers: list[nn.Module] = []
|
| 154 |
+
for out_ch, ks in zip(out_channels_list, kernel_sizes):
|
| 155 |
+
if ks == 4:
|
| 156 |
+
padding, output_padding = 1, 0
|
| 157 |
+
elif ks == 3:
|
| 158 |
+
padding, output_padding = 1, 1
|
| 159 |
+
elif ks == 2:
|
| 160 |
+
padding, output_padding = 0, 0
|
| 161 |
+
else:
|
| 162 |
+
raise ValueError(
|
| 163 |
+
f"Unsupported deconv kernel size {ks}. Use 2, 3, or 4."
|
| 164 |
+
)
|
| 165 |
+
layers += [
|
| 166 |
+
nn.ConvTranspose2d(
|
| 167 |
+
in_channels, out_ch,
|
| 168 |
+
kernel_size=ks, stride=2,
|
| 169 |
+
padding=padding, output_padding=output_padding,
|
| 170 |
+
bias=False,
|
| 171 |
+
),
|
| 172 |
+
nn.BatchNorm2d(out_ch),
|
| 173 |
+
nn.ReLU(inplace=True),
|
| 174 |
+
]
|
| 175 |
+
in_channels = out_ch
|
| 176 |
+
return nn.Sequential(*layers)
|
| 177 |
+
|
| 178 |
+
@staticmethod
|
| 179 |
+
def _make_conv_layers(
|
| 180 |
+
in_channels: int,
|
| 181 |
+
out_channels_list: Sequence[int],
|
| 182 |
+
kernel_sizes: Sequence[int],
|
| 183 |
+
) -> nn.Sequential:
|
| 184 |
+
layers: list[nn.Module] = []
|
| 185 |
+
for out_ch, ks in zip(out_channels_list, kernel_sizes):
|
| 186 |
+
padding = (ks - 1) // 2
|
| 187 |
+
layers += [
|
| 188 |
+
nn.Conv2d(in_channels, out_ch,
|
| 189 |
+
kernel_size=ks, stride=1, padding=padding),
|
| 190 |
+
nn.BatchNorm2d(out_ch),
|
| 191 |
+
nn.ReLU(inplace=True),
|
| 192 |
+
]
|
| 193 |
+
in_channels = out_ch
|
| 194 |
+
return nn.Sequential(*layers)
|
| 195 |
+
|
| 196 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 197 |
+
x = self.deconv_layers(x)
|
| 198 |
+
x = self.conv_layers(x)
|
| 199 |
+
x = self.final_layer(x)
|
| 200 |
+
return x
|
| 201 |
+
|
| 202 |
+
class EfficientNetB5PoseNet(nn.Module):
|
| 203 |
+
def __init__(
|
| 204 |
+
self,
|
| 205 |
+
num_keypoints: int = 17,
|
| 206 |
+
pretrained: bool = False,
|
| 207 |
+
frozen_stages: int = -1,
|
| 208 |
+
norm_eval: bool = False,
|
| 209 |
+
deconv_out_channels: Tuple[int, ...] = (256, 256, 256),
|
| 210 |
+
deconv_kernel_sizes: Tuple[int, ...] = (4, 4, 4),
|
| 211 |
+
) -> None:
|
| 212 |
+
super().__init__()
|
| 213 |
+
|
| 214 |
+
self.backbone = _EfficientNetBackbone(
|
| 215 |
+
variant='b5',
|
| 216 |
+
pretrained=pretrained,
|
| 217 |
+
out_indices=(8,),
|
| 218 |
+
frozen_stages=frozen_stages,
|
| 219 |
+
norm_eval=norm_eval,
|
| 220 |
+
)
|
| 221 |
+
backbone_out_ch = self.backbone.out_channels
|
| 222 |
+
self.head = HeatmapHead(
|
| 223 |
+
in_channels=backbone_out_ch,
|
| 224 |
+
out_channels=num_keypoints,
|
| 225 |
+
deconv_out_channels=deconv_out_channels,
|
| 226 |
+
deconv_kernel_sizes=deconv_kernel_sizes,
|
| 227 |
+
)
|
| 228 |
+
|
| 229 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 230 |
+
feats: Tuple[Tensor, ...] = self.backbone(x)
|
| 231 |
+
feat: Tensor = feats[-1]
|
| 232 |
+
heatmaps: Tensor = self.head(feat)
|
| 233 |
+
return heatmaps
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
DEFAULT_INPUT_SIZE = (192, 256)
|
| 237 |
+
|
| 238 |
+
class PoseEstimator:
|
| 239 |
+
def __init__(self, model_name, num_keypoints=17, device=None, input_size=DEFAULT_INPUT_SIZE):
|
| 240 |
+
if device is None:
|
| 241 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 242 |
+
self.device = torch.device(device)
|
| 243 |
+
self.input_size = input_size
|
| 244 |
+
self.model_name = model_name
|
| 245 |
+
self.model = EfficientNetB5PoseNet(num_keypoints=num_keypoints)
|
| 246 |
+
if os.path.isfile(model_name):
|
| 247 |
+
weights_path = model_name
|
| 248 |
+
elif os.path.isdir(model_name):
|
| 249 |
+
weights_path = os.path.join(model_name, "model.safetensors")
|
| 250 |
+
else:
|
| 251 |
+
weights_path = hf_hub_download(repo_id=model_name, filename="model.safetensors")
|
| 252 |
+
state_dict = load_file(weights_path, device=str(self.device))
|
| 253 |
+
self.model.load_state_dict(state_dict, strict=False)
|
| 254 |
+
self.model.to(self.device)
|
| 255 |
+
self.model.eval()
|
| 256 |
+
self.num_keypoints = num_keypoints
|
| 257 |
+
|
| 258 |
+
@staticmethod
|
| 259 |
+
def _get_centers_and_scales_xyxy(person_boxes, scale_factor=1.0):
|
| 260 |
+
centers, scales = [], []
|
| 261 |
+
for box in person_boxes:
|
| 262 |
+
x1, y1, x2, y2 = box
|
| 263 |
+
x1, x2 = sorted([x1, x2])
|
| 264 |
+
y1, y2 = sorted([y1, y2])
|
| 265 |
+
centers.append([(x1+x2)/2.0, (y1+y2)/2.0])
|
| 266 |
+
w, h = x2-x1, y2-y1
|
| 267 |
+
scales.append([(w/200.0)*scale_factor, (h/200.0)*scale_factor])
|
| 268 |
+
return np.array(centers), np.array(scales)
|
| 269 |
+
|
| 270 |
+
@staticmethod
|
| 271 |
+
def _process_image(image, bbox, target_size, angle=0, flip=False):
|
| 272 |
+
try:
|
| 273 |
+
if image is None or not isinstance(image, np.ndarray):
|
| 274 |
+
raise ValueError("Invalid image input.")
|
| 275 |
+
x1, y1, x2, y2 = map(lambda v: int(round(v)), bbox)
|
| 276 |
+
if x2-x1 <= 0 or y2-y1 <= 0:
|
| 277 |
+
raise ValueError(f"Invalid bbox: {{bbox}}")
|
| 278 |
+
x1, y1 = max(0, x1), max(0, y1)
|
| 279 |
+
x2, y2 = min(image.shape[1], x2), min(image.shape[0], y2)
|
| 280 |
+
if x2 <= x1 or y2 <= y1:
|
| 281 |
+
raise ValueError("Invalid bbox after clamping.")
|
| 282 |
+
cropped = image[y1:y2, x1:x2]
|
| 283 |
+
resized = cv2.resize(cropped, target_size)
|
| 284 |
+
if angle != 0:
|
| 285 |
+
center = (target_size[0]//2, target_size[1]//2)
|
| 286 |
+
rot = cv2.getRotationMatrix2D(center, angle, 1.0)
|
| 287 |
+
resized = cv2.warpAffine(resized, rot, target_size)
|
| 288 |
+
if flip:
|
| 289 |
+
resized = cv2.flip(resized, 1)
|
| 290 |
+
return resized, True
|
| 291 |
+
except Exception:
|
| 292 |
+
blank = np.zeros((target_size[1], target_size[0], 3), dtype=np.uint8)
|
| 293 |
+
return blank, False
|
| 294 |
+
|
| 295 |
+
@staticmethod
|
| 296 |
+
def _process(image, target_size=(192, 256), angle=0, flip=False, conf_threshold=0.5, model_weights="yolov8n.pt"):
|
| 297 |
+
try:
|
| 298 |
+
from ultralytics import YOLO
|
| 299 |
+
except ImportError:
|
| 300 |
+
raise ImportError("ultralytics is required. pip install ultralytics")
|
| 301 |
+
model = YOLO(model_weights)
|
| 302 |
+
crops, metadata = [], []
|
| 303 |
+
if image is None or not isinstance(image, np.ndarray):
|
| 304 |
+
raise ValueError("Invalid image input.")
|
| 305 |
+
results = model(image, conf=conf_threshold, classes=[0], verbose=False)
|
| 306 |
+
bboxes = []
|
| 307 |
+
for r in results:
|
| 308 |
+
for box in r.boxes:
|
| 309 |
+
bboxes.append(box.xyxy[0].cpu().numpy().tolist())
|
| 310 |
+
for idx, bbox in enumerate(bboxes):
|
| 311 |
+
processed, success = PoseEstimator._process_image(image, bbox, target_size, angle, flip)
|
| 312 |
+
crops.append(processed)
|
| 313 |
+
metadata.append({"bbox": bbox, "person_index": idx, "success": success})
|
| 314 |
+
if not crops:
|
| 315 |
+
return None, metadata
|
| 316 |
+
batch = np.stack(crops, axis=0).transpose(0, 3, 1, 2)
|
| 317 |
+
return np.ascontiguousarray(batch), metadata
|
| 318 |
+
|
| 319 |
+
def _preprocess(self, image_bgr):
|
| 320 |
+
batch, meta = self._process(image_bgr, target_size=self.input_size)
|
| 321 |
+
if batch is None:
|
| 322 |
+
return None, meta
|
| 323 |
+
t = torch.tensor(batch, dtype=torch.float32) / 255.0
|
| 324 |
+
return t.to(self.device), meta
|
| 325 |
+
|
| 326 |
+
@staticmethod
|
| 327 |
+
def _taylor(heatmap, coord):
|
| 328 |
+
H, W = heatmap.shape[:2]
|
| 329 |
+
px, py = int(coord[0]), int(coord[1])
|
| 330 |
+
if 1 < px < W-2 and 1 < py < H-2:
|
| 331 |
+
dx = 0.5*(heatmap[py][px+1]-heatmap[py][px-1])
|
| 332 |
+
dy = 0.5*(heatmap[py+1][px]-heatmap[py-1][px])
|
| 333 |
+
dxx = 0.25*(heatmap[py][px+2]-2*heatmap[py][px]+heatmap[py][px-2])
|
| 334 |
+
dxy = 0.25*(heatmap[py+1][px+1]-heatmap[py-1][px+1]-heatmap[py+1][px-1]+heatmap[py-1][px-1])
|
| 335 |
+
dyy = 0.25*(heatmap[py+2][px]-2*heatmap[py][px]+heatmap[py-2][px])
|
| 336 |
+
derivative = np.array([[dx],[dy]])
|
| 337 |
+
hessian = np.array([[dxx,dxy],[dxy,dyy]])
|
| 338 |
+
if dxx*dyy - dxy**2 != 0:
|
| 339 |
+
offset = -np.linalg.inv(hessian) @ derivative
|
| 340 |
+
coord += np.squeeze(offset.T, axis=0)
|
| 341 |
+
return coord
|
| 342 |
+
|
| 343 |
+
@staticmethod
|
| 344 |
+
def _get_max_preds(heatmaps):
|
| 345 |
+
N, K, _, W = heatmaps.shape
|
| 346 |
+
reshaped = heatmaps.reshape((N, K, -1))
|
| 347 |
+
idx = np.argmax(reshaped, 2).reshape((N, K, 1))
|
| 348 |
+
maxvals = np.amax(reshaped, 2).reshape((N, K, 1))
|
| 349 |
+
preds = np.tile(idx, (1, 1, 2)).astype(np.float32)
|
| 350 |
+
preds[:,:,0] = preds[:,:,0] % W
|
| 351 |
+
preds[:,:,1] = np.floor(preds[:,:,1] / W)
|
| 352 |
+
preds = np.where(np.tile(maxvals, (1, 1, 2)) > 0.0, preds, -1)
|
| 353 |
+
return preds, maxvals
|
| 354 |
+
|
| 355 |
+
@staticmethod
|
| 356 |
+
def _gaussian_blur(heatmaps, kernel=11):
|
| 357 |
+
border = (kernel-1)//2
|
| 358 |
+
B, J, H, W = heatmaps.shape
|
| 359 |
+
for i in range(B):
|
| 360 |
+
for j in range(J):
|
| 361 |
+
origin_max = np.max(heatmaps[i,j])
|
| 362 |
+
dr = np.zeros((H+2*border, W+2*border), dtype=np.float32)
|
| 363 |
+
dr[border:-border, border:-border] = heatmaps[i,j].copy()
|
| 364 |
+
dr = cv2.GaussianBlur(dr, (kernel, kernel), 0)
|
| 365 |
+
heatmaps[i,j] = dr[border:-border, border:-border].copy()
|
| 366 |
+
heatmaps[i,j] *= origin_max / np.max(heatmaps[i,j])
|
| 367 |
+
return heatmaps
|
| 368 |
+
|
| 369 |
+
@staticmethod
|
| 370 |
+
def transform_preds(coords, center, scale, output_size, use_udp=False):
|
| 371 |
+
scale = scale * 200.0
|
| 372 |
+
if use_udp:
|
| 373 |
+
sx = scale[0]/(output_size[0]-1.0)
|
| 374 |
+
sy = scale[1]/(output_size[1]-1.0)
|
| 375 |
+
else:
|
| 376 |
+
sx = scale[0]/output_size[0]
|
| 377 |
+
sy = scale[1]/output_size[1]
|
| 378 |
+
tc = np.ones_like(coords)
|
| 379 |
+
tc[:,0] = coords[:,0]*sx + center[0] - scale[0]*0.5
|
| 380 |
+
tc[:,1] = coords[:,1]*sy + center[1] - scale[1]*0.5
|
| 381 |
+
return tc
|
| 382 |
+
|
| 383 |
+
@staticmethod
|
| 384 |
+
def keypoints_from_heatmaps(heatmaps, center, scale, unbiased=False, post_process="default", kernel=11, use_udp=False, target_type="GaussianHeatmap"):
|
| 385 |
+
heatmaps = heatmaps.copy()
|
| 386 |
+
if unbiased:
|
| 387 |
+
assert post_process not in [False, None, "megvii"]
|
| 388 |
+
if post_process == "default" and unbiased:
|
| 389 |
+
post_process = "unbiased"
|
| 390 |
+
if post_process == "megvii":
|
| 391 |
+
heatmaps = PoseEstimator._gaussian_blur(heatmaps, kernel=kernel)
|
| 392 |
+
N, K, H, W = heatmaps.shape
|
| 393 |
+
preds, maxvals = PoseEstimator._get_max_preds(heatmaps)
|
| 394 |
+
if post_process == "unbiased":
|
| 395 |
+
heatmaps = np.log(np.maximum(PoseEstimator._gaussian_blur(heatmaps, kernel), 1e-10))
|
| 396 |
+
for n in range(N):
|
| 397 |
+
for k in range(K):
|
| 398 |
+
preds[n][k] = PoseEstimator._taylor(heatmaps[n][k], preds[n][k])
|
| 399 |
+
elif post_process is not None and post_process != "megvii":
|
| 400 |
+
for n in range(N):
|
| 401 |
+
for k in range(K):
|
| 402 |
+
hm = heatmaps[n][k]
|
| 403 |
+
px, py = int(preds[n][k][0]), int(preds[n][k][1])
|
| 404 |
+
if 1 < px < W-1 and 1 < py < H-1:
|
| 405 |
+
diff = np.array([hm[py][px+1]-hm[py][px-1], hm[py+1][px]-hm[py-1][px]])
|
| 406 |
+
preds[n][k] += np.sign(diff)*0.25
|
| 407 |
+
for i in range(N):
|
| 408 |
+
preds[i] = PoseEstimator.transform_preds(preds[i], center[i], scale[i], [W, H], use_udp=use_udp)
|
| 409 |
+
if post_process == "megvii":
|
| 410 |
+
maxvals = maxvals/255.0 + 0.5
|
| 411 |
+
return preds, maxvals
|
| 412 |
+
|
| 413 |
+
@torch.no_grad()
|
| 414 |
+
def predict(self, image_bgr):
|
| 415 |
+
tensor, meta = self._preprocess(image_bgr)
|
| 416 |
+
if tensor is None:
|
| 417 |
+
return np.array([]), np.array([])
|
| 418 |
+
centers, scales = self._get_centers_and_scales_xyxy([m["bbox"] for m in meta])
|
| 419 |
+
output = self.model(tensor).detach().cpu().numpy()
|
| 420 |
+
kps, scores = self.keypoints_from_heatmaps(output, centers, scales, unbiased=True, post_process="default", target_type="GaussianHeatmap", kernel=11)
|
| 421 |
+
return kps, scores
|
| 422 |
+
|
| 423 |
+
@staticmethod
|
| 424 |
+
def visualize(image_bgr, keypoints, scores, score_threshold=0.3, kp_radius=8, line_thickness=5):
|
| 425 |
+
canvas = image_bgr.copy()
|
| 426 |
+
if keypoints.ndim == 2:
|
| 427 |
+
keypoints = np.expand_dims(keypoints, axis=0)
|
| 428 |
+
scores = np.expand_dims(scores, axis=0)
|
| 429 |
+
edges = [(0,1),(0,2),(1,3),(2,4),(5,6),(5,11),(6,12),(11,12),(5,7),(7,9),(6,8),(8,10),(11,13),(13,15),(12,14),(14,16)]
|
| 430 |
+
colors = [(255,0,0),(255,85,0),(255,170,0),(255,255,0),(170,255,0),(85,255,0),(0,255,0),(0,255,85),(0,255,170),(0,255,255),(0,170,255),(0,85,255),(0,0,255),(85,0,255),(170,0,255),(255,0,255)]
|
| 431 |
+
for n in range(len(keypoints)):
|
| 432 |
+
kpts, scs = keypoints[n], scores[n].squeeze()
|
| 433 |
+
for i,(a,b) in enumerate(edges):
|
| 434 |
+
if scs[a]>=score_threshold and scs[b]>=score_threshold:
|
| 435 |
+
cv2.line(canvas,(int(kpts[a][0]),int(kpts[a][1])),(int(kpts[b][0]),int(kpts[b][1])),colors[i],thickness=line_thickness)
|
| 436 |
+
for k in range(len(kpts)):
|
| 437 |
+
if scs[k]>=score_threshold:
|
| 438 |
+
cv2.circle(canvas,(int(kpts[k,0]),int(kpts[k,1])),kp_radius,color=(255,255,255),thickness=-1)
|
| 439 |
+
return canvas
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c0747e25c2d435e77ff7cedb72e68cdb2e899e3041d2918c92c5cbc91b6ec8a4
|
| 3 |
+
size 156121564
|
pose.jpg
ADDED
|
Git LFS Details
|
requirements.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.0.0
|
| 2 |
+
torchvision>=0.15.0
|
| 3 |
+
safetensors>=0.4.0
|
| 4 |
+
numpy>=1.24.0
|
| 5 |
+
opencv-python>=4.8.0
|
| 6 |
+
ultralytics>=8.0.0
|
| 7 |
+
huggingface_hub>=0.20.0
|