| | --- |
| | library_name: py-feat |
| | pipeline_tag: image-feature-extraction |
| | license: mit |
| | --- |
| | |
| | # xgb_au |
| | |
| | ## Model Description |
| | xgb_au combines histogram of oriented gradient feature extraction with gradient boosting to predict facial action units from single frame images. |
| |
|
| | ## Model Details |
| | - **Model Type**: Gradient Boosting (XGB) |
| | - **Framework**: sklearn |
| |
|
| | ## Model Sources |
| | - **Repository**: [GitHub Repository](https://github.com/cosanlab/py-feat) |
| | - **Paper**: [Py-feat: Python facial expression analysis toolbox](https://link.springer.com/article/10.1007/s42761-023-00191-4) |
| |
|
| | ## Citation |
| | If you use the svm_au model in your research or application, please cite the following paper: |
| | |
| | Cheong, J.H., Jolly, E., Xie, T. et al. Py-Feat: Python Facial Expression Analysis Toolbox. Affec Sci 4, 781–796 (2023). https://doi.org/10.1007/s42761-023-00191-4 |
| | |
| | ``` |
| | @article{cheong2023py, |
| | title={Py-feat: Python facial expression analysis toolbox}, |
| | author={Cheong, Jin Hyun and Jolly, Eshin and Xie, Tiankang and Byrne, Sophie and Kenney, Matthew and Chang, Luke J}, |
| | journal={Affective Science}, |
| | volume={4}, |
| | number={4}, |
| | pages={781--796}, |
| | year={2023}, |
| | publisher={Springer} |
| | } |
| | ``` |
| | |
| | ## Example Useage |
| | |
| | ```python |
| | import numpy as np |
| | from skops.io import dump, load, get_untrusted_types |
| | from huggingface_hub import hf_hub_download |
| |
|
| | class XGBClassifier: |
| | def __init__(self) -> None: |
| | |
| | self.au_keys = [ |
| | "AU1", "AU2", "AU4", "AU5", "AU6", "AU7", "AU9", "AU10", "AU11", "AU12", |
| | "AU14", "AU15", "AU17", "AU20", "AU23", "AU24", "AU25", "AU26", "AU28", "AU43" |
| | ] |
| | self.weights_loaded = False |
| | |
| | def load_weights(self, scaler_upper=None, pca_model_upper=None, scaler_lower=None, pca_model_lower=None, scaler_full=None, pca_model_full=None, classifiers=None): |
| | self.scaler_upper = scaler_upper |
| | self.pca_model_upper = pca_model_upper |
| | self.scaler_lower = scaler_lower |
| | self.pca_model_lower = pca_model_lower |
| | self.scaler_full = scaler_full |
| | self.pca_model_full = pca_model_full |
| | self.classifiers = classifiers |
| | self.weights_loaded = True |
| | |
| | def pca_transform(self, frame, scaler, pca_model, landmarks): |
| | if not self.weights_loaded: |
| | raise ValueError('Need to load weights before running pca_transform') |
| | else: |
| | transformed_frame = pca_model.transform(scaler.transform(frame)) |
| | return np.concatenate((transformed_frame, landmarks), axis=1) |
| | |
| | def detect_au(self, frame, landmarks): |
| | if not self.weights_loaded: |
| | raise ValueError('Need to load weights before running detect_au') |
| | else: |
| | landmarks = np.concatenate(landmarks) |
| | landmarks = landmarks.reshape(-1, landmarks.shape[1] * landmarks.shape[2]) |
| | |
| | pca_transformed_upper = self.pca_transform(frame, self.scaler_upper, self.pca_model_upper, landmarks) |
| | pca_transformed_lower = self.pca_transform(frame, self.scaler_lower, self.pca_model_lower, landmarks) |
| | pca_transformed_full = self.pca_transform(frame, self.scaler_full, self.pca_model_full, landmarks) |
| | |
| | pred_aus = [] |
| | for key in self.au_keys: |
| | classifier = self.classifiers[key] |
| | |
| | if key in ["AU1", "AU2", "AU7"]: |
| | au_pred = classifier.predict_proba(pca_transformed_upper)[:, 1] |
| | elif key in ["AU11", "AU14", "AU17", "AU23", "AU24", "AU26"]: |
| | au_pred = classifier.predict_proba(pca_transformed_lower)[:, 1] |
| | else: |
| | au_pred = classifier.predict_proba(pca_transformed_full)[:, 1] |
| | |
| | pred_aus.append(au_pred) |
| | |
| | return np.array(pred_aus).T def __init__(self) -> None: |
| | self.weights_loaded = False |
| | |
| | def load_weights(self, scaler_upper=None, pca_model_upper=None, scaler_lower=None, pca_model_lower=None, scaler_full=None, pca_model_full=None, classifiers=None): |
| | self.scaler_upper = scaler_upper |
| | self.pca_model_upper = pca_model_upper |
| | self.scaler_lower = scaler_lower |
| | self.pca_model_lower = pca_model_lower |
| | self.scaler_full = scaler_full |
| | self.pca_model_full = pca_model_full |
| | self.classifiers = classifiers |
| | self.weights_loaded = True |
| | |
| | def pca_transform(self, frame, scaler, pca_model, landmarks): |
| | if not self.weights_loaded: |
| | raise ValueError('Need to load weights before running pca_transform') |
| | else: |
| | transformed_frame = pca_model.transform(scaler.transform(frame)) |
| | return np.concatenate((transformed_frame, landmarks), axis=1) |
| | |
| | def detect_au(self, frame, landmarks): |
| | """ |
| | Note that here frame is represented by hogs |
| | """ |
| | if not self.weights_loaded: |
| | raise ValueError('Need to load weights before running detect_au') |
| | else: |
| | landmarks = np.concatenate(landmarks) |
| | landmarks = landmarks.reshape(-1, landmarks.shape[1] * landmarks.shape[2]) |
| | |
| | pca_transformed_upper = self.pca_transform(frame, self.scaler_upper, self.pca_model_upper, landmarks) |
| | pca_transformed_lower = self.pca_transform(frame, self.scaler_lower, self.pca_model_lower, landmarks) |
| | pca_transformed_full = self.pca_transform(frame, self.scaler_full, self.pca_model_full, landmarks) |
| | |
| | aus_list = sorted(self.classifiers.keys(), key=lambda x: int(x[2::])) |
| | |
| | pred_aus = [] |
| | for keys in aus_list: |
| | if keys in ["AU1", "AU4", "AU6"]: |
| | au_pred = self.classifiers[keys].predict(pca_transformed_upper) |
| | elif keys in ["AU11", "AU12", "AU17"]: |
| | au_pred = self.classifiers[keys].predict(pca_transformed_lower) |
| | elif keys in [ |
| | "AU2", |
| | "AU5", |
| | "AU7", |
| | "AU9", |
| | "AU10", |
| | "AU14", |
| | "AU15", |
| | "AU20", |
| | "AU23", |
| | "AU24", |
| | "AU25", |
| | "AU26", |
| | "AU28", |
| | "AU43", |
| | ]: |
| | au_pred = self.classifiers[keys].predict(pca_transformed_full) |
| | else: |
| | raise ValueError("unknown AU detected") |
| | |
| | pred_aus.append(au_pred) |
| | pred_aus = np.array(pred_aus).T |
| | return pred_aus |
| | |
| | # Load model and weights |
| | au_model = XGBClassifier() |
| | model_path = hf_hub_download(repo_id="py-feat/xgb_au", filename="xgb_au_classifier.skops") |
| | unknown_types = get_untrusted_types(file=model_path) |
| | loaded_model = load(model_path, trusted=unknown_types) |
| | au_model.load_weights(scaler_upper = loaded_model.scaler_upper, |
| | pca_model_upper = loaded_model.pca_model_upper, |
| | scaler_lower = loaded_model.scaler_lower, |
| | pca_model_lower = loaded_model.scaler_full, |
| | pca_model_full=loaded_model.pca_model_full, |
| | classifiers=loaded_model.classifiers) |
| | |
| | # Test model |
| | frame = "path/to/your/test_image.jpg" # Replace with your loaded image |
| | landmarks = np.array([...]) # Replace with your landmarks data |
| | pred = au_model.detect_au(frame, landmarks) |
| | print(pred) |
| | |
| | ``` |
