EquiFusion / src /inference_utils.py
ceccurr's picture
Update to the released code
aed3cce verified
Raw History Blame Contribute Delete
12.4 kB
import os
import yaml
os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") # deterministic cuBLAS (as in the scripts)
from hydra import initialize, compose
from omegaconf import OmegaConf
import src.core as model_arch
from .utils.config import init_obj
from .utils.config import load_and_merge_autoenc_cfg
if not OmegaConf.has_resolver("eval"): # used by the configs (registered by the scripts when run from the command line)
OmegaConf.register_new_resolver("eval", eval)
def create_model(device, **kwargs,):
model = init_obj(kwargs, 'arch', model_arch).to(device)
return model
def load_model_config_exp(checkpoint_path):
exp_folder = os.path.dirname(os.path.dirname(checkpoint_path))
config_path = os.path.join(exp_folder, 'config.yaml')
with open(config_path, 'r') as stream:
cfg_diffusion = yaml.safe_load(stream)
ae_path = cfg_diffusion['pretrained_autoencoder_path']
if not os.path.isabs(ae_path): # released models: path relative to the model folder
cfg_diffusion['pretrained_autoencoder_path'] = os.path.join(exp_folder, ae_path)
cfg = load_and_merge_autoenc_cfg(cfg_diffusion)
return cfg, exp_folder
def load_model_config_exp_autoenc(checkpoint_path):
exp_folder = os.path.dirname(os.path.dirname(checkpoint_path))
config_path = os.path.join(exp_folder, 'config.yaml')
with open(config_path, 'r') as stream:
cfg = yaml.safe_load(stream)
return cfg, exp_folder
def quick_cfg_for_inference_no_hydra(checkpoint_path, dataset_name, num_samples=50, split='valid'):
with initialize(config_path="../configs/config_eval", version_base="1.3.2"):
cfg = compose(
config_name="config.yaml",
overrides=[
f"dataset={dataset_name}",
"task=hmp",
f"dataset_split={split}",
"stats_mode=deterministic",
f"num_samples={num_samples}",
f"batch_size=1",
f"checkpoint_path='{checkpoint_path}'",
]
)
cfg = OmegaConf.to_container(cfg, resolve=True)
# for backwards compatibility with the code
for subconf in ['task', 'dataset']:
if subconf in cfg:
cfg = {**cfg, **cfg[subconf]}
cfg.pop(subconf)
from .utils.config import merge_cfg, make_paths_absolute
cfg = make_paths_absolute(cfg)
# load config
cfg_orig, exp_folder = load_model_config_exp(checkpoint_path)
cfg_orig.pop("dataset", None) # training datasets: the skeleton is the one of the evaluated dataset
# merge original experiment config with the current evaluation config
cfg = merge_cfg(cfg, cfg_orig)
cfg = merge_cfg(cfg, cfg['method_specs'])
return cfg
# ----------------------------------------------------------------------------------------------------------
# Inference API (used by the notebooks and the demo)
# ----------------------------------------------------------------------------------------------------------
import types
import numpy as np
import torch
LIMB_NAMES = {"head": 0, "left_arm": 1, "right_arm": 2, "left_leg": 3, "right_leg": 4} # see Kinematic.idx_to_limb
def download_equifusion(repo_id: str, local_dir: str = None) -> str:
"""Downloads a released model folder from the Hugging Face Hub and returns the path of its checkpoint,
to be given to `load_equifusion` or eval.py (checkpoint_path=...)."""
import glob
from huggingface_hub import snapshot_download
folder = snapshot_download(repo_id, local_dir=local_dir)
return sorted(glob.glob(os.path.join(folder, "checkpoints", "checkpoint_*.safetensors")))[-1]
def missing_bones(skeleton, names, generate=False):
"""Bone indices for limb names (see LIMB_NAMES) and bone names (skeleton.bone_names) of a bone direction model.
The missing bones must not separate the remaining skeleton: every bone after a missing one in its kinematic chain
must be missing too. To generate missing bones, their mirrored bones (left <-> right) must be observed."""
assert skeleton.node_type == "bone", "missing limbs or bones are supported by the bone direction models; use missing_joints for the 3D joint models"
bones = []
for name in names:
if name in LIMB_NAMES:
bones += list(skeleton.limb_bones[name])
elif name in skeleton.bone_names:
bones.append(skeleton.bone_names.index(name))
else:
raise ValueError(f"Unknown limb or bone '{name}'. Limbs: {list(LIMB_NAMES)}; bones: {skeleton.bone_names}")
bones = sorted(set(bones))
children = {i: [j for j, b in enumerate(skeleton.bones) if b[0] == skeleton.bones[i][1]] for i in range(len(skeleton.bones))}
for b in bones:
stack, below = list(children[b]), []
while stack:
c = stack.pop(); below.append(c); stack += children[c]
if set(below) - set(bones):
raise ValueError(f"Missing bone '{skeleton.bone_names[b]}' would disconnect the bones after it in the chain; "
f"add {[skeleton.bone_names[c] for c in sorted(set(below) - set(bones))]} to the missing ones")
if generate:
mirror = {int(a): int(b) for a, b in skeleton.bone_symmetry} | {int(b): int(a) for a, b in skeleton.bone_symmetry}
for b in bones:
if b not in mirror or mirror[b] in bones:
raise ValueError(f"Bone '{skeleton.bone_names[b]}' can be generated only if its mirrored bone is observed")
return bones
def load_equifusion(checkpoint_path: str, dataset_name: str, missing_limbs=None, generate_missing_limbs: bool = False,
num_samples: int = 50):
"""Loads a released EquiFusion model (autoencoder + latent diffusion) for the skeleton of `dataset_name`.
missing_limbs: for the bone direction models, names of limbs (LIMB_NAMES) and/or bones (skeleton.bone_names) that are
not observed; they must not separate the remaining skeleton (see missing_bones). With generate_missing_limbs=True
their motion is generated (the latents of the mirrored limb are used as conditioning), otherwise the missing
bones are removed from the graph.
Returns (model, skeleton, config) to be used with `predict`.
"""
from .data import create_skeleton
from .eval_prepare_model import prepare_model
cfg = quick_cfg_for_inference_no_hydra(checkpoint_path, dataset_name, num_samples=num_samples)
if missing_limbs and all(l in LIMB_NAMES for l in missing_limbs):
cfg.update(if_eval_drop_limb=True, drop_limb_ids=[LIMB_NAMES[l] for l in missing_limbs],
if_eval_generate_dropped_limbs=generate_missing_limbs)
elif missing_limbs: # bone names (and limb names): explicit list of bones
cfg.update(if_eval_drop_limb=True, drop_limb_ids=[], if_eval_generate_dropped_limbs=generate_missing_limbs,
drop_bones=missing_bones(create_skeleton(**cfg), missing_limbs, generate_missing_limbs))
skeleton = create_skeleton(**cfg)
model, _, _ = prepare_model(cfg, skeleton, **{**cfg, "silent": True})
return model, skeleton, cfg
def _graph_adjacency(skeleton):
return (skeleton.bone_adj_matrix() if skeleton.node_type == "bone" else skeleton.adj_matrix).clone()
def _pad_nodes(x: torch.Tensor, num_nodes: int):
missing = num_nodes - x.shape[-2]
return torch.cat([x, x.new_zeros(*x.shape[:-2], missing, x.shape[-1])], dim=-2) if missing > 0 else x
def _pad_adj(adj: torch.Tensor, num_nodes: int):
out = adj.new_zeros(*adj.shape[:-2], num_nodes, num_nodes)
out[..., : adj.shape[-2], : adj.shape[-1]] = adj
return out
@torch.no_grad()
def predict(model, skeleton, config, obs, num_samples: int = 50, seed: int = None, missing_joints=None):
"""Predicts `num_samples` futures of 2 s from 0.5 s of observed motion.
obs: [T_obs, J, 3] or [B, T_obs, J, 3] joint positions in meters (including the root joint), at the frame rate of
the dataset given to `load_equifusion` (e.g. 30 frames for AMASS/Nymeria at 60 fps, 25 for H36M at 50 fps).
missing_joints: for the 3D joint models, names or indices of joints that are not observed (any joint except the
root); they are removed from the graph and returned as NaN.
Returns predictions [B, num_samples, T_pred, J-1, 3] in meters, relative to the root joint (the root is not
returned), in the joint order of the dataset; joints that are missing (not generated) are NaN.
"""
from .eval_prepare_model import get_prediction, process_evaluation_pair
device = next(model[0].parameters()).device
obs = torch.as_tensor(np.asarray(obs), dtype=torch.float32)
squeeze = obs.dim() == 3
obs = obs.unsqueeze(0) if squeeze else obs
if seed is not None:
from .utils.reproducibility import set_seed
set_seed(seed) # seeds and deterministic algorithms, as in the evaluation
# to model input space (bone directions or rescaled joints relative to the root)
x = skeleton.tranform_to_input_space(obs)
adj = _graph_adjacency(skeleton)
extra = {}
kept = None
if getattr(skeleton, "if_eval_drop_limb", False): # missing limbs (bone direction models)
x = skeleton.drop_limbs(x)
if skeleton.if_eval_generate_dropped_limbs:
extra["adj_orig"] = _pad_adj(adj, config["max_num_nodes"]).expand(x.shape[0], -1, -1).to(device)
adj = skeleton.adj_after_dropping_nodes(adj, skeleton.drop_joints_ids, keep_size=skeleton.if_eval_generate_dropped_limbs)
elif missing_joints: # missing joints (3D joint models)
assert skeleton.node_type == "joint", "arbitrary missing joints are supported by the 3D joint models"
name2node = {name: i for i, name in skeleton.node_dict.items()}
missing = sorted(name2node[j] if isinstance(j, str) else int(j) for j in missing_joints)
kept = [i for i in range(x.shape[-2]) if i not in missing]
x_kept = torch.zeros_like(x)
x_kept[..., : len(kept), :] = x[..., kept, :]
x = x_kept
adj = skeleton.adj_after_dropping_nodes(adj, missing)
x = _pad_nodes(x, config["max_num_nodes"])
adj = _pad_adj(adj, config["max_num_nodes"]).expand(x.shape[0], -1, -1)
# datasets not recorded at 60 fps are resampled to the 60-fps windows of the models
ctx = types.SimpleNamespace(skeleton=skeleton, if_eval_from60_at50fps=config.get("if_eval_from60_at50fps", False),
if_eval_from60_at30fps=False, if_eval_from60_at20fps=False)
if ctx.if_eval_from60_at50fps:
obs_len = x.shape[-3]
x = x[..., torch.from_numpy(np.floor(np.arange(0, obs_len, obs_len / 30))).long(), :, :]
ctx.mask_out = torch.from_numpy(np.ceil(np.arange(0, 120, 60 / 50))).long()
x, adj = x.to(device), adj.to(device)
kwargs = {k: v for k, v in config.items() if k not in ("num_samples", "pred_length", "adj", "extra")}
if getattr(skeleton, "if_eval_generate_dropped_limbs", False):
kwargs.update(sym_limb_idxs=skeleton.sym_limb_idxs, limb_idxs_dropped=skeleton.drop_joints_ids)
pred = get_prediction(x, model, num_samples=num_samples, pred_length=config["pred_length"], adj=adj, extra=extra, **kwargs)
# back to meters
if kept is not None:
pred_full = torch.full_like(pred, float("nan"))
pred_full[..., kept, :] = pred[..., : len(kept), :]
pred, x_full = pred_full, torch.zeros_like(x)
x_full[..., kept, :] = x[..., : len(kept), :]
x = x_full
T_out = len(ctx.mask_out) if ctx.if_eval_from60_at50fps else pred.shape[2]
target = torch.zeros(pred.shape[0], T_out, *pred.shape[3:], device=device) # no ground truth at inference
_, pred, _, _ = process_evaluation_pair(ctx, target=target, pred_dict={"pred": pred, "obs": x}, adj=adj, extra=extra)
if getattr(skeleton, "if_eval_drop_limb", False) and not skeleton.if_eval_generate_dropped_limbs:
# removed limbs: joints of the reduced skeleton -> original joint order, NaN for the missing joints
pred_full = torch.full((*pred.shape[:-2], skeleton.num_joints - 1, 3), float("nan"), device=pred.device)
for j_orig, j_new in skeleton.map_old2new_joint.items():
if j_orig > 0: # the root joint is not returned
pred_full[..., j_orig - 1, :] = pred[..., j_new - 1, :]
pred = pred_full
return pred[0] if squeeze else pred