| """MUGEN configuration. |
| |
| MUGEN couples an adaptive-length motion autoencoder (ALAE) with a GPT-2 |
| language model. The autoencoder compresses a motion clip of any length into K |
| continuous latent slots, and those slots are the only motion representation in |
| the system: the language model generates them for text-to-motion and reads |
| them back for motion understanding (captioning). |
| """ |
|
|
| from transformers import PretrainedConfig |
|
|
|
|
| class MugenConfig(PretrainedConfig): |
| """Configuration for :class:`MugenForConditionalGeneration`. |
| |
| Args: |
| motion_input_dim (`int`, *optional*, defaults to 263): |
| Dimensionality of one motion frame. 263 is the HumanML3D feature |
| layout (root velocities, RIC joint positions, 6D rotations, foot |
| contacts). |
| k_latent_slots (`int`, *optional*, defaults to 2): |
| Number of continuous latent slots K. The whole clip, whatever its |
| length, is represented by exactly K vectors. |
| latent_dim (`int`, *optional*, defaults to 512): |
| Width of one latent slot. |
| alae_hidden_dim (`int`, *optional*, defaults to 512): |
| Width of the autoencoder's convolutional trunk. |
| alae_depth (`int`, *optional*, defaults to 3): |
| Number of dilated convolutions inside one residual block. |
| alae_dilation_growth_rate (`int`, *optional*, defaults to 3): |
| Dilation growth factor across the residual stack. |
| alae_activation (`str`, *optional*, defaults to `"gelu"`): |
| Activation used throughout the autoencoder. |
| alae_norm (`str`, *optional*, defaults to `None`): |
| Normalisation inside the residual blocks (`None`, `"LN"`, `"GN"`, |
| `"BN"`). |
| alae_num_res_blocks (`int`, *optional*, defaults to 2): |
| Residual blocks in the encoder trunk and the decoder refiner. |
| alae_num_encoder_layers (`int`, *optional*, defaults to 4): |
| Cross-attention blocks that compress frames into the K slots. |
| alae_num_decoder_layers (`int`, *optional*, defaults to 4): |
| Cross-attention blocks that expand the K slots back to frames. |
| alae_nhead (`int`, *optional*, defaults to 8): |
| Attention heads in the autoencoder's cross-attention blocks. |
| alae_dim_feedforward (`int`, *optional*, defaults to 2048): |
| Feed-forward width in those blocks. |
| alae_dropout (`float`, *optional*, defaults to 0.05): |
| Dropout in those blocks (inactive at inference). |
| alae_max_decode_len (`int`, *optional*, defaults to 256): |
| Size of the decoder's positional table. Query positions are spread |
| across the whole table so a position encodes the frame's relative |
| phase in the clip rather than an absolute frame index, which is |
| what lets one decoder serve every clip length. |
| vocab_size (`int`, *optional*, defaults to 50261): |
| Language-model vocabulary: GPT-2's 50257 plus three reserved |
| motion-id slots and the `<MOT>` seed token. |
| n_positions, n_embd, n_layer, n_head, n_inner, activation_function, |
| resid_pdrop, embd_pdrop, attn_pdrop, layer_norm_epsilon, |
| initializer_range: |
| Standard GPT-2 hyperparameters. |
| num_cross_attn_layers (`int`, *optional*, defaults to 2): |
| Cross-attention blocks in the layer router's text encoder. |
| router_hidden (`int`, *optional*, defaults to 512): |
| Hidden width of the router MLP. |
| router_static_scale (`float`, *optional*, defaults to 4.0): |
| Bound on the router's text-independent logit component. |
| router_delta_scale (`float`, *optional*, defaults to 4.0): |
| Bound on the router's text-conditional logit component. Both parts |
| are tanh-capped, so no logit margin can saturate the softmax. |
| router_eval_tau (`float`, *optional*, defaults to 1.5): |
| Inference temperature of the routing softmax. This is the converged |
| end of the training anneal; changing it changes which transformer |
| depth each slot reads from. |
| latent_low_rank (`int`, *optional*, defaults to 64): |
| Rank r of the calibrated head's covariance. The head predicts |
| `N(mu, U diag(a)^2 U^T + diag(sigma^2))` over the flattened K*D |
| latent, so a single draw carries variance correlated across slots. |
| eval_sample_temperature (`float`, *optional*, defaults to 1.0): |
| Default multiplier on the sampled perturbation around `mu`. 1.0 is |
| the calibrated conditional distribution and the setting every |
| reported number uses; 0.0 decodes the mean. |
| m2t_max_new_tokens (`int`, *optional*, defaults to 64): |
| Caption length cap for motion-to-text generation. |
| m2t_num_beams (`int`, *optional*, defaults to 1): |
| Beam count for captioning. |
| mot_token (`str`, *optional*, defaults to `"<MOT>"`): |
| Seed token that ends the text-to-motion prompt. |
| mot_token_id (`int`, *optional*, defaults to 50260): |
| Token id of `mot_token` in the shipped tokenizer. |
| t2m_prompt_template (`str`, *optional*): |
| Prompt used for text-to-motion. Must end in `mot_token`. |
| m2t_prompt_prefix (`str`, *optional*): |
| Instruction placed before the motion slots for captioning. |
| fps (`int`, *optional*, defaults to 20): |
| Frame rate of the HumanML3D features this model was trained on. |
| num_joints (`int`, *optional*, defaults to 22): |
| Joint count recovered by `features_to_joints`. |
| length_multiple (`int`, *optional*, defaults to 4): |
| HumanML3D crops clips to a multiple of 4 frames; requesting a |
| length off this grid moves you away from the training distribution. |
| |
| Example: |
| ```python |
| from transformers import AutoConfig |
| |
| config = AutoConfig.from_pretrained("zy22b/MUGEN", trust_remote_code=True) |
| print(config.k_latent_slots) # 2 |
| ``` |
| """ |
|
|
| model_type = "mugen" |
|
|
| def __init__( |
| self, |
| |
| motion_input_dim: int = 263, |
| k_latent_slots: int = 2, |
| latent_dim: int = 512, |
| alae_hidden_dim: int = 512, |
| alae_depth: int = 3, |
| alae_dilation_growth_rate: int = 3, |
| alae_activation: str = "gelu", |
| alae_norm=None, |
| alae_num_res_blocks: int = 2, |
| alae_num_encoder_layers: int = 4, |
| alae_num_decoder_layers: int = 4, |
| alae_nhead: int = 8, |
| alae_dim_feedforward: int = 2048, |
| alae_dropout: float = 0.05, |
| alae_max_decode_len: int = 256, |
| |
| vocab_size: int = 50261, |
| n_positions: int = 1024, |
| n_embd: int = 768, |
| n_layer: int = 12, |
| n_head: int = 12, |
| n_inner=None, |
| activation_function: str = "gelu_new", |
| resid_pdrop: float = 0.1, |
| embd_pdrop: float = 0.1, |
| attn_pdrop: float = 0.1, |
| layer_norm_epsilon: float = 1e-5, |
| initializer_range: float = 0.02, |
| |
| num_cross_attn_layers: int = 2, |
| router_hidden: int = 512, |
| router_static_scale: float = 4.0, |
| router_delta_scale: float = 4.0, |
| router_eval_tau: float = 1.5, |
| |
| latent_low_rank: int = 64, |
| eval_sample_temperature: float = 1.0, |
| |
| m2t_max_new_tokens: int = 64, |
| m2t_num_beams: int = 1, |
| |
| mot_token: str = "<MOT>", |
| mot_token_id: int = 50260, |
| t2m_prompt_template: str = ( |
| "Please generate human motion based on the following textual " |
| "description: {text} {mot}" |
| ), |
| m2t_prompt_prefix: str = ( |
| "Please describe the following human motion using plain text:" |
| ), |
| |
| fps: int = 20, |
| num_joints: int = 22, |
| length_multiple: int = 4, |
| |
| bos_token_id: int = 50256, |
| eos_token_id: int = 50256, |
| pad_token_id: int = 50256, |
| **kwargs, |
| ): |
| self.motion_input_dim = motion_input_dim |
| self.k_latent_slots = k_latent_slots |
| self.latent_dim = latent_dim |
| self.alae_hidden_dim = alae_hidden_dim |
| self.alae_depth = alae_depth |
| self.alae_dilation_growth_rate = alae_dilation_growth_rate |
| self.alae_activation = alae_activation |
| self.alae_norm = alae_norm |
| self.alae_num_res_blocks = alae_num_res_blocks |
| self.alae_num_encoder_layers = alae_num_encoder_layers |
| self.alae_num_decoder_layers = alae_num_decoder_layers |
| self.alae_nhead = alae_nhead |
| self.alae_dim_feedforward = alae_dim_feedforward |
| self.alae_dropout = alae_dropout |
| self.alae_max_decode_len = alae_max_decode_len |
|
|
| self.vocab_size = vocab_size |
| self.n_positions = n_positions |
| self.n_embd = n_embd |
| self.n_layer = n_layer |
| self.n_head = n_head |
| self.n_inner = n_inner |
| self.activation_function = activation_function |
| self.resid_pdrop = resid_pdrop |
| self.embd_pdrop = embd_pdrop |
| self.attn_pdrop = attn_pdrop |
| self.layer_norm_epsilon = layer_norm_epsilon |
| self.initializer_range = initializer_range |
|
|
| self.num_cross_attn_layers = num_cross_attn_layers |
| self.router_hidden = router_hidden |
| self.router_static_scale = router_static_scale |
| self.router_delta_scale = router_delta_scale |
| self.router_eval_tau = router_eval_tau |
|
|
| self.latent_low_rank = latent_low_rank |
| self.eval_sample_temperature = eval_sample_temperature |
|
|
| self.m2t_max_new_tokens = m2t_max_new_tokens |
| self.m2t_num_beams = m2t_num_beams |
|
|
| self.mot_token = mot_token |
| self.mot_token_id = mot_token_id |
| self.t2m_prompt_template = t2m_prompt_template |
| self.m2t_prompt_prefix = m2t_prompt_prefix |
|
|
| self.fps = fps |
| self.num_joints = num_joints |
| self.length_multiple = length_multiple |
|
|
| super().__init__( |
| bos_token_id=bos_token_id, |
| eos_token_id=eos_token_id, |
| pad_token_id=pad_token_id, |
| **kwargs, |
| ) |
|
|
