{ "transformer_lm": { "dim": 1536, "num_heads": 24, "num_layers": 48, "hidden_scale": 4, "n_q": 4, "card": 2048, "dropout": 0.0, "emb_lr": null, "activation": "gelu", "norm_first": true, "bias_ff": false, "bias_attn": false, "bias_proj": false, "past_context": null, "causal": true, "custom": false, "memory_efficient": true, "attention_as_float32": false, "layer_scale": null, "positional_embedding": "sin", "xpos": false, "checkpointing": "torch", "weight_init": "gaussian", "depthwise_init": "current", "zero_bias_init": true, "norm": "layer_norm", "cross_attention": false, "qk_layer_norm": false, "qk_layer_norm_cross": false, "attention_dropout": null, "kv_repeat": 1, "two_step_cfg": false }, "conditioners": { "description": { "model": "t5", "t5": { "name": "t5-base", "finetune": false, "word_dropout": 0.3, "normalize_text": false } } }, "fuser": { "cross_attention_pos_emb": false, "cross_attention_pos_emb_scale": 1, "sum": [], "prepend": [], "cross": [ "description" ], "input_interpolate": [] }, "codebooks_pattern": { "delay": { "delays": [ 0, 1, 2, 3 ], "flatten_first": 0, "empty_initial": 0 } }, "classifier_free_guidance": { "training_dropout": 0.3, "inference_coef": 3.0 } }