-
Delete run_20260130_driftway_stinter
-
Upload folder using huggingface_hub
-
Upload 12 files
Model:
Tiny_MoE_Config(
name='Tiny_MoE',
dtype=jax.numpy.bfloat16,
param_dtype=jax.numpy.float32,
block_size=2048,
vocab_size=49152,
n_layer=30,
n_head=9,
n_kv_head=3,
n_embed=576,
n_experts=8,
mesh=None,
top_k=2,
load_factor=10.0,
expert_weight_priority=False,
load_balance_loss_coeff=0.01,
z_loss_coeff=0.0001,
n_mlp_hidden=1536,
mlp_bias=False,
attention_bias=False,
moe_bias=False,
ln_epsilon=1e-05,
glu_activation='silu',
sdpa_implementation='slow',
rope_theta=0.0001,
init_stddev=0.02,
use_cache=False,
glu_fc_kernel_sharding=(None,),
glu_fc_bias_sharding=(None,),
glu_gate_kernel_sharding=(None,),
glu_gate_bias_sharding=(None,),
glu_proj_kernel_sharding=(None,),
glu_proj_bias_sharding=(None,),
attn_wq_kernel_sharding=(None,),
attn_wq_bias_sharding=(None,),
attn_wkv_kernel_sharding=(None,),
attn_wkv_bias_sharding=(None,),
attn_wproj_kernel_sharding=(None,),
attn_wproj_bias_sharding=(None,),
embed_partition_spec=(None,),
rmsnorm_partition_spec=(None,),
)
Parameter Count: 413,275,968
MO… -
initial commit
