config.json

{
  "architectures": ["MDLMBPEV3"],
  "model_type": "mdlm",
  "auto_map": {
    "AutoConfig": "modeling_mdlm.MDLMConfig",
    "AutoModel": "modeling_mdlm.MDLMBPEV3"
  },
  "d_model": 1024,
  "n_heads": 16,
  "n_layers": 10,
  "vocab_size": 16000,
  "max_seq_len": 256,
  "d_ff": 4096,
  "dropout": 0.1,
  "pad_token_id": 0,
  "mask_token_id": 1,
  "bos_token_id": 2,
  "eos_token_id": 3,
  "torch_dtype": "float32",
  "transformers_version": "4.57.0",

  "training": {
    "params": 201258768,
    "params_human": "201.3M",
    "final_loss": 4.63,
    "final_ppl": 102.6,
    "best_step": 44000,
    "total_steps": 49779,
    "optimizer": "AdamW",
    "learning_rate_peak": 3e-4,
    "lr_schedule": "cosine with warmup",
    "weight_decay": 0.01,
    "warmup_steps": 1000,
    "epochs": 3,
    "batch_size": 32,
    "gradient_accumulation": 4,
    "effective_batch_size": 128,
    "seq_len": 128,
    "precision": "bf16",
    "gradient_clipping": 1.0
  },

  "training_data": {
    "dataset": "openbmb/Ultra-FineWeb",
    "dataset_url": "https://huggingface.co/datasets/openbmb/Ultra-FineWeb",
    "documents": 1000000,
    "documents_human": "1M",
    "total_tokens": 272000000,
    "tokens_human": "272M",
    "sequences": 2120000,
    "language": "English",
    "filtering": "Length >50 chars, deduplicated",
    "rejected_docs": 470122,
    "tokenization": "BPE 16K vocab, packed to 128 tokens per sequence"
  },

  "hardware": {
    "gpu": "NVIDIA GeForce RTX 3090",
    "vram_gb": 24,
    "system_ram_gb": 8,
    "training_time_hours": 6.75,
    "training_time_human": "6h 45min",
    "throughput_tokens_per_second": 33500,
    "optimizer_steps_per_second": 2.0,
    "location": "Single workstation, no distributed training"
  },

  "benchmark": {
    "forward_throughput_batch1_tps": 8443,
    "forward_throughput_batch32_tps": 57759,
    "generation_full_parallel_tps": 31.2,
    "generation_hybrid_validated_tps": 15.6,
    "repetition_score": 0.99,
    "oracle_log_prob": -3.55,
    "vs_qwen3_0.6b_speedup_forward": "1.8-3.9x",
    "vs_qwen3_0.6b_speedup_generation": "2.4x",
    "qwen3_0.6b_reference_lp": -1.18,
    "qwen3_0.6b_reference_ppl": "15-20"
  },

  "generation_config": {
    "mode": "full_parallel",
    "diffusion_steps": 32,
    "temperature": 0.7,
    "top_p": 0.95,
    "frequency_penalty": 0.4,
    "repetition_penalty": 1.3,
    "no_repeat_ngram": 2,
    "guidance": "adaptive (cooling temp + escalating penalties across blocks)"
  },

  "license": "MIT",
  "author": "Brian Tomas Schwabauer",
  "citation_bibtex_key": "schwabauer2026latentspace"
}