{
  "model_type": "loomformer",
  "architectures": [
    "LoomFormerForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "loomformer.LoomFormerConfig",
    "AutoModelForCausalLM": "loomformer.LoomFormerForCausalLM"
  },
  "vocab_size": 32768,
  "hidden_size": 576,
  "num_hidden_layers": 20,
  "num_attention_heads": 24,
  "num_key_value_heads": 6,
  "head_dim": 24,
  "intermediate_size": 3456,
  "max_position_embeddings": 4096,
  "tie_word_embeddings": true,
  "rope_theta": 10000.0,
  "rope_scaling": {
    "type": "linear",
    "factor": 4.0
  },
  "torch_dtype": "bf16",
  "ffn_type": "paraplex",
  "activation": "pvpowlu",
  "model_kind": "loomformer",
  "ablation": false,
  "loomformer_cfg": {
    "vocab": 32768,
    "tokenizer": "./tokenizers/tokenizer32768.json",
    "tied_embeddings": true,
    "doc_reset_attn": true,
    "device": "cuda:0, cuda:1",
    "train_dataset": "./datasets/sft/FSSFT1",
    "val_dataset": "./datasets/sft/FSSFT1/val/val_split.parquet",
    "auto_val_split_pct": 0.5,
    "dataset_format": "sft",
    "text_field": "text",
    "seq_len": 4096,
    "batch_size": 8,
    "model_dim": 576,
    "n_q_heads": 24,
    "head_dim": 24,
    "n_kv_heads": 6,
    "gqa_group_size": 4,
    "hidden": 3456,
    "hidden_mult": 6,
    "layers": 20,
    "phase_sectors": "open",
    "residual_init": "beta",
    "depth_attn_readout": "shared",
    "depth_attn_qkv_rms": true,
    "residual_branch_rms_cap": 1.0,
    "activation": "pvpowlu",
    "powlu_m": 3.0,
    "phase_grad_floor": 0.05,
    "phase_grad_mode": "secant",
    "use_cuda_phase_sin": true,
    "use_cuda_beta_space": true,
    "use_cuda_pvpowlu": true,
    "use_cuda_depth_attn": true,
    "amp_dtype": "bf16",
    "dataset_cache": "otf",
    "attn_impl": "auto",
    "attn_sdpa_compute_dtype": "auto",
    "attn_sdpa_value_fusion": true,
    "attn_sdpa_recompute_backward": false,
    "rope_theta": 10000.0,
    "rope_factor": 4.0,
    "rope_original_seq_len": 2048,
    "rope_beta_fast": 32.0,
    "rope_beta_slow": 1.0,
    "rope_attention_factor": null,
    "steps": 10000,
    "lr": 5e-05,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "grad_accum_steps": 1,
    "prefetch_batches": 16,
    "gpu_prefetch_batches": 4,
    "grad_checkpointing": false,
    "fsdp_full_shard": false,
    "optimizer_zero_shard": false,
    "save_every": 1000,
    "runpoints_path": "./runpoints",
    "save_initial_checkpoint": false,
    "save_final_checkpoint": true,
    "init_checkpoint": "./alt6/alt6.pt",
    "checkpoint": "./alt6_sft.pt",
    "tria_carry_enabled": true,
    "cloned": false,
    "cloned_from": null,
    "cloned_mapping": null,
    "train_lr": null,
    "resume": null,
    "resume_data_stream": "auto",
    "paraplex_gate_proj": true,
    "final_norm": true,
    "fused_linear_ce": true,
    "fused_linear_ce_chunk_size": 0,
    "use_cuda_tria": true,
    "compile": true,
    "graph": true,
    "save_graph": false,
    "tria_temporal_enabled": true,
    "tria_temporal_window": 256,
    "tria_temporal_window_min": 32,
    "tria_temporal_window_max": null,
    "tria_temporal_calibration": null,
    "tria_temporal_auto": false,
    "tria_carrier_alpha": 0.1,
    "tria_carrier_alpha_candidates": null,
    "tria_polarm_beta": 0.5,
    "tria_min_refeeds_per_sequence": 1,
    "tria_temporal_max_condition": 3.0,
    "tria_temporal_min_effective_rank": 2.7,
    "tria_temporal_population_pass_fraction": 0.9,
    "tria_temporal_calib_seeds": 3,
    "tria_temporal_calib_batch": 4,
    "tria_temporal_calib_tokens": null,
    "tria_temporal_calib_device": "auto",
    "tria_temporal_calib_parallel_sweep": 1,
    "tria_target_refeeds_per_sequence": 15,
    "tria_gamma_max": 0.8,
    "tria_raw_gamma_init": 0.01,
    "warmup_steps": 150,
    "min_lr_frac": 0.1,
    "seed": 1,
    "log_every": 10,
    "eval_every": 200,
    "eval_batches": 8,
    "optimizer": "atom"
  },
  "step": 10000,
  "tokens_seen": 327680000
}