| { |
| "model_type": "loomformer", |
| "architectures": [ |
| "LoomFormerForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "loomformer.LoomFormerConfig", |
| "AutoModelForCausalLM": "loomformer.LoomFormerForCausalLM" |
| }, |
| "vocab_size": 32768, |
| "hidden_size": 576, |
| "num_hidden_layers": 20, |
| "num_attention_heads": 24, |
| "num_key_value_heads": 6, |
| "head_dim": 24, |
| "intermediate_size": 3456, |
| "max_position_embeddings": 4096, |
| "tie_word_embeddings": true, |
| "rope_theta": 10000.0, |
| "rope_scaling": { |
| "type": "linear", |
| "factor": 4.0 |
| }, |
| "torch_dtype": "bf16", |
| "ffn_type": "paraplex", |
| "activation": "pvpowlu", |
| "model_kind": "loomformer", |
| "ablation": false, |
| "loomformer_cfg": { |
| "vocab": 32768, |
| "tokenizer": "./tokenizers/tokenizer32768.json", |
| "tied_embeddings": true, |
| "doc_reset_attn": true, |
| "device": "cuda:0, cuda:1", |
| "train_dataset": "./datasets/sft/FSSFT1", |
| "val_dataset": "./datasets/sft/FSSFT1/val/val_split.parquet", |
| "auto_val_split_pct": 0.5, |
| "dataset_format": "sft", |
| "text_field": "text", |
| "seq_len": 4096, |
| "batch_size": 8, |
| "model_dim": 576, |
| "n_q_heads": 24, |
| "head_dim": 24, |
| "n_kv_heads": 6, |
| "gqa_group_size": 4, |
| "hidden": 3456, |
| "hidden_mult": 6, |
| "layers": 20, |
| "phase_sectors": "open", |
| "residual_init": "beta", |
| "depth_attn_readout": "shared", |
| "depth_attn_qkv_rms": true, |
| "residual_branch_rms_cap": 1.0, |
| "activation": "pvpowlu", |
| "powlu_m": 3.0, |
| "phase_grad_floor": 0.05, |
| "phase_grad_mode": "secant", |
| "use_cuda_phase_sin": true, |
| "use_cuda_beta_space": true, |
| "use_cuda_pvpowlu": true, |
| "use_cuda_depth_attn": true, |
| "amp_dtype": "bf16", |
| "dataset_cache": "otf", |
| "attn_impl": "auto", |
| "attn_sdpa_compute_dtype": "auto", |
| "attn_sdpa_value_fusion": true, |
| "attn_sdpa_recompute_backward": false, |
| "rope_theta": 10000.0, |
| "rope_factor": 4.0, |
| "rope_original_seq_len": 2048, |
| "rope_beta_fast": 32.0, |
| "rope_beta_slow": 1.0, |
| "rope_attention_factor": null, |
| "steps": 10000, |
| "lr": 5e-05, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "grad_accum_steps": 1, |
| "prefetch_batches": 16, |
| "gpu_prefetch_batches": 4, |
| "grad_checkpointing": false, |
| "fsdp_full_shard": false, |
| "optimizer_zero_shard": false, |
| "save_every": 1000, |
| "runpoints_path": "./runpoints", |
| "save_initial_checkpoint": false, |
| "save_final_checkpoint": true, |
| "init_checkpoint": "./alt6/alt6.pt", |
| "checkpoint": "./alt6_sft.pt", |
| "tria_carry_enabled": true, |
| "cloned": false, |
| "cloned_from": null, |
| "cloned_mapping": null, |
| "train_lr": null, |
| "resume": null, |
| "resume_data_stream": "auto", |
| "paraplex_gate_proj": true, |
| "final_norm": true, |
| "fused_linear_ce": true, |
| "fused_linear_ce_chunk_size": 0, |
| "use_cuda_tria": true, |
| "compile": true, |
| "graph": true, |
| "save_graph": false, |
| "tria_temporal_enabled": true, |
| "tria_temporal_window": 256, |
| "tria_temporal_window_min": 32, |
| "tria_temporal_window_max": null, |
| "tria_temporal_calibration": null, |
| "tria_temporal_auto": false, |
| "tria_carrier_alpha": 0.1, |
| "tria_carrier_alpha_candidates": null, |
| "tria_polarm_beta": 0.5, |
| "tria_min_refeeds_per_sequence": 1, |
| "tria_temporal_max_condition": 3.0, |
| "tria_temporal_min_effective_rank": 2.7, |
| "tria_temporal_population_pass_fraction": 0.9, |
| "tria_temporal_calib_seeds": 3, |
| "tria_temporal_calib_batch": 4, |
| "tria_temporal_calib_tokens": null, |
| "tria_temporal_calib_device": "auto", |
| "tria_temporal_calib_parallel_sweep": 1, |
| "tria_target_refeeds_per_sequence": 15, |
| "tria_gamma_max": 0.8, |
| "tria_raw_gamma_init": 0.01, |
| "warmup_steps": 150, |
| "min_lr_frac": 0.1, |
| "seed": 1, |
| "log_every": 10, |
| "eval_every": 200, |
| "eval_batches": 8, |
| "optimizer": "atom" |
| }, |
| "step": 10000, |
| "tokens_seen": 327680000 |
| } |
