use_liger=False
xla_on_gpu=False
# ---------------------------------------------------------------------------
# Core shape -- chosen for GPU efficiency, not just "a number that fits"
# ---------------------------------------------------------------------------
CONTEXT = 1024
max_gen_tokens = 1024
gen_headroom = 128
vocab_size = 8192+4096
# ── Engram (static n-gram memory) ──────────────────────────────────
use_engram              = True
engram_layer_ids        = (1, 8)         # two layers, one early, one mid
engram_max_ngram        = 3              # 2-gram + 3-gram tables
engram_vocab_size       = (4096, 4096)   # halved from default
engram_embed_per_ngram  = 128
engram_n_heads          = 4
engram_kernel_size      = 4
engram_seed             = 0
engram_compressed_vocab = 0              # 0 → set from tokenizer lookup
bias = False
d_rope = None
top_k = 64
moe_top_k = 2
use_triton_fusions = False


gradient_checkpointing = False
num_kv_heads = 4
pattern = "dense"
sliding_window_size = 512

tensor_parallel_size = 1

# ─────────────────────────────────────────────────────────────────────
# Curriculum learning (token-level mixing across sources)
# ─────────────────────────────────────────────────────────────────────
# Master switch. False = old behavior (uniform random over the whole
# flat token array). True = per-source anchor schedule below.
CURRICULUM_ENABLED = False

# Number of training steps the anchor schedule spans. Set equal to
# MAX_STEPS in train.py so the ramp completes at run end.
CURRICULUM_TOTAL_STEPS = 20000

# Sources, in curriculum order (easiest first). Each entry:
#   (name, [glob_patterns], w@0, w@T/3, w@2T/3, w@T)
# Between anchors, weights interpolate linearly. Columns should sum
# to ~1.0 (the code renormalizes if not).
# Files matching none of the globs are EXCLUDED from training.
CURRICULUM_SOURCES = [
    ("tinystories", ["tinystories*.txt", "tiny_*.txt"],
     0.70, 0.10, 0.03, 0.02),
    ("simplewiki",  ["simplewiki*.txt",  "wiki_*.txt"],
     0.10, 0.70, 0.10, 0.03),
    ("cosmopedia",  ["cosmopedia*.txt",  "cosmo_*.txt"],
     0.10, 0.10, 0.70, 0.10),
    ("finephrase",  ["finephrase*.txt",  "fine_*.txt"],
     0.10, 0.10, 0.17, 0.85),
]

# Hard floor: no source ever drops below this fraction at any step.
# 0.0 disables. 0.02 = 2% minimum, prevents catastrophic forgetting.
CURRICULUM_FLOOR = 0.02

mod_capacity = 0.25
use_mtp    = False
mtp_depth  = 1      
mtp_lambda = 0.1 
use_nvfp4 = False
# config.py
use_fp8 = False
fp8_recipe = "rowwise_with_gw_hp"        # default
fp8_bf16_tail_frac = 0.15                # default
use_int8 = False
int8_group_size = 128
mod_budget_coeff = 0.05
mod_gate_entropy_coeff = 0.0
mod_hard_eval = False
use_flexattention = True
numberoflayers = 16     
numberofheads = 12
D_MODEL = 384        
ALPHA_COMPILE_MODE="reduce-overhead"  
num_kv_heads = 6 
sliding_window_size = 512  
use_sdpa=False
optimizer_8bit = True
use_wsd = True
wsd_decay_steps = 3500 
ffn_mult = 8 // 3           
use_moe = False            
n_experts = 8
n_shared = 1
# T4 path

PRECISION = 'bf16'         
GLOBAL_DTYPE = PRECISION
USE_ASYNC_LAYER_PREFETCH = True
intra_op_threads = 2
num_workers = 2
GPU_SIDE_BATCHING = True
DISABLE_MMAP = False

batch_size = 320
weight_decay = 0.01        
use_static_lr = False
static_lr = 1e-3        
learning_rate = 1e-3


attn_gate_window = 16
max_steps = 19000
data_parallel_only = True
use_fsdp = False
dropout = 0.0                   
use_muon = True
DATA_PATH = './training_data'
data_path = DATA_PATH
TOKENIZER_MODEL_PATH = './tokenizer'
use_streaming_batcher = False