Upload train_diffreaper_5l.py with huggingface_hub

train_diffreaper_5l.py

@@ -16,14 +16,14 @@ HF_TOKEN = os.getenv("HF_TOKEN")
 OUTPUT_DIR = "./training_output"
 LOG_FILE = "training.log"
 CHECKPOINT_LOG = "checkpoint_log.txt"
-BATCH_SIZE = 4 # Drastic reduction for 24-layer 2048-dim on 24GB VRAM
+BATCH_SIZE = 16 # Back to higher batch for 5090 (32GB VRAM)
 LEARNING_RATE = 1e-4
 SAVE_EVERY = 2500
 TEST_EVERY = 500
 
-N_EMBD = 1536 # Slightly downscale from 2048 to fit reliably
-N_HEAD = 24
-N_LAYER = 18 # Downscaled from 24 to save memory
+N_EMBD = 2048 # Back to full Large specs
+N_HEAD = 32
+N_LAYER = 24 # Full depth
 MAX_PROMPT_LEN = 32
 MAX_RESP_LEN = 128
 TOTAL_LEN = MAX_PROMPT_LEN + MAX_RESP_LEN