🔥 Mixtral-8x7B (47B MoE) + LoRA + 4-bit

app.py

@@ -81,8 +81,8 @@ def get_training_log():
 with gr.Blocks(title="D1337 CIPHER Training") as demo:
     gr.Markdown("# 🔥 D1337 CIPHER C2 V.1 - TRAINING")
     gr.Markdown("**Hardware**: L40S x4 (192GB VRAM)")
-    gr.Markdown("**Base**: GLM-4.7-Flash-abliterated (31B)")
-    gr.Markdown("**Dataset**: 92 samples | **Epochs**: 3")
+    gr.Markdown("**Base**: Mixtral-8x7B-Instruct (47B MoE) + LoRA")
+    gr.Markdown("**Dataset**: 92 samples | **Epochs**: 3 | **4-bit + LoRA**")
     
     with gr.Row():
         train_btn = gr.Button("🚀 START TRAINING", variant="primary")

requirements.txt

@@ -1,6 +1,8 @@
 torch>=2.0.0
-transformers @ git+https://github.com/huggingface/transformers.git
+transformers>=4.40.0
 datasets>=2.15.0
 accelerate>=0.25.0
 huggingface-hub>=0.20.0
-gradio>=5.0.0
+gradio>=5.0.0
+peft>=0.10.0
+bitsandbytes>=0.43.0

train.py

@@ -1,7 +1,8 @@
 
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, BitsAndBytesConfig
 from datasets import load_dataset
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
 import os
 
 # L40S x4 Multi-GPU setup
@@ -12,28 +13,53 @@ if torch.cuda.is_available():
     for i in range(torch.cuda.device_count()):
         print(f"  GPU {i}: {torch.cuda.get_device_name(i)}")
 
-# Load model
-model_name = "huihui-ai/Huihui-GLM-4.7-Flash-abliterated"
-print(f"\nLoading tokenizer: {model_name}")
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+# LARGE MODEL - Mixtral 8x7B (47B effective params, MoE)
+model_name = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+print(f"\n🔥 Loading BIG MODEL: {model_name}")
+
+# Tokenizer
+print("Loading tokenizer...")
+tokenizer = AutoTokenizer.from_pretrained(model_name)
 if tokenizer.pad_token is None:
     tokenizer.pad_token = tokenizer.eos_token
 
-print("Loading model (31B parameters)...")
+# Load model in 4-bit for memory efficiency on 192GB VRAM
+print("Loading model (47B MoE - this takes a few minutes)...")
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True
+)
+
 model = AutoModelForCausalLM.from_pretrained(
-    model_name, 
-    torch_dtype=torch.bfloat16,
-    trust_remote_code=True,
-    device_map="auto"
+    model_name,
+    quantization_config=bnb_config,
+    device_map="auto",
+    trust_remote_code=True
+)
+print("✅ Model loaded!")
+
+# LoRA config for efficient fine-tuning
+print("\nSetting up LoRA...")
+model = prepare_model_for_kbit_training(model)
+lora_config = LoraConfig(
+    r=64,
+    lora_alpha=128,
+    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+    lora_dropout=0.05,
+    bias="none",
+    task_type="CAUSAL_LM"
 )
-print("Model loaded!")
+model = get_peft_model(model, lora_config)
+model.print_trainable_parameters()
 
 # Load dataset
 print("\nLoading dataset...")
 dataset = load_dataset("Desorden1337/d1337-cipher-dataset", split="train")
 print(f"Dataset size: {len(dataset)} samples")
 
-# Tokenize with labels for causal LM
+# Tokenize
 def tokenize(examples):
     tokens = tokenizer(examples["text"], truncation=True, padding="max_length", max_length=2048)
     tokens["labels"] = tokens["input_ids"].copy()
@@ -41,13 +67,13 @@ def tokenize(examples):
 
 dataset = dataset.map(tokenize, batched=True, remove_columns=dataset.column_names)
 
-# Training args - L40S x4 OPTIMIZED
+# Training args
 training_args = TrainingArguments(
     output_dir="./d1337-cipher",
     num_train_epochs=3,
-    per_device_train_batch_size=4,
-    gradient_accumulation_steps=4,
-    learning_rate=2e-5,
+    per_device_train_batch_size=2,
+    gradient_accumulation_steps=8,
+    learning_rate=2e-4,
     lr_scheduler_type="cosine",
     warmup_ratio=0.1,
     weight_decay=0.01,
@@ -56,7 +82,7 @@ training_args = TrainingArguments(
     save_total_limit=2,
     bf16=True,
     gradient_checkpointing=True,
-    optim="adamw_torch",
+    optim="paged_adamw_8bit",
     push_to_hub=True,
     hub_model_id="Desorden1337/d1337-cipher-v1",
     hub_private_repo=True,
@@ -64,11 +90,11 @@ training_args = TrainingArguments(
 )
 
 # Train
-print("\n🚀 STARTING TRAINING...")
+print("\n🚀 STARTING TRAINING ON MIXTRAL 8x7B...")
 trainer = Trainer(
-    model=model, 
-    args=training_args, 
-    train_dataset=dataset, 
+    model=model,
+    args=training_args,
+    train_dataset=dataset,
     tokenizer=tokenizer
 )
 trainer.train()