Upload folder using huggingface_hub

README.md

@@ -14,6 +14,24 @@ license: mit
 
 This space is dedicated to training Microsoft's Phi-4 model using Unsloth optimizations for enhanced performance and efficiency. The training process utilizes 4-bit quantization and advanced memory optimizations.
 
+## Installation
+
+Before running the training, ensure you have all required dependencies installed:
+
+```bash
+pip install -r requirements.txt
+```
+
+### Critical Dependencies
+
+- **unsloth** (>=2024.3): Required for optimized 4-bit training
+- **peft** (>=0.9.0): Required for parameter-efficient fine-tuning
+- **transformers** (>=4.36.0): Required for model architecture and tokenization
+
+### Optional but Recommended
+
+- **flash-attn** (>=2.5.0): Significantly speeds up attention computations
+
 ## Features
 
 - 4-bit quantization using Unsloth

app.py

@@ -196,6 +196,19 @@ with gr.Blocks(title="Phi-4 Unsloth Training", theme=gr.themes.Soft(primary_hue=
             
             This interface allows you to manage training of the Phi-4 model with Unsloth 4-bit optimizations.
             
+            ### Installation
+            
+            Before starting training, ensure all dependencies are installed:
+            
+            ```bash
+            pip install -r requirements.txt
+            ```
+            
+            Critical packages:
+            - unsloth (>=2024.3)
+            - peft (>=0.9.0)
+            - transformers (>=4.36.0)
+            
             ### Quick Start
             
             1. Review the configuration in the Configuration tab
@@ -214,6 +227,7 @@ with gr.Blocks(title="Phi-4 Unsloth Training", theme=gr.themes.Soft(primary_hue=
             - Check the logs for out-of-memory errors
             - Verify the VRAM usage on each GPU
             - Check for CUDA version compatibility
+            - If you see "Unsloth not available" error, run: `pip install unsloth>=2024.3 peft>=0.9.0`
             """)
 
 # Launch the app

requirements.txt

@@ -1,20 +1,26 @@
 accelerate>=0.27.0
 bitsandbytes>=0.41.0
 datasets>=2.15.0
+einops>=0.7.0
 filelock>=3.13.1
+flash-attn>=2.5.0
 gradio>=5.17.0
 huggingface-hub>=0.19.0
 matplotlib>=3.7.0
 numpy>=1.24.0
 packaging>=23.0
+peft>=0.9.0
+protobuf>=4.23.4
 psutil>=5.9.0
 python-dotenv>=1.0.0
 pyyaml>=6.0.1
 regex>=2023.0.0
 requests>=2.31.0
 safetensors>=0.4.1
+sentencepiece>=0.1.99
 tensorboard>=2.15.0
 torch>=2.0.0
 tqdm>=4.65.0
 transformers>=4.36.0
 typing-extensions>=4.8.0
+unsloth>=2024.3

run_transformers_training.py

@@ -127,31 +127,33 @@ def parse_args():
 def load_model_and_tokenizer(config):
     """Load model and tokenizer with proper error handling and optimizations."""
     try:
-        if unsloth_available:
-            logger.info("Using Unsloth optimizations with pre-quantized model")
-            model, tokenizer = FastLanguageModel.from_pretrained(
-                model_name=config.get("model_name"),
-                max_seq_length=config.get("max_seq_length", 2048),
-                dtype=None,  # Let Unsloth choose optimal dtype
-                device_map="auto",
-            )
-            
-            # Apply Unsloth's training optimizations with config parameters
-            model = FastLanguageModel.get_peft_model(
-                model,
-                r=config.get("unsloth_r", 32),
-                target_modules=config.get("unsloth_target_modules", 
-                    ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]),
-                lora_alpha=config.get("unsloth_alpha", 16),
-                lora_dropout=config.get("unsloth_dropout", 0.05),
-                bias="none",
-                use_gradient_checkpointing=config.get("gradient_checkpointing", True),
-                random_state=config.get("seed", 42),
-            )
-            logger.info("Unsloth optimizations applied successfully")
-        else:
+        if not unsloth_available:
             logger.error("Unsloth is required for training with pre-quantized model")
-            raise ImportError("Unsloth is required for this training setup")
+            logger.error("Please install required packages with: pip install -r requirements.txt")
+            logger.error("Or directly install with: pip install unsloth>=2024.3 peft>=0.9.0")
+            raise ImportError("Required packages missing. See log for installation instructions.")
+            
+        logger.info("Using Unsloth optimizations with pre-quantized model")
+        model, tokenizer = FastLanguageModel.from_pretrained(
+            model_name=config.get("model_name"),
+            max_seq_length=config.get("max_seq_length", 2048),
+            dtype=None,  # Let Unsloth choose optimal dtype
+            device_map="auto",
+        )
+        
+        # Apply Unsloth's training optimizations with config parameters
+        model = FastLanguageModel.get_peft_model(
+            model,
+            r=config.get("unsloth_r", 32),
+            target_modules=config.get("unsloth_target_modules", 
+                ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]),
+            lora_alpha=config.get("unsloth_alpha", 16),
+            lora_dropout=config.get("unsloth_dropout", 0.05),
+            bias="none",
+            use_gradient_checkpointing=config.get("gradient_checkpointing", True),
+            random_state=config.get("seed", 42),
+        )
+        logger.info("Unsloth optimizations applied successfully")
 
         # Set up tokenizer settings
         if config.get("chat_template"):
@@ -168,6 +170,7 @@ def load_model_and_tokenizer(config):
     
     except Exception as e:
         logger.error(f"Error in model/tokenizer loading: {str(e)}")
+        logger.error("If missing dependencies, install with: pip install -r requirements.txt")
         raise
 
 def load_dataset_with_mapping(dataset_config):
@@ -359,6 +362,43 @@ class SimpleDataCollator:
         
         return batch
 
+def check_dependencies():
+    """Check if all required dependencies are installed."""
+    missing_packages = []
+    
+    # Critical packages
+    if not unsloth_available:
+        missing_packages.append("unsloth>=2024.3")
+    
+    if not peft_available:
+        missing_packages.append("peft>=0.9.0")
+    
+    # Optional but recommended packages
+    try:
+        import flash_attn
+    except ImportError:
+        logger.warning("flash-attn not found. Flash attention will not be used.")
+        missing_packages.append("flash-attn>=2.5.0 (optional)")
+    
+    # If critical packages are missing, exit with instructions
+    critical_missing = [pkg for pkg in missing_packages if "optional" not in pkg]
+    if critical_missing:
+        logger.error("Critical dependencies missing:")
+        for pkg in critical_missing:
+            logger.error(f"  - {pkg}")
+        logger.error("Please install required packages with: pip install -r requirements.txt")
+        logger.error(f"Or directly with: pip install {' '.join(critical_missing)}")
+        return False
+    
+    # If optional packages are missing, just warn
+    optional_missing = [pkg for pkg in missing_packages if "optional" in pkg]
+    if optional_missing:
+        logger.warning("Optional dependencies missing (recommended for best performance):")
+        for pkg in optional_missing:
+            logger.warning(f"  - {pkg.split(' ')[0]}")
+    
+    return True
+
 def main():
     # Set up logging
     logger.info("Starting training process")
@@ -366,6 +406,11 @@ def main():
     # Parse arguments
     args = parse_args()
     
+    # Check dependencies
+    if not check_dependencies():
+        logger.error("Aborting due to missing critical dependencies")
+        return 1
+    
     # Load environment variables
     load_env_variables()