tt

fix_compatibility.bat

@@ -0,0 +1,35 @@
+@echo off
+echo Fixing Python 3.12 compatibility issues for DPO training...
+echo.
+
+REM Uninstall problematic packages
+echo Removing conflicting packages...
+pip uninstall -y tensorflow keras protobuf
+
+REM Install tf-keras for compatibility
+echo Installing tf-keras...
+pip install tf-keras
+
+REM Install specific protobuf version
+echo Installing compatible protobuf...
+pip install protobuf==3.20.3
+
+REM Install PyTorch with CUDA 11.8 support
+echo Installing PyTorch...
+pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118
+
+REM Install other dependencies with specific versions
+echo Installing other dependencies...
+pip install transformers==4.36.2
+pip install accelerate==0.25.0
+pip install peft==0.7.1
+pip install trl==0.7.10
+pip install bitsandbytes==0.42.0
+pip install datasets
+pip install pandas
+pip install scipy
+pip install sentencepiece
+
+echo.
+echo Done! Now try running: python train_dpo_hf_fixed.py
+pause

fix_compatibility.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Fix Python 3.12 compatibility issues for DPO training
+
+echo "Fixing Python 3.12 compatibility issues for DPO training..."
+echo
+
+# Uninstall problematic packages
+echo "Removing conflicting packages..."
+pip uninstall -y tensorflow keras protobuf
+
+# Install tf-keras for compatibility
+echo "Installing tf-keras..."
+pip install tf-keras
+
+# Install specific protobuf version
+echo "Installing compatible protobuf..."
+pip install protobuf==3.20.3
+
+# Install PyTorch with CUDA 11.8 support
+echo "Installing PyTorch..."
+pip install torch==2.1.2 torchvision==0.16.2 torchaudio==2.1.2 --index-url https://download.pytorch.org/whl/cu118
+
+# Install other dependencies with specific versions
+echo "Installing other dependencies..."
+pip install transformers==4.36.2
+pip install accelerate==0.25.0
+pip install peft==0.7.1
+pip install trl==0.7.10
+pip install bitsandbytes==0.42.0
+pip install datasets
+pip install pandas
+pip install scipy
+pip install sentencepiece
+
+echo
+echo "Done! Now try running: python train_dpo_hf_fixed.py"

setup_environment.py

@@ -0,0 +1,120 @@
+"""
+Setup script to ensure all dependencies are correctly installed
+"""
+
+import subprocess
+import sys
+import os
+
+def run_command(cmd):
+    """Run a command and return success status"""
+    try:
+        subprocess.check_call(cmd, shell=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+def main():
+    print("🔧 Setting up environment for DPO training...")
+    print("="*60)
+    
+    # Python version check
+    python_version = sys.version_info
+    print(f"Python version: {python_version.major}.{python_version.minor}.{python_version.micro}")
+    
+    if python_version.major < 3 or (python_version.major == 3 and python_version.minor < 8):
+        print("❌ Python 3.8+ is required!")
+        sys.exit(1)
+    
+    # Fix protobuf issues
+    print("\n📦 Fixing protobuf issues...")
+    run_command(f"{sys.executable} -m pip uninstall -y protobuf")
+    run_command(f"{sys.executable} -m pip install protobuf==3.20.3")
+    
+    # Install tf-keras for compatibility
+    print("\n📦 Installing tf-keras for compatibility...")
+    run_command(f"{sys.executable} -m pip install tf-keras")
+    
+    # Core dependencies
+    print("\n📦 Installing core dependencies...")
+    dependencies = [
+        "torch>=2.0.0",
+        "transformers>=4.36.0",
+        "datasets",
+        "accelerate>=0.25.0",
+        "peft>=0.7.0",
+        "trl>=0.7.0",
+        "bitsandbytes>=0.41.0",
+        "pandas",
+        "scipy",
+        "sentencepiece",  # Required for some tokenizers
+        "protobuf==3.20.3",  # Specific version to avoid issues
+    ]
+    
+    for dep in dependencies:
+        print(f"Installing {dep}...")
+        if not run_command(f"{sys.executable} -m pip install {dep}"):
+            print(f"⚠️ Failed to install {dep}, continuing...")
+    
+    # BEIR dependencies (optional)
+    print("\n📦 Installing BEIR dependencies (optional)...")
+    beir_deps = ["beir", "scikit-learn", "tqdm"]
+    for dep in beir_deps:
+        print(f"Installing {dep}...")
+        run_command(f"{sys.executable} -m pip install {dep}")
+    
+    # Check CUDA
+    print("\n🔍 Checking CUDA availability...")
+    try:
+        import torch
+        if torch.cuda.is_available():
+            print(f"✅ CUDA is available!")
+            print(f"   Device: {torch.cuda.get_device_name(0)}")
+            print(f"   CUDA version: {torch.version.cuda}")
+        else:
+            print("⚠️ CUDA not available. Training will be slow on CPU.")
+    except Exception as e:
+        print(f"⚠️ Could not check CUDA: {e}")
+    
+    # Test imports
+    print("\n🧪 Testing imports...")
+    test_imports = [
+        "torch",
+        "transformers",
+        "trl",
+        "peft",
+        "datasets",
+        "accelerate",
+        "bitsandbytes",
+        "pandas"
+    ]
+    
+    failed = []
+    for module in test_imports:
+        try:
+            __import__(module)
+            print(f"✅ {module}")
+        except ImportError as e:
+            print(f"❌ {module}: {e}")
+            failed.append(module)
+    
+    if failed:
+        print(f"\n⚠️ Some imports failed: {', '.join(failed)}")
+        print("Try running: pip install --upgrade " + " ".join(failed))
+    else:
+        print("\n✅ All imports successful!")
+    
+    # Generate sample data if needed
+    if not os.path.exists("train.csv"):
+        print("\n📊 Generating sample data...")
+        try:
+            run_command(f"{sys.executable} generate_sample_data.py")
+        except:
+            print("⚠️ Could not generate sample data")
+    
+    print("\n✅ Setup complete!")
+    print("\nTo start training, run:")
+    print(f"  {sys.executable} train_dpo_hf_fixed.py")
+
+if __name__ == "__main__":
+    main()

train_dpo_hf_fixed.py

@@ -0,0 +1,366 @@
+"""
+DPO Training Script for Phi-3 Mini - Fixed version
+Handles dependency issues and provides cleaner error handling
+"""
+
+import os
+import sys
+import json
+import warnings
+warnings.filterwarnings("ignore")
+
+# Set environment variables to avoid TensorFlow issues
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
+
+try:
+    import torch
+    import pandas as pd
+    from transformers import (
+        AutoTokenizer, 
+        AutoModelForCausalLM,
+        TrainingArguments,
+        TrainerCallback,
+        TrainerState,
+        TrainerControl
+    )
+    from trl import DPOTrainer
+    from trl.trainer.dpo_config import DPOConfig
+    from datasets import Dataset
+    from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+    from datetime import datetime
+    import logging
+except ImportError as e:
+    print(f"Missing dependency: {e}")
+    print("\nPlease install required packages:")
+    print("pip install torch transformers trl peft datasets accelerate bitsandbytes pandas")
+    print("\nIf you get Keras errors, also run:")
+    print("pip install tf-keras")
+    sys.exit(1)
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Configuration
+MODEL_ID = "microsoft/Phi-3-mini-4k-instruct"
+HF_USERNAME = os.environ.get("HF_USERNAME", "your-username")
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+PROJECT_NAME = "phi3-dpo-beir"
+OUTPUT_DIR = f"./{PROJECT_NAME}-checkpoints"
+
+class ValidationCallback(TrainerCallback):
+    """Custom callback to evaluate model similar to evaluate.py"""
+    
+    def __init__(self, tokenizer, val_dataset, eval_freq=500):
+        self.tokenizer = tokenizer
+        self.val_dataset = val_dataset
+        self.eval_freq = eval_freq
+        
+    def format_prompt_for_inference(self, query, document):
+        """Format for inference matching evaluate.py style"""
+        prompt = f"""You are an AI content analyst.
+
+Task:
+1. Given the following content and a user query, decide if the content is relevant.
+2. If it is relevant:
+   - Extract the top 2-3 key sentences
+   - Suggest 3-5 relevant tags
+   - Provide a short explanation or content extension (~2-3 sentences)
+
+Format your response in JSON with:
+{{
+  "relevant": true or false,
+  "key_sentences": [...],
+  "tags": [...],
+  "expansion": "..."
+}}
+
+User Query:
+{query}
+
+Content:
+{document}
+
+Response:"""
+        return prompt
+    
+    def on_step_end(self, args, state: TrainerState, control: TrainerControl, **kwargs):
+        """Run validation every N steps"""
+        if state.global_step % self.eval_freq == 0 and state.global_step > 0:
+            logger.info(f"\n🔍 Running custom validation at step {state.global_step}")
+            
+            model = kwargs["model"]
+            model.eval()
+            
+            # Sample validation examples
+            sample_size = min(5, len(self.val_dataset))
+            samples = self.val_dataset.shuffle(seed=42).select(range(sample_size))
+            
+            correct = 0
+            for sample in samples:
+                try:
+                    # Extract query and document
+                    prompt_text = sample["prompt"]
+                    lines = prompt_text.split("\n")
+                    
+                    # Find query and document sections
+                    query_idx = -1
+                    doc_idx = -1
+                    for i, line in enumerate(lines):
+                        if line.strip() == "Query:":
+                            query_idx = i + 1
+                        elif line.strip() == "Document:":
+                            doc_idx = i + 1
+                    
+                    if query_idx == -1 or doc_idx == -1:
+                        continue
+                        
+                    query = lines[query_idx].strip()
+                    doc_parts = lines[doc_idx:]
+                    document = "\n".join(doc_parts).strip()
+                    
+                    # Generate response
+                    inference_prompt = self.format_prompt_for_inference(query, document)
+                    inputs = self.tokenizer(
+                        inference_prompt, 
+                        return_tensors="pt", 
+                        truncation=True,
+                        max_length=512
+                    )
+                    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+                    
+                    with torch.no_grad():
+                        outputs = model.generate(
+                            **inputs,
+                            max_new_tokens=256,
+                            temperature=0.1,
+                            do_sample=True,
+                            pad_token_id=self.tokenizer.eos_token_id
+                        )
+                    
+                    response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+                    response = response[len(inference_prompt):].strip()
+                    
+                    # Simple accuracy check
+                    expected = sample["chosen"].lower()
+                    if expected in response.lower():
+                        correct += 1
+                    
+                    logger.info(f"Expected: {expected}, Got: {response[:100]}...")
+                except Exception as e:
+                    logger.error(f"Validation error: {e}")
+                    continue
+            
+            if sample_size > 0:
+                accuracy = correct / sample_size
+                logger.info(f"✅ Validation accuracy: {accuracy:.2%}")
+        
+        return control
+
+def prepare_datasets():
+    """Load and prepare DPO datasets"""
+    logger.info("📊 Loading datasets...")
+    
+    # Check if data files exist
+    if not os.path.exists("train.csv"):
+        logger.error("train.csv not found!")
+        logger.info("Please run download_beir_datasets.py first or use generate_sample_data.py")
+        return None, None, None
+    
+    # Load CSVs
+    train_df = pd.read_csv("train.csv")
+    val_df = pd.read_csv("val.csv") if os.path.exists("val.csv") else None
+    test_df = pd.read_csv("test.csv") if os.path.exists("test.csv") else None
+    
+    # Convert to HF datasets
+    train_dataset = Dataset.from_pandas(train_df)
+    val_dataset = Dataset.from_pandas(val_df) if val_df is not None else None
+    test_dataset = Dataset.from_pandas(test_df) if test_df is not None else None
+    
+    logger.info(f"✅ Loaded {len(train_dataset)} training examples")
+    if val_dataset:
+        logger.info(f"✅ Loaded {len(val_dataset)} validation examples")
+    
+    return train_dataset, val_dataset, test_dataset
+
+def get_model_and_tokenizer():
+    """Load model and tokenizer with 4-bit quantization for A10G"""
+    logger.info(f"🤖 Loading model: {MODEL_ID}")
+    
+    # Tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.padding_side = "left"  # Important for DPO
+    
+    # Check if CUDA is available
+    if not torch.cuda.is_available():
+        logger.warning("⚠️ CUDA not available. Loading model in CPU mode (will be slow!)")
+        model = AutoModelForCausalLM.from_pretrained(
+            MODEL_ID,
+            torch_dtype=torch.float32,
+            device_map="cpu",
+            trust_remote_code=True
+        )
+    else:
+        # Model with 4-bit quantization
+        try:
+            model = AutoModelForCausalLM.from_pretrained(
+                MODEL_ID,
+                load_in_4bit=True,
+                torch_dtype=torch.float16,
+                device_map="auto",
+                trust_remote_code=True,
+                bnb_4bit_compute_dtype=torch.float16,
+                bnb_4bit_use_double_quant=True,
+                bnb_4bit_quant_type="nf4"
+            )
+            model = prepare_model_for_kbit_training(model)
+        except Exception as e:
+            logger.error(f"Failed to load model in 4-bit: {e}")
+            logger.info("Falling back to full precision...")
+            model = AutoModelForCausalLM.from_pretrained(
+                MODEL_ID,
+                torch_dtype=torch.float16,
+                device_map="auto",
+                trust_remote_code=True
+            )
+    
+    return model, tokenizer
+
+def get_peft_config():
+    """Get LoRA configuration optimized for A10G"""
+    return LoraConfig(
+        r=16,
+        lora_alpha=32,
+        target_modules=[
+            "q_proj", "v_proj", "k_proj", "o_proj",
+            "gate_proj", "up_proj", "down_proj"
+        ],
+        lora_dropout=0.1,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+
+def main():
+    logger.info("="*60)
+    logger.info("🚀 Starting DPO Training for Phi-3 Mini")
+    logger.info("="*60)
+    
+    # Load datasets
+    train_dataset, val_dataset, test_dataset = prepare_datasets()
+    if train_dataset is None:
+        return
+    
+    # Load model and tokenizer
+    try:
+        model, tokenizer = get_model_and_tokenizer()
+    except Exception as e:
+        logger.error(f"Failed to load model: {e}")
+        return
+    
+    # LoRA config
+    peft_config = get_peft_config()
+    
+    # Training arguments optimized for A10G
+    training_args = DPOConfig(
+        output_dir=OUTPUT_DIR,
+        num_train_epochs=3,
+        per_device_train_batch_size=2,  # A10G can handle this
+        per_device_eval_batch_size=2,
+        gradient_accumulation_steps=4,  # Effective batch size = 8
+        gradient_checkpointing=True,
+        learning_rate=5e-5,
+        lr_scheduler_type="cosine",
+        warmup_ratio=0.1,
+        logging_steps=10,
+        save_steps=100,  # Save every 100 steps
+        eval_steps=500,
+        save_total_limit=5,  # Keep last 5 checkpoints
+        load_best_model_at_end=True,
+        metric_for_best_model="loss",
+        greater_is_better=False,
+        
+        # DPO specific
+        beta=0.1,  # DPO regularization
+        
+        # Optimization
+        optim="paged_adamw_8bit" if torch.cuda.is_available() else "adamw_torch",
+        fp16=torch.cuda.is_available(),
+        
+        # Logging
+        report_to="none",  # Disable wandb for simplicity
+        run_name=f"{PROJECT_NAME}-{datetime.now().strftime('%Y%m%d-%H%M')}",
+        
+        # Hub integration
+        push_to_hub=True if HF_TOKEN else False,
+        hub_model_id=f"{HF_USERNAME}/{PROJECT_NAME}" if HF_TOKEN else None,
+        hub_strategy="checkpoint",  # Push every checkpoint
+        hub_token=HF_TOKEN,
+    )
+    
+    # Initialize trainer
+    try:
+        dpo_trainer = DPOTrainer(
+            model=model,
+            ref_model=None,  # Will create a reference model copy
+            args=training_args,
+            train_dataset=train_dataset,
+            eval_dataset=val_dataset,
+            tokenizer=tokenizer,
+            peft_config=peft_config,
+            max_prompt_length=512,
+            max_length=768,
+        )
+    except Exception as e:
+        logger.error(f"Failed to initialize trainer: {e}")
+        return
+    
+    # Add custom validation callback
+    if val_dataset:
+        val_callback = ValidationCallback(tokenizer, val_dataset)
+        dpo_trainer.add_callback(val_callback)
+    
+    # Start training
+    logger.info("🚀 Starting DPO training...")
+    logger.info(f"💾 Checkpoints will be saved to: {OUTPUT_DIR}")
+    if HF_TOKEN:
+        logger.info(f"🤗 Model will be pushed to: https://huggingface.co/{HF_USERNAME}/{PROJECT_NAME}")
+    
+    # Print some info about the data
+    logger.info("\n📊 Data Statistics:")
+    logger.info(f"Training samples: {len(train_dataset)}")
+    if val_dataset:
+        logger.info(f"Validation samples: {len(val_dataset)}")
+    
+    # Show a sample
+    logger.info("\n📝 Sample training data:")
+    sample = train_dataset[0]
+    logger.info(f"Prompt (first 200 chars): {sample['prompt'][:200]}...")
+    logger.info(f"Chosen: {sample['chosen']}")
+    logger.info(f"Rejected: {sample['rejected']}")
+    
+    try:
+        dpo_trainer.train()
+    except KeyboardInterrupt:
+        logger.info("\n⚠️ Training interrupted by user")
+    except Exception as e:
+        logger.error(f"\n❌ Training failed: {e}")
+        return
+    
+    # Save final model
+    logger.info("💾 Saving final model...")
+    dpo_trainer.save_model(f"{OUTPUT_DIR}/final")
+    
+    # Push to hub
+    if HF_TOKEN:
+        logger.info("🤗 Pushing final model to Hub...")
+        try:
+            dpo_trainer.push_to_hub()
+        except Exception as e:
+            logger.error(f"Failed to push to hub: {e}")
+    
+    logger.info("✅ Training complete!")
+    logger.info(f"📁 Model saved to: {OUTPUT_DIR}/final")
+
+if __name__ == "__main__":
+    main()