Upload folder using huggingface_hub

app.py

@@ -84,15 +84,17 @@ def display_config():
     <ul>
         <li><b>Model:</b> {model_name}</li>
         <li><b>Learning Rate:</b> {training_config.get('learning_rate', '2e-5')}</li>
-        <li><b>Batch Size:</b> {training_config.get('per_device_train_batch_size', 4)} × {training_config.get('gradient_accumulation_steps', 4)} = {training_config.get('per_device_train_batch_size', 4) * training_config.get('gradient_accumulation_steps', 4)}</li>
-        <li><b>Epochs:</b> {training_config.get('num_train_epochs', 3)}</li>
+        <li><b>Per-Device Batch Size:</b> {batch_size}</li>
+        <li><b>Gradient Accumulation:</b> {grad_accum}</li>
+        <li><b>Total Effective Batch Size:</b> {batch_size} × {gpu_count} × {grad_accum} = {batch_size * gpu_count * grad_accum}</li>
+        <li><b>Epochs:</b> {epochs}</li>
         <li><b>Precision:</b> {'BF16' if transformers_config.get('bf16', True) else 'FP16' if transformers_config.get('fp16', False) else 'FP32'}</li>
         <li><b>Max Sequence Length:</b> {transformers_config.get('tokenizer', {}).get('max_seq_length', 2048)}</li>
     </ul>
     
     <h3>Hardware</h3>
     <ul>
-        <li><b>GPU:</b> {gpu_count}× {gpu_type} ({vram} GB)</li>
+        <li><b>GPU:</b> {gpu_count}× {gpu_type} ({vram} GB VRAM per GPU, total: {vram * gpu_count} GB)</li>
         <li><b>Multi-GPU Strategy:</b> {hardware_config.get('training_optimizations', {}).get('multi_gpu_strategy', 'data_parallel')}</li>
         <li><b>Memory Optimizations:</b> {'Gradient Checkpointing' if hardware_config.get('training_optimizations', {}).get('memory_optimizations', {}).get('use_gradient_checkpointing', True) else 'None'}</li>
     </ul>
@@ -154,9 +156,10 @@ def create_interface():
                 gr.Markdown("## Training Information")
                 gr.Markdown("""
                 ### Hardware:
-                - 4× NVIDIA L4 GPUs (24GB VRAM each)
+                - 4× NVIDIA L4 GPUs (24GB VRAM per GPU, 96GB total)
                 - Training with BF16 precision
                 - Using Data Parallel for multi-GPU
+                - Effective batch size: 16 (per device) × 4 (GPUs) × 3 (gradient accumulation) = 192
                 
                 ### Notes:
                 - Training may take several hours depending on dataset size

requirements.txt

@@ -3,6 +3,7 @@ bitsandbytes>=0.41.0
 datasets>=2.15.0
 einops>=0.7.0
 filelock>=3.13.1
+flash-attn>=2.5.1
 gradio>=5.17.0
 huggingface-hub>=0.19.0
 matplotlib>=3.7.0

run_transformers_training.py

@@ -309,315 +309,58 @@ def load_dataset_with_mapping(dataset_config):
                     if source != target:  # Only rename if names are different
                         dataset = dataset.rename_column(source, target)
         
-        # Add prompt_number field that increments based on original order
-        def add_prompt_numbers(examples, indices):
-            # Defensive check to ensure indices is not None and is iterable
-            if indices is None:
-                logger.warning("Warning: indices is None in add_prompt_numbers, using empty list")
-                indices = []
-            elif isinstance(indices, int):
-                # Handle case where indices is a single integer
-                logger.warning(f"Warning: indices is an integer ({indices}) in add_prompt_numbers, converting to list")
-                indices = [indices]
+        # Add prompt_number field that increments based on original order - simple approach
+        logger.info("Adding prompt_number based on original dataset order (starting at 1)")
+        
+        # Simple approach 1: Add index as a column during dataset creation
+        # Create a list of dicts with indices
+        examples_with_idx = []
+        for i, example in enumerate(dataset):
+            example = dict(example)  # Make a copy to avoid modifying the original
+            example['prompt_number'] = i + 1  # 1-indexed
+            examples_with_idx.append(example)
+        
+        # Recreate dataset with prompt_number included
+        from datasets import Dataset
+        dataset = Dataset.from_list(examples_with_idx)
+        logger.info("Successfully added prompt_number to dataset")
+        
+        # If conversations is missing but text exists, attempt conversion
+        if "conversations" not in dataset.column_names and "text" in dataset.column_names:
+            logger.info("Converting 'text' field to 'conversations' format")
             
-            # Ensure indices is always a list/iterable
-            try:
-                # Create a new field with the dataset index as the prompt number, starting at 1
-                examples["prompt_number"] = [idx + 1 for idx in indices]  # Adding 1 to make it 1-indexed
-            except TypeError:
-                # Fallback for non-iterable types
-                logger.warning(f"Warning: non-iterable indices in add_prompt_numbers: {type(indices)}, using default")
-                examples["prompt_number"] = [1] * len(next(iter(examples.values())))
-                
-            return examples
-        
-        # Add prompt numbers to the dataset based on original order
-        logger.info("Adding prompt numbers based on original dataset order (starting at 1)")
-        try:
-            dataset = dataset.map(
-                add_prompt_numbers, 
-                with_indices=True,
-                desc="Adding prompt numbers"
-            )
-            logger.info(f"Successfully added prompt_number field to dataset")
-        except Exception as e:
-            logger.error(f"Error adding prompt numbers: {e}")
-            # Create a fallback implementation that doesn't rely on with_indices
-            logger.info("Attempting fallback method for adding prompt numbers")
-            
-            def add_prompt_numbers_fallback(example, idx):
-                example["prompt_number"] = idx + 1
+            def convert_text_to_conversations(example):
+                # Check if text is already a list of conversation turns
+                if isinstance(example.get("text"), list):
+                    example["conversations"] = example["text"]
+                # Otherwise, create a simple conversation with the text as user message
+                else:
+                    example["conversations"] = [
+                        {"role": "user", "content": str(example.get("text", ""))}
+                    ]
                 return example
             
-            # Process each example one by one with explicit indices
-            updated_examples = []
-            for i, example in enumerate(dataset):
-                updated_examples.append(add_prompt_numbers_fallback(dict(example), i))
-            
-            # Create a new dataset with the updated examples
-            from datasets import Dataset
-            dataset = Dataset.from_list(updated_examples)
-            logger.info(f"Successfully added prompt_number field using fallback method")
-        
-        # Rename 'id' to 'article_id' if it exists
-        if 'id' in dataset.column_names and 'article_id' not in dataset.column_names:
-            logger.info("Renaming 'id' column to 'article_id'")
-            dataset = dataset.rename_column('id', 'article_id')
-        
-        # Reorder columns to make prompt_number first if it exists
-        if 'prompt_number' in dataset.column_names:
-            logger.info("Reordering columns to place prompt_number first")
-            # Get current column names
-            current_columns = dataset.column_names
-            # Create new column order with prompt_number first
-            new_column_order = ['prompt_number'] + [col for col in current_columns if col != 'prompt_number']
-            # Reorder columns
-            dataset = dataset.select_columns(new_column_order)
-        
-        # Verify all new column names for logging
-        logger.info(f"Dataset loaded successfully with {len(dataset)} examples")
-        logger.info(f"Dataset columns: {dataset.column_names}")
-        
-        # Verify dataset is not empty
-        if len(dataset) == 0:
-            logger.error("Dataset is empty! This will cause errors during training.")
-            raise ValueError("Empty dataset loaded")
-            
-        # Check for required columns
-        required_columns = ['conversations']
-        for col in required_columns:
-            if col not in dataset.column_names:
-                logger.error(f"Required column '{col}' not found in dataset!")
-                raise ValueError(f"Required column '{col}' missing from dataset")
-        
-        # Verify expected columns exist
-        expected_columns = {"article_id", "conversations", "prompt_number"}
-        missing_columns = expected_columns - set(dataset.column_names)
-        if missing_columns:
-            logger.warning(f"Some expected columns are missing: {missing_columns}")
-            
-            # If "conversations" is missing but "text" exists, attempt conversion
-            if "conversations" not in dataset.column_names and "text" in dataset.column_names:
-                logger.info("Converting 'text' field to 'conversations' format")
-                
-                def convert_text_to_conversations(example):
-                    # Check if text is already a list of conversation turns
-                    if isinstance(example.get("text"), list):
-                        example["conversations"] = example["text"]
-                    # Otherwise, create a simple conversation with the text as user message
-                    else:
-                        example["conversations"] = [
-                            {"role": "user", "content": str(example.get("text", ""))}
-                        ]
-                    return example
-                
-                dataset = dataset.map(convert_text_to_conversations)
-                logger.info("Successfully converted 'text' to 'conversations'")
-        
-        # Verify data ordering requirements
-        processing_config = dataset_config.get("dataset", {}).get("processing", {})
-        data_loading_config = dataset_config.get("data_loading", {})
-        
-        # Check if sorting is required
-        sort_by_article_id = processing_config.get("sort_by_article_id", False)
-        if sort_by_article_id and 'article_id' in dataset.column_names:
-            logger.info("Sorting dataset by article_id as specified in config")
-            dataset = dataset.sort("article_id")
-            sorted_ids = [example['article_id'] for example in dataset.select(range(min(5, len(dataset))))]
-            logger.info(f"First few article_ids after sorting: {sorted_ids}")
-        
-        # Flag consolidation - we only need one flag to control sequence preservation
-        # Default to True to ensure safety
-        preserve_sequence = processing_config.get("preserve_entry_sequence", True)
-        shuffle_disabled = not data_loading_config.get("shuffle", False)
-        
-        if not preserve_sequence:
-            logger.warning("CRITICAL: preserve_entry_sequence is set to False. This is NOT RECOMMENDED!")
-            logger.warning("Data sequence integrity is essential for proper model training.")
-        
-        if not shuffle_disabled:
-            logger.error("CRITICAL: shuffle is enabled in the dataset config!")
-            logger.error("This will RANDOMIZE your dataset and break sequential order.")
-            logger.error("Please set shuffle: false in your data_loading configuration.")
-            # Actually enforce sequence preservation by raising an error
-            raise ValueError("Dataset shuffling is enabled but preserve_entry_sequence is required. " +
-                             "Please disable shuffling in your configuration.")
-        
-        # Verify the IDs are in sequential order if they're numeric
-        try:
-            if len(dataset) > 1:
-                # Check prompt numbers are sequential
-                sample_indices = range(min(10, len(dataset)))
-                sample_prompt_numbers = []
-                
-                # Defensive collection of prompt numbers
-                for i in sample_indices:
-                    try:
-                        if i < len(dataset) and "prompt_number" in dataset[i]:
-                            sample_prompt_numbers.append(dataset[i]["prompt_number"])
-                        else:
-                            # If prompt_number doesn't exist, use index+1 as fallback
-                            sample_prompt_numbers.append(i + 1)
-                            logger.warning(f"Sample at index {i} missing prompt_number, using {i+1} as fallback")
-                    except Exception as e:
-                        logger.warning(f"Error accessing sample at index {i}: {e}")
-                        sample_prompt_numbers.append(i + 1)  # Use fallback
-                
-                logger.info(f"Verifying sequential integrity with prompt numbers: {sample_prompt_numbers}")
-                
-                # Check if prompt numbers are sequential (1-indexed)
-                if sample_prompt_numbers:
-                    is_sequential = all(sample_prompt_numbers[i] == i + 1 for i in range(len(sample_prompt_numbers)))
-                    if not is_sequential:
-                        logger.warning("WARNING: Prompt numbers are not in sequential order!")
-                        logger.warning("This may indicate that data sequence is not preserved.")
-                    else:
-                        logger.info("Prompt numbers verify that samples are in sequential order.")
-                else:
-                    logger.warning("Could not verify sequential integrity: no prompt numbers collected")
-                
-                # Also check original IDs as a backup if numeric
-                try:
-                    sample_examples = []
-                    for i in sample_indices:
-                        try:
-                            if i < len(dataset):
-                                sample_examples.append(dataset[i])
-                        except Exception as e:
-                            logger.warning(f"Error accessing dataset at index {i}: {e}")
-                    
-                    if sample_examples:
-                        id_field = 'article_id' if 'article_id' in dataset.column_names else 'id'
-                        if all(isinstance(example.get(id_field, ''), (int, str)) for example in sample_examples):
-                            sample_ids = [example.get(id_field, '') for example in sample_examples if id_field in example]
-                            
-                            if sample_ids and all(isinstance(id, int) or (isinstance(id, str) and id.isdigit()) for id in sample_ids):
-                                numeric_ids = [int(id) if isinstance(id, str) else id for id in sample_ids]
-                                if len(numeric_ids) > 1:
-                                    is_ordered = all(numeric_ids[i] <= numeric_ids[i+1] for i in range(len(numeric_ids)-1))
-                                    if not is_ordered:
-                                        logger.warning(f"WARNING: Sample {id_field}s are not in sequential order.")
-                                    else:
-                                        logger.info(f"Sample {id_field}s appear to be in sequential order.")
-                except Exception as e:
-                    logger.warning(f"Error checking ID sequence: {e}")
-        except Exception as e:
-            logger.warning(f"Could not verify sequential integrity: {e}")
+            dataset = dataset.map(convert_text_to_conversations)
+            logger.info("Successfully converted 'text' to 'conversations'")
         
-        # Log examples without printing full content - with defensive coding
-        if "conversations" in dataset.column_names:
-            try:
-                # Safely get first few samples
-                first_few_indices = range(min(5, len(dataset)))
-                sample_prompt_numbers = []
-                sample_article_ids = []
-                
-                for i in first_few_indices:
-                    try:
-                        example = dataset[i]
-                        if 'prompt_number' in example:
-                            sample_prompt_numbers.append(example['prompt_number'])
-                        if 'article_id' in example:
-                            sample_article_ids.append(example['article_id'])
-                    except Exception as e:
-                        logger.warning(f"Error accessing sample at index {i}: {e}")
-                
-                logger.info(f"First few samples - Prompt numbers: {sample_prompt_numbers}, Article IDs: {sample_article_ids}")
-            
-                # Log conversation structure without full content
-                if len(dataset) > 0:
-                    try:
-                        sample_conv_structure = []
-                        first_example = dataset[0]
-                        
-                        if 'conversations' in first_example and first_example['conversations'] is not None:
-                            for msg in first_example['conversations']:
-                                if isinstance(msg, dict):
-                                    content = msg.get('content', '')
-                                    preview = content[:50] + "..." if len(content) > 50 else content
-                                    sample_conv_structure.append({
-                                        "role": msg.get('role', ''),
-                                        "content_length": len(content),
-                                        "preview": preview
-                                    })
-                            logger.info(f"Conversation structure: {sample_conv_structure}")
-                    except Exception as e:
-                        logger.warning(f"Error logging conversation structure: {e}")
-            except Exception as e:
-                logger.warning(f"Error logging sample examples: {e}")
+        # Verify we have the required columns
+        if "conversations" not in dataset.column_names:
+            logger.error("Required 'conversations' column not found in dataset!")
+            raise ValueError("Required 'conversations' column missing from dataset")
         
+        # Log column names and a sample
         logger.info(f"Dataset loaded successfully with {len(dataset)} examples")
         logger.info(f"Dataset columns: {dataset.column_names}")
         
-        # Verify dataset is not empty
-        if len(dataset) == 0:
-            logger.error("Dataset is empty! Cannot proceed with training.")
-            return dataset
+        # Log a sample for inspection
+        if len(dataset) > 0:
+            sample = dataset[0]
+            prompt_num = sample.get("prompt_number", "N/A")
+            article_id = sample.get("article_id", sample.get("id", "N/A"))
+            logger.info(f"First sample - Prompt number: {prompt_num}, ID: {article_id}")
             
-        # Check for required columns
-        required_cols = ['conversations', 'prompt_number']
-        for col in required_cols:
-            if col not in dataset.column_names:
-                logger.error(f"Required column '{col}' missing from dataset. Cannot proceed with training.")
-                return dataset
-        
-        # Validate at least one sample can be processed
-        try:
-            if len(dataset) > 0:
-                sample = dataset[0]
-                if 'conversations' not in sample or not sample['conversations']:
-                    logger.error("First sample has no conversations! Data format may be incorrect.")
-                    return dataset
-                if not isinstance(sample['conversations'], list):
-                    logger.error(f"Conversations field should be a list but got {type(sample['conversations'])}")
-                    return dataset
-        except Exception as e:
-            logger.error(f"Error validating first sample: {e}")
-            return dataset
-        
-        # Add metadata if specified
-        metadata_config = dataset_config.get("data_formatting", {}).get("metadata_handling", {})
-        if metadata_config:
-            include_article_id = metadata_config.get("include_article_id", False)
-            include_prompt_number = metadata_config.get("include_prompt_number", False)
-            metadata_format = metadata_config.get("metadata_format", "")
-            
-            if (include_article_id or include_prompt_number) and metadata_format:
-                logger.info("Adding metadata to conversations")
-                
-                def add_metadata(example):
-                    if not example.get("conversations"):
-                        return example
-                    
-                    # Prepare metadata
-                    metadata = metadata_format
-                    if include_article_id and "article_id" in example:
-                        metadata = metadata.replace("{article_id}", str(example.get("article_id", "")))
-                    if include_prompt_number and "prompt_number" in example:
-                        metadata = metadata.replace("{prompt_number}", str(example.get("prompt_number", "")))
-                    
-                    # Add system message with metadata if not empty
-                    if metadata.strip():
-                        if example["conversations"] and isinstance(example["conversations"], list):
-                            # Check if first message is already a system message
-                            if (isinstance(example["conversations"][0], dict) and 
-                                example["conversations"][0].get("role") == "system"):
-                                # Append to existing system message
-                                example["conversations"][0]["content"] += f"\n\nMetadata: {metadata}"
-                            else:
-                                # Add new system message at the beginning
-                                example["conversations"].insert(0, {
-                                    "role": "system",
-                                    "content": f"Metadata: {metadata}"
-                                })
-                    
-                    return example
-                
-                dataset = dataset.map(add_metadata)
-                logger.info("Metadata added to conversations")
-        
         return dataset
-        
+    
     except Exception as e:
         logger.error(f"Error loading dataset: {str(e)}")
         raise
@@ -1112,6 +855,10 @@ def main():
         per_device_batch_size = transformers_config.get("training", {}).get("per_device_train_batch_size", 16)
         gradient_accumulation_steps = transformers_config.get("training", {}).get("gradient_accumulation_steps", 3)
         
+        # Get multi-GPU strategy from hardware config (default to data_parallel)
+        multi_gpu_strategy = hardware_config.get("training_optimizations", {}).get("multi_gpu_strategy", "data_parallel")
+        logger.info(f"Multi-GPU strategy: {multi_gpu_strategy}")
+        
         # For multi-GPU setup, adjust for better balance
         if CUDA_AVAILABLE and NUM_GPUS > 1:
             log_info(f"Multi-GPU setup: Adjusting for {NUM_GPUS} GPUs")
@@ -1213,21 +960,17 @@ def main():
             """Custom dataloader that preserves original dataset order"""
             log_info("Creating sequential dataloader to maintain original dataset order")
             
-            # Verification of sequence preservation flags - consolidated
-            data_loading_config = dataset_config.get("data_loading", {})
-            sequential_processing = data_loading_config.get("sequential_processing", True)
-            shuffle_disabled = not data_loading_config.get("shuffle", False)
+            # Create a simple sequential sampler
+            sequential_sampler = torch.utils.data.SequentialSampler(dataset)
             
-            if not sequential_processing:
-                log_info("CRITICAL WARNING: sequential_processing flag is disabled! This may affect data order.")
-                log_info("Data sequence integrity is essential - using sequential sampler regardless of flag.")
-                # Force sequential processing regardless of flag
+            # Verification of sequence preservation flags - simplified
+            data_loading_config = dataset_config.get("data_loading", {})
+            shuffle_enabled = data_loading_config.get("shuffle", False)
             
-            if not shuffle_disabled:
-                log_info("CRITICAL ERROR: Shuffle is not disabled! This will randomize data entry order!")
-                # Actually handle the error rather than just logging it
+            if shuffle_enabled:
+                log_info("CRITICAL ERROR: Shuffle is enabled! This will randomize data entry order!")
                 raise ValueError("Dataset shuffling is enabled but sequential processing is required. " +
-                                 "Please disable shuffling in your configuration.")
+                              "Please disable shuffling in your configuration.")
             
             # Calculate batch size based on device availability
             if getattr(training_args, "no_cuda", False):