Upload folder using huggingface_hub

run_transformers_training.py

@@ -391,7 +391,8 @@ def load_dataset_with_mapping(dataset_config):
         return 1
 
 def format_phi_chat(messages, dataset_config):
-    """Format messages according to phi-4's chat template and dataset config."""
+    """Format messages according to phi-4's chat template and dataset config.
+    Only formats the conversation structure, preserves the actual content."""
     formatted_chat = ""
     
     # Get role templates from config
@@ -407,13 +408,13 @@ def format_phi_chat(messages, dataset_config):
             logger.warning(f"Skipping invalid message format: {message}")
             continue
             
-        content = message.get("content", "").strip()
+        content = message.get("content", "")  # Don't strip() - preserve exact content
         
         # Skip empty content
         if not content:
             continue
             
-        # Infer role based on content patterns
+        # Only add role prefixes based on position/content
         if "[RESEARCH INTRODUCTION]" in content:
             # System message
             template = roles.get("system", "System: {content}\n\n")
@@ -429,7 +430,7 @@ def format_phi_chat(messages, dataset_config):
                 template = roles.get("assistant", "Assistant: {content}\n\n")
             formatted_chat += template.format(content=content)
     
-    return formatted_chat.strip()
+    return formatted_chat
 
 class SimpleDataCollator:
     def __init__(self, tokenizer, dataset_config):
@@ -459,17 +460,25 @@ class SimpleDataCollator:
                     self.stats["skipped"] += 1
                     continue
                 
-                # Format the conversation using phi chat template
-                formatted_chat = format_phi_chat(conversations, self.dataset_config)
-                
-                # Skip if formatting resulted in empty content
-                if not formatted_chat:
-                    logger.warning(f"Empty formatted chat for paper_id {paper_id}, prompt {prompt_num}")
+                # Get the pre-tokenized content directly
+                # The content should already be properly tokenized and formatted
+                content = conversations[0].get("content", "")
+                if not content:
+                    logger.warning(f"Empty content for paper_id {paper_id}, prompt {prompt_num}")
                     self.stats["skipped"] += 1
                     continue
                 
-                # Create input IDs and attention mask
-                input_ids = self.tokenizer.encode(formatted_chat, add_special_tokens=False)
+                # Convert string of numbers to list of integers if needed
+                if isinstance(content, str):
+                    try:
+                        # Assuming content is space-separated numbers
+                        input_ids = [int(x) for x in content.split()]
+                    except ValueError:
+                        logger.warning(f"Invalid pre-tokenized content format for paper_id {paper_id}, prompt {prompt_num}")
+                        self.stats["skipped"] += 1
+                        continue
+                else:
+                    input_ids = content
                 
                 # Truncate if needed
                 if len(input_ids) > self.max_seq_length:
@@ -489,8 +498,7 @@ class SimpleDataCollator:
                 
                 # Log first few examples for verification
                 if self.stats["processed"] <= 3:
-                    logger.info(f"Sample {self.stats['processed']} formatted chat:")
-                    logger.info(f"{formatted_chat[:200]}...")
+                    logger.info(f"Sample {self.stats['processed']} token count: {len(input_ids)}")
                 
             except Exception as e:
                 logger.warning(f"Error processing example {paper_id}, prompt {prompt_num}: {str(e)}")