Upload run_cloud_training.py with huggingface_hub

run_cloud_training.py

@@ -110,14 +110,24 @@ def load_and_prepare_dataset(dataset_name, config):
         print(f"If it's a private dataset, ensure your HF_TOKEN has access to it.")
         raise
 
+def tokenize_string(text, tokenizer):
+    """Tokenize a string using the provided tokenizer"""
+    if not text:
+        return []
+    
+    # Tokenize the text
+    tokens = tokenizer.encode(text, add_special_tokens=False)
+    return tokens
+
 # Data collator for pre-tokenized dataset
 class PreTokenizedCollator(DataCollatorMixin):
     """
     Data collator for pre-tokenized datasets.
     Expects input_ids and labels already tokenized.
     """
-    def __init__(self, pad_token_id=0):
+    def __init__(self, pad_token_id=0, tokenizer=None):
         self.pad_token_id = pad_token_id
+        self.tokenizer = tokenizer  # Keep a reference to the tokenizer for string conversion
         
     def __call__(self, features):
         # Print a sample feature to understand structure
@@ -130,15 +140,58 @@ class PreTokenizedCollator(DataCollatorMixin):
             # If input_ids is not directly available, try to extract from conversations
             if 'input_ids' not in feature and 'conversations' in feature:
                 # Extract from conversations based on your dataset structure
-                # This is a placeholder - adjust based on actual structure
                 conversations = feature['conversations']
+                
+                # Debug the conversations structure
+                logger.info(f"Conversations type: {type(conversations)}")
                 if isinstance(conversations, list) and len(conversations) > 0:
-                    # Assuming input_ids might be in the content field
-                    if 'content' in conversations[0]:
-                        feature['input_ids'] = conversations[0]['content']
-                    # Or it might be the conversation itself
+                    logger.info(f"First conversation type: {type(conversations[0])}")
+                    logger.info(f"First conversation: {conversations[0]}")
+                
+                # Try different approaches to extract input_ids
+                if isinstance(conversations, list) and len(conversations) > 0:
+                    # Case 1: If conversations is a list of dicts with 'content' field
+                    if isinstance(conversations[0], dict) and 'content' in conversations[0]:
+                        content = conversations[0]['content']
+                        logger.info(f"Found content field: {type(content)}")
+                        
+                        # If content is a string, tokenize it
+                        if isinstance(content, str) and self.tokenizer:
+                            logger.info(f"Tokenizing string content: {content[:50]}...")
+                            feature['input_ids'] = self.tokenizer.encode(content, add_special_tokens=False)
+                        # If content is already a list of integers, use it directly
+                        elif isinstance(content, list) and all(isinstance(x, int) for x in content):
+                            feature['input_ids'] = content
+                        # If content is already tokenized in some other format
+                        else:
+                            logger.warning(f"Unexpected content format: {type(content)}")
+                            
+                    # Case 2: If conversations is a list of dicts with 'input_ids' field
                     elif isinstance(conversations[0], dict) and 'input_ids' in conversations[0]:
                         feature['input_ids'] = conversations[0]['input_ids']
+                    
+                    # Case 3: If conversations itself contains the input_ids
+                    elif all(isinstance(x, int) for x in conversations):
+                        feature['input_ids'] = conversations
+                    
+                    # Case 4: If conversations is a list of strings
+                    elif all(isinstance(x, str) for x in conversations) and self.tokenizer:
+                        # Join all strings and tokenize
+                        full_text = " ".join(conversations)
+                        feature['input_ids'] = self.tokenizer.encode(full_text, add_special_tokens=False)
+            
+            # Ensure input_ids is a list of integers
+            if 'input_ids' in feature:
+                # If input_ids is a string, tokenize it
+                if isinstance(feature['input_ids'], str) and self.tokenizer:
+                    logger.info(f"Converting string input_ids to tokens: {feature['input_ids'][:50]}...")
+                    feature['input_ids'] = self.tokenizer.encode(feature['input_ids'], add_special_tokens=False)
+                # If input_ids is not a list, convert it
+                elif not isinstance(feature['input_ids'], list):
+                    try:
+                        feature['input_ids'] = list(feature['input_ids'])
+                    except:
+                        logger.error(f"Could not convert input_ids to list: {type(feature['input_ids'])}")
             
             processed_features.append(feature)
             
@@ -380,7 +433,7 @@ def train(config_path, dataset_name, output_dir):
             model=model,
             args=training_args,
             train_dataset=training_dataset,
-            data_collator=PreTokenizedCollator(pad_token_id=tokenizer.pad_token_id),
+            data_collator=PreTokenizedCollator(pad_token_id=tokenizer.pad_token_id, tokenizer=tokenizer),
         )
         
         # Start training