Add dataset integration to Hello World model

- Updated model.py with load_dataset() and prepare_dataset_batch() methods
- Added example_with_dataset.py demonstrating full dataset usage
- Created dataset_integration_test.py for verifying setup
- Updated README with dataset references and usage examples
- Model now works with chiedo/hello-world dataset on Hugging Face

🤖 Generated with Claude Code (https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git push:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

README.md

@@ -18,6 +18,10 @@ A minimal "Hello World" transformer model for demonstration purposes on Hugging
 
 This is a simple transformer-based language model that serves as a basic example for uploading models to Hugging Face. It demonstrates the minimum required files and structure for a custom model.
 
+### Associated Dataset
+
+This model works with the [chiedo/hello-world dataset](https://huggingface.co/datasets/chiedo/hello-world), which contains 20 examples of "Hello World" variations for demonstration purposes.
+
 ### Architecture Details
 - **Model Type**: Custom Transformer (hello_world)
 - **Vocabulary Size**: 13 tokens
@@ -34,8 +38,9 @@ This is a simple transformer-based language model that serves as a basic example
 - `pytorch_model.bin` - Model weights (PyTorch format)
 - `tokenizer.json` - Tokenizer vocabulary and settings
 - `tokenizer_config.json` - Tokenizer configuration
-- `model.py` - Model implementation (HelloWorldModel class)
+- `model.py` - Model implementation (HelloWorldModel class with dataset loading methods)
 - `test_model.py` - Test script for local validation
+- `example_with_dataset.py` - Example script showing dataset integration
 
 ## Installation
 
@@ -251,6 +256,44 @@ with torch.no_grad():
     print(f"Model output shape: {logits.shape}")
 ```
 
+### Using the Model with Its Dataset
+
+This model includes built-in methods to work with the [chiedo/hello-world dataset](https://huggingface.co/datasets/chiedo/hello-world):
+
+#### Loading the Dataset Through the Model
+```python
+from transformers import AutoModel, AutoTokenizer
+from datasets import load_dataset
+
+# Load model and tokenizer
+model = AutoModel.from_pretrained("chiedo/hello-world", trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("chiedo/hello-world", trust_remote_code=True)
+
+# Method 1: Use the model's built-in dataset loading
+dataset = model.load_dataset("chiedo/hello-world")
+print(f"Dataset splits: {list(dataset.keys())}")
+
+# Method 2: Load dataset directly
+dataset = load_dataset("chiedo/hello-world")
+
+# Process a batch from the dataset
+texts = dataset["train"]["text"][:5]
+inputs = model.prepare_dataset_batch(texts, tokenizer)
+outputs = model(**inputs)
+```
+
+#### Complete Example with Dataset
+```python
+# Run the full example script
+python example_with_dataset.py
+```
+
+This will demonstrate:
+- Loading the model and dataset
+- Processing batches from the dataset
+- Running inference on dataset examples
+- Accessing dataset labels and features
+
 ## Model Vocabulary
 
 The model includes a minimal vocabulary:

dataset_integration_test.py

@@ -0,0 +1,88 @@
+"""
+Simple test to verify dataset integration setup.
+This test doesn't require external libraries to be installed.
+"""
+
+import json
+import os
+
+def test_dataset_files():
+    """Test that dataset files exist and are properly formatted."""
+    
+    dataset_path = os.path.expanduser("~/huggingface.co/datasets/chiedo/hello-world")
+    
+    print("Testing Dataset Integration Setup")
+    print("=" * 50)
+    
+    # Check dataset files exist
+    required_files = ["train.jsonl", "validation.jsonl", "test.jsonl", "README.md", "hello_world.py"]
+    
+    print("\n1. Checking dataset files:")
+    for file in required_files:
+        file_path = os.path.join(dataset_path, file)
+        if os.path.exists(file_path):
+            print(f"   ✓ {file} exists")
+        else:
+            print(f"   ✗ {file} missing")
+    
+    # Load and validate dataset content
+    print("\n2. Validating dataset content:")
+    splits = ["train", "validation", "test"]
+    
+    for split in splits:
+        file_path = os.path.join(dataset_path, f"{split}.jsonl")
+        try:
+            with open(file_path, 'r') as f:
+                lines = f.readlines()
+                print(f"\n   {split} split:")
+                print(f"   - Examples: {len(lines)}")
+                
+                # Parse first example
+                first_example = json.loads(lines[0])
+                print(f"   - First example: {first_example}")
+                
+                # Validate structure
+                if "text" in first_example and "label" in first_example:
+                    print(f"   - Structure: ✓ Valid")
+                else:
+                    print(f"   - Structure: ✗ Invalid")
+        except Exception as e:
+            print(f"   Error reading {split}: {e}")
+    
+    # Check model integration code
+    print("\n3. Checking model integration:")
+    model_file = "model.py"
+    
+    if os.path.exists(model_file):
+        with open(model_file, 'r') as f:
+            content = f.read()
+            
+        # Check for dataset integration methods
+        if "load_dataset" in content:
+            print("   ✓ load_dataset method found in model.py")
+        else:
+            print("   ✗ load_dataset method not found")
+            
+        if "prepare_dataset_batch" in content:
+            print("   ✓ prepare_dataset_batch method found in model.py")
+        else:
+            print("   ✗ prepare_dataset_batch method not found")
+            
+        if "from datasets import load_dataset" in content:
+            print("   ✓ datasets import found in model.py")
+        else:
+            print("   ✗ datasets import not found")
+    
+    print("\n4. Dataset URLs:")
+    print(f"   Model: https://huggingface.co/chiedo/hello-world")
+    print(f"   Dataset: https://huggingface.co/datasets/chiedo/hello-world")
+    
+    print("\n" + "=" * 50)
+    print("Dataset integration setup complete!")
+    print("\nTo use the dataset with the model, install dependencies:")
+    print("  pip install torch transformers datasets")
+    print("\nThen run:")
+    print("  python example_with_dataset.py")
+
+if __name__ == "__main__":
+    test_dataset_files()

example_with_dataset.py

@@ -0,0 +1,88 @@
+"""
+Example script showing how to use the Hello World model with its dataset.
+"""
+
+from transformers import PreTrainedTokenizerFast
+from model import HelloWorldModel, HelloWorldConfig
+from datasets import load_dataset
+import torch
+
+
+def main():
+    print("Loading Hello World Model and Dataset Example\n")
+    print("=" * 50)
+    
+    # Load model and tokenizer
+    print("Loading model and tokenizer...")
+    config = HelloWorldConfig.from_pretrained("chiedo/hello-world")
+    model = HelloWorldModel.from_pretrained("chiedo/hello-world")
+    tokenizer = PreTrainedTokenizerFast.from_pretrained("chiedo/hello-world")
+    
+    # Method 1: Load dataset using the model's built-in method
+    print("\n1. Loading dataset using model's load_dataset method:")
+    dataset = HelloWorldModel.load_dataset("chiedo/hello-world")
+    
+    if dataset:
+        print(f"Dataset loaded successfully!")
+        print(f"Splits available: {list(dataset.keys())}")
+        print(f"Train examples: {len(dataset['train'])}")
+        print(f"Validation examples: {len(dataset['validation'])}")
+        print(f"Test examples: {len(dataset['test'])}")
+        
+        # Show first few examples
+        print("\nFirst 3 training examples:")
+        for i in range(min(3, len(dataset['train']))):
+            example = dataset['train'][i]
+            print(f"  {i+1}. Text: '{example['text']}', Label: {example['label']}")
+    
+    # Method 2: Load dataset directly
+    print("\n2. Loading dataset directly with datasets library:")
+    dataset_direct = load_dataset("chiedo/hello-world")
+    
+    # Get label names
+    label_names = dataset_direct['train'].features['label'].names
+    print(f"Label categories: {label_names}")
+    
+    # Process a batch from the dataset
+    print("\n3. Processing a batch from the dataset:")
+    batch_texts = dataset_direct['train']['text'][:3]
+    print(f"Batch texts: {batch_texts}")
+    
+    # Prepare batch for model
+    inputs = model.prepare_dataset_batch(batch_texts, tokenizer)
+    print(f"Tokenized input shape: {inputs['input_ids'].shape}")
+    
+    # Run model inference
+    print("\n4. Running model inference on dataset batch:")
+    with torch.no_grad():
+        outputs = model(**inputs)
+        print(f"Model output shape: {outputs.logits.shape}")
+    
+    # Demonstrate the generate_hello_world function
+    print("\n5. Testing generate_hello_world function:")
+    result = model.generate_hello_world()
+    print(f"Generated output: {result}")
+    
+    # Show how to iterate through dataset
+    print("\n6. Iterating through test set:")
+    for i, example in enumerate(dataset_direct['test']):
+        if i >= 3:  # Only show first 3
+            break
+        text = example['text']
+        label_id = example['label']
+        label_name = label_names[label_id]
+        
+        # Tokenize and process
+        inputs = tokenizer(text, return_tensors="pt")
+        with torch.no_grad():
+            outputs = model(**inputs)
+            predicted_token = outputs.logits[0, -1].argmax().item()
+        
+        print(f"  Text: '{text}' | Label: {label_name} | Predicted next token ID: {predicted_token}")
+    
+    print("\n" + "=" * 50)
+    print("Example completed successfully!")
+
+
+if __name__ == "__main__":
+    main()

model.py

@@ -2,6 +2,7 @@ import torch
 import torch.nn as nn
 from transformers import PreTrainedModel, PretrainedConfig
 from transformers.modeling_outputs import CausalLMOutputWithPast
+from datasets import load_dataset
 
 
 class HelloWorldConfig(PretrainedConfig):
@@ -124,4 +125,46 @@ class HelloWorldModel(PreTrainedModel):
         with torch.no_grad():
             outputs = self.forward(input_ids)
         
-        return "Hello World!"
+        return "Hello World!"
+    
+    @classmethod
+    def load_dataset(cls, dataset_name="chiedo/hello-world", split=None):
+        """
+        Load the Hello World dataset.
+        
+        Args:
+            dataset_name (str): Name of the dataset on Hugging Face Hub
+            split (str, optional): Specific split to load ('train', 'validation', 'test')
+        
+        Returns:
+            Dataset or DatasetDict depending on split parameter
+        """
+        try:
+            if split:
+                return load_dataset(dataset_name, split=split)
+            else:
+                return load_dataset(dataset_name)
+        except Exception as e:
+            print(f"Error loading dataset: {e}")
+            print(f"Make sure the dataset exists at: https://huggingface.co/datasets/{dataset_name}")
+            return None
+    
+    def prepare_dataset_batch(self, texts, tokenizer, max_length=128):
+        """
+        Prepare a batch of texts from the dataset for model input.
+        
+        Args:
+            texts (list): List of text strings
+            tokenizer: Tokenizer to encode the texts
+            max_length (int): Maximum sequence length
+        
+        Returns:
+            dict: Dictionary with input_ids and attention_mask tensors
+        """
+        return tokenizer(
+            texts,
+            padding=True,
+            truncation=True,
+            max_length=max_length,
+            return_tensors="pt"
+        )