Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+custom_llm_model.pth filter=lfs diff=lfs merge=lfs -text
+hf_model/tokenizer.pkl filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,42 @@
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+# Load model and tokenizer from Hugging Face Hub
+model_name = "itriedcoding/Sage"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+def generate_text(prompt, max_length, temperature):
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=int(max_length),
+            temperature=temperature,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id
+        )
+    
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+demo = gr.Interface(
+    fn=generate_text,
+    inputs=[
+        gr.Textbox(label="Prompt", value="Hello", placeholder="Enter your prompt here"),
+        gr.Slider(minimum=10, maximum=100, value=30, label="Max Length"),
+        gr.Slider(minimum=0.1, maximum=2.0, value=0.8, label="Temperature")
+    ],
+    outputs=gr.Textbox(label="Generated Text"),
+    title="🤖 Sage Text Generator",
+    description="Generate text using the Sage custom character-level language model.",
+    examples=[
+        ["Hello", 30, 0.8],
+        ["The weather", 30, 0.8],
+        ["Deep learning", 30, 0.8]
+    ]
+)
+
+if __name__ == "__main__":
+    demo.launch()

custom_llm_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:accd9d82bd55ee686643f9e889f53e3d9938197f30fea126df1b596090c70382
+size 12805265

custom_llm_project/README.md

@@ -0,0 +1,79 @@
+# Custom LLM Model
+
+This is a custom-built language model trained on a small dataset of example sentences about AI and machine learning.
+
+## Model Description
+
+- **Model Type**: Transformer-based language model
+- **Vocabulary Size**: 40 characters
+- **Hidden Size**: 256
+- **Number of Layers**: 4
+- **Number of Attention Heads**: 8
+- **Feedforward Size**: 1024
+- **Max Sequence Length**: 64
+- **Training Epochs**: 10
+- **Parameters**: ~3.2M
+
+## Training Data
+
+The model was trained on a small custom dataset containing 10 example sentences about:
+- Greetings and small talk
+- Weather descriptions  
+- Machine learning concepts
+- Deep learning and transformers
+- Natural language processing
+- Model publishing and sharing
+
+## Usage
+
+```python
+import torch
+from train_model import TransformerLM, CharacterTokenizer
+
+# Load the saved model
+checkpoint = torch.load('custom_llm_model.pth', map_location='cpu')
+model_config = checkpoint['model_config']
+tokenizer = checkpoint['tokenizer']
+
+# Initialize model
+model = TransformerLM(**model_config)
+model.load_state_dict(checkpoint['model_state_dict'])
+model.eval()
+
+# Generate text
+def generate_text(prompt, max_length=50, temperature=0.8):
+    # Tokenize prompt
+    input_ids = tokenizer.encode(prompt, max_length=32, padding=False, return_tensors='pt')
+    generated = input_ids.clone()
+    
+    with torch.no_grad():
+        for _ in range(max_length):
+            logits = model(generated)
+            next_token_logits = logits[0, -1, :] / temperature
+            probs = torch.softmax(next_token_logits, dim=-1)
+            next_token = torch.multinomial(probs, num_samples=1)
+            generated = torch.cat([generated, next_token.unsqueeze(0)], dim=1)
+            
+            # Stop on period or max length
+            if next_token.item() == tokenizer.char_to_idx.get('.', tokenizer.unk_token_id):
+                break
+    
+    return tokenizer.decode(generated[0])
+
+# Example usage
+print(generate_text("Hello"))
+print(generate_text("The weather"))
+print(generate_text("Deep learning"))
+```
+
+## Limitations
+
+This is a small demonstration model trained on very limited data. For serious applications, consider:
+- Using larger datasets
+- Training for more epochs
+- Using larger model architectures
+- Implementing proper tokenization (BPE, WordPiece, etc.)
+
+## License
+
+This model is released under the MIT License.

custom_llm_project/hf_model/README.md

@@ -0,0 +1,83 @@
+# Custom LLM Model
+
+A small custom-built transformer language model trained on example sentences about AI and machine learning.
+
+## Model Description
+
+This is a demonstration model built to showcase how to create and publish a custom AI model to Hugging Face. The model is a transformer-based language model with:
+
+- **Architecture**: Transformer decoder
+- **Vocabulary Size**: 40 characters
+- **Hidden Size**: 256
+- **Number of Layers**: 4
+- **Number of Attention Heads**: 8
+- **Feedforward Size**: 1024
+- **Max Sequence Length**: 64
+- **Parameters**: ~3.2M
+
+## Training Data
+
+The model was trained on a small custom dataset containing 10 example sentences about:
+- Greetings and small talk
+- Weather descriptions  
+- Machine learning concepts
+- Deep learning and transformers
+- Natural language processing
+- Model publishing and sharing
+
+## Usage
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+# Load model and tokenizer
+model_name = "your-username/custom-llm-model"  # Replace with your HF username
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+# Generate text
+def generate_text(prompt, max_length=50, temperature=0.8):
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=max_length,
+            temperature=temperature,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id
+        )
+    
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+# Example usage
+print(generate_text("Hello"))
+print(generate_text("The weather"))
+print(generate_text("Deep learning"))
+```
+
+## Limitations
+
+This is a small demonstration model trained on very limited data. For serious applications, consider:
+- Using larger datasets
+- Training for more epochs
+- Using larger model architectures
+- Implementing proper tokenization (BPE, WordPiece, etc.)
+
+## License
+
+This model is released under the MIT License.
+
+## Citation
+
+```
+@misc{custom_llm_model,
+  author = {Your Name},
+  title = {Custom LLM Model},
+  year = {2026},
+  publisher = {Hugging Face},
+  journal = {Hugging Face Model Hub},
+  doi = {10.57967/hf/0000}
+}
+```

custom_llm_project/hf_model/__init__.py

@@ -0,0 +1,3 @@
+from .modeling_transformer_lm import TransformerLM, TransformerLMConfig
+
+__all__ = ["TransformerLM", "TransformerLMConfig"]

custom_llm_project/hf_model/config.json

@@ -0,0 +1,14 @@
+{
+  "architectures": ["TransformerLM"],
+  "model_type": "transformer_lm",
+  "vocab_size": 40,
+  "hidden_size": 256,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 64,
+  "pad_token_id": 0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "torch_dtype": "float32"
+}

custom_llm_project/hf_model/modeling_transformer_lm.py

@@ -0,0 +1,109 @@
+import torch
+import torch.nn as nn
+import math
+from transformers import PreTrainedModel
+from transformers.modeling_utils import PretrainedConfig
+
+class TransformerLMConfig(PretrainedConfig):
+    model_type = "transformer_lm"
+    
+    def __init__(
+        self,
+        vocab_size=40,
+        hidden_size=256,
+        num_hidden_layers=4,
+        num_attention_heads=8,
+        intermediate_size=1024,
+        max_position_embeddings=64,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        **kwargs
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs
+        )
+        
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+
+class TransformerLM(PreTrainedModel):
+    config_class = TransformerLMConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        
+        self.embedding = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.pos_embedding = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+        
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=config.hidden_size,
+            nhead=config.num_attention_heads,
+            dim_feedforward=config.intermediate_size,
+            batch_first=True
+        )
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=config.num_hidden_layers)
+        self.output_layer = nn.Linear(config.hidden_size, config.vocab_size)
+        
+        self.max_position_embeddings = config.max_position_embeddings
+        
+    def forward(self, input_ids, attention_mask=None, labels=None):
+        seq_len = input_ids.size(1)
+        pos = torch.arange(0, seq_len, device=input_ids.device).unsqueeze(0)
+        
+        # Embedding + positional encoding
+        src_emb = self.embedding(input_ids) * math.sqrt(self.config.hidden_size)
+        pos_emb = self.pos_embedding(pos)
+        src_emb = src_emb + pos_emb
+        
+        # Create key padding mask for transformer (True where we should mask)
+        if attention_mask is not None:
+            # Transformer expects True for positions to mask
+            src_key_padding_mask = ~attention_mask.bool()
+        else:
+            src_key_padding_mask = None
+        
+        # Transformer encoder
+        output = self.transformer_encoder(src_emb, src_key_padding_mask=src_key_padding_mask)
+        
+        # Output projection
+        logits = self.output_layer(output)
+        
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+        
+        return {
+            "loss": loss,
+            "logits": logits
+        }
+        
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # Only last token for inputs_ids if past is defined in kwargs
+        if "past_key_values" in kwargs:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+        
+        attention_mask = kwargs.get("attention_mask", None)
+        position_ids = kwargs.get("position_ids", None)
+        
+        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
+        if attention_mask is not None:
+            attention_mask = attention_mask
+        
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "position_ids": position_ids,
+        }

custom_llm_project/sage_model/README.md

@@ -0,0 +1,222 @@
+# Sage - Custom LLM Model
+
+Sage is a custom-built transformer language model designed for text generation tasks. This model demonstrates the full lifecycle of building and publishing a custom AI model to Hugging Face.
+
+## Model Overview
+
+- **Model Type**: Transformer-based language model
+- **Architecture**: Decoder-only transformer
+- **Vocabulary Size**: 40 characters
+- **Hidden Size**: 256
+- **Number of Layers**: 4
+- **Number of Attention Heads**: 8
+- **Feedforward Size**: 1024
+- **Max Sequence Length**: 64
+- **Parameters**: ~3,195,944
+- **Training Framework**: PyTorch
+- **License**: MIT
+
+## Training Data
+
+Sage was trained on a curated dataset of example sentences covering:
+- Conversational phrases and greetings
+- Weather and environmental descriptions
+- Machine learning and AI concepts
+- Deep learning architectures (transformers, neural networks)
+- Natural language processing applications
+- Model development and deployment practices
+
+The dataset consists of 10 examples designed to teach the model patterns in technical and conversational English.
+
+## Technical Specifications
+
+### Model Architecture
+```
+TransformerLM(
+  (embedding): Embedding(40, 256)
+  (pos_embedding): Embedding(64, 256)
+  (transformer_encoder): TransformerEncoder(
+    (layers): ModuleList(
+      (0-3): TransformerEncoderLayer(
+        (self_attn): MultiheadAttention(embed_dim=256, num_heads=8)
+        (linear1): Linear(256, 1024)
+        (linear2): Linear(1024, 256)
+        (norm1): LayerNorm(256)
+        (norm2): LayerNorm(256)
+        (dropout): Dropout(p=0.1)
+      )
+    )
+  )
+  (output_layer): Linear(256, 40)
+)
+```
+
+### Tokenization
+Sage uses a character-level tokenizer with:
+- Vocabulary: 40 unique characters including special tokens
+- Special tokens: `<PAD>` (0), `<UNK>` (1)
+- Encoding: UTF-8 character mapping
+- Maximum sequence length: 64 tokens
+
+## Usage
+
+### With Transformers Library
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+model_name = "itriedcoding/Sage"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+def generate_text(prompt, max_length=50, temperature=0.8):
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=max_length,
+            temperature=temperature,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id
+        )
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+print(generate_text("Hello"))
+print(generate_text("Deep learning"))
+```
+
+### Direct PyTorch Usage
+```python
+import torch
+from modeling_transformer_lm import TransformerLM
+
+model = TransformerLM.from_pretrained("itriedcoding/Sage")
+```
+
+## Model Card Metadata
+
+```
+library_name: transformers
+license: MIT
+base_model: custom-built
+tags:
+- text-generation
+- transformer
+- character-level
+- custom-model
+- educational
+pipeline_tag: text-generation
+```
+
+## Hugging Face Spaces Deployment
+
+You can run Sage in the dedicated Hugging Face Space:
+https://huggingface.co/spaces/itriedcoding/sage-space
+
+### Gradio Space
+The Space at `itriedcoding/sage-space` provides a Gradio interface for text generation.
+Create a new Space with `app.py`:
+```python
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
+
+model_name = "itriedcoding/Sage"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
+def generate_text(prompt, max_length, temperature):
+    inputs = tokenizer.encode(prompt, return_tensors="pt")
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            max_length=int(max_length),
+            temperature=temperature,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id
+        )
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+demo = gr.Interface(
+    fn=generate_text,
+    inputs=[
+        gr.Textbox(label="Prompt", value="Hello"),
+        gr.Slider(minimum=10, maximum=100, value=30, label="Max Length"),
+        gr.Slider(minimum=0.1, maximum=2.0, value=0.8, label="Temperature")
+    ],
+    outputs=gr.Textbox(label="Generated Text"),
+    title="Sage Text Generator",
+    description="Custom character-level language model"
+)
+
+if __name__ == "__main__":
+    demo.launch()
+```
+
+## GGUF Quantization
+
+WARNING: Sage uses a custom transformer architecture (not Llama-family). Standard GGUF conversion tools (llama.cpp) do not support this architecture. To generate GGUF files, you must write a custom conversion script that maps Sage's layers to GGUF format. A conversion script template is provided below.
+
+### GGUF Conversion Script Template
+```python
+import torch
+import struct
+
+def convert_to_gguf(model_path, output_path):
+    checkpoint = torch.load(model_path, map_location='cpu', weights_only=False)
+    state_dict = checkpoint['model_state_dict']
+    # Custom conversion logic to write GGUF format
+    # Standard GGUF tools like llama.cpp's convert.py will NOT work
+    # You need to implement the GGUF tensor serialization manually
+    print(f"Converting {len(state_dict)} tensors to GGUF format...")
+    # Implementation depends on the target runtime
+
+convert_to_gguf("pytorch_model.bin", "sage.gguf")
+```
+
+### Recommended Alternative Formats
+- **PyTorch**: Full precision (pytorch_model.bin) - already provided
+- **TorchScript**: torch.jit.trace for optimized CPU/GPU inference
+- **ONNX**: Use torch.onnx.export for cross-platform deployment
+
+## Performance & Limitations
+
+### Intended Use
+- Educational demonstrations of transformer architectures
+- Character-level language modeling experiments
+- Prototyping and testing custom model pipelines
+- Learning about model deployment on Hugging Face
+
+### Limitations
+- Character-level tokenization limits coherence
+- Small training dataset (10 examples)
+- Small model size (3.2M parameters)
+- Not suitable for production NLP applications
+- Best for short text generation (<50 tokens)
+
+## Citation
+
+```bibtex
+@misc{sage_model_2026,
+  author = {itriedcoding},
+  title = {Sage: Custom Character-Level Language Model},
+  year = {2026},
+  publisher = {Hugging Face},
+  journal = {Hugging Face Model Hub},
+  url = {https://huggingface.co/itriedcoding/Sage}
+}
+```
+
+## Training Reproducibility
+
+To reproduce this model:
+1. Clone the repository
+2. Install requirements: `pip install torch pandas`
+3. Run training: The model was trained using the script in `train_model.py`
+4. The trained checkpoint is saved as a PyTorch .pth file
+
+## Contact
+
+- Hugging Face: https://huggingface.co/itriedcoding
+- Model Space: https://huggingface.co/spaces/itriedcoding/sage-space
+- Issues: Use the "Issues" tab on this model page

custom_llm_project/sage_model/__init__.py

@@ -0,0 +1,3 @@
+from .modeling_transformer_lm import TransformerLM, TransformerLMConfig
+
+__all__ = ["TransformerLM", "TransformerLMConfig"]

custom_llm_project/sage_model/config.json

@@ -0,0 +1,14 @@
+{
+  "architectures": ["TransformerLM"],
+  "model_type": "transformer_lm",
+  "vocab_size": 40,
+  "hidden_size": 256,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 64,
+  "pad_token_id": 0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "torch_dtype": "float32"
+}

custom_llm_project/sage_model/modeling_transformer_lm.py

@@ -0,0 +1,109 @@
+import torch
+import torch.nn as nn
+import math
+from transformers import PreTrainedModel
+from transformers.modeling_utils import PretrainedConfig
+
+class TransformerLMConfig(PretrainedConfig):
+    model_type = "transformer_lm"
+    
+    def __init__(
+        self,
+        vocab_size=40,
+        hidden_size=256,
+        num_hidden_layers=4,
+        num_attention_heads=8,
+        intermediate_size=1024,
+        max_position_embeddings=64,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        **kwargs
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs
+        )
+        
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.max_position_embeddings = max_position_embeddings
+
+class TransformerLM(PreTrainedModel):
+    config_class = TransformerLMConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        
+        self.embedding = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.pos_embedding = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+        
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=config.hidden_size,
+            nhead=config.num_attention_heads,
+            dim_feedforward=config.intermediate_size,
+            batch_first=True
+        )
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=config.num_hidden_layers)
+        self.output_layer = nn.Linear(config.hidden_size, config.vocab_size)
+        
+        self.max_position_embeddings = config.max_position_embeddings
+        
+    def forward(self, input_ids, attention_mask=None, labels=None):
+        seq_len = input_ids.size(1)
+        pos = torch.arange(0, seq_len, device=input_ids.device).unsqueeze(0)
+        
+        # Embedding + positional encoding
+        src_emb = self.embedding(input_ids) * math.sqrt(self.config.hidden_size)
+        pos_emb = self.pos_embedding(pos)
+        src_emb = src_emb + pos_emb
+        
+        # Create key padding mask for transformer (True where we should mask)
+        if attention_mask is not None:
+            # Transformer expects True for positions to mask
+            src_key_padding_mask = ~attention_mask.bool()
+        else:
+            src_key_padding_mask = None
+        
+        # Transformer encoder
+        output = self.transformer_encoder(src_emb, src_key_padding_mask=src_key_padding_mask)
+        
+        # Output projection
+        logits = self.output_layer(output)
+        
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+        
+        return {
+            "loss": loss,
+            "logits": logits
+        }
+        
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # Only last token for inputs_ids if past is defined in kwargs
+        if "past_key_values" in kwargs:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+        
+        attention_mask = kwargs.get("attention_mask", None)
+        position_ids = kwargs.get("position_ids", None)
+        
+        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
+        if attention_mask is not None:
+            attention_mask = attention_mask
+        
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "position_ids": position_ids,
+        }

custom_llm_project/sage_model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:accd9d82bd55ee686643f9e889f53e3d9938197f30fea126df1b596090c70382
+size 12805265

custom_llm_project/train_model.py

@@ -0,0 +1,235 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+import json
+import pandas as pd
+from torch.utils.data import Dataset, DataLoader
+import math
+
+# Set device
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+print(f"Using device: {device}")
+
+# Custom Dataset Class
+class TextDataset(Dataset):
+    def __init__(self, texts, tokenizer, max_length=128):
+        self.texts = texts
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+    
+    def __len__(self):
+        return len(self.texts)
+    
+    def __getitem__(self, idx):
+        text = str(self.texts[idx])
+        tokens = self.tokenizer.encode(text, max_length=self.max_length, 
+                                     padding='max_length', truncation=True, return_tensors='pt')
+        input_ids = tokens.squeeze(0)
+        # For language modeling, target is input shifted by 1
+        target_ids = torch.cat([input_ids[1:], torch.tensor([self.tokenizer.pad_token_id])], dim=0)
+        return input_ids, target_ids
+
+# Simple Character-level Tokenizer
+class CharacterTokenizer:
+    def __init__(self):
+        self.char_to_idx = {}
+        self.idx_to_char = {}
+        self.vocab_size = 0
+        self.pad_token_id = 0
+        self.unk_token_id = 1
+        
+    def fit(self, texts):
+        # Build vocabulary from characters
+        chars = set()
+        for text in texts:
+            chars.update(list(str(text)))
+        
+        # Add special tokens
+        self.char_to_idx['<PAD>'] = 0
+        self.char_to_idx['<UNK>'] = 1
+        
+        # Add regular characters
+        for i, char in enumerate(sorted(chars)):
+            self.char_to_idx[char] = i + 2
+            
+        # Create reverse mapping
+        self.idx_to_char = {v: k for k, v in self.char_to_idx.items()}
+        self.vocab_size = len(self.char_to_idx)
+        
+    def encode(self, text, max_length=None, padding=False, truncation=False, return_tensors=None):
+        if isinstance(text, str):
+            text = [text]
+            
+        encoded = []
+        for t in text:
+            tokens = [self.char_to_idx.get(c, self.unk_token_id) for c in str(t)]
+            if truncation and max_length:
+                tokens = tokens[:max_length]
+            if padding and max_length:
+                tokens = tokens + [self.pad_token_id] * (max_length - len(tokens))
+            encoded.append(tokens)
+            
+        if return_tensors == 'pt':
+            return torch.tensor(encoded, dtype=torch.long)
+        return encoded
+    
+    def decode(self, token_ids):
+        if isinstance(token_ids, torch.Tensor):
+            token_ids = token_ids.tolist()
+        chars = [self.idx_to_char.get(idx, '<UNK>') for idx in token_ids]
+        return ''.join(chars)
+
+# Transformer Language Model
+class TransformerLM(nn.Module):
+    def __init__(self, vocab_size, d_model=256, nhead=8, num_layers=4, dim_feedforward=1024, max_seq_length=128):
+        super(TransformerLM, self).__init__()
+        self.d_model = d_model
+        self.embedding = nn.Embedding(vocab_size, d_model)
+        self.pos_embedding = nn.Embedding(max_seq_length, d_model)
+        
+        encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, 
+                                                 dim_feedforward=dim_feedforward, batch_first=True)
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
+        self.output_layer = nn.Linear(d_model, vocab_size)
+        
+        self.max_seq_length = max_seq_length
+        
+    def forward(self, src):
+        seq_len = src.size(1)
+        pos = torch.arange(0, seq_len, device=src.device).unsqueeze(0)
+        
+        # Embedding + positional encoding
+        src_emb = self.embedding(src) * math.sqrt(self.d_model)
+        pos_emb = self.pos_embedding(pos)
+        src_emb = src_emb + pos_emb
+        
+        # Create mask for padding (optional)
+        # src_key_padding_mask = (src == 0)  # Assuming 0 is pad token
+        
+        # Transformer encoder
+        output = self.transformer_encoder(src_emb)  # , src_key_padding_mask=src_key_padding_mask)
+        
+        # Output projection
+        logits = self.output_layer(output)
+        return logits
+
+# Load dataset
+print("Loading dataset...")
+df = pd.read_csv('data/dataset.csv')
+texts = df['text'].tolist()
+print(f"Loaded {len(texts)} text samples")
+
+# Initialize tokenizer
+tokenizer = CharacterTokenizer()
+tokenizer.fit(texts)
+print(f"Vocabulary size: {tokenizer.vocab_size}")
+
+# Create dataset and dataloader
+dataset = TextDataset(texts, tokenizer, max_length=64)
+dataloader = DataLoader(dataset, batch_size=4, shuffle=True)
+
+# Initialize model
+model = TransformerLM(
+    vocab_size=tokenizer.vocab_size,
+    d_model=256,
+    nhead=8,
+    num_layers=4,
+    dim_feedforward=1024,
+    max_seq_length=64
+).to(device)
+
+print(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
+
+# Loss and optimizer
+criterion = nn.CrossEntropyLoss(ignore_index=tokenizer.pad_token_id)
+optimizer = optim.AdamW(model.parameters(), lr=0.001)
+
+# Training loop
+num_epochs = 10
+model.train()
+
+print("Starting training...")
+for epoch in range(num_epochs):
+    total_loss = 0
+    num_batches = 0
+    
+    for batch_idx, (input_ids, target_ids) in enumerate(dataloader):
+        input_ids = input_ids.to(device)
+        target_ids = target_ids.to(device)
+        
+        # Forward pass
+        optimizer.zero_grad()
+        logits = model(input_ids)
+        
+        # Reshape for loss calculation: (batch_size * seq_len, vocab_size)
+        loss = criterion(logits.view(-1, logits.size(-1)), target_ids.view(-1))
+        
+        # Backward pass
+        loss.backward()
+        optimizer.step()
+        
+        total_loss += loss.item()
+        num_batches += 1
+        
+        if batch_idx % 10 == 0:
+            print(f'Epoch [{epoch+1}/{num_epochs}], Batch [{batch_idx}/{len(dataloader)}], Loss: {loss.item():.4f}')
+    
+    avg_loss = total_loss / num_batches
+    print(f'Epoch [{epoch+1}/{num_epochs}] Completed - Average Loss: {avg_loss:.4f}')
+
+# Save model and tokenizer
+print("Saving model and tokenizer...")
+torch.save({
+    'model_state_dict': model.state_dict(),
+    'tokenizer': tokenizer,
+    'model_config': {
+        'vocab_size': tokenizer.vocab_size,
+        'd_model': 256,
+        'nhead': 8,
+        'num_layers': 4,
+        'dim_feedforward': 1024,
+        'max_seq_length': 64
+    }
+}, 'custom_llm_model.pth')
+
+print("Training completed! Model saved as 'custom_llm_model.pth'")
+
+# Test generation
+def generate_text(model, tokenizer, prompt, max_length=50, temperature=0.8):
+    model.eval()
+    with torch.no_grad():
+        # Tokenize prompt
+        input_ids = tokenizer.encode(prompt, max_length=32, padding=False, return_tensors='pt')
+        input_ids = input_ids.to(device)
+        
+        generated = input_ids.clone()
+        
+        for _ in range(max_length):
+            # Get model predictions
+            logits = model(generated)
+            next_token_logits = logits[0, -1, :] / temperature
+            
+            # Apply softmax to get probabilities
+            probs = torch.softmax(next_token_logits, dim=-1)
+            
+            # Sample next token
+            next_token = torch.multinomial(probs, num_samples=1)
+            
+            # Append to generated sequence
+            generated = torch.cat([generated, next_token.unsqueeze(0)], dim=1)
+            
+            # Stop if we generate a period or reach reasonable length
+            if next_token.item() == tokenizer.char_to_idx.get('.', tokenizer.unk_token_id):
+                break
+                
+        # Decode generated text
+        generated_text = tokenizer.decode(generated[0])
+        return generated_text
+
+# Test the model
+print("\nTesting generation:")
+test_prompts = ["Hello", "The weather", "Deep learning"]
+for prompt in test_prompts:
+    generated = generate_text(model, tokenizer, prompt, max_length=30)
+    print(f"Prompt: '{prompt}' -> Generated: '{generated}'")

data/dataset.csv

@@ -0,0 +1,11 @@
+text
+"Hello, how are you today?"
+"The weather is nice and sunny."
+"I enjoy reading books about machine learning."
+"Deep learning has revolutionized artificial intelligence."
+"Natural language processing enables computers to understand text."
+"Transformers are a type of neural network architecture."
+"Training large language models requires significant computational resources."
+"Hugging Face provides a platform for sharing machine learning models."
+"I am excited to build my own custom AI model."
+"Publishing models to the hub allows others to use them."

hf_model/config.json

@@ -0,0 +1,16 @@
+{
+  "architectures": [
+    "TransformerLM"
+  ],
+  "model_type": "transformer_lm",
+  "vocab_size": 40,
+  "hidden_size": 256,
+  "num_hidden_layers": 4,
+  "num_attention_heads": 8,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 64,
+  "pad_token_id": 0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "torch_dtype": "float32"
+}

hf_model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972f246378970d2307ada18d34ee1834890e8241ffca6aec7c899eb4bc658426
+size 12803875

hf_model/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e5006ee164f745f1ed22e8fe7cff1c0139ed66a02f05b52037e906eab257fd
+size 509