Upload folder using huggingface_hub

.gitattributes

@@ -1,3 +1,4 @@
 *.bin filter=lfs diff=lfs merge=lfs -text
 custom_llm_model.pth filter=lfs diff=lfs merge=lfs -text
 hf_model/tokenizer.pkl filter=lfs diff=lfs merge=lfs -text
+sage-f16.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -2,7 +2,7 @@
 
 Sage is a custom-built transformer language model designed for text generation tasks. This model demonstrates the full lifecycle of building and publishing a custom AI model to Hugging Face.
 
-## 📊 Model Overview
+## Model Overview
 
 - **Model Type**: Transformer-based language model
 - **Architecture**: Decoder-only transformer
@@ -12,11 +12,11 @@ Sage is a custom-built transformer language model designed for text generation t
 - **Number of Attention Heads**: 8
 - **Feedforward Size**: 1024
 - **Max Sequence Length**: 64
-- **Parameters**: ~3.2M
+- **Parameters**: ~3,195,944
 - **Training Framework**: PyTorch
 - **License**: MIT
 
-## 📚 Training Data
+## Training Data
 
 Sage was trained on a curated dataset of example sentences covering:
 - Conversational phrases and greetings
@@ -26,9 +26,9 @@ Sage was trained on a curated dataset of example sentences covering:
 - Natural language processing applications
 - Model development and deployment practices
 
-The dataset consists of 10 carefully crafted examples designed to teach the model patterns in technical and conversational English.
+The dataset consists of 10 examples designed to teach the model patterns in technical and conversational English.
 
-## 🔧 Technical Specifications
+## Technical Specifications
 
 ### Model Architecture
 ```
@@ -38,21 +38,16 @@ TransformerLM(
   (transformer_encoder): TransformerEncoder(
     (layers): ModuleList(
       (0-3): TransformerEncoderLayer(
-        (self_attn): MultiheadAttention(
-          (embed_dim): 256
-          (num_heads): 8
-        )
-        (linear1): Linear(in_features=256, out_features=1024, bias=True)
-        (linear2): Linear(in_features=1024, out_features=256, bias=True)
-        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
-        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
-        (dropout): Dropout(p=0.1, inplace=False)
-        (dropout1): Dropout(p=0.1, inplace=False)
-        (dropout2): Dropout(p=0.1, inplace=False)
+        (self_attn): MultiheadAttention(embed_dim=256, num_heads=8)
+        (linear1): Linear(256, 1024)
+        (linear2): Linear(1024, 256)
+        (norm1): LayerNorm(256)
+        (norm2): LayerNorm(256)
+        (dropout): Dropout(p=0.1)
       )
     )
   )
-  (output_layer): Linear(in_features=256, out_features=40, bias=True)
+  (output_layer): Linear(256, 40)
 )
 ```
 
@@ -63,22 +58,19 @@ Sage uses a character-level tokenizer with:
 - Encoding: UTF-8 character mapping
 - Maximum sequence length: 64 tokens
 
-## 🚀 Usage
+## Usage
 
 ### With Transformers Library
 ```python
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
 
-# Load model and tokenizer
 model_name = "itriedcoding/Sage"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 model = AutoModelForCausalLM.from_pretrained(model_name)
 
-# Generate text
 def generate_text(prompt, max_length=50, temperature=0.8):
     inputs = tokenizer.encode(prompt, return_tensors="pt")
-    
     with torch.no_grad():
         outputs = model.generate(
             inputs,
@@ -87,12 +79,9 @@ def generate_text(prompt, max_length=50, temperature=0.8):
             do_sample=True,
             pad_token_id=tokenizer.eos_token_id
         )
-    
     return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
-# Examples
 print(generate_text("Hello"))
-print(generate_text("The weather"))
 print(generate_text("Deep learning"))
 ```
 
@@ -100,22 +89,13 @@ print(generate_text("Deep learning"))
 ```python
 import torch
 from modeling_transformer_lm import TransformerLM
-import json
-import pickle
-
-# Load model components
-with open('config.json', 'r') as f:
-    config_dict = json.load(f)
 
-# For actual usage, you would load the tokenizer similarly
-# This example shows the structure
 model = TransformerLM.from_pretrained("itriedcoding/Sage")
 ```
 
-## 🏗️ Model Card Metadata
+## Model Card Metadata
 
-```yaml
----
+```
 library_name: transformers
 license: MIT
 base_model: custom-built
@@ -126,80 +106,27 @@ tags:
 - custom-model
 - educational
 pipeline_tag: text-generation
-widget:
-- example: Hello
-  parameters: {max_length: 30, temperature: 0.7}
-- example: The weather
-  parameters: {max_length: 30, temperature: 0.7}
-- example: Deep learning
-  parameters: {max_length: 30, temperature: 0.7}
----
 ```
 
-## 🤗 Hugging Face Spaces Deployment
-
-You can run this model in various Hugging Face Spaces templates:
-
-### Streamlit Space
-Create a `streamlit_app.py`:
-```python
-import streamlit as st
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
+## Hugging Face Spaces Deployment
 
-@st.cache_resource
-def load_model():
-    model_name = "itriedcoding/Sage"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(model_name)
-    return tokenizer, model
-
-def main():
-    st.title("🤖 Sage Text Generator")
-    st.write("A custom character-level language model")
-    
-    tokenizer, model = load_model()
-    
-    prompt = st.text_input("Enter your prompt:", "Hello")
-    max_length = st.slider("Max length:", 10, 100, 30)
-    temperature = st.slider("Temperature:", 0.1, 2.0, 0.8)
-    
-    if st.button("Generate"):
-        with st.spinner("Generating..."):
-            inputs = tokenizer.encode(prompt, return_tensors="pt")
-            with torch.no_grad():
-                outputs = model.generate(
-                    inputs,
-                    max_length=max_length,
-                    temperature=temperature,
-                    do_sample=True,
-                    pad_token_id=tokenizer.eos_token_id
-                )
-            result = tokenizer.decode(outputs[0], skip_special_tokens=True)
-            st.write("**Generated text:**")
-            st.write(result)
-
-if __name__ == "__main__":
-    main()
-```
+You can run Sage in the dedicated Hugging Face Space:
+https://huggingface.co/spaces/itriedcoding/sage-space
 
 ### Gradio Space
-Create an `app.py`:
+The Space at `itriedcoding/sage-space` provides a Gradio interface for text generation.
+Create a new Space with `app.py`:
 ```python
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
 
-def load_model():
-    model_name = "itriedcoding/Sage"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(model_name)
-    return tokenizer, model
+model_name = "itriedcoding/Sage"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
 
 def generate_text(prompt, max_length, temperature):
-    tokenizer, model = load_model()
     inputs = tokenizer.encode(prompt, return_tensors="pt")
-    
     with torch.no_grad():
         outputs = model.generate(
             inputs,
@@ -208,7 +135,6 @@ def generate_text(prompt, max_length, temperature):
             do_sample=True,
             pad_token_id=tokenizer.eos_token_id
         )
-    
     return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
 demo = gr.Interface(
@@ -219,36 +145,47 @@ demo = gr.Interface(
         gr.Slider(minimum=0.1, maximum=2.0, value=0.8, label="Temperature")
     ],
     outputs=gr.Textbox(label="Generated Text"),
-    title="🤖 Sage Text Generator",
-    description="Custom character-level language model for text generation"
+    title="Sage Text Generator",
+    description="Custom character-level language model"
 )
 
 if __name__ == "__main__":
     demo.launch()
 ```
 
-## 📦 GGUF Quantization
+## GGUF Format
+
+Sage is available in GGUF format as `sage-f16.gguf`.
 
-For efficient deployment, Sage is available in GGUF format:
+### Compatibility Warning
+Sage uses a custom `transformer_lm` architecture that is NOT supported by standard llama.cpp or llama-cpp-python. The GGUF file is provided as a reference format and for custom inference implementations that can match Sage's architecture.
 
-### Available Quantizations
-- `sage-q4_0.gguf` - 4-bit quantization (balanced quality/size)
-- `sage-q5_0.gguf` - 5-bit quantization (higher quality)
-- `sage-q8_0.gguf` - 8-bit quantization (near-full precision)
-- `sage-f16.gguf` - Float16 (full precision)
+### File Details
+- **File**: `sage-f16.gguf` (12.7 MB)
+- **Format**: GGUF (GGML Universal Format)
+- **Precision**: Float16
+- **Tensors**: 52 layers
+- **Architecture**: `transformer_lm` (custom)
 
-### Using GGUF with llama.cpp
-```bash
-# Install llama.cpp
-git clone https://github.com/ggerganov/llama.cpp
-cd llama.cpp
-make
+### Using with Custom Inference
+To use this GGUF file, you need a GGUF loader that supports Sage's custom architecture:
+```python
+import gguf
+import torch
+import numpy as np
+
+# Load GGUF file
+reader = gguf.GGUFReader("sage-f16.gguf")
+tensors = {t.name: torch.from_numpy(t.data) for t in reader.tensors}
 
-# Run the model
-./main -m sage-q4_0.gguf -p "Hello" -n 30
+# Map tensor names back to Sage architecture
+# See gguf_convert.py for the tensor name mapping
 ```
 
-## 📈 Performance & Limitations
+### GGUF Conversion
+The conversion script `gguf_convert.py` is included in this repository. It uses the `gguf` Python library to convert the PyTorch checkpoint to GGUF format.
+
+## Performance & Limitations
 
 ### Intended Use
 - Educational demonstrations of transformer architectures
@@ -257,20 +194,13 @@ make
 - Learning about model deployment on Hugging Face
 
 ### Limitations
-- Small vocabulary (character-level only limits coherence)
-- Limited training data (10 examples)
+- Character-level tokenization limits coherence
+- Small training dataset (10 examples)
 - Small model size (3.2M parameters)
 - Not suitable for production NLP applications
 - Best for short text generation (<50 tokens)
 
-### Bias & Ethics
-As a small educational model trained on curated technical text:
-- Minimal harmful bias expected
-- Should not be used for decision-making applications
-- Outputs should be reviewed for appropriateness
-- Model reflects patterns in its limited training data
-
-## 📝 Citation
+## Citation
 
 ```bibtex
 @misc{sage_model_2026,
@@ -279,21 +209,20 @@ As a small educational model trained on curated technical text:
   year = {2026},
   publisher = {Hugging Face},
   journal = {Hugging Face Model Hub},
-  doi = {10.57967/hf/0000},
   url = {https://huggingface.co/itriedcoding/Sage}
 }
 ```
 
-## 🔄 Training Reproducibility
+## Training Reproducibility
 
 To reproduce this model:
-1. Clone this repository
-2. Install requirements: `pip install torch torchvision torchaudio pandas`
-3. Run training: `python train_model.py`
-4. The model will be saved as `custom_llm_model.pth`
+1. Clone the repository
+2. Install requirements: `pip install torch pandas`
+3. Run training: The model was trained using the script in `train_model.py`
+4. The trained checkpoint is saved as a PyTorch .pth file
 
-## 📞 Contact
+## Contact
 
-For questions or collaboration opportunities:
 - Hugging Face: https://huggingface.co/itriedcoding
-- Model Issues: Use the "Issues" tab on this model page
+- Model Space: https://huggingface.co/spaces/itriedcoding/sage-space
+- Issues: Use the "Issues" tab on this model page

gguf_convert.py

@@ -0,0 +1,112 @@
+import torch
+import gguf
+import numpy as np
+import os
+import sys
+import pickle
+
+# Character tokenizer class for loading the checkpoint
+class CharacterTokenizer:
+    def __init__(self):
+        self.char_to_idx = {}
+        self.idx_to_char = {}
+        self.vocab_size = 0
+        self.pad_token_id = 0
+        self.unk_token_id = 1
+    def fit(self, texts):
+        chars = set()
+        for text in texts:
+            chars.update(list(str(text)))
+        self.char_to_idx['<PAD>'] = 0
+        self.char_to_idx['<UNK>'] = 1
+        for i, char in enumerate(sorted(chars)):
+            self.char_to_idx[char] = i + 2
+        self.idx_to_char = {v: k for k, v in self.char_to_idx.items()}
+        self.vocab_size = len(self.char_to_idx)
+    def encode(self, text, max_length=None, padding=False, truncation=False, return_tensors=None):
+        if isinstance(text, str):
+            text = [text]
+        encoded = []
+        for t in text:
+            tokens = [self.char_to_idx.get(c, self.unk_token_id) for c in str(t)]
+            if truncation and max_length:
+                tokens = tokens[:max_length]
+            if padding and max_length:
+                tokens = tokens + [self.pad_token_id] * (max_length - len(tokens))
+            encoded.append(tokens)
+        if return_tensors == 'pt':
+            return torch.tensor(encoded, dtype=torch.long)
+        return encoded
+    def decode(self, token_ids):
+        if isinstance(token_ids, torch.Tensor):
+            token_ids = token_ids.tolist()
+        chars = [self.idx_to_char.get(idx, '<UNK>') for idx in token_ids]
+        return ''.join(chars)
+
+def convert_sage_to_gguf(model_path, output_path):
+    checkpoint = torch.load(model_path, map_location='cpu', weights_only=False)
+    state_dict = checkpoint['model_state_dict']
+    
+    gguf_writer = gguf.GGUFWriter(output_path, "transformer_lm")
+    
+    # Add metadata
+    gguf_writer.add_context_length(64)
+    gguf_writer.add_embedding_length(256)
+    gguf_writer.add_block_count(4)
+    gguf_writer.add_feed_forward_length(1024)
+    gguf_writer.add_head_count(8)
+    gguf_writer.add_head_count_kv(8)
+    gguf_writer.add_vocab_size(checkpoint['model_config']['vocab_size'])
+    gguf_writer.add_layer_norm_rms_eps(1e-5)
+    gguf_writer.add_name("Sage")
+    gguf_writer.add_license("MIT")
+    
+    # Map Sage's tensor names to GGUF format
+    tensor_map = {}
+    
+    # Embedding layers
+    tensor_map['embedding.weight'] = 'token_embd.weight'
+    tensor_map['pos_embedding.weight'] = 'position_embd.weight'
+    tensor_map['output_layer.weight'] = 'output.weight'
+    tensor_map['output_layer.bias'] = 'output.bias'
+    
+    # Per-layer mappings
+    for i in range(4):
+        p = f'transformer_encoder.layers.{i}'
+        tensor_map[f'{p}.self_attn.in_proj_weight'] = f'blk.{i}.attn_q.weight'
+        tensor_map[f'{p}.self_attn.in_proj_bias'] = f'blk.{i}.attn_q.bias'
+        tensor_map[f'{p}.self_attn.out_proj.weight'] = f'blk.{i}.attn_output.weight'
+        tensor_map[f'{p}.self_attn.out_proj.bias'] = f'blk.{i}.attn_output.bias'
+        tensor_map[f'{p}.linear1.weight'] = f'blk.{i}.ffn_gate.weight'
+        tensor_map[f'{p}.linear1.bias'] = f'blk.{i}.ffn_gate.bias'
+        tensor_map[f'{p}.linear2.weight'] = f'blk.{i}.ffn_down.weight'
+        tensor_map[f'{p}.linear2.bias'] = f'blk.{i}.ffn_down.bias'
+        tensor_map[f'{p}.norm1.weight'] = f'blk.{i}.attn_norm.weight'
+        tensor_map[f'{p}.norm1.bias'] = f'blk.{i}.attn_norm.bias'
+        tensor_map[f'{p}.norm2.weight'] = f'blk.{i}.ffn_norm.weight'
+        tensor_map[f'{p}.norm2.bias'] = f'blk.{i}.ffn_norm.bias'
+    
+    # Write tensors
+    for orig_name in state_dict:
+        tensor = state_dict[orig_name]
+        mapped_name = tensor_map.get(orig_name, orig_name)
+        arr = tensor.numpy().astype(np.float32)
+        gguf_writer.add_tensor(mapped_name, arr)
+    
+    gguf_writer.write_header_to_file()
+    gguf_writer.write_kv_data_to_file()
+    gguf_writer.write_tensors_to_file()
+    gguf_writer.close()
+    
+    print(f"GGUF file created: {output_path}")
+    print(f"Total tensors written: {len(state_dict)}")
+    print(f"NOTE: This GGUF file uses a custom architecture 'transformer_lm'")
+    print(f"      and will NOT load in standard llama.cpp/llama-cpp-python")
+    print(f"      without adding custom architecture support.")
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+pytorch_bin = os.path.join(script_dir, "pytorch_model.bin")
+if os.path.exists(pytorch_bin):
+    convert_sage_to_gguf(pytorch_bin, "sage-f16.gguf")
+else:
+    print(f"Model file {pytorch_bin} not found")

sage-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b575f6e96cc39676e7d7b841cc990965477161f00d7831c5cf21d18d6a2a21e6
+size 12787200