Added model files

README.md

@@ -0,0 +1,40 @@
+  ---
+  license: mit
+  tags:
+  - text-generation
+  - pytorch
+  - transformer
+  - custom-model
+  language:
+  - en
+  pipeline_tag: text-generation
+  ---
+
+  # VelocityLM - 2B Parameter Language Model
+
+  A custom transformer model with 2B parameters trained for text generation.
+
+  ## Model Details
+
+  - **Parameters:** ~2 billion
+  - **Architecture:** Custom Transformer with RoPE, RMSNorm, SwiGLU
+  - **Context Length:** 2,048 tokens
+  - **Tokenizer:** GPT-2 compatible
+  - **Training:** Falcon RefinedWeb dataset
+
+  ## Usage
+
+  ```python
+  from transformers import AutoTokenizer
+  import torch
+
+  # Load tokenizer
+  tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+  # Load model (you'll need custom loading code)
+  # See the Space implementation for details
+
+  Files
+
+  - config.json - Model configuration
+  - pytorch_model.bin - Model weights

config.json

@@ -0,0 +1,59 @@
+{
+  "model": {
+    "vocab_size": 50257,
+    "hidden_size": 2048,
+    "num_hidden_layers": 24,
+    "num_attention_heads": 32,
+    "intermediate_size": 8192,
+    "hidden_act": "silu",
+    "max_position_embeddings": 2048,
+    "initializer_range": 0.02,
+    "layer_norm_eps": 1e-05,
+    "use_cache": false,
+    "rope_theta": 10000.0,
+    "attention_dropout": 0.0,
+    "hidden_dropout": 0.0
+  },
+  "training": {
+    "batch_size_per_device": 4,
+    "gradient_accumulation_steps": 8,
+    "learning_rate": 0.0006,
+    "min_learning_rate": 6e-05,
+    "weight_decay": 0.1,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "adam_epsilon": 1e-08,
+    "max_grad_norm": 1.0,
+    "warmup_steps": 2000,
+    "num_training_steps": 100000,
+    "logging_steps": 10,
+    "save_steps": 1000,
+    "eval_steps": 500,
+    "save_total_limit": 5,
+    "fp16": true,
+    "gradient_checkpointing": true
+  },
+  "data": {
+    "dataset_name": "tiiuae/falcon-refinedweb",
+    "dataset_config": "default",
+    "text_column": "content",
+    "max_seq_length": 2048,
+    "num_workers": 8,
+    "preprocessing_num_workers": 16
+  },
+  "tokenizer": {
+    "tokenizer_name": "gpt2",
+    "add_special_tokens": true
+  },
+  "infrastructure": {
+    "num_gpus": 4,
+    "seed": 42,
+    "output_dir": "./checkpoints",
+    "logging_dir": "./logs",
+    "resume_from_checkpoint": "./checkpoints/checkpoint-4000"
+  },
+  "distributed": {
+    "backend": "nccl",
+    "find_unused_parameters": false
+  }
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:194cb060512a6a56887d2e0d3f623627cdef7231e8de1655d0f7c013883c21f0
+size 6854652150