Nano.Deep.Reasoner.11m | session complete | 20,000 examples | step 191

README.md

@@ -0,0 +1,69 @@
+---
+license: mit
+library_name: pytorch
+tags:
+- causal-lm
+- decoder-only
+- reasoning
+- chain-of-thought
+- think
+- thought
+- reasoning
+- pretraining
+---
+
+# Nano.Deep.Reasoner.11m
+
+A decoder-only language model trained completely from scratch.
+
+## Architecture
+
+- Parameters: 11,229,120
+- Context length: 1096 tokens
+- Hidden size: 360
+- Layers: 6
+- Attention heads: 8
+- Intermediate size: 1440
+- Vocabulary: 4,096
+- Weight tying: enabled
+
+## Core reasoning format
+
+The model vocabulary contains dedicated reasoning control tokens:
+
+- `<|input|>`
+- `<|think|>`
+- `<|thought|>`
+- `<|reasoning|>`
+- `<|answer|>`
+
+These tokens are part of the model vocabulary and training representation.
+
+## Training
+
+Dataset:
+
+`Plans11/Organized_PreTrain_1k_Context`
+
+Training is performed in sessions of up to:
+
+`20,000`
+
+new examples per session.
+
+Examples are tracked using SHA-256 content hashes so that dataset
+reordering does not intentionally cause the same example to be trained twice.
+
+Current progress:
+
+- Unique examples trained: 20,000
+- Global optimizer steps: 191
+
+## Important
+
+This model is a small experimental reasoning language model.
+The reasoning-token architecture does not guarantee factual correctness,
+logical correctness, or reliable reasoning.
+
+The model was trained with causal next-token prediction.
+

config.json

@@ -0,0 +1,39 @@
+{
+  "model_type": "nano_deep_reasoner",
+  "architectures": [
+    "NanoDeepReasonerForCausalLM"
+  ],
+  "vocab_size": 4096,
+  "hidden_size": 360,
+  "num_layers": 6,
+  "num_heads": 8,
+  "intermediate_size": 1440,
+  "max_position_embeddings": 1096,
+  "context_length": 1096,
+  "dropout": 0.05,
+  "layer_norm_eps": 1e-05,
+  "tie_word_embeddings": true,
+  "reasoning_tokens": [
+    "<pad>",
+    "<unk>",
+    "<|endoftext|>",
+    "<|input|>",
+    "<|think|>",
+    "<|thought|>",
+    "<|reasoning|>",
+    "<|answer|>"
+  ],
+  "num_parameters": 11229120,
+  "training": {
+    "dataset": "Plans11/Organized_PreTrain_1k_Context",
+    "session_examples": 20000,
+    "context_length": 1096,
+    "causal_next_token_prediction": true,
+    "reasoning_tokens": [
+      "<|think|>",
+      "<|thought|>",
+      "<|reasoning|>",
+      "<|answer|>"
+    ]
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea21473b0a0a15bb675da9f2eb36ba87f844b388ec6877ed1ffa10d94fce946c
+size 50821608

progress.json

@@ -0,0 +1,14 @@
+{
+  "model_name": "Nano.Deep.Reasoner.11m",
+  "dataset_repo": "Plans11/Organized_PreTrain_1k_Context",
+  "model_repo": "ITLL/Nano.Deep.Reasoner.11m",
+  "context_length": 1096,
+  "session_size": 20000,
+  "global_step": 191,
+  "unique_examples_completed": 20000,
+  "last_session_examples": 20000,
+  "remaining_estimate": 2693413,
+  "tokenizer_vocab_size": 4096,
+  "parameter_count": 11229120,
+  "updated_at_unix": 1787132858.6197612
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

training_metadata.json

@@ -0,0 +1,22 @@
+{
+  "model": "Nano.Deep.Reasoner.11m",
+  "architecture": "decoder-only",
+  "parameters": 11229120,
+  "context_length": 1096,
+  "vocab_size": 4096,
+  "dataset": "Plans11/Organized_PreTrain_1k_Context",
+  "global_step": 191,
+  "unique_examples_trained": 20000,
+  "last_session_size": 20000,
+  "learning_rate": 0.0003,
+  "batch_size": 8,
+  "gradient_accumulation": 4,
+  "weight_decay": 0.1,
+  "seed": 1147,
+  "reasoning_tokens": [
+    "<|think|>",
+    "<|thought|>",
+    "<|reasoning|>",
+    "<|answer|>"
+  ]
+}

training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a2c80b31fdca557676deadae88ae0e3e5f92df1ddfa967c92657a51b7bef7b7
+size 134839931