Upload MatterWaveAI/grt-30m-chat release checkpoint and model card

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,43 @@
+---
+license: apache-2.0
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+  - grt
+  - chat
+  - conversational
+  - rotational-transformer
+  - givens-rotations
+  - clifford-algebra
+  - matterwave
+base_model: MatterWaveAI/grt-30m
+---
+
+# GRT-30M-Chat (`MatterWaveAI/grt-30m-chat`)
+
+**GRT-30M-Chat** is the instruction and multi-turn conversational fine-tune of [`MatterWaveAI/grt-30m`](https://huggingface.co/MatterWaveAI/grt-30m) (`29,305,498` parameters, **zero `nn.Linear` layers**), formatted with the standard **ChatML** (`<|im_start|>` / `<|im_end|>`) template.
+
+## SFT Alignment Metrics
+
+- **Held-out Conversational Perplexity**: `182.79` (base) $\rightarrow$ **`1.37`** (Chat SFT)
+- **Held-out Assistant Loss**: `5.2083` $\rightarrow$ **`0.3129`**
+
+## Quickstart (`transformers`)
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "MatterWaveAI/grt-30m-chat"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.float32)
+
+messages = [
+    {"role": "user", "content": "What is GRT-30M?"}
+]
+prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
+
+out_ids = model.generate(**inputs, max_new_tokens=64, do_sample=False)
+print(tokenizer.decode(out_ids[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))
+```

SHA256SUMS

@@ -0,0 +1,8 @@
+f0aa0fd5bbba646dc7ae058d9e7e3436e4b2c1f8c422a2bfc06c8de09de076fc  chat_template.jinja
+c2b264a05e11914a8e41c1610f3c81e15d38151c459a1c66caaaa0381d804685  config.json
+4638cae59268889fb9731e481655c2da3a756f35d6d733f6e052ab76ba71b051  eval_results.json
+da90aebf9dbd74d88711c9cc69becf4b4d0e14d0f2f677d1351f4c48982bb23b  generation_config.json
+ee613803cb7b2c618def7e6c79476e1aab4e887cc08e234204b6bad12ad772bc  model.safetensors
+bf903f935c71e65018b3db05b5f48165c39a7efe90a49b5c2fe433768f23bcc8  README.md
+be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506  tokenizer.json
+b0b05688d53eec97348d23f9a203772fa28f12bdc8a0f6f5401f41568ec6ca31  tokenizer_config.json

chat_template.jinja

@@ -0,0 +1,6 @@
+{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system
+You are GRT-30M-Chat, a helpful, concise AI assistant built by MatterWave AI on the 100% rotational Givens Rotational Transformer architecture.<|im_end|>
+' }}{% endif %}{{ '<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>
+' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -0,0 +1,36 @@
+{
+  "angle_init_scale": 0.1,
+  "architectures": [
+    "GRTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "coupling_init_mode": "diagonal",
+  "coupling_init_scale": 0.01,
+  "dtype": "float32",
+  "eos_token_id": 151645,
+  "ffn_n_rotations": 21,
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "learnable_residual_gates": true,
+  "learnable_rope": true,
+  "log_magnitude_init": 0.0,
+  "max_position_embeddings": 2048,
+  "model_type": "grt",
+  "n_mixer_rotations": 1,
+  "n_qkv_rotations": 11,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 12,
+  "output_scale_init": 30.0,
+  "pad_token_id": 151643,
+  "peri_ln": false,
+  "phase_init_scale": 0.1,
+  "pos_freq_base": 10000.0,
+  "residual_gate_mode": "bounded_depth",
+  "residual_norm_mode": "none",
+  "tie_word_embeddings": true,
+  "transformers_version": "5.21.0.dev0",
+  "use_cache": true,
+  "vocab_size": 151669
+}

eval_results.json

@@ -0,0 +1,42 @@
+{
+  "model_id": "MatterWaveAI/grt-30m-chat",
+  "architecture": "GRTForCausalLM",
+  "parameters": 29305498,
+  "nn_linear_layers": 0,
+  "chat_template": "ChatML (<|im_start|> / <|im_end|>)",
+  "before_sft": {
+    "val_loss": 5.2083,
+    "val_ppl": 182.7854
+  },
+  "after_sft": {
+    "val_loss": 0.3129,
+    "val_ppl": 1.3674,
+    "samples": [
+      {
+        "user": "Can you explain this briefly? What is GRT-30M?",
+        "target": "GRT-30M is a 29.3M-parameter Givens Rotational Transformer developed by MatterWave AI that replaces every dense nn.Linear matrix with cascaded 2D Givens rotations and butterfly permutations.",
+        "generated": "GRT-30M is a 29.3M-parameter Givens Rotational Transformer developed by MatterWave AI that replaces every dense nn.Linear matrix with cascaded 2D Givens rotations and butterfly permutations."
+      },
+      {
+        "user": "Can you explain this briefly? How does a Givens Rotational Transformer replace dense linear layers?",
+        "target": "Instead of O(D^2) dense matrix multiplications, GRT represents hidden states as planar coordinate pairs and transforms them through butterfly-permuted 2D Givens rotations modulated by Clifford algebra features.",
+        "generated": "Instead of O(D^2) dense matrix multiplications, GRT represents hidden states as planar coordinate pairs and transforms them through butterfly-permuted 2D Givens rotations modulated by Clifford algebra features."
+      },
+      {
+        "user": "Can you explain this briefly? What is PearlNorm in the GRT architecture?",
+        "target": "PearlNorm is a pair-aware RMS normalization layer that divides each planar (x, y) pair by the root-mean-square of pair radii, rescaling magnitudes while preserving every pair's phase angle.",
+        "generated": "PearlNorm is a pair-aware RMS normalization layer that divides each planar (x, y) pair by the root-mean-square of pair radii, rescaling magnitudes while preserving every pair's phase angle."
+      },
+      {
+        "user": "Can you explain this briefly? What is RadialGELU?",
+        "target": "RadialGELU applies a GELU activation to the Euclidean radius r of each 2D coordinate pair as gelu(r) / r, introducing nonlinearity on pair magnitudes without distorting phase directions.",
+        "generated": "RadialGELU applies a GELU activation to the Euclidean radius r of each 2D coordinate pair as gelu(r) / r, introducing nonlinearity on pair magnitudes without distorting phase directions."
+      },
+      {
+        "user": "Can you explain this briefly? How are token embeddings parameterized in GRT?",
+        "target": "Each token in GRTEmbedding is stored as hidden_size // 2 phase angles plus a scalar log-magnitude, emitting exp(log_magnitude) * (cos(theta), sin(theta)) on the hypersphere.",
+        "generated": "Each token in GRTEmbedding is stored as hidden_size // 2 phase angles plus a scalar log-magnitude, emitting exp(log_magnitude) * (cos(theta), sin(theta)) on the hypersphere."
+      }
+    ]
+  }
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": false,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "max_new_tokens": 256,
+  "pad_token_id": 151643,
+  "transformers_version": "5.21.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee613803cb7b2c618def7e6c79476e1aab4e887cc08e234204b6bad12ad772bc
+size 121573808

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}