Upload NVFP4 quantized Behemoth-R1-123B-v2

README.md

@@ -0,0 +1,56 @@
+---
+license: other
+base_model: TheDrummer/Behemoth-R1-123B-v2
+tags:
+  - nvfp4
+  - modelopt
+  - quantized
+  - blackwell
+  - b200
+library_name: transformers
+---
+
+# Behemoth-R1-V2 ModelOpt NVFP4
+
+NVFP4 quantized version of [TheDrummer/Behemoth-R1-123B-v2](https://huggingface.co/TheDrummer/Behemoth-R1-123B-v2) using NVIDIA Model Optimizer.
+
+## Quantization Details
+
+| Property | Value |
+|----------|-------|
+| **Original Model** | TheDrummer/Behemoth-R1-123B-v2 |
+| **Quantization** | NVFP4 (FP4 weights, FP16 activations) |
+| **Method** | NVIDIA ModelOpt PTQ |
+| **Calibration Samples** | 512 |
+| **Max Sequence Length** | 4096 |
+
+## Hardware Requirements
+
+- **Optimal**: NVIDIA Blackwell GPUs (B100, B200, RTX PRO 6000 Blackwell)
+- **Compatible**: Hopper/Ampere (will use weight-only mode)
+
+## Usage with vLLM
+
+```python
+from vllm import LLM, SamplingParams
+
+llm = LLM(
+    model="TheHouseOfTheDude/Behemoth-R1-V2_ModelOpt-NVFP4",
+    quantization="modelopt",
+    trust_remote_code=True,
+)
+
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=512)
+outputs = llm.generate(["Write a story about..."], sampling_params)
+print(outputs[0].outputs[0].text)
+```
+
+## Chat Template
+
+Uses Mistral v7 (Non-Tekken) format. See the original model card for usage details.
+
+## Credits
+
+- Original Model: [TheDrummer](https://huggingface.co/TheDrummer)
+- Quantization: TheHouseOfTheDude
+- Quantization Framework: NVIDIA ModelOpt

chat_template.jinja

@@ -0,0 +1 @@
+{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + '[/INST]' }}{% elif message['role'] == 'system' %}{{ '[SYSTEM_PROMPT] ' + message['content'] + '[/SYSTEM_PROMPT]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + eos_token }}{% else %}{{ raise_exception('Only user, system and assistant roles are supported!') }}{% endif %}{% endfor %}

config.json

@@ -0,0 +1,56 @@
+{
+    "architectures": [
+        "MistralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "dtype": "bfloat16",
+    "eos_token_id": 2,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 12288,
+    "initializer_range": 0.02,
+    "intermediate_size": 28672,
+    "max_position_embeddings": 131072,
+    "model_type": "mistral",
+    "num_attention_heads": 96,
+    "num_hidden_layers": 88,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "transformers_version": "4.57.6",
+    "use_cache": true,
+    "vocab_size": 32768,
+    "quantization_config": {
+        "config_groups": {
+            "group_0": {
+                "input_activations": {
+                    "dynamic": false,
+                    "num_bits": 4,
+                    "type": "float",
+                    "group_size": 16
+                },
+                "weights": {
+                    "dynamic": false,
+                    "num_bits": 4,
+                    "type": "float",
+                    "group_size": 16
+                },
+                "targets": [
+                    "Linear"
+                ]
+            }
+        },
+        "ignore": [
+            "lm_head"
+        ],
+        "quant_algo": "NVFP4",
+        "producer": {
+            "name": "modelopt",
+            "version": "0.41.0"
+        },
+        "quant_method": "modelopt"
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.6"
+}

hf_quant_config.json

@@ -0,0 +1,14 @@
+{
+    "producer": {
+        "name": "modelopt",
+        "version": "0.41.0"
+    },
+    "quantization": {
+        "quant_algo": "NVFP4",
+        "kv_cache_quant_algo": null,
+        "group_size": 16,
+        "exclude_modules": [
+            "lm_head"
+        ]
+    }
+}

model-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3a51efb15092f6412b66fd307880c896b646985a1489c05c5348ae6ce98102
+size 4882434096

model-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:020ae3cc89ab5f131e93e207326d407f09a273a2d9f4bb320fb7232485c3e8be
+size 4869902096

model-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42bc43c141a7c229a7be3af07ec16fb27c90c151f1e17df50129314d09e3255f
+size 4869902232

model-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f96e7e22376c32f5305bdc3d7b992160ae2dd1324c823ed32c0d5172d261d405
+size 4969043384

model-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:318e86dd68b60b5810da52f5f69d03e524b9a27dfa8238adc0d7f90ae5ad3813
+size 4954837344

model-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82a68390d6c39e355eaafe8bcfe8d0c31344d1b22370ff907b6af5342a36c3b7
+size 4869902232

model-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:327ee4c4cb3b3ee8beb6b1176c964f205aa20a3e179fe13281f48ccec32affb6
+size 4969043384

model-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0a3d712ce2dd3c8915b14dccaa1b1f4013b0ef0a7c68bd23d24f53949c74d62
+size 4954837344

model-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2087aeb96c0cf23ec466aa4911ce5ff03caae8ba1a5cddefb03da391d3b54bc0
+size 4869902232

model-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839ab031b14b27218311a15fd032aea0585bb5a6c6ee03e187944d2a550a4fa7
+size 4969043384

model-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44b34d3d28ec84aa2efd527f601bdc62abb358133f7caa095570462e021bbc6f
+size 4954837344

model-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858adf31a2042971d09d3b1e3fca275d4fe346a1c8168db133df9ce6e20a9b93
+size 4869902232

model-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff3a9e6c820c60432dfad94de70a5e7b92b12563ef448acf9e415ce906d59e27
+size 4969043384

model-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0139ab9cc4e85f5d5da4adbffad444cdb67bfd2927bc1ea71d129d8cbb67379
+size 4954837344

model-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51445b6bf7b940e6c9134f28fef62fd02af1e4435f7d8a86c74aa91a92538bd7
+size 1201743136

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}