upload tiny test model
Browse files- README.md +2 -2
- config.json +4 -4
- model.safetensors +2 -2
- slice_d1.py +3 -3
README.md
CHANGED
|
@@ -20,8 +20,8 @@ GGUF version: [ggml-org-dev/tiny-d1-2B-for-testing-GGUF](https://huggingface.co/
|
|
| 20 |
|
| 21 |
Sliced from the safetensors with [slice_d1.py](slice_d1.py), which follows the `slice-model` skill of llama.cpp:
|
| 22 |
|
| 23 |
-
- Text model: 4 of 24 layers (0, 2, 3, 4), hidden size 2048 ->
|
| 24 |
-
- Vision encoder: 2 of 14 layers (0, 2), hidden size 1024 ->
|
| 25 |
- The Python code of the original repo is not included
|
| 26 |
|
| 27 |
## License
|
|
|
|
| 20 |
|
| 21 |
Sliced from the safetensors with [slice_d1.py](slice_d1.py), which follows the `slice-model` skill of llama.cpp:
|
| 22 |
|
| 23 |
+
- Text model: 4 of 24 layers (0, 2, 3, 4), hidden size 2048 -> 64, heads 16 -> 2, KV heads 4 -> 1, FFN 7680 -> 256 (head_dim 128 kept)
|
| 24 |
+
- Vision encoder: 2 of 14 layers (0, 2), hidden size 1024 -> 128, heads 16 -> 2, KV heads 8 -> 1, FFN 2560 -> 256, projector 4096 -> 256
|
| 25 |
- The Python code of the original repo is not included
|
| 26 |
|
| 27 |
## License
|
config.json
CHANGED
|
@@ -20,7 +20,7 @@
|
|
| 20 |
"dtype": "bfloat16",
|
| 21 |
"eos_token_id": 124900,
|
| 22 |
"head_dim": 128,
|
| 23 |
-
"hidden_size":
|
| 24 |
"initializer_range": 0.02,
|
| 25 |
"intermediate_size": 256,
|
| 26 |
"layer_types": [
|
|
@@ -59,7 +59,7 @@
|
|
| 59 |
"vision_config": {
|
| 60 |
"conv_kernel_size": 3,
|
| 61 |
"dtype": "bfloat16",
|
| 62 |
-
"hidden_size":
|
| 63 |
"in_channels": 3,
|
| 64 |
"initializer_range": 0.02,
|
| 65 |
"intermediate_size": 256,
|
|
@@ -69,9 +69,9 @@
|
|
| 69 |
],
|
| 70 |
"model_type": "lfm3_vl_vision",
|
| 71 |
"norm_eps": 1e-05,
|
| 72 |
-
"num_attention_heads":
|
| 73 |
"num_hidden_layers": 2,
|
| 74 |
-
"num_key_value_heads":
|
| 75 |
"patch_size": 16,
|
| 76 |
"rope_parameters": {
|
| 77 |
"rope_theta": 1000000.0,
|
|
|
|
| 20 |
"dtype": "bfloat16",
|
| 21 |
"eos_token_id": 124900,
|
| 22 |
"head_dim": 128,
|
| 23 |
+
"hidden_size": 64,
|
| 24 |
"initializer_range": 0.02,
|
| 25 |
"intermediate_size": 256,
|
| 26 |
"layer_types": [
|
|
|
|
| 59 |
"vision_config": {
|
| 60 |
"conv_kernel_size": 3,
|
| 61 |
"dtype": "bfloat16",
|
| 62 |
+
"hidden_size": 128,
|
| 63 |
"in_channels": 3,
|
| 64 |
"initializer_range": 0.02,
|
| 65 |
"intermediate_size": 256,
|
|
|
|
| 69 |
],
|
| 70 |
"model_type": "lfm3_vl_vision",
|
| 71 |
"norm_eps": 1e-05,
|
| 72 |
+
"num_attention_heads": 2,
|
| 73 |
"num_hidden_layers": 2,
|
| 74 |
+
"num_key_value_heads": 1,
|
| 75 |
"patch_size": 16,
|
| 76 |
"rope_parameters": {
|
| 77 |
"rope_theta": 1000000.0,
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8413e290e4758c9888b96f1536be3c900c1b74681272857707c7072567911b16
|
| 3 |
+
size 18369880
|
slice_d1.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
"""
|
| 3 |
Build a tiny slice of LiquidAI/d1-2B (lfm3-d1 decision model) for testing. Output is gibberish, only the shapes and code paths are real.
|
| 4 |
|
| 5 |
-
Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the 128000 x 2048 token embedding, so this one also narrows the hidden sizes.
|
| 6 |
|
| 7 |
Adapted from slice_d1omni.py of ggml-org-dev/tiny-d1-Omni-2B-for-testing. d1-2B has the LFM3-VL weights (no audio encoder, no depthformer).
|
| 8 |
|
|
@@ -20,8 +20,8 @@ from safetensors.torch import save_file
|
|
| 20 |
REPO = 'LiquidAI/d1-2B'
|
| 21 |
AUX = ['tokenizer.json', 'tokenizer_config.json', 'chat_template.jinja', 'processor_config.json', 'calibration.json', 'LICENSE']
|
| 22 |
|
| 23 |
-
E, H, KV, FF =
|
| 24 |
-
V, VH, VKV, VFF, VP =
|
| 25 |
TEXT_LAYERS = [0, 2, 3, 4] # conv, full attn, conv, sliding attn
|
| 26 |
VISION_LAYERS = [0, 2] # conv, full attn
|
| 27 |
|
|
|
|
| 2 |
"""
|
| 3 |
Build a tiny slice of LiquidAI/d1-2B (lfm3-d1 decision model) for testing. Output is gibberish, only the shapes and code paths are real.
|
| 4 |
|
| 5 |
+
Adapted from the slice-model skill of llama.cpp (skills/slice-model): tensors are fetched with HTTP range requests and config.json is patched to match, then convert_hf_to_gguf.py does the conversion. The stock script only drops layers, which cannot go below the 128000 x 2048 token embedding, so this one also narrows the hidden sizes (text hidden 64 keeps the Q4_0 GGUF under 10 MB).
|
| 6 |
|
| 7 |
Adapted from slice_d1omni.py of ggml-org-dev/tiny-d1-Omni-2B-for-testing. d1-2B has the LFM3-VL weights (no audio encoder, no depthformer).
|
| 8 |
|
|
|
|
| 20 |
REPO = 'LiquidAI/d1-2B'
|
| 21 |
AUX = ['tokenizer.json', 'tokenizer_config.json', 'chat_template.jinja', 'processor_config.json', 'calibration.json', 'LICENSE']
|
| 22 |
|
| 23 |
+
E, H, KV, FF = 64, 2, 1, 256 # text: hidden, heads, kv heads, ffn. head_dim stays 128
|
| 24 |
+
V, VH, VKV, VFF, VP = 128, 2, 1, 256, 256 # vision, head_dim stays 64, VP is the projector hidden size
|
| 25 |
TEXT_LAYERS = [0, 2, 3, 4] # conv, full attn, conv, sliding attn
|
| 26 |
VISION_LAYERS = [0, 2] # conv, full attn
|
| 27 |
|