Hugging Face's logo

cmeister
/
tokenizer-lm-ablations

Text Generation
Safetensors
tokenization
tokenizer-ablation
research
multilingual
code
Model card Files
xet
 Community
cmeister commited on
Commit
9fc882b
·
verified ·
1 Parent(s): d97be42

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +8 -0
  2. models/full-128k-allmulti-equal-bpe/model.safetensors +3 -0
  3. models/full-128k-apertus-seed42/model.safetensors +3 -0
  4. models/full-128k-apertus/model.safetensors +3 -0
  5. models/full-128k-bpe-nfc-clean-balanced-v128k/model.safetensors +3 -0
  6. models/full-128k-bpe-nfc-clean-balanced-v160k/model.safetensors +3 -0
  7. models/full-128k-bpe-nfc-clean-balanced-v160k/tokenizer.json +3 -0
  8. models/full-128k-bpe-nfc-clean-balanced-v200k/model.safetensors +3 -0
  9. models/full-128k-bpe-nfc-clean-balanced-v200k/tokenizer.json +3 -0
  10. models/full-128k-bpe-nfc-clean-balanced-v64k/model.safetensors +3 -0
  11. models/full-128k-bpe-nfc-clean-balanced-v96k/model.safetensors +3 -0
  12. models/full-128k-bpe-nfc-clean-balanced/model.safetensors +3 -0
  13. models/full-128k-bpe-nfc-clean-fineweb2full/config.json +28 -0
  14. models/full-128k-bpe-nfc-clean-fineweb2full/model.safetensors +3 -0
  15. models/full-128k-bpe-nfc-clean-nomark-balanced/model.safetensors +3 -0
  16. models/full-128k-bpe-nfc-gpt4-fineweb2full/model.safetensors +3 -0
  17. models/full-128k-bpe-nfc-plus2-balanced/model.safetensors +3 -0
  18. models/full-128k-bpe-nfc-plus3-balanced/model.safetensors +3 -0
  19. models/full-128k-claude-allmulti-equal-bpe/model.safetensors +3 -0
  20. models/full-128k-claude-balanced-bpe-seed42/model.safetensors +3 -0
  21. models/full-128k-claude-balanced-bpe/model.safetensors +3 -0
  22. models/full-128k-claude-balanced-nfc-bpe-seed42/model.safetensors +3 -0
  23. models/full-128k-claude-balanced-nfc-bpe/model.safetensors +3 -0
  24. models/full-128k-claude-balanced-unigram-seed42/model.safetensors +3 -0
  25. models/full-128k-claude-balanced-unigram/model.safetensors +3 -0
  26. models/full-128k-claude-english-bpe-seed42/model.safetensors +3 -0
  27. models/full-128k-claude-english-bpe/model.safetensors +3 -0
  28. models/full-128k-clauderev-balanced-bpe/model.safetensors +3 -0
  29. models/full-128k-gpt2-balanced-bpe/model.safetensors +3 -0
  30. models/full-128k-gpt4o-allmulti-equal-nfc-bpe/config.json +28 -0
  31. models/full-128k-gpt4o-allmulti-equal-nfc-bpe/generation_config.json +4 -0
  32. models/full-128k-gpt4o-allmulti-equal-nfc-bpe/model.safetensors +3 -0
  33. models/full-128k-gpt4o-allmulti-equal-norepeat-bpe/model.safetensors +3 -0
  34. models/full-128k-gpt4o-balanced-bpe-seed42/model.safetensors +3 -0
  35. models/full-128k-gpt4o-balanced-bpe/model.safetensors +3 -0
  36. models/full-128k-gpt4o-balanced-maxfiles50-bpe/model.safetensors +3 -0
  37. models/full-128k-gpt4o-balanced-nfc-bpe-seed42/model.safetensors +3 -0
  38. models/full-128k-gpt4o-balanced-nfc-bpe/model.safetensors +3 -0
  39. models/full-128k-gpt4o-balanced-unigram-seed42/model.safetensors +3 -0
  40. models/full-128k-gpt4o-balanced-unigram-tuned/config.json +28 -0
  41. models/full-128k-gpt4o-balanced-unigram-tuned/model.safetensors +3 -0
  42. models/full-128k-gpt4o-balanced-unigram-tuned/tokenizer.json +0 -0
  43. models/full-128k-gpt4o-balanced-unigram-tuned/tokenizer_config.json +7 -0
  44. models/full-128k-gpt4o-balanced-unigram/generation_config.json +4 -0
  45. models/full-128k-gpt4o-balanced-unigram/model.safetensors +3 -0
  46. models/full-128k-gpt4o-balanced-unigram/tokenizer.json +0 -0
  47. models/full-128k-gpt4o-code-bpe-seed42/model.safetensors +3 -0
  48. models/full-128k-gpt4o-code-bpe/model.safetensors +3 -0
  49. models/full-128k-gpt4o-english-bpe-seed42/model.safetensors +3 -0
  50. models/full-128k-gpt4o-english-bpe/model.safetensors +3 -0
.gitattributes CHANGED
@@ -43,3 +43,11 @@ tokenizers/pabpe-nfc-clean-fineweb2full/tokenizer.json filter=lfs diff=lfs merge
43
  tokenizers/bpe-nfc-clean-balanced-v200k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
44
  tokenizers/bpe-nfc-clean-balanced-v160k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
45
  models/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
43
  tokenizers/bpe-nfc-clean-balanced-v200k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
44
  tokenizers/bpe-nfc-clean-balanced-v160k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
45
  models/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
46
+ models/full-128k-pabpe-nfc-clean-fineweb2full/tokenizer.json filter=lfs diff=lfs merge=lfs -text
47
+ models/full-128k-highres-bpe/tokenizer.json filter=lfs diff=lfs merge=lfs -text
48
+ models/full-128k-pabpe-nfc-gpt4-balanced/tokenizer.json filter=lfs diff=lfs merge=lfs -text
49
+ models/full-128k-bpe-nfc-clean-balanced-v160k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
50
+ models/full-128k-pabpe-nfc-clean-balanced/tokenizer.json filter=lfs diff=lfs merge=lfs -text
51
+ models/full-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
52
+ models/full-128k-pabpe-nfc-gpt4-fineweb2full/tokenizer.json filter=lfs diff=lfs merge=lfs -text
53
+ models/full-128k-bpe-nfc-clean-balanced-v200k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
models/full-128k-allmulti-equal-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcc86bab58db7d88d7d960b0a90ae7c2ee88f79e007aec4e4670a1b089534cbd
3
+ size 4303381368
models/full-128k-apertus-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef6809fa8aae7417586e46f333ab29a73c2180190048265999f002d9e4f882c7
3
+ size 4337984376
models/full-128k-apertus/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2351a8a5b44d21a752d1246693142069b640ce21a14720e302ead959f910d0fe
3
+ size 4337984376
models/full-128k-bpe-nfc-clean-balanced-v128k/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8631855555a2df34e82dd86c58e0d0d74a2f193448bbd8c6e0eb5add82a19be9
3
+ size 4300235640
models/full-128k-bpe-nfc-clean-balanced-v160k/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d072a92601196367f7a57bff0cd98d7be8b10442f5a26905e93ec99165ac503f
3
+ size 4693451640
models/full-128k-bpe-nfc-clean-balanced-v160k/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d78690d98b4545f09a872071ccd7e13ab648e283ebff6dc7f485508b3b7f005
3
+ size 13001714
models/full-128k-bpe-nfc-clean-balanced-v200k/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:15b3bb329d8132f5875055fd343600b8a17ad93f8670010b31a535ff1dfefb2d
3
+ size 5184971640
models/full-128k-bpe-nfc-clean-balanced-v200k/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0e58ab71d6276edb32bb0e6e67b65c965047cb29f8061ff13b8757700348816
3
+ size 16470896
models/full-128k-bpe-nfc-clean-balanced-v64k/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47874f799e443b3b443cb435693cda0cb4b0cf12447f4e681d0250f52a4c7a0a
3
+ size 3513803624
models/full-128k-bpe-nfc-clean-balanced-v96k/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bd2270aa77b3295d60d9c5f851e5c658ceb9e52f0f7ccbfcff286fc52496cbc
3
+ size 3907019624
models/full-128k-bpe-nfc-clean-balanced/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7197874db0a291b48214378e35308e08675d28040251a9d02b0e9972dfba184f
3
+ size 4301022072
models/full-128k-bpe-nfc-clean-fineweb2full/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "NanoChatForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "nanochat_model.NanoChatConfig",
7
+ "AutoModelForCausalLM": "nanochat_model.NanoChatForCausalLM"
8
+ },
9
+ "bos_token_id": 127996,
10
+ "dtype": "float32",
11
+ "eos_token_id": null,
12
+ "head_dim": 128,
13
+ "kv_dim": 1536,
14
+ "logit_softcap": 15.0,
15
+ "model_type": "nanochat",
16
+ "n_embd": 1536,
17
+ "n_head": 12,
18
+ "n_kv_head": 12,
19
+ "n_layer": 24,
20
+ "pad_vocab_size_to": 64,
21
+ "padded_vocab_size": 128000,
22
+ "sequence_len": 2048,
23
+ "tie_word_embeddings": false,
24
+ "transformers_version": "5.3.0",
25
+ "ve_dim": 128,
26
+ "vocab_size": 128000,
27
+ "window_pattern": "SSSL"
28
+ }
models/full-128k-bpe-nfc-clean-fineweb2full/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8376387c9523fd19406a923d3cb2165e58353d039b268b5a9dac5ea6388dad85
3
+ size 4300235640
models/full-128k-bpe-nfc-clean-nomark-balanced/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:438791a33e441de08830694ef9e9a02b3435cbda64f3b962ec80d49ed9137f22
3
+ size 4300235640
models/full-128k-bpe-nfc-gpt4-fineweb2full/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd68b2d7efd9805e3839f0f1cba355d1f12a393a7db0e3230b15a3104b654fdb
3
+ size 4300235640
models/full-128k-bpe-nfc-plus2-balanced/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:80075ec472ce125fca45a114c156d5322401609ed0da4a02ded308f18e2d92f6
3
+ size 4303381368
models/full-128k-bpe-nfc-plus3-balanced/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f12c1307273292ea8d04112d545e543aa1dd73c6f6a88dd70abfd23bec6a7758
3
+ size 4303381368
models/full-128k-claude-allmulti-equal-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c9669f7dcfe2414eb4f9bd1967be98a02af72eae402c34931d365b7473383d5
3
+ size 4303381368
models/full-128k-claude-balanced-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21b9dc2923fe051dd1fc34da1949d9d4d415fa547cb3698044cccf58110c9061
3
+ size 4303381368
models/full-128k-claude-balanced-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f77d30a26c5a3adb4a05aa406f876dc903d255e9c3dd4e133f51d46863babfe0
3
+ size 4303381368
models/full-128k-claude-balanced-nfc-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50233d80b8d8b08275dadae596e5d0b9f72e8a1ce1dbae635437d231a417e5a6
3
+ size 4303381368
models/full-128k-claude-balanced-nfc-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4e63a92abacd80f590a6e57a3114cf53bd299305bcbf05331c8d4436995fae7
3
+ size 4303381368
models/full-128k-claude-balanced-unigram-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eed067437b857ffa6cf373d311a738d13348e2fabec677a30e816d7b1f38d165
3
+ size 4303381368
models/full-128k-claude-balanced-unigram/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2e055a2cdab6fa5afba9b4e3a7a341a6a9812079ad59a1338c9de23bbd1242f
3
+ size 4303381368
models/full-128k-claude-english-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f96f8b501d05f85c00d641a296587d1b1a978e3acd55a7f6c51b87f09e67fe86
3
+ size 4303381368
models/full-128k-claude-english-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6b56d70172ffc2be91a4f6555f70a732f1edb578aef415c935340ff0b7e3464
3
+ size 4303381368
models/full-128k-clauderev-balanced-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a2190e852dd82d777f4eca5820d28632e86ef2e87df93cddc8daa5bad53a36e0
3
+ size 4303381368
models/full-128k-gpt2-balanced-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79b0db9cc85331b834d2d58697e52981dadf0635571ada089d50f6c917da371c
3
+ size 4303381368
models/full-128k-gpt4o-allmulti-equal-nfc-bpe/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "NanoChatForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "nanochat_model.NanoChatConfig",
7
+ "AutoModelForCausalLM": "nanochat_model.NanoChatForCausalLM"
8
+ },
9
+ "bos_token_id": 0,
10
+ "dtype": "float32",
11
+ "eos_token_id": null,
12
+ "head_dim": 128,
13
+ "kv_dim": 1536,
14
+ "logit_softcap": 15.0,
15
+ "model_type": "nanochat",
16
+ "n_embd": 1536,
17
+ "n_head": 12,
18
+ "n_kv_head": 12,
19
+ "n_layer": 24,
20
+ "pad_vocab_size_to": 64,
21
+ "padded_vocab_size": 128256,
22
+ "sequence_len": 2048,
23
+ "tie_word_embeddings": false,
24
+ "transformers_version": "5.3.0",
25
+ "ve_dim": 128,
26
+ "vocab_size": 128256,
27
+ "window_pattern": "SSSL"
28
+ }
models/full-128k-gpt4o-allmulti-equal-nfc-bpe/generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 0,
3
+ "transformers_version": "5.3.0"
4
+ }
models/full-128k-gpt4o-allmulti-equal-nfc-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4f5f8356103747b48d0371479c2fdbf97f62be192ed1cdd40b272b477bfed29
3
+ size 4303381368
models/full-128k-gpt4o-allmulti-equal-norepeat-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f93bcd0df4748d97b0810c321f9f45d7a4feeec6bda8f87c27a8735070fad5b7
3
+ size 4303381368
models/full-128k-gpt4o-balanced-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f9fe0a6683cffbe7574cfccbfc9f8476682c2cfef4026eb067254dc91d4b4d3
3
+ size 4303381368
models/full-128k-gpt4o-balanced-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b5e88c9cbc97eb141a6a2a5f2a2326ec7e69550763183113f1be36e7e0470aae
3
+ size 4303381368
models/full-128k-gpt4o-balanced-maxfiles50-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5b8e28d1590071227cd8b41b97e2312dad442b4b3f56d9c3a525503177411ce
3
+ size 4303381368
models/full-128k-gpt4o-balanced-nfc-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:785da1b8c9b5a226429aa497fe64ed940e9577845f3e8a3794d09b38bf76ccbe
3
+ size 4303381368
models/full-128k-gpt4o-balanced-nfc-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cfbb7ab9c0bae6297652ef07ea8ca4295ab1fe9bec9eada5bb84789d92989b2
3
+ size 4303381368
models/full-128k-gpt4o-balanced-unigram-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dfdd8edef4cec0df8d9034e4619a2b5c5147f3be2102e62b831730749c7a1b56
3
+ size 4303381368
models/full-128k-gpt4o-balanced-unigram-tuned/config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "NanoChatForCausalLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "nanochat_model.NanoChatConfig",
7
+ "AutoModelForCausalLM": "nanochat_model.NanoChatForCausalLM"
8
+ },
9
+ "bos_token_id": 0,
10
+ "dtype": "float32",
11
+ "eos_token_id": null,
12
+ "head_dim": 128,
13
+ "kv_dim": 1536,
14
+ "logit_softcap": 15.0,
15
+ "model_type": "nanochat",
16
+ "n_embd": 1536,
17
+ "n_head": 12,
18
+ "n_kv_head": 12,
19
+ "n_layer": 24,
20
+ "pad_vocab_size_to": 64,
21
+ "padded_vocab_size": 128256,
22
+ "sequence_len": 2048,
23
+ "tie_word_embeddings": false,
24
+ "transformers_version": "5.3.0",
25
+ "ve_dim": 128,
26
+ "vocab_size": 128256,
27
+ "window_pattern": "SSSL"
28
+ }
models/full-128k-gpt4o-balanced-unigram-tuned/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5609881a5a6a24b0305c361e4730bdf9968ce33a495415683e4cad578cde1655
3
+ size 4303381368
models/full-128k-gpt4o-balanced-unigram-tuned/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
models/full-128k-gpt4o-balanced-unigram-tuned/tokenizer_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "model_max_length": 2048,
4
+ "bos_token": "<s>",
5
+ "add_bos_token": false,
6
+ "clean_up_tokenization_spaces": false
7
+ }
models/full-128k-gpt4o-balanced-unigram/generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 0,
3
+ "transformers_version": "5.3.0"
4
+ }
models/full-128k-gpt4o-balanced-unigram/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9877be1f936d771748a2386006be4988e9be4abf0edd6242af907f607978e5af
3
+ size 4303381368
models/full-128k-gpt4o-balanced-unigram/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
models/full-128k-gpt4o-code-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50565b39a3f2d4618629c68c954bd9ed120f3f789bc7c351cd2cc3672a9a19f6
3
+ size 4303381368
models/full-128k-gpt4o-code-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8606fbadffd597e6c18769e227512b9d3c76fbc957e6732a41a74d82d5868418
3
+ size 4303381368
models/full-128k-gpt4o-english-bpe-seed42/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c381c6fc69a78763fb0a3626406b711e338558693681ef81de78780eab5d9255
3
+ size 4303381368
models/full-128k-gpt4o-english-bpe/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1466f615860fc0b8f92bf4dd7fda5cb8690c21cdb84cfc9555f158136d522954
3
+ size 4303381368