Hugging Face's logo

hcisbmm
/
vial_fast_tokenizer

Transformers
Model card Files
xet
 Community
hcisbmm commited on
Commit
233750a
·
verified ·
1 Parent(s): e10f2db

Upload tokenizer metadata

Browse files
Files changed (1) hide show
  1. metadata.json +10 -10
metadata.json CHANGED
@@ -2,18 +2,18 @@
2
  "repo_id": "hcisbmm/vial_insertion",
3
  "vocab_size": 1024,
4
  "scale": 10.0,
5
- "encoded_dims": "7:11,11:14",
6
  "encoded_dim_ranges": [
7
  [
8
- 7,
9
- 11
10
  ],
11
  [
12
- 11,
13
  14
14
  ]
15
  ],
16
- "total_encoded_dims": 7,
17
  "delta_dims": null,
18
  "delta_dim_list": null,
19
  "use_delta_transform": false,
@@ -22,10 +22,10 @@
22
  "action_horizon": 10,
23
  "num_training_chunks": 1856,
24
  "compression_stats": {
25
- "compression_ratio": 7.15088364490755,
26
- "mean_token_length": 9.789,
27
- "p99_token_length": 22.0,
28
- "min_token_length": 2.0,
29
- "max_token_length": 29.0
30
  }
31
  }
 
2
  "repo_id": "hcisbmm/vial_insertion",
3
  "vocab_size": 1024,
4
  "scale": 10.0,
5
+ "encoded_dims": "0:7,7:14",
6
  "encoded_dim_ranges": [
7
  [
8
+ 0,
9
+ 7
10
  ],
11
  [
12
+ 7,
13
  14
14
  ]
15
  ],
16
+ "total_encoded_dims": 14,
17
  "delta_dims": null,
18
  "delta_dim_list": null,
19
  "use_delta_transform": false,
 
22
  "action_horizon": 10,
23
  "num_training_chunks": 1856,
24
  "compression_stats": {
25
+ "compression_ratio": 5.216289727635158,
26
+ "mean_token_length": 26.839,
27
+ "p99_token_length": 40.0,
28
+ "min_token_length": 7.0,
29
+ "max_token_length": 67.0
30
  }
31
  }