File size: 1,249 Bytes
6f22d12 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | {
"max_length": 512,
"context_candidates": [
null,
500,
300,
150,
80,
30,
0
],
"marker_tokens": [
"[ENT]",
"[/ENT]"
],
"marker_token_ids": {
"[ENT]": 65000,
"[/ENT]": 65001
},
"splits": {
"train": {
"rows": 100796,
"token_length_min": 5,
"token_length_mean": 53.74587285209731,
"token_length_median": 31.0,
"token_length_p95": 174.0,
"token_length_p99": 393.25000000001455,
"token_length_max": 512,
"context_counts": {
"None": 100590,
"500": 206
}
},
"validation": {
"rows": 12600,
"token_length_min": 5,
"token_length_mean": 52.443174603174604,
"token_length_median": 31.0,
"token_length_p95": 165.0,
"token_length_p99": 393.0,
"token_length_max": 512,
"context_counts": {
"None": 12570,
"500": 30
}
},
"test": {
"rows": 12600,
"token_length_min": 5,
"token_length_mean": 53.945238095238096,
"token_length_median": 31.0,
"token_length_p95": 178.0,
"token_length_p99": 409.0,
"token_length_max": 512,
"context_counts": {
"None": 12575,
"500": 25
}
}
}
} |