File size: 2,028 Bytes
f3d5a27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
{
  "tokenizer": "google/embeddinggemma-300m",
  "model_max_tokens": 2048,
  "corpora": {
    "independent_human": {
      "samples": 783,
      "raw_tokens": {
        "minimum": 1708,
        "p05": 1906,
        "median": 2094,
        "p95": 2258,
        "maximum": 3920
      },
      "effective_tokens": {
        "minimum": 1708,
        "p05": 1906,
        "median": 2048,
        "p95": 2048,
        "maximum": 2048
      },
      "truncated_fraction": 0.7075351213282248
    },
    "independent_ai": {
      "samples": 330,
      "raw_tokens": {
        "minimum": 891,
        "p05": 1271,
        "median": 1999,
        "p95": 2333,
        "maximum": 2425
      },
      "effective_tokens": {
        "minimum": 891,
        "p05": 1271,
        "median": 1999,
        "p95": 2048,
        "maximum": 2048
      },
      "truncated_fraction": 0.4636363636363636
    },
    "rewrite_human": {
      "samples": 1306,
      "raw_tokens": {
        "minimum": 377,
        "p05": 390,
        "median": 406,
        "p95": 430,
        "maximum": 785
      },
      "effective_tokens": {
        "minimum": 377,
        "p05": 390,
        "median": 406,
        "p95": 430,
        "maximum": 785
      },
      "truncated_fraction": 0.0
    },
    "rewrite_ai": {
      "samples": 1306,
      "raw_tokens": {
        "minimum": 240,
        "p05": 326,
        "median": 420,
        "p95": 514,
        "maximum": 680
      },
      "effective_tokens": {
        "minimum": 240,
        "p05": 326,
        "median": 420,
        "p95": 514,
        "maximum": 680
      },
      "truncated_fraction": 0.0
    },
    "sol_medium": {
      "samples": 30,
      "raw_tokens": {
        "minimum": 1737,
        "p05": 1974,
        "median": 2173,
        "p95": 2379,
        "maximum": 2474
      },
      "effective_tokens": {
        "minimum": 1737,
        "p05": 1974,
        "median": 2048,
        "p95": 2048,
        "maximum": 2048
      },
      "truncated_fraction": 0.8333333333333334
    }
  }
}