{
  "tokenizer": "google/embeddinggemma-300m",
  "model_max_tokens": 2048,
  "corpora": {
    "independent_human": {
      "samples": 783,
      "raw_tokens": {
        "minimum": 1708,
        "p05": 1906,
        "median": 2094,
        "p95": 2258,
        "maximum": 3920
      },
      "effective_tokens": {
        "minimum": 1708,
        "p05": 1906,
        "median": 2048,
        "p95": 2048,
        "maximum": 2048
      },
      "truncated_fraction": 0.7075351213282248
    },
    "independent_ai": {
      "samples": 330,
      "raw_tokens": {
        "minimum": 891,
        "p05": 1271,
        "median": 1999,
        "p95": 2333,
        "maximum": 2425
      },
      "effective_tokens": {
        "minimum": 891,
        "p05": 1271,
        "median": 1999,
        "p95": 2048,
        "maximum": 2048
      },
      "truncated_fraction": 0.4636363636363636
    },
    "rewrite_human": {
      "samples": 1306,
      "raw_tokens": {
        "minimum": 377,
        "p05": 390,
        "median": 406,
        "p95": 430,
        "maximum": 785
      },
      "effective_tokens": {
        "minimum": 377,
        "p05": 390,
        "median": 406,
        "p95": 430,
        "maximum": 785
      },
      "truncated_fraction": 0.0
    },
    "rewrite_ai": {
      "samples": 1306,
      "raw_tokens": {
        "minimum": 240,
        "p05": 326,
        "median": 420,
        "p95": 514,
        "maximum": 680
      },
      "effective_tokens": {
        "minimum": 240,
        "p05": 326,
        "median": 420,
        "p95": 514,
        "maximum": 680
      },
      "truncated_fraction": 0.0
    },
    "sol_medium": {
      "samples": 30,
      "raw_tokens": {
        "minimum": 1737,
        "p05": 1974,
        "median": 2173,
        "p95": 2379,
        "maximum": 2474
      },
      "effective_tokens": {
        "minimum": 1737,
        "p05": 1974,
        "median": 2048,
        "p95": 2048,
        "maximum": 2048
      },
      "truncated_fraction": 0.8333333333333334
    }
  }
}