{
  "max_length": 512,
  "context_candidates": [
    null,
    500,
    300,
    150,
    80,
    30,
    0
  ],
  "marker_tokens": [
    "[ENT]",
    "[/ENT]"
  ],
  "marker_token_ids": {
    "[ENT]": 65000,
    "[/ENT]": 65001
  },
  "splits": {
    "train": {
      "rows": 100796,
      "token_length_min": 5,
      "token_length_mean": 53.74587285209731,
      "token_length_median": 31.0,
      "token_length_p95": 174.0,
      "token_length_p99": 393.25000000001455,
      "token_length_max": 512,
      "context_counts": {
        "None": 100590,
        "500": 206
      }
    },
    "validation": {
      "rows": 12600,
      "token_length_min": 5,
      "token_length_mean": 52.443174603174604,
      "token_length_median": 31.0,
      "token_length_p95": 165.0,
      "token_length_p99": 393.0,
      "token_length_max": 512,
      "context_counts": {
        "None": 12570,
        "500": 30
      }
    },
    "test": {
      "rows": 12600,
      "token_length_min": 5,
      "token_length_mean": 53.945238095238096,
      "token_length_median": 31.0,
      "token_length_p95": 178.0,
      "token_length_p99": 409.0,
      "token_length_max": 512,
      "context_counts": {
        "None": 12575,
        "500": 25
      }
    }
  }
}