metadata
dataset_info:
features:
- name: text
dtype: string
- name: tokens
sequence: int64
- name: token_count
dtype: int64
splits:
- name: train
num_bytes: 167968257.38066393
num_examples: 69445
- name: test
num_bytes: 1726968.6193360796
num_examples: 714
download_size: 49543706
dataset_size: 169695226
configs:
- config_name: default
data_files:
- split: train
path: data/train-*
- split: test
path: data/test-*