[
  {
    "dataset": "DODa",
    "tokenizer": "DarijaBERT-ar",
    "vocab_size": 80000,
    "source": "external",
    "fertility_ar": 0.0,
    "fertility_az": 2.648,
    "fertility_overall": 2.648,
    "cpt_ar": 0.0,
    "cpt_az": 2.194,
    "cpt_overall": 2.194,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "DODa",
    "tokenizer": "DarijaBERT-az",
    "vocab_size": 110000,
    "source": "external",
    "fertility_ar": 0.0,
    "fertility_az": 1.246,
    "fertility_overall": 1.246,
    "cpt_ar": 0.0,
    "cpt_az": 4.677,
    "cpt_overall": 4.677,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "DODa",
    "tokenizer": "DarijaBERT-mix",
    "vocab_size": 160000,
    "source": "external",
    "fertility_ar": 0.0,
    "fertility_az": 1.318,
    "fertility_overall": 1.318,
    "cpt_ar": 0.0,
    "cpt_az": 4.441,
    "cpt_overall": 4.441,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "DODa",
    "tokenizer": "Ours (80K WP)",
    "vocab_size": 80000,
    "source": "ours",
    "fertility_ar": 0.0,
    "fertility_az": 1.72,
    "fertility_overall": 1.72,
    "cpt_ar": 0.0,
    "cpt_az": 3.467,
    "cpt_overall": 3.467,
    "gain_pct": 35.0,
    "n_texts": 10000
  },
  {
    "dataset": "DODa",
    "tokenizer": "Ours (110K WP)",
    "vocab_size": 110000,
    "source": "ours",
    "fertility_ar": 0.0,
    "fertility_az": 1.68,
    "fertility_overall": 1.68,
    "cpt_ar": 0.0,
    "cpt_az": 3.551,
    "cpt_overall": 3.551,
    "gain_pct": -34.8,
    "n_texts": 10000
  },
  {
    "dataset": "DODa",
    "tokenizer": "Ours (32K BPE)",
    "vocab_size": 32000,
    "source": "ours",
    "fertility_ar": 0.0,
    "fertility_az": 1.901,
    "fertility_overall": 1.901,
    "cpt_ar": 0.0,
    "cpt_az": 3.145,
    "cpt_overall": 3.145,
    "gain_pct": -44.2,
    "n_texts": 10000
  },
  {
    "dataset": "Darija-Wiki",
    "tokenizer": "DarijaBERT-ar",
    "vocab_size": 80000,
    "source": "external",
    "fertility_ar": 1.56,
    "fertility_az": 2.502,
    "fertility_overall": 1.562,
    "cpt_ar": 3.382,
    "cpt_az": 2.589,
    "cpt_overall": 3.381,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "Darija-Wiki",
    "tokenizer": "DarijaBERT-az",
    "vocab_size": 110000,
    "source": "external",
    "fertility_ar": 1.858,
    "fertility_az": 2.219,
    "fertility_overall": 1.858,
    "cpt_ar": 2.829,
    "cpt_az": 2.946,
    "cpt_overall": 2.829,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "Darija-Wiki",
    "tokenizer": "DarijaBERT-mix",
    "vocab_size": 160000,
    "source": "external",
    "fertility_ar": 1.502,
    "fertility_az": 2.142,
    "fertility_overall": 1.503,
    "cpt_ar": 3.512,
    "cpt_az": 3.04,
    "cpt_overall": 3.511,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "Darija-Wiki",
    "tokenizer": "Ours (80K WP)",
    "vocab_size": 80000,
    "source": "ours",
    "fertility_ar": 1.839,
    "fertility_az": 3.265,
    "fertility_overall": 1.841,
    "cpt_ar": 2.883,
    "cpt_az": 2.07,
    "cpt_overall": 2.882,
    "gain_pct": -17.9,
    "n_texts": 10000
  },
  {
    "dataset": "Darija-Wiki",
    "tokenizer": "Ours (110K WP)",
    "vocab_size": 110000,
    "source": "ours",
    "fertility_ar": 1.797,
    "fertility_az": 3.184,
    "fertility_overall": 1.799,
    "cpt_ar": 2.951,
    "cpt_az": 2.124,
    "cpt_overall": 2.95,
    "gain_pct": 3.2,
    "n_texts": 10000
  },
  {
    "dataset": "Darija-Wiki",
    "tokenizer": "Ours (32K BPE)",
    "vocab_size": 32000,
    "source": "ours",
    "fertility_ar": 2.102,
    "fertility_az": 3.805,
    "fertility_overall": 2.104,
    "cpt_ar": 2.519,
    "cpt_az": 1.743,
    "cpt_overall": 2.518,
    "gain_pct": -40.0,
    "n_texts": 10000
  },
  {
    "dataset": "Atlaset",
    "tokenizer": "DarijaBERT-ar",
    "vocab_size": 80000,
    "source": "external",
    "fertility_ar": 1.407,
    "fertility_az": 2.056,
    "fertility_overall": 1.41,
    "cpt_ar": 4.051,
    "cpt_az": 1.803,
    "cpt_overall": 4.041,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "Atlaset",
    "tokenizer": "DarijaBERT-az",
    "vocab_size": 110000,
    "source": "external",
    "fertility_ar": 1.772,
    "fertility_az": 1.918,
    "fertility_overall": 1.773,
    "cpt_ar": 3.22,
    "cpt_az": 1.965,
    "cpt_overall": 3.215,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "Atlaset",
    "tokenizer": "DarijaBERT-mix",
    "vocab_size": 160000,
    "source": "external",
    "fertility_ar": 1.379,
    "fertility_az": 1.922,
    "fertility_overall": 1.381,
    "cpt_ar": 4.131,
    "cpt_az": 1.963,
    "cpt_overall": 4.122,
    "gain_pct": 0.0,
    "n_texts": 10000
  },
  {
    "dataset": "Atlaset",
    "tokenizer": "Ours (80K WP)",
    "vocab_size": 80000,
    "source": "ours",
    "fertility_ar": 1.618,
    "fertility_az": 2.293,
    "fertility_overall": 1.621,
    "cpt_ar": 3.594,
    "cpt_az": 1.771,
    "cpt_overall": 3.586,
    "gain_pct": -15.0,
    "n_texts": 10000
  },
  {
    "dataset": "Atlaset",
    "tokenizer": "Ours (110K WP)",
    "vocab_size": 110000,
    "source": "ours",
    "fertility_ar": 1.566,
    "fertility_az": 2.183,
    "fertility_overall": 1.569,
    "cpt_ar": 3.714,
    "cpt_az": 1.846,
    "cpt_overall": 3.706,
    "gain_pct": 11.5,
    "n_texts": 10000
  },
  {
    "dataset": "Atlaset",
    "tokenizer": "Ours (32K BPE)",
    "vocab_size": 32000,
    "source": "ours",
    "fertility_ar": 1.864,
    "fertility_az": 2.571,
    "fertility_overall": 1.867,
    "cpt_ar": 3.119,
    "cpt_az": 1.492,
    "cpt_overall": 3.112,
    "gain_pct": -35.2,
    "n_texts": 10000
  }
]