{
  "version": "0.3.8",
  "rust_available": true,
  "cpu_count": 128,
  "samples": 10000,
  "total_bytes": 4353919,
  "thread_counts": [
    1,
    2,
    4,
    8
  ],
  "runs": 3,
  "results": {
    "SARFTokenizer (HF)": {
      "name": "SARFTokenizer (HF)",
      "type": "sarf",
      "vocab_size": 64641,
      "benchmarks": [
        {
          "threads": 1,
          "time_sec": 1.2484327759593725,
          "texts_per_sec": 8010.04282534587,
          "bytes_per_sec": 3487507.7648087065,
          "mb_per_sec": 3.325946583565432
        },
        {
          "threads": 2,
          "time_sec": 0.24223483664294085,
          "texts_per_sec": 41282.25377731365,
          "bytes_per_sec": 17973958.908386767,
          "mb_per_sec": 17.141302975069777,
          "speedup": 5.1538118681070975
        },
        {
          "threads": 4,
          "time_sec": 0.25055884880324203,
          "texts_per_sec": 39910.78362533811,
          "bytes_per_sec": 17376831.91312485,
          "mb_per_sec": 16.57183829605565,
          "speedup": 4.982593039209473
        },
        {
          "threads": 8,
          "time_sec": 0.23175926196078458,
          "texts_per_sec": 43148.22163047825,
          "bytes_per_sec": 18786386.197315022,
          "mb_per_sec": 17.916094014468214,
          "speedup": 5.38676541078482
        }
      ],
      "memory": {
        "current_mb": 6.103515625e-05,
        "peak_mb": 0.2074756622314453,
        "texts_processed": 10000
      }
    },
    "SARFTokenizer (Local)": {
      "name": "SARFTokenizer (Local)",
      "type": "sarf",
      "vocab_size": 64641,
      "benchmarks": [
        {
          "threads": 1,
          "time_sec": 1.2962954323738813,
          "texts_per_sec": 7714.2908555090635,
          "bytes_per_sec": 3358739.7527327165,
          "mb_per_sec": 3.2031438376738706
        },
        {
          "threads": 2,
          "time_sec": 0.24298642265299955,
          "texts_per_sec": 41154.562838602105,
          "bytes_per_sec": 17918363.307968363,
          "mb_per_sec": 17.08828287884556,
          "speedup": 5.3348471829023785
        },
        {
          "threads": 4,
          "time_sec": 0.23486537175873914,
          "texts_per_sec": 42577.58359658189,
          "bytes_per_sec": 18537935.019524623,
          "mb_per_sec": 17.679152507328627,
          "speedup": 5.51931271377662
        },
        {
          "threads": 8,
          "time_sec": 0.24369121653338274,
          "texts_per_sec": 41035.53727645379,
          "bytes_per_sec": 17866540.542316042,
          "mb_per_sec": 17.038860838237802,
          "speedup": 5.319417953647519
        }
      ],
      "memory": {
        "current_mb": 6.103515625e-05,
        "peak_mb": 0.2074756622314453,
        "texts_processed": 10000
      }
    },
    "tiktoken (o200k)": {
      "name": "tiktoken (o200k)",
      "type": "tiktoken",
      "vocab_size": 200019,
      "benchmarks": [
        {
          "threads": 1,
          "time_sec": 0.46731498713294667,
          "texts_per_sec": 21398.842911826185,
          "bytes_per_sec": 9316882.873181535,
          "mb_per_sec": 8.885271905118499
        },
        {
          "threads": 2,
          "time_sec": 0.52791434712708,
          "texts_per_sec": 18942.466811936807,
          "bytes_per_sec": 8247396.61593611,
          "mb_per_sec": 7.865330329834089,
          "speedup": 0.8852098634486518
        },
        {
          "threads": 4,
          "time_sec": 0.48348489341636497,
          "texts_per_sec": 20683.169497476425,
          "bytes_per_sec": 9005284.465528306,
          "mb_per_sec": 8.58810850670653,
          "speedup": 0.966555508758175
        },
        {
          "threads": 8,
          "time_sec": 0.8265109478185574,
          "texts_per_sec": 12099.053287065815,
          "bytes_per_sec": 5267829.798856831,
          "mb_per_sec": 5.023793982369262,
          "speedup": 0.5654068931165998
        }
      ],
      "memory": {
        "current_mb": 0.0,
        "peak_mb": 0.21294403076171875,
        "texts_processed": 10000
      }
    },
    "tiktoken (cl100k)": {
      "name": "tiktoken (cl100k)",
      "type": "tiktoken",
      "vocab_size": 100277,
      "benchmarks": [
        {
          "threads": 1,
          "time_sec": 0.3374793001761039,
          "texts_per_sec": 29631.447009584845,
          "bytes_per_sec": 12901292.013252463,
          "mb_per_sec": 12.303630841495956
        },
        {
          "threads": 2,
          "time_sec": 0.5219240722556909,
          "texts_per_sec": 19159.875030828225,
          "bytes_per_sec": 8342054.393434861,
          "mb_per_sec": 7.955603021082745,
          "speedup": 0.6466061216865516
        },
        {
          "threads": 4,
          "time_sec": 0.514385993902882,
          "texts_per_sec": 19440.653747442506,
          "bytes_per_sec": 8464303.172341112,
          "mb_per_sec": 8.072188541737663,
          "speedup": 0.6560818221652849
        },
        {
          "threads": 8,
          "time_sec": 0.8330062093834082,
          "texts_per_sec": 12004.71243473924,
          "bytes_per_sec": 5226754.555914744,
          "mb_per_sec": 4.984621578135246,
          "speedup": 0.4051341951291171
        }
      ],
      "memory": {
        "current_mb": 0.0,
        "peak_mb": 0.4065704345703125,
        "texts_processed": 10000
      }
    }
  }
}