token_analysis.json

{
  "text": "思考1.234和1.435哪个更大\n",
  "tokens": [
    {
      "id": 1,
      "token": "<s>"
    },
    {
      "id": 29871,
      "token": ""
    },
    {
      "id": 31579,
      "token": "思"
    },
    {
      "id": 235,
      "token": "�"
    },
    {
      "id": 131,
      "token": "�"
    },
    {
      "id": 134,
      "token": "�"
    },
    {
      "id": 29896,
      "token": "1"
    },
    {
      "id": 29889,
      "token": "."
    },
    {
      "id": 29906,
      "token": "2"
    },
    {
      "id": 29941,
      "token": "3"
    },
    {
      "id": 29946,
      "token": "4"
    },
    {
      "id": 30503,
      "token": "和"
    },
    {
      "id": 29896,
      "token": "1"
    },
    {
      "id": 29889,
      "token": "."
    },
    {
      "id": 29946,
      "token": "4"
    },
    {
      "id": 29941,
      "token": "3"
    },
    {
      "id": 29945,
      "token": "5"
    },
    {
      "id": 232,
      "token": "�"
    },
    {
      "id": 150,
      "token": "�"
    },
    {
      "id": 173,
      "token": "�"
    },
    {
      "id": 30502,
      "token": "个"
    },
    {
      "id": 31100,
      "token": "更"
    },
    {
      "id": 30257,
      "token": "大"
    },
    {
      "id": 13,
      "token": "\n"
    }
  ]
}