File size: 2,426 Bytes
74da6da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
{
  "add_prefix_space": false,
  "audio_end_token": "<longcat_audio_end>",
  "audio_pad_token": "<longcat_audio_pad>",
  "audio_start_token": "<longcat_audio_start>",
  "backend": "tokenizers",
  "bos_token": "<longcat_s>",
  "clean_up_tokenization_spaces": false,
  "eos_token": "</longcat_s>",
  "extra_special_tokens": [
    "<mask_131048>",
    "<mask_131049>",
    "<mask_131050>",
    "<mask_131051>",
    "<mask_131052>",
    "<mask_131053>",
    "<mask_131054>",
    "<mask_131055>",
    "<mask_131056>",
    "<mask_131057>",
    "<mask_131058>",
    "<mask_131059>",
    "<mask_131060>",
    "<mask_131061>",
    "<mask_131062>",
    "<mask_131063>",
    "<mask_131064>",
    "<mask_131065>",
    "<longcat_img_token_size>",
    "</longcat_img_token_size>",
    "<mask_131068>",
    "<mask_131069>",
    "<mask_131070>",
    "<mask_131071>",
    "<longcat_point_start>",
    "<longcat_point_end>",
    "<longcat_point_delim>",
    "<longcat_polygon_start>",
    "<longcat_polygon_end>",
    "<mask_131077>",
    "<mask_131078>",
    "<longcat_audio_start>",
    "<longcat_audio_end>",
    "<longcat_audio_pad>",
    "<longcat_img_start>",
    "<longcat_img_end>",
    "<longcat_img_pad>",
    "<longcat_img_newline>",
    "<longcat_box_start>",
    "<longcat_box_end>",
    "<longcat_box_delim>",
    "<longcat_ref_start>",
    "<longcat_ref_end>",
    "<longcat_img_delim>",
    "<longcat_audio_delim>",
    "<longcat_video_palce>",
    "<longcat_video_start>",
    "<longcat_video_end>",
    "<longcat_audiotext_start>",
    "<longcat_audiotext_end>",
    "<longcat_audiotext_pad>",
    "<longcat_audiogen_start>",
    "<longcat_audiogen_end>"
  ],
  "image_end_token": "<longcat_img_end>",
  "image_newline_token": "<longcat_img_newline>",
  "image_pad_token": "<longcat_img_pad>",
  "image_start_token": "<longcat_img_start>",
  "is_local": true,
  "model_max_length": 131072,
  "model_specific_special_tokens": {
    "audio_end_token": "<longcat_audio_end>",
    "audio_pad_token": "<longcat_audio_pad>",
    "audio_start_token": "<longcat_audio_start>",
    "image_end_token": "<longcat_img_end>",
    "image_newline_token": "<longcat_img_newline>",
    "image_pad_token": "<longcat_img_pad>",
    "image_start_token": "<longcat_img_start>"
  },
  "pad_token": "<longcat_pad>",
  "sp_model_kwargs": {},
  "tokenizer_class": "TokenizersBackend",
  "tool_parser_type": "longcat",
  "unk_token": "<longcat_unk>"
}