| { |
| "version": "1.0", |
| "truncation": null, |
| "padding": null, |
| "added_tokens": [ |
| { |
| "id": 0, |
| "content": "[PAD]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 1, |
| "content": "[SOS]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 2, |
| "content": "[EOS]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 3, |
| "content": "[UNK]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 4, |
| "content": "[MSK]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| }, |
| { |
| "id": 5, |
| "content": "[SEP]", |
| "single_word": false, |
| "lstrip": false, |
| "rstrip": false, |
| "normalized": false, |
| "special": true |
| } |
| ], |
| "normalizer": { |
| "type": "Sequence", |
| "normalizers": [ |
| { |
| "type": "NFD" |
| }, |
| { |
| "type": "StripAccents" |
| }, |
| { |
| "type": "Strip", |
| "strip_left": true, |
| "strip_right": true |
| } |
| ] |
| }, |
| "pre_tokenizer": { |
| "type": "Whitespace" |
| }, |
| "post_processor": null, |
| "decoder": null, |
| "model": { |
| "type": "WordLevel", |
| "vocab": { |
| "[PAD]": 0, |
| "[SOS]": 1, |
| "[EOS]": 2, |
| "[UNK]": 3, |
| "[MSK]": 4, |
| "[SEP]": 5, |
| "expression_statement__": 6, |
| "__expression_statement": 7, |
| "assignment": 8, |
| "__if_statement": 9, |
| "if_statement__": 10, |
| "expression_statement": 11, |
| "return_statement": 12, |
| "__parenthesized_expression": 13, |
| "parenthesized_expression__": 14, |
| "binary_expression": 15, |
| "binary_expression__": 16, |
| "__binary_expression": 17, |
| "__call_expression": 18, |
| "call_expression__": 19, |
| "assignment_expression__": 20, |
| "__assignment_expression": 21, |
| "member_expression": 22, |
| "selector_expression": 23, |
| "compound_statement__": 24, |
| "__compound_statement": 25, |
| "local_variable_declaration": 26, |
| "return_statement__": 27, |
| "__return_statement": 28, |
| "for_statement__": 29, |
| "__for_statement": 30, |
| "assignment_expression": 31, |
| "member_access_expression": 32, |
| "object_creation_expression": 33, |
| "call_statement": 34, |
| "member_call_expression": 35, |
| "subscript_expression": 36, |
| "local_variable_declaration__": 37, |
| "__local_variable_declaration": 38, |
| "member_call_expression__": 39, |
| "__member_call_expression": 40, |
| "parenthesized_expression": 41, |
| "__assignment_statement": 42, |
| "assignment_statement__": 43, |
| "try_statement__": 44, |
| "__try_statement": 45, |
| "call_expression": 46, |
| "unary_expression": 47, |
| "function_call_expression": 48, |
| "raise_statement": 49, |
| "__member_expression": 50, |
| "member_expression__": 51, |
| "function_call_expression__": 52, |
| "__function_call_expression": 53, |
| "__short_var_declaration": 54, |
| "short_var_declaration__": 55, |
| "subscript_expression__": 56, |
| "__subscript_expression": 57, |
| "call_statement__": 58, |
| "__call_statement": 59, |
| "array_creation_expression": 60, |
| "selector_expression__": 61, |
| "__selector_expression": 62, |
| "__throw_statement": 63, |
| "throw_statement__": 64, |
| "cast_expression": 65, |
| "update_expression": 66, |
| "object_creation_expression__": 67, |
| "__object_creation_expression": 68, |
| "augmented_assignment": 69, |
| "unary_expression__": 70, |
| "__unary_expression": 71, |
| "break_statement": 72, |
| "__unary_op_expression": 73, |
| "unary_op_expression__": 74, |
| "enhanced_for_statement__": 75, |
| "__enhanced_for_statement": 76, |
| "scoped_call_expression": 77, |
| "class_constant_access_expression": 78, |
| "assignment_statement": 79, |
| "while_statement__": 80, |
| "__while_statement": 81, |
| "if_statement": 82, |
| "continue_statement": 83, |
| "ternary_expression__": 84, |
| "__ternary_expression": 85, |
| "__foreach_statement": 86, |
| "foreach_statement__": 87, |
| "with_statement__": 88, |
| "__with_statement": 89, |
| "array_creation_expression__": 90, |
| "__array_creation_expression": 91, |
| "instanceof_expression": 92, |
| "throw_expression__": 93, |
| "__throw_expression": 94, |
| "assert_statement": 95, |
| "conditional_expression__": 96, |
| "__conditional_expression": 97, |
| "__scoped_call_expression": 98, |
| "scoped_call_expression__": 99, |
| "index_expression": 100, |
| "__member_access_expression": 101, |
| "member_access_expression__": 102, |
| "cast_expression__": 103, |
| "__cast_expression": 104, |
| "index_expression__": 105, |
| "__index_expression": 106, |
| "__augmented_assignment_expression": 107, |
| "augmented_assignment_expression__": 108, |
| "pass_statement": 109, |
| "scoped_property_access_expression": 110, |
| "new_expression": 111, |
| "import_from_statement": 112, |
| "unless_statement": 113, |
| "var_declaration": 114, |
| "else_statement": 115, |
| "__else_statement": 116, |
| "else_statement__": 117, |
| "method_declaration__": 118, |
| "__method_declaration": 119, |
| "new_expression__": 120, |
| "__new_expression": 121, |
| "unary_op_expression": 122, |
| "short_var_declaration": 123, |
| "__case_statement": 124, |
| "case_statement__": 125, |
| "__anonymous_function_creation_expression": 126, |
| "anonymous_function_creation_expression__": 127, |
| "augmented_assignment_expression": 128, |
| "__defer_statement": 129, |
| "defer_statement__": 130, |
| "delete_statement": 131, |
| "throw_statement": 132, |
| "parameter_declaration": 133, |
| "import_statement": 134, |
| "when_statement": 135, |
| "when_statement__": 136, |
| "__when_statement": 137, |
| "__lambda_expression": 138, |
| "lambda_expression__": 139, |
| "ternary_expression": 140, |
| "switch_expression__": 141, |
| "__switch_expression": 142, |
| "__synchronized_statement": 143, |
| "synchronized_statement__": 144, |
| "print_statement": 145, |
| "lambda_expression": 146, |
| "switch_statement__": 147, |
| "__switch_statement": 148, |
| "operator_assignment_statement": 149, |
| "type_assertion_expression__": 150, |
| "__type_assertion_expression": 151, |
| "rescue_statement": 152, |
| "__assert_statement": 153, |
| "assert_statement__": 154, |
| "for_in_statement__": 155, |
| "__for_in_statement": 156, |
| "__rescue_statement": 157, |
| "rescue_statement__": 158, |
| "__unset_statement": 159, |
| "unset_statement__": 160, |
| "type_assertion_expression": 161, |
| "__assignment": 162, |
| "assignment__": 163, |
| "__unless_statement": 164, |
| "unless_statement__": 165, |
| "expression_switch_statement__": 166, |
| "__expression_switch_statement": 167, |
| "inc_statement": 168, |
| "slice_expression": 169, |
| "__slice_expression": 170, |
| "slice_expression__": 171, |
| "__operator_assignment_statement": 172, |
| "operator_assignment_statement__": 173, |
| "__try_with_resources_statement": 174, |
| "try_with_resources_statement__": 175, |
| "global_statement": 176, |
| "receive_statement__": 177, |
| "__receive_statement": 178, |
| "field_declaration": 179, |
| "await_expression__": 180, |
| "__await_expression": 181, |
| "type_switch_statement__": 182, |
| "__type_switch_statement": 183, |
| "do_statement__": 184, |
| "__do_statement": 185, |
| "__go_statement": 186, |
| "go_statement__": 187, |
| "case_statement": 188, |
| "type_conversion_expression": 189, |
| "sequence_expression__": 190, |
| "__sequence_expression": 191, |
| "__update_expression": 192, |
| "update_expression__": 193, |
| "default_statement__": 194, |
| "__default_statement": 195, |
| "select_statement__": 196, |
| "__select_statement": 197, |
| "conditional_expression": 198, |
| "__send_statement": 199, |
| "send_statement__": 200, |
| "__echo_statement": 201, |
| "echo_statement__": 202, |
| "empty_statement": 203, |
| "throw_expression": 204, |
| "var_declaration__": 205, |
| "__var_declaration": 206, |
| "compound_statement": 207, |
| "__inc_statement": 208, |
| "inc_statement__": 209, |
| "__labeled_statement": 210, |
| "labeled_statement__": 211, |
| "jsx_expression": 212, |
| "echo_statement": 213, |
| "send_statement": 214, |
| "__type_conversion_expression": 215, |
| "type_conversion_expression__": 216, |
| "clone_expression": 217, |
| "unset_statement": 218, |
| "method_declaration": 219, |
| "yield_expression__": 220, |
| "__yield_expression": 221, |
| "dec_statement": 222, |
| "__jsx_expression": 223, |
| "jsx_expression__": 224, |
| "clone_expression__": 225, |
| "__clone_expression": 226, |
| "const_declaration__": 227, |
| "__const_declaration": 228, |
| "until_statement": 229, |
| "__require_once_expression": 230, |
| "require_once_expression__": 231, |
| "require_once_expression": 232, |
| "const_declaration": 233, |
| "field_declaration__": 234, |
| "__field_declaration": 235, |
| "include_expression__": 236, |
| "__include_expression": 237, |
| "__dec_statement": 238, |
| "dec_statement__": 239, |
| "__until_statement": 240, |
| "until_statement__": 241, |
| "goto_statement": 242, |
| "for_statement": 243, |
| "type_declaration": 244, |
| "__require_expression": 245, |
| "require_expression__": 246, |
| "nonlocal_statement": 247, |
| "yield_expression": 248, |
| "fallthrough_statement": 249, |
| "include_expression": 250, |
| "require_expression": 251, |
| "class_declaration__": 252, |
| "__class_declaration": 253, |
| "instanceof_expression__": 254, |
| "__instanceof_expression": 255, |
| "type_declaration__": 256, |
| "__type_declaration": 257, |
| "rest_assignment_statement": 258, |
| "variadic_parameter_declaration": 259, |
| "include_once_expression__": 260, |
| "__include_once_expression": 261, |
| "sequence_expression": 262, |
| "include_once_expression": 263, |
| "default_statement": 264, |
| "named_label_statement": 265, |
| "__exponentiation_expression": 266, |
| "exponentiation_expression__": 267, |
| "exponentiation_expression": 268, |
| "__constructor_declaration": 269, |
| "await_expression": 270, |
| "constructor_declaration__": 271, |
| "destructured_left_assignment_statement": 272, |
| "shell_command_expression": 273, |
| "enhanced_for_statement": 274, |
| "__class_constant_access_expression": 275, |
| "debugger_statement": 276, |
| "__parameter_declaration": 277, |
| "parameter_declaration__": 278, |
| "class_constant_access_expression__": 279, |
| "declare_statement": 280, |
| "constructor_declaration": 281, |
| "for_in_statement": 282, |
| "__continue_statement": 283, |
| "try_with_resources_statement": 284, |
| "continue_statement__": 285, |
| "select_statement": 286, |
| "try_statement": 287, |
| "labeled_statement": 288, |
| "expression_switch_statement": 289, |
| "augmented_assignment__": 290, |
| "scoped_property_access_expression__": 291, |
| "yield_statement__": 292, |
| "__yield_statement": 293, |
| "__scoped_property_access_expression": 294, |
| "__augmented_assignment": 295, |
| "with_statement": 296 |
| }, |
| "unk_token": "[UNK]" |
| } |
| } |
