diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..573c41fc61ac38851eb1366a98ab2a7eef60fb14 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +shared/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text +variants/cpu-int4/inference/frozen_base.onnx.data filter=lfs diff=lfs merge=lfs -text +variants/cpu-int4/inference/model.onnx_data filter=lfs diff=lfs merge=lfs -text +variants/cpu-int4/inference/tokenizer.json filter=lfs diff=lfs merge=lfs -text +variants/cpu-int4/train/checkpoint filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ca5b62f6a0f7595a584e62bcc0fec36bd375693f --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +--- +base_model: google/functiongemma-270m-it +library_name: mobiletransformers +pipeline_tag: text-generation +tags: + - mobiletransformers + - onnx + - on-device + - android + - lora + - int4 +--- + +# google/functiongemma-270m-it — MobileTransformers package + +On-device (Android) package exported from **google/functiongemma-270m-it** with MobileTransformers. + +## Provenance +- Base model: `google/functiongemma-270m-it` +- Selected task: `text-generation-with-past` +- PEFT methods: lora +- Quantization: int4 +- Toolchain: optimum-onnx 0.1.0, transformers 4.57.6, ort-training 1.23.0+cpu, ort-genai None + +## Licenses +- Framework: not declared in this package — see the repository +- Base model weights: see the base model above (this package redistributes an export of those weights, so their terms govern its contents) + +## Android runtime +- Minimum API: 28 +- Recommended device memory (MB): None +- Required ABIs: any + +## Variants + +| id | EP | quant | engines | features | min API | rec. RAM (MB) | +| --- | --- | --- | --- | --- | --- | --- | +| cpu-int4 | cpu | int4 | native | core, inference, train | 28 | None | + +Default variant: `cpu-int4`. diff --git a/mobiletransformers_manifest.json b/mobiletransformers_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..417ccc2bb3ac7f93f0eac666808a406f2aeb1d1b --- /dev/null +++ b/mobiletransformers_manifest.json @@ -0,0 +1,293 @@ +{ + "androidRuntime": { + "minimumAndroidApi": 28, + "recommendedDeviceMemoryMb": null, + "requiredAbis": [] + }, + "architectures": [ + "gemma3_text" + ], + "artifactFormatVersion": 1, + "baseModelId": "google/functiongemma-270m-it", + "defaultVariant": "cpu-int4", + "downloadPlan": { + "cpu-int4": { + "checksums": [ + "variants/cpu-int4/checksums.json" + ], + "core": [ + "mobiletransformers_manifest.json", + "shared/tokenizer/**", + "shared/chat_template.jinja", + "shared/config.json", + "shared/generation_config.json" + ], + "genai": [], + "inference": [ + "variants/cpu-int4/inference/**" + ], + "rag": [], + "train": [ + "variants/cpu-int4/train/**" + ] + } + }, + "exportedAt": "", + "fileSizes": { + "optimum/export_report.json": 822, + "optimum/supported_tasks.json": 34, + "shared/tokenizer/chat_template.jinja": 13792, + "shared/tokenizer/mobiletransformers_tokenizer_config.json": 356, + "shared/tokenizer/special_tokens_map.json": 706, + "shared/tokenizer/tokenizer.json": 33384899, + "shared/tokenizer/tokenizer_config.json": 1155714, + "variants/cpu-int4/checksums.json": 13424, + "variants/cpu-int4/inference/chat_template.jinja": 13792, + "variants/cpu-int4/inference/config.json": 1358, + "variants/cpu-int4/inference/frozen_base.onnx.data": 1684498944, + "variants/cpu-int4/inference/genai_config.json": 1044, + "variants/cpu-int4/inference/generation_config.json": 176, + "variants/cpu-int4/inference/merger_lora_fpin_fpout.onnx": 811, + "variants/cpu-int4/inference/model.layers.0.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.0.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.0.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.0.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.1.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.1.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.1.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.1.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.10.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.10.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.10.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.10.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.11.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.11.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.11.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.11.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.12.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.12.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.12.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.12.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.13.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.13.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.13.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.13.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.14.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.14.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.14.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.14.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.15.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.15.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.15.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.15.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.16.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.16.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.16.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.16.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.17.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.17.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.17.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.17.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.2.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.2.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.2.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.2.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.3.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.3.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.3.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.3.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.4.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.4.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.4.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.4.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.5.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.5.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.5.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.5.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.6.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.6.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.6.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.6.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.7.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.7.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.7.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.7.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.8.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.8.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.8.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.8.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.9.self_attn.q_proj.MatMul.weight.bin": 2621440, + "variants/cpu-int4/inference/model.layers.9.self_attn.q_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.layers.9.self_attn.v_proj.MatMul.weight.bin": 655360, + "variants/cpu-int4/inference/model.layers.9.self_attn.v_proj.MatMul.weight.bin.sha256": 65, + "variants/cpu-int4/inference/model.onnx": 920639, + "variants/cpu-int4/inference/model.onnx_data": 1743481344, + "variants/cpu-int4/inference/optimum_config.json": 334, + "variants/cpu-int4/inference/special_tokens_map.json": 706, + "variants/cpu-int4/inference/tokenizer.json": 33384899, + "variants/cpu-int4/inference/tokenizer_config.json": 1155714, + "variants/cpu-int4/inference/weight_handoff_map.json": 53841, + "variants/cpu-int4/train/checkpoint": 314028424, + "variants/cpu-int4/train/eval_model.onnx": 1901828, + "variants/cpu-int4/train/optimizer_model.onnx": 538, + "variants/cpu-int4/train/trainable_parameters.json": 7011, + "variants/cpu-int4/train/training_config.json": 19841, + "variants/cpu-int4/train/training_model.onnx": 1688100 + }, + "license": { + "baseModelWeights": null, + "framework": null, + "noticeFile": null + }, + "minReaderVersion": "1.0", + "mobiletransformersVersion": "0.1.0", + "onnxRuntimeGenAIVersion": null, + "onnxRuntimeTrainingVersion": "1.23.0+cpu", + "optimumOnnxVersion": "0.1.0", + "peftMethods": [ + "lora" + ], + "quantization": [ + "int4" + ], + "requiredFiles": [ + "mobiletransformers_manifest.json", + "shared/tokenizer/tokenizer.json", + "variants/cpu-int4/inference/model.onnx" + ], + "schemaVersion": "1.0", + "selectedTask": "text-generation-with-past", + "sha256": { + "optimum/export_report.json": "edc6a33e6199cbc47846e57f770d37b56822796cda4df3ebfe51eeb23dbec046", + "optimum/supported_tasks.json": "2591d52d3133d5bd71e384d09e57f48d29e84f4e9a358697595bb4bcb56dcfdf", + "shared/tokenizer/chat_template.jinja": "db61fb01017bd82401d3ffca4f8e066cd56ff6d38d2a30c4058770c0bf7ab49b", + "shared/tokenizer/mobiletransformers_tokenizer_config.json": "2b8628a7ad20677fd280311678611f19cf97860938f18ee678083cc0e550c668", + "shared/tokenizer/special_tokens_map.json": "d39df6f8ddb132a622394f05a603b166ba5f22d485483e30d06c1821ef4ac9b4", + "shared/tokenizer/tokenizer.json": "b6b09a0b4a803ad453063ca4bb49a784540e8120004e2450e025df2b27d41fb2", + "shared/tokenizer/tokenizer_config.json": "c33878647bdc69ce85d4b9aeef953815afd8405f8d85001ab0bbb89aa14d2554", + "variants/cpu-int4/checksums.json": "2fb0f6eee9752e511aab5cea2529e31a219233d36635a9458d2f328d512515be", + "variants/cpu-int4/inference/chat_template.jinja": "db61fb01017bd82401d3ffca4f8e066cd56ff6d38d2a30c4058770c0bf7ab49b", + "variants/cpu-int4/inference/config.json": "88a795b85d50b1642e66193935a64c34326fa76efe107f9fceeafd31aa3e2af9", + "variants/cpu-int4/inference/frozen_base.onnx.data": "9c14b3237169fd6023199017e1949b02bbfe0ef1e80ac10c2d72bba670ebfa82", + "variants/cpu-int4/inference/genai_config.json": "acac04545d607a84f676049824d2e113f9367179741494a46279551b438c6efc", + "variants/cpu-int4/inference/generation_config.json": "90f3b346c07b98c000b6de297e805a83f504d6856b01c405a746d805c05444c2", + "variants/cpu-int4/inference/merger_lora_fpin_fpout.onnx": "953a8ad4704235c19ec06150788107921ba7f2578a5880a32cd59b473544712c", + "variants/cpu-int4/inference/model.layers.0.self_attn.q_proj.MatMul.weight.bin": "de5441ec8bb19c3a0038e7f3bd3cb25d9b81599d1e6977ceecdc1d894ede2fa1", + "variants/cpu-int4/inference/model.layers.0.self_attn.q_proj.MatMul.weight.bin.sha256": "b303f006c606265b707027dccaa1c22f9f19b7920a8d51e442cbcfde886d3b65", + "variants/cpu-int4/inference/model.layers.0.self_attn.v_proj.MatMul.weight.bin": "aced1f8faec6416a6cbee9227400021af9b1f4c0d25feddc54e255f1210575fb", + "variants/cpu-int4/inference/model.layers.0.self_attn.v_proj.MatMul.weight.bin.sha256": "e41d2d8d47eaea11a011971b193d1ea7f15222a23dc2c2d7342c983d5fad5d58", + "variants/cpu-int4/inference/model.layers.1.self_attn.q_proj.MatMul.weight.bin": "cc0c039a777068e664ea4916413b822ce497007dc31f22f7e7cc27f9ed0bd820", + "variants/cpu-int4/inference/model.layers.1.self_attn.q_proj.MatMul.weight.bin.sha256": "1e1651baa468edd1ec0a787271c362646bcf4669d06816750fb9907bda6161ed", + "variants/cpu-int4/inference/model.layers.1.self_attn.v_proj.MatMul.weight.bin": "e2cde258f00749809d3b5e6a3838dbccb866dfdc228c86e5375a1c3fb3be14b2", + "variants/cpu-int4/inference/model.layers.1.self_attn.v_proj.MatMul.weight.bin.sha256": "d670153828a24efe77701e8491f5bc182fd88c9f27b924c750b9c20ca4fbf89c", + "variants/cpu-int4/inference/model.layers.10.self_attn.q_proj.MatMul.weight.bin": "864537f8b063185411edf60cc77bb704f3ddd3d39b6f19f04bd0da2bf1ba5b5e", + "variants/cpu-int4/inference/model.layers.10.self_attn.q_proj.MatMul.weight.bin.sha256": "286bfb981d94d988671f4572b770728b9a285f400ec4edc6ad83c9ea499d2c40", + "variants/cpu-int4/inference/model.layers.10.self_attn.v_proj.MatMul.weight.bin": "f8aedd8998857b74a4e93e3182911873cd9ad547f8f0ed8212856e8f0beaef00", + "variants/cpu-int4/inference/model.layers.10.self_attn.v_proj.MatMul.weight.bin.sha256": "528f124d7e9657be8b57303ec6248483bb0c6dde4c84084575e3c9afd0b8188b", + "variants/cpu-int4/inference/model.layers.11.self_attn.q_proj.MatMul.weight.bin": "534363c1244562ba5cc4043f27b4da97fa3a48dc2a60772532816b5f11865b91", + "variants/cpu-int4/inference/model.layers.11.self_attn.q_proj.MatMul.weight.bin.sha256": "19a221e71a3a79d359f1168ea1a3f88003340820f8ecffae5f5472d71be6700c", + "variants/cpu-int4/inference/model.layers.11.self_attn.v_proj.MatMul.weight.bin": "9b3cc8d43d80eae0f674534447d7845b17a4c779fa74adb1de07254f65de03f0", + "variants/cpu-int4/inference/model.layers.11.self_attn.v_proj.MatMul.weight.bin.sha256": "643bbdcb8d0381f20eaf58b1d8d9ff7c3f6435ec71d5ee7d4a93d7b60d541a83", + "variants/cpu-int4/inference/model.layers.12.self_attn.q_proj.MatMul.weight.bin": "5213011869521ddedba25e37944d311b30632ec3b2c061caeb6671d4f4b9596f", + "variants/cpu-int4/inference/model.layers.12.self_attn.q_proj.MatMul.weight.bin.sha256": "92e8c3fb1b76481b06c2ff2dd3ae3a99ad52477b274511b4ba3944dc9f376858", + "variants/cpu-int4/inference/model.layers.12.self_attn.v_proj.MatMul.weight.bin": "8dfd86b40b9e613538f07c8297cf3fb54cb3ebf99fe4918c30c8c51bea4da5c0", + "variants/cpu-int4/inference/model.layers.12.self_attn.v_proj.MatMul.weight.bin.sha256": "ca5d5b8d2723dcb8a31ed895ebd6f72d3111de86981d1f10a57fd64d393a5ceb", + "variants/cpu-int4/inference/model.layers.13.self_attn.q_proj.MatMul.weight.bin": "3e58dbbd1a83c01943979052519976f15dd2e13b4502fe72edf522f86960d618", + "variants/cpu-int4/inference/model.layers.13.self_attn.q_proj.MatMul.weight.bin.sha256": "8a75590cded9ed4eb8518b18d4f5694fd4131d8ef225bfff9d148412716fe7c6", + "variants/cpu-int4/inference/model.layers.13.self_attn.v_proj.MatMul.weight.bin": "21666b649c20492830d5f37db41e2ba54d270228c20a532eea2104fa3c61396e", + "variants/cpu-int4/inference/model.layers.13.self_attn.v_proj.MatMul.weight.bin.sha256": "327d2d9dd53b71a1662bb5751d554ef719aff537095da4bd8ac2aaaea47de555", + "variants/cpu-int4/inference/model.layers.14.self_attn.q_proj.MatMul.weight.bin": "ef8d82392a5ac5cf37f65c638a965c7c949111ec783b0c947ecfc8d261e1ad9f", + "variants/cpu-int4/inference/model.layers.14.self_attn.q_proj.MatMul.weight.bin.sha256": "c183ea62d344b6331a35070bdac7091347be27f629a00a3569acf40f01c87dd2", + "variants/cpu-int4/inference/model.layers.14.self_attn.v_proj.MatMul.weight.bin": "c1c727d66921e1dceec4142a2a2c36254189234bd90b09db4125927e70b8e458", + "variants/cpu-int4/inference/model.layers.14.self_attn.v_proj.MatMul.weight.bin.sha256": "75c2d4a4ac3606a197b259bb309ada6a55341a4c3c3e68ab2b5d29985699cd88", + "variants/cpu-int4/inference/model.layers.15.self_attn.q_proj.MatMul.weight.bin": "31f7849f39f1b1fe0d7ef163aae56279a8f48eac0ec81dceae7dd7fda77dd293", + "variants/cpu-int4/inference/model.layers.15.self_attn.q_proj.MatMul.weight.bin.sha256": "c314704b435abe1cd3ba672eb1a408e7c6f7c808a215bfb7938d27f5c4583dbe", + "variants/cpu-int4/inference/model.layers.15.self_attn.v_proj.MatMul.weight.bin": "8f5878be79b76139fc9cc5724e91074c742fffd021b2b7bd6f0406775ec69365", + "variants/cpu-int4/inference/model.layers.15.self_attn.v_proj.MatMul.weight.bin.sha256": "17e8852305a2dd0be3788629535015c3ba347bd8a84a318c1a2740846ea656d3", + "variants/cpu-int4/inference/model.layers.16.self_attn.q_proj.MatMul.weight.bin": "cc87b80c44df0b66e627944acd40e87a0b63c77a86d1cfa55085dad5c5610de7", + "variants/cpu-int4/inference/model.layers.16.self_attn.q_proj.MatMul.weight.bin.sha256": "331eece77b3bc5bea26c224ee395a6c864e2e98667cd6bd0ef8a20c48aeaa27c", + "variants/cpu-int4/inference/model.layers.16.self_attn.v_proj.MatMul.weight.bin": "0c1f91d46f8f885ca1970f61ada3e69fe7e2a44c4f103beb456e84fb0f39cb13", + "variants/cpu-int4/inference/model.layers.16.self_attn.v_proj.MatMul.weight.bin.sha256": "c25e491df56cb7902bc53dfe2fdc5f69cc6c94d56a73a1e354920ace0cdedf8c", + "variants/cpu-int4/inference/model.layers.17.self_attn.q_proj.MatMul.weight.bin": "75aa1f482c3a16420a092f912a073abc981fc584038eead4fc4dad219c4a18df", + "variants/cpu-int4/inference/model.layers.17.self_attn.q_proj.MatMul.weight.bin.sha256": "5dccc067c68edcfb2aafd0ecfe966965f274afc3455699e6add6a2f4b3c0151f", + "variants/cpu-int4/inference/model.layers.17.self_attn.v_proj.MatMul.weight.bin": "c331cf9885fc9a470111e2e8e59018a2ffa235ee1441aa005789e321aee80df3", + "variants/cpu-int4/inference/model.layers.17.self_attn.v_proj.MatMul.weight.bin.sha256": "b3058d47101dd69928aa1b6e78574dd47e8a9eedda003972accac1fcbc50c53f", + "variants/cpu-int4/inference/model.layers.2.self_attn.q_proj.MatMul.weight.bin": "0aab3b51d26366fda266a9a030b64fa0097a3c9fe0f2ac8b45cd455ed6d00844", + "variants/cpu-int4/inference/model.layers.2.self_attn.q_proj.MatMul.weight.bin.sha256": "0c44bd5f5b21e9b5292a13d4069c6b313c9b9aa80881c3b469ab0034c0bedd1e", + "variants/cpu-int4/inference/model.layers.2.self_attn.v_proj.MatMul.weight.bin": "f995a03e11e45e51c9038b610940e2388dea7f89b186c9c32bf0774c7ad56676", + "variants/cpu-int4/inference/model.layers.2.self_attn.v_proj.MatMul.weight.bin.sha256": "b7f5aa1ce6d2e1be556f65fb6ca619e7591e63dd975534dae239e71290ac5a7b", + "variants/cpu-int4/inference/model.layers.3.self_attn.q_proj.MatMul.weight.bin": "5f256353f99cd60325cd88345d8095a1d8bf3ec2b2ed8b7dab08368018fd8335", + "variants/cpu-int4/inference/model.layers.3.self_attn.q_proj.MatMul.weight.bin.sha256": "00990e7e4ec37977e4c3024989b9dedab6f8c94a76755aa59ff21bdde5eae41e", + "variants/cpu-int4/inference/model.layers.3.self_attn.v_proj.MatMul.weight.bin": "7412ad9549d423c2cfbe4922299f5759b2e30e6839ef89cf99f8880ea8872ee5", + "variants/cpu-int4/inference/model.layers.3.self_attn.v_proj.MatMul.weight.bin.sha256": "e832cd832df1e7c9249d968196ff8da050eb1f4bb733e6ed1f3df80988037bad", + "variants/cpu-int4/inference/model.layers.4.self_attn.q_proj.MatMul.weight.bin": "ad0a0d5c5db67a43ea9793d34d5ef09e081772e8e5fba2b771ee95983ecda45a", + "variants/cpu-int4/inference/model.layers.4.self_attn.q_proj.MatMul.weight.bin.sha256": "6baba6d67fea470291b3da65d1cf9834f6e103b9a6845c8d753c958a2d873b5e", + "variants/cpu-int4/inference/model.layers.4.self_attn.v_proj.MatMul.weight.bin": "f59967d4272b6bd14e86c0cdc9fed9f1bcb21f67faa8a1d769935f710247f1b3", + "variants/cpu-int4/inference/model.layers.4.self_attn.v_proj.MatMul.weight.bin.sha256": "294e36d47d7ea2796ec74012dba0d6e672eaade5d8fed5c4fc15885d735fd35a", + "variants/cpu-int4/inference/model.layers.5.self_attn.q_proj.MatMul.weight.bin": "ff1c5fe943f8959a8b9d09640aec44d881f035f771416ec88ac929e5d4fe6b4b", + "variants/cpu-int4/inference/model.layers.5.self_attn.q_proj.MatMul.weight.bin.sha256": "aa4e396ee86ecd0b1538e96230fc842659e7ec66c927493682fa914efce3047b", + "variants/cpu-int4/inference/model.layers.5.self_attn.v_proj.MatMul.weight.bin": "4ee8c2d65e018f46002157c99b0fd025cfc523e03b07ca6fe26da71659f51c70", + "variants/cpu-int4/inference/model.layers.5.self_attn.v_proj.MatMul.weight.bin.sha256": "31352189895dd2e1d92b88721c0950070cdd2caa8c68c8cae31233825ec5b667", + "variants/cpu-int4/inference/model.layers.6.self_attn.q_proj.MatMul.weight.bin": "353d2ac53c60dfce33c57e501010ec0dac799d9050617cf6f9f818d37e1a2296", + "variants/cpu-int4/inference/model.layers.6.self_attn.q_proj.MatMul.weight.bin.sha256": "e869d030595d5ed13ac046acfc08fd9487b3a9ba9e940edbb4724eba90b84167", + "variants/cpu-int4/inference/model.layers.6.self_attn.v_proj.MatMul.weight.bin": "e5d43600c7a68cc5889563120b46780f107945e26c9385b5fca04c3a22960843", + "variants/cpu-int4/inference/model.layers.6.self_attn.v_proj.MatMul.weight.bin.sha256": "fd3c65bcc2e5652073fdf480a8be666611a1e20d26662e648e5cdd7fb4e0340c", + "variants/cpu-int4/inference/model.layers.7.self_attn.q_proj.MatMul.weight.bin": "e9d722fef4ecc4e3dffddd5b1527bcca70af70978eb812df6d7c4452c4c10b90", + "variants/cpu-int4/inference/model.layers.7.self_attn.q_proj.MatMul.weight.bin.sha256": "5d11e119c112c0f2cd8a46a10856aa4e3a467048928ddd0de0891dec95164b18", + "variants/cpu-int4/inference/model.layers.7.self_attn.v_proj.MatMul.weight.bin": "d50fe1d3c9c265fc38d169bd63fa727062f145719419564a124a5c06fbe59b04", + "variants/cpu-int4/inference/model.layers.7.self_attn.v_proj.MatMul.weight.bin.sha256": "bb5995d820468db5e2c676bf0960061c85fff3ccaeebb0c76b89f905b400f0bc", + "variants/cpu-int4/inference/model.layers.8.self_attn.q_proj.MatMul.weight.bin": "eee1373ed2ee35607cda09ceeef4c92b7ab331a419ffb795c821a58c563e884b", + "variants/cpu-int4/inference/model.layers.8.self_attn.q_proj.MatMul.weight.bin.sha256": "ebf030acd3a19f40d679c3e8917c15a97e122f4f8076eff7043758fefbef1b4c", + "variants/cpu-int4/inference/model.layers.8.self_attn.v_proj.MatMul.weight.bin": "3b9e52ee458990d65fd339e4e544d5f379b1600dd4852fc3084a32e96bc00a0b", + "variants/cpu-int4/inference/model.layers.8.self_attn.v_proj.MatMul.weight.bin.sha256": "6142f9fd4e7ee9f118757cb582ce4d16c3105cb389343ee8aef7dbf4b83b939f", + "variants/cpu-int4/inference/model.layers.9.self_attn.q_proj.MatMul.weight.bin": "68c5c1b401f009fd33795a986edbef642ed6bb2e8cfdee72f8ef5d9930a608b4", + "variants/cpu-int4/inference/model.layers.9.self_attn.q_proj.MatMul.weight.bin.sha256": "ec94b226ebc6be2d0eff5dfd48edb112728ec2d7fb4968481f811a87b31a16ae", + "variants/cpu-int4/inference/model.layers.9.self_attn.v_proj.MatMul.weight.bin": "b029f98f39c8aef6ec3676ae87106eab90d9521ff05ab1c7812963f8a1929555", + "variants/cpu-int4/inference/model.layers.9.self_attn.v_proj.MatMul.weight.bin.sha256": "9004e7234ef3e8231241955e74f0cfd7880d2ba47a4b6f4f43b3e86759ea7793", + "variants/cpu-int4/inference/model.onnx": "7816d03ce0be987e340c51ab64916f13a0f70a68630c3a2de82370a32a6fabfa", + "variants/cpu-int4/inference/model.onnx_data": "984595b520feb718e9dde76cc6dcd2bcc590bb67987c4da01b15a6ea54cade05", + "variants/cpu-int4/inference/optimum_config.json": "78abd079b5f45b977fad47a63f173701f6681d555c1372a800221831ce811e4f", + "variants/cpu-int4/inference/special_tokens_map.json": "d39df6f8ddb132a622394f05a603b166ba5f22d485483e30d06c1821ef4ac9b4", + "variants/cpu-int4/inference/tokenizer.json": "b6b09a0b4a803ad453063ca4bb49a784540e8120004e2450e025df2b27d41fb2", + "variants/cpu-int4/inference/tokenizer_config.json": "c33878647bdc69ce85d4b9aeef953815afd8405f8d85001ab0bbb89aa14d2554", + "variants/cpu-int4/inference/weight_handoff_map.json": "2b809307b885be7b6bd2908831ab37e5f18ea6d867b6fa8be2d284138e94581f", + "variants/cpu-int4/train/checkpoint": "cd9ea126423c721891377e36136904e64aaa4c04db73f82b8f2732aec482b1be", + "variants/cpu-int4/train/eval_model.onnx": "b66f69dca13c1f8a43a760f9f5b9a4cad1a9518d4914e8d0e42bd025e5ce6b5c", + "variants/cpu-int4/train/optimizer_model.onnx": "8ed3b548bce52bf25c2154b1452314b3c48d079e28c52822dd348034f59dc054", + "variants/cpu-int4/train/trainable_parameters.json": "05fddff407edb77426860ce8637b046bffd277bb053b33aa6600a321ac993958", + "variants/cpu-int4/train/training_config.json": "4319ebadece693ac91fd1c0d5678e849521b9b3b54b70462df560653f5b1cb74", + "variants/cpu-int4/train/training_model.onnx": "8e6de37f3cccf0ccaa02815bbeaf774fa1fd0d0bc7e514102950937186a830cb" + }, + "supportedTasks": [ + "text-generation-with-past" + ], + "trainableParameterCount": 368640, + "trainingParameterCount": 268835639, + "transformersVersion": "4.57.6", + "trustRemoteCode": false, + "variants": [ + { + "abi": null, + "executionProvider": "cpu", + "features": [ + "core", + "inference", + "train" + ], + "id": "cpu-int4", + "minimumAndroidApi": 28, + "paths": { + "inference": "variants/cpu-int4/inference", + "tokenizer": "shared/tokenizer", + "train": "variants/cpu-int4/train" + }, + "quantization": "int4", + "recommendedDeviceMemoryMb": null, + "supportedEngines": [ + "native" + ], + "weightHandoff": "variants/cpu-int4/inference/weight_handoff_map.json" + } + ], + "weightHandoff": "variants/cpu-int4/inference/weight_handoff_map.json" +} diff --git a/optimum/export_report.json b/optimum/export_report.json new file mode 100644 index 0000000000000000000000000000000000000000..d6e6d1b1c736ce890f234322375f560a99af2fde --- /dev/null +++ b/optimum/export_report.json @@ -0,0 +1,35 @@ +{ + "androidRuntime": { + "minimumAndroidApi": 28, + "recommendedDeviceMemoryMb": null, + "requiredAbis": [] + }, + "architectures": [ + "gemma3_text" + ], + "license": { + "baseModelWeights": null, + "framework": null, + "noticeFile": null + }, + "mobiletransformersVersion": "0.1.0", + "onnxRuntimeTrainingVersion": "1.23.0+cpu", + "optimumOnnxVersion": "0.1.0", + "peftMethods": [ + "lora" + ], + "quantization": [ + "int4" + ], + "selectedTask": "text-generation-with-past", + "sourceParameterCount": 268098176, + "supportedTasks": [ + "text-generation-with-past" + ], + "trainInferenceLossDeltaNats": 0.8837, + "trainableParameterCount": 368640, + "trainingParameterCount": 268835639, + "trainingQuantizedParameterCount": 253296731, + "transformersVersion": "4.57.6", + "trustRemoteCode": false +} diff --git a/optimum/supported_tasks.json b/optimum/supported_tasks.json new file mode 100644 index 0000000000000000000000000000000000000000..5aaf6de0fc528213cd65f3e40f28f942d2b120f4 --- /dev/null +++ b/optimum/supported_tasks.json @@ -0,0 +1,3 @@ +[ + "text-generation-with-past" +] diff --git a/shared/tokenizer/chat_template.jinja b/shared/tokenizer/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..16294794d96bfe26bbf2da97af27ced085fd1683 --- /dev/null +++ b/shared/tokenizer/chat_template.jinja @@ -0,0 +1,279 @@ +{%- macro format_parameters(properties, required) -%} + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in properties | dictsort -%} + {%- if key not in standard_keys -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{- key }}:{description:{{ value['description'] }} + {%- if value['type'] | upper == 'STRING' -%} + {%- if value['enum'] -%} + ,enum:{{ format_argument(value['enum']) }} + {%- endif -%} + {%- elif value['type'] | upper == 'OBJECT' -%} + ,properties:{ + {%- if value['properties'] is defined and value['properties'] is mapping -%} + {{- format_parameters(value['properties'], value['required'] | default([])) -}} + {%- elif value is mapping -%} + {{- format_parameters(value, value['required'] | default([])) -}} + {%- endif -%} + } + {%- if value['required'] -%} + ,required:[ + {%- for item in value['required'] | default([]) -%} + {{- item -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- endif -%} + {%- elif value['type'] | upper == 'ARRAY' -%} + {%- if value['items'] is mapping and value['items'] -%} + ,items:{ + {%- set ns_items = namespace(found_first=false) -%} + {%- for item_key, item_value in value['items'] | dictsort -%} + {%- if item_value is not none -%} + {%- if ns_items.found_first %},{% endif -%} + {%- set ns_items.found_first = true -%} + {%- if item_key == 'properties' -%} + properties:{ + {%- if item_value is mapping -%} + {{- format_parameters(item_value, value['items']['required'] | default([])) -}} + {%- endif -%} + } + {%- elif item_key == 'required' -%} + required:[ + {%- for req_item in item_value -%} + {{- req_item -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + ] + {%- elif item_key == 'type' -%} + {%- if item_value is string -%} + type:{{ format_argument(item_value | upper) }} + {%- else -%} + type:{{ format_argument(item_value | map('upper') | list) }} + {%- endif -%} + {%- else -%} + {{ item_key }}:{{ format_argument(item_value) }} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + } + {%- endif -%} + {%- endif -%} + ,type:{{ value['type'] | upper }}} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} +{% macro format_function_declaration(tool_data) -%} +declaration:{{- tool_data['function']['name'] -}} +{description:{{- tool_data['function']['description'] -}} +{%- set params = tool_data['function']['parameters'] -%} +{%- if params -%} + ,parameters:{ + {%- if params['properties'] -%} + properties:{ {{- format_parameters(params['properties'], params['required']) -}} }, + {%- endif -%} + {%- if params['required'] -%} + required:[ + {%- for item in params['required'] -%} + {{- item -}} + {{- ',' if not loop.last -}} + {%- endfor -%} + ], + {%- endif -%} + {%- if params['type'] -%} + type:{{- params['type'] | upper -}}} + {%- endif -%} +{%- endif -%} +} +{%- endmacro -%} +{% macro format_argument(argument, escape_keys=True) -%} +{%- if argument is string -%} + {{- '' + argument + '' -}} +{%- elif argument is boolean -%} + {%- if argument -%} + {{- 'true' -}} + {%- else -%} + {{- 'false' -}} + {%- endif -%} +{%- elif argument is mapping -%} + {{- '{' -}} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in argument | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {%- if escape_keys -%} + {{- '' + key + '' -}} + {%- else -%} + {{- key -}} + {%- endif -%} + :{{- format_argument(value, escape_keys=escape_keys) -}} + {%- endfor -%} + {{- '}' -}} +{%- elif argument is sequence -%} + {{- '[' -}} + {%- for item in argument -%} + {{- format_argument(item, escape_keys=escape_keys) -}} + {%- if not loop.last %},{% endif -%} + {%- endfor -%} + {{- ']' -}} +{%- else -%} + {{- argument -}} +{%- endif -%} +{%- endmacro -%} +{{ bos_token }} +{%- set ns = namespace(prev_message_type=None) -%} +{#- Tool Declarations -#} +{%- set loop_messages = messages -%} +{%- if tools or messages[0]['role'] == 'system' or messages[0]['role'] == 'developer' -%} + {{- 'developer\n' -}} + {%- if messages[0]['role'] == 'system' or messages[0]['role'] == 'developer' -%} + {%- if messages[0]['content'] is string -%} + {{- messages[0]['content'] | trim -}} + {%- elif messages[0]['content'] is sequence -%} + {%- for item in messages[0]['content'] -%} + {%- if item['type'] == 'text' -%} + {{- item['text'] | trim -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} + {%- endif -%} + {%- if tools -%} + {%- for tool in tools %} + {{- '' -}} + {{- format_function_declaration(tool) | trim }} + {{- '' -}} + {%- endfor %} + {%- endif -%} + {{- '\n' }} +{%- endif %} +{#- Loop through messages. -#} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'assistant') -%} + {#- Rename "assistant" to "model". -#} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {%- if role != 'tool' -%} + {%- if ns.prev_message_type != 'tool_response' -%} + {{- '' + role + '\n' }} + {%- endif -%} + {%- set ns.prev_message_type = None -%} + {%- if 'content' in message and message['content'] is not none -%} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is sequence -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type in user/assistant message") }} + {%- endif -%} + {%- set ns.prev_message_type = 'content' -%} + {%- endif -%} + {%- if 'tool_calls' in message and message['tool_calls'] and message['tool_calls'] is iterable -%} + {#- Tool Calls -#} + {%- for tool_call in message['tool_calls'] -%} + {% set function = tool_call['function'] %} + {{- 'call:' + function['name'] + '{' -}} + {%- if 'arguments' in function -%} + {%- if function['arguments'] is mapping -%} + {%- set ns = namespace(found_first=false) -%} + {%- for key, value in function['arguments'] | dictsort -%} + {%- if ns.found_first %},{% endif -%} + {%- set ns.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {%- elif function['arguments'] is string -%} + {# This handles string-JSON, just in case #} + {{ function['arguments'] }} + {%- endif %} + {%- endif -%} + {{- '}' -}} + {%- endfor -%} + {%- if loop.last -%} + {{ '' }} + {%- endif -%} + {%- set ns.prev_message_type = 'tool_call' -%} + {%- endif -%} + {%- else -%} + {#- Tool Responses -#} + {%- if 'content' in message and message['content'] -%} + {%- if message['content'] is mapping -%} + {%- if 'name' in message['content'] and 'response' in message['content'] -%} + {{ 'response:' + message['content']['name'] | trim + '{' }} + {%- set response_ns = namespace(found_first=false) -%} + {%- for key, value in message['content']['response'] | dictsort -%} + {%- if response_ns.found_first %},{% endif -%} + {%- set response_ns.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif 'name' in message -%} + {{ 'response:' + message['name'] | trim + '{' }} + {%- set response_ns = namespace(found_first=false) -%} + {%- for key, value in message['content'] | dictsort -%} + {%- if response_ns.found_first %},{% endif -%} + {%- set response_ns.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{ raise_exception("Invalid tool response mapping: must contain 'name' and 'response' keys, or 'name' must be in the message.") }} + {%- endif -%} + {%- elif message['content'] is string -%} + {%- if 'name' in message -%} + {{ 'response:' + message['name'] | trim + '{value:' + format_argument(message['content'], escape_keys=False) + '}' }} + {%- else -%} + {{ raise_exception("Invalid tool response: 'name' must be provided.") }} + {%- endif -%} + {%- elif message['content'] is sequence -%} + {%- for item in message['content'] -%} + {%- if item is mapping -%} + {%- if 'name' in item and 'response' in item -%} + {{ 'response:' + item['name'] | trim + '{' }} + {%- set response_ns = namespace(found_first=false) -%} + {%- for key, value in item['response'] | dictsort -%} + {%- if response_ns.found_first %},{% endif -%} + {%- set response_ns.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {{- '}' -}} + {%- elif 'name' in message -%} + {{ 'response:' + message['name'] | trim + '{' }} + {%- set response_ns = namespace(found_first=false) -%} + {%- for key, value in item | dictsort -%} + {%- if response_ns.found_first %},{% endif -%} + {%- set response_ns.found_first = true -%} + {{- key -}}:{{- format_argument(value, escape_keys=False) -}} + {%- endfor -%} + {{- '}' -}} + {%- else -%} + {{ raise_exception("Invalid tool response mapping: must contain 'name' and 'response' keys, or 'name' must be in the message.") }} + {%- endif -%} + {%- else -%} + {{ raise_exception("Invalid tool response message: multiple responses must all be mappings") }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type in tool message: must be mapping, sequence of mappings, or string.") }} + {%- endif -%} + {%- endif -%} + {%- set ns.prev_message_type = 'tool_response' -%} + {%- endif -%} + {%- if ns.prev_message_type not in ['tool_call', 'tool_response'] -%} + {{ '\n' }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {%- if ns.prev_message_type != 'tool_response' -%} + {{- 'model\n' -}} + {%- endif -%} +{%- endif -%} diff --git a/shared/tokenizer/mobiletransformers_tokenizer_config.json b/shared/tokenizer/mobiletransformers_tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e535472705476ac122f5785c384cebbf2e84e3dd --- /dev/null +++ b/shared/tokenizer/mobiletransformers_tokenizer_config.json @@ -0,0 +1,17 @@ +{ + "model": { + "bos_token_id": null, + "context_length": 2048, + "num_attention_heads": 12, + "num_hidden_layers": 12, + "num_key_value_heads": 12, + "eos_token_id": [ + 1, + 50, + 106 + ], + "pad_token_id": 1, + "type": "unknown", + "vocab_size": 262146 + } +} \ No newline at end of file diff --git a/shared/tokenizer/special_tokens_map.json b/shared/tokenizer/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..bd5c7b5291da3a99a68c0c7c474787d8a2c28fbc --- /dev/null +++ b/shared/tokenizer/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "boi_token": "", + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eoi_token": "", + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "image_token": "", + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "sfr_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/shared/tokenizer/tokenizer.json b/shared/tokenizer/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..c3a88e473260042d50dd19fc19939c99c71b0b85 --- /dev/null +++ b/shared/tokenizer/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6b09a0b4a803ad453063ca4bb49a784540e8120004e2450e025df2b27d41fb2 +size 33384899 diff --git a/shared/tokenizer/tokenizer_config.json b/shared/tokenizer/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..22da9da2a646c193586c1de9fca8c26c5cbf30d8 --- /dev/null +++ b/shared/tokenizer/tokenizer_config.json @@ -0,0 +1,51355 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "[multimodal]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "108": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "217": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "218": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "219": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "220": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "221": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "222": { + "content": "