Push model using huggingface_hub.
Browse files- 1_Pooling/config.json +10 -0
- README.md +192 -0
- config.json +25 -0
- config_sentence_transformers.json +10 -0
- config_setfit.json +7 -0
- model.safetensors +3 -0
- model_head.pkl +3 -0
- modules.json +20 -0
- sentence_bert_config.json +4 -0
- special_tokens_map.json +37 -0
- tokenizer.json +0 -0
- tokenizer_config.json +65 -0
- vocab.txt +0 -0
1_Pooling/config.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"word_embedding_dimension": 384,
|
| 3 |
+
"pooling_mode_cls_token": false,
|
| 4 |
+
"pooling_mode_mean_tokens": true,
|
| 5 |
+
"pooling_mode_max_tokens": false,
|
| 6 |
+
"pooling_mode_mean_sqrt_len_tokens": false,
|
| 7 |
+
"pooling_mode_weightedmean_tokens": false,
|
| 8 |
+
"pooling_mode_lasttoken": false,
|
| 9 |
+
"include_prompt": true
|
| 10 |
+
}
|
README.md
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
tags:
|
| 3 |
+
- setfit
|
| 4 |
+
- sentence-transformers
|
| 5 |
+
- text-classification
|
| 6 |
+
- generated_from_setfit_trainer
|
| 7 |
+
widget:
|
| 8 |
+
- text: '"The Impact of Assessment for 21 st Century Skills in Higher Education Institutions:
|
| 9 |
+
A Narrative Literature Review" by Rany Sam You read the paper Assessing 21st century
|
| 10 |
+
skills: Integrating research findings. We found a related paper on Academia:\r\n\r\nThe
|
| 11 |
+
Impact of Assessment for 21 st Century Skills in Higher Education Institutions:
|
| 12 |
+
A Narrative Literature Review\r\nPaper Thumbnail\t\r\nAuthor Photo Rany Sam\r\n2024,
|
| 13 |
+
Multitech Publisher\r\n23 Views \r\nView PDF \u25B8\r\n \t\t\r\nDownload PDF \u2B07\r\n\r'
|
| 14 |
+
- text: '[Legal Notice] Update to Google Maps Platform terms and products effective
|
| 15 |
+
8 July 2025 \r\nHello Google Maps Platform customer,\r\n\r\nWe''re writing to
|
| 16 |
+
let you know about some important updates to the Google Maps Platform (GMP) Terms
|
| 17 |
+
of Service (ToS) and our product offerings for customers with any GMP project
|
| 18 |
+
linked to a billing account with an address in the European Economic Area (EEA
|
| 19 |
+
customers). These updates will be effective on 8 July 2025.\r\n\r\nThe changes
|
| 20 |
+
to our terms are a result of a recent proc'
|
| 21 |
+
- text: Update on our sub-processors list Dear Business Partner,\r\n\r\n \r\n\r\nTo
|
| 22 |
+
support our objectives of operational excellence and compliance with industry
|
| 23 |
+
best practices, we continuously monitor the best options to deliver our products
|
| 24 |
+
and services. \r\n\r\n \r\n\r\nAs of June 9, 2025 (for Enterprise Organizations
|
| 25 |
+
July 9, 2025), our current list of sub-processors will be replaced by the updated
|
| 26 |
+
list available here. No action is required on your part, and you may continue
|
| 27 |
+
to use your account as usual.\r\n\r\n
|
| 28 |
+
metrics:
|
| 29 |
+
- accuracy
|
| 30 |
+
pipeline_tag: text-classification
|
| 31 |
+
library_name: setfit
|
| 32 |
+
inference: true
|
| 33 |
+
base_model: sentence-transformers/all-MiniLM-L6-v2
|
| 34 |
+
---
|
| 35 |
+
|
| 36 |
+
# SetFit with sentence-transformers/all-MiniLM-L6-v2
|
| 37 |
+
|
| 38 |
+
This is a [SetFit](https://github.com/huggingface/setfit) model that can be used for Text Classification. This SetFit model uses [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) as the Sentence Transformer embedding model. A [LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) instance is used for classification.
|
| 39 |
+
|
| 40 |
+
The model has been trained using an efficient few-shot learning technique that involves:
|
| 41 |
+
|
| 42 |
+
1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning.
|
| 43 |
+
2. Training a classification head with features from the fine-tuned Sentence Transformer.
|
| 44 |
+
|
| 45 |
+
## Model Details
|
| 46 |
+
|
| 47 |
+
### Model Description
|
| 48 |
+
- **Model Type:** SetFit
|
| 49 |
+
- **Sentence Transformer body:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)
|
| 50 |
+
- **Classification head:** a [LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) instance
|
| 51 |
+
- **Maximum Sequence Length:** 256 tokens
|
| 52 |
+
- **Number of Classes:** 2 classes
|
| 53 |
+
<!-- - **Training Dataset:** [Unknown](https://huggingface.co/datasets/unknown) -->
|
| 54 |
+
<!-- - **Language:** Unknown -->
|
| 55 |
+
<!-- - **License:** Unknown -->
|
| 56 |
+
|
| 57 |
+
### Model Sources
|
| 58 |
+
|
| 59 |
+
- **Repository:** [SetFit on GitHub](https://github.com/huggingface/setfit)
|
| 60 |
+
- **Paper:** [Efficient Few-Shot Learning Without Prompts](https://arxiv.org/abs/2209.11055)
|
| 61 |
+
- **Blogpost:** [SetFit: Efficient Few-Shot Learning Without Prompts](https://huggingface.co/blog/setfit)
|
| 62 |
+
|
| 63 |
+
### Model Labels
|
| 64 |
+
| Label | Examples |
|
| 65 |
+
|:---------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
| 66 |
+
| 👨⚖️ Legal | <ul><li>'Airmoney Expiration Policy Update Hi alex ,\\r\\n\\r\\n \\r\\n\\r\\nFrom February 1, 2025, your Airmoney can expire \\u2014 this will always apply to your total balance, not partial amounts of Airmoney. \\r\\n\\r\\n \\r\\n\\r\\nYour Airmoney balance is set to expire on 1st February 2026.\\r\\n\\r\\nYour current Airmoney balance is 10.88 USD*.\\r\\n\\r\\n \\r\\n\\r\\nBelow, you\\u2019ll find details to help you understand how this change applies to you.\\r\\n\\r\\n \\r\\n\\r\\nDoes my Airmoney balance have to expire?\\r\\n\\r\\n \\r\\n\\r\\nNo, your Air'</li><li>'Meta Privacy Policy update Meta Privacy Policy update\\r\\n \\r\\nHi Alex,\\r\\n \\r\\nWe\\u2019re updating the Meta Privacy Policy to clarify some details.\\r\\n \\r\\nWhat you should know\\r\\n \\r\\nHere are the details that this update clarifies:\\r\\n \\r\\n\\u2022\\tHow we use information from third parties\\r\\n\\u2022\\tLegitimate interests is now our legal basis for using your information to improve Meta Products. Learn what this means for your rights\\r\\n\\u2022\\tWhen your information can be accessible to search engines\\r\\n \\'</li><li>"Google Play Developer Program Policy Update DEVELOPER UPDATE\\r\\nHello Google Play Developer,\\r\\nTo give users more control over their data, we're updating our Health Connect policy to strengthen safeguards regarding the handling of sensitive health record data. Health Connect is an Android platform that allows health and fitness apps to store and share the same on-device data, within a unified ecosystem. It also offers a single place for users to control which apps can read and write health and fitness data"</li></ul> |
|
| 67 |
+
| 👮🏽♂️ Security | <ul><li>"Petcube security: Sign-in notifications Hi, alexeysheiko.\\r\\n\\r\\nWe noticed a recent login to your Petcube account.\\r\\n\\r\\nTimestamp (UTC): 2025-05-04T08:19:58+00:00\\r\\n\\r\\nIP address: 85.114.207.94\\r\\n\\r\\nIf this was you, no action is required. If this wasn't you, follow the link to secure your account. Reset password\\r\\nWags & Purrs,\\r\\nPetcube Team"</li><li>'A new device is using your account A new device is using your account\\r\\nHi Oleksii,\\r\\nA new device signed in to your Netflix account, alexsheikodev@gmail.com.\\r\\n \\r\\nThe details\\r\\nDevice\\r\\nMac Chrome - Web Browser\\r\\nLocation\\r\\nMazovia, Poland\\r\\n(This location may not be exact.)\\r\\nTime\\r\\nJune 19th, 3:21 PM GMT+3\\r\\n \\r\\nIf this was you or someone in your household:\\r\\nEnjoy watching!\\r\\nIf it was someone else:\\r\\nPlease remember that we only allow the people in your household to use your account.\\r'</li></ul> |
|
| 68 |
+
|
| 69 |
+
## Uses
|
| 70 |
+
|
| 71 |
+
### Direct Use for Inference
|
| 72 |
+
|
| 73 |
+
First install the SetFit library:
|
| 74 |
+
|
| 75 |
+
```bash
|
| 76 |
+
pip install setfit
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
Then you can load this model and run inference.
|
| 80 |
+
|
| 81 |
+
```python
|
| 82 |
+
from setfit import SetFitModel
|
| 83 |
+
|
| 84 |
+
# Download from the 🤗 Hub
|
| 85 |
+
model = SetFitModel.from_pretrained("setfit_model_id")
|
| 86 |
+
# Run inference
|
| 87 |
+
preds = model("\"The Impact of Assessment for 21 st Century Skills in Higher Education Institutions: A Narrative Literature Review\" by Rany Sam You read the paper Assessing 21st century skills: Integrating research findings. We found a related paper on Academia:\r\n\r\nThe Impact of Assessment for 21 st Century Skills in Higher Education Institutions: A Narrative Literature Review\r\nPaper Thumbnail\t\r\nAuthor Photo Rany Sam\r\n2024, Multitech Publisher\r\n23 Views \r\nView PDF \u25B8\r\n \t\t\r\nDownload PDF \u2B07\r\n\r")
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
<!--
|
| 91 |
+
### Downstream Use
|
| 92 |
+
|
| 93 |
+
*List how someone could finetune this model on their own dataset.*
|
| 94 |
+
-->
|
| 95 |
+
|
| 96 |
+
<!--
|
| 97 |
+
### Out-of-Scope Use
|
| 98 |
+
|
| 99 |
+
*List how the model may foreseeably be misused and address what users ought not to do with the model.*
|
| 100 |
+
-->
|
| 101 |
+
|
| 102 |
+
<!--
|
| 103 |
+
## Bias, Risks and Limitations
|
| 104 |
+
|
| 105 |
+
*What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
|
| 106 |
+
-->
|
| 107 |
+
|
| 108 |
+
<!--
|
| 109 |
+
### Recommendations
|
| 110 |
+
|
| 111 |
+
*What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
|
| 112 |
+
-->
|
| 113 |
+
|
| 114 |
+
## Training Details
|
| 115 |
+
|
| 116 |
+
### Training Set Metrics
|
| 117 |
+
| Training set | Min | Median | Max |
|
| 118 |
+
|:-------------|:----|:-------|:----|
|
| 119 |
+
| Word count | 9 | 59.875 | 79 |
|
| 120 |
+
|
| 121 |
+
| Label | Training Sample Count |
|
| 122 |
+
|:---------------|:----------------------|
|
| 123 |
+
| 👨⚖️ Legal | 6 |
|
| 124 |
+
| 👮🏽♂️ Security | 2 |
|
| 125 |
+
|
| 126 |
+
### Training Hyperparameters
|
| 127 |
+
- batch_size: (16, 16)
|
| 128 |
+
- num_epochs: (2, 2)
|
| 129 |
+
- max_steps: -1
|
| 130 |
+
- sampling_strategy: oversampling
|
| 131 |
+
- num_iterations: 30
|
| 132 |
+
- body_learning_rate: (2e-05, 2e-05)
|
| 133 |
+
- head_learning_rate: 2e-05
|
| 134 |
+
- loss: CosineSimilarityLoss
|
| 135 |
+
- distance_metric: cosine_distance
|
| 136 |
+
- margin: 0.25
|
| 137 |
+
- end_to_end: False
|
| 138 |
+
- use_amp: False
|
| 139 |
+
- warmup_proportion: 0.1
|
| 140 |
+
- l2_weight: 0.01
|
| 141 |
+
- seed: 42
|
| 142 |
+
- eval_max_steps: -1
|
| 143 |
+
- load_best_model_at_end: False
|
| 144 |
+
|
| 145 |
+
### Training Results
|
| 146 |
+
| Epoch | Step | Training Loss | Validation Loss |
|
| 147 |
+
|:------:|:----:|:-------------:|:---------------:|
|
| 148 |
+
| 0.0333 | 1 | 0.2806 | - |
|
| 149 |
+
| 1.6667 | 50 | 0.038 | - |
|
| 150 |
+
|
| 151 |
+
### Framework Versions
|
| 152 |
+
- Python: 3.13.5
|
| 153 |
+
- SetFit: 1.1.2
|
| 154 |
+
- Sentence Transformers: 4.1.0
|
| 155 |
+
- Transformers: 4.52.4
|
| 156 |
+
- PyTorch: 2.7.1
|
| 157 |
+
- Datasets: 3.6.0
|
| 158 |
+
- Tokenizers: 0.21.1
|
| 159 |
+
|
| 160 |
+
## Citation
|
| 161 |
+
|
| 162 |
+
### BibTeX
|
| 163 |
+
```bibtex
|
| 164 |
+
@article{https://doi.org/10.48550/arxiv.2209.11055,
|
| 165 |
+
doi = {10.48550/ARXIV.2209.11055},
|
| 166 |
+
url = {https://arxiv.org/abs/2209.11055},
|
| 167 |
+
author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren},
|
| 168 |
+
keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences},
|
| 169 |
+
title = {Efficient Few-Shot Learning Without Prompts},
|
| 170 |
+
publisher = {arXiv},
|
| 171 |
+
year = {2022},
|
| 172 |
+
copyright = {Creative Commons Attribution 4.0 International}
|
| 173 |
+
}
|
| 174 |
+
```
|
| 175 |
+
|
| 176 |
+
<!--
|
| 177 |
+
## Glossary
|
| 178 |
+
|
| 179 |
+
*Clearly define terms in order to be accessible across audiences.*
|
| 180 |
+
-->
|
| 181 |
+
|
| 182 |
+
<!--
|
| 183 |
+
## Model Card Authors
|
| 184 |
+
|
| 185 |
+
*Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
|
| 186 |
+
-->
|
| 187 |
+
|
| 188 |
+
<!--
|
| 189 |
+
## Model Card Contact
|
| 190 |
+
|
| 191 |
+
*Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
|
| 192 |
+
-->
|
config.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"BertModel"
|
| 4 |
+
],
|
| 5 |
+
"attention_probs_dropout_prob": 0.1,
|
| 6 |
+
"classifier_dropout": null,
|
| 7 |
+
"gradient_checkpointing": false,
|
| 8 |
+
"hidden_act": "gelu",
|
| 9 |
+
"hidden_dropout_prob": 0.1,
|
| 10 |
+
"hidden_size": 384,
|
| 11 |
+
"initializer_range": 0.02,
|
| 12 |
+
"intermediate_size": 1536,
|
| 13 |
+
"layer_norm_eps": 1e-12,
|
| 14 |
+
"max_position_embeddings": 512,
|
| 15 |
+
"model_type": "bert",
|
| 16 |
+
"num_attention_heads": 12,
|
| 17 |
+
"num_hidden_layers": 6,
|
| 18 |
+
"pad_token_id": 0,
|
| 19 |
+
"position_embedding_type": "absolute",
|
| 20 |
+
"torch_dtype": "float32",
|
| 21 |
+
"transformers_version": "4.52.4",
|
| 22 |
+
"type_vocab_size": 2,
|
| 23 |
+
"use_cache": true,
|
| 24 |
+
"vocab_size": 30522
|
| 25 |
+
}
|
config_sentence_transformers.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"__version__": {
|
| 3 |
+
"sentence_transformers": "4.1.0",
|
| 4 |
+
"transformers": "4.52.4",
|
| 5 |
+
"pytorch": "2.7.1"
|
| 6 |
+
},
|
| 7 |
+
"prompts": {},
|
| 8 |
+
"default_prompt_name": null,
|
| 9 |
+
"similarity_fn_name": "cosine"
|
| 10 |
+
}
|
config_setfit.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"normalize_embeddings": false,
|
| 3 |
+
"labels": [
|
| 4 |
+
"\ud83d\udc68\u200d\u2696\ufe0f Legal",
|
| 5 |
+
"\ud83d\udc6e\ud83c\udffd\u200d\u2642\ufe0f Security"
|
| 6 |
+
]
|
| 7 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3bb5fa4b810037cc18b420e7657dbe3b96d01f2ff349c2a2bad96f6160857de5
|
| 3 |
+
size 90864192
|
model_head.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ea4959a66ccb207201304d69b7ba8654a3ae7e95712e19b48a0f992e122da79f
|
| 3 |
+
size 4031
|
modules.json
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"idx": 0,
|
| 4 |
+
"name": "0",
|
| 5 |
+
"path": "",
|
| 6 |
+
"type": "sentence_transformers.models.Transformer"
|
| 7 |
+
},
|
| 8 |
+
{
|
| 9 |
+
"idx": 1,
|
| 10 |
+
"name": "1",
|
| 11 |
+
"path": "1_Pooling",
|
| 12 |
+
"type": "sentence_transformers.models.Pooling"
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"idx": 2,
|
| 16 |
+
"name": "2",
|
| 17 |
+
"path": "2_Normalize",
|
| 18 |
+
"type": "sentence_transformers.models.Normalize"
|
| 19 |
+
}
|
| 20 |
+
]
|
sentence_bert_config.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"max_seq_length": 256,
|
| 3 |
+
"do_lower_case": false
|
| 4 |
+
}
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cls_token": {
|
| 3 |
+
"content": "[CLS]",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"mask_token": {
|
| 10 |
+
"content": "[MASK]",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "[PAD]",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"sep_token": {
|
| 24 |
+
"content": "[SEP]",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
},
|
| 30 |
+
"unk_token": {
|
| 31 |
+
"content": "[UNK]",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": false,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false
|
| 36 |
+
}
|
| 37 |
+
}
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"added_tokens_decoder": {
|
| 3 |
+
"0": {
|
| 4 |
+
"content": "[PAD]",
|
| 5 |
+
"lstrip": false,
|
| 6 |
+
"normalized": false,
|
| 7 |
+
"rstrip": false,
|
| 8 |
+
"single_word": false,
|
| 9 |
+
"special": true
|
| 10 |
+
},
|
| 11 |
+
"100": {
|
| 12 |
+
"content": "[UNK]",
|
| 13 |
+
"lstrip": false,
|
| 14 |
+
"normalized": false,
|
| 15 |
+
"rstrip": false,
|
| 16 |
+
"single_word": false,
|
| 17 |
+
"special": true
|
| 18 |
+
},
|
| 19 |
+
"101": {
|
| 20 |
+
"content": "[CLS]",
|
| 21 |
+
"lstrip": false,
|
| 22 |
+
"normalized": false,
|
| 23 |
+
"rstrip": false,
|
| 24 |
+
"single_word": false,
|
| 25 |
+
"special": true
|
| 26 |
+
},
|
| 27 |
+
"102": {
|
| 28 |
+
"content": "[SEP]",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": false,
|
| 32 |
+
"single_word": false,
|
| 33 |
+
"special": true
|
| 34 |
+
},
|
| 35 |
+
"103": {
|
| 36 |
+
"content": "[MASK]",
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"normalized": false,
|
| 39 |
+
"rstrip": false,
|
| 40 |
+
"single_word": false,
|
| 41 |
+
"special": true
|
| 42 |
+
}
|
| 43 |
+
},
|
| 44 |
+
"clean_up_tokenization_spaces": false,
|
| 45 |
+
"cls_token": "[CLS]",
|
| 46 |
+
"do_basic_tokenize": true,
|
| 47 |
+
"do_lower_case": true,
|
| 48 |
+
"extra_special_tokens": {},
|
| 49 |
+
"mask_token": "[MASK]",
|
| 50 |
+
"max_length": 128,
|
| 51 |
+
"model_max_length": 256,
|
| 52 |
+
"never_split": null,
|
| 53 |
+
"pad_to_multiple_of": null,
|
| 54 |
+
"pad_token": "[PAD]",
|
| 55 |
+
"pad_token_type_id": 0,
|
| 56 |
+
"padding_side": "right",
|
| 57 |
+
"sep_token": "[SEP]",
|
| 58 |
+
"stride": 0,
|
| 59 |
+
"strip_accents": null,
|
| 60 |
+
"tokenize_chinese_chars": true,
|
| 61 |
+
"tokenizer_class": "BertTokenizer",
|
| 62 |
+
"truncation_side": "right",
|
| 63 |
+
"truncation_strategy": "longest_first",
|
| 64 |
+
"unk_token": "[UNK]"
|
| 65 |
+
}
|
vocab.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|