Upload 12 files

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -0,0 +1,126 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+
+---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 4299 with parameters:
+```
+{'batch_size': 128, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.SoftmaxLoss.SoftmaxLoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 5,
+    "evaluation_steps": 1000,
+    "evaluator": "sentence_transformers.evaluation.EmbeddingSimilarityEvaluator.EmbeddingSimilarityEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 2150,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: BertModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "klue/bert-base",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 32000
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.27.1",
+    "pytorch": "2.0.0+cu117"
+  }
+}

eval/similarity_evaluation_sts-dev_results.csv

@@ -0,0 +1,26 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,1000,0.6659445577398593,0.6725674704728215,0.6723452888988094,0.6822943404791594,0.6734938259336746,0.6821095244744039,0.3542827688117064,0.375129710282081
+0,2000,0.7445418679821378,0.7582711568781528,0.7445695278411221,0.7491769678535929,0.7475482682410554,0.7505806033926292,0.6516841702963634,0.644625132585104
+0,3000,0.7611784379058513,0.7753377401337672,0.7624915976151901,0.7632756985425497,0.7652650662972923,0.7652178046176139,0.7098410157266418,0.7011994433368494
+0,4000,0.7664600822011544,0.7779506682370195,0.7650158186547508,0.7654782418404968,0.7667586961684713,0.7666540418535448,0.727029545107196,0.7192756402575862
+0,-1,0.7588258721580844,0.7711649812292414,0.7601431094101754,0.7596844164327338,0.7621491474434782,0.760718430958818,0.7313404813612134,0.72580007132318
+1,1000,0.7681415905613104,0.777508008176866,0.7682184877759901,0.7668035939752597,0.769902357010777,0.7680304845071181,0.7356156982138174,0.7283572995579198
+1,2000,0.7742120345102455,0.7860305900388165,0.7759786697854631,0.7749660088971759,0.7773815931917177,0.7761238737801722,0.7512480467845594,0.7458176276860289
+1,3000,0.7647251199713558,0.7771782972876436,0.7665557513521478,0.765477140678158,0.7676985718948486,0.7663204782455075,0.7376264851464495,0.7317939353845909
+1,4000,0.7668508603746315,0.7790608424134365,0.7701015147433578,0.7689631339375641,0.770994674550495,0.7695494001528361,0.7456712056866467,0.7405603180080812
+1,-1,0.7760705299905457,0.7865357437398377,0.7780168812553343,0.7771877811960934,0.7792550113966342,0.7779340379441579,0.7491237807112845,0.7426565371966939
+2,1000,0.7660973752800285,0.7762150485839853,0.7688767895910371,0.7677178084012513,0.7689664980913219,0.7676394568627904,0.7415639545225098,0.7360860159879333
+2,2000,0.7700804712898738,0.7788857557932253,0.771832930736482,0.7710945591804985,0.7720733512196799,0.771066148814079,0.7434970099554319,0.7377385108546101
+2,3000,0.7702978801686402,0.7778519973483153,0.7708395189129182,0.7697618760736129,0.771446051956762,0.7702827141041502,0.7453960977637094,0.7399275204183559
+2,4000,0.7752618062959603,0.7842447684883692,0.7752988490279625,0.7749458002964081,0.7765941762394749,0.7759540095833968,0.7522018583387206,0.746806940632618
+2,-1,0.7769924633068857,0.7853148326056885,0.7767178530645548,0.7761386300181663,0.7778289365232416,0.7771303993893972,0.7502759132840352,0.7443259887337296
+3,1000,0.7707249689780824,0.7778895006387035,0.7708852962346351,0.7701954497389991,0.7711964157647142,0.770465342070147,0.7485131409970578,0.7435443349631866
+3,2000,0.765345392865508,0.7731846481447531,0.7670651163456563,0.7658387065033504,0.7665886261301962,0.7659464699232981,0.7466526408746222,0.7426952109404683
+3,3000,0.770412475126673,0.777182000430369,0.7706925644799741,0.770204956042818,0.7707740381323195,0.7701862496240816,0.7486840138355021,0.7437404056119462
+3,4000,0.769441727952264,0.7762049685056753,0.7691023986611366,0.7687227127459159,0.7690761360339188,0.7689583893373857,0.745167440673268,0.7409386483057541
+3,-1,0.7708295482636287,0.7781358477883977,0.7718077788542748,0.7710027136441666,0.7717687967013193,0.7712107447231016,0.7491866506658181,0.7451200193260054
+4,1000,0.7691328551057761,0.7752461688945677,0.7695093852796154,0.7687449240924131,0.7693905324881671,0.7688976659206208,0.7484561215714696,0.7441252526243871
+4,2000,0.7695307937731882,0.7754710373811518,0.7692735938071544,0.7688034903820917,0.7691593254608746,0.7692798728634921,0.7461848158566525,0.7414715626070804
+4,3000,0.7690492243658177,0.7753930913959958,0.770085621692888,0.7694832559787814,0.7700100236708824,0.7698577832028968,0.7469965841675125,0.7425053620550033
+4,4000,0.7680466132561923,0.7742937251598457,0.768606388499023,0.7679242559927304,0.7684488738824462,0.768282112112849,0.7459098688242687,0.7414001691214792
+4,-1,0.7679647659540607,0.7742325596361201,0.768522572235958,0.7678292114428155,0.7683798142983765,0.7681453415553424,0.7460022278231219,0.7415731592238808

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:765e6d6553116c144ddfeb890861d235ed5d48a044e953fcddf356a2bc5a153e
+size 442540589

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/home/yyoo/.cache/huggingface/hub/models--klue--bert-base/snapshots/34b965303f98bc5214daca7f76b7fb82d2dc6183/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}