embedding model change max sequence length in sentence_bert_config.json

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -0,0 +1,135 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+datasets:
+- code_search_net
+---
+
+# flax-sentence-embeddings/st-codesearch-distilroberta-base
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+It was trained on the [code_search_net](https://huggingface.co/datasets/code_search_net) dataset and can be used to search program code given text.
+
+## Usage:
+
+```python
+from sentence_transformers import SentenceTransformer, util
+
+
+#This list the defines the different programm codes
+code = ["""def sort_list(x):
+   return sorted(x)""",
+"""def count_above_threshold(elements, threshold=0):
+    counter = 0
+    for e in elements:
+        if e > threshold:
+            counter += 1
+    return counter""",
+"""def find_min_max(elements):
+    min_ele = 99999
+    max_ele = -99999
+    for e in elements:
+        if e < min_ele:
+            min_ele = e
+        if e > max_ele:
+            max_ele = e
+    return min_ele, max_ele"""]
+    
+
+model = SentenceTransformer("flax-sentence-embeddings/st-codesearch-distilroberta-base")
+
+# Encode our code into the vector space
+code_emb = model.encode(code, convert_to_tensor=True)
+
+# Interactive demo: Enter queries, and the method returns the best function from the 
+# 3 functions we defined
+while True:
+    query = input("Query: ")
+    query_emb = model.encode(query, convert_to_tensor=True)
+    hits = util.semantic_search(query_emb, code_emb)[0]
+    top_hit = hits[0]
+
+    print("Cossim: {:.2f}".format(top_hit['score']))
+    print(code[top_hit['corpus_id']])
+    print("\n\n")
+```
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('flax-sentence-embeddings/st-codesearch-distilroberta-base')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+## Training
+
+The model was trained with a DistilRoBERTa-base model for 10k training steps on the codesearch dataset with batch_size 256 and MultipleNegativesRankingLoss. 
+
+It is some preliminary model. It was neither tested nor was the trained quite sophisticated 
+
+
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`MultiDatasetDataLoader.MultiDatasetDataLoader` of length 5371 with parameters:
+```
+{'batch_size': 256}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.MultipleNegativesRankingLoss.MultipleNegativesRankingLoss` with parameters:
+  ```
+  {'scale': 20, 'similarity_fct': 'dot_score'}
+  ```
+
+Parameters of the fit()-Method:
+```
+{
+    "callback": null,
+    "epochs": 1,
+    "evaluation_steps": 0,
+    "evaluator": "NoneType",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'transformers.optimization.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "warmupconstant",
+    "steps_per_epoch": 10000,
+    "warmup_steps": 500,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: RobertaModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+  (2): Normalize()
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "distilroberta-base",
+  "architectures": [
+    "RobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.0.0",
+    "transformers": "4.6.1",
+    "pytorch": "1.8.1"
+  }
+}

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eec3c58b3fd1ca767f783848b856c58c38dcaaab8904d267cd55e11387b28b16
+size 328520407

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "distilroberta-base"}

train_script.py

@@ -0,0 +1,120 @@
+import math
+from sentence_transformers import models, losses, datasets
+from sentence_transformers import LoggingHandler, SentenceTransformer, util, InputExample
+from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator
+import logging
+from datetime import datetime
+import sys
+import os
+import gzip
+import csv
+from MultiDatasetDataLoader import MultiDatasetDataLoader
+from shutil import copyfile
+import json
+import argparse
+
+#### Just some code to print debug information to stdout
+logging.basicConfig(format='%(asctime)s - %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S',
+                    level=logging.INFO,
+                    handlers=[LoggingHandler()])
+#### /print debug information to stdout
+
+
+#model_name = 'distilroberta-base' 
+#batch_size_pairs = 200
+#batch_size_triplets = 200 
+#steps_per_epoch = 10000
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--model', default='nreimers/MiniLM-L6-H384-uncased')
+parser.add_argument('--steps', type=int, default=2000)
+parser.add_argument('--batch_size_pairs', type=int, default=256)
+parser.add_argument('--batch_size_triplets', type=int, default=256)
+parser.add_argument('--data', nargs='+', default=[])
+parser.add_argument('--name')
+args = parser.parse_args()
+
+
+model_name = args.model #'nreimers/MiniLM-L6-H384-uncased'
+batch_size_pairs = args.batch_size_pairs #256
+batch_size_triplets = args.batch_size_triplets #256 
+steps_per_epoch = args.steps #2000
+
+num_epochs = 1
+max_seq_length = 128
+use_amp = True
+warmup_steps = 500
+
+#####
+
+output_path = 'output/training_data_benchmark-{}-norm-{}'.format(model_name.replace("/", "-"), args.name)
+logging.info("Output: "+output_path)
+if os.path.exists(output_path):
+    exit()
+
+
+# Write train script to output path
+os.makedirs(output_path, exist_ok=True)
+
+train_script_path = os.path.join(output_path, 'train_script.py')
+copyfile(__file__, train_script_path)
+with open(train_script_path, 'a') as fOut:
+    fOut.write("\n\n# Script was called via:\n#python " + " ".join(sys.argv))
+
+## SentenceTransformer model
+word_embedding_model = models.Transformer(model_name, max_seq_length=max_seq_length)
+pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
+norm = models.Normalize()
+model = SentenceTransformer(modules=[word_embedding_model, pooling_model, norm])
+
+datasets = []
+for filepath in args.data:
+    filepath = filepath.strip()
+    dataset = []
+    
+
+    with gzip.open(filepath, 'rt', encoding='utf8') as fIn:
+        for line in fIn:
+            data = json.loads(line.strip())
+            
+            if not isinstance(data, dict):
+                data = {'guid': None, 'texts': data}
+           
+            dataset.append(InputExample(guid=data.get('guid', None), texts=data['texts']))
+            if len(dataset) >= (steps_per_epoch * batch_size_pairs * 2):
+                break
+
+    datasets.append(dataset)
+    logging.info("{}: {}".format(filepath, len(dataset)))
+
+
+
+train_dataloader = MultiDatasetDataLoader(datasets, batch_size_pairs=batch_size_pairs, batch_size_triplets=batch_size_triplets, random_batch_fraction=0.25)
+
+
+# Our training loss
+train_loss = losses.MultipleNegativesRankingLoss(model, scale=20, similarity_fct=util.dot_score)
+
+
+
+#Read STSbenchmark dataset and use it as development set
+
+# Configure the training
+logging.info("Warmup-steps: {}".format(warmup_steps))
+
+# Train the model
+model.fit(train_objectives=[(train_dataloader, train_loss)],
+          evaluator=None,
+          epochs=1,
+          warmup_steps=warmup_steps,
+          steps_per_epoch=steps_per_epoch,
+          scheduler='warmupconstant',
+          use_amp=use_amp
+          )
+
+
+model.save(output_path)
+
+# Script was called via:
+#python training_data_benchmark_norm_cos.py --name codesearch-full --model distilroberta-base --steps 10000 --data data/codesearchnet.jsonl.gz