Release CodeEmbed v2: Multilingual code embeddings (MTEB 40.64% NDCG@10)

1_Pooling/config.json

@@ -0,0 +1,5 @@
+{
+    "embedding_dimension": 768,
+    "pooling_mode": "mean",
+    "include_prompt": true
+}

2_Dense/config.json

@@ -0,0 +1,8 @@
+{
+    "in_features": 768,
+    "out_features": 1536,
+    "bias": true,
+    "activation_function": "torch.nn.modules.activation.GELU",
+    "module_input_name": "sentence_embedding",
+    "module_output_name": "sentence_embedding"
+}

2_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84c6062f52b38491ffab8f5d8765889210e1710b867ea202830d86104249dab8
+size 4724896

3_Dense/config.json

@@ -0,0 +1,8 @@
+{
+    "in_features": 1536,
+    "out_features": 1536,
+    "bias": true,
+    "activation_function": "torch.nn.modules.linear.Identity",
+    "module_input_name": "sentence_embedding",
+    "module_output_name": "sentence_embedding"
+}

3_Dense/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d74a1b8b40906a3b9b125c6f01af726354406e19caeabac9d983bee25a597cc4
+size 9443488

4_Normalize/config.json

@@ -0,0 +1,4 @@
+{
+    "module_input_name": "sentence_embedding",
+    "module_output_name": "sentence_embedding"
+}

LICENSE

@@ -0,0 +1,49 @@
+Creative Commons Attribution-NonCommercial 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions.
+
+Section 1 – Definitions.
+
+a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor.
+b. Commercial means primarily intended for or directed towards commercial advantage or monetary compensation.
+c. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized.
+d. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License (including CodeEmbed model weights, architectures, and associated materials).
+e. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license.
+f. Licensor means Divakant Pandey.
+g. NonCommercial means not primarily intended for or directed towards commercial advantage or monetary compensation.
+h. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation.
+i. You means the individual or entity exercising the Licensed Rights under this Public License.
+
+Section 2 – Scope.
+
+a. License grant.
+   1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to:
+      a. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and
+      b. produce, reproduce, and Share Adapted Material for NonCommercial purposes only.
+   2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions.
+   3. Term. The term of this Public License is specified in Section 6(a).
+   4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so.
+
+b. Other rights.
+   1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights.
+   2. Patent and trademark rights are not licensed under this Public License.
+
+Section 3 – License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the following conditions.
+
+a. Attribution.
+   1. If You Share the Licensed Material (including in modified form), You must:
+      a. retain the identification of the creator(s) of the Licensed Material: Divakant Pandey (https://github.com/divakantpandey);
+      b. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and
+      c. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License.
+   2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material.
+
+b. NonCommercial.
+   You may not exercise any of the Licensed Rights granted under Section 2(a) for Commercial purposes.
+
+For inquiries regarding commercial licensing, enterprise deployment, or fine-tuning partnerships:
+Contact: Divakant Pandey
+Email: divakant01@gmail.com
+GitHub: https://github.com/divakantpandey
+

README.md

@@ -0,0 +1,343 @@
+---
+language:
+  - en
+  - code
+tags:
+  - python
+  - javascript
+  - go
+  - java
+  - ruby
+  - php
+  - code-search
+  - code-embeddings
+  - multilingual
+  - sentence-transformers
+  - matryoshka
+  - unixcoder
+  - mteb
+  - feature-extraction
+license: cc-by-nc-4.0
+pipeline_tag: feature-extraction
+library_name: sentence-transformers
+metrics:
+  - mrr
+  - ndcg
+  - map
+model-index:
+  - name: code-embed-v2
+    results:
+      - task:
+          type: retrieval
+          name: MTEB(Code, v1)
+        dataset:
+          name: MTEB(Code, v1) Macro-Average
+          type: mteb/code
+        metrics:
+          - type: ndcg_at_10
+            value: 0.4064
+            name: NDCG@10 (meanTask)
+          - type: mrr_at_10
+            value: 0.3781
+            name: MRR@10 (meanTask)
+      - task:
+          type: retrieval
+          name: CoIR
+        dataset:
+          name: CoIR Benchmark Macro-Average
+          type: coir
+        metrics:
+          - type: ndcg_at_10
+            value: 0.3729
+            name: NDCG@10 (meanTask)
+          - type: mrr_at_10
+            value: 0.3478
+            name: MRR@10 (meanTask)
+      - task:
+          type: retrieval
+          name: CoREB(v1)
+        dataset:
+          name: CoREB(v1) Macro-Average
+          type: coreb
+        metrics:
+          - type: ndcg_at_10
+            value: 0.2711
+            name: NDCG@10 (meanTask)
+      - task:
+          type: retrieval
+          name: Code Search
+        dataset:
+          name: CodeSearchNetRetrieval
+          type: code_search_net
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.7778
+            name: NDCG@10 (Multilingual)
+          - type: mrr_at_10
+            value: 0.7385
+            name: MRR@10 (Multilingual)
+      - task:
+          type: retrieval
+          name: COIR Code Search
+        dataset:
+          name: COIRCodeSearchNetRetrieval
+          type: coir_code_search_net
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.7443
+            name: NDCG@10 (Multilingual)
+          - type: mrr_at_10
+            value: 0.7137
+            name: MRR@10 (Multilingual)
+      - task:
+          type: retrieval
+          name: StackOverflow QA
+        dataset:
+          name: StackOverflowQA
+          type: stackoverflow_qa
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.5530
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.5159
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Code Feedback (Single-Turn)
+        dataset:
+          name: CodeFeedbackST
+          type: code_feedback_st
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.4693
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.4251
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Code-to-Code Search
+        dataset:
+          name: CodeSearchNetCCRetrieval
+          type: code_search_net_cc
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.4170
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.3784
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Code Edit Search
+        dataset:
+          name: CodeEditSearchRetrieval
+          type: code_edit_search
+          split: train
+        metrics:
+          - type: ndcg_at_10
+            value: 0.3703
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.3283
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Cross-Language Translation
+        dataset:
+          name: CodeTransOceanContest
+          type: code_trans_ocean_contest
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.3640
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.3150
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Framework Translation
+        dataset:
+          name: CodeTransOceanDL
+          type: code_trans_ocean_dl
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.3439
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.2204
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Text to SQL
+        dataset:
+          name: SyntheticText2SQL
+          type: synthetic_text2sql
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.3277
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.2513
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Code Feedback (Multi-Turn)
+        dataset:
+          name: CodeFeedbackMT
+          type: code_feedback_mt
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.3188
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.2882
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Web Query Code Search
+        dataset:
+          name: CosQA
+          type: cosqa
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.1599
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.1135
+            name: MRR@10
+      - task:
+          type: retrieval
+          name: Algorithmic Problem Retrieval
+        dataset:
+          name: AppsRetrieval
+          type: apps_retrieval
+          split: test
+        metrics:
+          - type: ndcg_at_10
+            value: 0.0306
+            name: NDCG@10
+          - type: mrr_at_10
+            value: 0.0241
+            name: MRR@10
+---
+
+# 💻 code-embed-v2
+
+**CodeEmbed v2** is a lightweight, state-of-the-art multilingual code embedding model (≤131M parameters) developed by **[Divakant Pandey](https://github.com/divakantpandey)**. Specialized for semantic code search, code-to-code retrieval, text-to-SQL mapping, and repository navigation across **6 major programming languages** with an extended **8,192 token context window** and native **Matryoshka Representation Learning (MRL)**.
+
+---
+
+## 🚀 Highlights & Functional Capabilities
+
+- **Multilingual Support**: High-accuracy semantic search across **Python, JavaScript, Go, Java, Ruby, and PHP**.
+- **Extended Context**: Full **8,192 token context window** allowing whole-file, multi-function, and long class embeddings without aggressive truncation.
+- **Matryoshka Representation Learning (MRL)**: Embeddings can be dynamically truncated from **1536** down to **768, 384, 256, 128, or 64 dimensions** with negligible loss in accuracy, enabling significant vector database memory and latency savings.
+- **Compact & Efficient**: ~131M parameters (**~484 MB** safetensors footprint) optimized for low-latency CPU inference and fast GPU batch indexing.
+- **Universal Compatibility**: Works out of the box with `sentence-transformers`, `transformers`, and PyTorch.
+
+---
+
+## 📊 Benchmark Performance
+
+CodeEmbed v2 was thoroughly evaluated across **18 official benchmark tasks** comprising the primary code retrieval suites on the Massive Text Embedding Benchmark (MTEB):
+
+### 🏆 Benchmark Suite Macro-Averages (`meanTask`)
+
+| Benchmark Suite | Total Tasks | Metric | Official Macro-Average | Coverage |
+| :--- | :---: | :---: | :---: | :---: |
+| **`MTEB(Code, v1)`** | **12 / 12** | **NDCG@10** | **`40.64%`** | **100% (Complete)** |
+| **`CoIR` (Code IR)** | **10 / 10** | **NDCG@10** | **`37.29%`** | **100% (Complete)** |
+| **`CoREB(v1)`** | **6 / 6** | **NDCG@10** | **`27.11%`** | **100% (Complete)** |
+
+### 📋 Detailed Task-by-Task Scorecard
+
+| Task | Category | NDCG@10 | MRR@10 | Evaluation Scope |
+| :--- | :--- | :---: | :---: | :--- |
+| **`CodeSearchNetRetrieval`** | Text-to-Code Search | **77.78%** | 73.85% | 6 Languages (Python, JS, Go, Java, Ruby, PHP) |
+| **`COIRCodeSearchNetRetrieval`** | Multilingual Search | **74.43%** | 71.37% | Full-scale 300k+ codebase corpus |
+| **`CorebC2TReranking`** | Code-to-Text Reranking | **66.74%** | 57.59% | Cross-lingual code description ranking |
+| **`StackOverflowQA`** | Developer Q&A Search | **55.30%** | 51.59% | Real-world developer query matching |
+| **`CodeFeedbackST`** | Instruction Feedback | **46.93%** | 42.51% | Single-turn instruction and code feedback |
+| **`CodeSearchNetCCRetrieval`** | Code-to-Code Search | **41.70%** | 37.84% | Cross-language code function matching |
+| **`CodeEditSearchRetrieval`** | Edit / Patch Search | **37.03%** | 32.83% | 13 Git commit and patch subsets |
+| **`CorebC2CReranking`** | Code-to-Code Reranking | **36.87%** | 34.40% | Direct code snippet similarity ranking |
+| **`CodeTransOceanContest`** | Code Translation | **36.40%** | 31.50% | Cross-language algorithmic translation |
+| **`CodeTransOceanDL`** | Framework Translation | **34.39%** | 22.04% | Deep learning API cross-matching |
+| **`SyntheticText2SQL`** | Text-to-SQL | **32.77%** | 25.13% | Natural language to structured SQL queries |
+| **`CodeFeedbackMT`** | Multi-Turn Feedback | **31.88%** | 28.82% | Conversational multi-turn code context |
+| **`CorebT2CReranking`** | Text-to-Code Reranking | **26.44%** | 27.65% | Natural language to code reranking |
+| **`CorebC2TRetrieval`** | Code-to-Text Search | **22.32%** | 18.83% | Source code to documentation retrieval |
+| **`CosQA`** | Web Query Code Search | **15.99%** | 11.35% | Real-world noisy web search queries |
+| **`CorebT2CRetrieval`** | Text-to-Code Search | **7.45%** | 8.10% | Zero-shot query to code retrieval |
+| **`AppsRetrieval`** | Algorithmic Problem Search | **3.06%** | 2.41% | Hard algorithmic programming specifications |
+| **`CorebC2CRetrieval`** | Code-to-Code Retrieval | **2.87%** | 2.25% | Direct code-to-code corpus retrieval |
+
+---
+
+## 💻 Quickstart Usage
+
+### 1. Basic Semantic Code Search with `sentence-transformers`
+
+```python
+from sentence_transformers import SentenceTransformer
+
+# Load model directly from Hugging Face Hub
+model = SentenceTransformer("divakantpandey/code-embed-v2")
+
+queries = [
+    "binary search implementation",
+    "fetch json from api endpoint",
+    "goroutine channel worker pool"
+]
+
+code_snippets = [
+    "def binary_search(arr, target):\n    left, right = 0, len(arr) - 1\n    while left <= right:\n        mid = (left + right) // 2\n        if arr[mid] == target: return mid\n        elif arr[mid] < target: left = mid + 1\n        else: right = mid - 1\n    return -1",
+    "async function fetchJSON(url) {\n    const response = await fetch(url);\n    return await response.json();\n}",
+    "func worker(id int, jobs <-chan int, results chan<- int) {\n    for j := range jobs {\n        results <- j * 2\n    }\n}"
+]
+
+# Generate normalized embeddings
+query_embeddings = model.encode(queries, normalize_embeddings=True)
+code_embeddings = model.encode(code_snippets, normalize_embeddings=True)
+
+# Compute cosine similarity
+similarity_matrix = query_embeddings @ code_embeddings.T
+print(similarity_matrix)
+```
+
+### 2. Matryoshka Dimension Truncation (Adaptive Dimensions)
+
+CodeEmbed v2 natively supports Matryoshka Representation Learning. You can truncate embeddings to smaller dimensions (e.g., 768, 384, 256) for faster vector database indexing with minimal accuracy impact:
+
+```python
+# Truncate to 768 dimensions (50% storage & compute reduction)
+compact_query_embs = model.encode(queries, truncate_dim=768, normalize_embeddings=True)
+compact_code_embs = model.encode(code_snippets, truncate_dim=768, normalize_embeddings=True)
+
+similarity = compact_query_embs @ compact_code_embs.T
+```
+
+---
+
+## 📜 License & Commercial Terms
+
+This model and its associated weights are licensed under the **Creative Commons Attribution-NonCommercial 4.0 International Public License (CC-BY-NC 4.0)**.
+
+- **Non-Commercial Use:** Free for academic research, educational use, personal exploration, open-source evaluations, and non-commercial benchmarking with appropriate attribution.
+- **Attribution:** Please attribute **Divakant Pandey** and reference the repository at [https://github.com/divakantpandey/code-embed](https://github.com/divakantpandey/code-embed).
+- **Commercial Licensing:** To license CodeEmbed v2 for commercial products, enterprise SaaS applications, closed-source systems, or proprietary developer tools, please contact:
+  - **Licensor / Maintainer:** Divakant Pandey
+  - **Email:** `divakant01@gmail.com`
+  - **GitHub:** [@divakantpandey](https://github.com/divakantpandey)

backbone/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19327aee05f08b6018ec4bfafef13c8f1174e75fd16a9be255f029d7ae90c668
+oid sha256:cee5b2e4cde154164d9dbabab7f8dedaecc81f5cc9e605b0ca1774ef38577837
 size 503741296

backbone/tokenizer_config.json

@@ -5,13 +5,20 @@
   "cls_token": "<s>",
   "eos_token": "</s>",
   "errors": "replace",
-  "is_local": false,
+  "is_local": true,
   "local_files_only": false,
   "mask_token": "<mask>",
-  "model_max_length": 1024,
+  "max_length": 1024,
+  "model_max_length": 8192,
+  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "</s>",
+  "stride": 0,
   "tokenizer_class": "RobertaTokenizer",
   "trim_offsets": true,
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

code_embed_config.json

@@ -1,5 +1,5 @@
 {
-  "base_model_name_or_path": "microsoft/unixcoder-base",
+  "base_model_name_or_path": "checkpoints_v2/stage2/last_model/backbone",
   "backbone_hidden_size": 768,
   "embedding_dim": 1536,
   "projection_hidden_dim": 1536,

config.json

@@ -0,0 +1,31 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "RobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 1026,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tie_word_embeddings": true,
+  "transformers_version": "5.20.1",
+  "type_vocab_size": 10,
+  "use_cache": true,
+  "vocab_size": 51416
+}

config_sentence_transformers.json

@@ -0,0 +1,14 @@
+{
+  "__version__": {
+    "pytorch": "2.6.0+cu124",
+    "sentence_transformers": "6.1.0",
+    "transformers": "5.20.1"
+  },
+  "default_prompt_name": null,
+  "model_type": "SentenceTransformer",
+  "prompts": {
+    "document": "",
+    "query": ""
+  },
+  "similarity_fn_name": "cosine"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cee5b2e4cde154164d9dbabab7f8dedaecc81f5cc9e605b0ca1774ef38577837
+size 503741296

modules.json

@@ -0,0 +1,32 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.base.modules.transformer.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.sentence_transformer.modules.pooling.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Dense",
+    "type": "sentence_transformers.base.modules.dense.Dense"
+  },
+  {
+    "idx": 3,
+    "name": "3",
+    "path": "3_Dense",
+    "type": "sentence_transformers.base.modules.dense.Dense"
+  },
+  {
+    "idx": 4,
+    "name": "4",
+    "path": "4_Normalize",
+    "type": "sentence_transformers.base.modules.normalize.Normalize"
+  }
+]

projection_head.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afd45d280664ff4c17c0926279327be1682d438a1955169f325c49c0d7db3b8f
+oid sha256:cab1e7ca82bb911a0b6362f2d543ae62d7053c74db66be77563faf8334b9d917
 size 14170320

pycode_embed_config.json

@@ -1,5 +1,5 @@
 {
-  "base_model_name_or_path": "microsoft/unixcoder-base",
+  "base_model_name_or_path": "checkpoints_v2/stage2/last_model/backbone",
   "backbone_hidden_size": 768,
   "embedding_dim": 1536,
   "projection_hidden_dim": 1536,

sentence_bert_config.json

@@ -0,0 +1,11 @@
+{
+  "transformer_task": "feature-extraction",
+  "modality_config": {
+    "text": {
+      "method": "forward",
+      "method_output_name": "last_hidden_state"
+    }
+  },
+  "module_output_name": "token_embeddings",
+  "max_seq_length": 8192
+}

tokenizer_config.json

@@ -5,13 +5,20 @@
   "cls_token": "<s>",
   "eos_token": "</s>",
   "errors": "replace",
-  "is_local": false,
+  "is_local": true,
   "local_files_only": false,
   "mask_token": "<mask>",
-  "model_max_length": 1024,
+  "max_length": 1024,
+  "model_max_length": 8192,
+  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "</s>",
+  "stride": 0,
   "tokenizer_class": "RobertaTokenizer",
   "trim_offsets": true,
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }