| { |
| "name": "CoIR (NDCG@10)", |
| "tasks": { |
| "CodeSearchNet-go": 68.71, |
| "CodeSearchNet-javascript": 50.41, |
| "CodeSearchNet-ruby": 56.52, |
| "CodeSearchNet-python": 87.84, |
| "CodeSearchNet-php": 53.36, |
| "synthetic-text2sql": 35.73, |
| "codefeedback-mt": 21.78, |
| "codefeedback-st": 48.58, |
| "codetrans-dl": 24.12, |
| "cosqa": 25.95, |
| "stackoverflow-qa": 55.35, |
| "apps": 2.83 |
| }, |
| "baselines": { |
| "v1 (file-level pairs)": { |
| "stackoverflow-qa": 58.4, |
| "cosqa": 20.91, |
| "apps": 3.08, |
| "codetrans-dl": 28.14, |
| "CodeSearchNet-ruby": 40.68, |
| "CodeSearchNet-javascript": 38.73, |
| "CodeSearchNet-go": 53.43, |
| "CodeSearchNet-java": 37.87 |
| }, |
| "BM25": { |
| "cosqa": 13.96, |
| "stackoverflow-qa": 56.8, |
| "codetrans-dl": 50.13 |
| }, |
| "UniXcoder": { |
| "cosqa": 25.14, |
| "stackoverflow-qa": 44.67, |
| "codetrans-dl": 41.82 |
| }, |
| "GTE-Base": { |
| "cosqa": 30.24, |
| "stackoverflow-qa": 62.71, |
| "codetrans-dl": 33.81 |
| }, |
| "E5-Base": { |
| "cosqa": 32.59, |
| "stackoverflow-qa": 86.86, |
| "codetrans-dl": 62.5 |
| } |
| }, |
| "note": "Measured with the coir-eval harness using mean pooling, 256 tokens and cosine similarity. The 'v1 (file-level pairs)' column is an earlier build of this model, run on the same machine with the same code. Blank cells were not measured.\n\nBM25, UniXcoder, GTE-Base and E5-Base come from Table 3 of the CoIR paper (arXiv:2407.02883). That table reports CodeSearchNet as one combined score across six languages, while the scores here are per language, so those cells are left blank rather than compared to something different.\n\nNo average is given. Two CoIR tests were not run, and an average over part of the benchmark is not the published average." |
| } |
