File size: 2,556 Bytes
6ced533 eb02943 6ced533 eb02943 6ced533 eb02943 6ced533 eb02943 6ced533 eb02943 6ced533 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | """Models under test and benchmark-wide settings.
`query_prefix` / `passage_prefix` are plain-text instruction prefixes prepended
before encoding (the E5 family expects "query: " / "passage: "; other model
families used here don't need one). This is applied manually rather than via
SentenceTransformer's `prompt_name=` API so behavior doesn't depend on whether
a given model repo happens to ship a config_sentence_transformers.json with
named prompts.
"""
MODELS = [
{
"name": "kazalbrur/bangla-embed-e5-small-banglish",
"query_prefix": "query: ",
"passage_prefix": "passage: ",
},
{
"name": "BAAI/bge-m3",
"query_prefix": "",
"passage_prefix": "",
},
{
# 4B/8B variants are skipped: their fp32 weights (~16GB / ~32GB) don't fit
# this machine's 8GB VRAM or its ~15GB system RAM.
"name": "Qwen/Qwen3-Embedding-0.6B",
"query_prefix": (
"Instruct: Given a web search query, retrieve relevant passages "
"that answer the query\nQuery:"
),
"passage_prefix": "",
},
{
"name": "intfloat/multilingual-e5-large",
"query_prefix": "query: ",
"passage_prefix": "passage: ",
},
{
# trust_remote_code is required to load Jina's custom modeling code; the
# model uses task-specific LoRA adapters selected via an encode()-time
# `task` kwarg rather than a text prefix.
"name": "jinaai/jina-embeddings-v3",
"query_prefix": "",
"passage_prefix": "",
"trust_remote_code": True,
"query_encode_kwargs": {"task": "retrieval.query"},
"passage_encode_kwargs": {"task": "retrieval.passage"},
},
{
"name": "sentence-transformers/LaBSE",
"query_prefix": "",
"passage_prefix": "",
},
{
# trust_remote_code is required to load Alibaba's custom modeling code.
"name": "Alibaba-NLP/gte-multilingual-base",
"query_prefix": "",
"passage_prefix": "",
"trust_remote_code": True,
},
]
QUERY_MODES = ["raw", "normalized"]
RETRIEVAL_MODES = ["dense", "hybrid"] # hybrid = BM25 + dense fused with RRF
TOP_K = 50 # depth retrieved per query; recall@10/@50 and nDCG@10 are sliced from this
FUSION_DEPTH = 50 # how deep each ranker (BM25, dense) is pulled before RRF fusion
RRF_K = 60 # RRF's rank-damping constant
BOOKS_PATH = "boi1_sample_books.json"
QUERIES_PATH = "boi1_eval_queries.json"
QDRANT_PATH = "./qdrant_data"
RESULTS_DIR = "./results"
|