File size: 11,653 Bytes
bc37111 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 |
"""
Dataset Tab Component
Displays task and dataset information.
"""
import gradio as gr
import pandas as pd
import html
class DatasetTab:
"""
Dataset information tab component.
Shows details about the evaluation tasks and datasets.
"""
def build(self) -> None:
"""Build the dataset tab UI."""
gr.Markdown("### MTEB Turkish + Turkish Legal Dataset Overview")
# Task name to dataset path mapping
task_to_dataset = {
'WebFAQRetrieval': 'PaDaS-Lab/webfaq-retrieval',
'XQuADRetrieval': 'google/xquad',
'TurHistQuadRetrieval': 'asparius/TurHistQuAD',
'MKQARetrieval': 'apple/mkqa',
'MassiveIntentClassification': 'mteb/amazon_massive_intent',
'MassiveScenarioClassification': 'mteb/amazon_massive_scenario',
'MultilingualSentimentClassification': 'mteb/multilingual-sentiment-classification',
'SIB200Classification': 'mteb/sib200',
'TurkishMovieSentimentClassification': 'asparius/Turkish-Movie-Review',
'TurkishProductSentimentClassification': 'asparius/Turkish-Product-Review',
'SIB200ClusteringS2S': 'mteb/sib200',
'XNLI': 'mteb/xnli',
'XNLIV2': 'mteb/xnli2.0-multi-pair',
'STS22.v2': 'mteb/sts22-crosslingual-sts'
}
# Create clickable task names
clickable_task_names = []
task_list = [
'WebFAQRetrieval', 'XQuADRetrieval', 'TurHistQuadRetrieval', 'MKQARetrieval',
'MassiveIntentClassification', 'MassiveScenarioClassification',
'MultilingualSentimentClassification', 'SIB200Classification',
'TurkishMovieSentimentClassification', 'TurkishProductSentimentClassification',
'SIB200ClusteringS2S', 'XNLI', 'XNLIV2', 'STS22.v2'
]
for task_name in task_list:
dataset_path = task_to_dataset[task_name]
hf_link = f"https://huggingface.co/datasets/{html.escape(dataset_path)}"
clickable_name = f'<a href="{hf_link}" target="_blank" style="color: #2563eb; text-decoration: underline;">{html.escape(task_name)}</a>'
clickable_task_names.append(clickable_name)
# Create dataset information table
dataset_data = pd.DataFrame({
'Task Name': clickable_task_names,
'Task Type': [
'Retrieval', 'Retrieval', 'Retrieval', 'Retrieval',
'Classification', 'Classification',
'Classification', 'Classification',
'Classification', 'Classification',
'Clustering', 'PairClassification', 'PairClassification', 'STS'
],
'Description': [
'Turkish FAQ retrieval task',
'Turkish question answering retrieval',
'Historical Turkish document retrieval',
'Multilingual knowledge QA retrieval',
'Intent classification for Turkish',
'Scenario classification for Turkish',
'Multilingual sentiment classification',
'SIB200 language identification',
'Turkish movie review sentiment',
'Turkish product review sentiment',
'SIB200 clustering task',
'Turkish natural language inference',
'Enhanced Turkish NLI task',
'Turkish semantic textual similarity'
],
'Domain': [
'FAQ/QA', 'QA', 'Historical', 'Knowledge QA',
'Intent', 'Scenario',
'Sentiment', 'Language ID',
'Movies', 'Products',
'Language ID', 'NLI', 'NLI', 'STS'
],
'Samples': [
'~145K', '~1.19K', '~1.33K', '~10K',
'~5K', '~5K',
'211', '~899',
'~2.64K', '800',
'99', '~7.5K', '~5.01K', '~208'
]
})
gr.Dataframe(
value=dataset_data,
label="MTEB Turkish Task Details",
interactive=False,
wrap=True,
datatype=["html", "str", "str", "str", "str"]
)
# Turkish Legal Tasks Section
self._build_legal_tasks_section()
# Task distribution
self._build_task_distribution_section()
# Metrics explanation
self._build_metrics_explanation_section()
def _build_legal_tasks_section(self):
"""Build the Turkish Legal Tasks section."""
gr.Markdown("---")
gr.Markdown("### Turkish Legal Tasks")
legal_task_to_dataset = {
'TurkishLegalQA': 'newmindai/contract-retrieval',
'TurkishTaxRulings': 'newmindai/regulation-retrieval',
'TurkishCourtOfCassation': 'newmindai/caselaw-retrieval'
}
clickable_legal_task_names = []
for task_name in ['TurkishLegalQA', 'TurkishTaxRulings', 'TurkishCourtOfCassation']:
dataset_path = legal_task_to_dataset[task_name]
hf_link = f"https://huggingface.co/datasets/{html.escape(dataset_path)}"
clickable_name = f'<a href="{hf_link}" target="_blank" style="color: #2563eb; text-decoration: underline;">{html.escape(task_name)}</a>'
clickable_legal_task_names.append(clickable_name)
legal_task_data = pd.DataFrame({
'Task Name': clickable_legal_task_names,
'Task Type': ['Contracts', 'Regulation', 'Case Law'],
'Description': [
'Turkish legal question answering retrieval',
'Turkish legal tax rulings retrieval',
'Turkish Court of Cassation caselaw retrieval'
],
'Domain': ['Contracts', 'Regulation', 'Caselaw'],
'Samples': ['272', '~120K', '~1.39K']
})
gr.Dataframe(
value=legal_task_data,
label="Turkish Legal Task Details",
interactive=False,
wrap=True,
datatype=["html", "str", "str", "str", "str"]
)
def _build_task_distribution_section(self):
"""Build the task distribution section."""
gr.Markdown("""
### Task Distribution:
**Turkish Tasks (14):**
- **Classification**: 6 tasks (sentiment, intent, scenario, language identification)
- **Retrieval**: 4 tasks (FAQ, QA, historical documents, knowledge QA)
- **Pair Classification**: 2 tasks (natural language inference)
- **Clustering**: 1 task (language clustering)
- **STS**: 1 task (semantic textual similarity)
**Turkish Legal Tasks (3):**
- **Contracts**: 1 task (Turkish legal QA retrieval)
- **Regulation**: 1 task (Turkish tax rulings retrieval)
- **Caselaw**: 1 task (Turkish Court of Cassation case law retrieval)
**Total: 17 tasks across 8 categories**
""")
# Statistics summary
stats_data = pd.DataFrame({
'Metric': [
'Total Tasks',
'Turkish Tasks',
'Legal Tasks',
'Task Categories',
'Languages',
'Avg. Tokens per Sample'
],
'Value': [
'17 tasks',
'14 tasks',
'3 tasks',
'8 categories',
'Turkish',
'~150 tokens'
],
'Notes': [
'Comprehensive evaluation: Turkish NLP + Legal',
'Classification, Retrieval, STS, NLI, Clustering',
'Contracts, Regulation, Caselaw',
'Turkish: 5 types, Legal: 3 types',
'Turkish-focused',
'Varies by task type and domain'
]
})
gr.Dataframe(
value=stats_data,
label="Dataset Statistics Summary",
interactive=False
)
def _build_metrics_explanation_section(self):
"""Build the metrics explanation section."""
gr.Markdown("""
---
### Metrics Explanation:
**Task Categories:**
- **MTEB Score**: Average performance by task categories (refers to Mean (TaskType))
- **Mean (Task)**: Average performance across all individual tasks
- **Classification**: Performance on Turkish classification tasks
- **Clustering**: Performance on Turkish clustering tasks
- **Pair Classification**: Performance on pair classification tasks (like NLI)
- **Retrieval**: Performance on Turkish information retrieval tasks
- **STS**: Performance on Semantic Textual Similarity tasks
**Turkish Legal Categories:**
- **Contracts**: Performance on Turkish legal contract analysis tasks
- **Regulation**: Performance on Turkish legal regulation analysis tasks
- **Caselaw**: Performance on Turkish Court of Cassation case law retrieval tasks
### Tokenizer Quality Metrics:
- **Unique Token Count**: Number of unique tokens generated by the tokenizer on Turkish MMLU dataset
- **Turkish Token Count**: How many unique tokens are valid Turkish words/morphemes
- **Turkish Token %**: Percentage of unique tokens that are linguistically valid Turkish
- **Pure Token Count**: How many unique tokens are morphologically pure (root words)
- **Pure Token %**: Percentage of unique tokens that are root words without suffixes
### Model Information:
- **Parameters**: Number of model parameters
- **Embed Dim**: Embedding dimension size
- **Max Seq Length**: Maximum sequence length the model can process
- **Vocab Size**: Size of the model's vocabulary
- **Model Architecture**: The underlying model architecture
- **Tokenizer Type**: The tokenizer implementation used
""")
# About, Contact, and Links section
self._build_about_section()
def _build_about_section(self):
"""Build the about, contact, and links section."""
gr.Markdown("""
---
### About Mizan:
This leaderboard presents results from the **Mizan** benchmark, which evaluates embedding models
on Turkish language tasks across multiple domains including:
- Text classification and sentiment analysis
- Information retrieval and search
- Semantic textual similarity
- Text clustering and pair classification
- **Turkish Legal**: Contract analysis, regulation, and case law retrieval
### Submit Your Model:
Use the **Submit** tab to submit your Turkish embedding model for evaluation.
Your request will be reviewed by administrators and you'll receive email notifications about the progress.
### Contact:
For any questions or feedback, please contact info@newmind.ai
### Links:
- **GitHub**: [embeddings-benchmark/mteb v1.38.51](https://github.com/embeddings-benchmark/mteb/tree/1.38.51) - Mizan is currently based on MTEB v1.38.51 (MTEB v2.0.0 support coming soon)
- **Github**: [malibayram/tokenizer_benchmark](https://github.com/malibayram/tokenizer_benchmark) - Tokenizer evaluation is done with code from this repository, developed by Mehmet Ali Bayram, which utilizes ITU NLP tools for Turkish linguistic analysis.
""")
|