Spaces:
Running
Running
fix bug evaluation code
Browse files
app.py
CHANGED
|
@@ -419,21 +419,33 @@ def search_kbli(text: str):
|
|
| 419 |
html += "</div>"
|
| 420 |
return html
|
| 421 |
|
| 422 |
-
def calculate_mrr(retrieved_kodes,
|
| 423 |
for i, kode in enumerate(retrieved_kodes):
|
| 424 |
-
if kode in
|
|
|
|
| 425 |
return 0.0
|
| 426 |
|
| 427 |
-
def calculate_recall(retrieved_kodes,
|
| 428 |
retrieved_k_set = set(retrieved_kodes[:k])
|
| 429 |
-
|
| 430 |
-
if not
|
| 431 |
-
|
|
|
|
|
|
|
| 432 |
|
| 433 |
def calculate_ndcg(retrieved_kodes, relevance_dict, k=10):
|
| 434 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
ideal_rels = sorted(relevance_dict.values(), reverse=True)[:k]
|
| 436 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 437 |
return dcg / idcg if idcg > 0 else 0.0
|
| 438 |
|
| 439 |
def run_evaluation(file_obj, scenario):
|
|
@@ -475,7 +487,7 @@ def run_evaluation(file_obj, scenario):
|
|
| 475 |
retrieved_kodes = [str(r.get('kode')) for r in candidates]
|
| 476 |
|
| 477 |
rel_dict = ground_truth.get(q_id, {})
|
| 478 |
-
|
| 479 |
|
| 480 |
mrr = calculate_mrr(retrieved_kodes, relevant_kodes)
|
| 481 |
recall = calculate_recall(retrieved_kodes, relevant_kodes, k=10)
|
|
|
|
| 419 |
html += "</div>"
|
| 420 |
return html
|
| 421 |
|
| 422 |
+
def calculate_mrr(retrieved_kodes, relevant_kodes_set):
|
| 423 |
for i, kode in enumerate(retrieved_kodes):
|
| 424 |
+
if kode in relevant_kodes_set:
|
| 425 |
+
return 1.0 / (i + 1)
|
| 426 |
return 0.0
|
| 427 |
|
| 428 |
+
def calculate_recall(retrieved_kodes, relevant_kodes_set, k=10):
|
| 429 |
retrieved_k_set = set(retrieved_kodes[:k])
|
| 430 |
+
|
| 431 |
+
if not relevant_kodes_set:
|
| 432 |
+
return 0.0
|
| 433 |
+
|
| 434 |
+
return len(retrieved_k_set & relevant_kodes_set) / len(relevant_kodes_set)
|
| 435 |
|
| 436 |
def calculate_ndcg(retrieved_kodes, relevance_dict, k=10):
|
| 437 |
+
|
| 438 |
+
dcg = 0
|
| 439 |
+
for i, kode in enumerate(retrieved_kodes[:k]):
|
| 440 |
+
rel = relevance_dict.get(kode, 0)
|
| 441 |
+
dcg += rel / math.log2(i + 2)
|
| 442 |
+
|
| 443 |
ideal_rels = sorted(relevance_dict.values(), reverse=True)[:k]
|
| 444 |
+
|
| 445 |
+
idcg = 0
|
| 446 |
+
for i, rel in enumerate(ideal_rels):
|
| 447 |
+
idcg += rel / math.log2(i + 2)
|
| 448 |
+
|
| 449 |
return dcg / idcg if idcg > 0 else 0.0
|
| 450 |
|
| 451 |
def run_evaluation(file_obj, scenario):
|
|
|
|
| 487 |
retrieved_kodes = [str(r.get('kode')) for r in candidates]
|
| 488 |
|
| 489 |
rel_dict = ground_truth.get(q_id, {})
|
| 490 |
+
relevant_kodes_set = {k for k, r in rel_dict.items() if r > 0}
|
| 491 |
|
| 492 |
mrr = calculate_mrr(retrieved_kodes, relevant_kodes)
|
| 493 |
recall = calculate_recall(retrieved_kodes, relevant_kodes, k=10)
|