allbibek commited on
Commit
f54c21c
·
verified ·
1 Parent(s): 22e39eb

fix bug evaluation code

Browse files
Files changed (1) hide show
  1. app.py +21 -9
app.py CHANGED
@@ -419,21 +419,33 @@ def search_kbli(text: str):
419
  html += "</div>"
420
  return html
421
 
422
- def calculate_mrr(retrieved_kodes, relevant_kodes):
423
  for i, kode in enumerate(retrieved_kodes):
424
- if kode in relevant_kodes: return 1.0 / (i + 1)
 
425
  return 0.0
426
 
427
- def calculate_recall(retrieved_kodes, relevant_kodes, k=10):
428
  retrieved_k_set = set(retrieved_kodes[:k])
429
- relevant_set = set(relevant_kodes)
430
- if not relevant_set: return 0.0
431
- return len(retrieved_k_set.intersection(relevant_set)) / len(relevant_set)
 
 
432
 
433
  def calculate_ndcg(retrieved_kodes, relevance_dict, k=10):
434
- dcg = sum(relevance_dict.get(k, 0) / math.log2(i + 2) for i, k in enumerate(retrieved_kodes[:k]))
 
 
 
 
 
435
  ideal_rels = sorted(relevance_dict.values(), reverse=True)[:k]
436
- idcg = sum(rel / math.log2(i + 2) for i, rel in enumerate(ideal_rels))
 
 
 
 
437
  return dcg / idcg if idcg > 0 else 0.0
438
 
439
  def run_evaluation(file_obj, scenario):
@@ -475,7 +487,7 @@ def run_evaluation(file_obj, scenario):
475
  retrieved_kodes = [str(r.get('kode')) for r in candidates]
476
 
477
  rel_dict = ground_truth.get(q_id, {})
478
- relevant_kodes = list(rel_dict.keys())
479
 
480
  mrr = calculate_mrr(retrieved_kodes, relevant_kodes)
481
  recall = calculate_recall(retrieved_kodes, relevant_kodes, k=10)
 
419
  html += "</div>"
420
  return html
421
 
422
+ def calculate_mrr(retrieved_kodes, relevant_kodes_set):
423
  for i, kode in enumerate(retrieved_kodes):
424
+ if kode in relevant_kodes_set:
425
+ return 1.0 / (i + 1)
426
  return 0.0
427
 
428
+ def calculate_recall(retrieved_kodes, relevant_kodes_set, k=10):
429
  retrieved_k_set = set(retrieved_kodes[:k])
430
+
431
+ if not relevant_kodes_set:
432
+ return 0.0
433
+
434
+ return len(retrieved_k_set & relevant_kodes_set) / len(relevant_kodes_set)
435
 
436
  def calculate_ndcg(retrieved_kodes, relevance_dict, k=10):
437
+
438
+ dcg = 0
439
+ for i, kode in enumerate(retrieved_kodes[:k]):
440
+ rel = relevance_dict.get(kode, 0)
441
+ dcg += rel / math.log2(i + 2)
442
+
443
  ideal_rels = sorted(relevance_dict.values(), reverse=True)[:k]
444
+
445
+ idcg = 0
446
+ for i, rel in enumerate(ideal_rels):
447
+ idcg += rel / math.log2(i + 2)
448
+
449
  return dcg / idcg if idcg > 0 else 0.0
450
 
451
  def run_evaluation(file_obj, scenario):
 
487
  retrieved_kodes = [str(r.get('kode')) for r in candidates]
488
 
489
  rel_dict = ground_truth.get(q_id, {})
490
+ relevant_kodes_set = {k for k, r in rel_dict.items() if r > 0}
491
 
492
  mrr = calculate_mrr(retrieved_kodes, relevant_kodes)
493
  recall = calculate_recall(retrieved_kodes, relevant_kodes, k=10)