Update app.py

app.py

@@ -48,6 +48,19 @@ def normalize_text(s: str):
     s = re.sub(r"\s+"," ",s).strip()
     return s
 
+import string
+
+def normalize_answer(text):
+
+    text = normalize_text(text)
+
+    # remove punctuation
+    text = re.sub(r"[^\w\s]", "", text)
+
+    # lowercase (safe for Hindi/Kannada)
+    text = text.lower()
+
+    return text.strip()
 
 df = pd.read_csv(CSV_PATH)
 df["context_norm"] = df["context"].apply(normalize_text)
@@ -158,27 +171,25 @@ def kn_to_en(text):
 
 
 
-def exact_match(pred,gold):
-
-    return int(normalize_text(pred)==normalize_text(gold))
-
+def exact_match(pred, gold):
 
-def token_f1(pred,gold):
+    return int(normalize_answer(pred) == normalize_answer(gold))
 
-    p=set(pred.split())
-    g=set(gold.split())
 
-    common=len(p & g)
+def token_f1(pred, gold):
 
-    if common==0:
+    pred_tokens = normalize_answer(pred).split()
+    gold_tokens = normalize_answer(gold).split()
 
-        return 0
+    common = set(pred_tokens) & set(gold_tokens)
 
-    precision=common/len(p)
+    if len(common) == 0:
+        return 0.0
 
-    recall=common/len(g)
+    precision = len(common) / len(pred_tokens)
+    recall = len(common) / len(gold_tokens)
 
-    return 2*precision*recall/(precision+recall)
+    return 2 * precision * recall / (precision + recall)
 
 
 def semantic_similarity(pred,gold):