first update

app.py

@@ -6,6 +6,7 @@ import requests
 import os
 from langdetect import detect
 import pandas as pd
+from utils import *
 
 # Load the source dataset
 source_dataset = load_dataset("vietdata/eng_echo", split="train")

requirements.txt

@@ -0,0 +1,3 @@
+deep-translator
+langdetect
+thefuzz

utils.py

@@ -0,0 +1,17 @@
+from thefuzz import fuzz
+from deep_translator import GoogleTranslator
+import time
+
+def gg_score(original, translation, target="en"):
+    while True:
+        try:
+            back_original = GoogleTranslator(source='auto', target=target).translate(translation)
+            gg_translation = GoogleTranslator(source='auto', target=target).translate(original)
+            break
+        except:
+            time.sleep(1)
+            continue
+    return (fuzz.token_set_ratio(original, back_original) + fuzz.token_set_ratio(gg_translation, translation))/200
+
+def miner_score(translation, grouth):
+    return fuzz.token_set_ratio(translation, grouth)/100