update text comparison

lib/report.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass, astuple
 
 from tabulate import tabulate
 from lib.log_parser import LogTag, LogItem, WebItem
-from lib.utils import save_csv, run_textdistance, highlight_diff, time_to_float
+from lib.utils import *
 
 class LogReport:
     """用于处理 log文件"""
@@ -240,8 +240,18 @@ class AccuracyItem:
     asr_accuracy: tuple= (0,1)
     text_compare: str = ""
     def __post_init__(self):
-        self.asr_accuracy = run_textdistance(self.audio_text, self.src_text)
-        self.text_compare = highlight_diff(self.audio_text, self.src_text)
+
+        if self.translation_type == "en2zh":
+            text1 = clean_text_for_comparison_en(self.audio_text)
+            text2 = clean_text_for_comparison_en(self.src_text)
+            spliter = " "
+        else:
+            text1 = clean_text_for_comparison_zh(self.audio_text)
+            text2 = clean_text_for_comparison_zh(self.src_text)
+            spliter = ""
+        self.asr_accuracy = run_textdistance(text1, text2)
+        self.text_compare = highlight_diff(text1, text2, spliter)
+
     def to_list(self):
         return [self.audio, self.translation_type, self.audio_length,
                 self.asr_accuracy[0], self.asr_accuracy[1],

lib/temp.py

@@ -0,0 +1,41 @@
+import re
+import difflib
+
+
+def replace_symbol(text):
+    symbol_pattern = "[,.，。、!?\n]"
+    to = ""
+    return re.sub(symbol_pattern, to, text).lower()
+def tokenize(text):
+    # return re.findall(r'\w+|\s+|[^\w\s]', text)
+    return text.split(" ")
+def highlight_diff(a, b):
+    a, b = replace_symbol(a), replace_symbol(b)
+    tokens_a = tokenize(a)
+    tokens_b = tokenize(b)
+    print(tokens_a)
+    print(tokens_b)
+    matcher = difflib.SequenceMatcher(None, tokens_a, tokens_b)
+    output = []
+
+    for tag, a_start, a_end, b_start, b_end in matcher.get_opcodes():
+        if tag == 'equal':
+            output.append(' '.join(tokens_a[a_start:a_end]))
+        elif tag == 'delete':
+            deleted = ' '.join(tokens_a[a_start:a_end])
+            output.append(f"[-{deleted}-]")
+        elif tag == 'insert':
+            inserted = ' '.join(tokens_b[b_start:b_end])
+            output.append(f"{{+{inserted}+}}")
+        elif tag == 'replace':
+            deleted = ' '.join(tokens_a[a_start:a_end])
+            inserted = ' '.join(tokens_b[b_start:b_end])
+            output.append(f"[-{deleted}-]{{+{inserted}+}}")
+
+    return ' '.join(output)
+
+
+if __name__ == '__main__':
+    text_1 = "We built computers to expand our brains. Originally scientists built computers to solve arithmetic, but they turned out to be incredibly useful for many other things as well. Running the entire internet, lifelike graphics, artificial brains, or simulating the universe. But amazingly all of it boils down to just flipping zeros and ones. Computers have become smaller and more powerful at an incredible rate. There was more computing power in your cell phone than there was in the entire world in the mid-1960s. And the entire Apollo moon landing could have been run on a couple of Nintendos. Computer science is a subject that studies what computers can do. It's a diverse and overlapping field, but I'm going to split it into three parts. The fundamental theory of computer science, computer engineering, and applications."
+    text_2 = "We built computers to expand our brains. Originally, scientists built computers to solve arithmetic, but they turned out to be incredibly useful for many other things as well. running the entire internet, life-like graphics, artificial brains, or simulating the universe. But amazingly, all of it boils down to just flipping zeros and ones. Computers have become smaller, more powerful and at an incredible rate. There's more computing power in your cell phone than there was in the entire world in the mid-1960s. And the entire Apollo moon landing could have been run on a couple of Nintendos. Computer science is a subject that studies what computers can do. It's a diverse and overlapping field, but I'm going to split it into three parts. The fundamental theory of computer science, computer engineering, and communications. "
+    print(highlight_diff(text_1, text_2))

lib/utils.py

@@ -37,34 +37,45 @@ def cmd(command: str, check=True, capture_output=False) -> CompletedProcess:
     print(ret.stdout)
     return ret
 
-def replace_symbol(text):
-    symbol_pattern = "[,.，。!?\n]"
+def clean_text_for_comparison_zh(text):
+    symbol_pattern = "[ ，。、!?\n]"
+    to = ""
+    return re.sub(symbol_pattern, to, text).lower()
+
+def clean_text_for_comparison_en(text):
+    symbol_pattern = "[,.\n]"
     to = ""
     return re.sub(symbol_pattern, to, text).lower()
 
 
 def run_textdistance(text1, text2):
-    text1 = replace_symbol(text1)
-    text2 = replace_symbol(text2)
     d = textdistance.levenshtein.distance(text1, text2)
     nd = d / len(text1)
     # print("Levenshtein distance of texts:", d, "normalized distance is:", nd)
     return d, nd
 
-def highlight_diff(a, b):
-    a, b = replace_symbol(a), replace_symbol(b)
+def highlight_diff(a, b, spliter=""):
+    if spliter:
+        a = a.split(spliter)
+        b = b.split(spliter)
     matcher = difflib.SequenceMatcher(None, a, b)
     output = []
+
     for tag, a_start, a_end, b_start, b_end in matcher.get_opcodes():
         if tag == 'equal':
-            output.append(a[a_start:a_end])
+            output.append(spliter.join(a[a_start:a_end]))
         elif tag == 'delete':
-            output.append(f"[-{a[a_start:a_end]}-]")
+            deleted = spliter.join(a[a_start:a_end])
+            output.append(f"[-{deleted}-]")
         elif tag == 'insert':
-            output.append(f"{{+{b[b_start:b_end]}+}}")
+            inserted = spliter.join(b[b_start:b_end])
+            output.append(f"{{+{inserted}+}}")
         elif tag == 'replace':
-            output.append(f"[-{a[a_start:a_end]}-]{{+{b[b_start:b_end]}+}}")
-    return ''.join(output)
+            deleted = spliter.join(a[a_start:a_end])
+            inserted = spliter.join(b[b_start:b_end])
+            output.append(f"[-{deleted}-]{{+{inserted}+}}")
+
+    return spliter.join(output)
 
 def time_to_float(s: str):
     if d := s.replace("s", ""):
@@ -73,7 +84,14 @@ def time_to_float(s: str):
 
 
 if __name__ == '__main__':
-    text_1 = "This sequence of events is an example of what is known as the Butterfly Effect, a manifestation of Chaos Theory. For many centuries, the world was explained through the laws of Isaac Newton in classical physics. According to these laws, if the current state of an object is known, its future behavior can be predicted with relative ease. Chaos Theory questions this deterministic vision. Not everything is predictable anymore, nor does it work like a quirk. Since the 1800s, mathematicians have raised the idea that not all phenomena could be predicted by Newtonian laws. But a meteorologist named Edward Lawrence made Chaos Theory a visible phenomenon. It all started in 1961, when he was working on a mathematical model to forecast the weather. Lawrence entered data such as temperature, humidity, pressure, and wind direction into his computer. His computer would draw a graph modeling what the weather would be like. Not always accurate, but very close to reality. G."
-    text_2 = "This sequence of events is an example of what is known as the butterfly effect, a manifestation of chaos theory. For many centuries, the world was explained through the laws of Isaac Newton in classical physics. According to these laws, if the current state of an object is known, Its future behavior can be predicted with relative ease. Chaos theory questions this deterministic vision. Not everything is predictable anymore. nor does it work like clockwork. Since the 1800s, mathematicians have raised the idea that not all phenomena could be predicted by Newtonian laws. But a meteorologist named Edward Lawrence made Kale's theory a visible phenomenon. It all started in 1961, when he was working on a mathematical model to forecast the weather. Lawrence entered data such as temperature, humidity, pressure, and wind direction into his computer. His computer would draw a graph, modeling what the weather would be like. Not always accurate, but very close to reality. "
+    text_1 = "This sequence of events is an example of what is known as the Butterfly Effect, a manifestation of Chaos Theory. For many centuries, the world was explained through the laws of Isaac Newton in classical physics. According to these laws, if the current state of an object is known, its future behavior can be predicted with relative ease. Chaos Theory questions this deterministic vision. Not everything is predictable anymore, nor does it work like a clockwork. Since the 1800s, mathematicians have raised the idea that not all phenomena could be predicted by Newtonian laws. But a meteorologist named Edward Lawrence made Chaos Theory a visible phenomenon. It all started in 1961, when he was working on a mathematical model to forecast the weather. Lawrence entered data such as temperature, humidity, pressure, and wind direction into his computer. His computer would draw a graph modeling what the weather would be like. Not always accurate, but very close to reality."
+    text_2 = "of events is an example of what is known as the butterfly effect, a manifestation of chaos theory. For many centuries, the world was explained through the laws of Isaac Newton in classical physics. According to these laws, if the current state of an object is known, Its future behavior can be predicted with relative ease. Chaos theory questions this deterministic vision. Not everything is predictable anymore. nor does it work like clockwork. Since the 1800s, mathematicians have raised the idea that not all phenomena could be predicted by Newtonian laws. but a meteorologist named Edward Lawrence made chaos theory a visible phenomenon. It all started in 1961. when he was working on a mathematical model to forecast the weather. Lawrence entered data such as temperature, humidity, pressure and wind direction into his computer. His computer would draw a graph, modeling what the weather would be like. not always accurate, but very close to reality. "
+    text_3 = "后来我自己总结啊，微积分这么难，入门主要有几个原因。首先呢大部分的教材为了追求严谨，从一开始就使用了现代数学的这个所谓极限的概念。在它的基础之上向你介绍微积分。呃，问题是它是一个非常抽象的概念。对于大部分在接触微积分之前啊，主要的学习经验就是刷题啊，甚至是连题也不刷的。同学们来说呢，这种抽象语言会很陌生。而且如果你去了解微积分的历史的时候，你会发现极限这个概念啊是微积分创立之后大概一两百年才出现的这么个东西。你等于说我们现在公认的这些微积分的创始人，这些大佬们、牛顿啊、莱布尼茨、欧拉啊，连他们都不知道极限是什么。但是人家就是凭着直觉创建了微积分，当然作为教科书嘛，追求严谨无可厚非啊。虽然对于我来讲，过早的追求这种严谨，导致学习的人入门困难甚至入不了门。"
+    text_4 = "后来我自己总结啊，微积分这么难，入门主要有几个原因。首先呢大部分的教材为了追求严谨。从一开始就使用了现代数学的这个所谓极限的概念在它的基础之上。向你介绍微积分呃，问题是它是一个非常抽象的概念。对于大部分在接触微积分之前。主要的学习经验就是刷题，甚至是连题也不刷的。同学们来说呢，这种抽象语言会很陌生。而且如果你去了解微积分的历史的时候，你会发现极限这个概念啊是微积分创立之后大概一两百年。才出现的这么个东西。你等于说我们现在公认的这些微积分的创始人，这些大佬。牛顿啊、莱布尼茨啊、欧拉啊啊，连他们都不知道极限是谁，但是人家就是凭着直觉。创建了危机，当然作为教科书嘛追求严谨，无可厚非啊，虽然对于我来讲。过早的追求这种严谨，导致学习的人入门困难，甚至入不了门。"
+
+    text_1, text_2 = clean_text_for_comparison_en(text_1), clean_text_for_comparison_en(text_2)
+    text_3, text_4 = clean_text_for_comparison_zh(text_3), clean_text_for_comparison_zh(text_4)
     print(run_textdistance(text_1, text_2))
-    print(highlight_diff(text_1, text_2))
+    print(highlight_diff(text_1, text_2, " "))
+    print(run_textdistance(text_3, text_4))
+    print(highlight_diff(text_3, text_4))