Update app.py
Browse files
app.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
-
#
|
| 2 |
-
# 実行方法: python
|
| 3 |
|
| 4 |
import re
|
| 5 |
import difflib
|
|
@@ -19,13 +19,22 @@ CJK_COMPATIBILITY = set(range(0x3300, 0x3400))
|
|
| 19 |
ENCLOSED_ALPHANUMERICS = set(range(0x2460, 0x2500))
|
| 20 |
ENCLOSED_CJK_LETTERS = set(range(0x3200, 0x3300))
|
| 21 |
|
| 22 |
-
HYPHENS = set([
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
KANJI_NUMBERS = set('零一二三四五六七八九十')
|
| 24 |
ARABIC_NUMBERS = set('01234567890123456789')
|
| 25 |
|
| 26 |
YAKUMONO = set(
|
| 27 |
-
'。、.,'
|
| 28 |
-
'〔〕《》〈〉【】[]()()'
|
| 29 |
'「」『』「」'
|
| 30 |
'〝〟'
|
| 31 |
'!?・…―'
|
|
@@ -151,21 +160,14 @@ def merge_ranges(ranges):
|
|
| 151 |
# ポストフィルタリング
|
| 152 |
# ============================================================
|
| 153 |
|
| 154 |
-
EQUAL_MERGE_THRESHOLD = 2
|
| 155 |
|
| 156 |
def filter_opcodes(opcodes):
|
| 157 |
-
"""
|
| 158 |
-
equal区間がTHRESHOLD以下の場合、前後の差分区間と結合する。
|
| 159 |
-
結果としてdiff区間のみのリストを返す。
|
| 160 |
-
"""
|
| 161 |
-
# まずopcodeをリスト化
|
| 162 |
ops = list(opcodes)
|
| 163 |
n = len(ops)
|
| 164 |
if n == 0:
|
| 165 |
-
return []
|
| 166 |
|
| 167 |
-
# equal区間が短い場合にdiffとして扱うフラグ
|
| 168 |
-
# Trueはdiff扱い、Falseはequal扱い
|
| 169 |
is_diff = []
|
| 170 |
for tag, i1, i2, j1, j2 in ops:
|
| 171 |
if tag == 'equal':
|
|
@@ -173,49 +175,33 @@ def filter_opcodes(opcodes):
|
|
| 173 |
else:
|
| 174 |
is_diff.append(True)
|
| 175 |
|
| 176 |
-
# 両端のequal
|
| 177 |
-
# 先頭のequal
|
| 178 |
for idx in range(n):
|
| 179 |
-
|
| 180 |
-
if tag == 'equal':
|
| 181 |
is_diff[idx] = False
|
| 182 |
else:
|
| 183 |
break
|
| 184 |
-
# 末尾のequal
|
| 185 |
for idx in range(n-1, -1, -1):
|
| 186 |
-
|
| 187 |
-
if tag == 'equal':
|
| 188 |
is_diff[idx] = False
|
| 189 |
else:
|
| 190 |
break
|
| 191 |
|
| 192 |
-
# diff区間を結合して返す
|
| 193 |
result_src = []
|
| 194 |
result_dst = []
|
| 195 |
|
| 196 |
i = 0
|
| 197 |
while i < n:
|
| 198 |
if is_diff[i]:
|
| 199 |
-
# 連続するdiff区間をまとめる
|
| 200 |
-
tag, i1, i2, j1, j2 = ops[i]
|
| 201 |
-
src_start, src_end = i1, i2
|
| 202 |
-
dst_start, dst_end = j1, j2
|
| 203 |
j = i + 1
|
| 204 |
while j < n and is_diff[j]:
|
| 205 |
-
_, ni1, ni2, nj1, nj2 = ops[j]
|
| 206 |
-
src_end = ni2
|
| 207 |
-
dst_end = nj2
|
| 208 |
j += 1
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
)
|
| 215 |
-
has_dst_diff = any(
|
| 216 |
-
ops[k][0] in ('replace', 'insert')
|
| 217 |
-
for k in range(i, j)
|
| 218 |
-
)
|
| 219 |
if has_src_diff:
|
| 220 |
result_src.append((src_start, src_end))
|
| 221 |
if has_dst_diff:
|
|
@@ -247,7 +233,6 @@ def compare_texts(src, dst, settings):
|
|
| 247 |
if src_active != '' or dst_active != '':
|
| 248 |
matcher = difflib.SequenceMatcher(None, src_active, dst_active, autojunk=False)
|
| 249 |
opcodes = matcher.get_opcodes()
|
| 250 |
-
|
| 251 |
result_src, result_dst = filter_opcodes(opcodes)
|
| 252 |
|
| 253 |
for a_start, a_end in result_src:
|
|
@@ -354,7 +339,7 @@ DST25 = (
|
|
| 354 |
# ============================================================
|
| 355 |
|
| 356 |
print("=" * 60)
|
| 357 |
-
print("バグ2 検証スクリプト
|
| 358 |
print("=" * 60)
|
| 359 |
|
| 360 |
run_test("TEST-01 相違なし",
|
|
|
|
| 1 |
+
# verify_bug2_ver5.py
|
| 2 |
+
# 実行方法: python verify_bug2_ver5.py
|
| 3 |
|
| 4 |
import re
|
| 5 |
import difflib
|
|
|
|
| 19 |
ENCLOSED_ALPHANUMERICS = set(range(0x2460, 0x2500))
|
| 20 |
ENCLOSED_CJK_LETTERS = set(range(0x3200, 0x3300))
|
| 21 |
|
| 22 |
+
HYPHENS = set([
|
| 23 |
+
'\u002D', # 半角ハイフンマイナス
|
| 24 |
+
'\uFF0D', # 全角ハイフンマイナス
|
| 25 |
+
'\u2010', # HYPHEN
|
| 26 |
+
'\u2011', # NON-BREAKING HYPHEN
|
| 27 |
+
'\u2012', # FIGURE DASH
|
| 28 |
+
'\u2013', # EN DASH
|
| 29 |
+
'\u2014', # EM DASH
|
| 30 |
+
])
|
| 31 |
+
|
| 32 |
KANJI_NUMBERS = set('零一二三四五六七八九十')
|
| 33 |
ARABIC_NUMBERS = set('01234567890123456789')
|
| 34 |
|
| 35 |
YAKUMONO = set(
|
| 36 |
+
'。、.,.,'
|
| 37 |
+
'〔〕《》〈〉【】[]()()[]'
|
| 38 |
'「」『』「」'
|
| 39 |
'〝〟'
|
| 40 |
'!?・…―'
|
|
|
|
| 160 |
# ポストフィルタリング
|
| 161 |
# ============================================================
|
| 162 |
|
| 163 |
+
EQUAL_MERGE_THRESHOLD = 2
|
| 164 |
|
| 165 |
def filter_opcodes(opcodes):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
ops = list(opcodes)
|
| 167 |
n = len(ops)
|
| 168 |
if n == 0:
|
| 169 |
+
return [], []
|
| 170 |
|
|
|
|
|
|
|
| 171 |
is_diff = []
|
| 172 |
for tag, i1, i2, j1, j2 in ops:
|
| 173 |
if tag == 'equal':
|
|
|
|
| 175 |
else:
|
| 176 |
is_diff.append(True)
|
| 177 |
|
| 178 |
+
# 両端のequal区間はdiff扱いにしない
|
|
|
|
| 179 |
for idx in range(n):
|
| 180 |
+
if ops[idx][0] == 'equal':
|
|
|
|
| 181 |
is_diff[idx] = False
|
| 182 |
else:
|
| 183 |
break
|
|
|
|
| 184 |
for idx in range(n-1, -1, -1):
|
| 185 |
+
if ops[idx][0] == 'equal':
|
|
|
|
| 186 |
is_diff[idx] = False
|
| 187 |
else:
|
| 188 |
break
|
| 189 |
|
|
|
|
| 190 |
result_src = []
|
| 191 |
result_dst = []
|
| 192 |
|
| 193 |
i = 0
|
| 194 |
while i < n:
|
| 195 |
if is_diff[i]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
j = i + 1
|
| 197 |
while j < n and is_diff[j]:
|
|
|
|
|
|
|
|
|
|
| 198 |
j += 1
|
| 199 |
+
src_start = ops[i][1]
|
| 200 |
+
src_end = ops[j-1][2]
|
| 201 |
+
dst_start = ops[i][3]
|
| 202 |
+
dst_end = ops[j-1][4]
|
| 203 |
+
has_src_diff = any(ops[k][0] in ('replace', 'delete') for k in range(i, j))
|
| 204 |
+
has_dst_diff = any(ops[k][0] in ('replace', 'insert') for k in range(i, j))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
if has_src_diff:
|
| 206 |
result_src.append((src_start, src_end))
|
| 207 |
if has_dst_diff:
|
|
|
|
| 233 |
if src_active != '' or dst_active != '':
|
| 234 |
matcher = difflib.SequenceMatcher(None, src_active, dst_active, autojunk=False)
|
| 235 |
opcodes = matcher.get_opcodes()
|
|
|
|
| 236 |
result_src, result_dst = filter_opcodes(opcodes)
|
| 237 |
|
| 238 |
for a_start, a_end in result_src:
|
|
|
|
| 339 |
# ============================================================
|
| 340 |
|
| 341 |
print("=" * 60)
|
| 342 |
+
print("バグ2 検証スクリプト ver5")
|
| 343 |
print("=" * 60)
|
| 344 |
|
| 345 |
run_test("TEST-01 相違なし",
|