Spaces:
Runtime error
Runtime error
hy commited on
Commit ยท
1225cdd
1
Parent(s): 8fd2b87
mismatch
Browse files- mismatch_model.py +27 -25
mismatch_model.py
CHANGED
|
@@ -47,41 +47,43 @@ def _split_sentences_ko(text: str):
|
|
| 47 |
return [p.strip() for p in parts if p.strip()]
|
| 48 |
|
| 49 |
def summarize_kobart_strict(text):
|
| 50 |
-
"""
|
| 51 |
-
[์์ ๋ฒ์ ]
|
| 52 |
-
- ์ ๊ท์(Regex) ๊ฒ์ฌ ๋ก์ง์ ๋ชจ๋ ์ ๊ฑฐํ์ต๋๋ค.
|
| 53 |
-
- KoBART๊ฐ ์์ฑํ ์์ฝ๋ฌธ์ ์กฐ๊ฑด ์์ด ๊ทธ๋๋ก ๋ฐํํฉ๋๋ค.
|
| 54 |
-
"""
|
| 55 |
text = _clean_text(text)
|
| 56 |
sents = _split_sentences_ko(text)
|
| 57 |
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
try:
|
| 64 |
-
# 2. KoBART ์์ฝ ์ํ
|
| 65 |
result = kobart_summarizer(
|
| 66 |
text,
|
| 67 |
-
min_length=30,
|
| 68 |
-
max_length=90,
|
| 69 |
-
num_beams=4,
|
| 70 |
-
no_repeat_ngram_size=3,
|
| 71 |
-
early_stopping=True
|
|
|
|
| 72 |
)[0]["summary_text"]
|
| 73 |
-
|
| 74 |
out = _clean_text(result)
|
| 75 |
-
|
| 76 |
-
|
|
|
|
| 77 |
if len(out) < 10:
|
| 78 |
-
|
|
|
|
| 79 |
|
| 80 |
return out
|
| 81 |
|
| 82 |
except Exception as e:
|
| 83 |
-
print(
|
| 84 |
-
return _clean_text(" ".join(sents[:3])) if sents else text
|
|
|
|
|
|
|
| 85 |
|
| 86 |
def get_cosine_similarity(title, summary):
|
| 87 |
"""(์ ์ง) SBERT ์ฝ์ฌ์ธ ์ ์ฌ๋"""
|
|
@@ -163,8 +165,8 @@ def get_mismatch_score(summary, title):
|
|
| 163 |
|
| 164 |
def calculate_mismatch_score(article_title, article_body):
|
| 165 |
"""
|
| 166 |
-
- w1 (SBERT ๊ฑฐ๋ฆฌ): 0.
|
| 167 |
-
- w2 (NLI ๋ถ์ผ์น): 0.
|
| 168 |
- Threshold: 0.45 ์ด์์ด๋ฉด '์ํ'
|
| 169 |
"""
|
| 170 |
# 1) ๋ณธ๋ฌธ ์์ฝ
|
|
@@ -178,7 +180,7 @@ def calculate_mismatch_score(article_title, article_body):
|
|
| 178 |
nli_mismatch, entail, neutral, contra = get_mismatch_score(summary, article_title)
|
| 179 |
|
| 180 |
# 4) ์ต์ข
์ ์(์์ ๊ณผ ๋์ผ ๊ตฌ์กฐ)
|
| 181 |
-
w1, w2 = 0.
|
| 182 |
final_score = (w1 * semantic_distance) + (w2 * nli_mismatch)
|
| 183 |
|
| 184 |
reason = (
|
|
@@ -200,4 +202,4 @@ def calculate_mismatch_score(article_title, article_body):
|
|
| 200 |
"score": round(final_score, 4),
|
| 201 |
"reason": reason,
|
| 202 |
"recommendation": recommendation
|
| 203 |
-
}
|
|
|
|
| 47 |
return [p.strip() for p in parts if p.strip()]
|
| 48 |
|
| 49 |
def summarize_kobart_strict(text):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
text = _clean_text(text)
|
| 51 |
sents = _split_sentences_ko(text)
|
| 52 |
|
| 53 |
+
print("[DEBUG] len(text) =", len(text), "len(sents) =", len(sents))
|
| 54 |
+
print("[DEBUG] first3 =", " | ".join(sents[:3]))
|
| 55 |
+
|
| 56 |
+
# โ
์ค์ง ๋ฌธ์ฅ ์ ๊ธฐ์ค๋ง ์ฌ์ฉ
|
| 57 |
+
if len(sents) <= 3:
|
| 58 |
+
print("[DEBUG] <=3 sentences -> return as-is")
|
| 59 |
+
return _clean_text(" ".join(sents)) if sents else text
|
| 60 |
|
| 61 |
try:
|
|
|
|
| 62 |
result = kobart_summarizer(
|
| 63 |
text,
|
| 64 |
+
min_length=30,
|
| 65 |
+
max_length=90,
|
| 66 |
+
num_beams=4,
|
| 67 |
+
no_repeat_ngram_size=3,
|
| 68 |
+
early_stopping=True,
|
| 69 |
+
truncation=True, # ๊ธธ์ด ์ด๊ณผ ๋ฐฉ์ง
|
| 70 |
)[0]["summary_text"]
|
| 71 |
+
|
| 72 |
out = _clean_text(result)
|
| 73 |
+
print("[DEBUG] kobart_out =", out)
|
| 74 |
+
|
| 75 |
+
# ์์ฝ์ด ๋ง๋ ์ ๋๊ฒ ์งง์ ๋๋ง fallback
|
| 76 |
if len(out) < 10:
|
| 77 |
+
print("[DEBUG] too short -> fallback to first 3 sentences")
|
| 78 |
+
return _clean_text(" ".join(sents[:3]))
|
| 79 |
|
| 80 |
return out
|
| 81 |
|
| 82 |
except Exception as e:
|
| 83 |
+
print("๐จ [Error] ์์ฝ ๋ชจ๋ธ ์๋ฌ:", repr(e))
|
| 84 |
+
return _clean_text(" ".join(sents[:3])) if sents else text
|
| 85 |
+
|
| 86 |
+
|
| 87 |
|
| 88 |
def get_cosine_similarity(title, summary):
|
| 89 |
"""(์ ์ง) SBERT ์ฝ์ฌ์ธ ์ ์ฌ๋"""
|
|
|
|
| 165 |
|
| 166 |
def calculate_mismatch_score(article_title, article_body):
|
| 167 |
"""
|
| 168 |
+
- w1 (SBERT ๊ฑฐ๋ฆฌ): 0.6
|
| 169 |
+
- w2 (NLI ๋ถ์ผ์น): 0.4
|
| 170 |
- Threshold: 0.45 ์ด์์ด๋ฉด '์ํ'
|
| 171 |
"""
|
| 172 |
# 1) ๋ณธ๋ฌธ ์์ฝ
|
|
|
|
| 180 |
nli_mismatch, entail, neutral, contra = get_mismatch_score(summary, article_title)
|
| 181 |
|
| 182 |
# 4) ์ต์ข
์ ์(์์ ๊ณผ ๋์ผ ๊ตฌ์กฐ)
|
| 183 |
+
w1, w2 = 0.6, 0.4
|
| 184 |
final_score = (w1 * semantic_distance) + (w2 * nli_mismatch)
|
| 185 |
|
| 186 |
reason = (
|
|
|
|
| 202 |
"score": round(final_score, 4),
|
| 203 |
"reason": reason,
|
| 204 |
"recommendation": recommendation
|
| 205 |
+
}
|