Spaces:
Running
Running
Commit ยท
725b08e
1
Parent(s): c540d93
fix: preserve bulk boundary \n + substring-count dedupe + 30-char prefix
Browse files- bulk join "" โ "\n" to keep paragraph boundaries across bulks
- apply_paragraph_dedupe: substring-count input to tolerate LLM paragraph restructuring
- prefix_len default 80 โ 30 to catch corrected+original echo pairs
- temperature 0 โ 0.0001 to loosen greedy decoding bias
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- pipelines.py +21 -11
- postprocess.py +14 -10
pipelines.py
CHANGED
|
@@ -416,7 +416,11 @@ def call_llm(
|
|
| 416 |
system_prompt: str,
|
| 417 |
user_content: str,
|
| 418 |
model: str = "solar-pro2",
|
| 419 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 420 |
reasoning_effort: str | None = None,
|
| 421 |
max_tokens: int | None = None,
|
| 422 |
response_format: dict | None = None,
|
|
@@ -804,12 +808,14 @@ def apply_rule(
|
|
| 804 |
)
|
| 805 |
elif rule == "paragraph_dedupe":
|
| 806 |
config = step.get("config", {})
|
| 807 |
-
|
| 808 |
-
|
| 809 |
-
|
| 810 |
-
|
| 811 |
-
|
| 812 |
-
|
|
|
|
|
|
|
| 813 |
|
| 814 |
return text
|
| 815 |
|
|
@@ -1068,10 +1074,14 @@ def run_pipeline(
|
|
| 1068 |
processed = process_bulks_parallel(
|
| 1069 |
bulks, original_bulks, step, model, prompts, client
|
| 1070 |
)
|
| 1071 |
-
#
|
| 1072 |
-
#
|
| 1073 |
-
#
|
| 1074 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1075 |
elif step["type"] == "specialist":
|
| 1076 |
text = run_specialist(step, text, original_text, prompts, model, client)
|
| 1077 |
# If a step produced an unusably short output (< 10% of input), the
|
|
|
|
| 416 |
system_prompt: str,
|
| 417 |
user_content: str,
|
| 418 |
model: str = "solar-pro2",
|
| 419 |
+
# Upstage ์๋น ์คํ์ด temperature=0 ์์ greedy ๋์ฝ๋ฉ ๊ฒฝ๋ก๋ก ๋ค์ด๊ฐ๋๋ฐ,
|
| 420 |
+
# ์ด ๊ฒฝ๋ก๊ฐ ํน์ ์
๋ ฅ(truncated article ๋ฑ)์์ ํ์ต ๋ฐ์ดํฐ์ "์ฌ์์" ํจํด์
|
| 421 |
+
# ์ฌํ์ฑ ๋๊ฒ ์ฌํํ๋ ๋ฐ์ด์ด์ค๊ฐ ๊ด์ฐฐ๋จ. 0.0001 ๋ก sampling ๊ฒฝ๋ก๋ก ์ด์ง
|
| 422 |
+
# ๋ฐ์ด ๋ฃ์ด bias ๋ฅผ ํ๋ค๋, argmax ํ๋ฅ ๋น์ค์ ๊ฑฐ์ ๊ทธ๋๋ก ์ ์ง.
|
| 423 |
+
temperature: float = 0.0001,
|
| 424 |
reasoning_effort: str | None = None,
|
| 425 |
max_tokens: int | None = None,
|
| 426 |
response_format: dict | None = None,
|
|
|
|
| 808 |
)
|
| 809 |
elif rule == "paragraph_dedupe":
|
| 810 |
config = step.get("config", {})
|
| 811 |
+
# prefix_len ๊ธฐ๋ณธ๊ฐ์ apply_paragraph_dedupe ์ ๊ธฐ๋ณธ๊ฐ(30)์ ๋ฐ๋ฅธ๋ค.
|
| 812 |
+
# ๋ช
์์ ์ผ๋ก config ์ ์ง์ ๋ ๊ฒฝ์ฐ๋ง override.
|
| 813 |
+
kwargs: dict = {}
|
| 814 |
+
if "min_len" in config:
|
| 815 |
+
kwargs["min_len"] = config["min_len"]
|
| 816 |
+
if "prefix_len" in config:
|
| 817 |
+
kwargs["prefix_len"] = config["prefix_len"]
|
| 818 |
+
return apply_paragraph_dedupe(text, original_text, **kwargs)
|
| 819 |
|
| 820 |
return text
|
| 821 |
|
|
|
|
| 1074 |
processed = process_bulks_parallel(
|
| 1075 |
bulks, original_bulks, step, model, prompts, client
|
| 1076 |
)
|
| 1077 |
+
# Bulk ๊ฒฝ๊ณ๋ ์๋ณธ์์ \n ๋ก ๋ถ๋ฆฌ๋ผ ์์์ผ๋ฏ๋ก ํฉ์น ๋๋ \n ๋ก
|
| 1078 |
+
# ์ฐ๊ฒฐํ๋ค. "".join ์ผ๋ก ๋ถ์ด๋ฉด ์ธ์ bulk ์ ๋ฌธ๋จ์ด ํ ์ค๋ก
|
| 1079 |
+
# ๋ญ๊ฐ์ง๊ณ , ์ดํ split_into_bulks ์ฌ์คํ ์ ๋ฌธ๋จ ์๊ฐ ์ค์ด
|
| 1080 |
+
# step1 ์ <์๋ฌธ>โ<๊ต์ด_๋ชจ๋ธ_์์ ๊ฒฐ๊ณผ> ์ ๋ ฌ์ด ์ด๊ธ๋๋ฉฐ ๊ตฌ์กฐ
|
| 1081 |
+
# ๋ณด์กด ๊ท์น๊น์ง ๊นจ์ง๋ค (์ฐธ์กฐ: run.py:326 "\n".join).
|
| 1082 |
+
text = "\n".join(
|
| 1083 |
+
str(p) if not isinstance(p, str) else p for p in processed
|
| 1084 |
+
)
|
| 1085 |
elif step["type"] == "specialist":
|
| 1086 |
text = run_specialist(step, text, original_text, prompts, model, client)
|
| 1087 |
# If a step produced an unusably short output (< 10% of input), the
|
postprocess.py
CHANGED
|
@@ -380,7 +380,7 @@ def apply_paragraph_dedupe(
|
|
| 380 |
text: str,
|
| 381 |
original: str,
|
| 382 |
min_len: int = 40,
|
| 383 |
-
prefix_len: int =
|
| 384 |
) -> str:
|
| 385 |
"""LLM์ด ๋ฑ์ ์ค๋ณต ๋ฌธ๋จ์ ์ ๊ฑฐํ๋ค.
|
| 386 |
|
|
@@ -411,13 +411,12 @@ def apply_paragraph_dedupe(
|
|
| 411 |
if not text:
|
| 412 |
return text
|
| 413 |
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
in_prefix[norm[:prefix_len]] += 1
|
| 421 |
|
| 422 |
out_paras = _split_output_paragraphs(text)
|
| 423 |
out_exact_seen: Counter[str] = Counter()
|
|
@@ -435,12 +434,17 @@ def apply_paragraph_dedupe(
|
|
| 435 |
out_exact_seen[norm] += 1
|
| 436 |
out_prefix_seen[prefix] += 1
|
| 437 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
exact_dup = (
|
| 439 |
-
out_exact_seen[norm] >
|
| 440 |
and out_exact_seen[norm] >= 2
|
| 441 |
)
|
| 442 |
near_dup = (
|
| 443 |
-
out_prefix_seen[prefix] >
|
| 444 |
and out_prefix_seen[prefix] >= 2
|
| 445 |
)
|
| 446 |
|
|
|
|
| 380 |
text: str,
|
| 381 |
original: str,
|
| 382 |
min_len: int = 40,
|
| 383 |
+
prefix_len: int = 30,
|
| 384 |
) -> str:
|
| 385 |
"""LLM์ด ๋ฑ์ ์ค๋ณต ๋ฌธ๋จ์ ์ ๊ฑฐํ๋ค.
|
| 386 |
|
|
|
|
| 411 |
if not text:
|
| 412 |
return text
|
| 413 |
|
| 414 |
+
# Input ์ "normalized ์ ์ฒด ๋ฌธ์์ด" ๋ก ๋๊ณ substring count ๋ฅผ ์ด๋ค.
|
| 415 |
+
# ์ด์ ๊ตฌํ์ input ์ ๋ฌธ๋จ Counter ๋ก ์
๋๋ฐ, LLM ์ด ๋ฌธ๋จ ๊ฒฝ๊ณ๋ฅผ ์ฌ๊ตฌ์กฐํ
|
| 416 |
+
# (์: ์ค๋ฐ๊ฟ ์์ด ์ค๋ณต ๋ฌธ์ฅ์ด ๋ค์ด์๋ ํ ๋ฌธ๋จ์ ์ฌ๋ฌ ๋ฌธ๋จ์ผ๋ก ์ชผ๊ฐฌ) ํ๋ฉด
|
| 417 |
+
# output ๋ฌธ๋จ์ด input ๋ฌธ๋จ๊ณผ exact match ๊ฐ ์ ๋์ด input_count=0 ์ผ๋ก ์กํ,
|
| 418 |
+
# ์ ๋นํ ์ค๋ณต (์ ์/์๋ณธ์ด ์๋ํ ๋ฐ๋ณต) ๊น์ง drop ๋๋ ๋ฒ๊ทธ๊ฐ ์์๋ค.
|
| 419 |
+
in_norm = _normalize_paragraph(original or "")
|
|
|
|
| 420 |
|
| 421 |
out_paras = _split_output_paragraphs(text)
|
| 422 |
out_exact_seen: Counter[str] = Counter()
|
|
|
|
| 434 |
out_exact_seen[norm] += 1
|
| 435 |
out_prefix_seen[prefix] += 1
|
| 436 |
|
| 437 |
+
# input ๋ฌธ์์ด ์ ์ฒด์์ ํด๋น ๋ฌธ๋จ(๋๋ prefix)์ด ๋ช ๋ฒ substring ์ผ๋ก
|
| 438 |
+
# ๋ฑ์ฅํ๋์ง ์ง๊ณ. output_count ๊ฐ input_count ๋ฅผ ์ด๊ณผํ ๋๋ง drop.
|
| 439 |
+
in_exact_count = in_norm.count(norm) if in_norm else 0
|
| 440 |
+
in_prefix_count = in_norm.count(prefix) if in_norm else 0
|
| 441 |
+
|
| 442 |
exact_dup = (
|
| 443 |
+
out_exact_seen[norm] > in_exact_count
|
| 444 |
and out_exact_seen[norm] >= 2
|
| 445 |
)
|
| 446 |
near_dup = (
|
| 447 |
+
out_prefix_seen[prefix] > in_prefix_count
|
| 448 |
and out_prefix_seen[prefix] >= 2
|
| 449 |
)
|
| 450 |
|