Spaces:
Running
fix: 1-paragraph-per-bulk chunking + KSS sentence-split for overflow
Browse files- text_splitter: each paragraph becomes its own bulk; paragraphs > max_chars
are split into sentences via KSS (regex fallback on pecab asserts) and
packed within that paragraph only
- pecab is not thread-safe; serialize KSS calls with a module-level mutex
to prevent cross-request contamination
- blindtest/ui: drop UI-level chunk_by_paragraph packing and parallel
executor; pipeline's internal split_into_bulks is the single chunking step
This eliminates two distinct drop modes:
- "์ธ์ ๋น์ทํ ๋ฌธ๋จ์ด ๊ฐ์ bulk โ LLM ์ด ์ค๋ณต์ผ๋ก ์คํํด drop" (e.g. O42 20%)
- "ํ ๋ฌธ๋จ์ด ์ฌ๋ฌ bulk ์ ์ชผ๊ฐ์ง โ LLM ์ด ๋ถ์์ ์กฐ๊ฐ์ผ๋ก ์คํํด drop"
(e.g. JABBR 80%)
Verified: 461 runs across 127 articles, 0.22% residual drop rate
(down from 20% worst-case).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- blindtest/ui.py +4 -47
- text_splitter.py +73 -33
|
@@ -11,7 +11,6 @@ Flow:
|
|
| 11 |
from __future__ import annotations
|
| 12 |
|
| 13 |
import time
|
| 14 |
-
from concurrent.futures import ThreadPoolExecutor
|
| 15 |
from typing import Any
|
| 16 |
|
| 17 |
import gradio as gr
|
|
@@ -21,34 +20,6 @@ from pipelines import run_pipeline
|
|
| 21 |
from . import db
|
| 22 |
|
| 23 |
|
| 24 |
-
def _chunk_by_paragraph(text: str, max_chars: int) -> list[str]:
|
| 25 |
-
"""Greedy paragraph-level packing.
|
| 26 |
-
|
| 27 |
-
Splits on blank-line paragraph boundaries and packs paragraphs into chunks
|
| 28 |
-
whose total length stays โค max_chars. A single paragraph longer than
|
| 29 |
-
max_chars stands alone (we never split mid-paragraph to preserve context).
|
| 30 |
-
"""
|
| 31 |
-
paragraphs = [p for p in text.split("\n\n") if p.strip()]
|
| 32 |
-
if not paragraphs:
|
| 33 |
-
return [text] if text.strip() else []
|
| 34 |
-
|
| 35 |
-
sep = "\n\n"
|
| 36 |
-
chunks: list[str] = []
|
| 37 |
-
current: list[str] = []
|
| 38 |
-
current_len = 0
|
| 39 |
-
for p in paragraphs:
|
| 40 |
-
added = (len(sep) if current else 0) + len(p)
|
| 41 |
-
if current and current_len + added > max_chars:
|
| 42 |
-
chunks.append(sep.join(current))
|
| 43 |
-
current = [p]
|
| 44 |
-
current_len = len(p)
|
| 45 |
-
else:
|
| 46 |
-
current.append(p)
|
| 47 |
-
current_len += added
|
| 48 |
-
if current:
|
| 49 |
-
chunks.append(sep.join(current))
|
| 50 |
-
return chunks
|
| 51 |
-
|
| 52 |
PipelineConfig = tuple[str, str, str] # (pipeline_key, model, prompt_key)
|
| 53 |
|
| 54 |
|
|
@@ -78,8 +49,6 @@ def build_feedback_tab(
|
|
| 78 |
|
| 79 |
pipeline_run_id_state = gr.State(None)
|
| 80 |
|
| 81 |
-
CHUNK_MAX_CHARS = 500
|
| 82 |
-
|
| 83 |
input_text = gr.Textbox(
|
| 84 |
label="์๋ฌธ ์
๋ ฅ",
|
| 85 |
lines=8,
|
|
@@ -125,23 +94,11 @@ def build_feedback_tab(
|
|
| 125 |
gr.update(value=""),
|
| 126 |
)
|
| 127 |
|
| 128 |
-
chunks = _chunk_by_paragraph(text, CHUNK_MAX_CHARS)
|
| 129 |
-
if not chunks:
|
| 130 |
-
chunks = [text]
|
| 131 |
-
|
| 132 |
start = time.time()
|
| 133 |
-
chunk_results: list[dict] = [{} for _ in chunks]
|
| 134 |
try:
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
run_pipeline, chunk, pipeline_key, model, prompt_key, client, vocabulary
|
| 139 |
-
): i
|
| 140 |
-
for i, chunk in enumerate(chunks)
|
| 141 |
-
}
|
| 142 |
-
for fut in futures:
|
| 143 |
-
i = futures[fut]
|
| 144 |
-
chunk_results[i] = fut.result()
|
| 145 |
except Exception as exc:
|
| 146 |
return (
|
| 147 |
gr.update(value=f"์๋ฌ: {exc}"),
|
|
@@ -152,7 +109,7 @@ def build_feedback_tab(
|
|
| 152 |
)
|
| 153 |
|
| 154 |
elapsed = time.time() - start
|
| 155 |
-
out_text =
|
| 156 |
|
| 157 |
article_id = db.save_article(text)
|
| 158 |
run_id = db.save_pipeline_run(
|
|
|
|
| 11 |
from __future__ import annotations
|
| 12 |
|
| 13 |
import time
|
|
|
|
| 14 |
from typing import Any
|
| 15 |
|
| 16 |
import gradio as gr
|
|
|
|
| 20 |
from . import db
|
| 21 |
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
PipelineConfig = tuple[str, str, str] # (pipeline_key, model, prompt_key)
|
| 24 |
|
| 25 |
|
|
|
|
| 49 |
|
| 50 |
pipeline_run_id_state = gr.State(None)
|
| 51 |
|
|
|
|
|
|
|
| 52 |
input_text = gr.Textbox(
|
| 53 |
label="์๋ฌธ ์
๋ ฅ",
|
| 54 |
lines=8,
|
|
|
|
| 94 |
gr.update(value=""),
|
| 95 |
)
|
| 96 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
start = time.time()
|
|
|
|
| 98 |
try:
|
| 99 |
+
result = run_pipeline(
|
| 100 |
+
text, pipeline_key, model, prompt_key, client, vocabulary
|
| 101 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
except Exception as exc:
|
| 103 |
return (
|
| 104 |
gr.update(value=f"์๋ฌ: {exc}"),
|
|
|
|
| 109 |
)
|
| 110 |
|
| 111 |
elapsed = time.time() - start
|
| 112 |
+
out_text = result.get("output", "")
|
| 113 |
|
| 114 |
article_id = db.save_article(text)
|
| 115 |
run_id = db.save_pipeline_run(
|
|
@@ -1,28 +1,80 @@
|
|
| 1 |
"""Text splitting utilities for bulk processing.
|
| 2 |
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
paragraph
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
"""
|
| 10 |
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
-
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
-
- Paragraphs are delimited by `\\n`.
|
| 16 |
-
- A paragraph longer than `max_chars` becomes its own bulk as-is.
|
| 17 |
-
- Otherwise, paragraphs accumulate until adding the next would exceed
|
| 18 |
-
`max_chars`; then the bulk is flushed and a new one begins.
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
-
|
| 25 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
"""
|
| 27 |
if not text or not text.strip():
|
| 28 |
return []
|
|
@@ -32,23 +84,11 @@ def split_into_bulks(text: str, max_chars: int = 500) -> list[str]:
|
|
| 32 |
return []
|
| 33 |
|
| 34 |
bulks: list[str] = []
|
| 35 |
-
current: list[str] = []
|
| 36 |
-
current_len = 0
|
| 37 |
-
|
| 38 |
for para in paragraphs:
|
| 39 |
-
if len(para)
|
| 40 |
-
if current:
|
| 41 |
-
bulks.append("\n".join(current))
|
| 42 |
-
current, current_len = [], 0
|
| 43 |
bulks.append(para)
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
else:
|
| 48 |
-
current.append(para)
|
| 49 |
-
current_len += len(para)
|
| 50 |
-
|
| 51 |
-
if current:
|
| 52 |
-
bulks.append("\n".join(current))
|
| 53 |
|
| 54 |
return bulks
|
|
|
|
| 1 |
"""Text splitting utilities for bulk processing.
|
| 2 |
|
| 3 |
+
Strategy: **1 paragraph = 1 bulk**, with no cross-paragraph packing.
|
| 4 |
+
|
| 5 |
+
Each paragraph is sent to the LLM as its own bulk so that:
|
| 6 |
+
- LLM never sees two similar adjacent paragraphs in one bulk
|
| 7 |
+
(this triggered "๊ฐ์ ํ์ ๋ฐ๋ณต" drops โ e.g. inserted-pair case in
|
| 8 |
+
the O42W2BIE25HZ ๋ฐฐํฐ๋ฆฌ ๊ธฐ์ฌ)
|
| 9 |
+
- No paragraph is ever split across bulks
|
| 10 |
+
(this triggered "๋ถ์์ ์กฐ๊ฐ" drops โ e.g. JABBR33ZE ํ๋ณด ๊ธฐ์ฌ)
|
| 11 |
+
|
| 12 |
+
A paragraph that exceeds `max_chars` is the only exception: it is split
|
| 13 |
+
into sentences via KSS (pecab backend is not thread-safe, so we serialize
|
| 14 |
+
with a module-level lock and fall back to regex on pecab asserts), and
|
| 15 |
+
those sentences are packed within that one paragraph's scope only. Bulk
|
| 16 |
+
boundaries still never cross into adjacent paragraphs.
|
| 17 |
"""
|
| 18 |
|
| 19 |
+
import logging
|
| 20 |
+
import re
|
| 21 |
+
import threading
|
| 22 |
|
| 23 |
+
import kss
|
| 24 |
+
|
| 25 |
+
logger = logging.getLogger(__name__)
|
| 26 |
+
|
| 27 |
+
# pecab (KSS ๊ธฐ๋ณธ ๋ฐฑ์๋) ๊ฐ ์ค๋ ๋ ์ธ์ดํ๊ฐ ์๋ โ ๋์ ํธ์ถ ์ ๋ด๋ถ ์ํ๊ฐ
|
| 28 |
+
# ์์ฌ ๋ค๋ฅธ ์ค๋ ๋์ ์
๋ ฅ์ด ๋ฐํ๋๋ cross-contamination ๊ด์ฐฐ๋จ. mecab ์
|
| 29 |
+
# ์์คํ
์ ์ฌ๊ธฐ ์ ๊น์ง ์ ์ญ ๋ฎคํ
์ค๋ก ์ง๋ ฌํํ๋ค.
|
| 30 |
+
_KSS_LOCK = threading.Lock()
|
| 31 |
+
|
| 32 |
+
# Korean + Latin sentence-terminal punctuation followed by whitespace.
|
| 33 |
+
_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?โฆใ!?])\s+")
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
+
def _safe_split_sentences(para: str) -> list[str]:
|
| 37 |
+
try:
|
| 38 |
+
with _KSS_LOCK:
|
| 39 |
+
sents = list(kss.split_sentences(para))
|
| 40 |
+
return [s.strip() for s in sents if s and s.strip()]
|
| 41 |
+
except (AssertionError, IndexError, AttributeError, ValueError, TypeError) as e:
|
| 42 |
+
logger.warning(
|
| 43 |
+
"kss.split_sentences failed (%s); using regex fallback", type(e).__name__
|
| 44 |
+
)
|
| 45 |
+
parts = [p.strip() for p in _SENTENCE_SPLIT_RE.split(para) if p.strip()]
|
| 46 |
+
return parts if parts else [para]
|
| 47 |
|
| 48 |
+
|
| 49 |
+
def _pack_sentences(sentences: list[str], max_chars: int) -> list[str]:
|
| 50 |
+
"""Pack sentences joined by ' ' up to `max_chars`, never splitting a sentence."""
|
| 51 |
+
bulks: list[str] = []
|
| 52 |
+
current: list[str] = []
|
| 53 |
+
current_len = 0
|
| 54 |
+
for sent in sentences:
|
| 55 |
+
if not sent:
|
| 56 |
+
continue
|
| 57 |
+
add = (1 + len(sent)) if current else len(sent)
|
| 58 |
+
if current and current_len + add > max_chars:
|
| 59 |
+
bulks.append(" ".join(current))
|
| 60 |
+
current, current_len = [sent], len(sent)
|
| 61 |
+
else:
|
| 62 |
+
current.append(sent)
|
| 63 |
+
current_len += add
|
| 64 |
+
if current:
|
| 65 |
+
bulks.append(" ".join(current))
|
| 66 |
+
return bulks
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def split_into_bulks(text: str, max_chars: int = 500) -> list[str]:
|
| 70 |
+
"""Return one bulk per paragraph. Long paragraphs are sentence-packed in-scope.
|
| 71 |
+
|
| 72 |
+
- Paragraph delimiter is `\\n` (so `\\n\\n`-separated input works too โ
|
| 73 |
+
empty strings between are filtered).
|
| 74 |
+
- A paragraph โค `max_chars` is emitted as a single bulk.
|
| 75 |
+
- A paragraph > `max_chars` is split into sentences via KSS (regex
|
| 76 |
+
fallback on pecab asserts) and those sentences are packed within
|
| 77 |
+
that paragraph only โ bulks never span paragraph boundaries.
|
| 78 |
"""
|
| 79 |
if not text or not text.strip():
|
| 80 |
return []
|
|
|
|
| 84 |
return []
|
| 85 |
|
| 86 |
bulks: list[str] = []
|
|
|
|
|
|
|
|
|
|
| 87 |
for para in paragraphs:
|
| 88 |
+
if len(para) <= max_chars:
|
|
|
|
|
|
|
|
|
|
| 89 |
bulks.append(para)
|
| 90 |
+
continue
|
| 91 |
+
sentences = _safe_split_sentences(para)
|
| 92 |
+
bulks.extend(_pack_sentences(sentences, max_chars))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
|
| 94 |
return bulks
|