dev-strender Claude Opus 4.7 (1M context) commited on
Commit
725b08e
ยท
1 Parent(s): c540d93

fix: preserve bulk boundary \n + substring-count dedupe + 30-char prefix

Browse files

- bulk join "" โ†’ "\n" to keep paragraph boundaries across bulks
- apply_paragraph_dedupe: substring-count input to tolerate LLM paragraph restructuring
- prefix_len default 80 โ†’ 30 to catch corrected+original echo pairs
- temperature 0 โ†’ 0.0001 to loosen greedy decoding bias

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files changed (2) hide show
  1. pipelines.py +21 -11
  2. postprocess.py +14 -10
pipelines.py CHANGED
@@ -416,7 +416,11 @@ def call_llm(
416
  system_prompt: str,
417
  user_content: str,
418
  model: str = "solar-pro2",
419
- temperature: float = 0.0,
 
 
 
 
420
  reasoning_effort: str | None = None,
421
  max_tokens: int | None = None,
422
  response_format: dict | None = None,
@@ -804,12 +808,14 @@ def apply_rule(
804
  )
805
  elif rule == "paragraph_dedupe":
806
  config = step.get("config", {})
807
- return apply_paragraph_dedupe(
808
- text,
809
- original_text,
810
- min_len=config.get("min_len", 40),
811
- prefix_len=config.get("prefix_len", 80),
812
- )
 
 
813
 
814
  return text
815
 
@@ -1068,10 +1074,14 @@ def run_pipeline(
1068
  processed = process_bulks_parallel(
1069
  bulks, original_bulks, step, model, prompts, client
1070
  )
1071
- # Defensive: coerce any non-string bulk result before joining
1072
- # so the next step (which may call re.sub) never sees a
1073
- # non-string input.
1074
- text = "".join(str(p) if not isinstance(p, str) else p for p in processed)
 
 
 
 
1075
  elif step["type"] == "specialist":
1076
  text = run_specialist(step, text, original_text, prompts, model, client)
1077
  # If a step produced an unusably short output (< 10% of input), the
 
416
  system_prompt: str,
417
  user_content: str,
418
  model: str = "solar-pro2",
419
+ # Upstage ์„œ๋น™ ์Šคํƒ์ด temperature=0 ์—์„œ greedy ๋””์ฝ”๋”ฉ ๊ฒฝ๋กœ๋กœ ๋“ค์–ด๊ฐ€๋Š”๋ฐ,
420
+ # ์ด ๊ฒฝ๋กœ๊ฐ€ ํŠน์ • ์ž…๋ ฅ(truncated article ๋“ฑ)์—์„œ ํ•™์Šต ๋ฐ์ดํ„ฐ์˜ "์žฌ์‹œ์ž‘" ํŒจํ„ด์„
421
+ # ์žฌํ˜„์„ฑ ๋†’๊ฒŒ ์žฌํ˜„ํ•˜๋Š” ๋ฐ”์ด์–ด์Šค๊ฐ€ ๊ด€์ฐฐ๋จ. 0.0001 ๋กœ sampling ๊ฒฝ๋กœ๋กœ ์‚ด์ง
422
+ # ๋ฐ€์–ด ๋„ฃ์–ด bias ๋ฅผ ํ”๋“ค๋˜, argmax ํ™•๋ฅ  ๋น„์ค‘์€ ๊ฑฐ์˜ ๊ทธ๋Œ€๋กœ ์œ ์ง€.
423
+ temperature: float = 0.0001,
424
  reasoning_effort: str | None = None,
425
  max_tokens: int | None = None,
426
  response_format: dict | None = None,
 
808
  )
809
  elif rule == "paragraph_dedupe":
810
  config = step.get("config", {})
811
+ # prefix_len ๊ธฐ๋ณธ๊ฐ’์€ apply_paragraph_dedupe ์˜ ๊ธฐ๋ณธ๊ฐ’(30)์„ ๋”ฐ๋ฅธ๋‹ค.
812
+ # ๋ช…์‹œ์ ์œผ๋กœ config ์— ์ง€์ •๋œ ๊ฒฝ์šฐ๋งŒ override.
813
+ kwargs: dict = {}
814
+ if "min_len" in config:
815
+ kwargs["min_len"] = config["min_len"]
816
+ if "prefix_len" in config:
817
+ kwargs["prefix_len"] = config["prefix_len"]
818
+ return apply_paragraph_dedupe(text, original_text, **kwargs)
819
 
820
  return text
821
 
 
1074
  processed = process_bulks_parallel(
1075
  bulks, original_bulks, step, model, prompts, client
1076
  )
1077
+ # Bulk ๊ฒฝ๊ณ„๋Š” ์›๋ณธ์—์„œ \n ๋กœ ๋ถ„๋ฆฌ๋ผ ์žˆ์—ˆ์œผ๋ฏ€๋กœ ํ•ฉ์น  ๋•Œ๋„ \n ๋กœ
1078
+ # ์—ฐ๊ฒฐํ•œ๋‹ค. "".join ์œผ๋กœ ๋ถ™์ด๋ฉด ์ธ์ ‘ bulk ์˜ ๋ฌธ๋‹จ์ด ํ•œ ์ค„๋กœ
1079
+ # ๋ญ‰๊ฐœ์ง€๊ณ , ์ดํ›„ split_into_bulks ์žฌ์‹คํ–‰ ์‹œ ๋ฌธ๋‹จ ์ˆ˜๊ฐ€ ์ค„์–ด
1080
+ # step1 ์˜ <์›๋ฌธ>โ†”<๊ต์—ด_๋ชจ๋ธ_์ˆ˜์ •๊ฒฐ๊ณผ> ์ •๋ ฌ์ด ์–ด๊ธ‹๋‚˜๋ฉฐ ๊ตฌ์กฐ
1081
+ # ๋ณด์กด ๊ทœ์น™๊นŒ์ง€ ๊นจ์ง„๋‹ค (์ฐธ์กฐ: run.py:326 "\n".join).
1082
+ text = "\n".join(
1083
+ str(p) if not isinstance(p, str) else p for p in processed
1084
+ )
1085
  elif step["type"] == "specialist":
1086
  text = run_specialist(step, text, original_text, prompts, model, client)
1087
  # If a step produced an unusably short output (< 10% of input), the
postprocess.py CHANGED
@@ -380,7 +380,7 @@ def apply_paragraph_dedupe(
380
  text: str,
381
  original: str,
382
  min_len: int = 40,
383
- prefix_len: int = 80,
384
  ) -> str:
385
  """LLM์ด ๋ฑ‰์€ ์ค‘๋ณต ๋ฌธ๋‹จ์„ ์ œ๊ฑฐํ•œ๋‹ค.
386
 
@@ -411,13 +411,12 @@ def apply_paragraph_dedupe(
411
  if not text:
412
  return text
413
 
414
- in_paras_norm = [_normalize_paragraph(p) for p in _split_output_paragraphs(original or "")]
415
- in_exact: Counter[str] = Counter()
416
- in_prefix: Counter[str] = Counter()
417
- for norm in in_paras_norm:
418
- if len(norm) >= min_len:
419
- in_exact[norm] += 1
420
- in_prefix[norm[:prefix_len]] += 1
421
 
422
  out_paras = _split_output_paragraphs(text)
423
  out_exact_seen: Counter[str] = Counter()
@@ -435,12 +434,17 @@ def apply_paragraph_dedupe(
435
  out_exact_seen[norm] += 1
436
  out_prefix_seen[prefix] += 1
437
 
 
 
 
 
 
438
  exact_dup = (
439
- out_exact_seen[norm] > in_exact.get(norm, 0)
440
  and out_exact_seen[norm] >= 2
441
  )
442
  near_dup = (
443
- out_prefix_seen[prefix] > in_prefix.get(prefix, 0)
444
  and out_prefix_seen[prefix] >= 2
445
  )
446
 
 
380
  text: str,
381
  original: str,
382
  min_len: int = 40,
383
+ prefix_len: int = 30,
384
  ) -> str:
385
  """LLM์ด ๋ฑ‰์€ ์ค‘๋ณต ๋ฌธ๋‹จ์„ ์ œ๊ฑฐํ•œ๋‹ค.
386
 
 
411
  if not text:
412
  return text
413
 
414
+ # Input ์€ "normalized ์ „์ฒด ๋ฌธ์ž์—ด" ๋กœ ๋‘๊ณ  substring count ๋ฅผ ์“ด๋‹ค.
415
+ # ์ด์ „ ๊ตฌํ˜„์€ input ์„ ๋ฌธ๋‹จ Counter ๋กœ ์…Œ๋Š”๋ฐ, LLM ์ด ๋ฌธ๋‹จ ๊ฒฝ๊ณ„๋ฅผ ์žฌ๊ตฌ์กฐํ™”
416
+ # (์˜ˆ: ์ค„๋ฐ”๊ฟˆ ์—†์ด ์ค‘๋ณต ๋ฌธ์žฅ์ด ๋“ค์–ด์žˆ๋˜ ํ•œ ๋ฌธ๋‹จ์„ ์—ฌ๋Ÿฌ ๋ฌธ๋‹จ์œผ๋กœ ์ชผ๊ฐฌ) ํ•˜๋ฉด
417
+ # output ๋ฌธ๋‹จ์ด input ๋ฌธ๋‹จ๊ณผ exact match ๊ฐ€ ์•ˆ ๋˜์–ด input_count=0 ์œผ๋กœ ์žกํ˜€,
418
+ # ์ •๋‹นํ•œ ์ค‘๋ณต (์ €์ž/์›๋ณธ์ด ์˜๋„ํ•œ ๋ฐ˜๋ณต) ๊นŒ์ง€ drop ๋˜๋Š” ๋ฒ„๊ทธ๊ฐ€ ์žˆ์—ˆ๋‹ค.
419
+ in_norm = _normalize_paragraph(original or "")
 
420
 
421
  out_paras = _split_output_paragraphs(text)
422
  out_exact_seen: Counter[str] = Counter()
 
434
  out_exact_seen[norm] += 1
435
  out_prefix_seen[prefix] += 1
436
 
437
+ # input ๋ฌธ์ž์—ด ์ „์ฒด์—์„œ ํ•ด๋‹น ๋ฌธ๋‹จ(๋˜๋Š” prefix)์ด ๋ช‡ ๋ฒˆ substring ์œผ๋กœ
438
+ # ๋“ฑ์žฅํ•˜๋Š”์ง€ ์ง‘๊ณ„. output_count ๊ฐ€ input_count ๋ฅผ ์ดˆ๊ณผํ•  ๋•Œ๋งŒ drop.
439
+ in_exact_count = in_norm.count(norm) if in_norm else 0
440
+ in_prefix_count = in_norm.count(prefix) if in_norm else 0
441
+
442
  exact_dup = (
443
+ out_exact_seen[norm] > in_exact_count
444
  and out_exact_seen[norm] >= 2
445
  )
446
  near_dup = (
447
+ out_prefix_seen[prefix] > in_prefix_count
448
  and out_prefix_seen[prefix] >= 2
449
  )
450