File size: 3,823 Bytes
8f6d79d 39cfcd1 8f6d79d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | """Long-document batching and safety tests."""
from __future__ import annotations
import os
os.environ.setdefault("LANGUAGE_TOOL_ENABLED", "false")
os.environ.setdefault("LANGUAGE_TOOL_URL", "")
os.environ.setdefault("GRAMMAR_FIX_OUTPUT", "false")
os.environ.setdefault("ENGINE_USE_MINILM_SAFETY", "false")
from app.engine.normalize import detect_blocks, normalize_text
from app.engine.orchestrator import rewrite_document
from app.engine.safety import check_safety
from app.engine.segment import iter_paragraph_batches, word_count
def _make_long_doc(target_words: int = 10500) -> str:
templates = [
"Ram went to school yesterday happily.",
"The committee approved the budget today carefully.",
"Workers cleaned the factory floor this morning quickly.",
"Students completed the assignment last night diligently.",
"Although rain fell heavily, traffic moved slowly through town.",
"# Section Heading\n\n",
"- Keep this list item unchanged forever.\n",
"She sent the report to https://files.example.org/a.pdf yesterday.\n\n",
]
parts: list[str] = []
while word_count("\n\n".join(parts)) < target_words:
for t in templates:
parts.append(t.strip())
if word_count("\n\n".join(parts)) >= target_words:
break
return "\n\n".join(parts)
def test_long_document_batching():
doc = _make_long_doc(10500)
assert word_count(doc) >= 10000
result = rewrite_document(doc, batch_paras=15)
assert result.stats.batches >= 2
assert result.input_words >= 10000
assert result.stats.sentences > 50
# Paragraph-ish structure preserved (blank-line separated blocks)
assert result.text.count("\n\n") >= 10
assert result.stats.seconds < 180 # CPU smoke budget
def test_bibliography_passthrough():
src = (
"Ram went to school yesterday happily.\n\n"
"References\n\n"
"Smith, J. (2020). A paper about schools. Journal of Education.\n"
"Doe, A. (2019). Another citation here with many words about learning."
)
result = rewrite_document(src)
assert "Smith, J." in result.text
assert "References" in result.text
def test_bibliography_ends_at_following_heading():
src = (
"References\n\n"
"Smith, J. (2020). A paper about schools.\n\n"
"# Appendix\n\n"
"Ram went to school yesterday happily."
)
result = rewrite_document(src)
appendix_sentence = next(
record for record in result.sentences if record.original.startswith("Ram ")
)
assert appendix_sentence.status == "rewritten"
assert "Yesterday, Ram happily went to school." in result.text
def test_safety_rejects_entity_loss():
original = "Alice met Bob yesterday at the park."
bad = "Someone met a friend yesterday at the park."
safety = check_safety(original, bad, use_minilm=False)
assert not safety.ok
assert any(r.startswith("entity:") or r == "negation" for r in safety.reasons) or (
"entity" in " ".join(safety.reasons)
)
def test_safety_rejects_negation_flip():
original = "Ram did not go to school yesterday."
bad = "Ram did go to school yesterday."
safety = check_safety(original, bad, use_minilm=False)
assert not safety.ok
assert "negation" in safety.reasons or "polarity" in safety.reasons
def test_normalize_and_blocks():
raw = "Hello world.\n\n# Title\n\n- item one\n\n```\ncode\n```"
norm = normalize_text(raw)
blocks = detect_blocks(norm)
kinds = {b.kind for b in blocks}
assert "heading" in kinds or "paragraph" in kinds
batches = list(iter_paragraph_batches(blocks, batch_paras=2))
assert batches
|