feat(rag): re-chunk rulebook by rule number for precise retrieval

Parser now emits a separate paragraph per top-level rule (NNN.) so
each keyword and sub-rule becomes its own chunk instead of one 30K-char
blob covering the entire section. Adds a fallback rule-split in
_chunk_section for any section that still arrives as a single oversized
paragraph. Corpus v1.2.0 re-ingested with 464 chunks (201 rulebook).

Also boosts official sources (rulebook, tournament_rules, patch_notes)
by 5% in RRF fusion so base-rule chunks rank above errata on tie.

backend/app/rag/retrieval.py

@@ -90,6 +90,10 @@ def fts_search(
     ]
 
 
+_OFFICIAL_SOURCES = frozenset({"rulebook", "tournament_rules", "patch_notes"})
+_OFFICIAL_BOOST = 1.05  # official rule sources get a 5% score boost over errata
+
+
 def _rrf_fuse(
     vector_results: list[Chunk],
     fts_results: list[Chunk],
@@ -102,6 +106,9 @@ def _rrf_fuse(
     1 / (rrf_k + rank_l(d)), where rank_l(d) is 1-based (only counted if d
     appears in l).
 
+    Rulebook chunks receive a _RULEBOOK_BOOST multiplier so base-rule chunks
+    rank above errata chunks when scores are comparable.
+
     Dedup key: chunk.id.
     Tie-break: chunk that appeared in vector_results wins (stable).
     Preserves original similarity from vector side; FTS-only chunks keep 0.0.
@@ -115,13 +122,15 @@ def _rrf_fuse(
 
     for rank_0, chunk in enumerate(vector_results):
         rank = rank_0 + 1  # 1-based
-        scores[chunk.id] = scores.get(chunk.id, 0.0) + 1.0 / (rrf_k + rank)
+        boost = _OFFICIAL_BOOST if chunk.source_type in _OFFICIAL_SOURCES else 1.0
+        scores[chunk.id] = scores.get(chunk.id, 0.0) + boost / (rrf_k + rank)
         chunks_by_id[chunk.id] = chunk  # vector side wins for Chunk object
         in_vector.add(chunk.id)
 
     for rank_0, chunk in enumerate(fts_results):
         rank = rank_0 + 1  # 1-based
-        scores[chunk.id] = scores.get(chunk.id, 0.0) + 1.0 / (rrf_k + rank)
+        boost = _OFFICIAL_BOOST if chunk.source_type in _OFFICIAL_SOURCES else 1.0
+        scores[chunk.id] = scores.get(chunk.id, 0.0) + boost / (rrf_k + rank)
         if chunk.id not in chunks_by_id:
             chunks_by_id[chunk.id] = chunk  # FTS-only: use FTS chunk (similarity=0.0)
 

backend/scripts/add_corpus_v1_2_0.py

@@ -0,0 +1,31 @@
+"""
+Migración de corpus v1.1.0 → v1.2.0 para chunks no-rulebook.
+
+Copia errata, tournament_rules y patch_notes desde v1.1.0.
+El rulebook se re-ingestea por separado con el parser re-chunkeado.
+"""
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+
+import psycopg2
+
+conn = psycopg2.connect(os.getenv("DATABASE_URL"))
+cur = conn.cursor()
+cur.execute("""
+    INSERT INTO corpus_chunks
+        (id, content, embedding, source_type, source_document,
+         section, parent_section, corpus_version, ingested_at)
+    SELECT
+        id, content, embedding, source_type, source_document,
+        section, parent_section, 'v1.2.0', NOW()
+    FROM corpus_chunks
+    WHERE corpus_version = 'v1.1.0'
+      AND source_type != 'rulebook'
+    ON CONFLICT (id) DO NOTHING
+""")
+conn.commit()
+print(f"Copiados: {cur.rowcount} chunks no-rulebook a v1.2.0")
+conn.close()

backend/scripts/ingest.py

@@ -26,10 +26,13 @@ DATABASE_URL = os.getenv("DATABASE_URL")
 EMBED_MODEL = "BAAI/bge-m3"
 CHUNK_SIZE = 512   # tokens aproximados
 CHUNK_OVERLAP = 50
+_RULE_SPLIT = re.compile(r"(?=\b\d{3,}\.\s)")
 
 SOURCES = [
     ("data/processed/rulebook.md", "rulebook"),
     ("data/processed/errata.md", "errata"),
+    ("data/processed/tournament_rules.md", "tournament_rules"),
+    ("data/processed/patch_notes.md", "patch_notes"),
 ]
 
 
@@ -69,6 +72,10 @@ def _chunk_section(section: dict, source_type: str, source_document: str) -> lis
 
     # Dividir en párrafos y agrupar respetando el tamaño
     paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()]
+
+    # Fallback: si sigue siendo 1 párrafo gigante, dividir por número de regla (NNN.)
+    if len(paragraphs) <= 1 and _approx_tokens(content) > CHUNK_SIZE:
+        paragraphs = [p.strip() for p in _RULE_SPLIT.split(content) if p.strip()]
     chunks = []
     current: list[str] = []
     current_tokens = 0

backend/scripts/parse_rulebook.py

@@ -7,10 +7,14 @@ Detecta headers por tamaño de fuente relativo al body text:
   - > 1.2x body → H3 (subsección)
   - else        → párrafo
 """
+import re
+
 import pymupdf
 from pathlib import Path
 from statistics import mode
 
+_RULE_BOUNDARY = re.compile(r"^\d{3,}\.")
+
 
 def _extract_spans(doc: pymupdf.Document) -> list[dict]:
     spans = []
@@ -59,6 +63,8 @@ def _spans_to_markdown(spans: list[dict], body_size: float) -> str:
     for span in spans:
         kind = _classify(span["size"], body_size)
         if kind == "body":
+            if current_body and _RULE_BOUNDARY.match(span["text"]):
+                flush_body()
             current_body.append(span["text"])
         else:
             flush_body()

backend/tests/test_add_corpus_v1_2_0.py

@@ -0,0 +1,34 @@
+"""Unit tests for add_corpus_v1_2_0 migration script."""
+import sys
+from unittest.mock import MagicMock, patch
+
+
+def _run_migration(database_url: str = "postgresql://fake"):
+    if "scripts.add_corpus_v1_2_0" in sys.modules:
+        del sys.modules["scripts.add_corpus_v1_2_0"]
+    mock_conn = MagicMock()
+    mock_cur = MagicMock()
+    mock_conn.cursor.return_value = mock_cur
+    mock_cur.rowcount = 150
+    with patch("psycopg2.connect", return_value=mock_conn), \
+         patch.dict("os.environ", {"DATABASE_URL": database_url}):
+        import scripts.add_corpus_v1_2_0  # noqa: F401
+    return mock_conn, mock_cur
+
+
+def test_migration_inserts_non_rulebook_chunks_from_v1_1_0():
+    _, mock_cur = _run_migration()
+    sql = mock_cur.execute.call_args[0][0]
+    assert "v1.1.0" in sql
+    assert "v1.2.0" in sql
+    assert "rulebook" in sql
+
+
+def test_migration_commits_transaction():
+    mock_conn, _ = _run_migration()
+    mock_conn.commit.assert_called_once()
+
+
+def test_migration_closes_connection():
+    mock_conn, _ = _run_migration()
+    mock_conn.close.assert_called_once()

backend/tests/test_ingest.py

@@ -192,3 +192,29 @@ def test_build_chunks_source_document_is_stem(tmp_path):
     f.write_text(md, encoding="utf-8")
     chunks = build_chunks(str(f), "errata")
     assert all(c["source_document"] == "errata" for c in chunks)
+
+
+# ---------------------------------------------------------------------------
+# _chunk_section — fallback rule-split
+# ---------------------------------------------------------------------------
+
+def _make_giant_rule_block() -> dict:
+    """Un único párrafo sin \\n\\n con números de regla embebidos (~3200 tokens)."""
+    rule_block = " ".join(
+        f"{800 + i}. This is a long rule description that takes up space. " * 15
+        for i in range(8)
+    )
+    return {"content": rule_block, "header": "Keywords", "level": 2}
+
+
+def test_chunk_section_falls_back_to_rule_split_for_single_huge_paragraph():
+    section = _make_giant_rule_block()
+    chunks = _chunk_section(section, "rulebook", "rulebook.md")
+    assert len(chunks) > 1, "Fallback rule-split debe generar múltiples chunks"
+
+
+def test_chunk_section_fallback_chunks_contain_rule_numbers():
+    section = _make_giant_rule_block()
+    chunks = _chunk_section(section, "rulebook", "rulebook.md")
+    rule_nums = {c["content"].split(".")[0].strip() for c in chunks}
+    assert any(n.isdigit() and len(n) == 3 for n in rule_nums)

backend/tests/test_parsers.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from scripts.parse_rulebook import parse_rulebook
+from scripts.parse_rulebook import parse_rulebook, _spans_to_markdown
 
 FIXTURES = Path(__file__).parent / "fixtures"
 SAMPLE_PDF = FIXTURES / "rulebook_sample.pdf"
@@ -41,3 +41,42 @@ def test_parse_rulebook_matches_expected_output():
     result = parse_rulebook(SAMPLE_PDF)
     expected = EXPECTED_MD.read_text(encoding="utf-8")
     assert result.strip() == expected.strip()
+
+
+# ---------------------------------------------------------------------------
+# _spans_to_markdown — rule boundary splitting
+# ---------------------------------------------------------------------------
+
+def _body_span(text: str, size: float = 10.0) -> dict:
+    return {"text": text, "size": size}
+
+
+def test_spans_to_markdown_splits_rules_into_separate_paragraphs():
+    spans = [
+        _body_span("805. Accelerate (Action): blah blah blah."),
+        _body_span("806. Ambush (Passive): blah blah blah."),
+        _body_span("807. Armor N (Passive): blah blah blah."),
+    ]
+    result = _spans_to_markdown(spans, body_size=10.0)
+    paragraphs = [p for p in result.split("\n\n") if p.strip()]
+    assert len(paragraphs) == 3
+
+
+def test_spans_to_markdown_non_rule_body_merges():
+    spans = [
+        _body_span("This is a sentence."),
+        _body_span("This continues the same paragraph."),
+    ]
+    result = _spans_to_markdown(spans, body_size=10.0)
+    paragraphs = [p for p in result.split("\n\n") if p.strip()]
+    assert len(paragraphs) == 1
+
+
+def test_spans_to_markdown_rule_content_is_preserved():
+    spans = [
+        _body_span("805. Accelerate (Action): Move this unit."),
+        _body_span("806. Ambush (Passive): React to attack."),
+    ]
+    result = _spans_to_markdown(spans, body_size=10.0)
+    assert "805. Accelerate" in result
+    assert "806. Ambush" in result