updated explanation parser for good retrival

db/chunker/bns_chunker.py

@@ -0,0 +1,430 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+from typing import List, Optional
+
+
+# =========================================================
+# CHUNK MODEL
+# =========================================================
+
+@dataclass
+class LegalChunk:
+
+    chunk_id: str
+
+    level: str
+
+    document: str
+
+    chapter_no: Optional[str]
+
+    section_no: Optional[str]
+
+    clause_no: Optional[str]
+
+    sub_clause_no: Optional[str]
+
+    roman_no: Optional[str]
+
+    title: str
+
+    text: str
+
+    enriched_text: str
+
+    parent_id: Optional[str]
+
+
+# =========================================================
+# CHUNKER
+# =========================================================
+
+class LegalChunker:
+
+    def __init__(
+        self,
+        document_name: str = "BNS"
+    ):
+        self.document_name = document_name
+
+    # -----------------------------------------------------
+    # Helpers
+    # -----------------------------------------------------
+
+    def build_context(
+        self,
+        title: str,
+        section_no: str = "",
+        clause_no: str = "",
+        sub_clause_no: str = "",
+        roman_no: str = ""
+    ) -> str:
+
+        parts = [
+            self.document_name
+        ]
+
+        if section_no:
+            parts.append(
+                f"Section {section_no}"
+            )
+
+        if title:
+            parts.append(
+                title
+            )
+
+        if clause_no:
+            parts.append(
+                f"Clause ({clause_no})"
+            )
+
+        if sub_clause_no:
+            parts.append(
+                f"SubClause ({sub_clause_no})"
+            )
+
+        if roman_no:
+            parts.append(
+                f"Roman Clause ({roman_no})"
+            )
+
+        return "\n".join(parts)
+
+    # -----------------------------------------------------
+    # Main
+    # -----------------------------------------------------
+
+    def chunk_document(
+        self,
+        document
+    ) -> List[LegalChunk]:
+
+        chunks = []
+
+        for chapter in document.chapters:
+
+            for section in chapter.sections:
+
+                section_id = (
+                    f"BNS-{section.section_no}"
+                )
+
+                title = (
+                    getattr(
+                        section,
+                        "title",
+                        ""
+                    )
+                )
+
+                # =================================
+                # SECTION CHUNK
+                # =================================
+
+                section_chunk = LegalChunk(
+                    chunk_id=section_id,
+
+                    level="section",
+
+                    document="bns",
+
+                    chapter_no=
+                        chapter.chapter_no,
+
+                    section_no=
+                        section.section_no,
+
+                    clause_no=None,
+
+                    sub_clause_no=None,
+
+                    roman_no=None,
+
+                    title=title,
+
+                    text=section.text,
+
+                    enriched_text=
+                        self.build_context(
+                            title=title,
+                            section_no=
+                                section.section_no
+                        )
+                        + "\n\n"
+                        + section.text,
+
+                    parent_id=None
+                )
+
+                chunks.append(
+                    section_chunk
+                )
+
+                # =================================
+                # CLAUSES
+                # =================================
+
+                for clause in section.clauses:
+
+                    clause_id = (
+                        f"{section_id}"
+                        f"({clause.clause_no})"
+                    )
+
+                    clause_chunk = LegalChunk(
+
+                        chunk_id=
+                            clause_id,
+
+                        level=
+                            "clause",
+
+                        document=
+                            "bns",
+
+                        chapter_no=
+                            chapter.chapter_no,
+
+                        section_no=
+                            section.section_no,
+
+                        clause_no=
+                            clause.clause_no,
+
+                        sub_clause_no=
+                            None,
+
+                        roman_no=
+                            None,
+
+                        title=
+                            title,
+
+                        text=
+                            clause.text,
+
+                        enriched_text=
+                            self.build_context(
+                                title=title,
+                                section_no=
+                                    section.section_no,
+                                clause_no=
+                                    clause.clause_no
+                            )
+                            + "\n\n"
+                            + clause.text,
+
+                        parent_id=
+                            section_id
+                    )
+
+                    chunks.append(
+                        clause_chunk
+                    )
+
+                    # =============================
+                    # SUB CLAUSES
+                    # =============================
+
+                    for sub in clause.sub_clauses:
+
+                        sub_id = (
+                            f"{clause_id}"
+                            f"({sub.sub_clause_no})"
+                        )
+
+                        sub_chunk = LegalChunk(
+
+                            chunk_id=
+                                sub_id,
+
+                            level=
+                                "subclause",
+
+                            document=
+                                "bns",
+
+                            chapter_no=
+                                chapter.chapter_no,
+
+                            section_no=
+                                section.section_no,
+
+                            clause_no=
+                                clause.clause_no,
+
+                            sub_clause_no=
+                                sub.sub_clause_no,
+
+                            roman_no=
+                                None,
+
+                            title=
+                                title,
+
+                            text=
+                                sub.text,
+
+                            enriched_text=
+                                self.build_context(
+                                    title=title,
+                                    section_no=
+                                        section.section_no,
+                                    clause_no=
+                                        clause.clause_no,
+                                    sub_clause_no=
+                                        sub.sub_clause_no
+                                )
+                                + "\n\n"
+                                + sub.text,
+
+                            parent_id=
+                                clause_id
+                        )
+
+                        chunks.append(
+                            sub_chunk
+                        )
+
+                        # =========================
+                        # ROMAN CLAUSES
+                        # =========================
+
+                        for roman in sub.roman_clauses:
+
+                            roman_id = (
+                                f"{sub_id}"
+                                f"({roman.roman_no})"
+                            )
+
+                            roman_chunk = LegalChunk(
+
+                                chunk_id=
+                                    roman_id,
+
+                                level=
+                                    "roman",
+
+                                document=
+                                    "bns",
+
+                                chapter_no=
+                                    chapter.chapter_no,
+
+                                section_no=
+                                    section.section_no,
+
+                                clause_no=
+                                    clause.clause_no,
+
+                                sub_clause_no=
+                                    sub.sub_clause_no,
+
+                                roman_no=
+                                    roman.roman_no,
+
+                                title=
+                                    title,
+
+                                text=
+                                    roman.text,
+
+                                enriched_text=
+                                    self.build_context(
+                                        title=title,
+                                        section_no=
+                                            section.section_no,
+                                        clause_no=
+                                            clause.clause_no,
+                                        sub_clause_no=
+                                            sub.sub_clause_no,
+                                        roman_no=
+                                            roman.roman_no
+                                    )
+                                    + "\n\n"
+                                    + roman.text,
+
+                                parent_id=
+                                    sub_id
+                            )
+
+                            chunks.append(
+                                roman_chunk
+                            )
+
+                # =================================
+                # EXPLANATIONS
+                # =================================
+
+                for explanation in (
+                    section.explanations
+                ):
+
+                    explanation_id = (
+                        f"{section_id}"
+                        f"-EXP-"
+                        f"{explanation.explanation_no}"
+                    )
+
+                    chunk = LegalChunk(
+
+                        chunk_id=
+                            explanation_id,
+
+                        level=
+                            "explanation",
+
+                        document=
+                            "bns",
+
+                        chapter_no=
+                            chapter.chapter_no,
+
+                        section_no=
+                            section.section_no,
+
+                        clause_no=None,
+
+                        sub_clause_no=None,
+
+                        roman_no=None,
+
+                        title=title,
+
+                        text=
+                            explanation.text,
+
+                        enriched_text=
+                            self.build_context(
+                                title=title,
+                                section_no=
+                                    section.section_no
+                            )
+                            + "\n\n"
+                            + explanation.text,
+
+                        parent_id=
+                            section_id
+                    )
+
+                    chunks.append(
+                        chunk
+                    )
+
+        return chunks
+
+
+# =========================================================
+# EXPORT
+# =========================================================
+
+def chunks_to_dicts(
+    chunks: List[LegalChunk]
+):
+
+    return [
+        asdict(chunk)
+        for chunk in chunks
+    ]

db/embbeder/bns_embedder.py

@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+from typing import List
+
+from tqdm import tqdm
+
+from sentence_transformers import (
+    SentenceTransformer
+)
+
+from qdrant_client import QdrantClient
+
+from qdrant_client.models import (
+    Distance,
+    VectorParams,
+    PointStruct
+)
+
+
+class QdrantEmbeddingGenerator:
+
+    def __init__(
+        self,
+        collection_name: str = "bns",
+        model_name: str = "BAAI/bge-large-en-v1.5",
+        host: str = "localhost",
+        port: int = 6333
+    ):
+
+        self.collection_name = (
+            collection_name
+        )
+
+        self.model = (
+            SentenceTransformer(
+                model_name
+            )
+        )
+
+        self.client = (
+            QdrantClient(
+                host=host,
+                port=port
+            )
+        )
+
+    # =====================================
+    # CREATE COLLECTION
+    # =====================================
+
+    def create_collection(self):
+
+        dimension = (
+            self.model
+            .get_sentence_embedding_dimension()
+        )
+
+        collections = (
+            self.client
+            .get_collections()
+        )
+
+        existing = {
+            c.name
+            for c in collections.collections
+        }
+
+        if (
+            self.collection_name
+            not in existing
+        ):
+
+            self.client.create_collection(
+                collection_name=
+                    self.collection_name,
+
+                vectors_config=
+                    VectorParams(
+                        size=dimension,
+                        distance=
+                            Distance.COSINE
+                    )
+            )
+
+            print(
+                f"Created collection "
+                f"{self.collection_name}"
+            )
+
+    # =====================================
+    # BUILD TEXT
+    # =====================================
+
+    def build_text(
+        self,
+        chunk: dict
+    ) -> str:
+
+        return chunk.get(
+            "enriched_text",
+            chunk["text"]
+        )
+
+    # =====================================
+    # INGEST
+    # =====================================
+
+    def ingest_chunks(
+        self,
+        chunks: List[dict],
+        batch_size: int = 64
+    ):
+
+        self.create_collection()
+
+        point_id = 1
+
+        for start in tqdm(
+            range(
+                0,
+                len(chunks),
+                batch_size
+            )
+        ):
+
+            batch = chunks[
+                start:
+                start + batch_size
+            ]
+
+            texts = [
+                self.build_text(
+                    chunk
+                )
+                for chunk in batch
+            ]
+
+            embeddings = (
+                self.model.encode(
+                    texts,
+                    normalize_embeddings=True,
+                    convert_to_numpy=True,
+                    show_progress_bar=False
+                )
+            )
+
+            points = []
+
+            for chunk, embedding in zip(
+                batch,
+                embeddings
+            ):
+
+                payload = {
+
+                    "chunk_id":
+                        chunk.get(
+                            "chunk_id"
+                        ),
+
+                    "level":
+                        chunk.get(
+                            "level"
+                        ),
+
+                    "chapter_no":
+                        chunk.get(
+                            "chapter_no"
+                        ),
+
+                    "section_no":
+                        chunk.get(
+                            "section_no"
+                        ),
+
+                    "clause_no":
+                        chunk.get(
+                            "clause_no"
+                        ),
+
+                    "sub_clause_no":
+                        chunk.get(
+                            "sub_clause_no"
+                        ),
+
+                    "roman_no":
+                        chunk.get(
+                            "roman_no"
+                        ),
+
+                    "title":
+                        chunk.get(
+                            "title"
+                        ),
+
+                    "text":
+                        chunk.get(
+                            "text"
+                        ),
+
+                    "parent_id":
+                        chunk.get(
+                            "parent_id"
+                        )
+                }
+
+                points.append(
+                    PointStruct(
+                        id=point_id,
+                        vector=
+                            embedding.tolist(),
+                        payload=
+                            payload
+                    )
+                )
+
+                point_id += 1
+
+            self.client.upsert(
+                collection_name=
+                    self.collection_name,
+
+                points=
+                    points
+            )
+
+        print(
+            f"Ingested "
+            f"{len(chunks)} chunks"
+        )

db/parsers/bns/chunk_temp.py

@@ -0,0 +1,430 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+from typing import List, Optional
+
+
+# =========================================================
+# CHUNK MODEL
+# =========================================================
+
+@dataclass
+class LegalChunk:
+
+    chunk_id: str
+
+    level: str
+
+    document: str
+
+    chapter_no: Optional[str]
+
+    section_no: Optional[str]
+
+    clause_no: Optional[str]
+
+    sub_clause_no: Optional[str]
+
+    roman_no: Optional[str]
+
+    title: str
+
+    text: str
+
+    enriched_text: str
+
+    parent_id: Optional[str]
+
+
+# =========================================================
+# CHUNKER
+# =========================================================
+
+class LegalChunker:
+
+    def __init__(
+        self,
+        document_name: str = "BNS"
+    ):
+        self.document_name = document_name
+
+    # -----------------------------------------------------
+    # Helpers
+    # -----------------------------------------------------
+
+    def build_context(
+        self,
+        title: str,
+        section_no: str = "",
+        clause_no: str = "",
+        sub_clause_no: str = "",
+        roman_no: str = ""
+    ) -> str:
+
+        parts = [
+            self.document_name
+        ]
+
+        if section_no:
+            parts.append(
+                f"Section {section_no}"
+            )
+
+        if title:
+            parts.append(
+                title
+            )
+
+        if clause_no:
+            parts.append(
+                f"Clause ({clause_no})"
+            )
+
+        if sub_clause_no:
+            parts.append(
+                f"SubClause ({sub_clause_no})"
+            )
+
+        if roman_no:
+            parts.append(
+                f"Roman Clause ({roman_no})"
+            )
+
+        return "\n".join(parts)
+
+    # -----------------------------------------------------
+    # Main
+    # -----------------------------------------------------
+
+    def chunk_document(
+        self,
+        document
+    ) -> List[LegalChunk]:
+
+        chunks = []
+
+        for chapter in document.chapters:
+
+            for section in chapter.sections:
+
+                section_id = (
+                    f"BNS-{section.section_no}"
+                )
+
+                title = (
+                    getattr(
+                        section,
+                        "title",
+                        ""
+                    )
+                )
+
+                # =================================
+                # SECTION CHUNK
+                # =================================
+
+                section_chunk = LegalChunk(
+                    chunk_id=section_id,
+
+                    level="section",
+
+                    document="bns",
+
+                    chapter_no=
+                        chapter.chapter_no,
+
+                    section_no=
+                        section.section_no,
+
+                    clause_no=None,
+
+                    sub_clause_no=None,
+
+                    roman_no=None,
+
+                    title=title,
+
+                    text=section.text,
+
+                    enriched_text=
+                        self.build_context(
+                            title=title,
+                            section_no=
+                                section.section_no
+                        )
+                        + "\n\n"
+                        + section.text,
+
+                    parent_id=None
+                )
+
+                chunks.append(
+                    section_chunk
+                )
+
+                # =================================
+                # CLAUSES
+                # =================================
+
+                for clause in section.clauses:
+
+                    clause_id = (
+                        f"{section_id}"
+                        f"({clause.clause_no})"
+                    )
+
+                    clause_chunk = LegalChunk(
+
+                        chunk_id=
+                            clause_id,
+
+                        level=
+                            "clause",
+
+                        document=
+                            "bns",
+
+                        chapter_no=
+                            chapter.chapter_no,
+
+                        section_no=
+                            section.section_no,
+
+                        clause_no=
+                            clause.clause_no,
+
+                        sub_clause_no=
+                            None,
+
+                        roman_no=
+                            None,
+
+                        title=
+                            title,
+
+                        text=
+                            clause.text,
+
+                        enriched_text=
+                            self.build_context(
+                                title=title,
+                                section_no=
+                                    section.section_no,
+                                clause_no=
+                                    clause.clause_no
+                            )
+                            + "\n\n"
+                            + clause.text,
+
+                        parent_id=
+                            section_id
+                    )
+
+                    chunks.append(
+                        clause_chunk
+                    )
+
+                    # =============================
+                    # SUB CLAUSES
+                    # =============================
+
+                    for sub in clause.sub_clauses:
+
+                        sub_id = (
+                            f"{clause_id}"
+                            f"({sub.sub_clause_no})"
+                        )
+
+                        sub_chunk = LegalChunk(
+
+                            chunk_id=
+                                sub_id,
+
+                            level=
+                                "subclause",
+
+                            document=
+                                "bns",
+
+                            chapter_no=
+                                chapter.chapter_no,
+
+                            section_no=
+                                section.section_no,
+
+                            clause_no=
+                                clause.clause_no,
+
+                            sub_clause_no=
+                                sub.sub_clause_no,
+
+                            roman_no=
+                                None,
+
+                            title=
+                                title,
+
+                            text=
+                                sub.text,
+
+                            enriched_text=
+                                self.build_context(
+                                    title=title,
+                                    section_no=
+                                        section.section_no,
+                                    clause_no=
+                                        clause.clause_no,
+                                    sub_clause_no=
+                                        sub.sub_clause_no
+                                )
+                                + "\n\n"
+                                + sub.text,
+
+                            parent_id=
+                                clause_id
+                        )
+
+                        chunks.append(
+                            sub_chunk
+                        )
+
+                        # =========================
+                        # ROMAN CLAUSES
+                        # =========================
+
+                        for roman in sub.roman_clauses:
+
+                            roman_id = (
+                                f"{sub_id}"
+                                f"({roman.roman_no})"
+                            )
+
+                            roman_chunk = LegalChunk(
+
+                                chunk_id=
+                                    roman_id,
+
+                                level=
+                                    "roman",
+
+                                document=
+                                    "bns",
+
+                                chapter_no=
+                                    chapter.chapter_no,
+
+                                section_no=
+                                    section.section_no,
+
+                                clause_no=
+                                    clause.clause_no,
+
+                                sub_clause_no=
+                                    sub.sub_clause_no,
+
+                                roman_no=
+                                    roman.roman_no,
+
+                                title=
+                                    title,
+
+                                text=
+                                    roman.text,
+
+                                enriched_text=
+                                    self.build_context(
+                                        title=title,
+                                        section_no=
+                                            section.section_no,
+                                        clause_no=
+                                            clause.clause_no,
+                                        sub_clause_no=
+                                            sub.sub_clause_no,
+                                        roman_no=
+                                            roman.roman_no
+                                    )
+                                    + "\n\n"
+                                    + roman.text,
+
+                                parent_id=
+                                    sub_id
+                            )
+
+                            chunks.append(
+                                roman_chunk
+                            )
+
+                # =================================
+                # EXPLANATIONS
+                # =================================
+
+                for explanation in (
+                    section.explanations
+                ):
+
+                    explanation_id = (
+                        f"{section_id}"
+                        f"-EXP-"
+                        f"{explanation.explanation_no}"
+                    )
+
+                    chunk = LegalChunk(
+
+                        chunk_id=
+                            explanation_id,
+
+                        level=
+                            "explanation",
+
+                        document=
+                            "bns",
+
+                        chapter_no=
+                            chapter.chapter_no,
+
+                        section_no=
+                            section.section_no,
+
+                        clause_no=None,
+
+                        sub_clause_no=None,
+
+                        roman_no=None,
+
+                        title=title,
+
+                        text=
+                            explanation.text,
+
+                        enriched_text=
+                            self.build_context(
+                                title=title,
+                                section_no=
+                                    section.section_no
+                            )
+                            + "\n\n"
+                            + explanation.text,
+
+                        parent_id=
+                            section_id
+                    )
+
+                    chunks.append(
+                        chunk
+                    )
+
+        return chunks
+
+
+# =========================================================
+# EXPORT
+# =========================================================
+
+def chunks_to_dicts(
+    chunks: List[LegalChunk]
+):
+
+    return [
+        asdict(chunk)
+        for chunk in chunks
+    ]

db/parsers/bns/embedder_temp.py

@@ -0,0 +1,29 @@
+from sentence_transformers import (
+    SentenceTransformer
+)
+
+
+class LegalEmbedder:
+
+    def __init__(
+        self,
+        model_name: str =
+        "BAAI/bge-large-en-v1.5"
+    ):
+
+        self.model = (
+            SentenceTransformer(
+                model_name
+            )
+        )
+
+    def embed(
+        self,
+        texts: list[str]
+    ):
+
+        return self.model.encode(
+            texts,
+            normalize_embeddings=True,
+            convert_to_numpy=True
+        )

db/parsers/bns/explanation_parser.py

@@ -30,6 +30,22 @@ class ExplanationParser:
     EXPLANATION_RE = re.compile(
         r"(?im)^Explanation\s*([0-9IVXLCDM]*)\s*[\.\-—:]"
     )
+    STOP_RE = re.compile(
+        r"(?im)^("
+        r"Explanation\s*"
+        r"|Illustration[s]?\."
+        r"|Exception\s*[.—:-]"
+        r"|\(\d+[A-Za-z]?\)"
+        r")"
+    )
+    NEXT_BLOCK_RE = re.compile(
+        r"(?im)^("
+        r"Explanation\s*"
+        r"|Illustration[s]?\."
+        r"|Exception\s*[.—:-]"
+        r"|\(\d+[A-Za-z]?\)"
+        r")"
+    )
 
     # =====================================================
     # ROMAN NUMERAL NORMALIZATION
@@ -95,14 +111,35 @@ class ExplanationParser:
 
             start = match.start()
 
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
+            if i + 1 < len(matches):
+
+                end = matches[i + 1].start()
+
+            else:
+
+                end = len(text)
+
+                remaining = text[start:]
+
+                
 
-            explanation_no = self.normalize_explanation_no(
-                match.group(1)
+                next_block = list(
+                    self.NEXT_BLOCK_RE.finditer(
+                        remaining
+                    )
+                )
+
+                if len(next_block) > 1:
+
+                    end = (
+                        start +
+                        next_block[1].start()
+                    )
+
+            explanation_no = (
+                self.normalize_explanation_no(
+                    match.group(1)
+                )
             )
 
             explanation_text = (
@@ -113,13 +150,15 @@ class ExplanationParser:
             explanations.append(
                 Explanation(
                     document="bns",
-                    explanation_no=explanation_no,
-                    text=explanation_text
+                    explanation_no=
+                        explanation_no,
+
+                    text=
+                        explanation_text
                 )
             )
 
         return explanations
-
     # =====================================================
     # VALIDATION
     # =====================================================

db/parsers/bns/ingest.py

@@ -0,0 +1,97 @@
+from tqdm import tqdm
+
+from qdrant_client.models import (
+    PointStruct
+)
+
+from embedder_temp import (
+    LegalEmbedder
+)
+
+from qdrant_store import (
+    QdrantStore
+)
+
+
+class LegalIngestionPipeline:
+
+    def __init__(self):
+
+        self.embedder = (
+            LegalEmbedder()
+        )
+
+        self.store = (
+            QdrantStore(
+                collection_name="bns"
+            )
+        )
+
+    def ingest(
+        self,
+        chunks,
+        batch_size=64
+    ):
+
+        dimension = (
+            self.embedder
+            .model
+            .get_sentence_embedding_dimension()
+        )
+
+        self.store.create_collection(
+            dimension
+        )
+
+        point_id = 1
+
+        for start in tqdm(
+            range(
+                0,
+                len(chunks),
+                batch_size
+            )
+        ):
+
+            batch = chunks[
+                start:
+                start + batch_size
+            ]
+
+            texts = [
+                chunk[
+                    "enriched_text"
+                ]
+                for chunk in batch
+            ]
+
+            embeddings = (
+                self.embedder.embed(
+                    texts
+                )
+            )
+
+            points = []
+
+            for chunk, embedding in zip(
+                batch,
+                embeddings
+            ):
+
+                points.append(
+                    PointStruct(
+                        id=point_id,
+
+                        vector=
+                            embedding.tolist(),
+
+                        payload=
+                            chunk
+                    )
+                )
+
+                point_id += 1
+
+            self.store.upsert(
+                points
+            )

db/parsers/bns/qdrant_store.py

@@ -0,0 +1,96 @@
+from qdrant_client import (
+    QdrantClient
+)
+
+from qdrant_client.models import (
+    Distance,
+    VectorParams,
+    PointStruct
+)
+
+
+class QdrantStore:
+
+    def __init__(
+        self,
+        collection_name: str,
+        host="localhost",
+        port=6333
+    ):
+
+        self.collection_name = (
+            collection_name
+        )
+
+        self.client = (
+            QdrantClient(
+                host=host,
+                port=port
+            )
+        )
+
+    def create_collection(
+        self,
+        vector_size: int
+    ):
+
+        collections = (
+            self.client
+            .get_collections()
+        )
+
+        existing = {
+            c.name
+            for c in collections.collections
+        }
+
+        if (
+            self.collection_name
+            not in existing
+        ):
+
+            self.client.create_collection(
+                collection_name=
+                    self.collection_name,
+
+                vectors_config=
+                    VectorParams(
+                        size=
+                            vector_size,
+
+                        distance=
+                            Distance.COSINE
+                    )
+            )
+
+    def upsert(
+        self,
+        points
+    ):
+
+        self.client.upsert(
+            collection_name=
+                self.collection_name,
+
+            points=
+                points
+        )
+
+    def search(
+        self,
+        query_vector,
+        limit=10
+    ):
+
+        return (
+            self.client.query_points(
+                collection_name=
+                    self.collection_name,
+
+                query=
+                    query_vector,
+
+                limit=
+                    limit
+            )
+        )

db/parsers/bns/search.py

@@ -0,0 +1,68 @@
+from sentence_transformers import (
+    SentenceTransformer
+)
+
+from qdrant_client import (
+    QdrantClient
+)
+
+
+class BNSRetriever:
+
+    def __init__(
+        self,
+        collection_name="bns",
+        model_name="BAAI/bge-large-en-v1.5"
+    ):
+
+        self.collection_name = (
+            collection_name
+        )
+
+        self.client = (
+            QdrantClient(
+                host="localhost",
+                port=6333
+            )
+        )
+
+        self.model = (
+            SentenceTransformer(
+                model_name
+            )
+        )
+
+    def search(
+        self,
+        query: str,
+        limit: int = 10
+    ):
+
+        query_vector = (
+            self.model.encode(
+                query,
+                normalize_embeddings=True
+            ).tolist()
+        )
+
+        hits = (
+            self.client.query_points(
+                collection_name=
+                    self.collection_name,
+
+                query=
+                    query_vector,
+
+                limit=
+                    limit
+            )
+        )
+
+        return hits
+    
+
+if __name__=="__main__":
+    query = "punishment for trafficking of person"
+    
+    bns_search=BNSRetriever()
+    print(bns_search.search(query))

db/parsers/bns/usage.py

@@ -0,0 +1,112 @@
+from collections import Counter
+
+from cleaner import BNSTextCleaner
+from bns_parser import BNSParser
+
+from chunk_temp import (
+    LegalChunker,
+    chunks_to_dicts
+)
+
+from ingest import (
+    LegalIngestionPipeline
+)
+
+# =====================================================
+# LOAD BNS
+# =====================================================
+
+with open(
+    "../../pdfs/bns.txt",
+    "r",
+    encoding="utf8"
+) as f:
+
+    text = f.read()
+
+# =====================================================
+# CLEAN
+# =====================================================
+
+cleaner = BNSTextCleaner()
+
+text = cleaner.clean(
+    text
+)
+
+# =====================================================
+# PARSE
+# =====================================================
+
+parser = BNSParser()
+
+document = parser.parse(
+    text
+)
+
+# =====================================================
+# CHUNK
+# =====================================================
+
+chunker = LegalChunker()
+
+chunks = chunks_to_dicts(
+    chunker.chunk_document(
+        document
+    )
+)
+
+# =====================================================
+# STATS
+# =====================================================
+
+print("\n========== CHUNK STATS ==========\n")
+
+print(
+    f"Total Chunks: {len(chunks)}"
+)
+
+levels = Counter(
+    chunk["level"]
+    for chunk in chunks
+)
+
+for level, count in sorted(
+    levels.items()
+):
+    print(
+        f"{level}: {count}"
+    )
+
+# =====================================================
+# INGEST
+# =====================================================
+
+pipeline = (
+    LegalIngestionPipeline()
+)
+
+# Optional:
+# wipe collection before indexing
+
+try:
+
+    pipeline.store.client.delete_collection(
+        collection_name="bns"
+    )
+
+    print(
+        "\nDeleted existing collection."
+    )
+
+except Exception:
+    pass
+
+pipeline.ingest(
+    chunks=chunks,
+    batch_size=64
+)
+
+print(
+    "\nBNS successfully indexed."
+)