combined results of all parsers and made a single pipeline

db/all_chunks.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+# =====================================================
+# BNS
+# =====================================================
+
+from db.parsers.bns.cleaner import (
+    BNSTextCleaner
+)
+
+from db.parsers.bns.bns_parser import (
+    BNSParser
+)
+
+from db.parsers.bns.chunker import (
+    LegalChunker as BNSChunker,
+    chunks_to_dicts as bns_to_dicts
+)
+
+# =====================================================
+# BNSS
+# =====================================================
+
+from db.parsers.bnss.cleaner import (
+    BNSSTextCleaner
+)
+
+from db.parsers.bnss.bnss_parser import (
+    BNSSParser
+)
+
+from db.parsers.bnss.chunker import (
+    LegalChunker as BNSSChunker,
+    chunks_to_dicts as bnss_to_dicts
+)
+
+# =====================================================
+# BSA
+# =====================================================
+
+from db.parsers.bsa.cleaner import (
+    BSATextCleaner
+)
+
+from db.parsers.bsa.bsa_parser import (
+    BSAParser
+)
+
+from db.parsers.bsa.chunker import (
+    LegalChunker as BSAChunker,
+    chunks_to_dicts as bsa_to_dicts
+)
+
+# =====================================================
+# CONSTITUTION
+# =====================================================
+
+from db.parsers.consitution.cleaner import (
+    ConstitutionTextCleaner
+)
+
+from db.parsers.consitution.constitution_parser import (
+    ConstitutionParser
+)
+
+from db.parsers.consitution.chunker import (
+    LegalChunker as ConstitutionChunker,
+    chunks_to_dicts as constitution_to_dicts
+)
+
+
+class LegalCorpusBuilder:
+
+    # =================================================
+    # LOAD FILE
+    # =================================================
+
+    @staticmethod
+    def load_text(
+        filepath
+    ):
+
+        with open(
+            filepath,
+            "r",
+            encoding="utf8"
+        ) as f:
+
+            return f.read()
+
+    # =================================================
+    # BNS
+    # =================================================
+
+    def build_bns_chunks(
+        self,
+        filepath
+    ):
+
+        text = self.load_text(
+            filepath
+        )
+
+        text = (
+            BNSTextCleaner()
+            .clean(text)
+        )
+
+        document = (
+            BNSParser()
+            .parse(text)
+        )
+
+        chunks = (
+            BNSChunker(
+                document_name="BNS"
+            )
+            .chunk_document(
+                document
+            )
+        )
+
+        return (
+            bns_to_dicts(
+                chunks
+            )
+        )
+
+    # =================================================
+    # BNSS
+    # =================================================
+
+    def build_bnss_chunks(
+        self,
+        filepath
+    ):
+
+        text = self.load_text(
+            filepath
+        )
+
+        text = (
+            BNSSTextCleaner()
+            .clean(text)
+        )
+
+        document = (
+            BNSSParser()
+            .parse(text)
+        )
+
+        chunks = (
+            BNSSChunker(
+                document_name="BNSS"
+            )
+            .chunk_document(
+                document
+            )
+        )
+
+        return (
+            bnss_to_dicts(
+                chunks
+            )
+        )
+
+    # =================================================
+    # BSA
+    # =================================================
+
+    def build_bsa_chunks(
+        self,
+        filepath
+    ):
+
+        text = self.load_text(
+            filepath
+        )
+
+        text = (
+            BSATextCleaner()
+            .clean(text)
+        )
+
+        document = (
+            BSAParser()
+            .parse(text)
+        )
+
+        chunks = (
+            BSAChunker(
+                document_name="BSA"
+            )
+            .chunk_document(
+                document
+            )
+        )
+
+        return (
+            bsa_to_dicts(
+                chunks
+            )
+        )
+
+    # =================================================
+    # CONSTITUTION
+    # =================================================
+
+    def build_constitution_chunks(
+        self,
+        filepath
+    ):
+
+        text = self.load_text(
+            filepath
+        )
+
+        text = (
+            ConstitutionTextCleaner()
+            .clean(text)
+        )
+
+        document = (
+            ConstitutionParser()
+            .parse(text)
+        )
+
+        chunker = ConstitutionChunker()
+
+        chunks = (
+            chunker.chunk_constitution(
+                document
+            )
+        )
+
+        return (
+            constitution_to_dicts(
+                chunks
+            )
+        )
+
+    # =================================================
+    # ALL CHUNKS
+    # =================================================
+
+    def build_all_chunks(
+        self,
+        bns_path,
+        bnss_path,
+        bsa_path,
+        constitution_path
+    ):
+
+        all_chunks = []
+
+        print(
+            "\nBuilding BNS..."
+        )
+
+        all_chunks.extend(
+            self.build_bns_chunks(
+                bns_path
+            )
+        )
+
+        print(
+            "Building BNSS..."
+        )
+
+        all_chunks.extend(
+            self.build_bnss_chunks(
+                bnss_path
+            )
+        )
+
+        print(
+            "Building BSA..."
+        )
+
+        all_chunks.extend(
+            self.build_bsa_chunks(
+                bsa_path
+            )
+        )
+
+        print(
+            "Building Constitution..."
+        )
+
+        all_chunks.extend(
+            self.build_constitution_chunks(
+                constitution_path
+            )
+        )
+
+        return all_chunks
+
+
+# =====================================================
+# TEST
+# =====================================================
+
+if __name__ == "__main__":
+
+    builder = (
+        LegalCorpusBuilder()
+    )
+
+    all_chunks = (
+        builder.build_all_chunks(
+            bns_path=
+                "db/pdfs/bns.txt",
+
+            bnss_path=
+                "db/pdfs/bnss.txt",
+
+            bsa_path=
+                "db/pdfs/bsa.txt",
+
+            constitution_path=
+                "db/pdfs/constitution.txt"
+        )
+    )
+
+    print()
+
+    print(
+        f"Total Chunks: "
+        f"{len(all_chunks)}"
+    ) 

db/embbeder/bns_embedder.py

@@ -1,230 +0,0 @@
-from __future__ import annotations
-
-from typing import List
-
-from tqdm import tqdm
-
-from sentence_transformers import (
-    SentenceTransformer
-)
-
-from qdrant_client import QdrantClient
-
-from qdrant_client.models import (
-    Distance,
-    VectorParams,
-    PointStruct
-)
-
-
-class QdrantEmbeddingGenerator:
-
-    def __init__(
-        self,
-        collection_name: str = "bns",
-        model_name: str = "BAAI/bge-large-en-v1.5",
-        host: str = "localhost",
-        port: int = 6333
-    ):
-
-        self.collection_name = (
-            collection_name
-        )
-
-        self.model = (
-            SentenceTransformer(
-                model_name
-            )
-        )
-
-        self.client = (
-            QdrantClient(
-                host=host,
-                port=port
-            )
-        )
-
-    # =====================================
-    # CREATE COLLECTION
-    # =====================================
-
-    def create_collection(self):
-
-        dimension = (
-            self.model
-            .get_sentence_embedding_dimension()
-        )
-
-        collections = (
-            self.client
-            .get_collections()
-        )
-
-        existing = {
-            c.name
-            for c in collections.collections
-        }
-
-        if (
-            self.collection_name
-            not in existing
-        ):
-
-            self.client.create_collection(
-                collection_name=
-                    self.collection_name,
-
-                vectors_config=
-                    VectorParams(
-                        size=dimension,
-                        distance=
-                            Distance.COSINE
-                    )
-            )
-
-            print(
-                f"Created collection "
-                f"{self.collection_name}"
-            )
-
-    # =====================================
-    # BUILD TEXT
-    # =====================================
-
-    def build_text(
-        self,
-        chunk: dict
-    ) -> str:
-
-        return chunk.get(
-            "enriched_text",
-            chunk["text"]
-        )
-
-    # =====================================
-    # INGEST
-    # =====================================
-
-    def ingest_chunks(
-        self,
-        chunks: List[dict],
-        batch_size: int = 64
-    ):
-
-        self.create_collection()
-
-        point_id = 1
-
-        for start in tqdm(
-            range(
-                0,
-                len(chunks),
-                batch_size
-            )
-        ):
-
-            batch = chunks[
-                start:
-                start + batch_size
-            ]
-
-            texts = [
-                self.build_text(
-                    chunk
-                )
-                for chunk in batch
-            ]
-
-            embeddings = (
-                self.model.encode(
-                    texts,
-                    normalize_embeddings=True,
-                    convert_to_numpy=True,
-                    show_progress_bar=False
-                )
-            )
-
-            points = []
-
-            for chunk, embedding in zip(
-                batch,
-                embeddings
-            ):
-
-                payload = {
-
-                    "chunk_id":
-                        chunk.get(
-                            "chunk_id"
-                        ),
-
-                    "level":
-                        chunk.get(
-                            "level"
-                        ),
-
-                    "chapter_no":
-                        chunk.get(
-                            "chapter_no"
-                        ),
-
-                    "section_no":
-                        chunk.get(
-                            "section_no"
-                        ),
-
-                    "clause_no":
-                        chunk.get(
-                            "clause_no"
-                        ),
-
-                    "sub_clause_no":
-                        chunk.get(
-                            "sub_clause_no"
-                        ),
-
-                    "roman_no":
-                        chunk.get(
-                            "roman_no"
-                        ),
-
-                    "title":
-                        chunk.get(
-                            "title"
-                        ),
-
-                    "text":
-                        chunk.get(
-                            "text"
-                        ),
-
-                    "parent_id":
-                        chunk.get(
-                            "parent_id"
-                        )
-                }
-
-                points.append(
-                    PointStruct(
-                        id=point_id,
-                        vector=
-                            embedding.tolist(),
-                        payload=
-                            payload
-                    )
-                )
-
-                point_id += 1
-
-            self.client.upsert(
-                collection_name=
-                    self.collection_name,
-
-                points=
-                    points
-            )
-
-        print(
-            f"Ingested "
-            f"{len(chunks)} chunks"
-        )

db/embbeder/constitution_embedding.py

@@ -1,244 +0,0 @@
-
-from __future__ import annotations
-
-import json
-import pickle
-from pathlib import Path
-
-import numpy as np
-from sentence_transformers import SentenceTransformer
-
-
-# =========================================================
-# EMBEDDING GENERATOR
-# =========================================================
-
-class EmbeddingGenerator:
-
-    def __init__(
-        self,
-        model_name: str = "BAAI/bge-large-en-v1.5"
-    ):
-
-        self.model = SentenceTransformer(
-            model_name
-        )
-
-    # =====================================================
-    # GENERATE
-    # =====================================================
-
-    def generate(
-        self,
-        chunks: list,
-        batch_size: int = 32
-    ):
-
-        texts = [
-            chunk.text
-            for chunk in chunks
-        ]
-
-        embeddings = self.model.encode(
-            texts,
-            batch_size=batch_size,
-            show_progress_bar=True,
-            convert_to_numpy=True,
-            normalize_embeddings=True
-        )
-
-        return embeddings
-
-    # =====================================================
-    # SAVE NUMPY
-    # =====================================================
-
-    def save_embeddings(
-        self,
-        embeddings,
-        output_path: str
-    ):
-
-        np.save(
-            output_path,
-            embeddings
-        )
-
-    # =====================================================
-    # SAVE COMPLETE DATASET
-    # =====================================================
-
-    def save_dataset(
-        self,
-        chunks,
-        embeddings,
-        output_file: str
-    ):
-
-        records = []
-
-        for chunk, embedding in zip(
-            chunks,
-            embeddings
-        ):
-
-            records.append(
-                {
-                    "chunk_id":
-                        chunk.chunk_id,
-
-                    "chunk_type":
-                        chunk.chunk_type,
-
-                    "text":
-                        chunk.text,
-
-                    "metadata":
-                        chunk.metadata,
-
-                    "references":
-                        chunk.references,
-
-                    "embedding":
-                        embedding.tolist()
-                }
-            )
-
-        with open(
-            output_file,
-            "w",
-            encoding="utf8"
-        ) as f:
-
-            json.dump(
-                records,
-                f,
-                ensure_ascii=False
-            )
-
-    # =====================================================
-    # PICKLE
-    # =====================================================
-
-    def save_pickle(
-        self,
-        chunks,
-        embeddings,
-        output_file
-    ):
-
-        data = []
-
-        for chunk, embedding in zip(
-            chunks,
-            embeddings
-        ):
-
-            data.append(
-                {
-                    "chunk_id":
-                        chunk.chunk_id,
-
-                    "chunk_type":
-                        chunk.chunk_type,
-
-                    "text":
-                        chunk.text,
-
-                    "metadata":
-                        chunk.metadata,
-
-                    "references":
-                        chunk.references,
-
-                    "embedding":
-                        embedding
-                }
-            )
-
-        with open(
-            output_file,
-            "wb"
-        ) as f:
-
-            pickle.dump(
-                data,
-                f
-            )
-
-
-# =========================================================
-# EXAMPLE
-# =========================================================
-
-if __name__ == "__main__":
-
-    from constitution_parser import ConstitutionParser
-    from legal_chunker import LegalChunker
-
-    with open(
-        "../../pdfs/constitution.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    # -----------------------------------------
-    # Parse
-    # -----------------------------------------
-
-    parser = ConstitutionParser()
-
-    constitution = parser.parse(
-        text
-    )
-
-    # -----------------------------------------
-    # Chunk
-    # -----------------------------------------
-
-    chunker = LegalChunker()
-
-    chunks = chunker.chunk_constitution(
-        constitution
-    )
-
-    print(
-        "Chunks:",
-        len(chunks)
-    )
-
-    # -----------------------------------------
-    # Embeddings
-    # -----------------------------------------
-
-    generator = EmbeddingGenerator()
-
-    embeddings = generator.generate(
-        chunks
-    )
-
-    print(
-        "Embeddings Shape:",
-        embeddings.shape
-    )
-
-    # -----------------------------------------
-    # Save
-    # -----------------------------------------
-
-    generator.save_embeddings(
-        embeddings,
-        "constitution_embeddings.npy"
-    )
-
-    generator.save_pickle(
-        chunks,
-        embeddings,
-        "constitution_dataset.pkl"
-    )
-
-    print(
-        "Saved successfully"
-    )

db/embedder.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from sentence_transformers import (
+    SentenceTransformer
+)
+
+
+class LegalEmbedder:
+
+    def __init__(
+        self,
+        model_name: str =
+        "BAAI/bge-large-en-v1.5"
+    ):
+
+        print(
+            f"Loading embedding model: "
+            f"{model_name}"
+        )
+
+        self.model = (
+            SentenceTransformer(
+                model_name
+            )
+        )
+
+    # =====================================================
+    # DOCUMENT EMBEDDINGS
+    # =====================================================
+
+    def embed(
+        self,
+        texts: list[str]
+    ):
+
+        return self.model.encode(
+            texts,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+            batch_size=16,
+            show_progress_bar=True
+        )
+
+    # =====================================================
+    # QUERY EMBEDDING
+    # =====================================================
+
+    def embed_query(
+        self,
+        query: str
+    ):
+
+        query = (
+            "Represent this sentence "
+            "for searching relevant "
+            f"passages: {query}"
+        )
+
+        return self.model.encode(
+            query,
+            normalize_embeddings=True,
+            convert_to_numpy=True
+        )
+
+    # =====================================================
+    # VECTOR SIZE
+    # =====================================================
+
+    def vector_size(
+        self
+    ) -> int:
+
+        return (
+            self.model
+            .get_sentence_embedding_dimension()
+        )
+
+
+# =========================================================
+# TEST
+# =========================================================
+
+if __name__ == "__main__":
+
+    embedder = (
+        LegalEmbedder()
+    )
+
+    texts = [
+        "Section 52. Facts of which Court shall take judicial notice.",
+        "Article 21. Protection of life and personal liberty."
+    ]
+
+    vectors = (
+        embedder.embed(
+            texts
+        )
+    )
+
+    print()
+
+    print(
+        "Vectors:",
+        len(vectors)
+    )
+
+    print(
+        "Dimension:",
+        len(vectors[0])
+    )
+
+    query_vector = (
+        embedder.embed_query(
+            "facts judicially noticed by court"
+        )
+    )
+
+    print(
+        "Query Dimension:",
+        len(query_vector)
+    )
+
+    print(
+        "Model Dimension:",
+        embedder.vector_size()
+    )

db/embeddings_generator.py

@@ -1,47 +0,0 @@
-from sentence_transformers import SentenceTransformer
-from typing import List, Dict
-
-
-class LegalEmbedder:
-    """
-    Converts legal records into embeddings.
-    """
-
-    def __init__(
-        self,
-        model_name: str = "BAAI/bge-large-en-v1.5"
-    ):
-        self.model = SentenceTransformer(
-            model_name
-        )
-
-    def embed_records(
-        self,
-        records: List[Dict]
-    ) -> List[Dict]:
-
-        if not records:
-            return []
-
-        texts = [
-            record["embedding_text"]
-            for record in records
-        ]
-
-        embeddings = self.model.encode(
-            texts,
-            batch_size=32,
-            show_progress_bar=True,
-            convert_to_numpy=True,
-            normalize_embeddings=True
-        )
-
-        for record, embedding in zip(
-            records,
-            embeddings
-        ):
-            record["embedding"] = (
-                embedding.tolist()
-            )
-
-        return records

db/ingest.py

@@ -1,172 +1,118 @@
-from collections import defaultdict
-import json
-import uuid
-import hashlib
+from __future__ import annotations
 
-from embeddings_generator import LegalEmbedder
-from graph_db import LegalGraphDB
-from vector_db import QdrantIngestor
-from parsers.constitution_parser import ConstitutionParser
-from parsers.bns_parser import BNSParser
-from parsers.bnss_parser import BNSSParser
-from parsers.bsa_parser import BSAParser
-from parsers.legal_normaliser import LegalNormalizer
-from generator.legal_record_generator import LegalRecordGenerator
-from generator.get_all_chunks import normalise_document
+from db.embedder import LegalEmbedder
+from db.vector_store import QdrantStore
 
 
-# --------------------------------------------------
-# STABLE ID GENERATOR (CORE FIX)
-# --------------------------------------------------
+class LegalIngestionPipeline:
 
-def stable_id(chunk: dict, document_uuid: str) -> str:
-    meta = chunk.get("meta", {})
+    def __init__(
+        self,
+        collection_name: str = "legal_rag"
+    ):
 
-    base = (
-        document_uuid + "|" +
-        chunk.get("type", "") + "|" +
-        str(meta.get("division_no", "")) + "|" +
-        str(meta.get("provision_no", "")) + "|" +
-        str(meta.get("clause_no", "")) + "|" +
-        str(meta.get("sub_clause_no", "")) + "|" +
-        str(meta.get("roman_no", "")) + "|" +
-        chunk.get("text", "")
-    )
+        self.embedder = LegalEmbedder()
 
-    return hashlib.sha256(base.encode("utf-8")).hexdigest()[:32]
+        self.store = QdrantStore(
+            collection_name=
+                collection_name
+        )
 
+    # =====================================================
+    # INGEST
+    # =====================================================
 
-# --------------------------------------------------
-# LOAD RAW TEXT
-# --------------------------------------------------
+    def ingest(
+        self,
+        chunks: list[dict],
+        recreate_collection: bool = False
+    ):
 
-with open("./pdfs/constitution.txt", "r", encoding="utf-8") as f:
-    constitution_text = f.read()
+        if not chunks:
 
-with open("./pdfs/bns.txt", "r", encoding="utf-8") as f:
-    bns_text = f.read()
+            print(
+                "No chunks found."
+            )
 
-with open("./pdfs/bnss.txt", "r", encoding="utf-8") as f:
-    bnss_text = f.read()
+            return
 
-with open("./pdfs/bsa.txt", "r", encoding="utf-8") as f:
-    bsa_text = f.read()
+        print(
+            f"\nChunks: {len(chunks)}"
+        )
 
+        texts = [
+            chunk[
+                "enriched_text"
+            ]
+            for chunk in chunks
+        ]
 
-# --------------------------------------------------
-# PARSERS
-# --------------------------------------------------
+        print(
+            "Generating embeddings..."
+        )
 
-constitution_parser = ConstitutionParser()
-bns_parser = BNSParser()
-bnss_parser = BNSSParser()
-bsa_parser = BSAParser()
+        embeddings = (
+            self.embedder.embed(
+                texts
+            )
+        )
 
-constitution_parsed = constitution_parser.parse(constitution_text)
-bns_parsed = bns_parser.parse(bns_text)
-bnss_parsed = bnss_parser.parse(bnss_text)
-bsa_parsed = bsa_parser.parse(bsa_text)
+        vector_size = len(
+            embeddings[0]
+        )
 
+        print(
+            f"Vector Size: "
+            f"{vector_size}"
+        )
 
-# --------------------------------------------------
-# NORMALISE
-# --------------------------------------------------
+        if recreate_collection:
 
-normaliser = LegalNormalizer()
+            self.store.recreate_collection(
+                vector_size
+            )
 
-constitution_normalised = normaliser.normalize(
-    constitution_parsed,
-    document_type="constitution"
-)
+        else:
 
-bns_normalised = normaliser.normalize(
-    bns_parsed.divisions,
-    document_type="bns"
-)
+            self.store.create_collection(
+                vector_size
+            )
 
-bnss_normalised = normaliser.normalize(
-    bnss_parsed,
-    document_type="bnss"
-)
+        points = []
 
+        for idx, (
+            chunk,
+            vector
+        ) in enumerate(
+            zip(
+                chunks,
+                embeddings
+            ),
+            start=1
+        ):
 
-# --------------------------------------------------
-# FLATTEN ALL CHUNKS
-# --------------------------------------------------
+            points.append(
+                {
+                    "id":
+                        idx,
 
-all_chunks = []
-for doc in [
-    constitution_normalised,
-    bns_normalised,
-    bnss_normalised,
-    bsa_parsed
-]:
-    all_chunks.extend(normalise_document(doc))
+                    "vector":
+                        vector.tolist(),
 
-
-# --------------------------------------------------
-# ⭐ DOCUMENT UUID (SAME FOR ENTIRE RUN)
-# --------------------------------------------------
-
-document_uuid = str(uuid.uuid4())
-
-
-# --------------------------------------------------
-# ��� ASSIGN STABLE IDS (IMPORTANT FIX)
-# --------------------------------------------------
-
-for chunk in all_chunks:
-    chunk["document_uuid"] = document_uuid
-    chunk["id"] = stable_id(chunk, document_uuid)
-
-
-# --------------------------------------------------
-# GRAPH DB (NEO4J)
-# --------------------------------------------------
-
-graph = LegalGraphDB(
-    uri="bolt://localhost:7687",
-    user="neo4j",
-    password="test12345"
-)
-
-graph.insert_chunks(all_chunks)
-print("✅ Data inserted into Neo4j")
-
-for chunk in all_chunks:
-    chunk["id"] = chunk.get("id") or stable_id(chunk)
-# --------------------------------------------------
-# RECORD GENERATION (FOR VECTOR DB)
-# --------------------------------------------------
-
-record_generator = LegalRecordGenerator()
-
-records = record_generator.generate(
-    normalized_chunks=all_chunks,
-    document_type="legal"
-)
-
-
-# --------------------------------------------------
-# EMBEDDINGS
-# --------------------------------------------------
-
-embedder = LegalEmbedder(model_name="BAAI/bge-large-en-v1.5")
-records = embedder.embed_records(records)
-
-
-# --------------------------------------------------
-# VECTOR DB (QDRANT)
-# --------------------------------------------------
-
-qdrant = QdrantIngestor()
-qdrant.upsert_chunks(records)
-
-print("✅ Data inserted into Qdrant")
-
-
-# --------------------------------------------------
-# CLEANUP
-# --------------------------------------------------
-
-graph.close()
+                    "payload":
+                        chunk
+                }
+            )
+
+        print(
+            f"Uploading "
+            f"{len(points)} points..."
+        )
+
+        self.store.upsert_points(
+            points
+        )
+
+        print(
+            "\nIngestion Complete."
+        )

db/parsers/base_parser.py

@@ -1,165 +0,0 @@
-from abc import ABC, abstractmethod
-import re
-
-
-class BaseLegalParser(ABC):
-
-    # ----------------------------------
-    # REFERENCES
-    # ----------------------------------
-
-    REFERENCE_RE = re.compile(
-        r"(?:article|section)[s]?\s+(\d+[A-Z]?)",
-        re.IGNORECASE
-    )
-
-    # ----------------------------------
-    # LEGAL HIERARCHY
-    # ----------------------------------
-
-    CLAUSE_RE = re.compile(
-        r"(?m)^\((\d+[A-Za-z]?)\)\s*(.*)"
-    )
-
-    SUBCLAUSE_RE = re.compile(
-        r"(?m)^\(([a-z])\)\s*(.*)"
-    )
-
-    ROMAN_RE = re.compile(
-        r"(?m)^\((i|ii|iii|iv|v|vi|vii|viii|ix|x)\)\s*(.*)"
-    )
-
-    # ----------------------------------
-    # CLEANER
-    # ----------------------------------
-
-    def clean(self, text: str):
-
-        text = text.replace("\r\n", "\n")
-
-        text = re.sub(
-            r"(?m)^\s*\d+\s*$",
-            "",
-            text
-        )
-
-        text = re.sub(
-            r"\n{3,}",
-            "\n\n",
-            text
-        )
-
-        return text.strip()
-
-    # ----------------------------------
-    # REFERENCES
-    # ----------------------------------
-
-    def extract_references(self, text):
-
-        return sorted(
-            set(
-                self.REFERENCE_RE.findall(text)
-            )
-        )
-
-    # ----------------------------------
-    # CLAUSES  (1) (2) (3)
-    # ----------------------------------
-
-    def extract_clauses(self, text):
-
-        matches = list(
-            self.CLAUSE_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, m in enumerate(matches):
-
-            start = m.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "clause_no": m.group(1),
-                    "text": text[start:end].strip()
-                }
-            )
-
-        return results
-
-    # ----------------------------------
-    # SUBCLAUSES (a)(b)(c)
-    # ----------------------------------
-
-    def extract_subclauses(self, text):
-
-        matches = list(
-            self.SUBCLAUSE_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, m in enumerate(matches):
-
-            start = m.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "sub_clause_no": m.group(1),
-                    "text": text[start:end].strip()
-                }
-            )
-
-        return results
-
-    # ----------------------------------
-    # ROMAN CLAUSES (i)(ii)(iii)
-    # ----------------------------------
-
-    def extract_roman_clauses(self, text):
-
-        matches = list(
-            self.ROMAN_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, m in enumerate(matches):
-
-            start = m.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "roman_no": m.group(1),
-                    "text": text[start:end].strip()
-                }
-            )
-
-        return results
-
-    # ----------------------------------
-    # ABSTRACT
-    # ----------------------------------
-
-    @abstractmethod
-    def parse(self, text):
-        pass

db/parsers/bns/bns_parser.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
-from chapter_parser import ChapterParser
-from section_parser import SectionParser
-from clause_parser import ClauseParser
-from explanation_parser import ExplanationParser
-from reference_extractor import ReferenceExtractor
-from cleaner import BNSTextCleaner
+from db.parsers.bns.chapter_parser import ChapterParser
+from db.parsers.bns.section_parser import SectionParser
+from db.parsers.bns.clause_parser import ClauseParser
+from db.parsers.bns.explanation_parser import ExplanationParser
+from db.parsers.bns.reference_extractor import ReferenceExtractor
+from db.parsers.bns.cleaner import BNSTextCleaner
 import re
-from legal_models import (
+from db.parsers.bns.legal_models import (
     BNSDocument,
     Chapter,
     Section

db/parsers/bns/chunk_temp.py

@@ -1,430 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass, asdict
-from typing import List, Optional
-
-
-# =========================================================
-# CHUNK MODEL
-# =========================================================
-
-@dataclass
-class LegalChunk:
-
-    chunk_id: str
-
-    level: str
-
-    document: str
-
-    chapter_no: Optional[str]
-
-    section_no: Optional[str]
-
-    clause_no: Optional[str]
-
-    sub_clause_no: Optional[str]
-
-    roman_no: Optional[str]
-
-    title: str
-
-    text: str
-
-    enriched_text: str
-
-    parent_id: Optional[str]
-
-
-# =========================================================
-# CHUNKER
-# =========================================================
-
-class LegalChunker:
-
-    def __init__(
-        self,
-        document_name: str = "BNS"
-    ):
-        self.document_name = document_name
-
-    # -----------------------------------------------------
-    # Helpers
-    # -----------------------------------------------------
-
-    def build_context(
-        self,
-        title: str,
-        section_no: str = "",
-        clause_no: str = "",
-        sub_clause_no: str = "",
-        roman_no: str = ""
-    ) -> str:
-
-        parts = [
-            self.document_name
-        ]
-
-        if section_no:
-            parts.append(
-                f"Section {section_no}"
-            )
-
-        if title:
-            parts.append(
-                title
-            )
-
-        if clause_no:
-            parts.append(
-                f"Clause ({clause_no})"
-            )
-
-        if sub_clause_no:
-            parts.append(
-                f"SubClause ({sub_clause_no})"
-            )
-
-        if roman_no:
-            parts.append(
-                f"Roman Clause ({roman_no})"
-            )
-
-        return "\n".join(parts)
-
-    # -----------------------------------------------------
-    # Main
-    # -----------------------------------------------------
-
-    def chunk_document(
-        self,
-        document
-    ) -> List[LegalChunk]:
-
-        chunks = []
-
-        for chapter in document.chapters:
-
-            for section in chapter.sections:
-
-                section_id = (
-                    f"BNS-{section.section_no}"
-                )
-
-                title = (
-                    getattr(
-                        section,
-                        "title",
-                        ""
-                    )
-                )
-
-                # =================================
-                # SECTION CHUNK
-                # =================================
-
-                section_chunk = LegalChunk(
-                    chunk_id=section_id,
-
-                    level="section",
-
-                    document="bns",
-
-                    chapter_no=
-                        chapter.chapter_no,
-
-                    section_no=
-                        section.section_no,
-
-                    clause_no=None,
-
-                    sub_clause_no=None,
-
-                    roman_no=None,
-
-                    title=title,
-
-                    text=section.text,
-
-                    enriched_text=
-                        self.build_context(
-                            title=title,
-                            section_no=
-                                section.section_no
-                        )
-                        + "\n\n"
-                        + section.text,
-
-                    parent_id=None
-                )
-
-                chunks.append(
-                    section_chunk
-                )
-
-                # =================================
-                # CLAUSES
-                # =================================
-
-                for clause in section.clauses:
-
-                    clause_id = (
-                        f"{section_id}"
-                        f"({clause.clause_no})"
-                    )
-
-                    clause_chunk = LegalChunk(
-
-                        chunk_id=
-                            clause_id,
-
-                        level=
-                            "clause",
-
-                        document=
-                            "bns",
-
-                        chapter_no=
-                            chapter.chapter_no,
-
-                        section_no=
-                            section.section_no,
-
-                        clause_no=
-                            clause.clause_no,
-
-                        sub_clause_no=
-                            None,
-
-                        roman_no=
-                            None,
-
-                        title=
-                            title,
-
-                        text=
-                            clause.text,
-
-                        enriched_text=
-                            self.build_context(
-                                title=title,
-                                section_no=
-                                    section.section_no,
-                                clause_no=
-                                    clause.clause_no
-                            )
-                            + "\n\n"
-                            + clause.text,
-
-                        parent_id=
-                            section_id
-                    )
-
-                    chunks.append(
-                        clause_chunk
-                    )
-
-                    # =============================
-                    # SUB CLAUSES
-                    # =============================
-
-                    for sub in clause.sub_clauses:
-
-                        sub_id = (
-                            f"{clause_id}"
-                            f"({sub.sub_clause_no})"
-                        )
-
-                        sub_chunk = LegalChunk(
-
-                            chunk_id=
-                                sub_id,
-
-                            level=
-                                "subclause",
-
-                            document=
-                                "bns",
-
-                            chapter_no=
-                                chapter.chapter_no,
-
-                            section_no=
-                                section.section_no,
-
-                            clause_no=
-                                clause.clause_no,
-
-                            sub_clause_no=
-                                sub.sub_clause_no,
-
-                            roman_no=
-                                None,
-
-                            title=
-                                title,
-
-                            text=
-                                sub.text,
-
-                            enriched_text=
-                                self.build_context(
-                                    title=title,
-                                    section_no=
-                                        section.section_no,
-                                    clause_no=
-                                        clause.clause_no,
-                                    sub_clause_no=
-                                        sub.sub_clause_no
-                                )
-                                + "\n\n"
-                                + sub.text,
-
-                            parent_id=
-                                clause_id
-                        )
-
-                        chunks.append(
-                            sub_chunk
-                        )
-
-                        # =========================
-                        # ROMAN CLAUSES
-                        # =========================
-
-                        for roman in sub.roman_clauses:
-
-                            roman_id = (
-                                f"{sub_id}"
-                                f"({roman.roman_no})"
-                            )
-
-                            roman_chunk = LegalChunk(
-
-                                chunk_id=
-                                    roman_id,
-
-                                level=
-                                    "roman",
-
-                                document=
-                                    "bns",
-
-                                chapter_no=
-                                    chapter.chapter_no,
-
-                                section_no=
-                                    section.section_no,
-
-                                clause_no=
-                                    clause.clause_no,
-
-                                sub_clause_no=
-                                    sub.sub_clause_no,
-
-                                roman_no=
-                                    roman.roman_no,
-
-                                title=
-                                    title,
-
-                                text=
-                                    roman.text,
-
-                                enriched_text=
-                                    self.build_context(
-                                        title=title,
-                                        section_no=
-                                            section.section_no,
-                                        clause_no=
-                                            clause.clause_no,
-                                        sub_clause_no=
-                                            sub.sub_clause_no,
-                                        roman_no=
-                                            roman.roman_no
-                                    )
-                                    + "\n\n"
-                                    + roman.text,
-
-                                parent_id=
-                                    sub_id
-                            )
-
-                            chunks.append(
-                                roman_chunk
-                            )
-
-                # =================================
-                # EXPLANATIONS
-                # =================================
-
-                for explanation in (
-                    section.explanations
-                ):
-
-                    explanation_id = (
-                        f"{section_id}"
-                        f"-EXP-"
-                        f"{explanation.explanation_no}"
-                    )
-
-                    chunk = LegalChunk(
-
-                        chunk_id=
-                            explanation_id,
-
-                        level=
-                            "explanation",
-
-                        document=
-                            "bns",
-
-                        chapter_no=
-                            chapter.chapter_no,
-
-                        section_no=
-                            section.section_no,
-
-                        clause_no=None,
-
-                        sub_clause_no=None,
-
-                        roman_no=None,
-
-                        title=title,
-
-                        text=
-                            explanation.text,
-
-                        enriched_text=
-                            self.build_context(
-                                title=title,
-                                section_no=
-                                    section.section_no
-                            )
-                            + "\n\n"
-                            + explanation.text,
-
-                        parent_id=
-                            section_id
-                    )
-
-                    chunks.append(
-                        chunk
-                    )
-
-        return chunks
-
-
-# =========================================================
-# EXPORT
-# =========================================================
-
-def chunks_to_dicts(
-    chunks: List[LegalChunk]
-):
-
-    return [
-        asdict(chunk)
-        for chunk in chunks
-    ]

db/parsers/bns/clause_parser.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 import re
 from typing import List
 
-from cleaner import BNSTextCleaner
-from legal_models import (
+from db.parsers.bns.cleaner import BNSTextCleaner
+from db.parsers.bns.legal_models import (
     Clause,
     SubClause,
     RomanClause

db/parsers/bns/explanation_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import re
 from typing import List, Optional
 
-from legal_models import Explanation
+from db.parsers.bns.legal_models import Explanation
 
 
 class ExplanationParser:

db/parsers/bns/ingest.py

@@ -4,11 +4,11 @@ from qdrant_client.models import (
     PointStruct
 )
 
-from embedder_temp import (
+from db.parsers.bns.embedder_temp import (
     LegalEmbedder
 )
 
-from qdrant_store import (
+from db.parsers.bns.qdrant_store import (
     QdrantStore
 )
 

db/parsers/bns/section_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 import re
 from typing import List, Dict
-from cleaner import BNSTextCleaner
+from db.parsers.bns.cleaner import BNSTextCleaner
 
 
 class SectionParser:

db/parsers/bns/temp.py

@@ -1,599 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import List
-
-from cleaner import BNSTextCleaner
-from legal_models import (
-    Clause,
-    SubClause,
-    RomanClause
-)
-
-
-class ClauseParser:
-
-    """
-    BNS Clause Parser
-
-    Supports:
-
-        (1)
-        (2)
-        (2A)
-
-    Sub Clauses:
-
-        (a)
-        (b)
-
-    Roman Clauses:
-
-        (i)
-        (ii)
-        (iii)
-
-    """
-
-    # =====================================================
-    # REGEX
-    # =====================================================
-    NUMERIC_RE = re.compile(
-        r'(?m)^\((\d+[A-Z]?)\)'
-    )
-
-    ALPHA_RE = re.compile(
-        r'(?m)^\(([a-z])\)'
-    )
-
-    CAPITAL_RE = re.compile(
-        r'(?m)^\(([A-Z])\)'
-    )
-
-    ROMAN_RE = re.compile(
-        r'(?:(?<=\n)|(?<=;)|(?<=:)|(?<=—)|(?<=\.)|^)\s*\((i|ii|iii|iv|v|vi|vii|viii|ix|x|xi|xii|xiii|xiv|xv|xvi|xvii|xviii|xix|xx)\)',
-        re.I
-    )
-    CLAUSE_RE = re.compile(
-        r'(?m)^\s*\((\d+[A-Z]?)\)'
-    )
-
-    SUBCLAUSE_RE = re.compile(
-        r'(?m)^\s*\(([a-z])\)'
-    )
-
-    # =====================================================
-    # SPLIT SECTIONS
-    # =====================================================
-
-    def split_sections(
-        self,
-        text: str,
-        pattern: re.Pattern
-    ):
-
-        matches = list(
-            pattern.finditer(text)
-        )
-
-        if not matches:
-            return []
-
-        sections = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            sections.append(
-                (
-                    match.group(1),
-                    text[start:end].strip()
-                )
-            )
-
-        return sections
-
-    # =====================================================
-    # ROMAN CLAUSES
-    # =====================================================
-
-    def parse_roman_clauses(
-        self,
-        text: str
-    ) -> List[RomanClause]:
-
-        romans = []
-
-        roman_sections = (
-            self.split_sections(
-                text,
-                self.ROMAN_RE
-            )
-        )
-
-        for roman_no, roman_text in roman_sections:
-
-            romans.append(
-                RomanClause(
-                    document="bns",
-                    roman_no=roman_no,
-                    text=roman_text
-                )
-            )
-
-        return romans
-
-    # =====================================================
-    # SUB CLAUSES
-    # =====================================================
-
-    ROMAN_VALUES = {
-    "i", "ii", "iii", "iv", "v",
-    "vi", "vii", "viii", "ix", "x",
-    "xi", "xii", "xiii", "xiv", "xv",
-    "xvi", "xvii", "xviii", "xix", "xx"
-    }
-    @staticmethod
-    def remove_illustrations(
-        self,
-        text: str
-    ):
-
-        return re.split(
-            r'Illustrations?\.',
-            text,
-            maxsplit=1,
-            flags=re.I
-        )[0]
-
-    def parse_subclauses(
-        self,
-        text: str
-    ) -> List[SubClause]:
-        text=self.remove_illustrations(text)
-        subclauses = []
-
-        sub_sections = (
-            self.split_sections(
-                text,
-                self.SUBCLAUSE_RE
-            )
-        )
-
-        for sub_no, sub_text in sub_sections:
-
-            # Skip roman numerals
-            if (
-                sub_no.lower()
-                in self.ROMAN_VALUES
-            ):
-                continue
-
-            sub = SubClause(
-                document="bns",
-                sub_clause_no=sub_no,
-                text=sub_text
-            )
-
-            sub.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    sub_text
-                )
-            )
-
-            subclauses.append(
-                sub
-            )
-
-        return subclauses
-    
-
-    def build_alpha_structure(
-        self,
-        text: str,
-        pattern: re.Pattern
-    ) -> List[Clause]:
-
-        clause = Clause(
-            document="bns",
-            clause_no="0",
-            text=text
-        )
-
-        sections = self.split_sections(
-            text,
-            pattern
-        )
-
-        for sub_no, sub_text in sections:
-
-            sub = SubClause(
-                document="bns",
-                sub_clause_no=sub_no,
-                text=sub_text
-            )
-
-            sub.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    sub_text
-                )
-            )
-
-            clause.sub_clauses.append(
-                sub
-            )
-
-        return [clause]
-    
-    def build_numeric_structure(
-        self,
-        text: str
-    ) -> List[Clause]:
-
-        clauses = []
-
-        clause_sections = self.split_sections(
-            text,
-            self.NUMERIC_RE
-        )
-
-        for clause_no, clause_text in clause_sections:
-
-            clause = Clause(
-                document="bns",
-                clause_no=clause_no,
-                text=clause_text
-            )
-
-            alpha_sections = self.split_sections(
-                clause_text,
-                self.ALPHA_RE
-            )
-
-            if alpha_sections:
-
-                for sub_no, sub_text in alpha_sections:
-
-                    sub = SubClause(
-                        document="bns",
-                        sub_clause_no=sub_no,
-                        text=sub_text
-                    )
-
-                    sub.roman_clauses.extend(
-                        self.parse_roman_clauses(
-                            sub_text
-                        )
-                    )
-
-                    clause.sub_clauses.append(
-                        sub
-                    )
-
-            clauses.append(
-                clause
-            )
-
-        return clauses
-    def parse_alpha_clauses(
-        self,
-        text: str
-    ):
-
-        return self.build_alpha_structure(
-            text,
-            self.ALPHA_RE
-        )
-    
-
-    def parse_alpha_roman(
-        self,
-        text: str
-    ):
-
-        return self.build_alpha_structure(
-            text,
-            self.ALPHA_RE
-        )
-    
-
-    def parse_capital_roman(
-        self,
-        text: str
-    ):
-
-        return self.build_alpha_structure(
-            text,
-            self.CAPITAL_RE
-        )
-
-
-    def parse_numeric_clauses(
-        self,
-        text: str
-    ):
-
-        clauses = []
-
-        sections = self.split_sections(
-            text,
-            self.NUMERIC_RE
-        )
-
-        for clause_no, clause_text in sections:
-
-            clauses.append(
-                Clause(
-                    document="bns",
-                    clause_no=clause_no,
-                    text=clause_text
-                )
-            )
-
-        return clauses
-
-    def parse_numeric_alpha(
-        self,
-        text: str
-    ):
-
-        return self.build_numeric_structure(
-            text
-        )
-    
-
-    def parse_numeric_alpha_roman(
-        self,
-        text: str
-    ):
-
-        return self.build_numeric_structure(
-            text
-        )
-
-
-
-
-    # =====================================================
-    # CLAUSES
-    # =====================================================
-
-    def parse_clauses(
-        self,
-        section_text: str
-    ) -> List[Clause]:
-
-        clauses = []
-
-        clause_sections = self.split_sections(
-            section_text,
-            self.CLAUSE_RE
-        )
-
-        # No numbered clauses present
-        if not clause_sections:
-
-            subclauses = self.parse_subclauses(
-                section_text
-            )
-
-            if subclauses:
-
-                synthetic_clause = Clause(
-                    document="bns",
-                    clause_no="0",
-                    text=section_text
-                )
-
-                synthetic_clause.sub_clauses.extend(
-                    subclauses
-                )
-
-                clauses.append(
-                    synthetic_clause
-                )
-
-            return clauses
-
-        for clause_no, clause_text in clause_sections:
-
-            clause = Clause(
-                document="bns",
-                clause_no=clause_no,
-                text=clause_text
-            )
-
-            clause.sub_clauses.extend(
-                self.parse_subclauses(
-                    clause_text
-                )
-            )
-            clause.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    clause_text
-                )
-            )
-
-            clauses.append(
-                clause
-            )
-
-        return clauses
-
-
-
-
-    def parse_section_structure(
-        self,
-        section_text: str
-    ) -> List[Clause]:
-
-        has_numeric = bool(
-            self.NUMERIC_RE.search(
-                section_text
-            )
-        )
-
-        has_alpha = bool(
-            self.ALPHA_RE.search(
-                section_text
-            )
-        )
-
-        has_capital = bool(
-            self.CAPITAL_RE.search(
-                section_text
-            )
-        )
-
-        has_roman = bool(
-            self.ROMAN_RE.search(
-                section_text
-            )
-        )
-
-        if has_numeric:
-            return self.build_numeric_structure(
-                section_text
-            )
-
-        if has_capital:
-            return self.build_alpha_structure(
-                section_text,
-                self.CAPITAL_RE
-            )
-
-        if has_alpha:
-            return self.build_alpha_structure(
-                section_text,
-                self.ALPHA_RE
-            )
-
-        return []
-
-
-
-
-
-
-
-    # =====================================================
-    # VALIDATION
-    # =====================================================
-
-    def validate_clauses(
-        self,
-        section_no,
-        clauses: List[Clause]
-    ) -> List[str]:
-
-        errors = []
-
-        seen = set()
-
-        for clause in clauses:
-
-            if clause.clause_no in seen:
-
-                print("\n" + "="*80)
-                print("section: ",section_no)
-                print("DUPLICATE SUBCLAUSE")
-                print("Clause:", clause.clause_no)
-                print("SubClause:", sub.sub_clause_no)
-                print(sub.text[:1000])
-
-                errors.append(
-                    f"Duplicate SubClause "
-                    f"{sub.sub_clause_no}"
-                )
-
-            seen.add(
-                clause.clause_no
-            )
-
-            sub_seen = set()
-
-            for sub in clause.sub_clauses:
-
-                if sub.sub_clause_no in sub_seen:
-
-                    errors.append(
-                        f"Duplicate SubClause "
-                        f"{sub.sub_clause_no}"
-                    )
-
-                sub_seen.add(
-                    sub.sub_clause_no
-                )
-
-                roman_seen = set()
-
-                for roman in sub.roman_clauses:
-
-                    if roman.roman_no in roman_seen:
-
-                        errors.append(
-                            f"Duplicate Roman "
-                            f"{roman.roman_no}"
-                        )
-
-                    roman_seen.add(
-                        roman.roman_no
-                    )
-
-        return errors
-
-
-# =========================================================
-# TEST
-# =========================================================
-
-if __name__ == "__main__":
-
-    with open(
-        "../../pdfs/bns.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    cleaner=BNSTextCleaner()
-    text=cleaner.clean(text)
-
- 
-
-    parser = ClauseParser()
-
-    print(len(parser.CLAUSE_RE.findall(text)))
-    print(len(parser.SUBCLAUSE_RE.findall(text)))
-    print(len(parser.ROMAN_RE.findall(text)))
-
-    clauses = parser.parse_section_structure(
-        text
-    )
-
-    print(
-        "Clauses:",
-        len(clauses)
-    )
-
-    # for clause in clauses:
-
-    #     print(
-    #         clause.clause_no,
-    #         len(clause.sub_clauses)
-    #     )
-
-    # print(
-    #     parser.validate_clauses(
-    #         clauses
-    #     )
-    # )

db/parsers/bns/usage.py

@@ -1,9 +1,9 @@
 from collections import Counter
 
-from cleaner import BNSTextCleaner
-from bns_parser import BNSParser
+from db.parsers.bns.cleaner import BNSTextCleaner
+from db.parsers.bns.bns_parser import BNSParser
 
-from chunk_temp import (
+from db.parsers.bns.chunker import (
     LegalChunker,
     chunks_to_dicts
 )

db/parsers/bns_parser.py

@@ -1,264 +0,0 @@
-# parsers/bns_parser.py
-
-from __future__ import annotations
-
-import re
-
-from parsers.base_parser import BaseLegalParser
-
-from models.legal_models import (
-    Chapter,
-    Section,
-    Clause,
-    SubClause,
-    RomanClause,
-    LegalDocument
-)
-
-
-class BNSParser(BaseLegalParser):
-
-    CHAPTER_RE = re.compile(
-        r"CHAPTER\s+([IVXLC]+)\s*\n+([^\n]+)",
-        re.IGNORECASE
-    )
-
-    SECTION_RE = re.compile(
-        r"(?m)^(\d+[A-Z]?)\.\s+(.+)$"
-    )
-
-    # -------------------------------------------------
-    # CHAPTERS
-    # -------------------------------------------------
-
-    def extract_chapters(self, text):
-
-        matches = list(
-            self.CHAPTER_RE.finditer(text)
-        )
-
-        chapters = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            chapters.append(
-                {
-                    "chapter_no":
-                        match.group(1).strip(),
-
-                    "chapter_title":
-                        match.group(2).strip(),
-
-                    "text":
-                        text[start:end]
-                }
-            )
-
-        return chapters
-
-    # -------------------------------------------------
-    # SECTIONS
-    # -------------------------------------------------
-
-    def extract_sections(self, chapter_text):
-
-        matches = list(
-            self.SECTION_RE.finditer(
-                chapter_text
-            )
-        )
-
-        sections = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(chapter_text)
-            )
-
-            sections.append(
-                {
-                    "section_no":
-                        match.group(1),
-
-                    "section_title":
-                        match.group(2).strip(),
-
-                    "text":
-                        chapter_text[
-                            start:end
-                        ].strip()
-                }
-            )
-
-        return sections
-
-    # -------------------------------------------------
-    # MAIN PARSER
-    # -------------------------------------------------
-
-    def parse(self, raw_text):
-
-        raw_text = self.clean(raw_text)
-
-        parsed_chapters = []
-
-        chapter_blocks = (
-            self.extract_chapters(
-                raw_text
-            )
-        )
-
-        for chapter_data in chapter_blocks:
-
-            chapter = Chapter(
-                document="bns",
-                chapter_no=
-                    chapter_data[
-                        "chapter_no"
-                    ],
-
-                chapter_title=
-                    chapter_data[
-                        "chapter_title"
-                    ]
-            )
-
-            section_blocks = (
-                self.extract_sections(
-                    chapter_data["text"]
-                )
-            )
-
-            for section_data in section_blocks:
-
-                section = Section(
-                    document="bns",
-
-                    section_no=
-                        section_data[
-                            "section_no"
-                        ],
-
-                    section_title=
-                        section_data[
-                            "section_title"
-                        ],
-
-                    text=
-                        section_data[
-                            "text"
-                        ]
-                )
-
-                # -------------------------
-                # SUBSECTIONS (1),(2),(3)
-                # -------------------------
-
-                clause_blocks = (
-                    self.extract_clauses(
-                        section.text
-                    )
-                )
-
-                for clause_data in clause_blocks:
-
-                    clause = Clause(
-                        document="bns",
-                        clause_no=
-                            clause_data[
-                                "clause_no"
-                            ],
-
-                        text=
-                            clause_data[
-                                "text"
-                            ]
-                    )
-
-                    # ---------------------
-                    # (a)(b)(c)
-                    # ---------------------
-
-                    sub_blocks = (
-                        self.extract_subclauses(
-                            clause.text
-                        )
-                    )
-
-                    for sub_data in sub_blocks:
-
-                        sub_clause = (
-                            SubClause(
-                                sub_clause_no=
-                                    sub_data[
-                                        "sub_clause_no"
-                                    ],
-
-                                text=
-                                    sub_data[
-                                        "text"
-                                    ]
-                            )
-                        )
-
-                        # -----------------
-                        # (i)(ii)(iii)
-                        # -----------------
-
-                        roman_blocks = (
-                            self.extract_roman_clauses(
-                                sub_clause.text
-                            )
-                        )
-
-                        for roman_data in roman_blocks:
-
-                            sub_clause.roman_clauses.append(
-                                RomanClause(
-                                    document="bns",
-                                    roman_no=
-                                        roman_data[
-                                            "roman_no"
-                                        ],
-
-                                    text=
-                                        roman_data[
-                                            "text"
-                                        ]
-                                )
-                            )
-
-                        clause.sub_clauses.append(
-                            sub_clause
-                        )
-
-                    section.clauses.append(
-                        clause
-                    )
-
-                chapter.sections.append(
-                    section
-                )
-
-            parsed_chapters.append(
-                chapter
-            )
-
-        return LegalDocument(
-            document="bns",
-            document_type="bns",
-            divisions=parsed_chapters
-        )

db/parsers/bnss/bnss_parser.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
-from chapter_parser import ChapterParser
-from section_parser import SectionParser
-from clause_parser import ClauseParser
-from explanation_parser import ExplanationParser
-from reference_extractor import ReferenceExtractor
-from cleaner import BNSSTextCleaner
+from db.parsers.bnss.chapter_parser import ChapterParser
+from db.parsers.bnss.section_parser import SectionParser
+from db.parsers.bnss.clause_parser import ClauseParser
+from db.parsers.bnss.explanation_parser import ExplanationParser
+from db.parsers.bnss.reference_extractor import ReferenceExtractor
+from db.parsers.bnss.cleaner import BNSSTextCleaner
 import re
-from legal_models import (
+from db.parsers.bnss.legal_models import (
     BNSSDocument,
     Chapter,
     Section

db/parsers/bnss/clause_parser.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 import re
 from typing import List
 
-from cleaner import BNSSTextCleaner
-from legal_models import (
+from db.parsers.bnss.cleaner import BNSSTextCleaner
+from db.parsers.bnss.legal_models import (
     Clause,
     SubClause,
     RomanClause

db/parsers/bnss/cleaner.py

@@ -132,7 +132,6 @@ class BNSSTextCleaner:
 
         text = re.sub(
             r"""
-            (?mx)
             ^
             (?:
                 [A-Z][A-Za-z' -]{0,40}
@@ -142,9 +141,10 @@ class BNSSTextCleaner:
             $
             """,
             "",
-            text
-        )   
-        
+            text,
+            flags=re.MULTILINE | re.VERBOSE
+        )  
+
         # =====================================
         # Collapse empty lines
         # =====================================

db/parsers/bnss/explanation_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import re
 from typing import List, Optional
 
-from legal_models import Explanation
+from db.parsers.bnss.legal_models import Explanation
 
 
 class ExplanationParser:

db/parsers/bnss/ingest.py

@@ -6,11 +6,11 @@ from qdrant_client.models import (
     PointStruct
 )
 
-from embedder_temp import (
+from db.parsers.bnss.embedder_temp import (
     LegalEmbedder
 )
 
-from qdrant_store import (
+from db.parsers.bnss.qdrant_store import (
     QdrantStore
 )
 

db/parsers/bnss/search.py

@@ -1,5 +1,5 @@
-from embedder_temp import LegalEmbedder
-from qdrant_store import QdrantStore
+from db.parsers.bnss.embedder_temp import LegalEmbedder
+from db.parsers.bnss.qdrant_store import QdrantStore
 
 
 class BNSSRetriever:

db/parsers/bnss/section_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 import re
 from typing import List, Dict
-from cleaner import BNSSTextCleaner
+from db.parsers.bnss.cleaner import BNSSTextCleaner
 
 
 class SectionParser:

db/parsers/bnss/temp.py

@@ -1,599 +0,0 @@
-from __future__ import annotations
-
-import re
-from typing import List
-
-from cleaner import BNSSTextCleaner
-from legal_models import (
-    Clause,
-    SubClause,
-    RomanClause
-)
-
-
-class ClauseParser:
-
-    """
-    BNS Clause Parser
-
-    Supports:
-
-        (1)
-        (2)
-        (2A)
-
-    Sub Clauses:
-
-        (a)
-        (b)
-
-    Roman Clauses:
-
-        (i)
-        (ii)
-        (iii)
-
-    """
-
-    # =====================================================
-    # REGEX
-    # =====================================================
-    NUMERIC_RE = re.compile(
-        r'(?m)^\((\d+[A-Z]?)\)'
-    )
-
-    ALPHA_RE = re.compile(
-        r'(?m)^\(([a-z])\)'
-    )
-
-    CAPITAL_RE = re.compile(
-        r'(?m)^\(([A-Z])\)'
-    )
-
-    ROMAN_RE = re.compile(
-        r'(?:(?<=\n)|(?<=;)|(?<=:)|(?<=—)|(?<=\.)|^)\s*\((i|ii|iii|iv|v|vi|vii|viii|ix|x|xi|xii|xiii|xiv|xv|xvi|xvii|xviii|xix|xx)\)',
-        re.I
-    )
-    CLAUSE_RE = re.compile(
-        r'(?m)^\s*\((\d+[A-Z]?)\)'
-    )
-
-    SUBCLAUSE_RE = re.compile(
-        r'(?m)^\s*\(([a-z])\)'
-    )
-
-    # =====================================================
-    # SPLIT SECTIONS
-    # =====================================================
-
-    def split_sections(
-        self,
-        text: str,
-        pattern: re.Pattern
-    ):
-
-        matches = list(
-            pattern.finditer(text)
-        )
-
-        if not matches:
-            return []
-
-        sections = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            sections.append(
-                (
-                    match.group(1),
-                    text[start:end].strip()
-                )
-            )
-
-        return sections
-
-    # =====================================================
-    # ROMAN CLAUSES
-    # =====================================================
-
-    def parse_roman_clauses(
-        self,
-        text: str
-    ) -> List[RomanClause]:
-
-        romans = []
-
-        roman_sections = (
-            self.split_sections(
-                text,
-                self.ROMAN_RE
-            )
-        )
-
-        for roman_no, roman_text in roman_sections:
-
-            romans.append(
-                RomanClause(
-                    document="bns",
-                    roman_no=roman_no,
-                    text=roman_text
-                )
-            )
-
-        return romans
-
-    # =====================================================
-    # SUB CLAUSES
-    # =====================================================
-
-    ROMAN_VALUES = {
-    "i", "ii", "iii", "iv", "v",
-    "vi", "vii", "viii", "ix", "x",
-    "xi", "xii", "xiii", "xiv", "xv",
-    "xvi", "xvii", "xviii", "xix", "xx"
-    }
-    @staticmethod
-    def remove_illustrations(
-        self,
-        text: str
-    ):
-
-        return re.split(
-            r'Illustrations?\.',
-            text,
-            maxsplit=1,
-            flags=re.I
-        )[0]
-
-    def parse_subclauses(
-        self,
-        text: str
-    ) -> List[SubClause]:
-        text=self.remove_illustrations(text)
-        subclauses = []
-
-        sub_sections = (
-            self.split_sections(
-                text,
-                self.SUBCLAUSE_RE
-            )
-        )
-
-        for sub_no, sub_text in sub_sections:
-
-            # Skip roman numerals
-            if (
-                sub_no.lower()
-                in self.ROMAN_VALUES
-            ):
-                continue
-
-            sub = SubClause(
-                document="bns",
-                sub_clause_no=sub_no,
-                text=sub_text
-            )
-
-            sub.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    sub_text
-                )
-            )
-
-            subclauses.append(
-                sub
-            )
-
-        return subclauses
-    
-
-    def build_alpha_structure(
-        self,
-        text: str,
-        pattern: re.Pattern
-    ) -> List[Clause]:
-
-        clause = Clause(
-            document="bns",
-            clause_no="0",
-            text=text
-        )
-
-        sections = self.split_sections(
-            text,
-            pattern
-        )
-
-        for sub_no, sub_text in sections:
-
-            sub = SubClause(
-                document="bns",
-                sub_clause_no=sub_no,
-                text=sub_text
-            )
-
-            sub.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    sub_text
-                )
-            )
-
-            clause.sub_clauses.append(
-                sub
-            )
-
-        return [clause]
-    
-    def build_numeric_structure(
-        self,
-        text: str
-    ) -> List[Clause]:
-
-        clauses = []
-
-        clause_sections = self.split_sections(
-            text,
-            self.NUMERIC_RE
-        )
-
-        for clause_no, clause_text in clause_sections:
-
-            clause = Clause(
-                document="bns",
-                clause_no=clause_no,
-                text=clause_text
-            )
-
-            alpha_sections = self.split_sections(
-                clause_text,
-                self.ALPHA_RE
-            )
-
-            if alpha_sections:
-
-                for sub_no, sub_text in alpha_sections:
-
-                    sub = SubClause(
-                        document="bns",
-                        sub_clause_no=sub_no,
-                        text=sub_text
-                    )
-
-                    sub.roman_clauses.extend(
-                        self.parse_roman_clauses(
-                            sub_text
-                        )
-                    )
-
-                    clause.sub_clauses.append(
-                        sub
-                    )
-
-            clauses.append(
-                clause
-            )
-
-        return clauses
-    def parse_alpha_clauses(
-        self,
-        text: str
-    ):
-
-        return self.build_alpha_structure(
-            text,
-            self.ALPHA_RE
-        )
-    
-
-    def parse_alpha_roman(
-        self,
-        text: str
-    ):
-
-        return self.build_alpha_structure(
-            text,
-            self.ALPHA_RE
-        )
-    
-
-    def parse_capital_roman(
-        self,
-        text: str
-    ):
-
-        return self.build_alpha_structure(
-            text,
-            self.CAPITAL_RE
-        )
-
-
-    def parse_numeric_clauses(
-        self,
-        text: str
-    ):
-
-        clauses = []
-
-        sections = self.split_sections(
-            text,
-            self.NUMERIC_RE
-        )
-
-        for clause_no, clause_text in sections:
-
-            clauses.append(
-                Clause(
-                    document="bns",
-                    clause_no=clause_no,
-                    text=clause_text
-                )
-            )
-
-        return clauses
-
-    def parse_numeric_alpha(
-        self,
-        text: str
-    ):
-
-        return self.build_numeric_structure(
-            text
-        )
-    
-
-    def parse_numeric_alpha_roman(
-        self,
-        text: str
-    ):
-
-        return self.build_numeric_structure(
-            text
-        )
-
-
-
-
-    # =====================================================
-    # CLAUSES
-    # =====================================================
-
-    def parse_clauses(
-        self,
-        section_text: str
-    ) -> List[Clause]:
-
-        clauses = []
-
-        clause_sections = self.split_sections(
-            section_text,
-            self.CLAUSE_RE
-        )
-
-        # No numbered clauses present
-        if not clause_sections:
-
-            subclauses = self.parse_subclauses(
-                section_text
-            )
-
-            if subclauses:
-
-                synthetic_clause = Clause(
-                    document="bns",
-                    clause_no="0",
-                    text=section_text
-                )
-
-                synthetic_clause.sub_clauses.extend(
-                    subclauses
-                )
-
-                clauses.append(
-                    synthetic_clause
-                )
-
-            return clauses
-
-        for clause_no, clause_text in clause_sections:
-
-            clause = Clause(
-                document="bns",
-                clause_no=clause_no,
-                text=clause_text
-            )
-
-            clause.sub_clauses.extend(
-                self.parse_subclauses(
-                    clause_text
-                )
-            )
-            clause.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    clause_text
-                )
-            )
-
-            clauses.append(
-                clause
-            )
-
-        return clauses
-
-
-
-
-    def parse_section_structure(
-        self,
-        section_text: str
-    ) -> List[Clause]:
-
-        has_numeric = bool(
-            self.NUMERIC_RE.search(
-                section_text
-            )
-        )
-
-        has_alpha = bool(
-            self.ALPHA_RE.search(
-                section_text
-            )
-        )
-
-        has_capital = bool(
-            self.CAPITAL_RE.search(
-                section_text
-            )
-        )
-
-        has_roman = bool(
-            self.ROMAN_RE.search(
-                section_text
-            )
-        )
-
-        if has_numeric:
-            return self.build_numeric_structure(
-                section_text
-            )
-
-        if has_capital:
-            return self.build_alpha_structure(
-                section_text,
-                self.CAPITAL_RE
-            )
-
-        if has_alpha:
-            return self.build_alpha_structure(
-                section_text,
-                self.ALPHA_RE
-            )
-
-        return []
-
-
-
-
-
-
-
-    # =====================================================
-    # VALIDATION
-    # =====================================================
-
-    def validate_clauses(
-        self,
-        section_no,
-        clauses: List[Clause]
-    ) -> List[str]:
-
-        errors = []
-
-        seen = set()
-
-        for clause in clauses:
-
-            if clause.clause_no in seen:
-
-                print("\n" + "="*80)
-                print("section: ",section_no)
-                print("DUPLICATE SUBCLAUSE")
-                print("Clause:", clause.clause_no)
-                print("SubClause:", sub.sub_clause_no)
-                print(sub.text[:1000])
-
-                errors.append(
-                    f"Duplicate SubClause "
-                    f"{sub.sub_clause_no}"
-                )
-
-            seen.add(
-                clause.clause_no
-            )
-
-            sub_seen = set()
-
-            for sub in clause.sub_clauses:
-
-                if sub.sub_clause_no in sub_seen:
-
-                    errors.append(
-                        f"Duplicate SubClause "
-                        f"{sub.sub_clause_no}"
-                    )
-
-                sub_seen.add(
-                    sub.sub_clause_no
-                )
-
-                roman_seen = set()
-
-                for roman in sub.roman_clauses:
-
-                    if roman.roman_no in roman_seen:
-
-                        errors.append(
-                            f"Duplicate Roman "
-                            f"{roman.roman_no}"
-                        )
-
-                    roman_seen.add(
-                        roman.roman_no
-                    )
-
-        return errors
-
-
-# =========================================================
-# TEST
-# =========================================================
-
-if __name__ == "__main__":
-
-    with open(
-        "../../pdfs/bns.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    cleaner=BNSSTextCleaner()
-    text=cleaner.clean(text)
-
- 
-
-    parser = ClauseParser()
-
-    print(len(parser.CLAUSE_RE.findall(text)))
-    print(len(parser.SUBCLAUSE_RE.findall(text)))
-    print(len(parser.ROMAN_RE.findall(text)))
-
-    clauses = parser.parse_section_structure(
-        text
-    )
-
-    print(
-        "Clauses:",
-        len(clauses)
-    )
-
-    # for clause in clauses:
-
-    #     print(
-    #         clause.clause_no,
-    #         len(clause.sub_clauses)
-    #     )
-
-    # print(
-    #     parser.validate_clauses(
-    #         clauses
-    #     )
-    # )

db/parsers/bnss/usage.py

@@ -2,16 +2,16 @@ from collections import Counter
 
 from cleaner import BNSSTextCleaner
 
-from bnss_parser import (
+from db.parsers.bnss.bnss_parser import (
     BNSSParser
 )
 
-from chunk_temp import (
+from db.parsers.bnss.chunker import (
     LegalChunker,
     chunks_to_dicts
 )
 
-from ingest import (
+from db.parsers.bnss.ingest import (
     LegalIngestionPipeline
 )
 

db/parsers/bnss_parser.py

@@ -1,221 +0,0 @@
-# parsers/bnss_parser.py
-
-from __future__ import annotations
-
-import re
-
-from dataclasses import dataclass, field
-from typing import List
-
-from parsers.base_parser import (
-    BaseLegalParser,
-    
-)
-from models.legal_models import (
-    BaseNode,
-    Chapter,
-    Section,
-    Clause,
-    SubClause,
-    RomanClause,
-)
-
-
-
-# ---------------------------------------------------
-# DATA MODELS
-# ---------------------------------------------------
-
-@dataclass
-class Section(BaseNode):
-    section_no: str
-    section_title: str
-    text: str
-
-    clauses: List[Clause] = field(default_factory=list)
-
-
-@dataclass
-class Chapter(BaseNode):
-    chapter_no: str
-    chapter_title: str
-
-    sections: List[Section] = field(default_factory=list)
-
-
-# ---------------------------------------------------
-# PARSER
-# ---------------------------------------------------
-
-class BNSSParser(BaseLegalParser):
-
-    CHAPTER_RE = re.compile(
-        r"CHAPTER\s+([IVXLC]+)\s*\n+([^\n]+)",
-        re.IGNORECASE
-    )
-
-    SECTION_RE = re.compile(
-        r"(?m)^(\d{1,3}[A-Z]?)\.\s*(.+)$"
-    )
-
-    # -------------------------------------------
-    # CHAPTER EXTRACTION
-    # -------------------------------------------
-
-    def extract_chapters(self, text):
-
-        matches = list(
-            self.CHAPTER_RE.finditer(text)
-        )
-
-        chapters = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            chapters.append(
-                {
-                    "chapter_no": match.group(1).strip(),
-                    "chapter_title": match.group(2).strip(),
-                    "text": text[start:end]
-                }
-            )
-
-        return chapters
-
-    # -------------------------------------------
-    # SECTION EXTRACTION
-    # -------------------------------------------
-
-    def extract_sections(self, chapter_text):
-
-        matches = list(
-            self.SECTION_RE.finditer(chapter_text)
-        )
-
-        sections = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(chapter_text)
-            )
-
-            sections.append(
-                {
-                    "section_no": match.group(1).strip(),
-                    "section_title": match.group(2).strip(),
-                    "text": chapter_text[start:end].strip()
-                }
-            )
-
-        return sections
-
-    # -------------------------------------------
-    # MAIN PARSE
-    # -------------------------------------------
-
-    def parse(self, raw_text):
-
-        raw_text = self.clean(raw_text)
-
-        chapters = []
-
-        for chapter_data in self.extract_chapters(raw_text):
-
-            chapter = Chapter(
-                document="bnss",
-                chapter_no=chapter_data["chapter_no"],
-                chapter_title=chapter_data["chapter_title"]
-            )
-
-            for section_data in self.extract_sections(
-                chapter_data["text"]
-            ):
-
-                section = Section(
-                    document="bnss",
-                    section_no=section_data["section_no"],
-                    section_title=section_data["section_title"],
-                    text=section_data["text"]
-                )
-
-                # -----------------------------
-                # Clauses
-                # -----------------------------
-
-                clause_data = self.extract_chapters(
-                    section.text
-                )
-
-                for c in clause_data:
-
-                    clause = Clause(
-                        clause_no=c["clause_no"],
-                        text=c["text"]
-                    )
-
-                    # -------------------------
-                    # Sub Clauses
-                    # -------------------------
-
-                    sub_clause_data = (
-                        self.extract_subclauses(
-                            clause.text
-                        )
-                    )
-
-                    for s in sub_clause_data:
-
-                        sub_clause = SubClause(
-                            document="bnss",
-                            sub_clause_no=s["sub_clause_no"],
-                            text=s["text"]
-                        )
-
-                        # ---------------------
-                        # Roman Clauses
-                        # ---------------------
-
-                        roman_data = (
-                            self.extract_roman_clauses(
-                                sub_clause.text
-                            )
-                        )
-
-                        for r in roman_data:
-
-                            sub_clause.roman_clauses.append(
-                                RomanClause(
-                                    document="bnss",
-                                    roman_no=r["roman_no"],
-                                    text=r["text"]
-                                )
-                            )
-
-                        clause.sub_clauses.append(
-                            sub_clause
-                        )
-
-                    section.clauses.append(
-                        clause
-                    )
-
-                chapter.sections.append(
-                    section
-                )
-
-            chapters.append(chapter)
-
-        return chapters

db/parsers/bsa/bsa_parser.py

@@ -1,35 +1,35 @@
 from __future__ import annotations
 
-from cleaner import BSATextCleaner
-from legal_models import (
+from db.parsers.bsa.cleaner import BSATextCleaner
+from db.parsers.bsa.legal_models import (
     LegalDocument
 )
 
-from part_parser import (
+from db.parsers.bsa.part_parser import (
     PartParser
 )
 
-from chapter_parser import (
+from db.parsers.bsa.chapter_parser import (
     ChapterParser
 )
 
-from section_parser import (
+from db.parsers.bsa.section_parser import (
     SectionParser
 )
 
-from clause_parser import (
+from db.parsers.bsa.clause_parser import (
     ClauseParser
 )
 
-from explaination_parser import (
+from db.parsers.bsa.explaination_parser import (
     ExplanationParser
 )
 
-from illustration_parser import (
+from db.parsers.bsa.illustration_parser import (
     IllustrationParser
 )
 
-from reference_parser import (
+from db.parsers.bsa.reference_parser import (
     ReferenceExtractor
 )
 

db/parsers/bsa/chapter_parser.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import re
 from typing import List, Dict
 
-from legal_models import Chapter
+from db.parsers.bsa.legal_models import Chapter
 
 
 class ChapterParser:

db/parsers/bsa/clause_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import (
+from db.parsers.bsa.legal_models import (
     Clause,
     SubClause,
     RomanClause

db/parsers/bsa/explaination_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import re
 from typing import List, Optional
 
-from legal_models import Explanation
+from db.parsers.bsa.legal_models import Explanation
 
 
 class ExplanationParser:

db/parsers/bsa/illustration_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import Illustration
+from db.parsers.bsa.legal_models import Illustration
 
 
 class IllustrationParser:

db/parsers/bsa/ingest.py

@@ -6,11 +6,11 @@ from qdrant_client.models import (
     PointStruct
 )
 
-from embedder import (
+from db.parsers.bsa.embedder import (
     LegalEmbedder
 )
 
-from qdrant_store import (
+from db.parsers.bsa.qdrant_store import (
     QdrantStore
 )
 

db/parsers/bsa/part_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import Part
+from db.parsers.bsa.legal_models import Part
 
 
 class PartParser:

db/parsers/bsa/section_parser.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 import re
 from typing import List
 
-from cleaner import BSATextCleaner
-from legal_models import Section
+from db.parsers.bsa.cleaner import BSATextCleaner
+from db.parsers.bsa.legal_models import Section
 
 
 class SectionParser:

db/parsers/bsa/usage.py

@@ -2,16 +2,16 @@ from collections import Counter
 
 from cleaner import BSATextCleaner
 
-from bsa_parser import (
+from db.parsers.bsa.bsa_parser import (
     BSAParser
 )
 
-from chunker import (
+from db.parsers.bsa.chunker import (
     LegalChunker,
     chunks_to_dicts
 )
 
-from ingest import (
+from db.parsers.bsa.ingest import (
     LegalIngestionPipeline
 )
 

db/parsers/bsa_parser.py

@@ -1,387 +0,0 @@
-from __future__ import annotations
-
-import re
-
-from parsers.base_parser import BaseLegalParser
-
-from models.legal_models import (
-    LegalDocument,
-    Division,
-    Provision,
-    Clause,
-    SubClause,
-    RomanClause
-)
-
-
-class BSAParser(BaseLegalParser):
-
-    # =====================================================
-    # REGEX
-    # =====================================================
-
-    PART_RE = re.compile(
-        r"(?m)^PART\s+([IVXLC]+)"
-    )
-
-    CHAPTER_RE = re.compile(
-        r"(?m)^CHAPTER\s+([IVXLC]+)\s*\n+([^\n]+)"
-    )
-
-    SECTION_RE = re.compile(
-        r"(?m)^(\d+[A-Z]?)\.\s*(.+)"
-    )
-
-    CLAUSE_RE = re.compile(
-        r"(?m)^\((\d+)\)\s*(.*)"
-    )
-
-    SUBCLAUSE_RE = re.compile(
-        r"(?m)^\(([a-z])\)\s*(.*)"
-    )
-
-    ROMAN_RE = re.compile(
-        r"(?m)^\((i|ii|iii|iv|v|vi|vii|viii|ix|x)\)\s*(.*)",
-        re.IGNORECASE
-    )
-
-    # =====================================================
-    # CLEAN
-    # =====================================================
-
-    def clean(self, text):
-
-        text = super().clean(text)
-
-        text = re.sub(
-            r"THE GAZETTE OF INDIA EXTRAORDINARY.*",
-            "",
-            text
-        )
-
-        text = re.sub(
-            r"_{5,}",
-            "",
-            text
-        )
-
-        return text
-
-    # =====================================================
-    # PARTS
-    # =====================================================
-
-    def extract_parts(self, text):
-
-        matches = list(
-            self.PART_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "part_no":
-                        match.group(1),
-
-                    "text":
-                        text[start:end]
-                }
-            )
-
-        return results
-
-    # =====================================================
-    # CHAPTERS
-    # =====================================================
-
-    def extract_chapters(self, text):
-
-        matches = list(
-            self.CHAPTER_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "chapter_no":
-                        match.group(1),
-
-                    "chapter_title":
-                        match.group(2).strip(),
-
-                    "text":
-                        text[start:end]
-                }
-            )
-
-        return results
-
-    # =====================================================
-    # SECTIONS
-    # =====================================================
-
-    def extract_sections(self, text):
-
-        matches = list(
-            self.SECTION_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            section_no = match.group(1)
-
-            title = match.group(2).strip()
-
-            results.append(
-                {
-                    "section_no":
-                        section_no,
-
-                    "section_title":
-                        title,
-
-                    "text":
-                        text[start:end]
-                }
-            )
-
-        return results
-
-    # =====================================================
-    # CLAUSES
-    # =====================================================
-
-    def extract_clauses(self, text):
-
-        matches = list(
-            self.CLAUSE_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "clause_no":
-                        match.group(1),
-
-                    "text":
-                        text[start:end].strip()
-                }
-            )
-
-        return results
-
-    # =====================================================
-    # SUB CLAUSES
-    # =====================================================
-
-    def extract_subclauses(self, text):
-
-        matches = list(
-            self.SUBCLAUSE_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "sub_clause_no":
-                        match.group(1),
-
-                    "text":
-                        text[start:end].strip()
-                }
-            )
-
-        return results
-
-    # =====================================================
-    # ROMAN CLAUSES
-    # =====================================================
-
-    def extract_roman_clauses(self, text):
-
-        matches = list(
-            self.ROMAN_RE.finditer(text)
-        )
-
-        results = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            results.append(
-                {
-                    "roman_no":
-                        match.group(1),
-
-                    "text":
-                        text[start:end].strip()
-                }
-            )
-
-        return results
-
-    # =====================================================
-    # MAIN PARSER
-    # =====================================================
-
-    def parse(self, raw_text):
-
-        raw_text = self.clean(raw_text)
-
-        divisions = []
-
-        for part_data in self.extract_parts(raw_text):
-
-            part_no = part_data["part_no"]
-
-            for chapter_data in self.extract_chapters(
-                part_data["text"]
-            ):
-
-                division = Division(
-                    document="bsa",
-                    division_no=chapter_data["chapter_no"],
-                    title=chapter_data["chapter_title"]
-                )
-
-                for section_data in self.extract_sections(
-                    chapter_data["text"]
-                ):
-
-                    provision = Provision(
-                        document="bsa",
-                        provision_no=section_data["section_no"],
-                        title=section_data["section_title"],
-                        text=section_data["text"]
-                    )
-
-                    # ---------------------
-                    # Clauses
-                    # ---------------------
-
-                    for clause_data in self.extract_clauses(
-                        provision.text
-                    ):
-
-                        clause = Clause(
-                            document="bsa",
-                            clause_no=clause_data["clause_no"],
-                            text=clause_data["text"]
-                        )
-
-                        # -----------------
-                        # Sub Clauses
-                        # -----------------
-
-                        for sub_data in self.extract_subclauses(
-                            clause.text
-                        ):
-
-                            sub = SubClause(
-                                document="bsa",
-                                sub_clause_no=sub_data["sub_clause_no"],
-                                text=sub_data["text"]
-                            )
-
-                            # -------------
-                            # Roman
-                            # -------------
-
-                            for roman_data in self.extract_roman_clauses(
-                                sub.text
-                            ):
-
-                                sub.roman_clauses.append(
-                                    RomanClause(
-                                        document="bsa",
-                                        roman_no=roman_data["roman_no"],
-                                        text=roman_data["text"]
-                                    )
-                                )
-
-                            clause.sub_clauses.append(sub)
-
-                        provision.clauses.append(clause)
-
-                    provision.references = (
-                        self.extract_references(
-                            provision.text
-                        )
-                    )
-
-                    division.provisions.append(
-                        provision
-                    )
-
-                division.part_no = part_no
-
-                divisions.append(
-                    division
-                )
-
-        return LegalDocument(
-            document="bsa",
-            document_type="bsa",
-            divisions=divisions
-        )

db/{chunker/constitution_chunker.py → parsers/consitution/chunker.py}

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from typing import List, Dict, Any
-from parsers.consitution.constitution_parser import ConstitutionParser
+from db.parsers.consitution.constitution_parser import ConstitutionParser
 
 # =========================================================
 # LEGAL CHUNK
@@ -407,26 +407,133 @@ class LegalChunker:
 
 
 
-if __name__ == "__main__":
+def build_enriched_text(
+    chunk: LegalChunk
+) -> str:
 
-    with open(
-        "../../pdfs/constitution.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
+    m = chunk.metadata
 
-        text = f.read()
+    parts = [
+        "Constitution of India"
+    ]
 
-    parser = ConstitutionParser()
+    if m.get("part_no"):
+        parts.append(
+            f"Part {m['part_no']}"
+        )
 
-    constitution = parser.parse(
-        text
-    )
-    chunker=LegalChunker()
-    chunks=chunker.chunk_constitution(constitution)
-    print(len(chunks))
-    print(
-        chunker.stats(chunks)
+    if m.get("chapter_no"):
+        parts.append(
+            f"Chapter {m['chapter_no']}"
+        )
+
+    if m.get("article_no"):
+        parts.append(
+            f"Article {m['article_no']}"
+        )
+
+    if m.get("clause_no"):
+        parts.append(
+            f"Clause ({m['clause_no']})"
+        )
+
+    if m.get("sub_clause_no"):
+        parts.append(
+            f"SubClause ({m['sub_clause_no']})"
+        )
+
+    if m.get("roman_no"):
+        parts.append(
+            f"Roman ({m['roman_no']})"
+        )
+
+    parts.append("")
+    parts.append(
+        chunk.text
     )
 
-    
+    return "\n".join(parts)
+
+
+
+
+def chunks_to_dicts(
+    chunks: List[LegalChunk]
+) -> List[dict]:
+
+    result = []
+
+    for chunk in chunks:
+
+        metadata = chunk.metadata
+
+        result.append(
+            {
+                "chunk_id":
+                    chunk.chunk_id,
+
+                "level":
+                    chunk.chunk_type,
+
+                "document":
+                    "constitution",
+
+                "part_no":
+                    metadata.get(
+                        "part_no"
+                    ),
+
+                "part_title":
+                    metadata.get(
+                        "part_title"
+                    ),
+
+                "chapter_no":
+                    metadata.get(
+                        "chapter_no"
+                    ),
+
+                "chapter_title":
+                    metadata.get(
+                        "chapter_title"
+                    ),
+
+                "article_no":
+                    metadata.get(
+                        "article_no"
+                    ),
+
+                "article_title":
+                    metadata.get(
+                        "article_title"
+                    ),
+
+                "clause_no":
+                    metadata.get(
+                        "clause_no"
+                    ),
+
+                "sub_clause_no":
+                    metadata.get(
+                        "sub_clause_no"
+                    ),
+
+                "roman_no":
+                    metadata.get(
+                        "roman_no"
+                    ),
+
+                "text":
+                    chunk.text,
+
+                "references":
+                    chunk.references,
+
+                "enriched_text":
+                    build_enriched_text(
+                        chunk
+                    )
+            }
+        )
+
+    return result

db/parsers/consitution/chunker_test.py

@@ -1,432 +0,0 @@
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import List, Dict, Any
-from constitution_parser import ConstitutionParser
-
-# =========================================================
-# LEGAL CHUNK
-# =========================================================
-
-@dataclass
-class LegalChunk:
-
-    chunk_id: str
-
-    chunk_type: str
-
-    text: str
-
-    metadata: Dict[str, Any] = field(
-        default_factory=dict
-    )
-
-    references: List[str] = field(
-        default_factory=list
-    )
-
-
-# =========================================================
-# LEGAL CHUNKER
-# =========================================================
-
-class LegalChunker:
-
-    """
-    Creates chunks at:
-
-        Article
-        Clause
-        Proviso
-        Explanation
-        Schedule
-
-    for RAG retrieval.
-    """
-
-    # =====================================================
-    # MAIN
-    # =====================================================
-
-    def chunk_constitution(
-        self,
-        constitution
-    ) -> List[LegalChunk]:
-
-        chunks = []
-
-        # -------------------------------------
-        # PARTS
-        # -------------------------------------
-
-        for part in constitution.parts:
-
-            part_meta = {
-                "part_no":
-                    part.part_no,
-
-                "part_title":
-                    part.part_title
-            }
-
-            # -------------------------
-            # Articles directly under part
-            # -------------------------
-
-            for article in getattr(
-                part,
-                "articles",
-                []
-            ):
-
-                chunks.extend(
-                    self.chunk_article(
-                        article,
-                        part_meta,
-                        {}
-                    )
-                )
-
-            # -------------------------
-            # Chapters
-            # -------------------------
-
-            for chapter in getattr(
-                part,
-                "chapters",
-                []
-            ):
-
-                chapter_meta = {
-                    "chapter_no":
-                        chapter.chapter_no,
-
-                    "chapter_title":
-                        chapter.chapter_title
-                }
-
-                for article in getattr(
-                    chapter,
-                    "articles",
-                    []
-                ):
-
-                    chunks.extend(
-                        self.chunk_article(
-                            article,
-                            part_meta,
-                            chapter_meta
-                        )
-                    )
-
-        # -------------------------------------
-        # SCHEDULES
-        # -------------------------------------
-
-        for schedule in getattr(
-            constitution,
-            "schedules",
-            []
-        ):
-
-            chunks.append(
-                LegalChunk(
-                    chunk_id=
-                        f"schedule_"
-                        f"{schedule.schedule_no}",
-
-                    chunk_type=
-                        "schedule",
-
-                    text=
-                        schedule.text,
-
-                    metadata={
-                        "schedule_no":
-                            schedule.schedule_no,
-
-                        "schedule_title":
-                            schedule.schedule_title
-                    }
-                )
-            )
-
-        return chunks
-
-    # =====================================================
-    # ARTICLE
-    # =====================================================
-
-    def chunk_article(
-        self,
-        article,
-        part_meta,
-        chapter_meta
-    ) -> List[LegalChunk]:
-
-        chunks = []
-
-        base_meta = {
-            **part_meta,
-            **chapter_meta,
-
-            "article_no":
-                article.article_no,
-
-            "article_title":
-                article.article_title
-        }
-
-        # -------------------------------------
-        # FULL ARTICLE
-        # -------------------------------------
-
-        chunks.append(
-            LegalChunk(
-                chunk_id=
-                    f"article_"
-                    f"{article.article_no}",
-
-                chunk_type=
-                    "article",
-
-                text=
-                    article.text,
-
-                metadata=
-                    base_meta,
-
-                references=[
-                    str(r)
-                    for r in getattr(
-                        article,
-                        "references",
-                        []
-                    )
-                ]
-            )
-        )
-
-        # -------------------------------------
-        # CLAUSES
-        # -------------------------------------
-
-        for clause in getattr(
-            article,
-            "clauses",
-            []
-        ):
-
-            clause_meta = {
-                **base_meta,
-
-                "clause_no":
-                    clause.clause_no
-            }
-
-            chunks.append(
-                LegalChunk(
-                    chunk_id=
-                        f"article_"
-                        f"{article.article_no}"
-                        f"_clause_"
-                        f"{clause.clause_no}",
-
-                    chunk_type=
-                        "clause",
-
-                    text=
-                        clause.text,
-
-                    metadata=
-                        clause_meta
-                )
-            )
-
-            # -----------------------------
-            # SUB CLAUSES
-            # -----------------------------
-
-            for sub in getattr(
-                clause,
-                "sub_clauses",
-                []
-            ):
-
-                sub_meta = {
-                    **clause_meta,
-
-                    "sub_clause_no":
-                        sub.sub_clause_no
-                }
-
-                chunks.append(
-                    LegalChunk(
-                        chunk_id=
-                            f"article_"
-                            f"{article.article_no}"
-                            f"_clause_"
-                            f"{clause.clause_no}"
-                            f"_sub_"
-                            f"{sub.sub_clause_no}",
-
-                        chunk_type=
-                            "sub_clause",
-
-                        text=
-                            sub.text,
-
-                        metadata=
-                            sub_meta
-                    )
-                )
-
-                # -------------------------
-                # Roman Clauses
-                # -------------------------
-
-                for roman in getattr(
-                    sub,
-                    "roman_clauses",
-                    []
-                ):
-
-                    chunks.append(
-                        LegalChunk(
-                            chunk_id=
-                                f"article_"
-                                f"{article.article_no}"
-                                f"_roman_"
-                                f"{roman.roman_no}",
-
-                            chunk_type=
-                                "roman_clause",
-
-                            text=
-                                roman.text,
-
-                            metadata={
-                                **sub_meta,
-
-                                "roman_no":
-                                    roman.roman_no
-                            }
-                        )
-                    )
-
-        # -------------------------------------
-        # PROVISOS
-        # -------------------------------------
-
-        for idx, proviso in enumerate(
-            getattr(
-                article,
-                "provisos",
-                []
-            ),
-            start=1
-        ):
-
-            chunks.append(
-                LegalChunk(
-                    chunk_id=
-                        f"article_"
-                        f"{article.article_no}"
-                        f"_proviso_"
-                        f"{idx}",
-
-                    chunk_type=
-                        "proviso",
-
-                    text=
-                        proviso.text,
-
-                    metadata=
-                        base_meta
-                )
-            )
-
-        # -------------------------------------
-        # EXPLANATIONS
-        # -------------------------------------
-
-        for idx, explanation in enumerate(
-            getattr(
-                article,
-                "explanations",
-                []
-            ),
-            start=1
-        ):
-
-            chunks.append(
-                LegalChunk(
-                    chunk_id=
-                        f"article_"
-                        f"{article.article_no}"
-                        f"_explanation_"
-                        f"{idx}",
-
-                    chunk_type=
-                        "explanation",
-
-                    text=
-                        explanation.text,
-
-                    metadata=
-                        base_meta
-                )
-            )
-
-        return chunks
-
-    # =====================================================
-    # STATS
-    # =====================================================
-
-    def stats(
-        self,
-        chunks: List[LegalChunk]
-    ):
-
-        result = {}
-
-        for chunk in chunks:
-
-            result.setdefault(
-                chunk.chunk_type,
-                0
-            )
-
-            result[
-                chunk.chunk_type
-            ] += 1
-
-        return result
-
-
-
-if __name__ == "__main__":
-
-    with open(
-        "../../pdfs/constitution.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    parser = ConstitutionParser()
-
-    constitution = parser.parse(
-        text
-    )
-    chunker=LegalChunker()
-    chunks=chunker.chunk_constitution(constitution)
-    print(len(chunks))
-    print(
-        chunker.stats(chunks)
-    )
-
-    

db/parsers/consitution/clause_parser.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import (
+from db.parsers.consitution.legal_models import (
     Clause,
     SubClause,
     RomanClause

db/parsers/consitution/cleaner.py

@@ -0,0 +1,61 @@
+import re
+
+class ConstitutionTextCleaner:
+    def clean(
+            self,
+            text: str
+        ) -> str:
+
+            text = text.replace(
+                "\r\n",
+                "\n"
+            )
+
+            text = re.sub(
+                r"\r",
+                "\n",
+                text
+            )
+
+            # remove page numbers
+
+            text = re.sub(
+                r"(?m)^\s*\d+\s*$",
+                "",
+                text
+            )
+
+            # remove repeated spaces
+
+            text = re.sub(
+                r"[ \t]+",
+                " ",
+                text
+            )
+
+            # remove constitution headers
+
+            text = re.sub(
+                r"THE CONSTITUTION OF INDIA",
+                "",
+                text,
+                flags=re.I
+            )
+
+            # remove separators
+
+            text = re.sub(
+                r"_{3,}",
+                "",
+                text
+            )
+
+            # collapse blank lines
+
+            text = re.sub(
+                r"\n{3,}",
+                "\n\n",
+                text
+            )
+
+            return text.strip()

db/parsers/consitution/constitution_parser.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 
 import re
 
-from legal_models import (
+from db.parsers.consitution.legal_models import (
     ConstitutionDocument,
     Preamble,
     Part,
@@ -11,14 +11,14 @@ from legal_models import (
     Article
 )
 
-from part_parser import PartParser
-from chapter_parser import ChapterParser
-from article_parser import ArticleParser
-from clause_parser import ClauseParser
-from proviso_parser import ProvisoParser
-from explanation_parser import ExplanationParser
-from schedule_parser import ScheduleParser
-from reference_extractor import ReferenceExtractor
+from db.parsers.consitution.part_parser import PartParser
+from db.parsers.consitution.chapter_parser import ChapterParser
+from db.parsers.consitution.article_parser import ArticleParser
+from db.parsers.consitution.clause_parser import ClauseParser
+from db.parsers.consitution.proviso_parser import ProvisoParser
+from db.parsers.consitution.explanation_parser import ExplanationParser
+from db.parsers.consitution.schedule_parser import ScheduleParser
+from db.parsers.consitution.reference_extractor import ReferenceExtractor
 
 
 class ConstitutionParser:

db/parsers/consitution/embeddings_test.py

@@ -174,7 +174,7 @@ class EmbeddingGenerator:
 if __name__ == "__main__":
 
     from constitution_parser import ConstitutionParser
-    from chunker_test import LegalChunker
+    from db.parsers.consitution.chunker import LegalChunker
 
     with open(
         "../../pdfs/constitution.txt",

db/parsers/consitution/explanation_parser.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import Explanation
+from db.parsers.consitution.legal_models import Explanation
 
 
 class ExplanationParser:

db/parsers/consitution/proviso_parser.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import Proviso
+from db.parsers.consitution.legal_models import Proviso
 
 
 class ProvisoParser:

db/parsers/consitution/reference_extractor.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import re
 from typing import List
 
-from legal_models import Reference
+from db.parsers.consitution.legal_models import Reference
 
 
 class ReferenceExtractor:

db/parsers/consitution/schedule_parser.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import re
 from typing import List, Dict
 
-from legal_models import (
+from db.parsers.consitution.legal_models import (
     Schedule,
     ScheduleParagraph
 )

db/parsers/consitution/usage.py

@@ -1,12 +1,12 @@
-from embeddings_test import (
+from db.parsers.consitution.embeddings_test import (
     EmbeddingGenerator
 )
 
-from qdrant_store import (
+from db.parsers.consitution.qdrant_store import (
     QdrantStore
 )
-from constitution_parser import ConstitutionParser
-from chunker_test import LegalChunker
+from db.parsers.consitution.constitution_parser import ConstitutionParser
+from db.parsers.consitution.chunker import LegalChunker
 
 
 with open(

db/parsers/constitution_parser.py

@@ -1,1823 +0,0 @@
-# from __future__ import annotations
-
-# import re
-# from dataclasses import dataclass, field, asdict
-# from typing import List
-# from models.legal_models import Article, Part, Explanation, Proviso, RomanClause, SubClause, Clause
-
-# # ==========================================================
-# # DATA MODELS
-# # ==========================================================
-
-# @dataclass
-# class Reference:
-#     article_no: str
-
-
-# # @dataclass
-# # class Explanation:
-# #     text: str
-
-
-# # @dataclass
-# # class Proviso:
-# #     text: str
-
-
-# # @dataclass
-# # class RomanClause:
-# #     roman_no: str
-# #     text: str
-
-
-# # @dataclass
-# # class SubClause:
-# #     sub_clause_no: str
-# #     text: str
-# #     roman_clauses: List[RomanClause] = field(default_factory=list)
-
-
-# # @dataclass
-# # class Clause:
-# #     clause_no: str
-# #     text: str
-# #     sub_clauses: List[SubClause] = field(default_factory=list)
-
-
-# # @dataclass
-# # class Article:
-# #     article_no: str
-# #     article_title: str
-# #     text: str
-
-# #     clauses: List[Clause] = field(default_factory=list)
-# #     provisos: List[Proviso] = field(default_factory=list)
-# #     explanations: List[Explanation] = field(default_factory=list)
-# #     references: List[Reference] = field(default_factory=list)
-
-
-# # @dataclass
-# # class Part:
-# #     part_no: str
-# #     part_title: str
-# #     articles: List[Article] = field(default_factory=list)
-
-
-# # ==========================================================
-# # PARSER
-# # ==========================================================
-
-# class ConstitutionParser:
-
-#     PART_RE = re.compile(
-#         r"PART\s+([IVXLC]+)\s*\n+([^\n]+)",
-#         re.IGNORECASE
-#     )
-
-#     ARTICLE_RE = re.compile(
-#         r"(?m)^(\d{1,3}[A-Z]?)\.\s*(.+)$"
-#     )
-
-#     CLAUSE_RE = re.compile(
-#         r"(?m)^\((\d+[A-Za-z]?)\)\s*(.*)"
-#     )
-
-#     SUBCLAUSE_RE = re.compile(
-#         r"(?m)^\(([a-hj-z])\)\s*(.*)"
-#     )
-
-#     ROMAN_RE = re.compile(
-#         r"(?m)^\((i|ii|iii|iv|v|vi|vii|viii|ix|x)\)\s*(.*)"
-#     )
-
-#     PROVISO_RE = re.compile(
-#         r"(?is)"
-#         r"(Provided(?:\s+further)?(?:\s+also)?\s+that.*?)(?="
-#         r"Provided(?:\s+further)?(?:\s+also)?\s+that"
-#         r"|Explanation"
-#         r"|$)"
-#     )
-
-#     EXPLANATION_RE = re.compile(
-#         r"(?is)"
-#         r"(Explanation.*?)(?="
-#         r"Provided"
-#         r"|$)"
-#     )
-
-#     REFERENCE_RE = re.compile(
-#         r"article[s]?\s+(\d+[A-Z]?)",
-#         re.IGNORECASE
-#     )
-
-#     # ======================================================
-#     # OCR CLEANING
-#     # ======================================================
-
-#     def clean(self, text: str):
-
-#         text = text.replace("\r\n", "\n")
-
-#         # remove page numbers
-#         text = re.sub(
-#             r"(?m)^\s*\d+\s*$",
-#             "",
-#             text
-#         )
-
-#         # remove constitution headers
-#         text = re.sub(
-#             r"THE CONSTITUTION OF INDIA",
-#             "",
-#             text,
-#             flags=re.IGNORECASE
-#         )
-
-#         # remove separators
-#         text = re.sub(
-#             r"_{5,}",
-#             "",
-#             text
-#         )
-
-#         # remove amendment notes
-#         text = re.sub(
-#             r"(?m)^\d+\.\s+(Subs\.|Ins\.|Added|Omitted).*?$",
-#             "",
-#             text
-#         )
-
-#         text = re.sub(
-#             r"\n{3,}",
-#             "\n\n",
-#             text
-#         )
-
-#         return text.strip()
-
-#     # ======================================================
-#     # PARTS
-#     # ======================================================
-
-#     def extract_parts(self, text):
-
-#         matches = list(self.PART_RE.finditer(text))
-
-#         results = []
-
-#         for i, m in enumerate(matches):
-
-#             start = m.start()
-
-#             end = (
-#                 matches[i + 1].start()
-#                 if i + 1 < len(matches)
-#                 else len(text)
-#             )
-
-#             results.append({
-#                 "part_no": m.group(1).strip(),
-#                 "part_title": m.group(2).strip(),
-#                 "text": text[start:end]
-#             })
-
-#         return results
-
-#     # ======================================================
-#     # ARTICLES
-#     # ======================================================
-
-#     def extract_articles(self, text):
-
-#         matches = list(self.ARTICLE_RE.finditer(text))
-
-#         results = []
-
-#         for i, m in enumerate(matches):
-
-#             start = m.start()
-
-#             end = (
-#                 matches[i + 1].start()
-#                 if i + 1 < len(matches)
-#                 else len(text)
-#             )
-
-#             results.append({
-#                 "article_no": m.group(1),
-#                 "article_title": m.group(2).strip(),
-#                 "text": text[start:end].strip()
-#             })
-
-#         return results
-
-#     # ======================================================
-#     # CLAUSES
-#     # ======================================================
-
-#     def extract_clauses(self, text):
-
-#         matches = list(self.CLAUSE_RE.finditer(text))
-
-#         results = []
-
-#         for i, m in enumerate(matches):
-
-#             start = m.start()
-
-#             end = (
-#                 matches[i + 1].start()
-#                 if i + 1 < len(matches)
-#                 else len(text)
-#             )
-
-#             results.append({
-#                 "clause_no": m.group(1),
-#                 "text": text[start:end].strip()
-#             })
-
-#         return results
-
-#     # ======================================================
-#     # SUB CLAUSES
-#     # ======================================================
-
-#     def extract_subclauses(self, text):
-
-#         matches = list(self.SUBCLAUSE_RE.finditer(text))
-
-#         results = []
-
-#         for i, m in enumerate(matches):
-
-#             start = m.start()
-
-#             end = (
-#                 matches[i + 1].start()
-#                 if i + 1 < len(matches)
-#                 else len(text)
-#             )
-
-#             results.append({
-#                 "sub_clause_no": m.group(1),
-#                 "text": text[start:end].strip()
-#             })
-
-#         return results
-
-#     # ======================================================
-#     # ROMAN CLAUSES
-#     # ======================================================
-
-#     def extract_roman_clauses(self, text):
-
-#         matches = list(self.ROMAN_RE.finditer(text))
-
-#         results = []
-
-#         for i, m in enumerate(matches):
-
-#             start = m.start()
-
-#             end = (
-#                 matches[i + 1].start()
-#                 if i + 1 < len(matches)
-#                 else len(text)
-#             )
-
-#             results.append({
-#                 "roman_no": m.group(1),
-#                 "text": text[start:end].strip()
-#             })
-
-#         return results
-
-#     # ======================================================
-#     # PROVISOS
-#     # ======================================================
-
-#     def extract_provisos(self, text):
-
-#         return [
-#             Proviso(text=m.strip())
-#             for m in self.PROVISO_RE.findall(text)
-#         ]
-
-#     # ======================================================
-#     # EXPLANATIONS
-#     # ======================================================
-
-#     def extract_explanations(self, text):
-
-#         return [
-#             Explanation(text=m.strip())
-#             for m in self.EXPLANATION_RE.findall(text)
-#         ]
-
-#     # ======================================================
-#     # REFERENCES
-#     # ======================================================
-
-#     def extract_references(self, text):
-
-#         refs = sorted(
-#             set(
-#                 self.REFERENCE_RE.findall(text)
-#             )
-#         )
-
-#         return [
-#             Reference(article_no=r)
-#             for r in refs
-#         ]
-
-#     # ======================================================
-#     # MAIN PARSER
-#     # ======================================================
-
-#     def parse(self, raw_text):
-
-#         raw_text = self.clean(raw_text)
-
-#         parsed_parts = []
-
-#         for p in self.extract_parts(raw_text):
-
-#             part = Part(
-#                 document="constitution",
-#                 part_no=p["part_no"],
-#                 part_title=p["part_title"]
-#             )
-
-#             for a in self.extract_articles(p["text"]):
-
-#                 article = Article(
-#                     document="constitution",
-#                     article_no=a["article_no"],
-#                     article_title=a["article_title"],
-#                     text=a["text"]
-#                 )
-
-#                 article.provisos = self.extract_provisos(
-#                     article.text
-#                 )
-
-#                 article.explanations = self.extract_explanations(
-#                     article.text
-#                 )
-
-#                 article.references = self.extract_references(
-#                     article.text
-#                 )
-
-#                 for c in self.extract_clauses(article.text):
-
-#                     clause = Clause(
-#                         document="constitution",
-#                         clause_no=c["clause_no"],
-#                         text=c["text"]
-#                     )
-
-#                     for s in self.extract_subclauses(
-#                         clause.text
-#                     ):
-
-#                         sub = SubClause(
-#                             document="constitution",
-#                             sub_clause_no=s["sub_clause_no"],
-#                             text=s["text"]
-#                         )
-
-#                         for r in self.extract_roman_clauses(
-#                             sub.text
-#                         ):
-
-#                             sub.roman_clauses.append(
-#                                 RomanClause(
-#                                     document="constitution",
-#                                     roman_no=r["roman_no"],
-#                                     text=r["text"]
-#                                 )
-#                             )
-
-#                         clause.sub_clauses.append(sub)
-
-#                     article.clauses.append(clause)
-
-#                 part.articles.append(article)
-
-#             parsed_parts.append(part)
-
-#         return parsed_parts
-
-
-# # ==========================================================
-# # EXAMPLE
-# # ==========================================================
-
-# if __name__ == "__main__":
-
-#     with open("text_extracted_ocr_output.txt", "r", encoding="utf8") as f:
-#         text = f.read()
-
-#     parser = ConstitutionParser()
-
-#     result = parser.parse(text)
-
-#     print(asdict(result[0]))
-
-
-
-
-
-
-
-
-
-
-
-
-
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import List, Optional
-
-
-# ==========================================================
-# BASE NODE
-# ==========================================================
-
-@dataclass
-class LegalNode:
-    """
-    Base node used by all legal entities.
-    """
-
-    document: str
-
-    id: Optional[str] = None
-
-    parent_id: Optional[str] = None
-
-    source_text: Optional[str] = None
-
-    start_offset: Optional[int] = None
-
-    end_offset: Optional[int] = None
-
-
-# ==========================================================
-# REFERENCES
-# ==========================================================
-
-@dataclass
-class Reference:
-
-    reference_type: str
-
-    reference_value: str
-
-    text: str = ""
-
-
-# ==========================================================
-# PROVISO
-# ==========================================================
-
-@dataclass
-class Proviso(LegalNode):
-
-    text: str = ""
-
-
-# ==========================================================
-# EXPLANATION
-# ==========================================================
-
-@dataclass
-class Explanation(LegalNode):
-
-    explanation_no: str = ""
-
-    text: str = ""
-
-
-# ==========================================================
-# ILLUSTRATION
-# ==========================================================
-
-@dataclass
-class Illustration(LegalNode):
-
-    illustration_no: str = ""
-
-    text: str = ""
-
-
-# ==========================================================
-# ROMAN CLAUSE
-# ==========================================================
-
-@dataclass
-class RomanClause(LegalNode):
-
-    roman_no: str = ""
-
-    text: str = ""
-
-
-# ==========================================================
-# SUB CLAUSE
-# ==========================================================
-
-@dataclass
-class SubClause(LegalNode):
-
-    sub_clause_no: str = ""
-
-    text: str = ""
-
-    roman_clauses: List[RomanClause] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# CLAUSE
-# ==========================================================
-
-@dataclass
-class Clause(LegalNode):
-
-    clause_no: str = ""
-
-    text: str = ""
-
-    sub_clauses: List[SubClause] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# ARTICLE
-# ==========================================================
-
-@dataclass
-class Article(LegalNode):
-
-    article_no: str = ""
-
-    article_title: str = ""
-
-    text: str = ""
-
-    clauses: List[Clause] = field(
-        default_factory=list
-    )
-
-    provisos: List[Proviso] = field(
-        default_factory=list
-    )
-
-    explanations: List[Explanation] = field(
-        default_factory=list
-    )
-
-    illustrations: List[Illustration] = field(
-        default_factory=list
-    )
-
-    references: List[Reference] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# CHAPTER
-# ==========================================================
-
-@dataclass
-class Chapter(LegalNode):
-
-    chapter_no: str = ""
-
-    chapter_title: str = ""
-
-    articles: List[Article] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# PART
-# ==========================================================
-
-@dataclass
-class Part(LegalNode):
-
-    part_no: str = ""
-
-    part_title: str = ""
-
-    chapters: List[Chapter] = field(
-        default_factory=list
-    )
-
-    articles: List[Article] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# SCHEDULE PARAGRAPH
-# ==========================================================
-
-@dataclass
-class ScheduleParagraph(LegalNode):
-
-    paragraph_no: str = ""
-
-    text: str = ""
-
-
-# ==========================================================
-# SCHEDULE
-# ==========================================================
-
-@dataclass
-class Schedule(LegalNode):
-
-    schedule_no: str = ""
-
-    schedule_title: str = ""
-
-    text: str = ""
-
-    paragraphs: List[ScheduleParagraph] = field(
-        default_factory=list
-    )
-
-    references: List[Reference] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# PREAMBLE
-# ==========================================================
-
-@dataclass
-class Preamble(LegalNode):
-
-    text: str = ""
-
-
-# ==========================================================
-# CONSTITUTION DOCUMENT
-# ==========================================================
-
-@dataclass
-class ConstitutionDocument(LegalNode):
-
-    title: str = "Constitution of India"
-
-    preamble: Optional[Preamble] = None
-
-    parts: List[Part] = field(
-        default_factory=list
-    )
-
-    schedules: List[Schedule] = field(
-        default_factory=list
-    )
-
-
-# ==========================================================
-# UTILITIES
-# ==========================================================
-
-def build_id(
-    prefix: str,
-    value: str
-) -> str:
-
-    value = value.replace(" ", "_")
-
-    return f"{prefix}:{value}"
-
-
-def assign_ids(
-    constitution: ConstitutionDocument
-):
-
-    constitution.id = "constitution"
-
-    for part in constitution.parts:
-
-        part.id = build_id(
-            "part",
-            part.part_no
-        )
-
-        part.parent_id = constitution.id
-
-        for chapter in part.chapters:
-
-            chapter.id = build_id(
-                "chapter",
-                chapter.chapter_no
-            )
-
-            chapter.parent_id = part.id
-
-            for article in chapter.articles:
-
-                article.id = build_id(
-                    "article",
-                    article.article_no
-                )
-
-                article.parent_id = chapter.id
-
-        for article in part.articles:
-
-            article.id = build_id(
-                "article",
-                article.article_no
-            )
-
-            article.parent_id = part.id
-
-    for schedule in constitution.schedules:
-
-        schedule.id = build_id(
-            "schedule",
-            schedule.schedule_no
-        )
-
-        schedule.parent_id = constitution.id
-
-
-
-from __future__ import annotations
-
-import re
-from typing import List, Dict
-
-
-class PartParser:
-    """
-    Constitution Part Parser
-
-    Extracts:
-
-        PART I
-        THE UNION AND ITS TERRITORY
-
-        PART II
-        CITIZENSHIP
-
-    and returns the entire text
-    belonging to each Part.
-    """
-
-    # =====================================================
-    # PART PATTERN
-    # =====================================================
-
-    PART_RE = re.compile(
-        r"(?im)^PART\s+([IVXLCDM]+)\s*$"
-    )
-
-    # =====================================================
-    # CLEAN TITLE
-    # =====================================================
-
-    @staticmethod
-    def clean_title(title: str) -> str:
-
-        title = title.strip()
-
-        title = re.sub(
-            r"\s+",
-            " ",
-            title
-        )
-
-        return title
-
-    # =====================================================
-    # EXTRACT TITLE
-    # =====================================================
-
-    def extract_part_title(
-        self,
-        text: str,
-        part_start: int
-    ) -> str:
-
-        """
-        Usually title is within next 5 lines
-        after PART I
-        """
-
-        chunk = text[part_start:part_start + 1000]
-
-        lines = [
-            x.strip()
-            for x in chunk.split("\n")
-            if x.strip()
-        ]
-
-        if len(lines) < 2:
-            return ""
-
-        title_lines = []
-
-        for line in lines[1:6]:
-
-            # stop if article starts
-
-            if re.match(
-                r"^\d+[A-Z]{0,3}\.",
-                line
-            ):
-                break
-
-            # stop if chapter starts
-
-            if re.match(
-                r"^CHAPTER\s+",
-                line,
-                re.I
-            ):
-                break
-
-            title_lines.append(line)
-
-        return self.clean_title(
-            " ".join(title_lines)
-        )
-
-    # =====================================================
-    # EXTRACT PARTS
-    # =====================================================
-
-    def extract_parts(
-        self,
-        text: str
-    ) -> List[Dict]:
-
-        matches = list(
-            self.PART_RE.finditer(text)
-        )
-
-        if not matches:
-            return []
-
-        parts = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            part_no = match.group(1)
-
-            part_title = self.extract_part_title(
-                text,
-                start
-            )
-
-            part_text = text[start:end].strip()
-
-            parts.append(
-                {
-                    "part_no": part_no,
-                    "part_title": part_title,
-                    "text": part_text,
-                    "start": start,
-                    "end": end
-                }
-            )
-
-        return parts
-
-    # =====================================================
-    # VALIDATION
-    # =====================================================
-
-    def validate_parts(
-        self,
-        parts: List[Dict]
-    ) -> List[str]:
-
-        errors = []
-
-        if not parts:
-
-            errors.append(
-                "No Parts Found"
-            )
-
-            return errors
-
-        seen = set()
-
-        for part in parts:
-
-            part_no = part["part_no"]
-
-            if part_no in seen:
-
-                errors.append(
-                    f"Duplicate Part {part_no}"
-                )
-
-            seen.add(part_no)
-
-            if not part["part_title"]:
-
-                errors.append(
-                    f"Missing title for Part {part_no}"
-                )
-
-        return errors
-
-
-# =========================================================
-# EXAMPLE
-# =========================================================
-
-if __name__ == "__main__":
-
-    with open(
-        "constitution.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    parser = PartParser()
-
-    parts = parser.extract_parts(text)
-
-    print(
-        f"Parts Found: {len(parts)}"
-    )
-
-    for part in parts[:5]:
-
-        print(
-            part["part_no"],
-            part["part_title"]
-        )
-
-    print(
-        parser.validate_parts(parts)
-    )
-
-
-from __future__ import annotations
-
-import re
-from typing import List, Dict
-
-
-class ChapterParser:
-    """
-    Constitution Chapter Parser
-
-    Extracts:
-
-        CHAPTER I
-        THE EXECUTIVE
-
-        CHAPTER II
-        PARLIAMENT
-
-    and returns text belonging
-    to each chapter.
-    """
-
-    # =====================================================
-    # CHAPTER REGEX
-    # =====================================================
-
-    CHAPTER_RE = re.compile(
-        r"(?im)^CHAPTER\s+([IVXLCDM]+[A-Z]?)\s*$"
-    )
-
-    # =====================================================
-    # TITLE CLEANER
-    # =====================================================
-
-    @staticmethod
-    def clean_title(title: str) -> str:
-
-        title = title.strip()
-
-        title = re.sub(
-            r"\s+",
-            " ",
-            title
-        )
-
-        return title
-
-    # =====================================================
-    # EXTRACT CHAPTER TITLE
-    # =====================================================
-
-    def extract_chapter_title(
-        self,
-        text: str,
-        chapter_start: int
-    ) -> str:
-
-        chunk = text[
-            chapter_start:
-            chapter_start + 1000
-        ]
-
-        lines = [
-            line.strip()
-            for line in chunk.split("\n")
-            if line.strip()
-        ]
-
-        if len(lines) < 2:
-            return ""
-
-        title_lines = []
-
-        for line in lines[1:6]:
-
-            # stop at article
-
-            if re.match(
-                r"^\d+[A-Z]{0,3}\.",
-                line
-            ):
-                break
-
-            # stop at next chapter
-
-            if re.match(
-                r"^CHAPTER\s+",
-                line,
-                re.I
-            ):
-                break
-
-            title_lines.append(line)
-
-        return self.clean_title(
-            " ".join(title_lines)
-        )
-
-    # =====================================================
-    # EXTRACT CHAPTERS
-    # =====================================================
-
-    def extract_chapters(
-        self,
-        text: str
-    ) -> List[Dict]:
-
-        matches = list(
-            self.CHAPTER_RE.finditer(text)
-        )
-
-        if not matches:
-            return []
-
-        chapters = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            chapter_no = match.group(1)
-
-            chapter_title = (
-                self.extract_chapter_title(
-                    text,
-                    start
-                )
-            )
-
-            chapter_text = (
-                text[start:end]
-                .strip()
-            )
-
-            chapters.append(
-                {
-                    "chapter_no":
-                        chapter_no,
-
-                    "chapter_title":
-                        chapter_title,
-
-                    "text":
-                        chapter_text,
-
-                    "start":
-                        start,
-
-                    "end":
-                        end
-                }
-            )
-
-        return chapters
-
-    # =====================================================
-    # VALIDATION
-    # =====================================================
-
-    def validate_chapters(
-        self,
-        chapters: List[Dict]
-    ) -> List[str]:
-
-        errors = []
-
-        seen = set()
-
-        for chapter in chapters:
-
-            chapter_no = (
-                chapter["chapter_no"]
-            )
-
-            if chapter_no in seen:
-
-                errors.append(
-                    f"Duplicate Chapter "
-                    f"{chapter_no}"
-                )
-
-            seen.add(chapter_no)
-
-            if not chapter[
-                "chapter_title"
-            ]:
-
-                errors.append(
-                    f"Missing title for "
-                    f"Chapter {chapter_no}"
-                )
-
-        return errors
-
-
-# =========================================================
-# EXAMPLE
-# =========================================================
-
-if __name__ == "__main__":
-
-    with open(
-        "constitution.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    parser = ChapterParser()
-
-    chapters = (
-        parser.extract_chapters(
-            text
-        )
-    )
-
-    print(
-        f"Found "
-        f"{len(chapters)} "
-        f"chapters"
-    )
-
-    for chapter in chapters[:10]:
-
-        print(
-            chapter["chapter_no"],
-            chapter["chapter_title"]
-        )
-
-    print(
-        parser.validate_chapters(
-            chapters
-        )
-    )
-
-
-from __future__ import annotations
-
-import re
-from typing import List, Dict
-
-
-class ArticleParser:
-    """
-    Constitution Article Parser
-
-    Supports:
-
-        1.
-        21A.
-        239AA.
-        243ZG.
-
-    Returns:
-
-        article_no
-        article_title
-        article_text
-        start
-        end
-    """
-
-    # =====================================================
-    # ARTICLE HEADER
-    # =====================================================
-
-    ARTICLE_RE = re.compile(
-        r"(?m)^(\d{1,3}[A-Z]{0,3})\.\s*(.*?)\s*$"
-    )
-
-    # =====================================================
-    # ARTICLE NUMBER VALIDATOR
-    # =====================================================
-
-    ARTICLE_NUMBER_RE = re.compile(
-        r"^\d{1,3}[A-Z]{0,3}$"
-    )
-
-    # =====================================================
-    # CLEAN TITLE
-    # =====================================================
-
-    @staticmethod
-    def clean_title(title: str) -> str:
-
-        title = title.strip()
-
-        title = re.sub(
-            r"\s+",
-            " ",
-            title
-        )
-
-        title = title.rstrip(
-            "."
-        )
-
-        title = title.rstrip(
-            "—"
-        )
-
-        title = title.strip()
-
-        return title
-
-    # =====================================================
-    # RECOVER MULTI LINE TITLE
-    # =====================================================
-
-    def recover_title(
-        self,
-        article_text: str,
-        current_title: str
-    ) -> str:
-
-        if current_title:
-            return self.clean_title(
-                current_title
-            )
-
-        lines = [
-            line.strip()
-            for line in article_text.split(
-                "\n"
-            )
-            if line.strip()
-        ]
-
-        if len(lines) < 2:
-            return ""
-
-        candidate = lines[1]
-
-        if len(candidate) > 150:
-            return ""
-
-        return self.clean_title(
-            candidate
-        )
-
-    # =====================================================
-    # EXTRACT ARTICLES
-    # =====================================================
-
-    def extract_articles(
-        self,
-        text: str
-    ) -> List[Dict]:
-
-        matches = list(
-            self.ARTICLE_RE.finditer(
-                text
-            )
-        )
-
-        if not matches:
-            return []
-
-        articles = []
-
-        for i, match in enumerate(
-            matches
-        ):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            article_no = (
-                match.group(1)
-                .strip()
-            )
-
-            article_title = (
-                match.group(2)
-                .strip()
-            )
-
-            article_text = (
-                text[start:end]
-                .strip()
-            )
-
-            article_title = (
-                self.recover_title(
-                    article_text,
-                    article_title
-                )
-            )
-
-            articles.append(
-                {
-                    "article_no":
-                        article_no,
-
-                    "article_title":
-                        article_title,
-
-                    "text":
-                        article_text,
-
-                    "start":
-                        start,
-
-                    "end":
-                        end
-                }
-            )
-
-        return articles
-
-    # =====================================================
-    # VALIDATION
-    # =====================================================
-
-    def validate_articles(
-        self,
-        articles: List[Dict]
-    ) -> List[str]:
-
-        errors = []
-
-        seen = set()
-
-        for article in articles:
-
-            article_no = (
-                article["article_no"]
-            )
-
-            if article_no in seen:
-
-                errors.append(
-                    f"Duplicate "
-                    f"Article "
-                    f"{article_no}"
-                )
-
-            seen.add(
-                article_no
-            )
-
-            if not self.ARTICLE_NUMBER_RE.match(
-                article_no
-            ):
-
-                errors.append(
-                    f"Invalid Article "
-                    f"Number "
-                    f"{article_no}"
-                )
-
-            if not article[
-                "article_title"
-            ]:
-
-                errors.append(
-                    f"Missing title "
-                    f"for Article "
-                    f"{article_no}"
-                )
-
-        return errors
-
-    # =====================================================
-    # FIND ARTICLE
-    # =====================================================
-
-    def get_article(
-        self,
-        articles: List[Dict],
-        article_no: str
-    ):
-
-        for article in articles:
-
-            if (
-                article["article_no"]
-                == article_no
-            ):
-                return article
-
-        return None
-
-
-# =========================================================
-# EXAMPLE
-# =========================================================
-
-if __name__ == "__main__":
-
-    with open(
-        "constitution.txt",
-        "r",
-        encoding="utf8"
-    ) as f:
-
-        text = f.read()
-
-    parser = ArticleParser()
-
-    articles = (
-        parser.extract_articles(
-            text
-        )
-    )
-
-    print(
-        f"Articles Found: "
-        f"{len(articles)}"
-    )
-
-    for article in articles[:10]:
-
-        print(
-            article["article_no"],
-            article["article_title"]
-        )
-
-    errors = (
-        parser.validate_articles(
-            articles
-        )
-    )
-
-    print(
-        f"Errors: {len(errors)}"
-    )
-
-    for e in errors[:20]:
-        print(e)
-
-
-
-from __future__ import annotations
-
-import re
-from typing import List
-
-from models.legal_models import (
-    Clause,
-    SubClause,
-    RomanClause
-)
-
-
-class ClauseParser:
-
-    """
-    Recursive Clause Parser
-
-    Handles:
-
-        (1)
-        (2)
-        (2A)
-
-    Sub Clauses:
-
-        (a)
-        (b)
-
-    Roman Clauses:
-
-        (i)
-        (ii)
-        (iii)
-        (xiv)
-
-    """
-
-    # =====================================================
-    # REGEX
-    # =====================================================
-
-    CLAUSE_RE = re.compile(
-        r"(?m)^\((\d+[A-Z]?)\)\s"
-    )
-
-    SUBCLAUSE_RE = re.compile(
-        r"(?m)^\(([a-z])\)\s"
-    )
-
-    ROMAN_RE = re.compile(
-        r"(?m)^\(([ivxlcdm]+)\)\s",
-        re.IGNORECASE
-    )
-
-    # =====================================================
-    # SPLIT BY MARKER
-    # =====================================================
-
-    def split_sections(
-        self,
-        text: str,
-        pattern: re.Pattern
-    ):
-
-        matches = list(
-            pattern.finditer(text)
-        )
-
-        if not matches:
-            return []
-
-        sections = []
-
-        for i, match in enumerate(matches):
-
-            start = match.start()
-
-            end = (
-                matches[i + 1].start()
-                if i + 1 < len(matches)
-                else len(text)
-            )
-
-            sections.append(
-                (
-                    match.group(1),
-                    text[start:end].strip()
-                )
-            )
-
-        return sections
-
-    # =====================================================
-    # ROMAN CLAUSES
-    # =====================================================
-
-    def parse_roman_clauses(
-        self,
-        text: str
-    ) -> List[RomanClause]:
-
-        romans = []
-
-        for roman_no, roman_text in self.split_sections(
-            text,
-            self.ROMAN_RE
-        ):
-
-            romans.append(
-                RomanClause(
-                    document="constitution",
-                    roman_no=roman_no,
-                    text=roman_text
-                )
-            )
-
-        return romans
-
-    # =====================================================
-    # SUB CLAUSES
-    # =====================================================
-
-    def parse_subclauses(
-        self,
-        text: str
-    ) -> List[SubClause]:
-
-        subclauses = []
-
-        for sub_no, sub_text in self.split_sections(
-            text,
-            self.SUBCLAUSE_RE
-        ):
-
-            sub = SubClause(
-                document="constitution",
-                sub_clause_no=sub_no,
-                text=sub_text
-            )
-
-            sub.roman_clauses.extend(
-                self.parse_roman_clauses(
-                    sub_text
-                )
-            )
-
-            subclauses.append(sub)
-
-        return subclauses
-
-    # =====================================================
-    # CLAUSES
-    # =====================================================
-
-    def parse_clauses(
-        self,
-        article_text: str
-    ) -> List[Clause]:
-
-        clauses = []
-
-        clause_sections = self.split_sections(
-            article_text,
-            self.CLAUSE_RE
-        )
-
-        if not clause_sections:
-            return clauses
-
-        for clause_no, clause_text in clause_sections:
-
-            clause = Clause(
-                document="constitution",
-                clause_no=clause_no,
-                text=clause_text
-            )
-
-            clause.sub_clauses.extend(
-                self.parse_subclauses(
-                    clause_text
-                )
-            )
-
-            clauses.append(clause)
-
-        return clauses
-
-    # =====================================================
-    # VALIDATION
-    # =====================================================
-
-    def validate_clauses(
-        self,
-        clauses: List[Clause]
-    ) -> List[str]:
-
-        errors = []
-
-        seen = set()
-
-        for clause in clauses:
-
-            if clause.clause_no in seen:
-
-                errors.append(
-                    f"Duplicate Clause "
-                    f"{clause.clause_no}"
-                )
-
-            seen.add(
-                clause.clause_no
-            )
-
-            sub_seen = set()
-
-            for sub in clause.sub_clauses:
-
-                if sub.sub_clause_no in sub_seen:
-
-                    errors.append(
-                        f"Duplicate SubClause "
-                        f"{sub.sub_clause_no} "
-                        f"in Clause "
-                        f"{clause.clause_no}"
-                    )
-
-                sub_seen.add(
-                    sub.sub_clause_no
-                )
-
-                roman_seen = set()
-
-                for roman in sub.roman_clauses:
-
-                    if roman.roman_no in roman_seen:
-
-                        errors.append(
-                            f"Duplicate Roman "
-                            f"{roman.roman_no}"
-                        )
-
-                    roman_seen.add(
-                        roman.roman_no
-                    )
-
-        return errors
-
-
-# =========================================================
-# EXAMPLE
-# =========================================================
-
-if __name__ == "__main__":
-
-    sample = '''
-(1) The State may make laws.
-
-(a) for education
-
-(i) primary education
-
-(ii) secondary education
-
-(b) for health
-
-(2) Parliament may legislate.
-'''
-
-    parser = ClauseParser()
-
-    clauses = parser.parse_clauses(
-        sample
-    )
-
-    print(
-        f"Clauses: "
-        f"{len(clauses)}"
-    )
-
-    for clause in clauses:
-
-        print(
-            clause.clause_no,
-            len(clause.sub_clauses)
-        )
-
-    print(
-        parser.validate_clauses(
-            clauses
-        )
-    )