File size: 2,067 Bytes
eaff197 dbabef2 eaff197 dbabef2 eaff197 dbabef2 eaff197 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | from collections import Counter
from cleaner import BNSSTextCleaner
from db.parsers.bnss.bnss_parser import (
BNSSParser
)
from db.parsers.bnss.chunker import (
LegalChunker,
chunks_to_dicts
)
from db.parsers.bnss.ingest import (
LegalIngestionPipeline
)
# =====================================================
# CONFIG
# =====================================================
DOCUMENT_NAME = "BNSS"
COLLECTION_NAME = "bnss"
PDF_PATH = "../../pdfs/bnss.txt"
# =====================================================
# LOAD
# =====================================================
with open(
PDF_PATH,
"r",
encoding="utf8"
) as f:
text = f.read()
# =====================================================
# CLEAN
# =====================================================
cleaner = BNSSTextCleaner()
text = cleaner.clean(
text
)
# =====================================================
# PARSE
# =====================================================
parser = BNSSParser()
document = parser.parse(
text
)
# =====================================================
# CHUNK
# =====================================================
chunker = LegalChunker(
document_name=DOCUMENT_NAME
)
chunks = chunks_to_dicts(
chunker.chunk_document(
document
)
)
# =====================================================
# STATS
# =====================================================
print(
"\n========== CHUNK STATS ==========\n"
)
print(
f"Total Chunks: {len(chunks)}"
)
levels = Counter(
chunk["level"]
for chunk in chunks
)
for level, count in sorted(
levels.items()
):
print(
f"{level}: {count}"
)
# =====================================================
# INGEST
# =====================================================
pipeline = LegalIngestionPipeline(
collection_name=
COLLECTION_NAME
)
pipeline.ingest(
chunks=
chunks,
batch_size=
64,
recreate_collection=
True
)
print(
f"\n{DOCUMENT_NAME} successfully indexed."
) |