File size: 1,405 Bytes
dbabef2 5baef2a dbabef2 5baef2a dbabef2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | from __future__ import annotations
from db.all_chunks import (
LegalCorpusBuilder
)
from db.ingest import (
LegalIngestionPipeline
)
from db.graph_builders.all_loader import AllFilesLoader
def main():
print("Inserting into neo4j")
loader=AllFilesLoader()
loader.main()
print(
"\nBuilding Legal Corpus...\n"
)
builder = (
LegalCorpusBuilder()
)
all_chunks = (
builder.build_all_chunks(
bns_path=
"db/pdfs/bns.txt",
bnss_path=
"db/pdfs/bnss.txt",
bsa_path=
"db/pdfs/bsa.txt",
constitution_path=
"db/pdfs/constitution.txt"
)
)
print(
f"\nTotal Chunks: "
f"{len(all_chunks)}"
)
# =====================================
# SAMPLE
# =====================================
print(
"\nSample Chunk:\n"
)
print(
all_chunks[0]
)
# =====================================
# INGEST
# =====================================
pipeline = (
LegalIngestionPipeline(
collection_name=
"legal_rag"
)
)
pipeline.ingest(
chunks=
all_chunks,
recreate_collection=
True
)
print(
"\nFinished."
)
if __name__ == "__main__":
main() |