finished with ingestion and embedding

app/embeddings.py

@@ -0,0 +1,12 @@
+import chromadb
+from ingestion import extract
+def embeddings(file_path):
+    client = chromadb.Client()
+    v_db = client.get_or_create_collection(name="emb_db")
+    chunks, _, _ = extract(file_path)
+    v_db.add(
+        ids = chunks["ids"],
+        documents=chunks["text"],
+        metadatas=chunks["metadata"]
+    )
+    return v_db

app/ingestion.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+import fitz
+import re
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+
+
+def extract(file_path):
+    filename = Path(file_path).name
+    splitter = RecursiveCharacterTextSplitter(
+    chunk_size=500,
+    chunk_overlap=75,
+    separators=["\n\n", "\n", ". ", " "]
+    )
+    p = {}
+    p["ids"], p["text"], p["metadata"] = [], [], []
+    #skipping the first 24 pages
+    skips = 0
+    with fitz.open(file_path) as doc:
+        for page in doc:
+
+            text = page.get_text("text")
+            text = re.sub(r"[ \t]+", " ", text)
+            text = re.sub(r"\n{3,}", "\n\n", text)
+            text = re.sub(r"(?m)^\d+\s*$", "", text)
+            text = re.sub(r"-\n", "", text)
+            num = page.number + 1
+
+            split_text = splitter.split_text(text)
+
+            if num >= 25:
+                for i, t in enumerate(split_text):
+                    id = f"{filename}_{num}_{i}"
+                    p["ids"].append(id)
+                    p["text"].append(t)
+                    p["metadata"].append(
+                        {"page_number":num}
+                    )
+
+            else:
+                skips+=1
+    return p, skips, num
+
+p, skips, num = extract("../corpus/d2l-en.pdf")
+print(f"Total number of pages:{num}")
+print(f"Pages skipped:{skips}")
+print(f"Total number of chunks: {len(p["ids"])}")
+total_char = 0
+for x in p["text"]:
+    total_char += len(x)
+
+average_chunk_len = total_char/len(p["ids"])
+print(f"Average chunk length: {average_chunk_len}")