Adding .csv ingestion functionality

src/ingest.py

@@ -8,13 +8,23 @@ from langchain_community.embeddings import OpenAIEmbeddings
 
 
 def load_documents(data_dir: str):
-    loader = DirectoryLoader(
-        data_dir,
-        glob="**/*.txt",
-        loader_cls=TextLoader,
-        show_progress=True,
-    )
-    docs = loader.load()
+    from pathlib import Path
+    from langchain_community.document_loaders import CSVLoader, TextLoader
+
+    docs = []
+    for path in Path(data_dir).rglob("*"):
+        if not path.is_file():
+            continue
+        suffix = path.suffix.lower()
+        if suffix == ".txt":
+            loader = TextLoader(str(path))
+        elif suffix == ".csv":
+            loader = CSVLoader(file_path=str(path))
+        else:
+            continue
+        loaded = loader.load()
+        docs.extend(loaded)
+
     print(f"Loaded {len(docs)} documents from {data_dir}")
     print("Documents ingested:")
     for d in docs: