Upload folder using huggingface_hub

src/doc_loading.py

@@ -1,6 +1,9 @@
 from langchain.document_loaders import PyPDFLoader, TextLoader
 from langchain.docstore.document import Document
 from typing import List
+from langchain.text_splitter import (
+    RecursiveCharacterTextSplitter,
+)
 
 
 def load_docs(file_path: str, with_pageinfo: bool = True) -> List[Document]:
@@ -17,8 +20,14 @@ def load_docs(file_path: str, with_pageinfo: bool = True) -> List[Document]:
         List[Document]: List of documents.
     """
     if file_path.endswith(".pdf"):
+        # load documents
         loader = PyPDFLoader(file_path)
         docs = loader.load()
+        # # split documents
+        # text_splitter = RecursiveCharacterTextSplitter(
+        #     chunk_size=1000, chunk_overlap=150
+        # )
+        # docs = text_splitter.split_documents(docs)
     elif file_path.endswith(".txt"):
         loader = TextLoader(file_path)
         docs = loader.load()

src/llm_utils.py

@@ -20,11 +20,18 @@ async def async_generate_llmchain(
     Returns:
         dict: Dictionary with the summarization.
     """
+
     print(f"Starting summarization for {k}")
     now = time.time()
-    chain = LLMChain(llm=llm, **llm_kwargs)
 
-    resp = await chain.arun(text=docs)
+    # chain = LLMChain(llm=llm, **llm_kwargs)
+    chain = load_summarize_chain(
+        llm=llm,
+        **llm_kwargs,
+    )
+
+    # resp = await chain.arun(text=docs)
+    resp = await chain.arun(docs)
     print(f"Time taken for {k}: ", time.time() - now)
     return {k: resp}
 

src/summarization.py

@@ -5,7 +5,7 @@ from src.prompts import (
     prompts_parallel_summary,
 )
 from src.doc_loading import load_docs
-from src.llm_utils import async_generate_llmchain
+from src.llm_utils import async_generate_llmchain, async_generate_summary_chain
 import time
 from typing import Dict, List
 import asyncio