Hugging Face's logo

Spaces:
ATTENTION-01
/
Insurance_rag_sys
Sleeping

App Files Community
1
singhankur01 commited on
Commit
b7c7987
·
verified ·
1 Parent(s): f5bf326

Update utils/DocsLoader.py

Browse files
Files changed (1) hide show
  1. utils/DocsLoader.py +6 -1
utils/DocsLoader.py CHANGED
@@ -10,7 +10,12 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
10
  from langchain_text_splitters.sentence_transformers import SentenceTransformersTokenTextSplitter # give better results but slow can use later for project
11
  from langchain.schema import Document
12
  # Download NLTK sentence tokenizer
13
- nltk.download('punkt', quiet=True)
 
 
 
 
 
14
  MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
15
 
16
  def load_and_chunk(url: str) -> list[Document]:
 
10
  from langchain_text_splitters.sentence_transformers import SentenceTransformersTokenTextSplitter # give better results but slow can use later for project
11
  from langchain.schema import Document
12
  # Download NLTK sentence tokenizer
13
+ NLTK_PATH = "/tmp/nltk_data"
14
+ os.makedirs(NLTK_PATH, exist_ok=True)
15
+ nltk.data.path.append(NLTK_PATH)
16
+ nltk.download("punkt", download_dir=NLTK_PATH, quiet=True)
17
+
18
+
19
  MODEL_DIR = os.path.join("/tmp", "e5-large-v2")
20
 
21
  def load_and_chunk(url: str) -> list[Document]: