Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -3,7 +3,7 @@ import torch
|
|
| 3 |
import os
|
| 4 |
from transformers import AutoTokenizer, BitsAndBytesConfig, AutoModelForCausalLM
|
| 5 |
from huggingface_hub import login
|
| 6 |
-
from
|
| 7 |
from langchain.embeddings import SentenceTransformerEmbeddings
|
| 8 |
from langchain.vectorstores import FAISS
|
| 9 |
from langchain.chains import RetrievalQA
|
|
@@ -108,7 +108,7 @@ documents = [
|
|
| 108 |
|
| 109 |
|
| 110 |
# 2. Chunk Documents
|
| 111 |
-
text_splitter =
|
| 112 |
docs = text_splitter.split_documents(documents)
|
| 113 |
|
| 114 |
# 3. Create Embeddings and Vector Store (FAISS)
|
|
|
|
| 3 |
import os
|
| 4 |
from transformers import AutoTokenizer, BitsAndBytesConfig, AutoModelForCausalLM
|
| 5 |
from huggingface_hub import login
|
| 6 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 7 |
from langchain.embeddings import SentenceTransformerEmbeddings
|
| 8 |
from langchain.vectorstores import FAISS
|
| 9 |
from langchain.chains import RetrievalQA
|
|
|
|
| 108 |
|
| 109 |
|
| 110 |
# 2. Chunk Documents
|
| 111 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
|
| 112 |
docs = text_splitter.split_documents(documents)
|
| 113 |
|
| 114 |
# 3. Create Embeddings and Vector Store (FAISS)
|