Duke-911 commited on
Commit
e8b5a9a
·
1 Parent(s): 69df689

Refactor retriever.py to initialize SentenceSplitter and update BM25Retriever instantiation

Browse files
Files changed (1) hide show
  1. retriever.py +8 -1
retriever.py CHANGED
@@ -1,6 +1,7 @@
1
  from llama_index.core.schema import Document
2
  from llama_index.core.tools import FunctionTool
3
  from llama_index.retrievers.bm25 import BM25Retriever
 
4
 
5
  import datasets
6
 
@@ -22,7 +23,13 @@ docs = [
22
  for guest in guest_dataset
23
  ]
24
 
25
- bm25_retriever = BM25Retriever.from_documents(docs)
 
 
 
 
 
 
26
 
27
  def get_guest_info_retreiver(query: str) -> str:
28
  """Fetches guest information based on the query."""
 
1
  from llama_index.core.schema import Document
2
  from llama_index.core.tools import FunctionTool
3
  from llama_index.retrievers.bm25 import BM25Retriever
4
+ from llama_index.core.node_parser import SentenceSplitter
5
 
6
  import datasets
7
 
 
23
  for guest in guest_dataset
24
  ]
25
 
26
+
27
+
28
+
29
+ # initialize node parser
30
+ splitter = SentenceSplitter(chunk_size=512)
31
+ nodes = splitter.get_nodes_from_documents(docs)
32
+ bm25_retriever = BM25Retriever.from_defaults(nodes=nodes)
33
 
34
  def get_guest_info_retreiver(query: str) -> str:
35
  """Fetches guest information based on the query."""