Cachoups commited on
Commit
5fd7d46
·
verified ·
1 Parent(s): f41da4b

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -3
app.py CHANGED
@@ -52,8 +52,8 @@ tokenizer_name = EMBEDDING_MODEL_NAME
52
  # Token splitting for more context split
53
  text_splitter = TokenTextSplitter.from_huggingface_tokenizer(
54
  tokenizer=AutoTokenizer.from_pretrained(tokenizer_name),
55
- chunk_size=200,
56
- chunk_overlap=20
57
  )
58
 
59
  chunks = text_splitter.split_documents(documents) # chunks used for LLM generation
@@ -68,7 +68,7 @@ for i, doc in enumerate(chunks):
68
  doc.metadata["index"] = i # Add an index for tracking
69
 
70
  """Retriever"""
71
- bm25_retriever = BM25Retriever.from_documents(chunks_bm25, k = 4) # 2 most similar contexts
72
 
73
  def retriever(query):
74
  tmp = bm25_retriever.invoke(preprocess_for_bm25(query))
@@ -149,6 +149,7 @@ Instructions:
149
  """
150
  system_message = f"""
151
  You are an expert in League of Legends (LoL) lore. You will only answer questions related to the champions and their stories within the game.
 
152
  Instructions:
153
  1. Only use the context provided below to answer the question. Reference the context directly for accuracy.
154
  2. If the question is outside the scope of League of Legends lore, respond: "Please ask something related to League of Legends lore."
 
52
  # Token splitting for more context split
53
  text_splitter = TokenTextSplitter.from_huggingface_tokenizer(
54
  tokenizer=AutoTokenizer.from_pretrained(tokenizer_name),
55
+ chunk_size=300,
56
+ chunk_overlap=30
57
  )
58
 
59
  chunks = text_splitter.split_documents(documents) # chunks used for LLM generation
 
68
  doc.metadata["index"] = i # Add an index for tracking
69
 
70
  """Retriever"""
71
+ bm25_retriever = BM25Retriever.from_documents(chunks_bm25, k = 2) # 2 most similar contexts
72
 
73
  def retriever(query):
74
  tmp = bm25_retriever.invoke(preprocess_for_bm25(query))
 
149
  """
150
  system_message = f"""
151
  You are an expert in League of Legends (LoL) lore. You will only answer questions related to the champions and their stories within the game.
152
+
153
  Instructions:
154
  1. Only use the context provided below to answer the question. Reference the context directly for accuracy.
155
  2. If the question is outside the scope of League of Legends lore, respond: "Please ask something related to League of Legends lore."