feat: add summerizer map-reduce

utils/gpt_processor.py

@@ -24,7 +24,7 @@ class GPTAgent:
         response = self.agent.complete(messages=messages)
         return response.choices[0].message["content"]
 
-    def split_into_many(text):
+    def split_into_many(self, text):
         tokenizer = tiktoken.get_encoding("cl100k_base")
 
         sentences = text.split("。")
@@ -36,12 +36,12 @@ class GPTAgent:
 
         for sentence, token in zip(sentences, n_tokens):
 
-            if tokens_so_far + token > 500:
+            if tokens_so_far + token > 3000:
                 chunks.append("。".join(chunk) + "。")
                 chunk = []
                 tokens_so_far = 0
 
-            if token > 500:
+            if token > 3000:
                     continue
             chunk.append(sentence)
             tokens_so_far += token + 1
@@ -345,4 +345,4 @@ class QuestionAnswerer(GPTAgent):
             if history[i][1] is not None:
                 messages.append({"role": "assistant", "content": history[i][1]})
 
-        return messages
+        return messages