Try to split query into chunks to prevent overloading model

app.py

@@ -13,27 +13,39 @@ def read_dataset(dataset):
         text.append(item['text'])
     return "\n".join(text)
 
+def chunk_text(text, max_length): 
+    words = text.split() 
+    for i in range(0, len(words), max_length): 
+        yield " ".join(words[i:i+ max_length])
+
 context = read_dataset(dataset)
 
+max_chunk_length = 30000;
 st.title("Adrega AI Help")
 user_input = st.text_input('Ask me a question') 
 
 if st.button("Submit"): 
     if user_input: 
-        messages = [
-            {"role": "system", "content": f"Context: {context}"},
-            {"role": "user","content": user_input}
-        ]
-
-        completion = client.chat.completions.create(
-            model= "Qwen/Qwen2.5-72B-Instruct",
-            #model="Qwen/Qwen2.5-Coder-32B-Instruct", 
-            #model="HuggingFaceTB/SmolLM2-1.7B-Instruct",
-        	messages=messages, 
-        	max_tokens=500
-        )
+        responses = []
+
+        for chunk in chunk_text(context, max_chunk_length):
+            messages = [
+                {"role": "system", "content": f"Context: {context}"},
+                {"role": "user","content": user_input}
+            ]
+
+            completion = client.chat.completions.create(
+                model= "Qwen/Qwen2.5-72B-Instruct",
+                #model="Qwen/Qwen2.5-Coder-32B-Instruct", 
+                #model="HuggingFaceTB/SmolLM2-1.7B-Instruct",
+            	messages=messages, 
+            	max_tokens=500
+            )
+        
+            answer = completion.choices[0].message['content']
+            responses.append(answer)
         
-        answer = completion.choices[0].message['content']
-        st.write(f"Adrega AI: {answer}") 
+        final_response = " ".join(responses)
+        st.write(f"Adrega AI: {final_response}") 
     else: 
         st.write("Please enter a question.")