transcript-text generation limit increased, response updated

app.py

@@ -104,14 +104,30 @@ def predict_image(image_url, text, file_pref):
         # Process the inputs and move to the appropriate device
         inputs = processor(image, input_text, return_tensors="pt").to(device)
 
-        outputs = model.generate(**inputs, max_new_tokens=100)
+        # outputs = model.generate(**inputs, max_new_tokens=100)
     
-        # Decode the output to return the final response
-        response = processor.decode(outputs[0], skip_special_tokens=True)
+        # # Decode the output to return the final response
+        # response = processor.decode(outputs[0], skip_special_tokens=True)
+               
+        # return response
+
+        streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
+
+        generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=2048)
+        generated_text = ""
+        
+        thread = Thread(target=model.generate, kwargs=generation_kwargs)
+        thread.start()
+        buffer = ""
+        
+        for new_text in streamer:
+            buffer += new_text
+            # generated_text_without_prompt = buffer
+            # # time.sleep(0.01)
+            # yield buffer
+        
+        return buffer
         
-        # return buffer
-        return response
-    
     except Exception as e:
         raise ValueError(f"Error during prediction: {str(e)}")