Update app.py

app.py

@@ -429,7 +429,13 @@ class Phi3MiniEducationalLLM(Runnable):
             prompt = str(input)
 
         try:
-            # Format using Phi-3 chat template
+            # Load model inside GPU context
+            model = self._load_model_if_needed()
+        
+            # Clear GPU cache
+            if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+            
             text = self._format_chat_template(prompt)
 
             inputs = self.tokenizer(
@@ -440,8 +446,8 @@ class Phi3MiniEducationalLLM(Runnable):
                 max_length=3072
             )
 
-            # Move inputs to model device
-            inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
+            # Move inputs to model device - now model is not None
+            inputs = {k: v.to(model.device) for k, v in inputs.items()}
 
             # Initialize TextIteratorStreamer
             streamer = TextIteratorStreamer(
@@ -458,7 +464,7 @@ class Phi3MiniEducationalLLM(Runnable):
                 "temperature": 0.7,
                 "top_p": 0.9,
                 "top_k": 50,
-                "repetition_penalty": 1.2,  # Slightly stronger to help with loop prevention
+                "repetition_penalty": 1.2,
                 "pad_token_id": self.tokenizer.eos_token_id,
                 "streamer": streamer,
                 "use_cache": True
@@ -466,7 +472,7 @@ class Phi3MiniEducationalLLM(Runnable):
 
             # Start generation in background
             generation_thread = threading.Thread(
-                target=self.model.generate,
+                target=model.generate,  # Use the loaded model
                 kwargs=generation_kwargs
             )
             generation_thread.start()
@@ -474,43 +480,43 @@ class Phi3MiniEducationalLLM(Runnable):
             # Track outputs
             generated_text = ""
             token_history = []
-            loop_window = 20  # Number of tokens to compare
-            loop_threshold = 3  # Allow N repetitions before aborting
-
+            loop_window = 20
+            loop_threshold = 3
+    
             try:
                 for new_text in streamer:
                     if not new_text:
                         continue
-
+    
                     generated_text += new_text
-
+    
                     # Tokenize and track
                     tokens = self.tokenizer.tokenize(new_text)
                     token_history.extend(tokens)
-
+    
                     # Check for loops
                     if len(token_history) >= 2 * loop_window:
                         recent = token_history[-loop_window:]
                         prev = token_history[-2*loop_window:-loop_window]
                         overlap = sum(1 for r, p in zip(recent, prev) if r == p)
-
+    
                         if overlap >= loop_threshold:
                             logger.warning(f"Looping detected (overlap: {overlap}/{loop_window}). Aborting generation.")
                             yield "[Looping detected — generation stopped early]"
                             break
-
+    
                     yield generated_text
             except Exception as e:
                 logger.error(f"Error in streaming iteration: {e}")
                 yield f"[Streaming error: {str(e)}]"
-
+    
             generation_thread.join()
-
+    
             end_stream_time = time.perf_counter()
             stream_time = end_stream_time - start_stream_time
             log_metric(f"LLM Stream time: {stream_time:0.4f} seconds. Generated length: {len(generated_text)} chars. Model: {self.model_name}. Timestamp: {current_time:%Y-%m-%d %H:%M:%S}")
             logger.info(f"Stream generation completed: {len(generated_text)} chars in {stream_time:.2f}s")
-
+    
         except Exception as e:
             logger.error(f"Streaming generation error: {e}")
             end_stream_time = time.perf_counter()