Testing Stable LM 2 1.6B Zephyr - subprocess issues

app.py

@@ -22,6 +22,7 @@ def generate_response(user_message):
     return output.decode()
 """
 
+"""
 def generate_response(user_message):
     print("Before request")
     cmd = [
@@ -34,7 +35,33 @@ def generate_response(user_message):
     result = subprocess.run(cmd, capture_output=True, text=True)
     print("After response")
     return result.stdout
+"""
+
+def generate_response(user_message):
+    print("Before request")
+    cmd = [
+        "/app/llama.cpp/main",  # Path to the executable
+        "-m", "/app/llama.cpp/models/stablelm-2-zephyr-1_6b-Q4_0.gguf",
+        "-p", user_message,
+        "-n", "400",
+        "-e"
+    ]
+    
+    # Start the subprocess
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    # Yield each line of output as it becomes available
+    for line in process.stdout:
+        yield line
+    
+    # Wait for the subprocess to finish if it hasn't already
+    process.wait()
 
+    print("After response")
+    # Check for any errors
+    if process.returncode != 0:
+        error_message = process.stderr.read()
+        print(f"Error: {error_message}")
 
 iface = gr.Interface(
     fn=generate_response,