Process timeouts

app.py

@@ -159,9 +159,9 @@ class HuggingFaceModelProcessor:
             "-o", quant_config.imatrix_file,
         ]
 
-        process = subprocess.Popen(imatrix_command, shell=False)
+        process = subprocess.Popen(imatrix_command, shell=False, stderr=subprocess.STDOUT)
         try:
-            process.wait(timeout=300)
+            process.wait(timeout=600)
         except subprocess.TimeoutExpired:
             print("Imatrix computation timed out. Sending SIGINT to allow graceful termination...")
             process.send_signal(signal.SIGINT)
@@ -170,9 +170,10 @@ class HuggingFaceModelProcessor:
             except subprocess.TimeoutExpired:
                 print("Imatrix proc still didn't term. Forcefully terminating process...")
                 process.kill()
+            raise GGUFConverterError("Error generating imatrix: Operation timed out.")
 
         if process.returncode != 0:
-             raise GGUFConverterError(f"Error generating imatrix")
+            raise GGUFConverterError(f"Error generating imatrix: code={process.returncode}.")
 
         print(f"Importance matrix generation completed: {os.path.abspath(quant_config.imatrix_file)}")
 
@@ -195,16 +196,21 @@ class HuggingFaceModelProcessor:
         split_cmd.extend([quant_config.quantized_gguf, model_path_prefix])
 
         print(f"Split command: {split_cmd}")
-        process = subprocess.Popen(split_cmd, shell=False)
+        process = subprocess.Popen(split_cmd, shell=False, stderr=subprocess.STDOUT)
         try:
             process.wait(timeout=300)
         except subprocess.TimeoutExpired:
-            print("Splitting timed out. Killing process...")
-            process.kill()
+            print("Splitting timed out. Sending SIGINT to allow graceful termination...")
+            process.send_signal(signal.SIGINT)
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                print("Splitting timed out. Killing process...")
+                process.kill()
             raise GGUFConverterError("Error splitting the model: Operation timed out.")
 
         if process.returncode != 0:
-            raise GGUFConverterError(f"Error splitting the model")
+            raise GGUFConverterError(f"Error splitting the model: code={process.returncode}")
 
         print("Model split successfully!")
 
@@ -282,16 +288,21 @@ class HuggingFaceModelProcessor:
                 "python3", "/app/convert_hf_to_gguf.py", local_dir,
                 "--outtype", "f16", "--outfile", processing_config.quant_config.fp16_model
             ]
-            process = subprocess.Popen(convert_command, shell=False)
+            process = subprocess.Popen(convert_command, shell=False, stderr=subprocess.STDOUT)
             try:
                 process.wait(timeout=600)
             except subprocess.TimeoutExpired:
-                print("Conversion timed out. Killing process...")
-                process.kill()
+                print("Conversion timed out. Sending SIGINT to allow graceful termination...")
+                process.send_signal(signal.SIGINT)
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    print("Conversion timed out. Killing process...")
+                    process.kill()
                 raise GGUFConverterError("Error converting to fp16: Operation timed out.")
 
             if process.returncode != 0:
-                raise GGUFConverterError(f"Error converting to fp16")
+                raise GGUFConverterError(f"Error converting to fp16: code={process.returncode}")
 
             print("Model converted to fp16 successfully!")
             print(f"Converted model path: {os.path.abspath(processing_config.quant_config.fp16_model)}")
@@ -328,16 +339,21 @@ class HuggingFaceModelProcessor:
         print(f"Quantizing model with {quantize_cmd}")
 
         # Use Popen for quantization
-        process = subprocess.Popen(quantize_cmd, shell=False)
+        process = subprocess.Popen(quantize_cmd, shell=False, stderr=subprocess.STDOUT)
         try:
             process.wait(timeout=3600)
         except subprocess.TimeoutExpired:
-            print("Quantization timed out. Killing process...")
-            process.kill()
+            print("Quantization timed out. Sending SIGINT to allow graceful termination...")
+            process.send_signal(signal.SIGINT)
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                print("Quantization timed out. Killing process...")
+                process.kill()
             raise GGUFConverterError("Error quantizing: Operation timed out.")
 
         if process.returncode != 0:
-            raise GGUFConverterError(f"Error quantizing")
+            raise GGUFConverterError(f"Error quantizing: code={process.returncode}")
 
         print(f"Quantized successfully with {quant_config.imatrix_method if quant_config.use_imatrix else quant_config.method} option!")
         print(f"Quantized model path: {os.path.abspath(quant_config.quantized_gguf)}")