Update app.py

app.py

@@ -9,34 +9,22 @@ from fastapi import FastAPI
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
 from typing import List, Optional, Dict
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, AwqConfig
+from transformers import AutoTokenizer, TextIteratorStreamer
+from awq import AutoAWQForCausalLM
 from threading import Thread
 import gradio as gr
 
-# Desactivar kernels Marlin incompatibles con el init en CPU de ZeroGPU
-os.environ["GPTQMODEL_DISABLE_MARLIN"] = "1"
-os.environ["AUTOAWQ_USE_MARLIN"] = "0"
-
 MODEL_ID = "Qwen/Qwen2.5-Coder-32B-Instruct-AWQ"
 
-# Cargar Tokenizer
+# 1. Cargar Tokenizer
 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
 
-# Forzar backend GEMM estándar de AWQ para evitar fallos de repacking en CPU
-quant_config = AwqConfig(
-    bits=4,
-    version="gemm",
-    fuse_max_seq_len=4096,
-    do_fuse=False
-)
-
-# Cargar modelo en modo device_map="auto" compatible con ZeroGPU
-model = AutoModelForCausalLM.from_pretrained(
+# 2. Cargar modelo directamente con AutoAWQ
+model = AutoAWQForCausalLM.from_quantized(
     MODEL_ID,
-    dtype=torch.float16,
-    quantization_config=quant_config,
-    device_map="auto",
-    trust_remote_code=True
+    fuse_layers=True,
+    trust_remote_code=True,
+    safetensors=True
 )
 
 app = FastAPI(title="Qwen2.5-Coder-32B OpenAI API")