Create app.py

app.py

@@ -0,0 +1,98 @@
+import gradio as gr
+import torch
+from transformers import AutoTokenizer
+
+# loaders for different quant types
+from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
+from awq import AutoAWQForCausalLM
+
+# all models & type
+MODEL_OPTIONS = {
+    "Llama-3.2-3B": ("meta-llama/Llama-3.2-3B-Instruct", "transformers"),
+    "Llama-3.2-1B": ("meta-llama/Llama-3.2-1B-Instruct", "transformers"),
+    "Qwen2.5-3B-Instruct": ("Qwen/Qwen2.5-3B-Instruct", "transformers"),
+    "Qwen2.5-1.5B-Instruct": ("Qwen/Qwen2.5-1.5B-Instruct", "transformers"),
+    "OpenChat-3.5-0106-GPTQ": ("TheBloke/openchat-3.5-0106-GPTQ", "gptq"),
+    "Gemma-3-4b-it-GPTQ": ("ISTA-DASLab/gemma-3-4b-it-GPTQ-4b-128g", "gptq"),
+    "LLaMA2-7B-GPTQ": ("TheBloke/Llama-2-7B-GPTQ", "gptq"),
+    "LLaMA2-7B-AWQ": ("TitanML/llama2-7b-base-4bit-AWQ", "awq"),
+    "BTLM-3B-8k-base": ("cerebras/btlm-3b-8k-base", "transformers"),
+    "SmolLM3-3B": ("HuggingFaceTB/SmolLM3-3B", "transformers"),
+    "StableLM2-1.6B": ("stabilityai/stablelm-2-zephyr-1_6b", "transformers"),
+    "Falcon-H1-1.5B-Deep": ("unsloth/Falcon-H1-1.5B-Deep-Instruct", "transformers"),
+    "Mistral-7B-Instruct": ("mistralai/Mistral-7B-Instruct-v0.1", "transformers")
+}
+
+loaded = {}
+SYSTEM_PROMPT = "You are HugginGPT — a helpful assistant that remembers context and follows instructions."
+
+def load_model(model_key):
+    model_id, mtype = MODEL_OPTIONS[model_key]
+    # return cached if loaded
+    if model_key in loaded:
+        return loaded[model_key]
+
+    # transformers regular
+    if mtype == "transformers":
+        from transformers import AutoModelForCausalLM
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id,
+            device_map="auto",
+            torch_dtype=torch.float16
+        )
+
+    # GPTQ quant
+    elif mtype == "gptq":
+        quant_cfg = BaseQuantizeConfig(bits=4, group_size=64, desc_act=False)
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        model = AutoGPTQForCausalLM.from_quantized(
+            model_id,
+            use_safetensors=True,
+            device="cuda:0",
+            quantize_config=quant_cfg
+        )
+
+    # AWQ quant
+    elif mtype == "awq":
+        tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=False)
+        model = AutoAWQForCausalLM.from_quantized(
+            model_id,
+            fuse_layers=True,
+            trust_remote_code=False,
+            safetensors=True
+        )
+
+    loaded[model_key] = (tokenizer, model)
+    return tokenizer, model
+
+def generate_response(message, history, model_choice):
+    tokenizer, model = load_model(model_choice)
+
+    # build prompt with system + memory
+    context = f"system: {SYSTEM_PROMPT}\n"
+    if history:
+        for u, a in history:
+            context += f"user: {u}\nassistant: {a}\n"
+    context += f"user: {message}\nassistant:"
+
+    inputs = tokenizer(context, return_tensors="pt").to(model.device)
+    output = model.generate(
+        **inputs,
+        max_new_tokens=200,
+        do_sample=True,
+        top_p=0.9,
+        temperature=0.8
+    )
+    text = tokenizer.decode(output[0], skip_special_tokens=True)
+    reply = text.split("assistant:")[-1].strip()
+    return reply
+
+with gr.Blocks() as demo:
+    gr.ChatInterface(
+        fn=generate_response,
+        title="HugginGPT",
+        inputs=[gr.Dropdown(choices=list(MODEL_OPTIONS.keys()), value="Llama-3.2-3B")]
+    )
+
+demo.launch()