Update app.py

app.py

@@ -1,18 +1,23 @@
 import spaces
 import gradio as gr
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
-from huggingface_hub import hf_hub_download
 import torch
 import random
 
-# Load model and tokenizer from Hugging Face
-model_name = "AstroMLab/AstroSage-8B-GGUF"
+# Define model parameters for 8-bit quantized loading
+model_name = "AstroMLab/AstroSage-8B"
+
+# Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+# Load the model with 8-bit quantization using bitsandbytes
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
     torch_dtype=torch.float16,
-    device_map="auto"
+    load_in_8bit=True,               # Enable 8-bit quantization
+    device_map="auto"                # Automatically assign layers to available GPUs
 )
+
 streamer = TextStreamer(tokenizer)
 
 # Placeholder responses for when context is empty