Upload folder using huggingface_hub

demo.py

@@ -8,21 +8,18 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 def load_model():
-    """Load fine-tuned model without quantization for CPU compatibility."""
-    logger.info("Loading model...")
+    """Load the GGUF model from Hugging Face."""
+    logger.info("Loading GGUF model...")
     
-    # Use explicit AutoTokenizer instead of LlamaTokenizer
-    tokenizer = AutoTokenizer.from_pretrained(
-        "Zwounds/boolean-search-model"
-    )
-    
-    # Load model in the most compatible way for Spaces
+    # Load model directly from HF Hub
     model = AutoModelForCausalLM.from_pretrained(
-        "Zwounds/boolean-search-model",
-        low_cpu_mem_usage=True,
-        torch_dtype=torch.float32  # Use standard floating point for CPU
+        "Zwounds/boolean-search-model", 
+        model_file="boolean-model.gguf",  # Specify the exact filename
+        model_type="llama",
+        gpu_layers=0  # Use CPU only for HF Spaces compatibility
     )
-    return model, tokenizer
+    
+    return model
 
 def format_prompt(query):
     """Format query with instruction prompt."""
@@ -82,28 +79,21 @@ Example conversions showing proper quoting:
 def get_boolean_query(query):
     """Generate boolean query from natural language."""
     prompt = format_prompt(query)
-    device = "cuda" if torch.cuda.is_available() else "cpu"
-    
-    # Tokenize and generate response
-    inputs = tokenizer(prompt, return_tensors="pt").to(device)
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=32,
-        do_sample=False,
-        use_cache=True,
-        eos_token_id=tokenizer.eos_token_id
-    )
+
+    
+    # Generate response
+    response = model(prompt, max_new_tokens=64, temperature=0)
+    
+    # Extract response section
+    if "### Response:" in response:
+        response = response.split("### Response:")[-1].strip()
     
-    # Extract response section and clean output
-    full_response = tokenizer.decode(outputs[0])
-    response = full_response.split("### Response:")[-1].strip()
-    # Remove end of text token if present
     cleaned_response = response.replace("<|end_of_text|>", "").strip()
     return cleaned_response
 
 # Load model globally
 logger.info("Initializing model...")
-model, tokenizer = load_model()
+model = load_model()
 logger.info("Model loaded successfully")
 
 # Example queries using more natural language