updated

backend/services/resume_parser.py

@@ -6,7 +6,6 @@ import zipfile
 import json
 import torch
 from typing import List
-from transformers import AutoModelForCausalLM, AutoTokenizer
 
 os.environ["OMP_NUM_THREADS"] = "1"
 os.environ["OPENBLAS_NUM_THREADS"] = "1"
@@ -14,18 +13,25 @@ os.environ["MKL_NUM_THREADS"] = "1"
 os.environ["NUMEXPR_NUM_THREADS"] = "1"
 os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
 
-MODEL_ID = "tiiuae/falcon-7b-instruct"
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+import torch
 
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4"
+)
 
-print(f"Loading {MODEL_ID}... (This may take some time on first run)")
-tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b-instruct")
 model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    torch_dtype=torch.float32,  
+    "tiiuae/falcon-7b-instruct",
+    quantization_config=bnb_config,
     device_map="auto"
 )
 
 
+
 # ===============================
 # Text Extraction (PDF/DOCX)
 # ===============================

requirements.txt

@@ -61,4 +61,5 @@ requests>=2.31.0
 # Spaces.  The version is pinned for reproducibility and compatibility.
 psycopg2-binary
 matplotlib
+bitsandbytes>=0.41.0