broadfield-dev commited on
Commit
8436872
·
verified ·
1 Parent(s): 5e7e419

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -4
app.py CHANGED
@@ -42,7 +42,7 @@ DATA_DIR.mkdir(exist_ok=True)
42
  # Using NVIDIA Nemotron 3 Nano 4B (BF16) - a compact Mamba2-Transformer hybrid SLM
43
  # 4-bit quantization via BitsAndBytes may not support Mamba layers fully;
44
  # we attempt it first, then fall back to BF16 (model is ~8GB, fits on A10G/T4)
45
- MODEL_NAME = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
46
 
47
  print("[Overthinker] Attempting to load Nemotron 3 Nano 4B with 4-bit quantization...")
48
 
@@ -58,10 +58,9 @@ try:
58
  tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=False)
59
  model = AutoModelForCausalLM.from_pretrained(
60
  MODEL_NAME,
61
- quantization_config=bnb_config,
62
- device_map="auto",
63
  trust_remote_code=True,
64
- torch_dtype=torch.bfloat16
65
  )
66
  print(f"[Overthinker] Model loaded in 4-bit quantization on device: {model.device}")
67
  loaded_quantized = True
 
42
  # Using NVIDIA Nemotron 3 Nano 4B (BF16) - a compact Mamba2-Transformer hybrid SLM
43
  # 4-bit quantization via BitsAndBytes may not support Mamba layers fully;
44
  # we attempt it first, then fall back to BF16 (model is ~8GB, fits on A10G/T4)
45
+ MODEL_NAME = "nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8"
46
 
47
  print("[Overthinker] Attempting to load Nemotron 3 Nano 4B with 4-bit quantization...")
48
 
 
58
  tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=False)
59
  model = AutoModelForCausalLM.from_pretrained(
60
  MODEL_NAME,
61
+ torch_dtype=torch.bfloat16,
 
62
  trust_remote_code=True,
63
+ device_map="auto"
64
  )
65
  print(f"[Overthinker] Model loaded in 4-bit quantization on device: {model.device}")
66
  loaded_quantized = True