Spaces:
Build error
Build error
Update app.py
Browse filesGo back to Float8DynamicActivationFloat8WeightConfig. It appears this had done a little better [against rule of thumb].
app.py
CHANGED
|
@@ -30,8 +30,8 @@ import gradio as gr
|
|
| 30 |
import torch
|
| 31 |
from transformers import TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
|
| 32 |
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig
|
| 33 |
-
quant_config = Float8WeightOnlyConfig()
|
| 34 |
-
|
| 35 |
quantization_config = TorchAoConfig(quant_type=quant_config)
|
| 36 |
|
| 37 |
MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
|
|
|
|
| 30 |
import torch
|
| 31 |
from transformers import TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
|
| 32 |
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig
|
| 33 |
+
# quant_config = Float8WeightOnlyConfig()
|
| 34 |
+
quant_config = Float8DynamicActivationFloat8WeightConfig()
|
| 35 |
quantization_config = TorchAoConfig(quant_type=quant_config)
|
| 36 |
|
| 37 |
MODEL_ID = "HuggingFaceTB/SmolLM3-3B"
|