quantization_config.json

{
  "base_model": "BAAI/bge-m3",
  "quantization": {
    "method": "PyTorch dynamic quantization",
    "dtype": "torch.qint8",
    "quantized_modules": [
      "torch.nn.Linear"
    ],
    "device": "cpu"
  },
  "sentence_transformers": {
    "max_seq_length": 128,
    "normalize_embeddings": true
  },
  "checkpoint": {
    "filename": "bge_m3_quantized_state_dict.pt",
    "format": "PyTorch state_dict"
  },
  "evaluation": {
    "fast_mode": true,
    "base_inference_time_seconds": 5.61921859998256,
    "quantized_inference_time_seconds": 0.5463337999826763,
    "speedup": 10.285321172076008,
    "base_state_dict_size_mib": 2165.9678840637207,
    "quantized_state_dict_size_mib": 1299.0863275527954,
    "size_reduction_percent": 40.0228259564269
  }
}