quantization_config.json

{
  "model_id": "google/madlad400-3b-mt",
  "output_dir": "madlad400_int8",
  "target_languages": [
    "en",
    "fr",
    "de",
    "es",
    "hi",
    "zh",
    "ar",
    "ru",
    "sq"
  ],
  "dtype_load": "float32",
  "quantize_linear": true,
  "quantize_embedding": false,
  "quantize_layernorm": false,
  "max_length": 256,
  "batch_size_validation": 4,
  "num_validation_sentences": 20,
  "export_onnx": true,
  "onnx_opset": 17,
  "onnx_optimize": true,
  "validate_parity": true,
  "parity_rtol": 0.01,
  "parity_atol": 0.01,
  "device": "cpu",
  "num_threads": 4
}