evaluation/export_check.json

{
  "source": "results_v6/models/legalbert",
  "quantisation": {
    "weight_type": "QUInt8",
    "per_channel": false,
    "reduce_range": false
  },
  "variants_tried": [
    {
      "weight_type": "QInt8",
      "per_channel": true,
      "reduce_range": false,
      "val_agreement": 0.9506,
      "mb": 35.6
    },
    {
      "weight_type": "QInt8",
      "per_channel": true,
      "reduce_range": true,
      "val_agreement": 0.9753,
      "mb": 35.6
    },
    {
      "weight_type": "QInt8",
      "per_channel": false,
      "reduce_range": false,
      "val_agreement": 0.963,
      "mb": 35.5
    },
    {
      "weight_type": "QInt8",
      "per_channel": false,
      "reduce_range": true,
      "val_agreement": 0.9506,
      "mb": 35.5
    },
    {
      "weight_type": "QUInt8",
      "per_channel": true,
      "reduce_range": false,
      "val_agreement": 0.9753,
      "mb": 35.6
    },
    {
      "weight_type": "QUInt8",
      "per_channel": true,
      "reduce_range": true,
      "val_agreement": 0.9691,
      "mb": 35.6
    },
    {
      "weight_type": "QUInt8",
      "per_channel": false,
      "reduce_range": false,
      "val_agreement": 0.9753,
      "mb": 35.5
    },
    {
      "weight_type": "QUInt8",
      "per_channel": false,
      "reduce_range": true,
      "val_agreement": 0.9568,
      "mb": 35.5
    }
  ],
  "int8_mb": 35.5,
  "fp32_mb": 140.4,
  "agreement": 0.9905,
  "n_rows": 210,
  "disagreements": [
    {
      "text": "Proposed use case: a financial newsletter auto-generating market commentary articles published directly to subscribers.",
      "true": "limited_risk",
      "pytorch": "limited_risk",
      "onnx_int8": "minimal_risk"
    },
    {
      "text": "Our client wants to deploy a language-learning app's AI that decides a learner's official proficiency level, steering their curriculum path.",
      "true": "high_risk",
      "pytorch": "minimal_risk",
      "onnx_int8": "high_risk"
    }
  ],
  "test_accuracy_pytorch": 0.8333,
  "test_accuracy_onnx_int8": 0.8333
}