quant_manifest.json

{
  "config_id": "c1sim",
  "source_model": "meta-llama/Llama-3.1-8B-Instruct",
  "scheme": "W8A16",
  "method": "gptq-simulated",
  "simulated": true,
  "simulated_reason": "vLLM 0.11 compressed-tensors supports only 4/8-bit; 2/3-bit have no kernel",
  "bits_weight": 8,
  "bits_activation": 16,
  "group_size": 128,
  "symmetric": true,
  "calibration_file": "calibration_c4_512x2048.jsonl",
  "calibration_sha256": "b5736155d5b24bf5eb15faae0eb4a742aea4a9d0532db9684085a49750606e19",
  "n_calibration_samples": 512,
  "calibration_seq_len": 2048,
  "grid_check_distinct_values": [
    [
      "model.layers.0.self_attn.q_proj",
      80
    ],
    [
      "model.layers.0.self_attn.k_proj",
      67
    ],
    [
      "model.layers.0.self_attn.v_proj",
      83
    ]
  ],
  "grid_check_max_allowed": 256,
  "quantize_seconds": 1707.2,
  "output_dir": "/scratch/jeesup0103/safety_quant/models/Llama-3.1-8B-Instruct-c1sim"
}