impl: Benchmark results of Mistral

results/mistral-7b/benchmark_results.json

@@ -0,0 +1,67 @@
+{
+  "model": "mistral-7b",
+  "avg_bits": 6.95,
+  "compression": [
+    {
+      "context_len": 512,
+      "fp16_mb": 67.11,
+      "uniform8_mb": 33.55,
+      "mixed_precision_mb": 29.17,
+      "compression_vs_fp16": 2.3,
+      "compression_vs_8bit": 1.15
+    },
+    {
+      "context_len": 1024,
+      "fp16_mb": 134.22,
+      "uniform8_mb": 67.11,
+      "mixed_precision_mb": 58.33,
+      "compression_vs_fp16": 2.3,
+      "compression_vs_8bit": 1.15
+    },
+    {
+      "context_len": 2048,
+      "fp16_mb": 268.44,
+      "uniform8_mb": 134.22,
+      "mixed_precision_mb": 116.66,
+      "compression_vs_fp16": 2.3,
+      "compression_vs_8bit": 1.15
+    },
+    {
+      "context_len": 4096,
+      "fp16_mb": 536.87,
+      "uniform8_mb": 268.44,
+      "mixed_precision_mb": 233.31,
+      "compression_vs_fp16": 2.3,
+      "compression_vs_8bit": 1.15
+    },
+    {
+      "context_len": 8192,
+      "fp16_mb": 1073.74,
+      "uniform8_mb": 536.87,
+      "mixed_precision_mb": 466.62,
+      "compression_vs_fp16": 2.3,
+      "compression_vs_8bit": 1.15
+    }
+  ],
+  "memory": [
+    {
+      "context": 1024,
+      "peak_memory_gb": 14.76
+    },
+    {
+      "context": 4096,
+      "peak_memory_gb": 15.53
+    },
+    {
+      "context": 8192,
+      "peak_memory_gb": 16.56
+    }
+  ],
+  "decode_tokens_per_sec": 37.2,
+  "perplexity": 14.23,
+  "summary": {
+    "fp16_8k_mb": 1073.74,
+    "ours_8k_mb": 466.62,
+    "compression_8k": 2.3
+  }
+}