impl: Benchmark results of Llama

results/llama-3-8b/benchmark_results.json

@@ -0,0 +1,67 @@
+{
+  "model": "llama-3-8b",
+  "avg_bits": 7.84,
+  "compression": [
+    {
+      "context_len": 512,
+      "fp16_mb": 67.11,
+      "uniform8_mb": 33.55,
+      "mixed_precision_mb": 32.9,
+      "compression_vs_fp16": 2.04,
+      "compression_vs_8bit": 1.02
+    },
+    {
+      "context_len": 1024,
+      "fp16_mb": 134.22,
+      "uniform8_mb": 67.11,
+      "mixed_precision_mb": 65.8,
+      "compression_vs_fp16": 2.04,
+      "compression_vs_8bit": 1.02
+    },
+    {
+      "context_len": 2048,
+      "fp16_mb": 268.44,
+      "uniform8_mb": 134.22,
+      "mixed_precision_mb": 131.6,
+      "compression_vs_fp16": 2.04,
+      "compression_vs_8bit": 1.02
+    },
+    {
+      "context_len": 4096,
+      "fp16_mb": 536.87,
+      "uniform8_mb": 268.44,
+      "mixed_precision_mb": 263.2,
+      "compression_vs_fp16": 2.04,
+      "compression_vs_8bit": 1.02
+    },
+    {
+      "context_len": 8192,
+      "fp16_mb": 1073.74,
+      "uniform8_mb": 536.87,
+      "mixed_precision_mb": 526.39,
+      "compression_vs_fp16": 2.04,
+      "compression_vs_8bit": 1.02
+    }
+  ],
+  "memory": [
+    {
+      "context": 1024,
+      "peak_memory_gb": 16.47
+    },
+    {
+      "context": 4096,
+      "peak_memory_gb": 17.69
+    },
+    {
+      "context": 8192,
+      "peak_memory_gb": 19.31
+    }
+  ],
+  "decode_tokens_per_sec": 36.7,
+  "perplexity": 20.7,
+  "summary": {
+    "fp16_8k_mb": 1073.74,
+    "ours_8k_mb": 526.39,
+    "compression_8k": 2.04
+  }
+}